dofs: Add bounded filesystem primitives

.changeset/dofs-bounded-filesystem.md

@@ -0,0 +1,5 @@
+---
+"@cloudflare/dofs": minor
+---
+
+Add stable directory pagination with metadata, bounded byte reads, single-character globs, and configurable bounded grep results.

packages/dofs/src/fs/filesystem.ts

@@ -46,6 +46,11 @@ export class WorkspaceFilesystem {
 
   readFile(path: string): Promise<ReadableStream<Uint8Array>>;
   readFile(path: string, encoding: "utf8"): Promise<string>;
+  readFile(
+    path: string,
+    options: ReadFileOptions & { encoding?: undefined },
+  ): Promise<ReadableStream<Uint8Array>>;
+  readFile(path: string, options: ReadFileOptions & { encoding: "utf8" }): Promise<string>;
   readFile(path: string, options: ReadFileOptions): Promise<string | ReadableStream<Uint8Array>>;
   readFile(
     path: string,

packages/dofs/src/fs/find.test.ts

@@ -54,6 +54,17 @@ describe("find", () => {
     });
   });
 
+  it("matches ? as one non-separator character", async () => {
+    await withDB(async (db) => {
+      mkdir(db, "/a", {}, () => 0);
+      await writeFile(db, "/a/a.ts", "", {}, () => 0);
+      await writeFile(db, "/a/ab.ts", "", {}, () => 0);
+      await writeFile(db, "/a/b.ts", "", {}, () => 0);
+      const paths = find(db, "/a", "?.ts").map((entry) => entry.path);
+      expect(paths).toEqual(["/a/a.ts", "/a/b.ts"]);
+    });
+  });
+
   it("matches ** recursively", async () => {
     await withDB(async (db) => {
       mkdir(db, "/a/b/c", { recursive: true }, () => 0);

packages/dofs/src/fs/find.ts

@@ -65,6 +65,7 @@ function walk(db: Database, parentInode: number, parentPath: string, out: Worksp
 // Compile a simple glob into a regex. Supported:
 //   *  matches any run of characters except '/'
 //   ** matches any run of characters including '/'
+//   ?  matches one character except '/'
 // Anything else is a literal. Regex metacharacters in literals are
 // escaped so '.' in '*.ts' doesn't match an arbitrary character.
 function compileGlob(pattern: string): RegExp {
@@ -89,6 +90,11 @@ function compileGlob(pattern: string): RegExp {
       }
       continue;
     }
+    if (ch === "?") {
+      re += "[^/]";
+      i += 1;
+      continue;
+    }
     if (REGEX_METACHARS.has(ch)) {
       re += `\\${ch}`;
     } else {

packages/dofs/src/fs/grep.test.ts

@@ -43,14 +43,97 @@ describe("grep", () => {
     });
   });
 
-  it("respects ignoreCase", async () => {
+  it("respects the ignoreCase option", async () => {
     await withDB(async (db) => {
       await writeFile(db, "/a.txt", "todo\nTODO\nTodo\n", {}, () => 0);
       expect((await grep(db, "TODO", "/a.txt", { ignoreCase: true })).length).toBe(3);
+      expect((await grep(db, "TODO", "/a.txt", { ignoreCase: false })).length).toBe(1);
       expect((await grep(db, "TODO", "/a.txt")).length).toBe(1);
     });
   });
 
+  it("supports opt-in regular expressions and literal strings by default", async () => {
+    await withDB(async (db) => {
+      await writeFile(db, "/a.txt", "task 12\ntask \\d+\ntask xx\n", {}, () => 0);
+      expect(
+        (await grep(db, String.raw`task \d+`, "/a.txt", { regex: true })).map(
+          (match) => match.line,
+        ),
+      ).toEqual([1]);
+      expect((await grep(db, String.raw`task \d+`, "/a.txt")).map((match) => match.line)).toEqual([
+        2,
+      ]);
+      await expect(grep(db, "[", "/a.txt", { regex: true })).rejects.toThrow(
+        "Invalid regular expression",
+      );
+    });
+  });
+
+  it("returns numbered context around matches", async () => {
+    await withDB(async (db) => {
+      await writeFile(db, "/a.txt", "one\ntwo\nTODO\nfour\nfive\n", {}, () => 0);
+      expect(await grep(db, "TODO", "/a.txt", { context: 1 })).toEqual([
+        {
+          path: "/a.txt",
+          line: 3,
+          text: "TODO",
+          context: [
+            { line: 2, text: "two", isMatch: false },
+            { line: 3, text: "TODO", isMatch: true },
+            { line: 4, text: "four", isMatch: false },
+          ],
+        },
+      ]);
+    });
+  });
+
+  it("marks adjacent matches as matching context", async () => {
+    await withDB(async (db) => {
+      await writeFile(db, "/a.txt", "TODO one\nTODO two\nplain\n", {}, () => 0);
+
+      const matches = await grep(db, "TODO", "/a.txt", { context: 1 });
+      expect(matches).toEqual([
+        {
+          path: "/a.txt",
+          line: 1,
+          text: "TODO one",
+          context: [
+            { line: 1, text: "TODO one", isMatch: true },
+            { line: 2, text: "TODO two", isMatch: true },
+          ],
+        },
+        {
+          path: "/a.txt",
+          line: 2,
+          text: "TODO two",
+          context: [
+            { line: 1, text: "TODO one", isMatch: true },
+            { line: 2, text: "TODO two", isMatch: true },
+            { line: 3, text: "plain", isMatch: false },
+          ],
+        },
+      ]);
+
+      if (matches[0].context === undefined || matches[1].context === undefined) {
+        throw new Error("expected grep context");
+      }
+      matches[0].context[0].text = "changed";
+      expect(matches[1].context[0].text).toBe("TODO one");
+    });
+  });
+
+  it("applies offset and limit across files in path and line order", async () => {
+    await withDB(async (db) => {
+      await writeFile(db, "/a.txt", "TODO a1\nTODO a2\n", {}, () => 0);
+      await writeFile(db, "/b.txt", "TODO b1\nTODO b2\n", {}, () => 0);
+      expect(
+        (await grep(db, "TODO", "/", { offset: 1, limit: 2 })).map(
+          (match) => `${match.path}:${match.line}`,
+        ),
+      ).toEqual(["/a.txt:2", "/b.txt:1"]);
+    });
+  });
+
   it("matches across a chunk boundary", async () => {
     await withDB(async (db) => {
       // Lay out a file whose line straddles the 512KiB chunk boundary.