fix(core): expose partial filesystem scan results

Author: thdxr

packages/core/src/filesystem/search.ts

@@ -1,7 +1,7 @@
 export * as FileSystemSearch from "./search"
 
 import path from "path"
-import { Context, Effect, Fiber, Layer, Scope } from "effect"
+import { Context, Effect, Layer, Scope } from "effect"
 import { Fff } from "#fff"
 import fuzzysort from "fuzzysort"
 import { FileSystem } from "../filesystem"
@@ -28,22 +28,20 @@ export const ripgrepLayer = Layer.effect(
     const state = {
       files: [] as string[],
       directories: [] as string[],
-      scan: undefined as Fiber.Fiber<void, never> | undefined,
     }
-    state.scan = yield* ripgrep.find({ cwd: location.directory, pattern: "*", limit: 100_000 }).pipe(
-      Effect.tap((result) =>
+    const directories = new Set<string>()
+    yield* ripgrep.find({
+      cwd: location.directory,
+      pattern: "*",
+      limit: location.vcs ? Number.MAX_SAFE_INTEGER : 100_000,
+      onEntry: (entry) =>
         Effect.sync(() => {
-          state.files = result.map((item) => item.path)
-          state.directories = Array.from(
-            new Set(
-              state.files.flatMap((file) => {
-                const parts = file.split("/")
-                return parts.slice(0, -1).map((_, index) => parts.slice(0, index + 1).join("/") + path.sep)
-              }),
-            ),
-          )
+          state.files.push(entry.path)
+          const parts = entry.path.split("/")
+          parts.slice(0, -1).forEach((_, index) => directories.add(parts.slice(0, index + 1).join("/") + path.sep))
+          state.directories = Array.from(directories)
         }),
-      ),
+    }).pipe(
       Effect.orDie,
       Effect.asVoid,
       Effect.forkIn(scope),
@@ -104,7 +102,6 @@ export const ripgrepLayer = Layer.effect(
         }),
       find: (input) =>
         Effect.gen(function* () {
-          if (input.query) yield* Fiber.join(state.scan!)
           const items =
             input.type === "file"
               ? state.files

packages/core/src/ripgrep.ts

@@ -56,6 +56,7 @@ export interface FindInput {
   readonly hidden?: boolean
   readonly follow?: boolean
   readonly signal?: AbortSignal
+  readonly onEntry?: (entry: Entry) => Effect.Effect<void>
 }
 
 export interface GlobInput {
@@ -102,6 +103,7 @@ export const layer = Layer.effect(
       readonly signal?: AbortSignal
       readonly parse: (line: string) => Effect.Effect<A | undefined, Error>
       readonly pattern?: string
+      readonly onItem?: (item: A) => Effect.Effect<void>
     }) => {
       const program = Effect.scoped(
         Effect.gen(function* () {
@@ -112,11 +114,16 @@ export const layer = Layer.effect(
             Effect.map((output) => output.buffer.toString("utf8")),
             Effect.forkScoped,
           )
+          let observed = 0
           const rows = yield* Stream.decodeText(handle.stdout).pipe(
             Stream.splitLines,
             Stream.filter((line) => line.length > 0),
             Stream.mapEffect(input.parse),
             Stream.filter((row): row is A => row !== undefined),
+            Stream.tap((row) => {
+              if (!input.onItem || observed++ >= input.limit) return Effect.void
+              return input.onItem(row)
+            }),
             Stream.take(input.limit + 1),
             Stream.runCollect,
             Effect.map((chunk) => [...chunk]),
@@ -181,7 +188,7 @@ export const layer = Layer.effect(
           Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
         ),
       find: (input) =>
-        run<string>({
+        run<Entry>({
           cwd: input.cwd,
           limit: input.limit,
           signal: input.signal,
@@ -194,24 +201,22 @@ export const layer = Layer.effect(
             `--glob=${input.pattern}`,
             ".",
           ],
-          parse: (line) =>
-            Effect.succeed(
-              line
-                .replace(/^(?:\.[\\/])+/u, "")
-                .replace(/^[\\/]+/u, "")
-                .replaceAll("\\", "/"),
-            ),
-        }).pipe(
-          Effect.map((result) =>
-            result.items.map((relative) => {
-              const absolute = path.resolve(input.cwd, relative)
-              return new Entry({
+          parse: (line) => {
+            const relative = line
+              .replace(/^(?:\.[\\/])+/u, "")
+              .replace(/^[\\/]+/u, "")
+              .replaceAll("\\", "/")
+            return Effect.succeed(
+              new Entry({
                 path: RelativePath.make(relative),
                 type: "file",
-                mime: FSUtil.mimeType(absolute),
-              })
-            }),
-          ),
+                mime: FSUtil.mimeType(path.resolve(input.cwd, relative)),
+              }),
+            )
+          },
+          onItem: input.onEntry,
+        }).pipe(
+          Effect.map((result) => result.items),
           Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
         ),
       grep: (input) =>

packages/core/test/ripgrep.test.ts

@@ -25,6 +25,15 @@ describe("Ripgrep", () => {
           expect(files.map((item) => item.path)).toContain(RelativePath.make(".opencode/config"))
           expect(files.map((item) => item.path)).toContain(RelativePath.make(".git/config"))
 
+          const observed: string[] = []
+          const limited = yield* ripgrep.find({
+            cwd: tmp.path,
+            pattern: "**/*",
+            limit: 1,
+            onEntry: (entry) => Effect.sync(() => observed.push(entry.path)),
+          })
+          expect(observed).toEqual(limited.map((item) => item.path))
+
           const matches = yield* ripgrep.grep({ cwd: tmp.path, pattern: "needle", include: "config", limit: 10 })
           expect(matches.map((item) => item.entry.path)).toContain(RelativePath.make(".opencode/config"))
           expect(matches.map((item) => item.entry.path)).toContain(RelativePath.make(".git/config"))