fix(core): reduce file index allocations

Co-authored-by: Hona <10430890+Hona@users.noreply.github.com>

packages/core/src/filesystem/search.ts

@@ -26,16 +26,22 @@ const REFRESH_INTERVAL = Duration.toMillis("10 seconds")
 type Prepared = ReturnType<typeof fuzzysort.prepare>
 
 function emptyIndex() {
-  return { files: new Map<string, Prepared>(), directories: new Map<string, Prepared>() }
+  return {
+    files: new Map<string, Prepared>(),
+    directories: new Map<string, Prepared>(),
+    fileTargets: [] as Prepared[],
+    directoryTargets: [] as Prepared[],
+    combinedTargets: undefined as Prepared[] | undefined,
+  }
 }
 
 function search(index: ReturnType<typeof emptyIndex>, input: FileSystem.FindInput) {
   const items =
     input.type === "file"
-      ? Array.from(index.files.values())
+      ? index.fileTargets
       : input.type === "directory"
-        ? Array.from(index.directories.values())
-        : [...index.files.values(), ...index.directories.values()]
+        ? index.directoryTargets
+        : (index.combinedTargets ??= [...index.fileTargets, ...index.directoryTargets])
   const result = fuzzysort.go(input.query, items, { limit: input.limit ?? 50 })
   // Targets are owned by the current location index. The only global fuzzysort
   // state left is its query cache, which must not retain every query forever.
@@ -66,20 +72,28 @@ export const ripgrepLayer = Layer.effect(
       const next = emptyIndex()
       const previous = index
       if (!initialized) index = next
-      yield* ripgrep.find({
+      yield* ripgrep.scan({
         cwd: location.directory,
         pattern: "*",
         limit: location.vcs && !home ? Number.MAX_SAFE_INTEGER : 100_000,
         exclude: home ? [...Protected.names()].map((name) => `${name}/**`) : undefined,
         onEntry: (entry) =>
           Effect.sync(() => {
-            next.files.set(entry.path, previous.files.get(entry.path) ?? fuzzysort.prepare(entry.path))
+            const file = previous.files.get(entry.path) ?? fuzzysort.prepare(entry.path)
+            next.files.set(entry.path, file)
+            next.fileTargets.push(file)
+            next.combinedTargets = undefined
             const parts = entry.path.split("/")
-            parts.slice(0, -1).forEach((_, offset) => {
-              const directory = parts.slice(0, offset + 1).join("/") + path.sep
-              if (!next.directories.has(directory))
-                next.directories.set(directory, previous.directories.get(directory) ?? fuzzysort.prepare(directory))
-            })
+            let prefix = ""
+            for (const [offset, part] of parts.entries()) {
+              if (offset === parts.length - 1) break
+              prefix = prefix ? `${prefix}/${part}` : part
+              const directory = prefix + path.sep
+              if (next.directories.has(directory)) continue
+              const prepared = previous.directories.get(directory) ?? fuzzysort.prepare(directory)
+              next.directories.set(directory, prepared)
+              next.directoryTargets.push(prepared)
+            }
           }),
       })
       index = next

packages/core/src/ripgrep.ts

@@ -60,6 +60,10 @@ export interface FindInput {
   readonly onEntry?: (entry: Entry) => Effect.Effect<void>
 }
 
+export interface ScanInput extends Omit<FindInput, "onEntry"> {
+  readonly onEntry: (entry: Entry) => Effect.Effect<void>
+}
+
 export interface GlobInput {
   readonly cwd: string
   readonly pattern: string
@@ -80,6 +84,7 @@ export interface GrepInput {
 
 export interface Interface {
   readonly find: (input: FindInput) => Effect.Effect<readonly Entry[], Error>
+  readonly scan: (input: ScanInput) => Effect.Effect<void, Error>
   readonly glob: (input: GlobInput) => Effect.Effect<readonly Entry[], Error>
   readonly grep: (input: GrepInput) => Effect.Effect<readonly Match[], Error | InvalidPatternError>
 }
@@ -105,6 +110,7 @@ const layer = Layer.effect(
       readonly parse: (line: string) => Effect.Effect<A | undefined, Error>
       readonly pattern?: string
       readonly onItem?: (item: A) => Effect.Effect<void>
+      readonly collect?: boolean
     }) => {
       const program = Effect.scoped(
         Effect.gen(function* () {
@@ -127,11 +133,17 @@ const layer = Layer.effect(
               return input.onItem(row)
             }),
             Stream.take(input.limit + 1),
-            Stream.runCollect,
-            Effect.map((chunk) => [...chunk]),
+            Stream.runFold(
+              () => ({ count: 0, items: [] as A[] }),
+              (result, row) => {
+                result.count++
+                if (input.collect !== false) result.items.push(row)
+                return result
+              },
+            ),
           )
-          const truncated = rows.length > input.limit
-          if (truncated) return { items: rows.slice(0, input.limit), truncated, partial: false }
+          const truncated = rows.count > input.limit
+          if (truncated) return { items: rows.items.slice(0, input.limit), truncated, partial: false }
 
           const code = yield* handle.exitCode
           const stderr = yield* Fiber.join(stderrFiber)
@@ -141,7 +153,7 @@ const layer = Layer.effect(
           if (code !== 0 && code !== 1 && code !== 2) {
             return yield* failure(stderr.trim() || `ripgrep failed with code ${code}`)
           }
-          return { items: code === 1 ? [] : rows, truncated: false, partial: code === 2 }
+          return { items: code === 1 ? [] : rows.items, truncated: false, partial: code === 2 }
         }),
       )
       const abortable = input.signal ? program.pipe(Effect.raceFirst(waitForAbort(input.signal))) : program
@@ -154,6 +166,40 @@ const layer = Layer.effect(
       )
     }
 
+    const find = (input: FindInput, collect = true) =>
+      run<Entry>({
+        cwd: input.cwd,
+        limit: input.limit,
+        signal: input.signal,
+        args: [
+          "--no-config",
+          "--files",
+          ...(input.hidden ? ["--hidden"] : []),
+          ...(input.follow ? ["--follow"] : []),
+          ...(input.pattern === "*" ? [] : [`--glob=${input.pattern}`]),
+          ...(input.exclude ?? []).map((pattern) => `--glob=!${pattern}`),
+          "--glob=!**/.git/**",
+          ".",
+        ],
+        parse: (line) => {
+          const relative = line
+            .replace(/^(?:\.[\\/])+/u, "")
+            .replace(/^[\\/]+/u, "")
+            .replaceAll("\\", "/")
+          return Effect.succeed(
+            Entry.make({
+              path: RelativePath.make(relative),
+              type: "file",
+            }),
+          )
+        },
+        onItem: input.onEntry,
+        collect,
+      }).pipe(
+        Effect.map((result) => result.items),
+        Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
+      )
+
     return Service.of({
       glob: (input) =>
         run<string>({
@@ -187,38 +233,8 @@ const layer = Layer.effect(
           ),
           Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
         ),
-      find: (input) =>
-        run<Entry>({
-          cwd: input.cwd,
-          limit: input.limit,
-          signal: input.signal,
-          args: [
-            "--no-config",
-            "--files",
-            ...(input.hidden ? ["--hidden"] : []),
-            ...(input.follow ? ["--follow"] : []),
-            ...(input.pattern === "*" ? [] : [`--glob=${input.pattern}`]),
-            ...(input.exclude ?? []).map((pattern) => `--glob=!${pattern}`),
-            "--glob=!**/.git/**",
-            ".",
-          ],
-          parse: (line) => {
-            const relative = line
-              .replace(/^(?:\.[\\/])+/u, "")
-              .replace(/^[\\/]+/u, "")
-              .replaceAll("\\", "/")
-            return Effect.succeed(
-              Entry.make({
-                path: RelativePath.make(relative),
-                type: "file",
-              }),
-            )
-          },
-          onItem: input.onEntry,
-        }).pipe(
-          Effect.map((result) => result.items),
-          Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
-        ),
+      find,
+      scan: (input) => find(input, false).pipe(Effect.asVoid),
       grep: (input) =>
         run<RawMatchData>({
           ...input,

packages/core/test/filesystem/search.test.ts

@@ -13,6 +13,15 @@ import { Ripgrep } from "@opencode-ai/core/ripgrep"
 import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
 import { location } from "../fixture/location"
 
+function ripgrep(find: Ripgrep.Interface["find"]) {
+  return Ripgrep.Service.of({
+    find,
+    scan: (input) => find(input).pipe(Effect.asVoid),
+    glob: () => Effect.succeed([]),
+    grep: () => Effect.succeed([]),
+  })
+}
+
 describe("FileSystemSearch", () => {
   test("bounds a home scan even when home is detected as a repository", async () => {
     let observed: Ripgrep.FindInput | undefined
@@ -31,17 +40,14 @@ describe("FileSystemSearch", () => {
         Ripgrep.node,
         Layer.succeed(
           Ripgrep.Service,
-          Ripgrep.Service.of({
-            find: (input) =>
+          ripgrep((input) =>
               Effect.gen(function* () {
                 observed = input
                 if (input.onEntry)
                   yield* input.onEntry(FileSystem.Entry.make({ path: RelativePath.make("src/index.ts"), type: "file" }))
                 return []
               }),
-            glob: () => Effect.succeed([]),
-            grep: () => Effect.succeed([]),
-          }),
+          ),
         ),
       ],
     ])
@@ -78,8 +84,7 @@ describe("FileSystemSearch", () => {
         Ripgrep.node,
         Layer.succeed(
           Ripgrep.Service,
-          Ripgrep.Service.of({
-            find: (input) =>
+          ripgrep((input) =>
               Effect.gen(function* () {
                 scans++
                 if (scans > 1) {
@@ -94,9 +99,7 @@ describe("FileSystemSearch", () => {
                 if (scans === 1) yield* Deferred.succeed(initial, undefined)
                 return [entry]
               }),
-            glob: () => Effect.succeed([]),
-            grep: () => Effect.succeed([]),
-          }),
+          ),
         ),
       ],
     ])
@@ -145,8 +148,7 @@ describe("FileSystemSearch", () => {
         Ripgrep.node,
         Layer.succeed(
           Ripgrep.Service,
-          Ripgrep.Service.of({
-            find: (input) =>
+          ripgrep((input) =>
               Effect.gen(function* () {
                 scans++
                 const entry = FileSystem.Entry.make({ path: RelativePath.make("src/index.ts"), type: "file" })
@@ -154,9 +156,7 @@ describe("FileSystemSearch", () => {
                 yield* Deferred.succeed(scans === 1 ? first : second, undefined)
                 return [entry]
               }),
-            glob: () => Effect.succeed([]),
-            grep: () => Effect.succeed([]),
-          }),
+          ),
         ),
       ],
     ])
@@ -178,4 +178,54 @@ describe("FileSystemSearch", () => {
     prepare.mockRestore()
     cleanup.mockRestore()
   })
+
+  test("invalidates a mixed target list while the initial scan is still running", async () => {
+    const first = Effect.runSync(Deferred.make<void>())
+    const release = Effect.runSync(Deferred.make<void>())
+    const complete = Effect.runSync(Deferred.make<void>())
+    const layer = AppNodeBuilder.build(FileSystemSearch.node, [
+      [
+        Location.node,
+        Layer.succeed(
+          Location.Service,
+          Location.Service.of(
+            location({ directory: AbsolutePath.make(path.join(os.tmpdir(), "opencode-search-partial")) }),
+          ),
+        ),
+      ],
+      [
+        Ripgrep.node,
+        Layer.succeed(
+          Ripgrep.Service,
+          ripgrep((input) =>
+              Effect.gen(function* () {
+                if (input.onEntry)
+                  yield* input.onEntry(
+                    FileSystem.Entry.make({ path: RelativePath.make("src/first.ts"), type: "file" }),
+                  )
+                yield* Deferred.succeed(first, undefined)
+                yield* Deferred.await(release)
+                if (input.onEntry)
+                  yield* input.onEntry(
+                    FileSystem.Entry.make({ path: RelativePath.make("src/second.ts"), type: "file" }),
+                  )
+                yield* Deferred.succeed(complete, undefined)
+                return []
+              }),
+          ),
+        ),
+      ],
+    ])
+
+    await Effect.runPromise(
+      Effect.gen(function* () {
+        const search = yield* FileSystemSearch.Service
+        yield* Deferred.await(first)
+        expect((yield* search.find({ query: "first" }))[0]?.path).toBe(RelativePath.make("src/first.ts"))
+        yield* Deferred.succeed(release, undefined)
+        yield* Deferred.await(complete)
+        expect((yield* search.find({ query: "second" }))[0]?.path).toBe(RelativePath.make("src/second.ts"))
+      }).pipe(Effect.provide(layer), Effect.scoped),
+    )
+  })
 })

packages/core/test/ripgrep.test.ts

@@ -104,6 +104,28 @@ describe("Ripgrep", () => {
     ),
   )
 
+  it.live("streams find entries without retaining a result array", () =>
+    Effect.acquireUseRelease(
+      Effect.promise(() => tmpdir()),
+      (tmp) =>
+        Effect.gen(function* () {
+          yield* Effect.promise(() => fs.writeFile(path.join(tmp.path, "one.txt"), "one\n"))
+          yield* Effect.promise(() => fs.writeFile(path.join(tmp.path, "two.txt"), "two\n"))
+          const observed: RelativePath[] = []
+          const ripgrep = yield* Ripgrep.Service
+          yield* ripgrep.scan({
+            cwd: tmp.path,
+            pattern: "*",
+            limit: 10,
+            onEntry: (entry) => Effect.sync(() => observed.push(entry.path)),
+          })
+
+          expect(new Set(observed)).toEqual(new Set([RelativePath.make("one.txt"), RelativePath.make("two.txt")]))
+        }),
+      (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()),
+    ),
+  )
+
   it.live("excludes protected directory trees from catch-all find results", () =>
     Effect.acquireUseRelease(
       Effect.promise(() => tmpdir()),