refactor: enhance workspace entry search ranking and add tests

- Introduced a new scoring mechanism for workspace entries that prioritizes exact basename matches over broader path matches.
- Updated the search logic to utilize shared ranking functions for improved performance and maintainability.
- Added a test case to validate the new ranking behavior for exact matches in workspace entries.
- Removed deprecated functions related to scoring and ranking to streamline the codebase.

Author: juliusmarminge

apps/server/src/workspace/Layers/WorkspaceEntries.test.ts

@@ -129,6 +129,18 @@ it.layer(TestLayer)("WorkspaceEntriesLive", (it) => {
       }),
     );
 
+    it.effect("prioritizes exact basename matches ahead of broader path matches", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir({ prefix: "t3code-workspace-exact-ranking-" });
+        yield* writeTextFile(cwd, "src/components/Composer.tsx");
+        yield* writeTextFile(cwd, "docs/composer.tsx-notes.md");
+
+        const result = yield* searchWorkspaceEntries({ cwd, query: "Composer.tsx", limit: 5 });
+
+        expect(result.entries[0]?.path).toBe("src/components/Composer.tsx");
+      }),
+    );
+
     it.effect("tracks truncation without sorting every fuzzy match", () =>
       Effect.gen(function* () {
         const cwd = yield* makeTempDir({ prefix: "t3code-workspace-fuzzy-limit-" });

apps/server/src/workspace/Layers/WorkspaceEntries.ts

@@ -4,6 +4,12 @@ import type { Dirent } from "node:fs";
 import { Cache, Duration, Effect, Exit, Layer, Option, Path } from "effect";
 
 import { type ProjectEntry } from "@t3tools/contracts";
+import {
+  insertRankedSearchResult,
+  normalizeSearchQuery,
+  scoreQueryMatch,
+  type RankedSearchResult,
+} from "@t3tools/shared/searchRanking";
 
 import { GitCore } from "../../git/Services/GitCore.ts";
 import {
@@ -40,10 +46,7 @@ interface SearchableWorkspaceEntry extends ProjectEntry {
   normalizedName: string;
 }
 
-interface RankedWorkspaceEntry {
-  entry: SearchableWorkspaceEntry;
-  score: number;
-}
+type RankedWorkspaceEntry = RankedSearchResult<SearchableWorkspaceEntry>;
 
 function toPosixPath(input: string): string {
   return input.replaceAll("\\", "/");
@@ -74,127 +77,39 @@ function toSearchableWorkspaceEntry(entry: ProjectEntry): SearchableWorkspaceEnt
   };
 }
 
-function normalizeQuery(input: string): string {
-  return input
-    .trim()
-    .replace(/^[@./]+/, "")
-    .toLowerCase();
-}
-
-function scoreSubsequenceMatch(value: string, query: string): number | null {
-  if (!query) return 0;
-
-  let queryIndex = 0;
-  let firstMatchIndex = -1;
-  let previousMatchIndex = -1;
-  let gapPenalty = 0;
-
-  for (let valueIndex = 0; valueIndex < value.length; valueIndex += 1) {
-    if (value[valueIndex] !== query[queryIndex]) {
-      continue;
-    }
-
-    if (firstMatchIndex === -1) {
-      firstMatchIndex = valueIndex;
-    }
-    if (previousMatchIndex !== -1) {
-      gapPenalty += valueIndex - previousMatchIndex - 1;
-    }
-
-    previousMatchIndex = valueIndex;
-    queryIndex += 1;
-    if (queryIndex === query.length) {
-      const spanPenalty = valueIndex - firstMatchIndex + 1 - query.length;
-      const lengthPenalty = Math.min(64, value.length - query.length);
-      return firstMatchIndex * 2 + gapPenalty * 3 + spanPenalty + lengthPenalty;
-    }
-  }
-
-  return null;
-}
-
 function scoreEntry(entry: SearchableWorkspaceEntry, query: string): number | null {
   if (!query) {
     return entry.kind === "directory" ? 0 : 1;
   }
 
   const { normalizedPath, normalizedName } = entry;
 
-  if (normalizedName === query) return 0;
-  if (normalizedPath === query) return 1;
-  if (normalizedName.startsWith(query)) return 2;
-  if (normalizedPath.startsWith(query)) return 3;
-  if (normalizedPath.includes(`/${query}`)) return 4;
-  if (normalizedName.includes(query)) return 5;
-  if (normalizedPath.includes(query)) return 6;
-
-  const nameFuzzyScore = scoreSubsequenceMatch(normalizedName, query);
-  if (nameFuzzyScore !== null) {
-    return 100 + nameFuzzyScore;
-  }
-
-  const pathFuzzyScore = scoreSubsequenceMatch(normalizedPath, query);
-  if (pathFuzzyScore !== null) {
-    return 200 + pathFuzzyScore;
+  const scores = [
+    scoreQueryMatch({
+      value: normalizedName,
+      query,
+      exactBase: 0,
+      prefixBase: 2,
+      includesBase: 5,
+      fuzzyBase: 100,
+    }),
+    scoreQueryMatch({
+      value: normalizedPath,
+      query,
+      exactBase: 1,
+      prefixBase: 3,
+      boundaryBase: 4,
+      includesBase: 6,
+      fuzzyBase: 200,
+      boundaryMarkers: ["/"],
+    }),
+  ].filter((score): score is number => score !== null);
+
+  if (scores.length === 0) {
+    return null;
   }
 
-  return null;
-}
-
-function compareRankedWorkspaceEntries(
-  left: RankedWorkspaceEntry,
-  right: RankedWorkspaceEntry,
-): number {
-  const scoreDelta = left.score - right.score;
-  if (scoreDelta !== 0) return scoreDelta;
-  return left.entry.path.localeCompare(right.entry.path);
-}
-
-function findInsertionIndex(
-  rankedEntries: RankedWorkspaceEntry[],
-  candidate: RankedWorkspaceEntry,
-): number {
-  let low = 0;
-  let high = rankedEntries.length;
-
-  while (low < high) {
-    const middle = low + Math.floor((high - low) / 2);
-    const current = rankedEntries[middle];
-    if (!current) {
-      break;
-    }
-
-    if (compareRankedWorkspaceEntries(candidate, current) < 0) {
-      high = middle;
-    } else {
-      low = middle + 1;
-    }
-  }
-
-  return low;
-}
-
-function insertRankedEntry(
-  rankedEntries: RankedWorkspaceEntry[],
-  candidate: RankedWorkspaceEntry,
-  limit: number,
-): void {
-  if (limit <= 0) {
-    return;
-  }
-
-  const insertionIndex = findInsertionIndex(rankedEntries, candidate);
-  if (rankedEntries.length < limit) {
-    rankedEntries.splice(insertionIndex, 0, candidate);
-    return;
-  }
-
-  if (insertionIndex >= limit) {
-    return;
-  }
-
-  rankedEntries.splice(insertionIndex, 0, candidate);
-  rankedEntries.pop();
+  return Math.min(...scores);
 }
 
 function isPathInIgnoredDirectory(relativePath: string): boolean {
@@ -469,7 +384,9 @@ export const makeWorkspaceEntries = Effect.gen(function* () {
       const normalizedCwd = yield* normalizeWorkspaceRoot(input.cwd);
       return yield* Cache.get(workspaceIndexCache, normalizedCwd).pipe(
         Effect.map((index) => {
-          const normalizedQuery = normalizeQuery(input.query);
+          const normalizedQuery = normalizeSearchQuery(input.query, {
+            trimLeadingPattern: /^[@./]+/,
+          });
           const limit = Math.max(0, Math.floor(input.limit));
           const rankedEntries: RankedWorkspaceEntry[] = [];
           let matchedEntryCount = 0;
@@ -481,11 +398,15 @@ export const makeWorkspaceEntries = Effect.gen(function* () {
             }
 
             matchedEntryCount += 1;
-            insertRankedEntry(rankedEntries, { entry, score }, limit);
+            insertRankedSearchResult(
+              rankedEntries,
+              { item: entry, score, tieBreaker: entry.path },
+              limit,
+            );
           }
 
           return {
-            entries: rankedEntries.map((candidate) => candidate.entry),
+            entries: rankedEntries.map((candidate) => candidate.item),
             truncated: index.truncated || matchedEntryCount > limit,
           };
         }),

apps/web/src/components/chat/ChatComposer.tsx

@@ -101,6 +101,7 @@ import type { PendingUserInputDraftAnswer } from "../../pendingUserInput";
 import type { PendingApproval, PendingUserInput } from "../../session-logic";
 import { deriveLatestContextWindowSnapshot } from "../../lib/contextWindow";
 import { formatProviderSkillDisplayName } from "../../providerSkillPresentation";
+import { searchProviderSkills } from "../../providerSkillSearch";
 
 const IMAGE_SIZE_LIMIT_LABEL = `${Math.round(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / (1024 * 1024))}MB`;
 
@@ -757,30 +758,20 @@ export const ChatComposer = memo(
         );
       }
       if (composerTrigger.kind === "skill") {
-        const query = composerTrigger.query.trim().toLowerCase();
-        return (selectedProviderStatus?.skills ?? [])
-          .filter((skill) => skill.enabled)
-          .filter((skill) => {
-            if (!query) return true;
-            return (
-              skill.name.toLowerCase().includes(query) ||
-              skill.displayName?.toLowerCase().includes(query) ||
-              skill.shortDescription?.toLowerCase().includes(query) ||
-              skill.description?.toLowerCase().includes(query) ||
-              skill.scope?.toLowerCase().includes(query)
-            );
-          })
-          .map((skill) => ({
-            id: `skill:${selectedProvider}:${skill.name}`,
-            type: "skill" as const,
-            provider: selectedProvider,
-            skill,
-            label: formatProviderSkillDisplayName(skill),
-            description:
-              skill.shortDescription ??
-              skill.description ??
-              (skill.scope ? `${skill.scope} skill` : "Run provider skill"),
-          }));
+        return searchProviderSkills(
+          selectedProviderStatus?.skills ?? [],
+          composerTrigger.query,
+        ).map((skill) => ({
+          id: `skill:${selectedProvider}:${skill.name}`,
+          type: "skill" as const,
+          provider: selectedProvider,
+          skill,
+          label: formatProviderSkillDisplayName(skill),
+          description:
+            skill.shortDescription ??
+            skill.description ??
+            (skill.scope ? `${skill.scope} skill` : "Run provider skill"),
+        }));
       }
       return searchableModelOptions
         .filter(({ searchSlug, searchName, searchProvider }) => {

apps/web/src/providerSkillSearch.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+
+import type { ServerProviderSkill } from "@t3tools/contracts";
+
+import { searchProviderSkills } from "./providerSkillSearch";
+
+function makeSkill(input: Partial<ServerProviderSkill> & Pick<ServerProviderSkill, "name">) {
+  return {
+    path: `/tmp/${input.name}/SKILL.md`,
+    enabled: true,
+    ...input,
+  } satisfies ServerProviderSkill;
+}
+
+describe("searchProviderSkills", () => {
+  it("moves exact ui matches ahead of broader ui matches", () => {
+    const skills = [
+      makeSkill({
+        name: "agent-browser",
+        displayName: "Agent Browser",
+        shortDescription: "Browser automation CLI for AI agents",
+      }),
+      makeSkill({
+        name: "building-native-ui",
+        displayName: "Building Native Ui",
+        shortDescription: "Complete guide for building beautiful apps with Expo Router",
+      }),
+      makeSkill({
+        name: "ui",
+        displayName: "Ui",
+        shortDescription: "Explore, build, and refine UI.",
+      }),
+    ];
+
+    expect(searchProviderSkills(skills, "ui").map((skill) => skill.name)).toEqual([
+      "ui",
+      "building-native-ui",
+    ]);
+  });
+
+  it("uses fuzzy ranking for abbreviated queries", () => {
+    const skills = [
+      makeSkill({ name: "gh-fix-ci", displayName: "Gh Fix Ci" }),
+      makeSkill({ name: "github", displayName: "Github" }),
+      makeSkill({ name: "agent-browser", displayName: "Agent Browser" }),
+    ];
+
+    expect(searchProviderSkills(skills, "gfc").map((skill) => skill.name)).toEqual(["gh-fix-ci"]);
+  });
+
+  it("omits disabled skills from results", () => {
+    const skills = [
+      makeSkill({ name: "ui", displayName: "Ui", enabled: false }),
+      makeSkill({ name: "frontend-design", displayName: "Frontend Design" }),
+    ];
+
+    expect(searchProviderSkills(skills, "ui").map((skill) => skill.name)).toEqual([]);
+  });
+});