feat: Trigger.dev index-repo task with R2 storage (#88)

* feat: Trigger.dev index-repo task with R2 storage and webhook trigger

- Scaffold apps/trigger with Trigger.dev SDK v4 (#81)
- GitHub App JWT generation, installation tokens, tarball download (#82)
- R2 upload helpers: tarball, file-tree.json, individual files (#83)
- DrizzleSyncStorage: Merkle state via Drizzle/Neon (#84)
- index-repo task: walk → chunk → embed → upsert → Merkle persist → R2 (#85)
- Wire webhook to trigger index-repo after repo upsert (#86)
- 170 tests passing (153 core + 17 web)

* fix: address code review findings

- Fix SHA extraction: use redirect:manual to capture SHA from Location URL
- Bulk chunk cache inserts instead of sequential per-chunk DB round-trips
- Remove redundant hashFiles call — reuse fileHashMap from computeChanges
- Return 500 on webhook errors so GitHub retries delivery
- Reuse S3Client singleton instead of creating one per file upload

* style: fix formatting in index-repo.ts

Author: HrushiBorhade

apps/trigger/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@codeindexer/trigger",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "npx trigger.dev@latest dev",
+    "deploy": "npx trigger.dev@latest deploy",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@trigger.dev/sdk": "^4.0.0",
+    "@aws-sdk/client-s3": "^3.0.0",
+    "@codeindexer/core": "workspace:*",
+    "@codeindexer/db": "workspace:*",
+    "tar": "^7.0.0",
+    "uuid": "^11.0.0",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@trigger.dev/build": "^4.0.0",
+    "@codeindexer/tsconfig": "workspace:*",
+    "@types/uuid": "^10.0.0",
+    "typescript": "^5"
+  }
+}

apps/trigger/src/lib/drizzle-sync-storage.ts

@@ -0,0 +1,74 @@
+import type { SyncStorage } from '@codeindexer/core';
+import { type Database, fileHashes, dirHashes, eq, and } from '@codeindexer/db';
+
+export class DrizzleSyncStorage implements SyncStorage {
+  constructor(
+    private db: Database,
+    private repoId: string,
+  ) {}
+
+  async getFileHash(filePath: string): Promise<string | null> {
+    const row = await this.db.query.fileHashes.findFirst({
+      where: and(eq(fileHashes.repoId, this.repoId), eq(fileHashes.filePath, filePath)),
+    });
+    return row?.sha256 ?? null;
+  }
+
+  async setFileHash(filePath: string, hash: string): Promise<void> {
+    await this.db
+      .insert(fileHashes)
+      .values({
+        repoId: this.repoId,
+        filePath,
+        sha256: hash,
+        updatedAt: new Date(),
+      })
+      .onConflictDoUpdate({
+        target: [fileHashes.repoId, fileHashes.filePath],
+        set: { sha256: hash, updatedAt: new Date() },
+      });
+  }
+
+  async getDirHash(dirPath: string): Promise<string | null> {
+    const row = await this.db.query.dirHashes.findFirst({
+      where: and(eq(dirHashes.repoId, this.repoId), eq(dirHashes.dirPath, dirPath)),
+    });
+    return row?.merkleHash ?? null;
+  }
+
+  async setDirHash(dirPath: string, hash: string): Promise<void> {
+    await this.db
+      .insert(dirHashes)
+      .values({
+        repoId: this.repoId,
+        dirPath,
+        merkleHash: hash,
+        updatedAt: new Date(),
+      })
+      .onConflictDoUpdate({
+        target: [dirHashes.repoId, dirHashes.dirPath],
+        set: { merkleHash: hash, updatedAt: new Date() },
+      });
+  }
+
+  async clearDirHashes(): Promise<void> {
+    await this.db.delete(dirHashes).where(eq(dirHashes.repoId, this.repoId));
+  }
+
+  async getAllFileHashes(): Promise<Map<string, string>> {
+    const rows = await this.db.query.fileHashes.findMany({
+      where: eq(fileHashes.repoId, this.repoId),
+    });
+    const map = new Map<string, string>();
+    for (const row of rows) {
+      map.set(row.filePath, row.sha256);
+    }
+    return map;
+  }
+
+  async transaction<T>(fn: () => Promise<T>): Promise<T> {
+    // Neon HTTP driver does not support interactive transactions.
+    // Each query is auto-committed, so we just run fn sequentially.
+    return fn();
+  }
+}

apps/trigger/src/lib/github.ts

@@ -0,0 +1,76 @@
+import { createSign, createPrivateKey } from 'node:crypto';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { extract } from 'tar';
+
+function createAppJWT(appId: string, privateKey: string): string {
+  const now = Math.floor(Date.now() / 1000);
+  const header = Buffer.from(JSON.stringify({ alg: 'RS256', typ: 'JWT' })).toString('base64url');
+  const payload = Buffer.from(
+    JSON.stringify({ iat: now - 60, exp: now + 600, iss: appId }),
+  ).toString('base64url');
+  const key = createPrivateKey(privateKey);
+  const signature = createSign('RSA-SHA256').update(`${header}.${payload}`).sign(key, 'base64url');
+  return `${header}.${payload}.${signature}`;
+}
+
+async function getInstallationToken(installationId: number, appJwt: string): Promise<string> {
+  const res = await fetch(
+    `https://api.github.com/app/installations/${installationId}/access_tokens`,
+    {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${appJwt}`,
+        Accept: 'application/vnd.github+json',
+        'User-Agent': 'CodeIndexer/1.0',
+        'X-GitHub-Api-Version': '2022-11-28',
+      },
+    },
+  );
+  if (!res.ok) throw new Error(`Failed to get installation token: ${res.status}`);
+  const data = (await res.json()) as { token: string };
+  return data.token;
+}
+
+async function downloadAndExtractTarball(
+  fullName: string,
+  ref: string,
+  token: string,
+  destDir: string,
+  tarballPath: string,
+): Promise<string> {
+  // Use redirect: 'manual' to capture the SHA from the redirect Location URL
+  // GitHub returns 302 → codeload.github.com/.../{sha}.tar.gz
+  const headRes = await fetch(`https://api.github.com/repos/${fullName}/tarball/${ref}`, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+      Accept: 'application/vnd.github+json',
+      'User-Agent': 'CodeIndexer/1.0',
+    },
+    redirect: 'manual',
+  });
+
+  const location = headRes.headers.get('location') ?? '';
+  const shaMatch =
+    location.match(/\/([a-f0-9]{40})\.tar\.gz/) ?? location.match(/\/([a-f0-9]{7,40})/);
+  const headSha = shaMatch?.[1] ?? 'unknown';
+
+  if (!location) throw new Error(`Tarball download failed: no redirect (status ${headRes.status})`);
+
+  // Follow the redirect and save tarball to disk
+  const res = await fetch(location);
+  if (!res.ok) throw new Error(`Tarball download failed: ${res.status}`);
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  await pipeline(Readable.fromWeb(res.body as any), createWriteStream(tarballPath));
+
+  // Extract tarball with strip: 1 to remove GitHub's wrapper directory
+  await mkdir(destDir, { recursive: true });
+  await extract({ file: tarballPath, cwd: destDir, strip: 1 });
+
+  return headSha;
+}
+
+export { createAppJWT, getInstallationToken, downloadAndExtractTarball };

apps/trigger/src/lib/r2.ts

@@ -0,0 +1,81 @@
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+let _client: S3Client | null = null;
+
+function getR2Client(): S3Client {
+  if (!_client) {
+    _client = new S3Client({
+      region: 'auto',
+      endpoint: process.env.R2_ENDPOINT!,
+      credentials: {
+        accessKeyId: process.env.R2_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
+      },
+    });
+  }
+  return _client;
+}
+
+const BUCKET = () => process.env.R2_BUCKET!;
+
+async function uploadBuffer(key: string, body: Buffer, contentType = 'application/octet-stream') {
+  await getR2Client().send(
+    new PutObjectCommand({
+      Bucket: BUCKET(),
+      Key: key,
+      Body: body,
+      ContentType: contentType,
+    }),
+  );
+}
+
+async function uploadFile(key: string, filePath: string) {
+  const body = await readFile(filePath);
+  await uploadBuffer(key, body);
+}
+
+async function uploadRepoFiles(
+  repoId: string,
+  files: string[],
+  cloneDir: string,
+  concurrency = 20,
+) {
+  const client = getR2Client();
+  const bucket = BUCKET();
+
+  for (let i = 0; i < files.length; i += concurrency) {
+    const batch = files.slice(i, i + concurrency);
+    await Promise.all(
+      batch.map(async (file) => {
+        const relativePath = path.relative(cloneDir, file);
+        const body = await readFile(file);
+        await client.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: `repos/${repoId}/files/${relativePath}`,
+            Body: body,
+          }),
+        );
+      }),
+    );
+  }
+}
+
+function buildFileTree(files: string[], cloneDir: string): Record<string, unknown> {
+  const tree: Record<string, unknown> = {};
+  for (const file of files) {
+    const rel = path.relative(cloneDir, file);
+    const parts = rel.split(path.sep);
+    let node = tree as Record<string, unknown>;
+    for (let i = 0; i < parts.length - 1; i++) {
+      node[parts[i]] = (node[parts[i]] as Record<string, unknown>) || {};
+      node = node[parts[i]] as Record<string, unknown>;
+    }
+    node[parts[parts.length - 1]] = null; // leaf = file
+  }
+  return tree;
+}
+
+export { uploadBuffer, uploadFile, uploadRepoFiles, buildFileTree, getR2Client };