diff --git a/backend/src/api-key.ts b/backend/src/api-key.ts new file mode 100644 index 0000000..b111857 --- /dev/null +++ b/backend/src/api-key.ts @@ -0,0 +1,90 @@ +import { createHash, randomBytes } from "node:crypto"; +import type { FastifyReply, FastifyRequest } from "fastify"; + +import { env } from "./env.js"; +import { convex, internal } from "./convex.js"; +import { LOCAL_USER_ID } from "./local-credentials.js"; + +const KEY_PREFIX_LEN = 8; + +function hashKey(key: string): string { + return createHash("sha256").update(key).digest("hex"); +} + +export function generateApiKey(): { key: string; keyHash: string; keyPrefix: string } { + const random = randomBytes(32).toString("base64url"); + const key = `bsk_${random}`; + const keyHash = hashKey(key); + const keyPrefix = key.slice(0, KEY_PREFIX_LEN); + return { key, keyHash, keyPrefix }; +} + +export function apiKeyHash(key: string): string { + return hashKey(key); +} + +export async function requireApiKey( + req: FastifyRequest, + reply: FastifyReply, +): Promise { + const header = req.headers["x-api-key"]; + const authHeader = req.headers.authorization; + let rawKey: string | undefined; + if (typeof header === "string" && header.startsWith("bsk_")) { + rawKey = header; + } else if (typeof authHeader === "string" && authHeader.startsWith("Bearer bsk_")) { + rawKey = authHeader.slice("Bearer ".length); + } + if (!rawKey) return false; + + const keyHash = hashKey(rawKey); + const record = await convex.query(internal.apiKeys.lookupByHash, { keyHash }); + if (!record) { + await reply.code(401).send({ error: "Invalid API key" }); + return true; + } + + req.auth = { userId: record.ownerId }; + + void convex + .mutation(internal.apiKeys.touchLastUsed, { + id: record._id, + lastUsedAt: Date.now(), + }) + .catch(() => {}); + + return true; +} + +/** + * Try API-key auth. Returns: + * - `true` if the request had an X-API-Key / Bearer bsk_… header + * (either the key was accepted and req.auth is set, or the response + * was already sent with 401). Caller must NOT continue. + * - `false` if no API key was presented. Caller should fall through + * to Clerk (or whatever its next auth mechanism is). + */ +export async function tryApiKeyAuth( + req: FastifyRequest, + reply: FastifyReply, +): Promise { + const header = req.headers["x-api-key"]; + const authHeader = req.headers.authorization; + const hasKey = + (typeof header === "string" && header.startsWith("bsk_")) || + (typeof authHeader === "string" && authHeader.startsWith("Bearer bsk_")); + if (!hasKey) return false; + await requireApiKey(req, reply); + return true; +} + +export async function requireApiKeyOrCli( + req: FastifyRequest, + reply: FastifyReply, +): Promise { + if (env.IS_LOCAL_MODE) { + req.auth = { userId: LOCAL_USER_ID }; + return true; + } + return requireApiKey(req, reply); +} diff --git a/backend/src/clerk-auth.ts b/backend/src/clerk-auth.ts index de28dca..85f4583 100644 --- a/backend/src/clerk-auth.ts +++ b/backend/src/clerk-auth.ts @@ -9,6 +9,7 @@ import { createClerkClient, type ClerkClient } from "@clerk/backend"; import { env } from "./env.js"; import { LOCAL_USER_ID } from "./local-credentials.js"; +import { tryApiKeyAuth } from "./api-key.js"; /** * Clerk JWT verification for the Fastify backend. @@ -79,13 +80,10 @@ export async function getUserEmail( export default fp(clerkPlugin, { name: "clerk-auth" }); /** - * Fastify preHandler that requires a valid Clerk session token. - * - * Reads `Authorization: Bearer `, verifies it via Clerk's - * `authenticateRequest`, and attaches `req.auth = { userId }` on success. - * Returns 401 otherwise. + * Fastify preHandler that requires a valid Clerk session token — no API key fallback. + * Use for routes that must only be accessible via Clerk (e.g. API key management). */ -export async function requireAuth( +export async function requireClerkAuth( req: FastifyRequest, reply: FastifyReply, ): Promise { @@ -100,8 +98,6 @@ export async function requireAuth( return; } - // Wrap the Fastify request just enough for Clerk's authenticateRequest API. - // Clerk accepts a Web Request; build one from the headers we care about. const headers = new Headers(); for (const [k, v] of Object.entries(req.headers)) { if (typeof v === "string") headers.set(k, v); @@ -116,7 +112,6 @@ export async function requireAuth( const requestState = await req.server.clerk.authenticateRequest( clerkRequest, { - // Anyone consuming our backend is our own frontend; lock to its origin. authorizedParties: [env.CLIENT_ORIGIN], }, ); @@ -134,3 +129,23 @@ export async function requireAuth( req.auth = { userId: auth.userId }; } + +/** + * Fastify preHandler that accepts either a Clerk session token or a valid API key. + * Use for routes that should be accessible from both browser (Clerk) and + * programmatic clients (API key). + */ +export async function requireAuth( + req: FastifyRequest, + reply: FastifyReply, +): Promise { + if (env.IS_LOCAL_MODE) { + req.auth = { userId: LOCAL_USER_ID }; + return; + } + + const apiKeyHandled = await tryApiKeyAuth(req, reply); + if (apiKeyHandled) return; + + await requireClerkAuth(req, reply); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 4d63f42..cb97ed2 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -4,7 +4,7 @@ import type { ClerkClient } from "@clerk/backend"; import { z } from "zod"; import { env } from "./env.js"; -import clerkAuthPlugin, { requireAuth, getUserEmail } from "./clerk-auth.js"; +import clerkAuthPlugin, { requireAuth, requireClerkAuth, getUserEmail } from "./clerk-auth.js"; import { inferSchema } from "./pipeline/schema-inference.js"; import { datasetContextSchema, type DatasetContext } from "./pipeline/populate.js"; import { populateWorkflow } from "./mastra/workflows/populate.js"; @@ -25,6 +25,7 @@ import { verifyOpenRouterApiKey, verifyTinyFishApiKey, } from "./local-credentials.js"; +import { generateApiKey } from "./api-key.js"; /** Domain part of an email, for analytics (we never log full addresses). */ function emailDomain(email: string): string { @@ -362,6 +363,82 @@ async function runUpdateWorkflowInBackground({ } } +/** + * Background runner for the enrichment workflow. + * Processes rows concurrently to fill empty columns with AI-researched data. + */ +async function runEnrichWorkflowInBackground({ + input, + run, + authorizedUserId, + logger, +}: { + input: { + datasetId: string; + sourceColumns: string[]; + targetColumns: Array<{ name: string; type: "text" | "number" | "boolean" | "url" | "date"; description?: string }>; + authContext: { + authorizedUserId: string; + workflowRunId: string; + modelConfig: { + schemaInference: string; + populateOrchestrator: string; + investigateSubagent: string; + }; + }; + enrichmentRunId: string; + }; + run: EnrichWorkflowRun; + authorizedUserId: string; + logger: FastifyBaseLogger; +}): Promise { + const datasetId = input.datasetId; + + try { + const result = await run.start({ + inputData: input, + }); + + logger.info( + { + workflowStatus: result.status, + steps: JSON.stringify(result.steps).slice(0, 2000), + }, + "Enrich workflow completed", + ); + + if (result.status !== "success") { + throw new Error(`Enrich workflow ended with status: ${result.status}`); + } + + logger.info( + { datasetId, result: JSON.stringify(result) }, + "Enrich workflow finished", + ); + } catch (err) { + const lastStatusError = statusErrorMessage(err); + logger.error({ err, datasetId }, "Enrich background workflow failed"); + + // Mark the enrichment run as failed + try { + await convex.mutation(internal.sheetsEnrichmentRuns.complete, { + id: input.enrichmentRunId as any, + rowsProcessed: 0, + rowsUpdated: 0, + rowsFound: 0, + status: "failed", + }); + } catch (statusErr) { + logger.error( + { err: statusErr, datasetId }, + "Failed to mark enrichment run as failed", + ); + } + } finally { + deregisterDataset(datasetId); + } +} + async function runScheduledUpdateWorkflowInBackground({ input, run, @@ -693,7 +770,7 @@ if (env.IS_LOCAL_MODE) { await fastify.register(fastifyCors, { origin: Array.from(allowedCorsOrigins), methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], - allowedHeaders: ["Content-Type", "Authorization", "Cookie"], + allowedHeaders: ["Content-Type", "Authorization", "Cookie", "X-API-Key"], credentials: true, maxAge: 86400, }); @@ -1145,6 +1222,7 @@ await fastify.register(async (instance) => { instance.post("/populate", async (req, reply) => { const parsed = datasetContextSchema.safeParse(req.body); if (!parsed.success) { + req.log.info({ validationErrors: parsed.error.flatten().fieldErrors }, "populate validation failed"); return reply.code(400).send({ error: "Invalid request", details: parsed.error.flatten().fieldErrors, @@ -1368,6 +1446,420 @@ await fastify.register(async (instance) => { return reply.code(502).send({ error: "Failed to stop dataset run. Please try again." }); } }); + + // ──────────────────────────────────────────────────────────────────────── + // API key management — generate / list / revoke. Only accessible with + // a Clerk session (browser/frontend). Non-browser clients use the issued + // key via the X-API-Key header on the other protected routes. + // ──────────────────────────────────────────────────────────────────────── + + await instance.register(async (apiKeysInstance) => { + apiKeysInstance.addHook("preHandler", requireClerkAuth); + + apiKeysInstance.post("/api-keys", async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + const body = req.body as { name?: string }; + const name = (body?.name ?? "Default key").toString().slice(0, 60) || "Default key"; + + try { + const { key, keyHash, keyPrefix } = generateApiKey(); + const { id } = await convex.mutation(internal.apiKeys.create, { + ownerId: auth.userId, + name, + keyHash, + keyPrefix, + createdAt: Date.now(), + }); + return reply.code(201).send({ id, key, keyPrefix, name }); + } catch (err) { + req.log.error(err, "Failed to create API key"); + return reply.code(500).send({ error: "Failed to create API key" }); + } + }); + + apiKeysInstance.get("/api-keys", async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + try { + const rows = await convex.query(internal.apiKeys.listByOwner, { ownerId: auth.userId }); + return { + keys: rows + .filter((k: { revokedAt?: number }) => !k.revokedAt) + .map((k: { _id: unknown; name: string; keyPrefix: string; createdAt: number; lastUsedAt?: number }) => ({ + id: k._id, + name: k.name, + keyPrefix: k.keyPrefix, + createdAt: k.createdAt, + lastUsedAt: k.lastUsedAt ?? null, + })), + }; + } catch (err) { + req.log.error(err, "Failed to list API keys"); + return reply.code(500).send({ error: "Failed to list API keys" }); + } + }); + + apiKeysInstance.delete<{ Params: { id: string } }>("/api-keys/:id", async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + const id = req.params?.id; + if (!id) return reply.code(400).send({ error: "id is required" }); + + try { + const result = await convex.mutation(internal.apiKeys.revoke, { + id: id as never, + ownerId: auth.userId, + revokedAt: Date.now(), + }); + if (!result) return reply.code(404).send({ error: "Key not found" }); + return { success: true }; + } catch (err) { + req.log.error(err, "Failed to revoke API key"); + return reply.code(500).send({ error: "Failed to revoke API key" }); + } + }); + }); +}); + +// ──────────────────────────────────────────────────────────────────────── +// Sheets add-on routes — accept either Clerk JWT OR API key. +// These mirror the CLI routes but live in the protected (or local) auth +// path so the add-on works for both self-hosted local users and +// Clerk-authenticated cloud users. +// ──────────────────────────────────────────────────────────────────────── + +const addonCreateDatasetSchema = z.object({ + name: z.string().trim().min(1).max(120), + description: z.string().trim().min(1), + columns: z.array( + z.object({ + name: z.string().trim().min(1), + type: z.enum(["text", "number", "boolean", "url", "date"]), + description: z.optional(z.string()), + isPrimaryKey: z.optional(z.boolean()), + }), + ), + retrievalStrategy: z.optional( + z.enum(["search_fetch", "browser", "hybrid"]), + ), + sourceHint: z.optional(z.string()), + maxRowCount: z.number().int().min(1).max(2500).default(100), + refreshCadence: refreshCadenceSchema.default("manual"), +}); + +await fastify.register(async (instance) => { + instance.addHook("preHandler", requireAuth); + + instance.post("/addon/datasets", async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + const parsed = addonCreateDatasetSchema.safeParse(req.body); + if (!parsed.success) { + return reply.code(400).send({ + error: "Invalid request", + details: parsed.error.flatten().fieldErrors, + }); + } + + try { + const datasetId = await convex.mutation( + internal.datasets.createForOwnerInternal, + { + ownerId: auth.userId, + name: parsed.data.name, + description: parsed.data.description, + refreshCadence: parsed.data.refreshCadence, + maxRowCount: parsed.data.maxRowCount, + columns: parsed.data.columns, + retrievalStrategy: parsed.data.retrievalStrategy, + sourceHint: parsed.data.sourceHint, + }, + ); + + const dataset = await convex.query(internal.datasets.getInternal, { + id: datasetId, + }); + return reply.code(201).send({ dataset }); + } catch (err) { + req.log.error(err, "Addon dataset create failed"); + return reply.code(502).send({ error: "Failed to create dataset" }); + } + }); + + instance.get("/addon/datasets", async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + try { + const datasets = await convex.query(internal.datasets.listByOwnerInternal, { + ownerId: auth.userId, + }); + return { datasets }; + } catch (err) { + req.log.error(err, "Addon dataset list failed"); + return reply.code(502).send({ error: "Failed to list datasets" }); + } + }); + + instance.get("/addon/datasets/public", async (req, reply) => { + try { + const datasets = await convex.query(internal.datasets.listPublicInternal, {}); + return { datasets }; + } catch (err) { + req.log.error(err, "Addon public dataset list failed"); + return reply.code(502).send({ error: "Failed to list public datasets" }); + } + }); + + instance.get<{ Params: { datasetId: string } }>( + "/addon/datasets/public/:datasetId/rows", + async (req, reply) => { + const params = cliDatasetIdParamsSchema.safeParse(req.params); + if (!params.success) { + return reply.code(400).send({ error: "datasetId is required" }); + } + + try { + const dataset = await convex.query(internal.datasets.getInternal, { + id: params.data.datasetId, + }); + if (!dataset || dataset.visibility !== "public") { + return reply.code(404).send({ error: "Dataset not found" }); + } + const rows = await convex.query(internal.datasetRows.listInternal, { + datasetId: params.data.datasetId, + }); + return { rows, dataset }; + } catch (err) { + req.log.error(err, "Addon public rows get failed"); + return reply.code(400).send({ error: "Invalid datasetId" }); + } + }, + ); + + instance.get<{ Params: { datasetId: string } }>( + "/addon/datasets/:datasetId", + async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + const params = cliDatasetIdParamsSchema.safeParse(req.params); + if (!params.success) { + return reply.code(400).send({ error: "datasetId is required" }); + } + + try { + const dataset = await convex.query(internal.datasets.getInternal, { + id: params.data.datasetId, + }); + if (!dataset || dataset.ownerId !== auth.userId) { + return reply.code(404).send({ error: "Dataset not found" }); + } + return { dataset }; + } catch (err) { + req.log.error(err, "Addon dataset get failed"); + return reply.code(400).send({ error: "Invalid datasetId" }); + } + }, + ); + + instance.get<{ Params: { datasetId: string } }>( + "/addon/datasets/:datasetId/rows", + async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + const params = cliDatasetIdParamsSchema.safeParse(req.params); + if (!params.success) { + return reply.code(400).send({ error: "datasetId is required" }); + } + + try { + const dataset = await convex.query(internal.datasets.getInternal, { + id: params.data.datasetId, + }); + const isPublic = (dataset as Record | null)?.visibility === "public"; + if (!dataset || (dataset.ownerId !== auth.userId && !isPublic)) { + return reply.code(404).send({ error: "Dataset not found" }); + } + const rows = await convex.query(internal.datasetRows.listInternal, { + datasetId: params.data.datasetId, + }); + if (!rows) return reply.code(404).send({ error: "Dataset not found" }); + return { rows, dataset }; + } catch (err) { + req.log.error(err, "Addon rows get failed"); + return reply.code(400).send({ error: "Invalid datasetId" }); + } + }, + ); + + // ──────────────────────────────────────────────────────────────────────── + // Data Enrichment — fill empty columns directly from sheet data + // Independent of BigSet datasets + // ──────────────────────────────────────────────────────────────────────── + + /** + * Enrich rows directly from sheet data. + * POST /sheets/enrich + * + * Request body: + * { + * sourceColumns: string[], // Columns with identifying data + * targetColumns: string[], // Columns to fill + * rows: Array<{ // Rows to enrich + * rowIndex: number, // Row number in the sheet + * sourceData: Record // Current values in source columns + * }> + * } + * + * Response: + * { + * results: Array<{ + * rowIndex: number, + * values: Record, // Enriched values to write + * error?: string + * }>, + * stats: { + * rowsProcessed: number, + * rowsEnriched: number, + * rowsWithErrors: number + * } + * } + */ + instance.post<{ + Body: { + sourceColumns: string[]; + targetColumns: string[]; + rows: Array<{ + rowIndex: number; + sourceData: Record; + targetColumns?: string[]; + }>; + }; + }>("/sheets/enrich", async (req, reply) => { + const auth = req.auth; + if (!auth) return reply.code(401).send({ error: "Authentication required" }); + + const body = req.body; + if (!Array.isArray(body.sourceColumns) || body.sourceColumns.length === 0) { + return reply.code(400).send({ error: "At least one source column is required" }); + } + if (!Array.isArray(body.targetColumns) || body.targetColumns.length === 0) { + return reply.code(400).send({ error: "At least one target column is required" }); + } + if (!Array.isArray(body.rows) || body.rows.length === 0) { + return reply.code(400).send({ error: "At least one row is required" }); + } + + if (!(await ensureLocalSetupReady(reply))) return; + + try { + const { getModelConfig } = await import("./config/models.js"); + const modelConfig = await getModelConfig(auth.userId); + const { requireOpenRouterApiKey } = await import("./local-credentials.js"); + const openRouterApiKey = await requireOpenRouterApiKey(); + + const { buildEnrichAgent, parseEnrichResponse } = await import("./mastra/agents/enrich-agent.js"); + + const results: Array<{ + rowIndex: number; + values: Record; + error?: string; + }> = new Array(body.rows.length); + let rowsEnriched = 0; + let rowsWithErrors = 0; + + const mockAuthContext = { + authorizedUserId: auth.userId, + workflowRunId: `enrich-${Date.now()}`, + modelConfig, + }; + + const MAX_CONCURRENT = 5; + let idx = 0; + + const workers = Array.from( + { length: Math.min(MAX_CONCURRENT, body.rows.length) }, + async () => { + while (idx < body.rows.length) { + const i = idx++; + const row = body.rows[i]; + try { + const sourceData: Record = {}; + for (const col of body.sourceColumns) { + if (row.sourceData[col] !== undefined && row.sourceData[col] !== null && row.sourceData[col] !== "") { + sourceData[col] = row.sourceData[col]; + } + } + + if (Object.keys(sourceData).length === 0) { + results[i] = { rowIndex: row.rowIndex, values: {}, error: "No source data" }; + continue; + } + + const rowTargetCols = row.targetColumns ?? body.targetColumns; + const columnsToFill = rowTargetCols.map((name: string) => ({ + name, + type: "text" as const, + })); + + const agent = buildEnrichAgent( + "sheet-enrichment", + mockAuthContext as any, + sourceData, + columnsToFill, + openRouterApiKey, + ); + + const result = await agent.generate( + `Research and find values for these columns based on the entity information provided. Return ONLY a JSON object with the found values.`, + { maxSteps: 10 } + ); + + const foundValues = parseEnrichResponse(result.text, rowTargetCols); + + if (foundValues && Object.keys(foundValues).length > 0) { + const cleanValues: Record = {}; + for (const [col, val] of Object.entries(foundValues)) { + if (val !== null && val !== undefined && val !== "") { + cleanValues[col] = val; + } + } + results[i] = { rowIndex: row.rowIndex, values: cleanValues }; + if (Object.keys(cleanValues).length > 0) rowsEnriched++; + } else { + results[i] = { rowIndex: row.rowIndex, values: {}, error: "No values found" }; + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + results[i] = { rowIndex: row.rowIndex, values: {}, error: msg }; + rowsWithErrors++; + } + } + } + ); + + await Promise.allSettled(workers); + + return { + results, + stats: { + rowsProcessed: body.rows.length, + rowsEnriched, + rowsWithErrors, + }, + }; + } catch (err) { + req.log.error(err, "Enrich processing failed"); + return reply.code(502).send({ error: "Failed to process enrichment. Please try again." }); + } + }); }); try { diff --git a/backend/src/mastra/agents/enrich-agent.ts b/backend/src/mastra/agents/enrich-agent.ts new file mode 100644 index 0000000..538f5a5 --- /dev/null +++ b/backend/src/mastra/agents/enrich-agent.ts @@ -0,0 +1,106 @@ +import { Agent } from "@mastra/core/agent"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; +import type { AuthContext } from "../workflows/populate.js"; + +export interface EnrichColumn { + name: string; + type: "text" | "number" | "boolean" | "url" | "date"; + description?: string; +} + +function buildEnrichInstructions( + sourceData: Record, + targetColumns: EnrichColumn[], +): string { + const sources = Object.entries(sourceData) + .map(([k, v]) => `- ${k}: ${v}`) + .join("\n"); + + const targets = targetColumns + .map((c) => `- "${c.name}" (${c.type})${c.description ? `: ${c.description}` : ""}`) + .join("\n"); + + return `You research one entity and fill in missing fields. Be fast — you have very few steps. + +KNOWN INFORMATION (use this to identify the entity): +${sources} + +FIELDS TO FIND (research these): +${targets} + +RULES: +- Do NOT fetch the same URL twice. +- You have at most 6 tool calls total. Budget them: 1 search + 1-2 fetches = done. +- ALWAYS return a value for every field, even if partial. Use "" for unknown text fields, 0 for unknown numbers. +- Never fabricate values. Use "" or 0 for anything you cannot verify. +- Return ONLY a JSON object — no extra text, no markdown, no explanation. + +TOOL CALL FORMAT: + search_web: {"query": "your search terms"} + fetch_page: {"url": "https://example.com/page"} + +WORKFLOW: +1. Search the web for this entity using the known information. +2. Fetch 1-2 of the best results. +3. Extract the missing field values from fetched content. +4. Return JSON with ONLY the target fields as keys. + +EXAMPLE RESPONSE: +${JSON.stringify(Object.fromEntries(targetColumns.map((c) => [c.name, "value"])))} + +Return your JSON response now.`; +} + +export function buildEnrichAgent( + _authorizedDatasetId: string, + authContext: AuthContext, + sourceData: Record, + targetColumns: EnrichColumn[], + openRouterApiKey: string, +): Agent { + if (!authContext.modelConfig?.investigateSubagent) { + throw new Error("modelConfig.investigateSubagent is not configured"); + } + const modelSlug = authContext.modelConfig.investigateSubagent; + const openrouter = createOpenRouter({ + apiKey: openRouterApiKey, + baseURL: process.env.OPENROUTER_BASE_URL, + }); + + return new Agent({ + id: "enrich-agent", + name: "Data Enrichment Agent", + instructions: buildEnrichInstructions(sourceData, targetColumns), + model: openrouter(modelSlug), + tools: { + search_web: searchWebTool, + fetch_page: fetchPageTool, + }, + }); +} + +export function parseEnrichResponse( + text: string, + targetColumns: string[], +): Record | null { + let jsonStr = text.trim(); + if (jsonStr.startsWith("```")) { + const match = jsonStr.match(/```(?:json)?\n?([\s\S]*?)\n?```/); + if (match) jsonStr = match[1]; + } + const braceMatch = jsonStr.match(/\{[\s\S]*\}/); + if (!braceMatch) return null; + try { + const parsed = JSON.parse(braceMatch[0]); + const result: Record = {}; + for (const col of targetColumns) { + if (col in parsed && parsed[col] !== null && parsed[col] !== "") { + result[col] = parsed[col]; + } + } + return result; + } catch { + return null; + } +} \ No newline at end of file diff --git a/bigset-addon/.claspignore b/bigset-addon/.claspignore new file mode 100644 index 0000000..f145211 --- /dev/null +++ b/bigset-addon/.claspignore @@ -0,0 +1,12 @@ +node_modules/ +sidebar/ +.git/ +*.md +.clasp.json +package*.json +tsconfig.json +vite.config.ts +*.ts + +!src/** +src/**/*.ts diff --git a/bigset-addon/.gitignore b/bigset-addon/.gitignore new file mode 100644 index 0000000..bd59355 --- /dev/null +++ b/bigset-addon/.gitignore @@ -0,0 +1,29 @@ +node_modules/ +package-lock.json +appsscript.json +.clasp.json +sidebar/node_modules +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/bigset-addon/README.md b/bigset-addon/README.md new file mode 100644 index 0000000..d41644c --- /dev/null +++ b/bigset-addon/README.md @@ -0,0 +1,145 @@ +# BigSet Google Sheet Add-on + +Build and enrich datasets from any Google Sheet using natural language. The add-on lets you: + +- **Generate** — describe a dataset in plain English and the add-on creates the schema and populates it via web search +- **Enrich** — fill in missing cells in an existing sheet using AI-powered research +- **Insert** — write any dataset back into the active sheet + +## Architecture + +```text +bigset-addon/ +├── src/ # Google Apps Script (TypeScript → pushed to Google) +│ ├── index.ts # Add-on entry: menu, sidebar, HTTP proxy, sheet ops +│ ├── appsscript.json # Manifest (OAuth scopes, triggers) +│ └── sidebar.html # Built sidebar UI (from sidebar/ build) +│ +└── sidebar/ # Svelte SPA (builds into src/) + ├── src/ + │ ├── api/ # google.script.run client + │ ├── components/ # Shared UI components + │ ├── pages/ # Route pages + enrich/ sub-pages + │ ├── stores/ # Svelte writable stores + │ ├── App.svelte # Router (#/, #/settings, etc.) + │ └── main.ts # Sidebar entry point + └── sidebar.html # HTML shell for the SPA +``` + +**Deployment flow:** `sidebar/` builds → `src/` receives built HTML/JS → `tsc` compiles `index.ts` → `index.js` → `clasp push` deploys to Google Apps Script. + +## Setup + +### 1. Install dependencies + +```bash +npm install +``` + +### 2. Create the Apps Script project + +You need an Apps Script project to push to. Create one at : + +1. Click **+ New project** (top left). +2. Rename it to `BigSet Addon` (click the title in the toolbar). +3. Copy the **script ID** from the URL bar — it looks like: + ```text + https://script.google.com/home/projects/1Lu2qMd7YE8QUvECiKQpPdv-1_tQU-LAMWRvVUHQ-m5lEIG-q0HPQ1X-t/edit + ``` + The long string between `/projects/` and `/edit` is your script ID. +4. Open `.clasp.json` in this repo and paste the ID as `scriptId`. + +> **Alternatively:** `npm run clasp:create` will create a brand-new Apps Script project and write the ID to `.clasp.json` for you. Note: this only works for the very first setup — running it again will overwrite the existing project. + +### 3. Install the manifest + +The repo ships an `appsscript.json` that declares the OAuth scopes and add-on metadata. To load it into your new project: + +1. In the Apps Script editor, open **Project Settings** (gear icon, left sidebar). +2. Enable **"Show 'appsscript.json' manifest in editor"**. +3. Go back to **Editor** and click `appsscript.json`. +4. Paste the contents of `src/appsscript.json` from this repo and save. + +### 4. Log in with clasp + +```bash +npm run clasp:login +``` + +This opens a browser window for Google OAuth. After authorizing, your credentials are stored locally. + +### 5. Build and push + +```bash +npm run build && clasp push +``` + +This: +- Builds the Svelte sidebar → copies HTML/JS into `src/` +- Compiles `src/index.ts` → `src/index.js` (TypeScript types are stripped) +- Pushes everything in `src/` to the Apps Script project + +After the first push, the add-on is live in your Apps Script project. Open any Google Sheet, refresh, and the **BigSet → Open** menu item should appear. + +## Developing the sidebar + +```bash +cd sidebar +npm install +npm run dev # Vite dev server at localhost:5173 +``` + +The sidebar dev server proxies `google.script.run` calls to the Apps Script environment. Changes to Svelte files hot-reload without a full deploy. + +To test against a live backend, set the sidebar's environment variable: + +```bash +VITE_DASHBOARD_URL=https://your-dashboard-url.com npm run dev +``` + +## Sheet permissions + +The add-on uses `spreadsheets.currentonly` scope — it can only read the active spreadsheet. Required scopes are declared in `src/appsscript.json`. + +## Backend requirements + +The add-on communicates with the BigSet backend API. On first use, enter your backend URL and API key in **Settings**. Per-user credentials are stored via `PropertiesService.getUserProperties()` so each user maintains their own authentication. + +## Config files reference + +The project has two layers (Apps Script side and Svelte SPA side), each with its own config. They look similar but serve different toolchains — don't merge them. + +### Apps Script side (repo root) + +**`package.json`** — Build orchestration for the Apps Script side. Scripts: `build` (chains sidebar build + tsc + copy), `build:tsc` (compiles `src/index.ts` → `src/index.js`), `clasp:create` / `clasp:login` / `deploy`. Dev deps: `typescript`, `@google/clasp`, `@types/google-apps-script`, `shx`, `npm-run-all`. + +**`tsconfig.json`** — TypeScript config for the server-side code. `target: ES2020`, `module: none` (Apps Script uses globals, not modules), `outDir: ./src` (emits `index.js` next to `index.ts`), `types: ["google-apps-script"]` (loads globals like `SpreadsheetApp`, `HtmlService`, `PropertiesService`). Includes only `src/**/*.ts`, excludes `node_modules` and `sidebar/`. + +**`.clasp.json`** — clasp config. `scriptId` is the Apps Script project ID (see Setup step 2). `rootDir: "src"` tells clasp what to push. `scriptExtensions` lists which file types to upload. + +**`.claspignore`** — Excludes everything except `src/` from the push (drops `node_modules/`, `sidebar/`, configs, docs). Also excludes `*.ts` after `!src/**` so the source `index.ts` isn't pushed — only the compiled `index.js` goes to Apps Script. + +**`src/appsscript.json`** — Apps Script manifest. Declares OAuth scopes (`spreadsheets.currentonly`), add-on metadata, runtime version. **Don't regenerate** this with `clasp create` or you'll overwrite scopes and add-on config. + +### Sidebar side (`sidebar/`) + +**`package.json`** — Svelte SPA dependencies and scripts. `dev` (Vite dev server), `build` (Vite production build → `dist/`). Dev deps: `vite`, `svelte`, `@sveltejs/vite-plugin-svelte`, `vite-plugin-singlefile` (inlines everything into one HTML file), `typescript`. + +**`tsconfig.json`** — TS config for the SPA. `target: ESNext`, `module: ESNext`, `moduleResolution: bundler`. Extends `@tsconfig/svelte` which sets the compiler options Svelte expects. Type-checks `src/**/*.ts` and `*.svelte` files for editor IntelliSense. + +**`tsconfig.node.json`** — Separate tsconfig for `vite.config.ts` itself, since the build config runs in Node (not the browser). Referenced via `references` in the main tsconfig. + +**`vite.config.ts`** — Vite build config. Registers the Svelte plugin and `vite-plugin-singlefile` (collapses all JS/CSS into a single HTML file — required for Apps Script's `HtmlService`). `rollupOptions.input` points at `sidebar.html` as the entry HTML shell. + +**`svelte.config.js`** — Tells the Svelte compiler to use Vite's preprocessor (handles ` + + +
+ + + diff --git a/bigset-addon/sidebar/src/App.svelte b/bigset-addon/sidebar/src/App.svelte new file mode 100644 index 0000000..848bfb9 --- /dev/null +++ b/bigset-addon/sidebar/src/App.svelte @@ -0,0 +1,22 @@ + + + + + diff --git a/bigset-addon/sidebar/src/api/client.ts b/bigset-addon/sidebar/src/api/client.ts new file mode 100644 index 0000000..a3e4306 --- /dev/null +++ b/bigset-addon/sidebar/src/api/client.ts @@ -0,0 +1,238 @@ +/** + * Thin client over `google.script.run` for the sidebar. Apps Script can't + * make HTTP calls directly from the iframe, but the server-side + * (src/index.ts in the Apps Script project) can via UrlFetchApp.fetch(). + * + * Every method here returns a Promise that resolves with whatever the + * Apps Script server function returned, or rejects with an Error whose + * message contains the backend's error string. + * + * For Svelte code: + * import { api } from "../api/client"; + * const rows = await api.listRows(datasetId); + * + * The runtime also exposes `invoke(name, payload)` for one-off server + * functions (useful for testing or new endpoints). + */ +export interface BackendError { + error: string; + details?: Record; +} + +function isBackendError(value: unknown): value is BackendError { + return ( + typeof value === "object" && + value !== null && + "error" in value && + typeof (value as { error: unknown }).error === "string" + ); +} + +export interface DatasetSummary { + id: string; + name: string; + status: "live" | "building" | "updating" | "failed" | "paused"; + rowCount: number; + columns: Array<{ name: string; type: string; description?: string; isPrimaryKey?: boolean }>; + description?: string; + lastStatusError?: string; +} + +export type ColumnType = "text" | "number" | "boolean" | "url" | "date"; + +export interface DatasetSchema { + dataset_name: string; + description: string; + columns: Array<{ + name: string; + display_name: string; + type: string; + retrieval_hint?: string; + is_primary_key?: boolean; + }>; + primary_key?: string; + retrieval_strategy?: "search_fetch" | "browser" | "hybrid"; + source_hint?: string; +} + +export interface DatasetRow { + _id: string; + data: Record; + rowSummary?: string; + howFound?: string; + sources?: string[]; +} + +export interface InsertRowsResult { + rowsInserted: number; + startCell: string; + endCell: string; +} + +interface ServerApi { + callBackend(path: string, method: string, body: unknown): Promise; + getApiKey(): Promise; + setApiKey(key: string): Promise; + getBackendUrl(): Promise; + setBackendUrl(url: string): Promise; + insertRowsIntoActiveSheet( + headers: string[], + rows: Array>, + clearFirst: boolean, + ): Promise; +} + +function invoke(name: keyof ServerApi, ...args: unknown[]): Promise { + return new Promise((resolve, reject) => { + if (typeof google === "undefined" || !google.script?.run) { + reject(new Error("Not running inside Google Apps Script (google.script.run unavailable).")); + return; + } + const runner = google.script.run + .withSuccessHandler((value: T) => resolve(value)) + .withFailureHandler((err: Error | string) => { + const message = + typeof err === "string" + ? err + : err?.message || "Apps Script call failed"; + reject(new Error(message)); + }); + const fn = runner[name] as (...a: unknown[]) => unknown; + if (typeof fn !== "function") { + reject(new Error(`Server function "${String(name)}" is not available`)); + return; + } + fn(...args); + }); +} + +export const api = { + /** Call any backend HTTP route. */ + callBackend(path: string, method: "GET" | "POST" | "PUT" | "DELETE" = "POST", body: unknown = null): Promise { + return invoke("callBackend", path, method, body).then((result) => { + if (isBackendError(result)) { + throw new Error(result.error); + } + return result as T; + }); + }, + + getApiKey: () => invoke("getApiKey"), + setApiKey: (key: string) => invoke("setApiKey", key), + getBackendUrl: () => invoke("getBackendUrl"), + setBackendUrl: (url: string) => invoke("setBackendUrl", url), + + /** Insert rows into the active Google Sheet. */ + insertRows: (headers: string[], rows: Array>, clearFirst = true) => + invoke("insertRowsIntoActiveSheet", headers, rows, clearFirst), + + /** High-level helpers wrapping the backend endpoints. */ + inferSchema(prompt: string) { + return this.callBackend("/infer-schema", "POST", { prompt }); + }, + + createDataset(schema: { + name: string; + description: string; + columns: Array<{ name: string; type: ColumnType; description?: string; isPrimaryKey?: boolean }>; + retrievalStrategy?: "search_fetch" | "browser" | "hybrid"; + sourceHint?: string; + maxRowCount: number; + refreshCadence?: "manual" | "30m" | "6h" | "12h" | "daily" | "weekly"; + }) { + return this.callBackend<{ dataset: DatasetSummary & { _id: string } }>("/addon/datasets", "POST", schema).then( + (r) => ({ ...r.dataset, id: r.dataset._id }), + ); + }, + + populate( + datasetId: string, + datasetName: string, + description: string, + maxRowCount: number, + columns: Array<{ name: string; type: ColumnType; description?: string; isPrimaryKey?: boolean }>, + ) { + return this.callBackend<{ success: boolean; runId: string }>("/populate", "POST", { + datasetId, + datasetName, + description, + maxRowCount, + columns, + }); + }, + + getDataset(id: string) { + return this.callBackend<{ dataset: DatasetSummary & { _id: string } }>(`/addon/datasets/${encodeURIComponent(id)}`, "GET").then( + (r) => ({ ...r.dataset, id: r.dataset._id }), + ); + }, + + listRows(id: string) { + return this.callBackend<{ rows: DatasetRow[]; dataset: DatasetSummary & { _id: string } }>( + `/addon/datasets/${encodeURIComponent(id)}/rows`, + "GET", + ).then((r) => ({ rows: r.rows, dataset: { ...r.dataset, id: r.dataset._id } })); + }, + + stopDataset(id: string) { + return this.callBackend<{ success: boolean }>("/stop", "POST", { datasetId: id }); + }, + + listDatasets() { + return this.callBackend<{ datasets: Array }>("/addon/datasets", "GET").then( + (r) => r.datasets.map((d) => ({ ...d, id: d._id })), + ); + }, + + listPublicDatasets() { + return this.callBackend<{ datasets: Array }>("/addon/datasets/public", "GET").then( + (r) => r.datasets.map((d) => ({ ...d, id: d._id })), + ); + }, + + listPublicRows(datasetId: string) { + return this.callBackend<{ rows: DatasetRow[]; dataset: DatasetSummary & { _id: string } }>( + `/addon/datasets/public/${encodeURIComponent(datasetId)}/rows`, + "GET", + ).then((r) => ({ rows: r.rows, dataset: { ...r.dataset, id: r.dataset._id } })); + }, + + // ──────────────────────────────────────────────────────────────────────── + // Data Enrichment API + // ──────────────────────────────────────────────────────────────────────── + + /** + * Enrich rows directly from sheet data. + * This is the independent enrichment that doesn't require a BigSet dataset. + */ + enrichRows(params: { + sourceColumns: string[]; + targetColumns: string[]; + rows: Array<{ + rowIndex: number; + sourceData: Record; + targetColumns?: string[]; + }>; + }) { + return this.callBackend<{ + results: Array<{ + rowIndex: number; + values: Record; + error?: string; + }>; + stats: { + rowsProcessed: number; + rowsEnriched: number; + rowsWithErrors: number; + }; + }>("/sheets/enrich", "POST", { + sourceColumns: params.sourceColumns, + targetColumns: params.targetColumns, + rows: params.rows, + }); + }, +}; + +/** Where in the Svelte app we are. Useful for the route guard. */ +export const ROUTES = ["/", "/settings"] as const; +export type Route = (typeof ROUTES)[number]; diff --git a/bigset-addon/sidebar/src/app.css b/bigset-addon/sidebar/src/app.css new file mode 100644 index 0000000..9915879 --- /dev/null +++ b/bigset-addon/sidebar/src/app.css @@ -0,0 +1,468 @@ +/* ============================================================ + BigSet — Google Sheets Add-on Theme + Mirrors frontend/app/globals.css for visual consistency + ============================================================ */ + +:root, +[data-theme="light"] { + --background: rgb(243, 244, 240); + --surface: rgb(229, 231, 224); + --surface-2: rgb(220, 222, 215); + --border: rgb(205, 208, 198); + --foreground: #1d1b16; + --foreground-soft: rgba(29, 27, 22, 0.7); + --foreground-faint: rgba(29, 27, 22, 0.5); + --foreground-ghost: rgba(29, 27, 22, 0.3); + --muted: #7c7f74; + --accent: #1d1b16; + --accent-text: rgb(243, 244, 240); + --link: #2563eb; + --link-decoration: rgba(37, 99, 235, 0.3); + --link-decoration-hover: rgba(37, 99, 235, 0.6); + + --emerald-soft: rgba(16, 185, 129, 0.05); + --emerald-border: rgba(16, 185, 129, 0.2); + --emerald-text: rgb(4, 120, 87); + --emerald-dot: rgb(16, 185, 129); + + --amber-soft: rgba(217, 119, 6, 0.05); + --amber-border: rgba(217, 119, 6, 0.2); + --amber-text: rgb(180, 83, 9); + --amber-dot: rgb(217, 119, 6); + + --blue-soft: rgba(37, 99, 235, 0.05); + --blue-border: rgba(37, 99, 235, 0.2); + --blue-text: rgb(29, 78, 216); + --blue-dot: rgb(37, 99, 235); + + --red-soft: rgba(220, 38, 38, 0.05); + --red-border: rgba(220, 38, 38, 0.2); + --red-text: rgb(185, 28, 28); + --red-dot: rgb(220, 38, 38); +} + +[data-theme="dark"] { + --background: #141210; + --surface: #1c1a17; + --surface-2: #23211d; + --border: #2e2b26; + --foreground: #e8e4de; + --foreground-soft: rgba(232, 228, 222, 0.7); + --foreground-faint: rgba(232, 228, 222, 0.5); + --foreground-ghost: rgba(232, 228, 222, 0.3); + --muted: #7a756c; + --accent: #e8e4de; + --accent-text: #141210; + --link: #93b4f5; + --link-decoration: rgba(147, 180, 245, 0.3); + --link-decoration-hover: rgba(147, 180, 245, 0.6); + + --emerald-soft: rgba(16, 185, 129, 0.1); + --emerald-border: rgba(16, 185, 129, 0.25); + --emerald-text: rgb(110, 231, 183); + --emerald-dot: rgb(52, 211, 153); + + --amber-soft: rgba(217, 119, 6, 0.1); + --amber-border: rgba(217, 119, 6, 0.25); + --amber-text: rgb(252, 211, 77); + --amber-dot: rgb(245, 158, 11); + + --blue-soft: rgba(37, 99, 235, 0.1); + --blue-border: rgba(37, 99, 235, 0.25); + --blue-text: rgb(147, 180, 245); + --blue-dot: rgb(96, 165, 250); + + --red-soft: rgba(220, 38, 38, 0.1); + --red-border: rgba(220, 38, 38, 0.25); + --red-text: rgb(252, 165, 165); + --red-dot: rgb(248, 113, 113); +} + +:root { + font-family: "Geist", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + Roboto, sans-serif; + font-size: 13px; + line-height: 1.5; + font-weight: 400; + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + -webkit-text-size-adjust: 100%; + font-feature-settings: "ss01", "cv11"; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + background-color: var(--background); + color: var(--foreground); + width: 100%; + height: 100%; + overflow: hidden; + transition: background-color 0.3s ease, color 0.3s ease; +} + +#app { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; +} + +/* Typography helpers */ +.eyebrow { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.15em; + color: var(--muted); + font-weight: 600; +} + +.muted { + color: var(--muted); +} + +.soft { + color: var(--foreground-soft); +} + +.faint { + color: var(--foreground-faint); +} + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + border-radius: 8px; + font-weight: 600; + cursor: pointer; + transition: opacity 0.15s ease, background-color 0.15s ease, + border-color 0.15s ease; + font-family: inherit; + white-space: nowrap; +} + +.btn:disabled { + opacity: 0.4; + cursor: not-allowed; +} + +.btn-primary { + background-color: var(--accent); + color: var(--accent-text); + border: 1px solid var(--accent); + padding: 8px 16px; + font-size: 13px; +} + +.btn-primary:hover:not(:disabled) { + opacity: 0.9; +} + +.btn-secondary { + background-color: transparent; + color: var(--foreground); + border: 1px solid var(--border); + padding: 7px 14px; + font-size: 13px; +} + +.btn-secondary:hover:not(:disabled) { + background-color: var(--foreground-soft); + background-color: rgba(0, 0, 0, 0.04); +} + +[data-theme="dark"] .btn-secondary:hover:not(:disabled) { + background-color: rgba(255, 255, 255, 0.06); +} + +.btn-ghost { + background-color: transparent; + color: var(--foreground); + border: 1px solid transparent; + padding: 6px 10px; + font-size: 12px; + font-weight: 500; +} + +.btn-ghost:hover:not(:disabled) { + background-color: rgba(0, 0, 0, 0.04); +} + +[data-theme="dark"] .btn-ghost:hover:not(:disabled) { + background-color: rgba(255, 255, 255, 0.06); +} + +.btn-danger { + background-color: transparent; + color: var(--red-text); + border: 1px solid var(--red-border); + padding: 6px 12px; + font-size: 12px; + font-weight: 500; +} + +.btn-danger:hover:not(:disabled) { + background-color: var(--red-soft); +} + +/* Form inputs */ +.input, +.textarea, +.select { + width: 100%; + border-radius: 8px; + border: 1px solid var(--border); + background-color: var(--surface); + padding: 8px 12px; + font-size: 13px; + color: var(--foreground); + outline: none; + transition: border-color 0.15s ease; + font-family: inherit; +} + +.input::placeholder, +.textarea::placeholder { + color: var(--foreground-ghost); +} + +.input:focus, +.textarea:focus, +.select:focus { + border-color: var(--foreground-ghost); +} + +.textarea { + resize: vertical; + min-height: 88px; + line-height: 1.5; +} + +.select { + appearance: none; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='6 9 12 15 18 9'/%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 10px center; + padding-right: 28px; + cursor: pointer; +} + +.label { + display: block; + font-size: 12px; + font-weight: 500; + color: var(--foreground); + margin-bottom: 6px; +} + +/* Surface cards */ +.card { + background-color: var(--surface); + border: 1px solid var(--border); + border-radius: 12px; + padding: 12px; +} + +/* Pill badge */ +.pill { + display: inline-flex; + align-items: center; + gap: 6px; + border-radius: 999px; + padding: 3px 10px; + font-size: 11px; + font-weight: 600; + border: 1px solid var(--border); + background-color: var(--surface); + color: var(--foreground); + white-space: nowrap; +} + +.pill-live { + border-color: var(--emerald-border); + background-color: var(--emerald-soft); + color: var(--emerald-text); +} + +.pill-building { + border-color: var(--amber-border); + background-color: var(--amber-soft); + color: var(--amber-text); +} + +.pill-updating { + border-color: var(--blue-border); + background-color: var(--blue-soft); + color: var(--blue-text); +} + +.pill-failed { + border-color: var(--red-border); + background-color: var(--red-soft); + color: var(--red-text); +} + +.pill-paused { + color: var(--muted); +} + +.pill-dot { + width: 6px; + height: 6px; + border-radius: 50%; + background-color: currentColor; + flex-shrink: 0; +} + +.pill-dot-pulse { + animation: pulse-dot 1.6s ease-in-out infinite; +} + +@keyframes pulse-dot { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.4; } +} + +/* Alert */ +.alert { + border-radius: 8px; + padding: 10px 12px; + font-size: 12px; + line-height: 1.5; +} + +.alert-error { + border: 1px solid var(--red-border); + background-color: var(--red-soft); + color: var(--red-text); +} + +.alert-info { + border: 1px solid var(--border); + background-color: var(--surface); + color: var(--foreground-soft); +} + +/* Spinner */ +.spinner { + width: 16px; + height: 16px; + border: 2px solid var(--foreground-ghost); + border-top-color: var(--foreground); + border-radius: 50%; + animation: spin 0.8s linear infinite; + flex-shrink: 0; +} + +.spinner-lg { + width: 24px; + height: 24px; + border-width: 2.5px; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +/* Scrollbar */ +::-webkit-scrollbar { + width: 6px; + height: 6px; +} +::-webkit-scrollbar-track { + background: transparent; +} +::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} +::-webkit-scrollbar-thumb:hover { + background: var(--muted); +} + +/* Icon button */ +.icon-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + border-radius: 6px; + border: none; + background: transparent; + color: var(--muted); + cursor: pointer; + transition: background-color 0.15s ease, color 0.15s ease; +} + +.icon-btn:hover { + background-color: rgba(0, 0, 0, 0.04); + color: var(--foreground); +} + +[data-theme="dark"] .icon-btn:hover { + background-color: rgba(255, 255, 255, 0.06); +} + +/* Column type glyphs (BigSet-style: ≡ # ■ ⇗ ☆) */ +.col-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + border-radius: 4px; + background-color: var(--surface-2); + font-size: 11px; + font-weight: 700; + flex-shrink: 0; + color: var(--foreground-ghost); +} + +.col-icon-text { color: var(--foreground-ghost); } +.col-icon-number { color: rgba(139, 92, 246, 0.7); } +.col-icon-bool { color: rgba(16, 185, 129, 0.7); } +.col-icon-url { color: var(--link); opacity: 0.7; } +.col-icon-date { color: rgba(217, 119, 6, 0.7); } + +@keyframes shimmer { + 0% { background-position: -200% 0; } + 100% { background-position: 200% 0; } +} + +.shimmer { + background: linear-gradient( + 90deg, + transparent 0%, + var(--foreground) 50%, + transparent 100% + ); + background-size: 200% 100%; + animation: shimmer 1.8s ease-in-out infinite; + opacity: 0.04; + border-radius: inherit; + pointer-events: none; +} + +/* Theme transition */ +body, +.card, +.btn, +.input, +.textarea, +.select, +.pill, +.alert { + transition: background-color 0.3s ease, color 0.3s ease, + border-color 0.3s ease; +} diff --git a/bigset-addon/sidebar/src/components/ColumnIcon.svelte b/bigset-addon/sidebar/src/components/ColumnIcon.svelte new file mode 100644 index 0000000..2951a68 --- /dev/null +++ b/bigset-addon/sidebar/src/components/ColumnIcon.svelte @@ -0,0 +1,15 @@ + + +{glyph} + + diff --git a/bigset-addon/sidebar/src/components/Header.svelte b/bigset-addon/sidebar/src/components/Header.svelte new file mode 100644 index 0000000..233b6c3 --- /dev/null +++ b/bigset-addon/sidebar/src/components/Header.svelte @@ -0,0 +1,117 @@ + + +
+ {#if showBack} + + {/if} +
+ + {title} +
+
+ + +
+
+ + diff --git a/bigset-addon/sidebar/src/components/Icon.svelte b/bigset-addon/sidebar/src/components/Icon.svelte new file mode 100644 index 0000000..14b07cf --- /dev/null +++ b/bigset-addon/sidebar/src/components/Icon.svelte @@ -0,0 +1,129 @@ + + +{#if name === "sun"} + + + + +{:else if name === "moon"} + + + +{:else if name === "settings"} + + + + +{:else if name === "close"} + + + + +{:else if name === "check"} + + + +{:else if name === "chevron-right"} + + + +{:else if name === "chevron-down"} + + + +{:else if name === "chevron-left"} + + + +{:else if name === "plus"} + + + + +{:else if name === "trash"} + + + + + + +{:else if name === "stop"} + + + +{:else if name === "key"} + + + +{:else if name === "link"} + + + + +{:else if name === "external"} + + + + + +{:else if name === "play"} + + + +{:else if name === "arrow-right"} + + + + +{:else if name === "arrow-left"} + + + + +{:else if name === "menu"} + + + + + +{:else if name === "info"} + + + + + +{:else if name === "alert"} + + + + + +{:else if name === "sparkle"} + + + +{/if} diff --git a/bigset-addon/sidebar/src/components/Spinner.svelte b/bigset-addon/sidebar/src/components/Spinner.svelte new file mode 100644 index 0000000..1c70f0f --- /dev/null +++ b/bigset-addon/sidebar/src/components/Spinner.svelte @@ -0,0 +1,9 @@ + + + + + diff --git a/bigset-addon/sidebar/src/components/StatusBadge.svelte b/bigset-addon/sidebar/src/components/StatusBadge.svelte new file mode 100644 index 0000000..9a4c131 --- /dev/null +++ b/bigset-addon/sidebar/src/components/StatusBadge.svelte @@ -0,0 +1,19 @@ + + + + + {text} + + + diff --git a/bigset-addon/sidebar/src/components/Stepper.svelte b/bigset-addon/sidebar/src/components/Stepper.svelte new file mode 100644 index 0000000..9fdb4c1 --- /dev/null +++ b/bigset-addon/sidebar/src/components/Stepper.svelte @@ -0,0 +1,112 @@ + + +
+ {#each Array.from({ length: total }, (_, i) => i + 1) as step} + {@const isDone = step < current} + {@const isCurrent = step === current} +
+
+ {#if isDone} + + + + {:else} + {step} + {/if} +
+ {#if step < total} +
+ {/if} +
+ {/each} + {#if labels.length} +
+ {#each labels as label, i} + + {label} + + {/each} +
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/components/ThemeToggle.svelte b/bigset-addon/sidebar/src/components/ThemeToggle.svelte new file mode 100644 index 0000000..c4b6f7d --- /dev/null +++ b/bigset-addon/sidebar/src/components/ThemeToggle.svelte @@ -0,0 +1,46 @@ + + + + + diff --git a/bigset-addon/sidebar/src/lib/format.ts b/bigset-addon/sidebar/src/lib/format.ts new file mode 100644 index 0000000..9009579 --- /dev/null +++ b/bigset-addon/sidebar/src/lib/format.ts @@ -0,0 +1,14 @@ +/** + * Convert a snake_case identifier from the backend's schema inference + * into a human-readable Title Case label for display. Identical to + * the dashboard's transform at `frontend/app/dataset/new/page.tsx:139-143`. + * + * nhl_teams_and_head_coaches → "Nhl Teams And Head Coaches" + * yc_hiring_startups → "Yc Hiring Startups" + */ +export function snakeToTitleCase(value: string): string { + return value + .split("_") + .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) + .join(" "); +} diff --git a/bigset-addon/sidebar/src/main.ts b/bigset-addon/sidebar/src/main.ts new file mode 100644 index 0000000..028364b --- /dev/null +++ b/bigset-addon/sidebar/src/main.ts @@ -0,0 +1,8 @@ +import './app.css' +import App from './App.svelte' + +const target = document.getElementById('app'); +if (!target) throw new Error('Mount element #app not found'); +const app = new App({ target }); + +export default app; diff --git a/bigset-addon/sidebar/src/pages/DatasetDetail.svelte b/bigset-addon/sidebar/src/pages/DatasetDetail.svelte new file mode 100644 index 0000000..2a83fae --- /dev/null +++ b/bigset-addon/sidebar/src/pages/DatasetDetail.svelte @@ -0,0 +1,368 @@ + + +
+
+ +
+ + {#if !ds} +
No dataset selected.
+ {:else} +
+
+

{ds.name}

+ +
+ {#if ds.description} +

{ds.description}

+ {/if} + +
+
+ {rows.length} + rows +
+
+ {ds.columns.length} + columns +
+
+
+ +
+
Columns
+ {#each ds.columns as col} +
+ {col.name} + {col.type} +
+ {/each} +
+ + {#if insertResult} +
+ Inserted {insertResult.rowsInserted} rows +
Range {insertResult.startCell} : {insertResult.endCell}
+ +
+ {:else if insertError} +
{insertError}
+ {:else if showConfirm} +
+

+ This will replace all existing data in the active sheet with {rows.length} rows. + This cannot be undone. +

+
+ + +
+
+ {:else} +
+ +
+ {/if} + {/if} +
+ + diff --git a/bigset-addon/sidebar/src/pages/Datasets.svelte b/bigset-addon/sidebar/src/pages/Datasets.svelte new file mode 100644 index 0000000..83e33a6 --- /dev/null +++ b/bigset-addon/sidebar/src/pages/Datasets.svelte @@ -0,0 +1,190 @@ + + +
+
+ Your datasets + +
+ + {#if loading} +
+ +
+ {:else if loadError} +
{loadError}
+ {:else if datasets.length === 0} +
+

No datasets yet.

+

Generate one from the Generate tab.

+
+ {:else} +
+ {#each datasets as ds (ds.id)} + + {/each} +
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/pages/Home.svelte b/bigset-addon/sidebar/src/pages/Home.svelte new file mode 100644 index 0000000..f8dc29c --- /dev/null +++ b/bigset-addon/sidebar/src/pages/Home.svelte @@ -0,0 +1,128 @@ + + +
+
+ + {#if $wizard.selectedForInsert} + + {:else} + + +
+ {#if tab === "datasets"} + + {:else if tab === "enrich"} + + {:else if tab === "public"} + + {:else} + {#if $wizard.step === "describe"} + + {:else if $wizard.step === "generating"} + + {:else if $wizard.step === "review"} + + {:else if $wizard.step === "populating"} + + {:else if $wizard.step === "done"} + + {/if} + {/if} +
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/pages/Public.svelte b/bigset-addon/sidebar/src/pages/Public.svelte new file mode 100644 index 0000000..d8e84ee --- /dev/null +++ b/bigset-addon/sidebar/src/pages/Public.svelte @@ -0,0 +1,189 @@ + + +
+
+ Public datasets + +
+ + {#if loading} +
+ +
+ {:else if loadError} +
{loadError}
+ {:else if datasets.length === 0} +
+

No public datasets available.

+
+ {:else} +
+ {#each datasets as ds (ds.id)} + + {/each} +
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/pages/Settings.svelte b/bigset-addon/sidebar/src/pages/Settings.svelte new file mode 100644 index 0000000..d7b5a08 --- /dev/null +++ b/bigset-addon/sidebar/src/pages/Settings.svelte @@ -0,0 +1,255 @@ + + +
+
push("/")} /> + + {#if loading} +
+ +
+ {:else} +
+
+ Backend +

BigSet API endpoint

+

Where the add-on should send requests. Use the local Docker URL during development.

+ +
+ +
+ Authentication +

API key

+

+ Generate one at BigSet dashboard → Settings → API keys and paste it here. +

+
+ {#if showKey} + (apiKey = e.currentTarget.value)} placeholder="bsk_..." autocomplete="off" spellcheck="false" /> + {:else} + (apiKey = e.currentTarget.value)} placeholder="bsk_..." autocomplete="off" spellcheck="false" /> + {/if} + +
+ {#if savedApiKey && !apiKey} +
+ + Saved: {maskedKey(savedApiKey)} +
+ {/if} +
+ + {#if testResult} +
+ {testResult.ok ? "Connected" : "Couldn't connect"} +
{testResult.message}
+
+ {/if} + +
+ + +
+
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/pages/StepDescribe.svelte b/bigset-addon/sidebar/src/pages/StepDescribe.svelte new file mode 100644 index 0000000..62e14f9 --- /dev/null +++ b/bigset-addon/sidebar/src/pages/StepDescribe.svelte @@ -0,0 +1,208 @@ + + +
+
+ Datasets +

What do you want to track?

+

Describe the data and BigSet will design the columns and sources for you.

+
+ +
+ +
+ ⌘ + Enter to generate +
+
+ + {#if localError} +
+ Couldn't generate schema. +
{localError}
+
+ {/if} + +
+ +
+ +
+
Try a prompt
+ + + +
+
+ + diff --git a/bigset-addon/sidebar/src/pages/StepDone.svelte b/bigset-addon/sidebar/src/pages/StepDone.svelte new file mode 100644 index 0000000..77df0cb --- /dev/null +++ b/bigset-addon/sidebar/src/pages/StepDone.svelte @@ -0,0 +1,196 @@ + + +
+
+ +
+ +
+
+ +
+

Dataset is live

+

+ {$wizard.rowCount} row{$wizard.rowCount === 1 ? "" : "s"} ready to insert into + {$wizard.dataset?.name ?? "your sheet"} +

+
+ + {#if insertResult} +
+ Inserted {insertResult.rowsInserted} rows +
Range {insertResult.startCell} : {insertResult.endCell}
+
+ {:else if localError} +
{localError}
+ {/if} + +
+ + +
+ +
+ +
+
+ + diff --git a/bigset-addon/sidebar/src/pages/StepGenerating.svelte b/bigset-addon/sidebar/src/pages/StepGenerating.svelte new file mode 100644 index 0000000..4811c4c --- /dev/null +++ b/bigset-addon/sidebar/src/pages/StepGenerating.svelte @@ -0,0 +1,72 @@ + + +
+
+ +

Analyzing your request…

+

Figuring out what columns and data sources to use

+
+ {#if $wizard.prompt} +
+ Your prompt +

“{$wizard.prompt.trim()}”

+
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/pages/StepPopulating.svelte b/bigset-addon/sidebar/src/pages/StepPopulating.svelte new file mode 100644 index 0000000..c076aee --- /dev/null +++ b/bigset-addon/sidebar/src/pages/StepPopulating.svelte @@ -0,0 +1,257 @@ + + +
+
+ + {fmtElapsed(elapsedSec)} +
+ +
+
+ {$wizard.rowCount} + rows collected +
+
+ target: {$wizard.schema?.maxRowCount ?? 100} +
+ +
+
+
+ +
+ + + {#if $wizard.dataset?.status === "building"} + Searching the web and inserting rows… + {:else if $wizard.dataset?.status === "updating"} + Refreshing rows… + {:else} + Working… + {/if} + +
+
+ +
+ +
+ +
+ Tip: keep this sidebar open. Rows appear in your Sheet when population completes. +
+
+ + diff --git a/bigset-addon/sidebar/src/pages/StepReview.svelte b/bigset-addon/sidebar/src/pages/StepReview.svelte new file mode 100644 index 0000000..57eff72 --- /dev/null +++ b/bigset-addon/sidebar/src/pages/StepReview.svelte @@ -0,0 +1,433 @@ + + +{#if $wizard.schema} +
+
+
+ Review schema +

{$wizard.schema.columns.length} column{$wizard.schema.columns.length === 1 ? "" : "s"}

+
+ + {#if localError} +
{localError}
+ {/if} + +
+ + +
+ +
+ + +
+ +
+
+ + +
+
+ +
+ {$wizard.schema.retrievalStrategy ?? "auto"} +
+
+
+ +
+ Columns ({$wizard.schema.columns.length}) + +
+ +
+ {#each $wizard.schema.columns as col, i (i)} +
+ + + handleNameChange(i, e.currentTarget.value)} + placeholder="column_name" + /> + + {#if col.description} + + {/if} +
+ {/each} +
+ + {#if $wizard.schema.primaryKey} +
+ +
+ {/if} +
+ +
+ + +
+
+{/if} + + diff --git a/bigset-addon/sidebar/src/pages/enrich/EnrichConfirm.svelte b/bigset-addon/sidebar/src/pages/enrich/EnrichConfirm.svelte new file mode 100644 index 0000000..f1af1dc --- /dev/null +++ b/bigset-addon/sidebar/src/pages/enrich/EnrichConfirm.svelte @@ -0,0 +1,220 @@ + + +
+
+ Confirm Enrichment + +
+ +
+ Range + {state.range} +
+ +
+
+ {state.rows.length} + rows +
+
+ {state.headers.length} + columns +
+
+ {totalEmpty} + to fill +
+
+ +
+

Source columns

+
+ {#each state.sourceColumns as col} + {col} + {/each} +
+
+ +
+

Target columns

+
+ {#each state.targetColumns as col} + {col} + {/each} +
+
+ + {#if !state.targetColumns.length} +
+

All columns already have data. Nothing to enrich.

+
+ {/if} + +
+ {#if state.targetColumns.length} + + {/if} + +
+
+ + diff --git a/bigset-addon/sidebar/src/pages/enrich/EnrichDone.svelte b/bigset-addon/sidebar/src/pages/enrich/EnrichDone.svelte new file mode 100644 index 0000000..13c6c6d --- /dev/null +++ b/bigset-addon/sidebar/src/pages/enrich/EnrichDone.svelte @@ -0,0 +1,224 @@ + + +
+
+
+ + + +
+ +
+ +
+
+ {filledResults.length} + rows filled +
+ {#if errorResults.length > 0} +
+ {errorResults.length} + errors +
+ {/if} + {#if skippedResults.length > 0} +
+ {skippedResults.length} + no data +
+ {/if} +
+ + {#if filledResults.length > 0} +
+

Filled cells

+ {#each filledResults as r} +
+ {#each Object.entries(r.values) as [col, val]} +
+ {col} + {String(val)} +
+ {/each} +
+ {/each} +
+ {/if} + + {#if errorResults.length > 0} +
+

Errors

+ {#each errorResults as r} +
+ Row {r.rowIndex} + {r.error} +
+ {/each} +
+ {/if} + +

Written directly to your sheet. Only empty cells were filled — existing data was never overwritten.

+ +
+ + +
+
+ + diff --git a/bigset-addon/sidebar/src/pages/enrich/EnrichTab.svelte b/bigset-addon/sidebar/src/pages/enrich/EnrichTab.svelte new file mode 100644 index 0000000..2b6ce86 --- /dev/null +++ b/bigset-addon/sidebar/src/pages/enrich/EnrichTab.svelte @@ -0,0 +1,162 @@ + + +
+ {#if status === "idle"} +
+
+ + + + +
+

Select cells to enrich

+

+ Highlight a range in your sheet including the header row, then click below. +

+ +
+ {:else if status === "loading"} +
+ +

Reading selection…

+
+ {:else if status === "confirm"} + + {:else if status === "enriching"} +
+ +

Researching and filling cells…

+

This may take a minute for larger selections.

+
+ {:else if status === "done"} + + {:else if status === "error"} +
+
!
+

Something went wrong

+

{error}

+

Select a range in your sheet (including header row), then try again.

+ +
+ {/if} +
+ + diff --git a/bigset-addon/sidebar/src/stores/enrichStore.ts b/bigset-addon/sidebar/src/stores/enrichStore.ts new file mode 100644 index 0000000..d53a78b --- /dev/null +++ b/bigset-addon/sidebar/src/stores/enrichStore.ts @@ -0,0 +1,203 @@ +import { writable, derived } from "svelte/store"; + +export interface SelectionData { + headers: string[]; + rows: Array<{ rowIndex: number; data: Record }>; + range: string; +} + +export type EnrichStatus = "idle" | "loading" | "confirm" | "enriching" | "done" | "error"; + +export interface RowResult { + rowIndex: number; + values: Record; + error?: string; +} + +interface EnrichState { + status: EnrichStatus; + headers: string[]; + rows: Array<{ rowIndex: number; data: Record }>; + sourceColumns: string[]; + targetColumns: string[]; + enrichedCount: number; + errorCount: number; + error: string | null; + range: string; + results: RowResult[]; +} + +const initial: EnrichState = { + status: "idle", + headers: [], + rows: [], + sourceColumns: [], + targetColumns: [], + enrichedCount: 0, + errorCount: 0, + error: null, + range: "", + results: [], +}; + +function createStore() { + const { subscribe, set, update } = writable(initial); + + return { + subscribe, + + reset() { + set(initial); + }, + + async loadSelection() { + update((s) => ({ ...s, status: "loading" })); + try { + const data: SelectionData = await new Promise((resolve, reject) => { + if (typeof google === "undefined" || !google.script?.run) { + reject(new Error("Not running in Google Apps Script")); + return; + } + google.script.run + .withSuccessHandler(resolve) + .withFailureHandler((e: Error | string) => + reject(new Error(typeof e === "string" ? e : e.message)) + ) + .getSelectedRange(); + }); + + if (!data.headers.length || !data.rows.length) { + update((s) => ({ + ...s, + status: "error", + error: `Detected range ${data.range || "(none)"} has ${data.headers.length} header(s) and ${data.rows.length} data row(s). Select a range with at least one header row and one data row, then try again.`, + range: data.range, + })); + return; + } + + const sourceCols: string[] = []; + const targetCols: string[] = []; + for (const h of data.headers) { + const nonEmpty = data.rows.filter((r) => r.data[h] != null && r.data[h] !== ""); + const hasAnyData = nonEmpty.length > 0; + const hasAnyEmpty = nonEmpty.length < data.rows.length; + if (hasAnyData) sourceCols.push(h); + if (hasAnyEmpty) targetCols.push(h); + } + + update((s) => ({ + ...s, + status: "confirm", + headers: data.headers, + rows: data.rows, + range: data.range, + sourceColumns: sourceCols, + targetColumns: targetCols, + })); + } catch (err) { + update((s) => ({ + ...s, + status: "error", + error: err instanceof Error ? err.message : "Failed to read selection", + })); + } + }, + + async enrich() { + update((s) => ({ ...s, status: "enriching", results: [] })); + + try { + const state = getState(); + + const rowsToSend = state.rows + .filter((r) => + state.targetColumns.some( + (c) => r.data[c] == null || r.data[c] === "" + ) + ) + .map((r) => ({ + rowIndex: r.rowIndex, + sourceData: Object.fromEntries( + Object.entries(r.data).filter( + ([k, v]) => + state.sourceColumns.includes(k) && + v != null && + v !== "" + ) + ), + targetColumns: state.targetColumns.filter( + (c) => r.data[c] == null || r.data[c] === "" + ), + })); + + const { api } = await import("../api/client.js"); + const resp = await api.enrichRows({ + sourceColumns: state.sourceColumns, + targetColumns: state.targetColumns, + rows: rowsToSend, + }); + + const updates: Array<{ rowIndex: number; columnName: string; value: unknown }> = []; + for (const result of resp.results) { + if (result.error) continue; + for (const [col, val] of Object.entries(result.values)) { + updates.push({ rowIndex: result.rowIndex, columnName: col, value: val }); + } + } + + if (updates.length > 0) { + await new Promise((resolve, reject) => { + if (typeof google === "undefined" || !google.script?.run) { + reject(new Error("Not running in Google Apps Script")); + return; + } + google.script.run + .withSuccessHandler(() => resolve()) + .withFailureHandler((e: Error | string) => + reject(new Error(typeof e === "string" ? e : e.message)) + ) + .updateSheetCells(updates, state.range); + }); + } + + update((s) => ({ + ...s, + status: "done", + enrichedCount: resp.stats.rowsEnriched, + errorCount: resp.stats.rowsWithErrors, + results: resp.results, + })); + } catch (err) { + update((s) => ({ + ...s, + status: "error", + error: err instanceof Error ? err.message : "Enrichment failed", + })); + } + }, + }; +} + +let _state: EnrichState = initial; + +function getState(): EnrichState { + return _state; +} + +export const enrichStore = createStore(); + +enrichStore.subscribe((s) => { + _state = s; +}); + +export const eligibleCount = derived( + { subscribe: enrichStore.subscribe }, + ($) => { + if (!$.targetColumns.length) return 0; + const allEmpty = $.rows.map( + (r) => $.targetColumns.filter((c) => r.data[c] == null || r.data[c] === "").length + ); + return allEmpty.reduce((a, b) => a + b, 0); + } +); diff --git a/bigset-addon/sidebar/src/stores/theme.ts b/bigset-addon/sidebar/src/stores/theme.ts new file mode 100644 index 0000000..4a5716c --- /dev/null +++ b/bigset-addon/sidebar/src/stores/theme.ts @@ -0,0 +1,53 @@ +/** + * Lightweight, reactive theme store. The actual DOM attribute is set + * pre-hydration in the inline + + + + +
+ + + diff --git a/bigset-addon/tsconfig.json b/bigset-addon/tsconfig.json new file mode 100644 index 0000000..1535b59 --- /dev/null +++ b/bigset-addon/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "none", + "outDir": "./src", + "rootDir": "./src", + "strict": false, + "noImplicitAny": false, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "noEmitOnError": true, + "removeComments": false, + "newLine": "lf", + "types": ["google-apps-script"], + "lib": ["ES2020"] + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "sidebar"] +} diff --git a/frontend/app/dashboard/settings/api-keys/page.tsx b/frontend/app/dashboard/settings/api-keys/page.tsx new file mode 100644 index 0000000..5e29566 --- /dev/null +++ b/frontend/app/dashboard/settings/api-keys/page.tsx @@ -0,0 +1,247 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; +import { Copy, Check, Plus, Trash2, KeyRound } from "lucide-react"; +import { SettingsPageLayout } from "@/components/settings/SettingsPageLayout"; +import { SettingsHeader } from "@/components/settings/SettingsHeader"; +import { useAppAuth } from "@/lib/app-auth"; +import { + listApiKeys, + createApiKey, + revokeApiKey, + type ApiKeyInfo, + type CreatedApiKey, +} from "@/lib/backend"; + +const navItems = [ + { + label: "Models", + href: "/dashboard/settings/models", + icon: ( + + + + + ), + }, + { + label: "API Keys", + href: "/dashboard/settings/api-keys", + icon: ( + + + + + + ), + }, + { + label: "Account", + href: "/dashboard/settings/account", + disabled: true, + icon: ( + + + + + ), + }, + { + label: "Billing", + href: "/dashboard/settings/billing", + disabled: true, + icon: ( + + + + + ), + }, +]; + +export default function ApiKeysSettingsPage() { + const { getToken } = useAppAuth(); + const [keys, setKeys] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const [creating, setCreating] = useState(false); + const [newName, setNewName] = useState(""); + const [createdKey, setCreatedKey] = useState(null); + const [copied, setCopied] = useState(false); + const [revokingId, setRevokingId] = useState(null); + + const loadKeys = useCallback(async () => { + setLoading(true); + setError(null); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + const { keys } = await listApiKeys(token); + setKeys(keys); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load API keys."); + } finally { + setLoading(false); + } + }, [getToken]); + + useEffect(() => { + loadKeys(); + }, [loadKeys]); + + async function handleCreate() { + setCreating(true); + setError(null); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + const created = await createApiKey(newName.trim() || "Default key", token); + setCreatedKey(created); + setNewName(""); + await loadKeys(); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to create API key."); + } finally { + setCreating(false); + } + } + + async function handleRevoke(id: string) { + setRevokingId(id); + try { + const token = await getToken(); + if (!token) throw new Error("Not authenticated"); + await revokeApiKey(id, token); + await loadKeys(); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to revoke API key."); + } finally { + setRevokingId(null); + } + } + + function copyKey() { + if (!createdKey) return; + navigator.clipboard.writeText(createdKey.key); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } + + return ( + +
+ + + {createdKey && ( +
+
+
+

+ Copy your API key now +

+

+ You won't be able to see it again. +

+
+ +
+ + {createdKey.key} + + +
+ )} + +
+ setNewName(e.target.value)} + placeholder="Key name (e.g. Sheets add-on)" + className="flex-1 rounded-lg border border-border bg-surface px-3 py-2 text-sm outline-none focus:border-foreground/30" + /> + +
+ + {error && ( +
+ {error} +
+ )} + + {loading ? ( +
Loading…
+ ) : keys && keys.length === 0 ? ( +
+ +

No API keys yet.

+

+ Create one above to connect the Sheets add-on. +

+
+ ) : ( +
+ {keys?.map((k) => ( +
+
+

{k.name}

+

+ {k.keyPrefix}…•••••• +

+
+
+
+

+ Created {new Date(k.createdAt).toLocaleDateString()} +

+ {k.lastUsedAt && ( +

+ Last used {new Date(k.lastUsedAt).toLocaleDateString()} +

+ )} +
+ +
+
+ ))} +
+ )} +
+ + ); +} diff --git a/frontend/app/dashboard/settings/models/page.tsx b/frontend/app/dashboard/settings/models/page.tsx index 6a3acb4..626e593 100644 --- a/frontend/app/dashboard/settings/models/page.tsx +++ b/frontend/app/dashboard/settings/models/page.tsx @@ -90,6 +90,17 @@ export default function ModelSettingsPage() { ), }, + { + label: "API Keys", + href: "/dashboard/settings/api-keys", + icon: ( + + + + + + ), + }, { label: "Account", href: "/dashboard/settings/account", diff --git a/frontend/convex/_generated/api.d.ts b/frontend/convex/_generated/api.d.ts index bc8dc15..982dca9 100644 --- a/frontend/convex/_generated/api.d.ts +++ b/frontend/convex/_generated/api.d.ts @@ -8,6 +8,7 @@ * @module */ +import type * as apiKeys from "../apiKeys.js"; import type * as datasetRows from "../datasetRows.js"; import type * as datasets from "../datasets.js"; import type * as lib_authz from "../lib/authz.js"; @@ -19,6 +20,7 @@ import type * as openRouterModels from "../openRouterModels.js"; import type * as publicSeed from "../publicSeed.js"; import type * as quota from "../quota.js"; import type * as runStats from "../runStats.js"; +import type * as sheetsEnrichmentRuns from "../sheetsEnrichmentRuns.js"; import type { ApiFromModules, @@ -27,6 +29,7 @@ import type { } from "convex/server"; declare const fullApi: ApiFromModules<{ + apiKeys: typeof apiKeys; datasetRows: typeof datasetRows; datasets: typeof datasets; "lib/authz": typeof lib_authz; @@ -38,6 +41,7 @@ declare const fullApi: ApiFromModules<{ publicSeed: typeof publicSeed; quota: typeof quota; runStats: typeof runStats; + sheetsEnrichmentRuns: typeof sheetsEnrichmentRuns; }>; /** diff --git a/frontend/convex/apiKeys.ts b/frontend/convex/apiKeys.ts new file mode 100644 index 0000000..ff0d7d7 --- /dev/null +++ b/frontend/convex/apiKeys.ts @@ -0,0 +1,64 @@ +import { + internalMutation, + internalQuery, +} from "./_generated/server.js"; +import { v } from "convex/values"; + +export const create = internalMutation({ + args: { + ownerId: v.string(), + name: v.string(), + keyHash: v.string(), + keyPrefix: v.string(), + createdAt: v.number(), + }, + handler: async (ctx, args) => { + const id = await ctx.db.insert("apiKeys", args); + return { id }; + }, +}); + +export const revoke = internalMutation({ + args: { + id: v.id("apiKeys"), + ownerId: v.string(), + revokedAt: v.number(), + }, + handler: async (ctx, args) => { + const existing = await ctx.db.get(args.id); + if (!existing || existing.ownerId !== args.ownerId) return null; + await ctx.db.patch(args.id, { revokedAt: args.revokedAt }); + return { success: true }; + }, +}); + +export const listByOwner = internalQuery({ + args: { ownerId: v.string() }, + handler: async (ctx, { ownerId }) => { + return await ctx.db + .query("apiKeys") + .withIndex("by_owner", (q) => q.eq("ownerId", ownerId)) + .collect(); + }, +}); + +export const lookupByHash = internalQuery({ + args: { keyHash: v.string() }, + handler: async (ctx, { keyHash }) => { + const matches = await ctx.db + .query("apiKeys") + .withIndex("by_hash", (q) => q.eq("keyHash", keyHash)) + .collect(); + return matches.find((k) => !k.revokedAt) ?? null; + }, +}); + +export const touchLastUsed = internalMutation({ + args: { + id: v.id("apiKeys"), + lastUsedAt: v.number(), + }, + handler: async (ctx, args) => { + await ctx.db.patch(args.id, { lastUsedAt: args.lastUsedAt }); + }, +}); diff --git a/frontend/convex/datasetRows.ts b/frontend/convex/datasetRows.ts index a4a877c..f5ae3d1 100644 --- a/frontend/convex/datasetRows.ts +++ b/frontend/convex/datasetRows.ts @@ -379,3 +379,143 @@ export const listInternal = internalQuery({ .collect(); }, }); + +/** + * Update specific columns in a row without touching other columns. + * Used by the enrichment workflow to fill empty cells while preserving + * existing data. + * + * Capability-scoped like `update`: caller MUST pass expectedDatasetId. + */ +export const updateCells = internalMutation({ + args: { + id: v.id("datasetRows"), + expectedDatasetId: v.id("datasets"), + data: v.record(v.string(), v.any()), + }, + handler: async (ctx, args) => { + const existing = await assertRowInDataset( + ctx, + args.id, + args.expectedDatasetId, + ); + + const oldData = existing.data as Record; + const newData = args.data; + + // Merge: new data overwrites matching keys, other keys preserved + const mergedData: Record = { ...oldData }; + for (const [key, newVal] of Object.entries(newData)) { + mergedData[key] = newVal; + // Track history for cells that changed + const oldVal = oldData[key]; + if (oldVal !== undefined && String(oldVal) !== String(newVal)) { + await ctx.db.insert("datasetHistory", { + datasetRowId: args.id, + columnName: key, + oldValue: String(oldVal ?? ""), + newValue: String(newVal ?? ""), + changedAt: Date.now(), + }); + } + } + + await ctx.db.patch(args.id, { data: mergedData }); + return { success: true }; + }, +}); + +/** + * Query rows eligible for enrichment. + * + * A row qualifies if: + * - ALL sourceColumns have non-null values + * - At least one targetColumn has a null/missing value + * + * Used by the enrichment workflow to determine which rows need filling. + */ +export const getRowsForEnrich = internalQuery({ + args: { + datasetId: v.id("datasets"), + sourceColumns: v.array(v.string()), + targetColumns: v.array(v.string()), + }, + handler: async (ctx, args) => { + const rows = await ctx.db + .query("datasetRows") + .withIndex("by_dataset", (q) => q.eq("datasetId", args.datasetId)) + .collect(); + + return rows.filter((row) => { + const data = row.data as Record; + + // Check all source columns have values + const hasAllSources = args.sourceColumns.every( + (col) => data[col] !== undefined && data[col] !== null && data[col] !== "" + ); + + // Check at least one target column is empty + const hasEmptyTargets = args.targetColumns.some( + (col) => data[col] === undefined || data[col] === null || data[col] === "" + ); + + return hasAllSources && hasEmptyTargets; + }); + }, +}); + +/** + * Bulk update multiple rows' cells in a single transaction. + * Used by the enrichment workflow after processing rows. + */ +export const bulkUpdateCells = internalMutation({ + args: { + datasetId: v.id("datasets"), + updates: v.array( + v.object({ + rowId: v.id("datasetRows"), + data: v.record(v.string(), v.any()), + }) + ), + }, + handler: async (ctx, args) => { + const dataset = await ctx.db.get(args.datasetId); + if (!dataset) throw new Error("Dataset not found"); + + const results: Array<{ rowId: string; success: boolean; error?: string }> = []; + + for (const update of args.updates) { + try { + const existing = await assertRowInDataset(ctx, update.rowId, args.datasetId); + + const oldData = existing.data as Record; + const mergedData: Record = { ...oldData }; + + for (const [key, newVal] of Object.entries(update.data)) { + mergedData[key] = newVal; + const oldVal = oldData[key]; + if (oldVal !== undefined && String(oldVal) !== String(newVal)) { + await ctx.db.insert("datasetHistory", { + datasetRowId: update.rowId, + columnName: key, + oldValue: String(oldVal ?? ""), + newValue: String(newVal ?? ""), + changedAt: Date.now(), + }); + } + } + + await ctx.db.patch(update.rowId, { data: mergedData }); + results.push({ rowId: update.rowId, success: true }); + } catch (err) { + results.push({ + rowId: update.rowId, + success: false, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + return results; + }, +}); diff --git a/frontend/convex/datasets.ts b/frontend/convex/datasets.ts index 791671b..3ed6ae6 100644 --- a/frontend/convex/datasets.ts +++ b/frontend/convex/datasets.ts @@ -450,6 +450,17 @@ export const listByOwnerInternal = internalQuery({ }, }); +export const listPublicInternal = internalQuery({ + args: {}, + handler: async (ctx) => { + return await ctx.db + .query("datasets") + .withIndex("by_visibility", (q) => q.eq("visibility", "public")) + .order("desc") + .collect(); + }, +}); + export const getOwnedInternal = internalQuery({ args: { id: v.id("datasets"), diff --git a/frontend/convex/schema.ts b/frontend/convex/schema.ts index 83ea007..2f1f72b 100644 --- a/frontend/convex/schema.ts +++ b/frontend/convex/schema.ts @@ -186,4 +186,53 @@ export default defineSchema({ .index("by_dataset", ["datasetId"]) .index("by_user", ["userId"]) .index("by_workflow_run", ["workflowRunId"]), + + // API keys for non-browser clients (Google Sheets add-on, CLI, etc.). + // We never store the plaintext key — only a SHA-256 hash. The `keyPrefix` + // (first 8 chars) is returned in listings so users can identify a key + // without leaking the secret. Created by the backend, listed/revoked + // through HTTP routes under requireAuth. + apiKeys: defineTable({ + ownerId: v.string(), + name: v.string(), + keyHash: v.string(), + keyPrefix: v.string(), + lastUsedAt: v.optional(v.number()), + createdAt: v.number(), + revokedAt: v.optional(v.number()), + }) + .index("by_owner", ["ownerId"]) + .index("by_hash", ["keyHash"]), + + /** + * Tracks individual data enrichment runs initiated by the Google Sheets addon. + * Each run fills empty columns in existing rows using AI-powered research. + */ + sheetsEnrichmentRuns: defineTable({ + datasetId: v.id("datasets"), + userId: v.string(), + sourceColumns: v.array(v.string()), + targetColumns: v.array(v.string()), + status: v.union( + v.literal("running"), + v.literal("completed"), + v.literal("failed"), + v.literal("stopped") + ), + rowsProcessed: v.number(), + rowsUpdated: v.number(), + rowsFound: v.number(), + errors: v.array( + v.object({ + rowId: v.string(), + message: v.string(), + }) + ), + startedAt: v.number(), + completedAt: v.optional(v.number()), + workflowRunId: v.optional(v.string()), + }) + .index("by_dataset", ["datasetId"]) + .index("by_user", ["userId"]) + .index("by_workflow_run", ["workflowRunId"]), }); diff --git a/frontend/convex/sheetsEnrichmentRuns.ts b/frontend/convex/sheetsEnrichmentRuns.ts new file mode 100644 index 0000000..89f0004 --- /dev/null +++ b/frontend/convex/sheetsEnrichmentRuns.ts @@ -0,0 +1,160 @@ +import { + query, + mutation, + internalMutation, + internalQuery, +} from "./_generated/server.js"; +import { v } from "convex/values"; +import type { Id } from "./_generated/dataModel.js"; +import { loadOwnedDataset, requireIdentity } from "./lib/authz.js"; + +/** + * Creates a new enrichment run tracking record. + * Called when the user starts an enrichment operation from the Sheets addon. + */ +export const create = internalMutation({ + args: { + datasetId: v.id("datasets"), + userId: v.string(), + sourceColumns: v.array(v.string()), + targetColumns: v.array(v.string()), + workflowRunId: v.optional(v.string()), + }, + handler: async (ctx, args) => { + return await ctx.db.insert("sheetsEnrichmentRuns", { + datasetId: args.datasetId, + userId: args.userId, + sourceColumns: args.sourceColumns, + targetColumns: args.targetColumns, + status: "running", + rowsProcessed: 0, + rowsUpdated: 0, + rowsFound: 0, + errors: [], + startedAt: Date.now(), + workflowRunId: args.workflowRunId, + }); + }, +}); + +/** + * Update enrichment run progress. + * Called by the enrichment workflow as rows are processed. + */ +export const updateProgress = internalMutation({ + args: { + id: v.id("sheetsEnrichmentRuns"), + rowsProcessed: v.number(), + rowsUpdated: v.number(), + rowsFound: v.number(), + }, + handler: async (ctx, args) => { + const run = await ctx.db.get(args.id); + if (!run || run.status !== "running") return; + + await ctx.db.patch(args.id, { + rowsProcessed: args.rowsProcessed, + rowsUpdated: args.rowsUpdated, + rowsFound: args.rowsFound, + }); + }, +}); + +/** + * Record an error for a specific row during enrichment. + */ +export const addError = internalMutation({ + args: { + id: v.id("sheetsEnrichmentRuns"), + rowId: v.string(), + message: v.string(), + }, + handler: async (ctx, args) => { + const run = await ctx.db.get(args.id); + if (!run || run.status !== "running") return; + + const errors = [...run.errors, { rowId: args.rowId, message: args.message }]; + await ctx.db.patch(args.id, { errors }); + }, +}); + +/** + * Mark enrichment run as completed. + */ +export const complete = internalMutation({ + args: { + id: v.id("sheetsEnrichmentRuns"), + rowsProcessed: v.number(), + rowsUpdated: v.number(), + rowsFound: v.number(), + status: v.union( + v.literal("completed"), + v.literal("failed"), + v.literal("stopped") + ), + }, + handler: async (ctx, args) => { + await ctx.db.patch(args.id, { + status: args.status, + rowsProcessed: args.rowsProcessed, + rowsUpdated: args.rowsUpdated, + rowsFound: args.rowsFound, + completedAt: Date.now(), + }); + }, +}); + +/** + * Get enrichment run by ID. + */ +export const get = internalQuery({ + args: { id: v.id("sheetsEnrichmentRuns") }, + handler: async (ctx, args) => { + return await ctx.db.get(args.id); + }, +}); + +/** + * Get enrichment runs for a dataset. + */ +export const listByDataset = internalQuery({ + args: { datasetId: v.id("datasets") }, + handler: async (ctx, args) => { + return await ctx.db + .query("sheetsEnrichmentRuns") + .withIndex("by_dataset", (q) => q.eq("datasetId", args.datasetId)) + .order("desc") + .collect(); + }, +}); + +/** + * User-facing: list enrichment runs for datasets owned by the authenticated user. + */ +export const listByUser = query({ + args: {}, + handler: async (ctx) => { + const identity = await requireIdentity(ctx); + + return await ctx.db + .query("sheetsEnrichmentRuns") + .withIndex("by_user", (q) => q.eq("userId", identity.subject)) + .order("desc") + .take(50); + }, +}); + +/** + * Get the most recent running enrichment run for a dataset, if any. + */ +export const getActiveRunForDataset = internalQuery({ + args: { datasetId: v.id("datasets") }, + handler: async (ctx, args) => { + const runs = await ctx.db + .query("sheetsEnrichmentRuns") + .withIndex("by_dataset", (q) => q.eq("datasetId", args.datasetId)) + .collect(); + + return runs.find((r) => r.status === "running") ?? null; + }, +}); diff --git a/frontend/lib/backend.ts b/frontend/lib/backend.ts index f5ea3e5..22a70dd 100644 --- a/frontend/lib/backend.ts +++ b/frontend/lib/backend.ts @@ -344,6 +344,51 @@ export async function update( return res.json(); } +export interface ApiKeyInfo { + id: string; + name: string; + keyPrefix: string; + createdAt: number; + lastUsedAt: number | null; +} + +export interface CreatedApiKey { + id: string; + key: string; + keyPrefix: string; + name: string; +} + +export async function listApiKeys(token: string): Promise<{ keys: ApiKeyInfo[] }> { + const res = await fetch(`${BACKEND_URL}/api-keys`, { + method: "GET", + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(await errorMessage(res)); + return res.json(); +} + +export async function createApiKey(name: string, token: string): Promise { + const res = await fetch(`${BACKEND_URL}/api-keys`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ name }), + }); + if (!res.ok) throw new Error(await errorMessage(res)); + return res.json(); +} + +export async function revokeApiKey(id: string, token: string): Promise { + const res = await fetch(`${BACKEND_URL}/api-keys/${id}`, { + method: "DELETE", + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(await errorMessage(res)); +} + export async function stopPopulation( datasetId: string, token: string,