Merge pull request #236 from Deodat-Lawson/main

Update Mar 5th

Author: Deodat-Lawson

.env.example

@@ -0,0 +1,47 @@
+ # Since the ".env" file is gitignored, you can use the ".env.example" file to
+# build a new ".env" file when you clone the repo. Keep this file up-to-date
+# when you add new variables to `.env`.
+
+# This file will be committed to version control, so make sure not to have any
+# secrets in it. If you are cloning this repo, create a copy of this file named
+# ".env" and populate it with your secrets.
+
+# When adding additional environment variables, the schema in "/src/env.js"
+# should be updated accordingly.
+
+# Database 
+DATABASE_URL="" 
+
+# Docker Compose: password for PostgreSQL (used by db service)
+# POSTGRES_PASSWORD=password
+
+# Clerk Authentication (get from https://clerk.com/)
+
+NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=
+CLERK_SECRET_KEY=
+
+# OpenAI API (get from https://platform.openai.com/)
+
+OPENAI_API_KEY=
+
+# UploadThing (get from https://uploadthing.com/)
+UPLOADTHING_SECRET="your_uploadthing_secret"
+UPLOADTHING_APP_ID="your_uploadthing_app_id"
+
+# Datalab OCR API (optional - get from https://www.datalab.to/)
+# Required only if you want to enable OCR processing for scanned documents
+DATALAB_API_KEY="your_datalab_api_key"
+
+# Landing.AI OCR API (optional - get from https://www.landing.ai/)
+LANDING_AI_API_KEY="your_landing_ai_api_key"
+
+# Tavily API (optional - get from https://www.tavily.com/)
+TAVILY_API_KEY="your_tavily_api_key"
+
+# Azure Document Intelligence OCR API (optional - get from https://learn.microsoft.com/en-us/azure/applied-ai-services/document-intelligence/quickstarts/get-started-with-rest-api?pivots=programming-language-rest-api)
+AZURE_DOC_INTELLIGENCE_ENDPOINT="your_azure_doc_intelligence_endpoint"
+AZURE_DOC_INTELLIGENCE_KEY="your_azure_doc_intelligence_key"
+
+# Inngest (required for background document processing - https://inngest.com/)
+INNGEST_EVENT_KEY="dev_placeholder"
+INNGEST_SIGNING_KEY="signkey-dev-xxxxx"

README.md

@@ -124,6 +124,7 @@ Create `.env` from `.env.example` and fill required values:
 - `DATABASE_URL`
 - `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY`
 - `CLERK_SECRET_KEY`
+- `BLOB_READ_WRITE_TOKEN` (Vercel Blob read/write token)
 - `OPENAI_API_KEY`
 - `INNGEST_EVENT_KEY`, as placeholder
 
@@ -137,6 +138,18 @@ Optional integrations:
 - `LANGCHAIN_TRACING_V2`, `LANGCHAIN_API_KEY`, `LANGCHAIN_PROJECT`
 - `DEBUG_PERF` (`1` or `true`) to enable dev perf logs for middleware and key auth/dashboard APIs
 
+### 2.1) Configure Vercel Blob Storage
+
+Vercel Blob is used for storing uploaded documents. Both **public** and **private** stores are supported -- the upload logic auto-detects which mode the store uses and adapts automatically.
+
+1. In the Vercel dashboard, go to **Storage → Blob → Create Store**.
+2. Choose either **Public** or **Private** access. Both work:
+   - **Public** stores produce URLs the browser can load directly (faster for previews).
+   - **Private** stores keep files behind authentication; the app proxies content through `/api/documents/[id]/content` and `/api/files/[id]` so previews still work.
+3. Generate a **Read/Write token** for the store and add it as `BLOB_READ_WRITE_TOKEN` in your environment (`.env` locally, or Vercel Project Settings for deploys).
+4. Redeploy so the token is available at build and runtime.
+5. Verify: sign in to the Employer Upload page, upload a small PDF, and confirm `/api/upload-local` returns a `vercel-storage.com` URL without errors.
+
 ### 3) Start database and apply schema
 
 ```bash

__tests__/api/fetchDocument/fetchDocument.test.ts

@@ -1,7 +1,8 @@
-import { POST } from "~/app/api/fetchDocument/route";
-import { auth } from "@clerk/nextjs/server";
-import { validateRequestBody } from "~/lib/validation";
-import { dbCore } from "~/server/db/core";
+jest.mock("~/server/storage/vercel-blob", () => ({
+  isPrivateBlobUrl: jest.fn(() => false),
+  fetchBlob: jest.fn(),
+  putFile: jest.fn(),
+}));
 
 jest.mock("@clerk/nextjs/server", () => ({
   auth: jest.fn(),
@@ -11,13 +12,17 @@ jest.mock("~/lib/validation", () => ({
   validateRequestBody: jest.fn(),
 }));
 
-// Route uses dbCore from core, not db from index
 jest.mock("~/server/db/core", () => ({
   dbCore: {
     select: jest.fn(),
   },
 }));
 
+import { POST } from "~/app/api/fetchDocument/route";
+import { auth } from "@clerk/nextjs/server";
+import { validateRequestBody } from "~/lib/validation";
+import { dbCore } from "~/server/db/core";
+
 describe("POST /api/fetchDocument", () => {
   beforeEach(() => {
     jest.clearAllMocks();

__tests__/api/trendSearch/web-search.pbt.test.ts

@@ -115,8 +115,8 @@ describe("Unit: one sub-query returns 0 results, pipeline continues", () => {
         const result = await executeSearch(subQueries);
 
         expect(fetchSpy).toHaveBeenCalledTimes(3);
-        expect(result).toHaveLength(2);
-        expect(result.map((r) => r.url)).toEqual(["https://b.com", "https://c.com"]);
+        expect(result.results).toHaveLength(2);
+        expect(result.results.map((r) => r.url)).toEqual(["https://b.com", "https://c.com"]);
     });
 });
 
@@ -158,7 +158,7 @@ describe("Unit: Tavily fails, retries 2 times then marks sub-query failed", () =
 
         // 1 + 2 retries for first sub-query, then 1 for second
         expect(fetchSpy).toHaveBeenCalledTimes(4);
-        expect(result).toHaveLength(1);
-        expect(result[0].url).toBe("https://ok.com");
+        expect(result.results).toHaveLength(1);
+        expect(result.results[0].url).toBe("https://ok.com");
     });
 });

__tests__/lib/ocr/complexity.test.ts

@@ -1,3 +1,9 @@
+jest.mock("~/server/storage/vercel-blob", () => ({
+  fetchBlob: jest.fn(),
+  putFile: jest.fn(),
+  isPrivateBlobUrl: jest.fn(() => false),
+}));
+
 import { selectSamplePages } from "~/lib/ocr/complexity";
 
 describe("OCR Complexity Module", () => {

dev-output.log

@@ -51,7 +51,7 @@ docker compose --env-file .env --profile dev up
 
 1. Import repository into Vercel.
 2. Configure managed PostgreSQL (Vercel Postgres, Neon, Supabase, etc.).
-3. Set `DATABASE_URL` and app environment variables.
+3. Set `DATABASE_URL`, `BLOB_READ_WRITE_TOKEN`, and the other app environment variables.
 4. Deploy with Vercel defaults.
 5. Apply schema once:
 
@@ -65,6 +65,12 @@ Optional integrations:
 - LangSmith for tracing
 - Sidecar (deploy separately and set `SIDECAR_URL`)
 
+### Verifying Blob uploads on Vercel
+
+1. After deploy, sign in to the Employer portal and open `/employer/upload`.
+2. Upload any small PDF or DOCX. The `/api/upload-local` response should return a `vercel-storage.com` URL.
+3. Paste that URL into a new tab. The file should download directly, confirming Blob access end to end.
+
 ## Option 3: VPS self-hosted (Node + reverse proxy)
 
 1. Install Node.js 18+, pnpm, Nginx, and PostgreSQL with pgvector.
@@ -89,7 +95,8 @@ Optional: Run the sidecar separately and point `SIDECAR_URL` to it.
 | `CLERK_SECRET_KEY` | Yes | Clerk secret key |
 | `OPENAI_API_KEY` | Yes | OpenAI API key |
 | `INNGEST_EVENT_KEY` | Yes (prod) | Inngest event key for background jobs |
-| `UPLOADTHING_TOKEN` | Optional | UploadThing for cloud storage |
+| `BLOB_READ_WRITE_TOKEN` | Yes (Vercel) | Required for Vercel Blob uploads |
+| `UPLOADTHING_TOKEN` | Optional | UploadThing legacy uploader |
 | `SIDECAR_URL` | Optional | Sidecar URL for reranking and Graph RAG |
 | `TAVILY_API_KEY` | Optional | Web search for analysis |
 | `AZURE_DOC_INTELLIGENCE_*` | Optional | OCR for scanned PDFs |

docs/deployment.md

@@ -0,0 +1,39 @@
+-- Company Metadata Tables
+-- One canonical JSON row per company + append-only audit history.
+
+CREATE TABLE IF NOT EXISTS "company_metadata" (
+    "id"                          serial PRIMARY KEY NOT NULL,
+    "company_id"                  bigint NOT NULL REFERENCES "company"("id") ON DELETE CASCADE,
+    "schema_version"              varchar(20) NOT NULL DEFAULT '1.0.0',
+    "metadata"                    jsonb NOT NULL,
+    "last_extraction_document_id" bigint REFERENCES "document"("id") ON DELETE SET NULL,
+    "created_at"                  timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at"                  timestamptz
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS "company_metadata_company_id_unique"
+    ON "company_metadata" ("company_id");
+
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS "company_metadata_history" (
+    "id"          serial PRIMARY KEY NOT NULL,
+    "company_id"  bigint NOT NULL REFERENCES "company"("id") ON DELETE CASCADE,
+    "document_id" bigint REFERENCES "document"("id") ON DELETE SET NULL,
+    "change_type" varchar(32) NOT NULL,   -- extraction | merge | manual_override | deprecation
+    "diff"        jsonb NOT NULL,
+    "changed_by"  varchar(256) NOT NULL,
+    "created_at"  timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS "company_metadata_history_company_id_idx"
+    ON "company_metadata_history" ("company_id");
+
+CREATE INDEX IF NOT EXISTS "company_metadata_history_document_id_idx"
+    ON "company_metadata_history" ("document_id");
+
+CREATE INDEX IF NOT EXISTS "company_metadata_history_created_at_idx"
+    ON "company_metadata_history" ("created_at");
+
+CREATE INDEX IF NOT EXISTS "company_metadata_history_change_type_idx"
+    ON "company_metadata_history" ("change_type");

drizzle/0002_company_metadata.sql

@@ -0,0 +1,8 @@
+ALTER TABLE "file_uploads"
+    ADD COLUMN IF NOT EXISTS "storage_provider" varchar(64) NOT NULL DEFAULT 'database',
+    ADD COLUMN IF NOT EXISTS "storage_url" varchar(1024),
+    ADD COLUMN IF NOT EXISTS "storage_pathname" varchar(1024),
+    ADD COLUMN IF NOT EXISTS "blob_checksum" varchar(128);
+
+ALTER TABLE "file_uploads"
+    ALTER COLUMN "file_data" DROP NOT NULL;

drizzle/0002_vercel_blob.sql

@@ -0,0 +1,4 @@
+-- Add onboarding profile columns to company table
+ALTER TABLE "company"
+    ADD COLUMN IF NOT EXISTS "description" text,
+    ADD COLUMN IF NOT EXISTS "industry" varchar(256);