Add async processing of PDF imports and other general improvements (#22)

Author: danielnaab

CLAUDE.md

@@ -114,6 +114,45 @@ common → (no dependencies)
 - **Testing**: Vitest (unit/integration), Playwright (E2E), @vitest/browser (Storybook)
 - **Language**: TypeScript throughout
 
+### Development Workflow with Conditional Exports
+
+This monorepo uses **conditional exports** for zero-build development workflow:
+
+**In Development:**
+- Library packages (common, database, forms-core, auth, design) are consumed directly from TypeScript source files
+- No build step required when editing library code - changes are immediately reflected in consuming apps
+- Hot module replacement (HMR) works instantly across package boundaries
+- Consumer apps (server, spotlight, etc.) are configured with `customConditions: ["development"]` in tsconfig.json
+- Vite/Astro resolve the `development` export condition to use `./src/**/*.ts` files
+
+**In Production:**
+- Library packages are built and published from `dist/` folders
+- Production builds use optimized, transpiled artifacts
+- The `development` export condition is not used
+
+**How it Works:**
+Each library package.json has exports like:
+```json
+{
+  "exports": {
+    ".": {
+      "development": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts"
+      },
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  }
+}
+```
+
+**Build Scripts:**
+- **Important:** The `design` package requires CSS/SASS compilation:
+  - First time setup: Run `pnpm --filter @flexion/forms-design build:styles` (one-time)
+  - Or use `pnpm dev` which includes `dev:styles` (gulp watch) for the design package
+- Production builds (`pnpm build`) are still required before publishing
+
 ### Pattern System
 
 Patterns are the platform's primary building blocks. Each pattern has:
@@ -137,5 +176,9 @@ Use `describeDatabase` helper for testing database routines against both SQLite
 - Node version is specified in `.nvmrc` - use `nvm install` to ensure correct version
 - Requires Docker or Podman for running tests (PostgreSQL container)
 - Playwright version must match exactly (1.51.1) across local and CI environments
-- Build is required before running `pnpm dev`
+- **Development**:
+  - No TypeScript build required - packages are consumed from source via conditional exports
+  - CSS/styles must be built once: `pnpm --filter @flexion/forms-design build:styles`
+  - Or run `pnpm dev` which includes style watching
+- **Production/Publishing**: Run `pnpm build` to create optimized artifacts before publishing
 - Pre-commit hook runs `pnpm format` automatically

apps/cli/src/cli-controller/forms.ts

@@ -1,9 +1,9 @@
 import { promises as fs } from 'fs';
 import { Command } from 'commander';
 
-import { commands } from '@flexion/forms-infra-core';
 import { type Context } from './types.js';
-import { createFormService, createFormsRepository, defaultFormConfig, parsePdf as parsePdfCore } from '@flexion/forms-core';
+import { createFormService, defaultFormConfig, parsePdf as parsePdfCore } from '@flexion/forms-core';
+import { createFormsRepository } from '@flexion/forms-core/repository';
 import { createTestPdfParser } from '@flexion/forms-core/documents/pdf/context';
 import { createFilesystemDatabaseContext } from '@flexion/forms-database/context';
 

apps/sandbox/tsconfig.json

@@ -4,7 +4,8 @@
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "outDir": "./dist",
-    "emitDeclarationOnly": true
+    "emitDeclarationOnly": true,
+    "customConditions": ["development"]
   },
   "include": ["./src"],
   "references": []

apps/server-doj/tsconfig.json

@@ -4,7 +4,8 @@
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "outDir": "./dist",
-    "emitDeclarationOnly": true
+    "emitDeclarationOnly": true,
+    "customConditions": ["development"]
   },
   "include": ["./src"],
   "references": []

apps/spotlight/astro.config.mjs

@@ -21,6 +21,11 @@ export default defineConfig({
     define: {
       'import.meta.env.GITHUB': JSON.stringify(githubRepository),
     },
+    resolve: {
+      conditions: process.env.NODE_ENV === 'production'
+        ? ['production', 'import', 'module', 'browser', 'default']
+        : ['development', 'import', 'module', 'browser', 'default'],
+    },
   },
 });
 

apps/spotlight/tsconfig.json

@@ -6,7 +6,8 @@
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "jsx": "react",
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
+    "customConditions": ["development"]
   },
   "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.astro"],
   "exclude": [".astro", "dist", "node_modules"]

packages/auth/package.json

@@ -6,6 +6,17 @@
   "license": "CC0",
   "main": "dist/index.js",
   "types": "dist/index.d.js",
+  "exports": {
+    ".": {
+      "development": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts"
+      },
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
   "publishConfig": {
     "registry": "https://npm.pkg.github.com"
   },

packages/common/package.json

@@ -6,6 +6,17 @@
   "license": "CC0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "development": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts"
+      },
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
   "publishConfig": {
     "registry": "https://npm.pkg.github.com"
   },

packages/database/migrations/20251007000000_create_form_jobs.mjs

@@ -0,0 +1,58 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export async function up(knex) {
+  await knex.schema.createTable('form_jobs', table => {
+    table.uuid('id').primary();
+
+    // Foreign key to forms (CASCADE delete: jobs belong to forms)
+    table
+      .uuid('form_id')
+      .notNullable()
+      .references('id')
+      .inTable('forms')
+      .onDelete('CASCADE');
+
+    // Job type - extensible for future operations
+    table.text('job_type').notNullable();
+    // Values: 'import-pdf', 'validate-schema', 'publish', 'export', etc.
+
+    // Job status - represents operation state
+    table.text('status').notNullable();
+    // Values: 'pending', 'processing', 'completed', 'failed'
+
+    // Timing information
+    table.timestamp('created_at').notNullable().defaultTo(knex.fn.now());
+    table.timestamp('started_at').nullable();
+    table.timestamp('completed_at').nullable();
+
+    // Error tracking
+    table.text('error_message').nullable();
+    table.text('error_stack').nullable();
+
+    // Job metadata (input parameters, varies by job_type)
+    // For 'import-pdf': { documentId, fileName, userId }
+    // For 'publish': { targetEnvironment, publisherId }
+    table.text('metadata').nullable();
+
+    // Job result (output data, varies by job_type)
+    // For 'import-pdf': { patternsAdded: 5, fieldsExtracted: 12 }
+    // For 'validate': { errorsFound: 2, warningsFound: 5 }
+    table.text('result').nullable();
+
+    // Indexes for common queries
+    table.index('form_id', 'idx_form_jobs_form_id');
+    table.index('status', 'idx_form_jobs_status');
+    table.index(['form_id', 'job_type'], 'idx_form_jobs_form_type');
+    table.index('created_at', 'idx_form_jobs_created');
+  });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export async function down(knex) {
+  await knex.schema.dropTableIfExists('form_jobs');
+}

packages/database/package.json

@@ -12,16 +12,28 @@
   },
   "exports": {
     ".": {
+      "development": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts"
+      },
       "types": "./dist/types/index.d.ts",
       "import": "./dist/esm/index.js",
       "require": "./dist/cjs/index.js"
     },
     "./context": {
+      "development": {
+        "types": "./src/context/index.ts",
+        "import": "./src/context/index.ts"
+      },
       "types": "./dist/types/context/index.d.ts",
       "import": "./dist/esm/context.js",
       "require": "./dist/cjs/context.js"
     },
     "./testing": {
+      "development": {
+        "types": "./src/testing.ts",
+        "import": "./src/testing.ts"
+      },
       "types": "./dist/types/testing.d.ts",
       "import": "./dist/esm/testing.js",
       "require": "./dist/cjs/testing.js"