Merge pull request #41 from constructive-io/feat/portable-functions

feat(packages): extract fn-types; make fn-runtime/fn-app publishable (Wave 1 of portable-functions toolkit)

Author: Anmol1696

.github/workflows/docker.yaml

@@ -59,8 +59,14 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v6
 
+      # Pinned to v4: setup-node@v5 auto-detects pnpm via the
+      # packageManager field in package.json and tries to cache the
+      # store, which fails here because this workflow doesn't run
+      # `pnpm install` on the runner — the store path doesn't exist
+      # and v5 promotes the "missing path" warning to an error.
+      # Sticking to v4 until v5's auto-cache can be opted out cleanly.
       - name: Setup Node.js
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v4
         with:
           node-version: '22'
 

.github/workflows/publish.yaml

@@ -0,0 +1,73 @@
+name: Publish toolkit packages
+
+on:
+  push:
+    tags:
+      - 'fn-v*'   # toolkit release tag, e.g. fn-v0.1.0
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Run pnpm publish --dry-run only'
+        type: boolean
+        default: true
+
+concurrency:
+  group: publish-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    name: Publish @constructive-io/fn-* to npm
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write   # required for npm provenance
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v5
+        with:
+          node-version: '22'
+          registry-url: 'https://registry.npmjs.org'
+          cache: 'pnpm'
+
+      - name: Generate function packages
+        run: node --experimental-strip-types scripts/generate.ts
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build toolkit packages
+        run: |
+          pnpm --filter @constructive-io/fn-types build
+          pnpm --filter @constructive-io/knative-job-fn build
+          pnpm --filter @constructive-io/fn-runtime build
+          pnpm --filter @constructive-io/fn-generator build
+          pnpm --filter @constructive-io/fn-client build
+          pnpm --filter @constructive-io/fn-cli build
+
+      - name: Verify generator snapshot
+        run: pnpm --filter @constructive-io/fn-generator test
+
+      - name: Publish (dry run for workflow_dispatch when requested)
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true'
+        run: |
+          for pkg in fn-types fn-app fn-runtime fn-generator fn-client fn-cli; do
+            (cd "packages/$pkg" && pnpm publish --dry-run --no-git-checks --access public)
+          done
+
+      - name: Publish to npm with provenance
+        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != 'true')
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_CONFIG_PROVENANCE: 'true'
+        run: |
+          # Order matters: deps first, dependents last.
+          for pkg in fn-types fn-app fn-runtime fn-generator fn-client fn-cli; do
+            (cd "packages/$pkg" && pnpm publish --no-git-checks --access public)
+          done

.github/workflows/test.yaml

@@ -40,3 +40,52 @@ jobs:
       - run: pnpm install
       - run: pnpm build
       - run: pnpm test:integration
+
+  toolkit:
+    name: Toolkit package tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '22'
+          cache: 'pnpm'
+      - run: node --experimental-strip-types scripts/generate.ts
+      - run: pnpm install
+      - name: Build toolkit packages
+        run: |
+          pnpm --filter @constructive-io/fn-types build
+          pnpm --filter @constructive-io/knative-job-fn build
+          pnpm --filter @constructive-io/fn-runtime build
+          pnpm --filter @constructive-io/fn-generator build
+          pnpm --filter @constructive-io/fn-client build
+          pnpm --filter @constructive-io/fn-cli build
+      - name: Run toolkit unit tests
+        run: |
+          pnpm --filter @constructive-io/fn-generator test
+          pnpm --filter @constructive-io/fn-client test
+          pnpm --filter @constructive-io/fn-cli test
+
+  fn-init-e2e:
+    name: fn init end-to-end
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '22'
+          cache: 'pnpm'
+      - run: node --experimental-strip-types scripts/generate.ts
+      - run: pnpm install
+      - name: Build fn-cli (transitive)
+        run: |
+          pnpm --filter @constructive-io/fn-types build
+          pnpm --filter @constructive-io/knative-job-fn build
+          pnpm --filter @constructive-io/fn-runtime build
+          pnpm --filter @constructive-io/fn-generator build
+          pnpm --filter @constructive-io/fn-client build
+          pnpm --filter @constructive-io/fn-cli build
+      - name: Binary integration test
+        run: pnpm exec jest tests/integration/fn-init.test.ts

README.md

@@ -2,7 +2,22 @@
 
 Functions playground for Constructive — isolated workspace for building, testing, and deploying Knative-style HTTP functions.
 
-## Quick Start
+This repo is also the source of the **Portable Functions Toolkit**: a set of `@constructive-io/fn-*` npm packages that any external repo can `pnpm add` to get the same code-gen + Docker + k8s pipeline against its own `functions/` directory. See [docs/portable-functions-toolkit.md](docs/portable-functions-toolkit.md) for the full toolkit guide.
+
+## Quick start (in another repo)
+
+```bash
+pnpm add -D @constructive-io/fn-cli
+pnpm add @constructive-io/fn-runtime
+
+pnpm fn init send-welcome --no-tty           # scaffold functions/send-welcome/
+pnpm fn generate                             # stamp out generated/<name>/ packages
+pnpm install                                 # link the new workspaces
+pnpm fn build                                # compile
+pnpm fn dev                                  # run functions as local Node processes
+```
+
+## Quick start (this repo, dogfood)
 
 ```bash
 # Install dependencies

docs/portable-functions-toolkit.md

@@ -0,0 +1,119 @@
+# Portable Functions Toolkit
+
+The Constructive Functions toolkit lets any external repo `pnpm add` a small set of npm packages, drop in its own `functions/` directory, and get a full code-gen + Docker + k8s + manifest-registry pipeline — without git submodules or copy-paste.
+
+## Package layout (Starship V2 style)
+
+```
+fn-cli ──► fn-client ──► fn-generator ──► fn-types
+                  └────────────────────► fn-types
+        └────────► fn-types
+fn-runtime ──► fn-types         (handlers import this + fn-types)
+knative-job-fn (fn-app)         (low-level Express middleware)
+```
+
+| Package | Single responsibility |
+|---|---|
+| `@constructive-io/fn-types` | Source-of-truth TS types: `FunctionHandler`, `FunctionContext`, `HandlerManifest`, `FnRegistry`, `FnConfig` + `defineConfig()`. No logic. |
+| `@constructive-io/fn-runtime` | Express server factory + GraphQL clients + log + job-callback wiring. The contract handlers import. |
+| `@constructive-io/knative-job-fn` | Low-level Express middleware for Knative job request/response shape. fn-runtime depends on it. |
+| `@constructive-io/fn-generator` | Programmatic builders that emit Dockerfiles, k8s YAML, configmaps, skaffold profiles, manifest registry. Pure functions; idempotent file I/O at the boundary. |
+| `@constructive-io/fn-client` | Importable `FnClient` API — config loading, manifest reading, `pnpm build`, child-process orchestration for `dev`. |
+| `@constructive-io/fn-cli` | The `fn` executable. Subcommands: `init`, `generate`, `build`, `dev`, `manifest`, `verify`. |
+
+## Quick start
+
+```bash
+# In a fresh project
+pnpm add -D @constructive-io/fn-cli
+pnpm add @constructive-io/fn-runtime
+
+# Scaffold a function
+pnpm fn init send-welcome --no-tty --description "Welcome email sender"
+# → functions/send-welcome/{handler.json, handler.ts}
+
+# Stamp out the workspace package, build, run
+pnpm fn generate
+pnpm install                           # link the just-created generated/* workspaces
+pnpm fn build
+pnpm fn dev                            # functions run as local Node processes
+```
+
+`fn init` uses [`genomic`](https://www.npmjs.com/package/genomic) under the hood — the same template engine `pgpm init` uses — so the prompt conventions and `--no-tty` flag-mapping match the rest of the Constructive ecosystem. Two handler types ship today: `--type=node-graphql` (default) and `--type=python`.
+
+## Repo layout the toolkit expects
+
+```
+my-app/
+├── functions/
+│   └── send-welcome/
+│       ├── handler.json   # {"name":"send-welcome","version":"0.1.0","type":"node-graphql"}
+│       └── handler.ts     # default-exported FunctionHandler
+├── fn.config.json         # FnConfig (typed via fn-types) — optional
+└── package.json
+```
+
+## CLI surface
+
+```bash
+fn init <name> [--type=node-graphql|python] [--description=<d>] [--force] [--no-tty]
+fn generate [--only=<name>] [--packages-only]
+fn build [--only=<name>]
+fn dev [--only=<name>]
+fn manifest                            # print on-disk functions-manifest.json
+fn verify                              # check manifest matches functions/
+fn --version                           # print fn-cli version
+```
+
+Common flags: `--root=<dir>`, `--config=<file>`.
+
+## Job-service registry (when running the `jobs-bundle` preset)
+
+The job-service no longer hardcodes function names. It loads its registry at startup from one of three sources, in priority order:
+
+1. `FUNCTIONS_REGISTRY` env var  
+   Format: `name:moduleName:port,...` — `moduleName` and `port` are optional (missing `moduleName` falls back to `@constructive-io/<name>-fn`).
+
+2. `FUNCTIONS_MANIFEST_PATH` env var pointing to a JSON file with the existing `functions-manifest.json` shape. Manifest entries can carry an optional `moduleName` field; otherwise convention applies.
+
+3. Default file: `<cwd>/generated/functions-manifest.json` — what `fn generate` produces.
+
+Empty registry is allowed; lookups still throw `Unknown function "<name>"` to preserve the legacy behaviour.
+
+## Releasing the toolkit (Wave 5)
+
+The CI workflow at `.github/workflows/publish.yaml` publishes all six packages with [npm provenance](https://docs.npmjs.com/generating-provenance-statements) when a `fn-v*` tag is pushed. Steps:
+
+1. Update versions in each `packages/fn-*/package.json` (and `packages/fn-app/package.json`). Bump in lock-step for now; we'll move to changesets later.
+2. Verify locally:
+   ```bash
+   pnpm --filter '@constructive-io/fn-*' build
+   pnpm --filter @constructive-io/fn-generator test
+   pnpm --filter @constructive-io/fn-client test
+   for pkg in fn-types fn-app fn-runtime fn-generator fn-client fn-cli; do
+     (cd "packages/$pkg" && pnpm publish --dry-run --no-git-checks --access public)
+   done
+   ```
+3. Tag and push:
+   ```bash
+   git tag fn-v0.1.0
+   git push origin fn-v0.1.0
+   ```
+4. CI publishes in dependency order: `fn-types` → `fn-app` → `fn-runtime` → `fn-generator` → `fn-client` → `fn-cli`.
+
+You can also run the workflow with `workflow_dispatch` (default `dry_run: true`) to verify packing before tagging.
+
+## Verification checklist (manual, before first release)
+
+- [ ] **Snapshot regression**: `pnpm --filter @constructive-io/fn-generator test` passes (asserts byte-identical output vs `scripts/generate.ts`).
+- [ ] **Job-registry tests**: `pnpm exec jest tests/integration/job-registry.test.ts` — six cases pass.
+- [ ] **Brasilia E2E**: with the live k8s stack running (`make skaffold-dev`), `pnpm test:e2e` still picks up jobs end-to-end.
+- [ ] **Scratch repo**: in a fresh `/tmp/test-fn-app` repo, `pnpm add -D @constructive-io/fn-cli && pnpm add @constructive-io/fn-runtime`, add `functions/hello/handler.{json,ts}`, run `fn generate && fn build && fn manifest`. Confirm output is sensible and `docker build -f generated/hello/Dockerfile .` succeeds.
+- [ ] **Hub integration**: in `constructive-hub/istanbul`, `pnpm bootstrap && pnpm start` still launches `send-email-link` and processes a job (the hub does not yet consume the new toolkit; this confirms Wave 1-3 didn't regress the existing submodule path).
+
+## Deferred follow-ups (not in this branch)
+
+- **Wave 4c — replace hand-written `k8s/base/functions/*.yaml` with generator output**. The hand-written manifests carry mailgun secrets, dry-run env vars, and a different image strategy (single bundled image, args-driven entry vs per-function image with Dockerfile CMD). Migrating safely requires either teaching `KnativeServiceBuilder` to emit those fields or providing a Kustomize patch overlay. Tracked separately.
+- **fn.config.ts/.js loading** — JSON only for now. Adding `.ts` requires an `esbuild`/`jiti` loader.
+- **`fn init` and `fn dockerfile` / `fn k8s` standalone subcommands** — the underlying builders exist (`buildPackages`, `buildSkaffold`); these are thin CLI wrappers to add later.
+- **Templates packaging** — currently `templatesDir` is a constructor option pointing at the host repo's `templates/`. A future change can ship templates inside `fn-generator` (or a separate `fn-templates` package) so customer repos don't need their own copy.

job/service/src/index.ts

@@ -25,22 +25,12 @@ import {
   KnativeJobsSvcResult,
   StartedFunction
 } from './types';
+import {
+  FunctionRegistryEntry,
+  loadFunctionRegistry
+} from './registry';
 
-type FunctionRegistryEntry = {
-  moduleName: string;
-  defaultPort: number;
-};
-
-const functionRegistry: Record<FunctionName, FunctionRegistryEntry> = {
-  'simple-email': {
-    moduleName: '@constructive-io/simple-email-fn',
-    defaultPort: 8081
-  },
-  'send-email-link': {
-    moduleName: '@constructive-io/send-email-link-fn',
-    defaultPort: 8082
-  }
-};
+const functionRegistry = loadFunctionRegistry();
 
 const log = new Logger('knative-job-service');
 const requireFn = createRequire(__filename);

job/service/src/registry.ts

@@ -0,0 +1,84 @@
+/**
+ * Function registry loader for the in-process function server.
+ *
+ * Sources, in priority order:
+ *   1. FUNCTIONS_REGISTRY env var
+ *      Format: "name:moduleName:port,..." (port optional)
+ *      Example: "simple-email:@org/simple-email-fn:8081,foo:@org/foo-fn"
+ *   2. FUNCTIONS_MANIFEST_PATH env var pointing to a JSON file with shape
+ *        { functions: [{ name, dir, port, type, moduleName? }] }
+ *   3. Default file: <cwd>/generated/functions-manifest.json
+ *
+ * If no source resolves, the registry is empty; callers throw on lookup of
+ * an unknown function (preserves the legacy "Unknown function X" behaviour).
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+
+export interface FunctionRegistryEntry {
+  moduleName: string;
+  defaultPort: number;
+}
+
+export type FunctionRegistry = Record<string, FunctionRegistryEntry>;
+
+const DEFAULT_MODULE_PREFIX = '@constructive-io/';
+const DEFAULT_MODULE_SUFFIX = '-fn';
+
+const conventionalModuleName = (name: string): string =>
+  `${DEFAULT_MODULE_PREFIX}${name}${DEFAULT_MODULE_SUFFIX}`;
+
+const parseEnvRegistry = (raw: string): FunctionRegistry => {
+  const out: FunctionRegistry = {};
+  for (const pair of raw.split(',')) {
+    const trimmed = pair.trim();
+    if (!trimmed) continue;
+    const [name, moduleName, portStr] = trimmed.split(':').map((s) => s.trim());
+    if (!name) continue;
+    const portNumber = portStr ? Number(portStr) : NaN;
+    out[name] = {
+      moduleName: moduleName || conventionalModuleName(name),
+      defaultPort: Number.isFinite(portNumber) ? portNumber : 0,
+    };
+  }
+  return out;
+};
+
+interface ManifestEntry {
+  name: string;
+  dir?: string;
+  port?: number;
+  type?: string;
+  moduleName?: string;
+}
+
+const fromManifestEntry = (entry: ManifestEntry): FunctionRegistryEntry => ({
+  moduleName: entry.moduleName ?? conventionalModuleName(entry.name),
+  defaultPort: typeof entry.port === 'number' ? entry.port : 0,
+});
+
+const loadManifestFile = (manifestPath: string): FunctionRegistry => {
+  const raw = fs.readFileSync(manifestPath, 'utf-8');
+  const parsed = JSON.parse(raw) as { functions?: ManifestEntry[] };
+  const out: FunctionRegistry = {};
+  for (const entry of parsed.functions ?? []) {
+    if (!entry.name) continue;
+    out[entry.name] = fromManifestEntry(entry);
+  }
+  return out;
+};
+
+export const loadFunctionRegistry = (
+  env: NodeJS.ProcessEnv = process.env,
+  cwd: string = process.cwd()
+): FunctionRegistry => {
+  if (env.FUNCTIONS_REGISTRY) {
+    return parseEnvRegistry(env.FUNCTIONS_REGISTRY);
+  }
+  const manifestPath =
+    env.FUNCTIONS_MANIFEST_PATH ?? path.join(cwd, 'generated', 'functions-manifest.json');
+  if (fs.existsSync(manifestPath)) {
+    return loadManifestFile(manifestPath);
+  }
+  return {};
+};