Merge pull request #4 from knowledgefutures/tr/record-level-schemas

Author: isTravis

README.md

@@ -68,6 +68,7 @@ src/
 │   │   ├── collections.ts    # Collection CRUD
 │   │   ├── versions.ts   # Version push/pull/diff + privacy filtering
 │   │   ├── files.ts      # Content-addressed file storage
+│   │   ├── schemas.ts    # Schema discovery, search, labeling
 │   │   └── health.ts     # Health check
 │   └── server.ts         # Fastify entry point
 ├── db/
@@ -185,6 +186,56 @@ npm run secrets:decrypt      # Decrypt .env.enc → .env
 npm run secrets:decrypt:dev  # Decrypt .env.dev.enc → .env.dev
 ```
 
+## Schema System
+
+Underlay uses **globally deduplicated, content-addressed schemas** for record validation and interoperability.
+
+### How it works
+
+- Each record type in a collection has its own JSON Schema, stored as an immutable, content-addressed row in the global `schemas` table.
+- A version declares its full set of type→schema bindings via the `version_schemas` join table.
+- If two collections define the same fields and types for a record type, they produce the same schema hash — alignment is automatic.
+- Schemas are never modified. Evolving a type produces a new hash and a new row.
+
+### Push payload
+
+```json
+{
+  "schemas": {
+    "Author": { "type": "object", "properties": { "name": { "type": "string" } } },
+    "Pub": { "type": "object", "properties": { "title": { "type": "string" }, "authorId": { "type": "string", "x-ref-type": "Author" } } }
+  },
+  "changes": { "added": [...] }
+}
+```
+
+### Relationship annotations
+
+Fields that hold record IDs of another type use `"x-ref-type": "TypeName"` to document the relationship. This enables linked-record navigation in the UI and helps LLMs understand the relational graph.
+
+### Schema labeling
+
+Schemas can be labeled post-hoc with human-readable names or URIs (e.g. `schema.org/Person`, `dc.author.v1`). Labels enable discovery across collections without upfront coordination.
+
+- `POST /api/schemas/:id/labels` — Add a label
+- `DELETE /api/schemas/:id/labels/:label` — Remove a label
+- `GET /api/schemas?label=...` — Search by label
+- Labels are injected as `x-underlay-labels` in schema exports (opt-out via `?raw=true`)
+
+### Schema discovery API
+
+| Endpoint | Purpose |
+|----------|--------|
+| `GET /api/schemas` | Global search (filter by `q`, `slug`, `label`, `schema_hash`) |
+| `GET /api/schemas/:id` | Single schema with labels + usage info |
+| `GET /api/collections/:owner/:slug/schemas` | Collection's schemas (with label enrichment) |
+
+### Versioning semantics
+
+- **Major bump**: Schema set changed (type added, removed, or schema modified)
+- **Minor bump**: Records changed, schema set identical
+- **Patch bump**: Only metadata changed (readme, message)
+
 ## Maintenance Checklist
 
 When adding or changing features, update these locations:
@@ -198,13 +249,15 @@ When adding or changing features, update these locations:
 | Quick start | `src/pages/docs/quickstart.astro` | Getting started tutorial |
 | Self-hosting | `src/pages/docs/self-host.astro` | Deployment instructions |
 | DB schema | `src/db/schema.ts` → `npm run db:generate` | Schema changes need a migration |
+| Schema discovery | `src/api/routes/schemas.ts` | Schema search, labeling, cross-referencing |
 | Encrypted secrets | `.env.enc` / `.env.dev.enc` | Re-encrypt after changing .env files |
 
 ### Privacy features
 
-The system supports three levels of privacy (type-level, field-level, record-level) via `"private": true` annotations. When changing how privacy works, update:
-- `src/api/routes/versions.ts` — filtering logic
+The system supports three levels of privacy (type-level, field-level, record-level) via `"private": true` annotations in per-type schemas. When changing how privacy works, update:
+- `src/api/routes/versions.ts` — filtering logic (reads from `version_schemas` JOIN `schemas`)
 - `src/api/routes/files.ts` — file access checks
+- `src/api/routes/schemas.ts` — public schema filtering
 - `public/.well-known/ai.txt` — Privacy section
 - `src/pages/docs/concepts.astro` — Privacy section
 - `src/pages/docs/api/versions.astro` — Push endpoint docs

public/.well-known/ai.txt

@@ -35,7 +35,8 @@ Delete endpoints require "admin" scope.
 - Version: an immutable snapshot containing a JSON Schema, records, and file references. Sequential integer numbers, auto-derived semver.
 - Record: a flat JSON object with { id, type, data }. Records reference other records by id and files by hash.
 - File: a binary blob stored by SHA-256 hash, referenced in record data as {"$file": "sha256:<hex>"}.
-- Schema: a JSON Schema document describing the record types in a version. Schema changes trigger a major version bump.
+- Schema: a JSON Schema document for a single record type, stored as a global, immutable, content-addressed entity. Each type gets its own schema. Schema changes trigger a major version bump.
+- Schema labeling: schemas can be labeled post-hoc with URIs or names (e.g. "schema.org/Person") for cross-collection discovery.
 
 ---
 
@@ -109,33 +110,39 @@ Authorization: Bearer ul_<key>
   "message": "Daily archive 2026-04-27",
   "app_id": "my-app",
   "actor_id": "my-app:cron-job",
-  "schema": {
-    "type": "object",
-    "properties": {
-      "Article": {
-        "type": "object",
-        "properties": {
-          "title": {"type": "string"},
-          "body": {"type": "string"},
-          "publishedAt": {"type": "string", "format": "date-time"}
-        }
+  "schemas": {
+    "Article": {
+      "type": "object",
+      "properties": {
+        "title": {"type": "string"},
+        "body": {"type": "string"},
+        "publishedAt": {"type": "string", "format": "date-time"},
+        "authorId": {"type": "string", "x-ref-type": "Author"}
+      }
+    },
+    "Author": {
+      "type": "object",
+      "properties": {
+        "name": {"type": "string"},
+        "email": {"type": "string", "private": true}
       }
     }
   },
   "changes": {
     "added": [
-      {"id": "article-42", "type": "Article", "data": {"title": "New Paper", "body": "...", "publishedAt": "2026-04-27T00:00:00Z"}}
+      {"id": "article-42", "type": "Article", "data": {"title": "New Paper", "body": "...", "publishedAt": "2026-04-27T00:00:00Z", "authorId": "author-1"}}
     ],
     "updated": [
-      {"id": "article-10", "type": "Article", "data": {"title": "Updated Title", "body": "...", "publishedAt": "2025-01-01T00:00:00Z"}}
+      {"id": "article-10", "type": "Article", "data": {"title": "Updated Title", "body": "...", "publishedAt": "2025-01-01T00:00:00Z", "authorId": "author-2"}}
     ],
     "removed": ["article-5"]
   }
 }
 
 Field reference:
 - base_version: the version number you diffed against. null for first push. Used for optimistic locking.
-- schema: required on first push and whenever the schema changes. Omit if unchanged.
+- schemas: per-type JSON Schema map. Required on first push. If omitted on subsequent pushes and records validate against carried-forward schemas, they are carried forward automatically.
+- schemas[TypeName]: JSON Schema for that record type. Use "x-ref-type" to annotate foreign key fields.
 - message: human-readable commit message (optional).
 - app_id: identifier for the pushing application (optional).
 - actor_id: identifier for the user or process that triggered the push (optional).
@@ -157,7 +164,7 @@ Missing files (422 — records reference files not yet uploaded):
 → Upload the listed files, then retry the push.
 
 ### First push (no existing versions)
-Set base_version to null. Put all records in changes.added. Include the schema.
+Set base_version to null. Put all records in changes.added. Include schemas for all types.
 
 ---
 
@@ -182,11 +189,26 @@ The schema declares which fields are references, so tools can resolve them at re
 
 ---
 
+## Schema Discovery
+
+Schemas are globally deduplicated by content hash. If two collections use the same type shape, they share the same schema row. This enables cross-collection discovery.
+
+GET /api/schemas                                        → search schemas (?q=text&slug=TypeName&label=uri&schema_hash=sha256:...&limit=50&offset=0)
+GET /api/schemas/:id                                    → single schema with labels and usage info
+GET /api/collections/:owner/:slug/schemas               → schemas for latest version (?version=N for specific, ?raw=true to skip label enrichment)
+POST /api/schemas/:id/labels                            → add label {"label": "schema.org/Person"} (requires write scope)
+DELETE /api/schemas/:id/labels/:label                   → remove label (requires admin scope)
+
+When schemas are returned via the collection schemas endpoint, known labels are injected as "x-underlay-labels" on the schema body (opt-out with ?raw=true).
+
+---
+
 ## Versioning
 
 - Versions are sequential integers (1, 2, 3, ...).
 - Semver is derived automatically: schema change → major, record change → minor, metadata-only → patch.
-- Each version has a content-addressed hash computed from schema + sorted records + sorted file hashes.
+- Schema change = any type's schema_id changed, or types added/removed between versions.
+- Each version has a content-addressed hash computed from sorted type→schema_hash map + sorted records + sorted file hashes.
 - Versions are immutable once created.
 
 ---
@@ -206,27 +228,24 @@ Underlay supports fine-grained privacy at three levels: types, fields, and indiv
 Private data is stored alongside public data in the same version but is only visible to the collection owner.
 
 ### Private Types
-Mark an entire type as private in the schema. All records of that type are hidden from public readers.
+Mark an entire type as private in its schema. All records of that type are hidden from public readers.
 
-"schema": {
-  "type": "object",
-  "properties": {
-    "Article": {
-      "type": "object",
-      "properties": { "title": {"type": "string"} }
-    },
-    "InternalNote": {
-      "type": "object",
-      "private": true,
-      "properties": { "note": {"type": "string"}, "articleId": {"type": "string"} }
-    }
+"schemas": {
+  "Article": {
+    "type": "object",
+    "properties": { "title": {"type": "string"} }
+  },
+  "InternalNote": {
+    "type": "object",
+    "private": true,
+    "properties": { "note": {"type": "string"}, "articleId": {"type": "string"} }
   }
 }
 
 Public readers see only the Article type. InternalNote is completely hidden (including from the schema response).
 
 ### Private Fields
-Mark individual fields as private within a type. The type itself is visible, but those fields are stripped for public readers.
+Mark individual fields as private within a type's schema. The type itself is visible, but those fields are stripped for public readers.
 
 "Author": {
   "type": "object",

src/api/routes/collections.ts

@@ -188,7 +188,6 @@ export async function collectionsRoutes(app: FastifyInstance) {
         id: schema.versions.id,
         number: schema.versions.number,
         semver: schema.versions.semver,
-        schema: schema.versions.schema,
         recordCount: schema.versions.recordCount,
         fileCount: schema.versions.fileCount,
         totalBytes: schema.versions.totalBytes,
@@ -417,6 +416,20 @@ export async function collectionsRoutes(app: FastifyInstance) {
     // Pipe tar → gzip → response
     const outputStream = pack.pipe(gzip);
 
+    // Load schemas for this version
+    const versionSchemaEntries = await db
+      .select({
+        slug: schema.versionSchemas.slug,
+        schemaBody: schema.schemas.schema,
+      })
+      .from(schema.versionSchemas)
+      .innerJoin(schema.schemas, eq(schema.versionSchemas.schemaId, schema.schemas.id))
+      .where(eq(schema.versionSchemas.versionId, version.id));
+
+    const schemasMap = Object.fromEntries(
+      versionSchemaEntries.map((e) => [e.slug, e.schemaBody]),
+    );
+
     // Add manifest.json
     const manifest = {
       collection: { owner, slug, name: collection.name, description: collection.description },
@@ -430,7 +443,7 @@ export async function collectionsRoutes(app: FastifyInstance) {
         totalBytes: version.totalBytes,
         createdAt: version.createdAt,
       },
-      schema: version.schema,
+      schemas: schemasMap,
     };
     const manifestBuf = Buffer.from(JSON.stringify(manifest, null, 2));
     pack.entry({ name: "manifest.json", size: manifestBuf.length }, manifestBuf);

src/api/routes/files.ts

@@ -35,7 +35,7 @@ async function isFilePubliclyAccessible(
 
   // Get the latest version
   const [latest] = await db
-    .select({ id: schema.versions.id, schema: schema.versions.schema })
+    .select({ id: schema.versions.id })
     .from(schema.versions)
     .where(eq(schema.versions.collectionId, collection.id))
     .orderBy(sql`${schema.versions.number} desc`)
@@ -54,14 +54,22 @@ async function isFilePubliclyAccessible(
 
   if (!vf) return false;
 
-  // Get schema to determine private types and fields
-  const schemaDoc = (latest.schema ?? {}) as Record<string, any>;
+  // Load version schemas to determine private types and fields
+  const schemaEntries = await db
+    .select({
+      slug: schema.versionSchemas.slug,
+      schemaBody: schema.schemas.schema,
+    })
+    .from(schema.versionSchemas)
+    .innerJoin(schema.schemas, eq(schema.versionSchemas.schemaId, schema.schemas.id))
+    .where(eq(schema.versionSchemas.versionId, latest.id));
+
   const privateTypes = new Set<string>();
-  const props = schemaDoc?.properties as Record<string, any> | undefined;
-  if (props) {
-    for (const [typeName, typeDef] of Object.entries(props)) {
-      if (typeDef?.private === true) privateTypes.add(typeName);
-    }
+  const typeSchemaMap = new Map<string, Record<string, any>>();
+  for (const entry of schemaEntries) {
+    const body = entry.schemaBody as Record<string, any>;
+    typeSchemaMap.set(entry.slug, body);
+    if (body?.private === true) privateTypes.add(entry.slug);
   }
 
   // Find public records that reference this file hash
@@ -83,7 +91,8 @@ async function isFilePubliclyAccessible(
     if (privateTypes.has(rec.type)) continue;
 
     // Get private fields for this type
-    const typeProps = props?.[rec.type]?.properties as Record<string, any> | undefined;
+    const typeSchema = typeSchemaMap.get(rec.type);
+    const typeProps = typeSchema?.properties as Record<string, any> | undefined;
     if (!typeProps) return true; // no schema constraints, allow
 
     const privateFields = new Set<string>();