docs: refactor Lance format specification structure

.github/workflows/docs-deploy.yml

@@ -50,6 +50,11 @@ jobs:
         with:
           repository: lance-format/lance-namespace-impls
           path: lance-namespace-impls
+      - name: Checkout lance-duckdb
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          repository: lance-format/lance-duckdb
+          path: lance-duckdb
       - name: Checkout lance-trino
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
@@ -67,84 +72,17 @@ jobs:
         uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
           enable-cache: true
-      - name: Copy lance-namespace docs
-        run: |
-          cp -r lance-namespace/docs/src docs/src/format/namespace
-          cat >> docs/src/format/.pages << 'EOF'
-            - Namespace Spec: namespace
-          EOF
-          cp docs/src/format/namespace/rest.yaml docs/src/rest.yaml
-      - name: Copy lance-namespace-impls docs
-        run: |
-          # Copy implementation specs to the integrations folder
-          cp lance-namespace-impls/docs/src/*.md docs/src/format/namespace/integrations/
-
-          # Copy .pages from lance-namespace-impls and append template entry
-          cp lance-namespace-impls/docs/src/.pages docs/src/format/namespace/integrations/.pages
-          echo "  - Template for New Integrations: template.md" >> docs/src/format/namespace/integrations/.pages
-      - name: Copy lance-huggingface docs
-        run: |
-          cp -r lance-huggingface/docs/src docs/src/integrations/huggingface
-          cat >> docs/src/integrations/.pages << 'EOF'
-            - Huggingface: huggingface
-          EOF
-      - name: Copy lance-spark docs
-        run: |
-          cp -r lance-spark/docs/src docs/src/integrations/spark
-          cat >> docs/src/integrations/.pages << 'EOF'
-            - Apache Spark: spark
-          EOF
-      - name: Copy lance-ray docs
+      - name: Assemble full docs website
+        env:
+          LANCE_NAMESPACE_REPO: ${{ github.workspace }}/lance-namespace
+          LANCE_NAMESPACE_IMPLS_REPO: ${{ github.workspace }}/lance-namespace-impls
+          LANCE_SPARK_REPO: ${{ github.workspace }}/lance-spark
+          LANCE_RAY_REPO: ${{ github.workspace }}/lance-ray
+          LANCE_TRINO_REPO: ${{ github.workspace }}/lance-trino
+          LANCE_DUCKDB_REPO: ${{ github.workspace }}/lance-duckdb
+          LANCE_HUGGINGFACE_REPO: ${{ github.workspace }}/lance-huggingface
         run: |
-          cp -r lance-ray/docs/src docs/src/integrations/ray
-          cat >> docs/src/integrations/.pages << 'EOF'
-            - Ray: ray
-          EOF
-      - name: Copy lance-trino docs
-        run: |
-          cp -r lance-trino/docs/src docs/src/integrations/trino
-          cat >> docs/src/integrations/.pages << 'EOF'
-            - Trino: trino
-          EOF
-      - name: Copy contributing docs
-        run: |
-          mkdir -p docs/src/community/project-specific/lance
-
-          # Lance project files
-          cp CONTRIBUTING.md docs/src/community/project-specific/lance/general.md
-          cp release_process.md docs/src/community/project-specific/lance/release.md
-          cp rust/CONTRIBUTING.md docs/src/community/project-specific/lance/rust.md
-          cp python/CONTRIBUTING.md docs/src/community/project-specific/lance/python.md
-          cp docs/CONTRIBUTING.md docs/src/community/project-specific/lance/docs.md
-
-          # External project files
-          cp lance-ray/CONTRIBUTING.md docs/src/community/project-specific/ray.md
-          cp lance-spark/CONTRIBUTING.md docs/src/community/project-specific/spark.md
-          cp lance-namespace/CONTRIBUTING.md docs/src/community/project-specific/namespace.md
-          cp lance-namespace-impls/CONTRIBUTING.md docs/src/community/project-specific/namespace-impls.md || true
-          cp lance-trino/CONTRIBUTING.md docs/src/community/project-specific/trino.md
-
-          # Create .pages for project-specific
-          cat > docs/src/community/project-specific/.pages << 'EOF'
-          nav:
-            - index.md
-            - Lance: lance
-            - Lance Namespace: namespace.md
-            - Lance Namespace Impls: namespace-impls.md
-            - Lance Ray: ray.md
-            - Lance Spark: spark.md
-            - Lance Trino: trino.md
-          EOF
-
-          # Create .pages for lance subfolder
-          cat > docs/src/community/project-specific/lance/.pages << 'EOF'
-          nav:
-            - General: general.md
-            - Release: release.md
-            - Rust: rust.md
-            - Python: python.md
-            - Docs: docs.md
-          EOF
+          docs/make-full-website.sh
       - name: Deploy
         working-directory: docs
         run: uv run mkdocs gh-deploy --force

.gitignore

@@ -110,5 +110,18 @@ test_data/venv
 .benchmarks/
 
 # Auto-built docs
+docs/src/format/catalog/**
 docs/src/format/namespace/**
+docs/src/format/layout.png
+docs/src/integrations/duckdb/**
+docs/src/integrations/huggingface/**
+docs/src/integrations/ray/**
 docs/src/integrations/spark/**
+docs/src/integrations/trino/**
+docs/src/community/project-specific/.pages
+docs/src/community/project-specific/lance/**
+docs/src/community/project-specific/namespace.md
+docs/src/community/project-specific/namespace-impls.md
+docs/src/community/project-specific/ray.md
+docs/src/community/project-specific/spark.md
+docs/src/community/project-specific/trino.md

docs/Makefile

@@ -1,22 +1,32 @@
-.PHONY: help serve build clean install dev-install check-links sync
+.PHONY: help serve build clean check-links sync make-full-website clean-full-website
 
 help:
 	@echo "Available commands:"
 	@echo "  make serve        - Serve documentation locally (auto-installs deps)"
 	@echo "  make build        - Build documentation (auto-installs deps)"
+	@echo "  make make-full-website  - Assemble the full website from local repo checkouts"
+	@echo "  make clean-full-website - Remove generated website content"
+	@echo "                       Run `make make-full-website` before `make build` or `make serve` for the full multi-repo site"
+	@echo "                       Override any repo with env vars like LANCE_NAMESPACE_REPO=..., LANCE_SPARK_REPO=..., LANCE_RAY_REPO=..."
 	@echo "  make clean        - Clean build artifacts"
 	@echo "  make check-links  - Check for broken links"
 	@echo "  make sync         - Sync dependencies with pyproject.toml"
 
 sync:
 	uv sync --all-extras
 
-serve:
+serve: make-full-website
 	uv run mkdocs serve
 
-build:
+build: make-full-website
 	uv run mkdocs build
 
+make-full-website:
+	./make-full-website.sh
+
+clean-full-website:
+	./clean-full-website.sh
+
 clean:
 	rm -rf site/
 	rm -rf .cache/

docs/clean-full-website.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+repo_root=$(cd -- "$script_dir/.." && pwd)
+docs_src="$script_dir/src"
+
+rm -rf "$docs_src/format/catalog"
+rm -rf "$docs_src/format/namespace"
+rm -f "$docs_src/format/layout.png"
+rm -f "$docs_src/format/overview.png"
+rm -f "$docs_src/format/java-sdk-example.png"
+rm -rf "$docs_src/integrations/huggingface"
+rm -rf "$docs_src/integrations/duckdb"
+rm -rf "$docs_src/integrations/spark"
+rm -rf "$docs_src/integrations/ray"
+rm -rf "$docs_src/integrations/trino"
+rm -f "$docs_src/community/project-specific/.pages"
+rm -rf "$docs_src/community/project-specific/lance"
+rm -f "$docs_src/community/project-specific/namespace.md"
+rm -f "$docs_src/community/project-specific/namespace-impls.md"
+rm -f "$docs_src/community/project-specific/ray.md"
+rm -f "$docs_src/community/project-specific/spark.md"
+rm -f "$docs_src/community/project-specific/trino.md"
+
+cat > "$docs_src/format/.pages" <<'EOF'
+nav:
+  - Overview: index.md
+  - File Format: file
+  - Table Format: table
+  - Index Formats: index
+  - Catalog Specs: catalog
+  - Namespace Client Spec: namespace
+EOF
+
+cat > "$docs_src/integrations/.pages" <<'EOF'
+nav:
+  - Apache DataFusion: datafusion.md
+  - PostgreSQL: https://github.com/lancedb/pglance
+  - PyTorch: pytorch.md
+  - Tensorflow: tensorflow.md
+EOF
+
+mkdir -p "$docs_src/format/catalog/dir"
+mkdir -p "$docs_src/format/catalog/rest"
+mkdir -p "$docs_src/format/namespace/operations/models"
+mkdir -p "$docs_src/format/namespace/supported-catalogs"
+
+cat > "$docs_src/format/catalog/.pages" <<'EOF'
+title: Catalog Specs
+nav:
+  - Overview: index.md
+  - Directory Catalog: dir
+  - REST Catalog: rest
+EOF
+
+cat > "$docs_src/format/catalog/index.md" <<'EOF'
+# Catalog Specs
+
+This section describes how Lance catalogs organize, discover, and coordinate Lance tables.
+
+When a local `lance-namespace` checkout with split catalog docs is available, `docs/make-full-website.sh` replaces these placeholders with the latest source content.
+
+See also:
+
+- [Directory Catalog](dir/index.md)
+- [REST Catalog](rest/index.md)
+- [Namespace Client Spec](../namespace/index.md)
+EOF
+
+cat > "$docs_src/format/catalog/dir/index.md" <<'EOF'
+# Directory Catalog
+
+The Directory Catalog is the storage-native catalog format for Lance.
+
+This placeholder page keeps the local website buildable when external catalog docs are not available.
+Run `docs/make-full-website.sh` with `LANCE_NAMESPACE_REPO` pointing at a split `lance-namespace` checkout to populate the full specification.
+EOF
+
+cat > "$docs_src/format/catalog/rest/index.md" <<'EOF'
+# REST Catalog
+
+The REST Catalog is the service-oriented catalog specification for Lance.
+
+This placeholder page keeps the local website buildable when external catalog docs are not available.
+Run `docs/make-full-website.sh` with `LANCE_NAMESPACE_REPO` pointing at a split `lance-namespace` checkout to populate the full specification.
+EOF
+
+cat > "$docs_src/format/namespace/.pages" <<'EOF'
+title: Namespace Client Spec
+nav:
+  - Overview: index.md
+  - Objects & Relationships: object-relationship.md
+  - Operations: operations
+  - Supported Catalogs: supported-catalogs
+EOF
+
+cat > "$docs_src/format/namespace/index.md" <<'EOF'
+# Namespace Client Spec
+
+The Lance Namespace Client Spec defines the interface that engines and tools use to discover tables, resolve locations, and coordinate table operations through catalogs.
+
+When a local `lance-namespace` checkout with split namespace docs is available, `docs/make-full-website.sh` replaces these placeholders with the latest source content.
+
+See also:
+
+- [Objects & Relationships](object-relationship.md)
+- [Operations](operations/index.md)
+- [Supported Catalogs](supported-catalogs/index.md)
+EOF
+
+cat > "$docs_src/format/namespace/object-relationship.md" <<'EOF'
+# Objects & Relationships
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+
+Run `docs/make-full-website.sh` with `LANCE_NAMESPACE_REPO` pointing at a split `lance-namespace` checkout to populate the full object model description.
+EOF
+
+cat > "$docs_src/format/namespace/operations/.pages" <<'EOF'
+title: Operations
+nav:
+  - Overview: index.md
+  - Models: models
+EOF
+
+cat > "$docs_src/format/namespace/operations/index.md" <<'EOF'
+# Operations
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+
+Run `docs/make-full-website.sh` with `LANCE_NAMESPACE_REPO` pointing at a split `lance-namespace` checkout to populate the operation reference.
+EOF
+
+cat > "$docs_src/format/namespace/operations/models/.pages" <<'EOF'
+title: Models
+EOF
+
+cat > "$docs_src/format/namespace/operations/models/index.md" <<'EOF'
+# Operation Models
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+EOF
+
+cat > "$docs_src/format/namespace/supported-catalogs/.pages" <<'EOF'
+title: Supported Catalogs
+nav:
+  - Overview: index.md
+  - Lance Directory Catalog: lance-dir.md
+  - Lance REST Catalog: lance-rest.md
+  - Template: template.md
+EOF
+
+cat > "$docs_src/format/namespace/supported-catalogs/index.md" <<'EOF'
+# Supported Catalogs
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+
+Run `docs/make-full-website.sh` with `LANCE_NAMESPACE_REPO` and `LANCE_NAMESPACE_IMPLS_REPO` set to local checkouts to populate the full integration catalog list.
+EOF
+
+cat > "$docs_src/format/namespace/supported-catalogs/lance-dir.md" <<'EOF'
+# Lance Directory Catalog
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+EOF
+
+cat > "$docs_src/format/namespace/supported-catalogs/lance-rest.md" <<'EOF'
+# Lance REST Catalog
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+EOF
+
+cat > "$docs_src/format/namespace/supported-catalogs/template.md" <<'EOF'
+# Template
+
+This placeholder page keeps the local website buildable when external namespace docs are not available.
+EOF