diff --git a/.gitignore b/.gitignore
index e170cbd..4e7ad72 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
.databricks/
-build/
+/build/
+docs/build/
dist/
__pycache__/
*.egg-info
diff --git a/VERSION b/VERSION
index 8c20c52..759e855 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-v0.20.0
\ No newline at end of file
+v0.21.0
diff --git a/docs/Makefile b/docs/Makefile
index 21073bb..4fc15a1 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -35,7 +35,7 @@ spelling:
html-multiversion:
@./scripts/build_versioned_docs.sh
-# Local preview build including all local branches.
+# Local preview build: current branch as local-branch-preview (current stays main).
html-multiversion-preview:
@./scripts/build_versioned_docs.sh --preview
diff --git a/docs/README.md b/docs/README.md
index b8b6da4..a6ba15d 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -9,7 +9,7 @@ This folder contains the Sphinx documentation source and build tooling for local
- [Multiversion Implementation (No sphinx-multiversion)](#multiversion-implementation-no-sphinx-multiversion)
- [What Gets Built](#what-gets-built)
- [Version Selection Rules](#version-selection-rules)
- - [How the Sidebar Switcher Works](#how-the-sidebar-switcher-works)
+ - [How the Version Dropdown Works](#how-the-version-dropdown-works)
- [Purge and Rebuild Behavior](#purge-and-rebuild-behavior)
- [Local Build and Testing](#local-build-and-testing)
- [GitHub Pages Deployment](#github-pages-deployment)
@@ -40,7 +40,7 @@ The `docs/Makefile` supports these explicit targets:
- `make html-multiversion`
- Builds versioned docs without `sphinx-multiversion` using `scripts/build_versioned_docs.sh`.
- `make html-multiversion-preview`
- - Same as above, plus local branches for preview/testing.
+ - Same as above, plus the current git branch as `local-branch-preview`.
Catch-all behavior is enabled:
@@ -59,6 +59,10 @@ Versioned docs are generated by:
- `main` branch is always built and published as `current`.
- Selected release tags are built from git tags.
+- **Docs IA cutover at v0.21.0:**
+ - **`current` and tags `>= v0.21.0`** use **main's** `docs/conf.py` with that ref's `docs/source` (sphinx-immaterial / section-folder IA).
+ - **Tags `<= v0.20.x`** use **`v0.20.0`'s** `docs/conf.py` (RTD / flat layout) with each tag's own `docs/source`.
+- **`local-branch-preview`** (optional) uses the current branch's own conf + source (pre-merge validation).
- Output layout:
- `docs/build/html/current/index.html`
- `docs/build/html/vX.Y.Z/index.html`
@@ -70,9 +74,10 @@ Versioned docs are generated by:
`scripts/select_versions.py` applies these rules:
1. Identify the highest major version present in tags.
-2. Include the latest patch release for the last 5 minor series in that major.
-3. Include the latest available release for each of the last 3 majors.
-4. De-duplicate and sort descending (newest first).
+2. For major `0`, include the latest patch release for every minor from the current minor down to `0.12` (inclusive).
+3. For other majors, include the latest patch release for the last 5 minor series in that major.
+4. Include the latest available release for each of the last 3 majors.
+5. De-duplicate and sort descending (newest first).
Example ordering:
@@ -82,13 +87,27 @@ Example ordering:
- `v0.14.0`
- `v0.13.0`
- `v0.12.1`
+- … through `v0.12.x` when older minors exist
-### How the Sidebar Switcher Works
+### How the Version Dropdown Works
-- `scripts/build_versioned_docs.py` writes `versions.json` with `name`, `url`, and `is_latest`.
-- `docs/conf.py` loads this manifest via `DOCS_VERSIONS_FILE`.
-- `docs/_templates/versions.html` renders the version list in the RTD sidebar.
-- Links target `.../index.html` explicitly to avoid directory-listing behavior in local `file://` browsing.
+- `scripts/build_versioned_docs.py` writes a **superset** `versions.json` with:
+ - mike / sphinx-immaterial fields: `version`, `title`, `aliases`
+ - legacy RTD fields: `name`, `display_version`, `url`, `status`, `is_latest`, `release_date`
+- RTD-era refs load the manifest at build time via `DOCS_VERSIONS_FILE` into `versions.html`.
+- Immaterial refs enable `version_dropdown` and fetch the parent `../versions.json` over HTTP (`version_json` in `conf.py`).
+- Use a local HTTP server to test the immaterial switcher (see below).
+
+### Preview builds
+
+```bash
+make -C docs html-multiversion-preview
+```
+
+- Still publishes `current` from `main`.
+- Also builds the **current git branch** once as `local-branch-preview` (skipped if you are on `main` or detached HEAD).
+- To review a feature branch before merge, check it out, run preview, then select `local-branch-preview` in the version menu.
+- Preview builds from the **working tree** (uncommitted conf/theme/source edits are included).
## Purge and Rebuild Behavior
@@ -122,7 +141,7 @@ make -C docs spelling
make -C docs html-multiversion
```
-1. If you need to test branch previews in the version menu:
+1. If you need to preview the current branch against `main` / release tags:
```bash
make -C docs html-multiversion-preview
@@ -132,7 +151,7 @@ Practical rule of thumb:
- Doc text/layout change only: `html` + `spelling`
- Versioning/switcher/build-script change: `html` + `spelling` + `html-multiversion`
-- Branch preview verification: add `html-multiversion-preview`
+- Branch theme/IA preview: `html-multiversion-preview`, then select `local-branch-preview`
### Open docs
@@ -145,7 +164,9 @@ cd docs/build/html
python3 -m http.server 8000
```
-Then open [http://localhost:8000](http://localhost:8000).
+Then open [http://localhost:8000/current/index.html](http://localhost:8000/current/index.html) to exercise the version dropdown.
+
+Note: `make html` builds a single-version tree without a parent-level `versions.json`; use `html-multiversion` to test version switching.
## GitHub Pages Deployment
diff --git a/docs/conf.py b/docs/conf.py
index afb2892..7015d14 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -5,18 +5,27 @@
# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-import json
import os
-import subprocess
import sys
-from datetime import datetime
sys.path.append(os.path.abspath(".")) # Ensure the script is discoverable
from custom_markdown_builder import CustomMarkdownTranslator
+# Ensure sphinx-immaterial registers domain synopses before env init.
+from sphinx.domains.python import PythonDomain
+import sphinx.domains.std
+
+PythonDomain.initial_data.setdefault("synopses", {})
+sphinx.domains.std.StandardDomain.initial_data.setdefault("synopses", {})
+import sphinx_immaterial.apidoc.python.synopses # noqa: F401
+import sphinx_immaterial.apidoc.generic_synopses # noqa: F401
+
project = 'Lakeflow Framework'
copyright = '2026, Databricks'
author = 'Erik Seefeld, Haille Woldegebriel'
+# Site title in the header / — project name only (no version / "documentation").
+html_title = project
+
# Read version from the VERSION file at the repo root so conf.py never
# needs a manual update when a release is cut.
_here = os.path.dirname(os.path.abspath(__file__))
@@ -36,144 +45,463 @@
'sphinx_design',
'myst_parser',
'sphinx_tabs.tabs',
+ 'sphinx_copybutton',
'custom_markdown_builder',
"sphinxcontrib.spelling",
+ # Theme directives (md-mermaid, task-list) register via html_theme setup,
+ # which does not run for the spelling builder — load them explicitly.
+ "sphinx_immaterial.mermaid_diagrams",
+ "sphinx_immaterial.task_lists",
]
autosectionlabel_prefix_document = True
+# MyST — task-list checkboxes (Quick Start prerequisites) and fenced directives
+myst_enable_extensions = [
+ "colon_fence",
+ "tasklist",
+]
+myst_enable_checkboxes = True
+
+# sphinx-immaterial ``.. task-list::`` (RST) — interactive like MyST checkboxes
+clickable_checkbox = True
+
templates_path = ['_templates', 'source/_templates']
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+# Code blocks: when :linenos: is set, use Cursor/VS Code-style gutter (table layout).
+html_codeblock_linenos_style = 'table'
+
+# Copy button — WAF-style command snippets only (not dark spec/code panels)
+copybutton_selector = "div.lf-command-block div.highlight pre"
+copybutton_prompt_is_regexp = True
+copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,6}\.\.\. | {5,8}: "
+copybutton_only_copy_prompt_lines = False
+copybutton_remove_prompts = True
+
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-import sphinx_rtd_theme
intersphinx_mapping = {
'rtd': ('https://docs.readthedocs.io/en/stable/', None),
'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
}
-html_theme = "sphinx_rtd_theme"
+html_theme = "sphinx_immaterial"
html_baseurl = "https://databricks-solutions.github.io/lakeflow_framework/"
+html_logo = "source/_static/lff-logo.png"
+html_favicon = "source/_static/lff-logo.png"
-# Inject the version switcher into the sidebar.
-html_sidebars = {
- '**': [
- 'versions.html',
- 'globaltoc.html',
- 'relations.html',
- 'sourcelink.html',
- 'searchbox.html',
+html_theme_options = {
+ "site_url": html_baseurl,
+ "repo_url": "https://github.com/databricks-solutions/lakeflow_framework",
+ "repo_name": "lakeflow_framework",
+ "icon": {
+ "repo": "fontawesome/brands/github",
+ },
+ "features": [
+ "navigation.instant",
+ "navigation.top",
+ "navigation.tabs",
+ "navigation.tabs.sticky",
+ "toc.integrate",
+ "toc.follow",
+ "search.suggest",
+ "content.code.copy",
+ ],
+ # Version dropdown needs parent-level versions.json (written by html-multiversion).
+ # Flat ``make html`` builds omit it; fetching a missing manifest breaks search init.
+ "version_dropdown": os.environ.get("DOCS_CURRENT_VERSION") is not None,
+ # Parent of each version folder (mike default). Plain "versions.json"
+ # resolves inside the version dir and 404s in multiversion layouts.
+ "version_json": "../versions.json",
+ "palette": [
+ {
+ "media": "(prefers-color-scheme: light)",
+ "scheme": "default",
+ "primary": "red",
+ "accent": "blue-grey",
+ "toggle": {
+ "icon": "material/brightness-7",
+ "name": "Switch to dark mode",
+ },
+ },
+ {
+ "media": "(prefers-color-scheme: dark)",
+ "scheme": "slate",
+ "primary": "red",
+ "accent": "blue-grey",
+ "toggle": {
+ "icon": "material/brightness-4",
+ "name": "Switch to light mode",
+ },
+ },
],
+ "font": {
+ "text": "Roboto",
+ "code": "Roboto Mono",
+ },
}
html_static_path = ['source/_static']
html_css_files = [
+ 'databricks-theme.css',
'custom.css',
]
+html_js_files = [
+ 'mermaid-zoom.js',
+]
-# Suppress generic Sphinx "Last updated" text; custom version metadata is shown.
+# Suppress generic Sphinx "Last updated" text.
html_last_updated_fmt = None
+html_context = {
+ "docs_current_version": os.environ.get("DOCS_CURRENT_VERSION", "current"),
+}
+
-def _head_release_date() -> str:
- try:
- return subprocess.run(
- ["git", "log", "-1", "--format=%cs"],
- cwd=_here,
- capture_output=True,
- text=True,
- check=True,
- ).stdout.strip()
- except Exception:
- return ""
-
-
-def _format_release_date(date_str: str) -> str:
- if not date_str:
- return ""
- try:
- dt = datetime.strptime(date_str, "%Y-%m-%d")
- return dt.strftime("%B %d, %Y")
- except ValueError:
- return date_str
-
-
-def _fallback_versions() -> list[dict[str, str]]:
- head_release_date = _head_release_date()
- return [
- {
- "name": "current",
- "display_version": release,
- "url": "../current/index.html",
- "is_latest": True,
- "status": "current",
- "release_date": head_release_date,
- "release_date_human": _format_release_date(head_release_date),
- }
+def _patch_landing_nav(app, pagename, templatename, context, doctree):
+ """Use index.html as the Home landing page; WAF-style section nav."""
+ nav = context.get("nav")
+ if nav is None:
+ return
+
+ from sphinx_immaterial.nav_adapt import MkdocsNavEntry
+
+ pathto = context["pathto"]
+ master = app.config.master_doc
+ home_url = pathto(master)
+ is_home = pagename == master
+
+ # Section index pages: non-selectable sidebar title, tab lands on first child.
+ # Architecture / Deploy / Samples / Features / Build / Contributors / Get Started
+ # are real top-level hub links.
+ section_index_pages = frozenset({
+ "build/spec-reference/index",
+ "deploy/framework/index",
+ "features/metadata/index",
+ "features/authoring/index",
+ "features/configuration/index",
+ "features/sources-targets/index",
+ "features/platform/index",
+ "features/python/index",
+ "features/data-quality/index",
+ "features/environments/index",
+ "features/migrations/index",
+ })
+
+ def _docname_from_url(url):
+ if not url or url == "#":
+ return None
+ path = url.split("#", 1)[0]
+ if path.endswith(".html"):
+ return path[:-5]
+ return path
+
+ def apply_section_captions(entries):
+ for entry in entries:
+ if entry.children:
+ apply_section_captions(entry.children)
+ docname = _docname_from_url(entry.url)
+ if docname in section_index_pages and entry.children:
+ first_child = entry.children[0]
+ if first_child.url:
+ entry.caption_only = True
+ entry.url = first_child.url
+
+ apply_section_captions(nav)
+
+ def rewrite_home_urls(entries):
+ for entry in entries:
+ if entry.url:
+ path = entry.url.split("#", 1)[0]
+ if path == "home.html" or path.endswith("/home.html"):
+ fragment = entry.url[len(path):]
+ entry.url = home_url + fragment
+ rewrite_home_urls(entry.children)
+
+ rewrite_home_urls(nav)
+
+ nav[:] = [
+ entry
+ for entry in nav
+ if not (entry.url and entry.url.split("#", 1)[0] == home_url and "Home" in entry.title)
]
+ nav.insert(
+ 0,
+ MkdocsNavEntry(
+ title_text="Home",
+ url=home_url,
+ children=[],
+ active=is_home,
+ current=is_home,
+ active_or_section_within_active=is_home,
+ caption_only=False,
+ ),
+ )
-def _load_versions() -> list[dict[str, str]]:
- versions_file = os.environ.get("DOCS_VERSIONS_FILE")
- if not versions_file:
- default_versions_file = os.path.join(_here, "build", "html", "versions.json")
- if os.path.exists(default_versions_file):
- versions_file = default_versions_file
-
- if not versions_file:
- return _fallback_versions()
-
- try:
- with open(versions_file, encoding="utf-8") as f:
- versions = json.load(f)
- except FileNotFoundError:
- return _fallback_versions()
-
- # For per-version pages, switch from site-root links (e.g. "v1.2.3/") to
- # sibling paths (e.g. "../v1.2.3/"), which work under GitHub project pages.
- adapted = []
- for item in versions:
- adapted.append(
- {
- "name": item["name"],
- "display_version": item.get("display_version", item["name"].lstrip("v")),
- "url": f"../{item['name']}/index.html",
- "is_latest": item.get("is_latest", False),
- "status": item.get("status", "release"),
- "release_date": item.get("release_date", ""),
- "release_date_human": _format_release_date(item.get("release_date", "")),
- }
- )
- return adapted
-
-
-def _current_version_meta(versions: list[dict[str, str]], current: str) -> dict[str, str]:
- for item in versions:
- if item["name"] == current:
- return item
- return {
- "name": current,
- "display_version": current.lstrip("v"),
- "url": f"../{current}/index.html",
- "is_latest": current == "current",
- "status": "current" if current == "current" else "release",
- "release_date": "",
- "release_date_human": "",
+ if is_home:
+ for entry in nav[1:]:
+ entry.active = False
+ entry.current = False
+ entry.active_or_section_within_active = False
+
+ page = context.get("page")
+ if page is not None:
+ if "hide" not in page["meta"]:
+ page["meta"]["hide"] = []
+ page["meta"]["hide"].append("navigation")
+ page["meta"]["hide"].append("toc")
+
+ context["nav"] = nav
+
+
+def _flatten_mermaid_diagrams(app, doctree):
+ """Emit plain diagram source for md-mermaid (no Pygments wrapper inside <pre>)."""
+ from sphinx_immaterial.mermaid_diagrams import mermaid_node
+
+ for diagram in doctree.findall(mermaid_node):
+ content = diagram.get('content', '')
+ if not content:
+ for child in diagram.children:
+ if child.tagname == 'literal_block':
+ content = child.astext()
+ break
+ if content:
+ diagram['content'] = content
+ # Drop child nodes — content lives on the node attribute. Replacing
+ # children with nodes.Text() without append() orphans text nodes that
+ # the spelling builder treats as prose (<unknown>:None: br, subgraph, …).
+ diagram.children[:] = []
+
+
+def _ignore_spelling_in_nonprose(app, doctree):
+ """Skip diagram markup and raw HTML during sphinxcontrib-spelling checks."""
+ from docutils import nodes
+
+ from sphinx_immaterial.mermaid_diagrams import mermaid_node
+
+ skip_tags = {
+ 'literal_block',
+ 'raw',
+ 'fixed_text_block',
+ 'math_block',
+ mermaid_node.__name__,
}
+ for node in doctree.findall(nodes.Element):
+ classes = set(node.get('classes', []))
+ if node.tagname in skip_tags or classes.intersection(
+ {'mermaid-diagram', 'lf-mermaid-source', 'lf-arch'}
+ ):
+ for text in node.findall(nodes.Text):
+ text.spellingIgnore = True
-_docs_current_version = os.environ.get("DOCS_CURRENT_VERSION", "current")
-_docs_versions = _load_versions()
-html_context = {
- "docs_current_version": _docs_current_version,
- "docs_versions": _docs_versions,
- "docs_current_version_meta": _current_version_meta(_docs_versions, _docs_current_version),
-}
+def _embed_mermaid_source_templates(app, doctree):
+ """Persist diagram source beside the rendered host (survives pre→div swap)."""
+ import html
+
+ from docutils import nodes
+
+ from sphinx_immaterial.mermaid_diagrams import mermaid_node
+
+ for diagram in doctree.findall(mermaid_node):
+ content = diagram.get('content') or diagram.astext()
+ if not content:
+ continue
+
+ parent = diagram.parent
+ if parent is None:
+ continue
+
+ if any(
+ isinstance(child, nodes.raw)
+ and 'lf-mermaid-source' in child.astext()
+ for child in parent.children
+ ):
+ continue
+
+ template = nodes.raw(
+ '',
+ f'<template class="lf-mermaid-source">{html.escape(content)}</template>',
+ format='html',
+ )
+ parent.insert(parent.children.index(diagram), template)
+
+
+def _table_column_count(table) -> int:
+ from docutils import nodes
+
+ for tgroup in table.findall(nodes.tgroup):
+ cols = tgroup.get('cols')
+ if cols:
+ return int(cols)
+ return 0
+
+
+def _is_page_metadata_table(table) -> bool:
+ """Feature-page metadata strip: 2 cols, Applies To / Configuration Scope rows."""
+ from docutils import nodes
+
+ if _table_column_count(table) != 2:
+ return False
+
+ rows = list(table.findall(nodes.row))
+ if not rows:
+ return False
+
+ entries = list(rows[0].findall(nodes.entry))
+ if not entries:
+ return False
+
+ first_cell = entries[0].astext()
+ return (
+ 'Applies To' in first_cell
+ or 'Configuration Scope' in first_cell
+ or 'Databricks Docs' in first_cell
+ )
+
+
+def _mark_content_tables(app, doctree, docname=None):
+ """Tag body list-tables for theme CSS (excludes h1 metadata strips)."""
+ from docutils import nodes
+
+ for table in doctree.findall(nodes.table):
+ if _is_page_metadata_table(table):
+ continue
+
+ classes = list(table.get('classes', []))
+ if 'lf-content-table' not in classes:
+ classes.append('lf-content-table')
+ if 'data' not in classes:
+ classes.append('data')
+
+ col_count = _table_column_count(table)
+ col_class = f'lf-table-cols-{col_count}'
+ if 2 <= col_count <= 6 and col_class not in classes:
+ classes.append(col_class)
+
+ table['classes'] = classes
+
+
+def _upgrade_mermaid_dist(app, env=None):
+ """Replace theme mermaid (11.12) with vendored 11.16 for treeView-beta support."""
+ from pathlib import Path
+ import shutil
+
+ import sphinx_immaterial
+ from sphinx.builders.html import StandaloneHTMLBuilder
+
+ if not isinstance(app.builder, StandaloneHTMLBuilder):
+ return
+
+ if env is None:
+ env = app.env
+ dst_dir = Path(app.outdir) / "_static" / "mermaid"
+ dst = dst_dir / "mermaid.min.js"
+ theme_mermaid = Path(sphinx_immaterial.__file__).parent / "bundles" / "mermaid"
+
+ needs_mermaid = getattr(env, "sphinx_immaterial_copy_mermaid_dist", False)
+ if not needs_mermaid and not dst.is_file():
+ return
+
+ # Theme copy runs on env-check-consistency (skipped on no-op incremental
+ # builds). Ensure the dist exists before overriding with the vendored build.
+ if not dst.is_file() and theme_mermaid.is_dir():
+ dst_dir.mkdir(parents=True, exist_ok=True)
+ shutil.copytree(theme_mermaid, dst_dir, dirs_exist_ok=True)
+
+ vendor = Path(_here) / "source" / "_static" / "vendor" / "mermaid.min.js"
+ if vendor.is_file():
+ dst_dir.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(vendor, dst)
+
+
+def _upgrade_mermaid_dist_on_finish(app, exception):
+ if exception is not None:
+ return
+ _upgrade_mermaid_dist(app)
+
+
+def _fix_language_data_for_immaterial_search(app, exception):
+ """Sphinx 8+ emits stopwords as a Set; immaterial search expects an array."""
+ if exception is not None:
+ return
+
+ from pathlib import Path
+ import re
+
+ path = Path(app.outdir) / "_static" / "language_data.js"
+ if not path.is_file():
+ return
+
+ text = path.read_text(encoding="utf-8")
+ if "new Set(" not in text:
+ return
+
+ updated, count = re.subn(
+ r"const stopwords = new Set\((\[[\s\S]*?\])\);\s*\nwindow\.stopwords = stopwords;",
+ r"var stopwords = \1;\nwindow.stopwords = stopwords;",
+ text,
+ count=1,
+ )
+ if count:
+ path.write_text(updated, encoding="utf-8")
+
+
+def _override_mermaid_pre_class(app):
+ """Use lf-mermaid-src so sphinx-immaterial does not shadow-DOM render diagrams."""
+ from sphinx_immaterial.mermaid_diagrams import (
+ depart_mermaid_node_html,
+ mermaid_node,
+ )
+
+ def visit_mermaid_node_html_lf(self, node):
+ attributes = {'class': 'lf-mermaid-src'}
+ self.body.append(self.starttag(node, 'pre', **attributes))
+
+ app.add_node(
+ mermaid_node,
+ override=True,
+ html=(visit_mermaid_node_html_lf, depart_mermaid_node_html),
+ )
+
+
+def _patch_mermaid_pre_tags(app, exception):
+ """Ensure theme does not shadow-render our diagrams (class lf-mermaid-src)."""
+ if exception is not None:
+ return
+
+ from pathlib import Path
+
+ outdir = Path(app.outdir)
+ for html_path in outdir.rglob('*.html'):
+ text = html_path.read_text(encoding='utf-8')
+ updated = text.replace('<pre class="mermaid">', '<pre class="lf-mermaid-src">')
+ if updated != text:
+ html_path.write_text(updated, encoding='utf-8')
+
def setup(app):
app.set_translator("markdown", CustomMarkdownTranslator)
- app.add_css_file('custom.css')
+
+ def _init_domain_synopses(app):
+ for domain_name in ("py", "std"):
+ domain = app.env.get_domain(domain_name)
+ if "synopses" not in domain.data:
+ domain.data["synopses"] = {}
+
+ app.connect("builder-inited", _init_domain_synopses)
+ _override_mermaid_pre_class(app)
+ app.connect("doctree-read", _flatten_mermaid_diagrams)
+ app.connect("doctree-read", _ignore_spelling_in_nonprose)
+ app.connect("doctree-read", _embed_mermaid_source_templates)
+ app.connect("doctree-read", _mark_content_tables)
+ app.connect("env-check-consistency", _upgrade_mermaid_dist, priority=1000)
+ app.connect("build-finished", _upgrade_mermaid_dist_on_finish, priority=1000)
+ app.connect("build-finished", _fix_language_data_for_immaterial_search)
+ app.connect("build-finished", _patch_mermaid_pre_tags)
+ app.connect("html-page-context", _patch_landing_nav, priority=999)
#app.add_builder(MarkdownBuilder)
diff --git a/docs/decisions/0001-bundle-src-layout.md b/docs/decisions/0001-bundle-src-layout.md
index 9aa1af0..ab17607 100644
--- a/docs/decisions/0001-bundle-src-layout.md
+++ b/docs/decisions/0001-bundle-src-layout.md
@@ -13,7 +13,7 @@ modules and lifecycle scripts. This caused:
- No separation between "code that gets installed on the cluster" and "code that spec
references by module path".
-- No clear, fork-safe area for customer customisation.
+- No clear, fork-safe area for customer customization.
- Ambiguity between `extensions/` (sys.path) and `extensions/libraries/` (a variant
that was never released to `main`).
@@ -69,6 +69,6 @@ user-facing documentation to avoid confusion with callback-registration APIs.
code. It is **not** required if libraries are sourced from PyPI, UC Volumes, or an
artifact repository.
- New bundles should place spec-referenced Python in `src/python/` and cluster-install
- artefacts in `src/libraries/` (if bundled).
+ artifacts in `src/libraries/` (if bundled).
- The `pipeline_bundle_template/` and samples are migrated to this layout in this PR.
- Existing bundles using `extensions/` continue to work until `v1.0.0` (see ADR-0002).
diff --git a/docs/decisions/0002-extensions-deprecation.md b/docs/decisions/0002-extensions-deprecation.md
index f8d4dae..a2e7074 100644
--- a/docs/decisions/0002-extensions-deprecation.md
+++ b/docs/decisions/0002-extensions-deprecation.md
@@ -22,12 +22,12 @@ The new canonical path (`src/python/`) was introduced in ADR-0001.
## Decision
-| Legacy behaviour | Released? | Action in v0.13.0 | Removed in |
+| Legacy behavior | Released? | Action in v0.13.0 | Removed in |
|-----------------|-----------|-----------------|------------|
| Flat `extensions/` on `sys.path` | Yes | Emit `DeprecationWarning` + structured log | **v1.0.0** |
| `extensions/libraries/` on `sys.path` | No | Drop immediately, no deprecation window | **v0.13.0** (this PR) |
-**One-minor deprecation window:** deprecated behaviour is removed in the first minor
+**One-minor deprecation window:** deprecated behavior is removed in the first minor
version after the deprecation warning lands — giving consumers one release cycle to
migrate before the path disappears.
diff --git a/docs/decisions/0004-mirror-to-stdout-default.md b/docs/decisions/0004-mirror-to-stdout-default.md
index 2eb3118..e0bd432 100644
--- a/docs/decisions/0004-mirror-to-stdout-default.md
+++ b/docs/decisions/0004-mirror-to-stdout-default.md
@@ -62,7 +62,7 @@ default logger is returned unchanged.
- The `lakeflowframework` `logging.Logger` instance is modified in-place when
silenced. Any code that obtained a direct `logging.getLogger("lakeflowframework")`
reference before logger resolution will stop producing output — this is the
- intended behaviour.
+ intended behavior.
- `mirror_to_stdout: false` is the breaking change relative to the previous
hard-coded stdout logger. Teams relying on plain-text stdout output and also
enabling a custom logger must set `mirror_to_stdout: true` explicitly.
diff --git a/docs/decisions/0005-syspath-registration-decoupled-from-logger.md b/docs/decisions/0005-syspath-registration-decoupled-from-logger.md
index 33ebe02..360a77e 100644
--- a/docs/decisions/0005-syspath-registration-decoupled-from-logger.md
+++ b/docs/decisions/0005-syspath-registration-decoupled-from-logger.md
@@ -1,4 +1,4 @@
-# ADR-0005: `sys.path` registration decoupled from logger initialisation
+# ADR-0005: `sys.path` registration decoupled from logger initialization
**Date:** 2026-05-17
**Status:** Accepted
diff --git a/docs/decisions/0006-local-config-sparse-overlay.md b/docs/decisions/0006-local-config-sparse-overlay.md
index d21baea..07d67da 100644
--- a/docs/decisions/0006-local-config-sparse-overlay.md
+++ b/docs/decisions/0006-local-config-sparse-overlay.md
@@ -30,7 +30,7 @@ The original framework config mechanism offered a binary choice: use
resolution like `GLOBAL_CONFIG`).
The `src/local/` fork-safe area introduced in ADR-0001 provides a natural home
-for customer customisations. Config overrides should follow the same pattern.
+for customer customizations. Config overrides should follow the same pattern.
## Decision
@@ -47,7 +47,7 @@ Override detection and deprecation warnings are the **caller's responsibility**.
active, stores the resolved root in `self._active_config_path`, and passes it
to `load_framework_config`. `self._active_config_path` is reused by
`_setup_operational_metadata` so the override detection only runs once per
- pipeline initialisation.
+ pipeline initialization.
- **`load_framework_logger_config`** independently checks
`config/override/logger.json`, emits its own `DeprecationWarning` if active,
@@ -66,7 +66,7 @@ load_framework_config(
) -> Dict
```
-Behaviour:
+Behavior:
1. When `name` is a **sequence** (e.g. `FrameworkPaths.GLOBAL_CONFIG =
("global.json", "global.yaml", "global.yml")`):
diff --git a/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md b/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md
index 13363cd..17bfe02 100644
--- a/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md
+++ b/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md
@@ -32,16 +32,21 @@ The pipeline builds:
- `main` as `current`
- selected release tags
+- optional `local-branch-preview` (current branch) with `--preview`
- `docs/build/html/versions.json`
- root redirect `docs/build/html/index.html -> current/index.html`
-### 2) Keep selector UI scope minimal and theme-aligned
+Each published version (`current` and release tags) is built with **main's**
+`docs/conf.py` + templates and that ref's `docs/source`, so the RTD version
+selector stays available on historical docs. `--preview` adds
+`local-branch-preview` using the current branch's own conf/source.
-- Keep the RTD selector at the bottom (`docs/_templates/versions.html`).
-- Keep breadcrumb version/release-date metadata (`docs/_templates/breadcrumbs.html`).
-- Do **not** keep a second custom header selector path.
+### 2) Shared superset `versions.json` + theme-native selectors
-This limits ongoing CSS/template overrides and reduces regressions from theme changes.
+- Write one `versions.json` with both mike fields (`version`, `title`, `aliases`) and legacy RTD fields (`name`, `display_version`, `url`, …).
+- RTD refs (`current` / tags, via main's conf) consume it at build time via `DOCS_VERSIONS_FILE` / `versions.html`.
+- Immaterial preview refs enable `version_dropdown` and load the same file over HTTP.
+- Do **not** use a feature-branch conf for historical tag builds — that drops the RTD selector on older tags.
### 3) Place docs tooling under `docs/scripts/`
diff --git a/docs/decisions/0008-lakeflow-framework-package.md b/docs/decisions/0008-lakeflow-framework-package.md
index 1fcb79c..2b2e0ab 100644
--- a/docs/decisions/0008-lakeflow-framework-package.md
+++ b/docs/decisions/0008-lakeflow-framework-package.md
@@ -137,7 +137,7 @@ module is added.
risk with customer code.
- Existing flat-deploy customers are unaffected: the `src/` shims preserve
backward compatibility, and `framework.sourcePath` + Workspace Files-first
- resolution (ADR-0009) means behaviour is identical to pre-v0.20.0.
+ resolution (ADR-0009) means behavior is identical to pre-v0.20.0.
- Editable installs (`pip install -e ".[contrib]"`) are supported for local
development; the `src/` layout ensures the installed package and the source
tree are the same directory.
diff --git a/docs/decisions/0009-strategy-b-workspace-files-first-resolver.md b/docs/decisions/0009-strategy-b-workspace-files-first-resolver.md
index dbd8512..71e9cdd 100644
--- a/docs/decisions/0009-strategy-b-workspace-files-first-resolver.md
+++ b/docs/decisions/0009-strategy-b-workspace-files-first-resolver.md
@@ -32,7 +32,7 @@ first when both a Workspace Files path and the wheel package data are potentiall
- `+` Simple: the wheel is always the canonical source of defaults.
- `−` **Breaking for existing flat-deploy customers.** Their defaults live in
Workspace Files; if the wheel shadow-contains a newer (or different) version of a default
- file, behaviour silently changes on upgrade.
+ file, behavior silently changes on upgrade.
- `−` Contradicts the "explicit wins" principle: a customer who set
`framework.sourcePath` explicitly expects Workspace Files to be authoritative.
@@ -91,7 +91,7 @@ similar validators.
- **Zero-change upgrade for flat-deploy customers.** Their `framework.sourcePath`
points to the deployed `src/` directory in Workspace Files; those files are
- found at step 1 and the wheel is never consulted. Behaviour is identical to
+ found at step 1 and the wheel is never consulted. Behavior is identical to
pre-v0.20.0.
- **Explicit wins.** Setting `framework.sourcePath` is a deliberate act.
Workspace Files-first honours that intent; package-first would silently override it.
@@ -108,7 +108,7 @@ similar validators.
config and schema reads; call sites that previously used
`os.path.join(framework_path, FrameworkPaths.CONFIG_PATH, name)` are
migrated to it.
-- Flat-deploy customers see **no behaviour change** on upgrade to v0.20.0.
+- Flat-deploy customers see **no behavior change** on upgrade to v0.20.0.
- Wheel-install customers get defaults from the wheel with no `framework.sourcePath`
required.
- The `src/local/config/` sparse overlay (ADR-0006) continues to work in all
diff --git a/docs/redirects/architecture.yaml b/docs/redirects/architecture.yaml
new file mode 100644
index 0000000..d7bbf4b
--- /dev/null
+++ b/docs/redirects/architecture.yaml
@@ -0,0 +1,3 @@
+redirects:
+- from: concepts.html
+ to: architecture/index.html
diff --git a/docs/redirects/build.yaml b/docs/redirects/build.yaml
new file mode 100644
index 0000000..8e1ed39
--- /dev/null
+++ b/docs/redirects/build.yaml
@@ -0,0 +1,45 @@
+redirects:
+- from: build_pipeline_bundle.html
+ to: build/index.html
+- from: build_pipeline_bundle_steps.html
+ to: build/bundle-steps.html
+- from: build_pipeline_bundle_structure.html
+ to: build/bundle-structure.html
+- from: dataflow_spec_ref_cdc.html
+ to: build/spec-reference/cdc.html
+- from: dataflow_spec_ref_data_quality.html
+ to: build/spec-reference/data-quality.html
+- from: dataflow_spec_ref_main_flows.html
+ to: build/spec-reference/flows.html
+- from: dataflow_spec_ref_main_materialized_views.html
+ to: build/spec-reference/materialized-views.html
+- from: dataflow_spec_ref_main_standard.html
+ to: build/spec-reference/standard.html
+- from: dataflow_spec_ref_source_details.html
+ to: build/spec-reference/source-details.html
+- from: dataflow_spec_ref_table_migration.html
+ to: build/spec-reference/table-migration.html
+- from: dataflow_spec_ref_target_details.html
+ to: build/spec-reference/target-details.html
+- from: dataflow_spec_reference.html
+ to: build/spec-reference/index.html
+- from: patterns.html
+ to: build/patterns/index.html
+- from: patterns_streaming_basic_1_1.html
+ to: build/patterns/basic-1-1.html
+- from: build/patterns/silver-1-1.html
+ to: build/patterns/basic-1-1.html
+- from: build/patterns/common-medallion-path.html
+ to: build/patterns/base-patterns.html
+- from: patterns_streaming_cdc_stream_from_snapshot.html
+ to: build/patterns/cdc-stream-from-snapshot.html
+- from: patterns_streaming_flow_groups.html
+ to: build/patterns/flow-groups.html
+- from: patterns_streaming_multi_source_streaming.html
+ to: build/patterns/multi-source-streaming.html
+- from: patterns_streaming_stream_static_basic.html
+ to: build/patterns/stream-static-basic.html
+- from: patterns_streaming_stream_static_streaming_dwh.html
+ to: build/patterns/stream-static-streaming-dwh.html
+- from: splitting_dataflow_spec.html
+ to: build/spec-reference/splitting-dataflow-spec.html
diff --git a/docs/redirects/contributors.yaml b/docs/redirects/contributors.yaml
new file mode 100644
index 0000000..046b187
--- /dev/null
+++ b/docs/redirects/contributors.yaml
@@ -0,0 +1,15 @@
+redirects:
+- from: contributor.html
+ to: contributors/index.html
+- from: contributor_contrib.html
+ to: contributors/contrib.html
+- from: contributor_dev_docs.html
+ to: contributors/dev-docs.html
+- from: contributor_dev_env.html
+ to: contributors/dev-env.html
+- from: contributor_dev_git.html
+ to: contributors/dev-git.html
+- from: contributor_dev_steps.html
+ to: contributors/dev-steps.html
+- from: contributor_imports.html
+ to: contributors/imports.html
diff --git a/docs/redirects/deploy.yaml b/docs/redirects/deploy.yaml
new file mode 100644
index 0000000..1bd2a48
--- /dev/null
+++ b/docs/redirects/deploy.yaml
@@ -0,0 +1,17 @@
+redirects:
+- from: deploy.html
+ to: deploy/index.html
+- from: deploy_before_you_deploy.html
+ to: deploy/before-you-deploy.html
+- from: deploy_ci_cd.html
+ to: deploy/ci-cd.html
+- from: deploy_framework.html
+ to: deploy/framework/index.html
+- from: deploy_framework_options.html
+ to: deploy/framework/options.html
+- from: deploy_local_framework.html
+ to: deploy/framework/local-framework.html
+- from: deploy_local_pipeline_bundle.html
+ to: deploy/pipeline-bundle/local.html
+- from: deploy_wheel.html
+ to: deploy/framework/wheel.html
diff --git a/docs/redirects/features.yaml b/docs/redirects/features.yaml
new file mode 100644
index 0000000..456f84d
--- /dev/null
+++ b/docs/redirects/features.yaml
@@ -0,0 +1,93 @@
+redirects:
+- from: feature_a_z.html
+ to: features/a-z.html
+- from: feature_authoring.html
+ to: features/authoring/index.html
+- from: feature_auto_complete.html
+ to: features/authoring/auto-complete.html
+- from: feature_builder_parallelization.html
+ to: features/configuration/builder-parallelization.html
+- from: feature_cdc.html
+ to: features/platform/cdc.html
+- from: feature_cdf.html
+ to: features/platform/cdf.html
+- from: feature_configuration.html
+ to: features/configuration/index.html
+- from: feature_data_quality.html
+ to: features/data-quality/index.html
+- from: feature_data_quality_expectations.html
+ to: features/data-quality/expectations.html
+- from: feature_data_quality_quarantine.html
+ to: features/data-quality/quarantine.html
+- from: feature_databricks_schema_features.html
+ to: features/platform/databricks-schema-features.html
+- from: feature_environments.html
+ to: features/environments/index.html
+- from: feature_framework_configuration.html
+ to: features/configuration/framework-configuration.html
+- from: feature_liquid_clustering.html
+ to: features/platform/liquid-clustering.html
+- from: feature_logging.html
+ to: features/configuration/logging.html
+- from: feature_logical_environment.html
+ to: features/environments/logical-environments.html
+- from: feature_mandatory_table_properties.html
+ to: features/configuration/mandatory-table-properties.html
+- from: feature_materialized_views.html
+ to: features/platform/materialized-views.html
+- from: feature_metadata.html
+ to: features/metadata/index.html
+- from: feature_migrations.html
+ to: features/migrations/index.html
+- from: feature_multi_source_streaming.html
+ to: features/platform/multi-source-streaming.html
+- from: feature_operational_metadata.html
+ to: features/configuration/operational-metadata.html
+- from: feature_platform.html
+ to: features/platform/index.html
+- from: feature_python.html
+ to: features/python/index.html
+- from: feature_python_dependency_management.html
+ to: features/python/dependency-management.html
+- from: feature_python_extensions.html
+ to: features/python/extensions.html
+- from: feature_python_functions.html
+ to: features/python/functions.html
+- from: feature_python_source.html
+ to: features/python/source.html
+- from: feature_schemas.html
+ to: features/metadata/schema-management.html
+- from: feature_secrets.html
+ to: features/configuration/secrets-management.html
+- from: feature_soft_deletes.html
+ to: features/platform/soft-deletes.html
+- from: feature_source_types.html
+ to: features/sources-targets/source-types.html
+- from: feature_sources_targets.html
+ to: features/sources-targets/index.html
+- from: feature_spark_configuration.html
+ to: features/configuration/spark-configuration.html
+- from: feature_spec_format.html
+ to: features/metadata/spec-format.html
+- from: feature_sql_source.html
+ to: features/sources-targets/sql-source.html
+- from: feature_substitutions.html
+ to: features/metadata/substitutions.html
+- from: feature_table_migration.html
+ to: features/migrations/table-migration.html
+- from: feature_target_catalog_schema.html
+ to: features/platform/target-catalog-schema.html
+- from: feature_target_types.html
+ to: features/sources-targets/target-types.html
+- from: feature_templates.html
+ to: features/metadata/templates.html
+- from: feature_ui_integration.html
+ to: features/authoring/ui-integration.html
+- from: feature_validation.html
+ to: features/metadata/validation.html
+- from: feature_versioning_dataflow_spec.html
+ to: features/environments/versioning-dataflow-specs.html
+- from: feature_versioning_framework.html
+ to: features/environments/versioning-framework.html
+- from: features.html
+ to: features/index.html
diff --git a/docs/redirects/get-started.yaml b/docs/redirects/get-started.yaml
new file mode 100644
index 0000000..26c8d6f
--- /dev/null
+++ b/docs/redirects/get-started.yaml
@@ -0,0 +1,7 @@
+redirects:
+- from: get_started.html
+ to: get-started/index.html
+- from: quick_start.html
+ to: get-started/quick-start.html
+- from: what_is_lakeflow_framework.html
+ to: get-started/what-is-lakeflow-framework.html
diff --git a/docs/redirects/legacy.yaml b/docs/redirects/legacy.yaml
new file mode 100644
index 0000000..765000a
--- /dev/null
+++ b/docs/redirects/legacy.yaml
@@ -0,0 +1,17 @@
+redirects:
+- from: deploy_framework_bundle.html
+ to: deploy/framework/index.html
+- from: deploy_framework_overview.html
+ to: deploy/framework/options.html
+- from: deploy_pipeline_bundle.html
+ to: deploy/pipeline-bundle/local.html
+- from: feature_direct_publishing_mode.html
+ to: features/platform/target-catalog-schema.html
+- from: getting_started.html
+ to: get-started/index.html
+- from: home.html
+ to: index.html
+- from: orchestration.html
+ to: build/index.html
+- from: pipeline_execution.html
+ to: build/index.html
diff --git a/docs/redirects/samples.yaml b/docs/redirects/samples.yaml
new file mode 100644
index 0000000..4e8518e
--- /dev/null
+++ b/docs/redirects/samples.yaml
@@ -0,0 +1,3 @@
+redirects:
+- from: deploy_samples.html
+ to: samples/index.html
diff --git a/docs/scripts/build_versioned_docs.py b/docs/scripts/build_versioned_docs.py
index dc57dac..2eaa43e 100755
--- a/docs/scripts/build_versioned_docs.py
+++ b/docs/scripts/build_versioned_docs.py
@@ -2,10 +2,17 @@
"""Build versioned docs without sphinx-multiversion.
Build strategy:
-- Always build `main` as the default development docs, published as `current`.
+- Always build ``main`` as ``current``.
- Build selected release tags from docs/scripts/select_versions.py.
+- **Docs IA cutover at v0.21.0:** ``current`` and tags ``>= v0.21.0`` use
+ **main's** ``docs/conf.py`` with that ref's ``docs/source`` (immaterial /
+ section-folder IA). Tags ``<= v0.20.x`` use **v0.20.0's** ``docs/conf.py``
+ (RTD / flat layout) with each tag's own ``docs/source``.
+- ``--preview`` builds the current branch as ``local-branch-preview`` using
+ **that branch's** conf + source (pre-merge IA validation).
- Write per-version output under docs/build/html/<version>/.
-- Generate docs/build/html/versions.json for the sidebar switcher.
+- Generate docs/build/html/versions.json as a **superset** manifest consumed by
+ both sphinx-immaterial (mike fields) and legacy RTD ``versions.html``.
"""
from __future__ import annotations
@@ -13,12 +20,25 @@
import argparse
import json
import os
+import re
import shutil
import subprocess
import sys
from pathlib import Path
+_SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_SCRIPTS_DIR))
+
DOCS_BASEURL = "https://databricks-solutions.github.io/lakeflow_framework/"
+PREVIEW_VERSION_NAME = "local-branch-preview"
+MAIN_CONF_WORKTREE = "_conf_main"
+LEGACY_CONF_REF = "v0.20.0"
+LEGACY_CONF_WORKTREE = "_conf_legacy"
+# Tags at or above this semver use main conf; older tags use LEGACY_CONF_REF conf.
+DOCS_CONF_CUTOVER = (0, 21, 0)
+REDIRECT_VERSIONS = frozenset({"current", PREVIEW_VERSION_NAME})
+_TAG_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
def _run(command: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None) -> None:
@@ -39,13 +59,11 @@ def _selected_tags(repo_root: Path) -> list[str]:
return json.loads(output) if output else []
-def _list_preview_branches(repo_root: Path) -> list[str]:
- raw = _run_capture(
- ["git", "for-each-ref", "--format=%(refname:short)", "refs/heads"],
- cwd=repo_root,
- )
- branches = [line.strip() for line in raw.splitlines() if line.strip()]
- return [b for b in branches if b != "main"]
+def _current_branch(repo_root: Path) -> str:
+ try:
+ return _run_capture(["git", "branch", "--show-current"], cwd=repo_root)
+ except subprocess.CalledProcessError:
+ return ""
def _safe_remove(path: Path) -> None:
@@ -53,10 +71,56 @@ def _safe_remove(path: Path) -> None:
shutil.rmtree(path)
+def _cleanup_docs_worktrees(repo_root: Path, worktrees_root: Path) -> None:
+ """Unregister git worktrees under ``worktrees_root``, then delete the dir.
+
+ ``shutil.rmtree`` alone leaves stale registrations ("missing but already
+ registered") that break the next ``git worktree add``.
+ """
+ if worktrees_root.exists():
+ for child in sorted(worktrees_root.iterdir()):
+ if not child.is_dir():
+ continue
+ subprocess.run(
+ ["git", "worktree", "remove", "--force", str(child)],
+ cwd=repo_root,
+ check=False,
+ capture_output=True,
+ )
+ shutil.rmtree(worktrees_root, ignore_errors=True)
+ subprocess.run(
+ ["git", "worktree", "prune"],
+ cwd=repo_root,
+ check=False,
+ capture_output=True,
+ )
+
+
def _normalize_name(ref: str) -> str:
return ref.replace("/", "-")
+def _parse_release_tag(tag: str) -> tuple[int, int, int] | None:
+ match = _TAG_RE.match(tag)
+ if not match:
+ return None
+ return int(match.group(1)), int(match.group(2)), int(match.group(3))
+
+
+def _conf_ref_for(version_name: str) -> str | None:
+ """Git ref for ``docs/conf.py``, or ``None`` for the working tree (preview)."""
+ if version_name == PREVIEW_VERSION_NAME:
+ return None
+ if version_name == "current":
+ return "main"
+ parsed = _parse_release_tag(version_name)
+ if parsed is None:
+ return "main"
+ if parsed >= DOCS_CONF_CUTOVER:
+ return "main"
+ return LEGACY_CONF_REF
+
+
def _release_date_for_ref(repo_root: Path, ref: str) -> str:
# ISO-style date (YYYY-MM-DD) for deterministic display in docs headers.
return _run_capture(["git", "log", "-1", "--format=%cs", ref], cwd=repo_root)
@@ -72,43 +136,174 @@ def _version_for_ref(repo_root: Path, ref: str, fallback: str) -> str:
return fallback
+def _superset_versions(links: list[dict[str, str]]) -> list[dict[str, object]]:
+ """Build a versions.json entry set readable by immaterial and legacy RTD.
+
+ Immaterial / mike fields: ``version``, ``title``, ``aliases``
+ Legacy RTD fields: ``name``, ``display_version``, ``url``, ``is_latest``,
+ ``status``, ``release_date``
+ """
+ payload: list[dict[str, object]] = []
+ for item in links:
+ name = item["name"]
+ display = item["display_version"]
+ is_current = item.get("status") == "current"
+ title = f"{display} (current)" if is_current else display
+ aliases: list[str] = ["latest"] if is_current else []
+ payload.append(
+ {
+ # mike / sphinx-immaterial
+ "version": name,
+ "title": title,
+ "aliases": aliases,
+ # legacy RTD versions.html (via conf.py _load_versions)
+ "name": name,
+ "display_version": display,
+ "url": f"{name}/index.html",
+ "is_latest": is_current,
+ "status": "current" if is_current else item.get("status", "release"),
+ "release_date": item.get("release_date", ""),
+ }
+ )
+ return payload
+
+
+def _ensure_worktree(
+ *,
+ repo_root: Path,
+ worktrees_root: Path,
+ name: str,
+ ref: str,
+) -> Path:
+ path = worktrees_root / name
+ if path.exists():
+ return path
+ try:
+ _run(["git", "worktree", "add", "--detach", str(path), ref], cwd=repo_root)
+ except subprocess.CalledProcessError:
+ # Recover from a previous run that deleted the dir without unregistering.
+ subprocess.run(
+ ["git", "worktree", "prune"],
+ cwd=repo_root,
+ check=False,
+ capture_output=True,
+ )
+ _run(
+ ["git", "worktree", "add", "--force", "--detach", str(path), ref],
+ cwd=repo_root,
+ )
+ return path
+
+
+def _sphinx_build(
+ *,
+ repo_root: Path,
+ conf_dir: Path,
+ source_dir: Path,
+ out_dir: Path,
+ version_name: str,
+ versions_file: Path,
+) -> None:
+ out_dir.mkdir(parents=True, exist_ok=True)
+ env = os.environ.copy()
+ env["DOCS_CURRENT_VERSION"] = version_name
+ # Legacy RTD conf.py loads the shared manifest at build time.
+ env["DOCS_VERSIONS_FILE"] = str(versions_file)
+ _run(
+ [
+ sys.executable,
+ "-m",
+ "sphinx",
+ "-b",
+ "html",
+ "-c",
+ str(conf_dir),
+ str(source_dir),
+ str(out_dir),
+ ],
+ cwd=repo_root,
+ env=env,
+ )
+
+
def _build_ref(
*,
repo_root: Path,
- docs_dir: Path,
worktrees_root: Path,
output_root: Path,
+ versions_file: Path,
ref: str,
version_name: str,
+ conf_ref: str | None,
) -> None:
- worktree_path = worktrees_root / _normalize_name(version_name)
- _run(["git", "worktree", "add", "--detach", str(worktree_path), ref], cwd=repo_root)
- try:
- out_dir = output_root / version_name
- out_dir.mkdir(parents=True, exist_ok=True)
- env = os.environ.copy()
- env["DOCS_CURRENT_VERSION"] = version_name
- env["DOCS_VERSIONS_FILE"] = str(output_root / "versions.json")
- _run(
- [
- "sphinx-build",
- "-b",
- "html",
- "-c",
- str(docs_dir),
- str(worktree_path / "docs" / "source"),
- str(out_dir),
- ],
- cwd=repo_root,
- env=env,
+ """Build one version folder.
+
+ ``conf_ref`` selects which git ref supplies ``docs/conf.py``:
+ ``main`` (modern IA), ``v0.20.0`` (legacy RTD), or ``None`` (working tree).
+ Source always comes from ``ref`` (except preview, which uses the working tree).
+ """
+ if conf_ref is None:
+ # Preview: pick up local conf.py / theme / source edits immediately.
+ source_wt = repo_root
+ conf_dir = repo_root / "docs"
+ conf_label = f"{ref} (working tree)"
+ else:
+ source_wt = _ensure_worktree(
+ repo_root=repo_root,
+ worktrees_root=worktrees_root,
+ name=_normalize_name(version_name),
+ ref=ref,
+ )
+ conf_worktree_name = (
+ MAIN_CONF_WORKTREE if conf_ref == "main" else LEGACY_CONF_WORKTREE
)
- finally:
- _run(["git", "worktree", "remove", "--force", str(worktree_path)], cwd=repo_root)
+ conf_wt = _ensure_worktree(
+ repo_root=repo_root,
+ worktrees_root=worktrees_root,
+ name=conf_worktree_name,
+ ref=conf_ref,
+ )
+ conf_dir = conf_wt / "docs"
+ conf_label = conf_ref
+
+ source_dir = source_wt / "docs" / "source"
+ print(
+ f"Building docs for {ref} -> {version_name} (conf={conf_label})",
+ file=sys.stderr,
+ )
+ out_dir = output_root / version_name
+ _sphinx_build(
+ repo_root=repo_root,
+ conf_dir=conf_dir,
+ source_dir=source_dir,
+ out_dir=out_dir,
+ version_name=version_name,
+ versions_file=versions_file,
+ )
+
+ if version_name in REDIRECT_VERSIONS:
+ from write_redirects import write_redirects
+
+ count = write_redirects(
+ out_dir,
+ version_name=version_name,
+ docs_dir=repo_root / "docs",
+ )
+ print(f"Wrote {count} redirect stub(s) for {version_name}", file=sys.stderr)
def main() -> None:
parser = argparse.ArgumentParser()
- parser.add_argument("--preview", action="store_true", help="Include local branches in build.")
+ parser.add_argument(
+ "--preview",
+ action="store_true",
+ help=(
+ "Also build the current git branch as ``local-branch-preview`` "
+ "using that branch's own conf/source. "
+ "``current`` uses main; tags >= v0.21.0 use main conf; "
+ f"older tags use {LEGACY_CONF_REF} conf."
+ ),
+ )
args = parser.parse_args()
docs_dir = Path(__file__).resolve().parent.parent
@@ -116,17 +311,30 @@ def main() -> None:
build_root = docs_dir / "build"
output_root = build_root / "html"
worktrees_root = build_root / ".worktrees"
+ versions_file = output_root / "versions.json"
_safe_remove(output_root)
- _safe_remove(worktrees_root)
+ _cleanup_docs_worktrees(repo_root, worktrees_root)
output_root.mkdir(parents=True, exist_ok=True)
worktrees_root.mkdir(parents=True, exist_ok=True)
tags = _selected_tags(repo_root)
- versions = [{"name": "current", "ref": "main"}] + [{"name": t, "ref": t} for t in tags]
+ versions: list[dict[str, str]] = [{"name": "current", "ref": "main"}]
+ versions.extend({"name": t, "ref": t} for t in tags)
if args.preview:
- for branch in _list_preview_branches(repo_root):
- versions.append({"name": _normalize_name(branch), "ref": branch})
+ preview_branch = _current_branch(repo_root)
+ if not preview_branch:
+ print(
+ "Warning: --preview ignored (detached HEAD; no current branch).",
+ file=sys.stderr,
+ )
+ elif preview_branch == "main":
+ print(
+ "Warning: --preview ignored (already on main; current covers it).",
+ file=sys.stderr,
+ )
+ else:
+ versions.append({"name": PREVIEW_VERSION_NAME, "ref": preview_branch})
# De-duplicate while preserving order.
deduped: list[dict[str, str]] = []
@@ -140,7 +348,11 @@ def main() -> None:
links = []
for item in deduped:
is_current = item["name"] == "current"
- display_version = _version_for_ref(repo_root, item["ref"], item["name"].lstrip("v"))
+ is_preview = item["name"] == PREVIEW_VERSION_NAME
+ if is_preview:
+ display_version = f"{PREVIEW_VERSION_NAME} ({item['ref']})"
+ else:
+ display_version = _version_for_ref(repo_root, item["ref"], item["name"].lstrip("v"))
links.append(
{
"name": item["name"],
@@ -149,6 +361,7 @@ def main() -> None:
"is_latest": is_current,
"status": "current" if is_current else "release",
"release_date": _release_date_for_ref(repo_root, item["ref"]),
+ "ref": item["ref"],
}
)
@@ -162,19 +375,30 @@ def main() -> None:
if item["name"] == "current" or item["display_version"] != current_version
]
- (output_root / "versions.json").write_text(json.dumps(links, indent=2) + "\n", encoding="utf-8")
+ versions_file.write_text(
+ json.dumps(_superset_versions(links), indent=2) + "\n",
+ encoding="utf-8",
+ )
for item in deduped:
- print(f"Building docs for {item['ref']} -> {item['name']}", file=sys.stderr)
_build_ref(
repo_root=repo_root,
- docs_dir=docs_dir,
worktrees_root=worktrees_root,
output_root=output_root,
+ versions_file=versions_file,
ref=item["ref"],
version_name=item["name"],
+ conf_ref=_conf_ref_for(item["name"]),
)
+ # Immaterial's mike client resolves version_json relative to the version
+ # folder. Copy the site-root manifest into each version dir so both
+ # ``versions.json`` and ``../versions.json`` resolve successfully.
+ for item in deduped:
+ dest = output_root / item["name"] / "versions.json"
+ if (output_root / item["name"]).is_dir():
+ shutil.copy2(versions_file, dest)
+
# Root redirect for GitHub Pages.
index_html = """<!DOCTYPE html>
<html>
@@ -219,8 +443,13 @@ def main() -> None:
)
(output_root / "robots.txt").write_text(robots_txt, encoding="utf-8")
- _safe_remove(worktrees_root)
- _run(["git", "worktree", "prune"], cwd=repo_root)
+ from write_redirects import write_root_mirror_redirects
+
+ root_count = write_root_mirror_redirects(output_root, docs_dir=repo_root / "docs")
+ if root_count:
+ print(f"Wrote {root_count} root-level redirect mirror(s)", file=sys.stderr)
+
+ _cleanup_docs_worktrees(repo_root, worktrees_root)
if __name__ == "__main__":
diff --git a/docs/scripts/select_versions.py b/docs/scripts/select_versions.py
index 6a4ea33..03a5cd0 100755
--- a/docs/scripts/select_versions.py
+++ b/docs/scripts/select_versions.py
@@ -5,6 +5,8 @@
--------
* For the current release major (highest major present), include the latest
patch release for the last 5 minor series.
+* For major ``0``, include the latest patch for every minor from the current
+ minor down to ``0.12`` (inclusive), not capped at five minors.
* Also include the latest available release for each of the last 3 major
versions.
@@ -27,6 +29,9 @@
TAG_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
+# For 0.x releases, publish docs back to this minor (latest patch per minor).
+MIN_MINOR_MAJOR_0 = 12
+
def _get_tags() -> list[str]:
result = subprocess.run(
@@ -75,19 +80,27 @@ def select_versions(tags: list[str]) -> list[str]:
majors_desc = sorted(best_per_major.keys(), reverse=True)
current_major = majors_desc[0]
- # Last 5 minor series for current major (latest patch in each minor).
+ # Latest patch for each selected minor in the current major.
current_major_minors = sorted(
[minor for (major, minor) in best_per_minor if major == current_major],
reverse=True,
)
- current_major_last_5 = {
- best_per_minor[(current_major, minor)][1] for minor in current_major_minors[:5]
- }
+ if current_major == 0:
+ current_major_selected = {
+ best_per_minor[(0, minor)][1]
+ for minor in current_major_minors
+ if minor >= MIN_MINOR_MAJOR_0
+ }
+ else:
+ current_major_selected = {
+ best_per_minor[(current_major, minor)][1]
+ for minor in current_major_minors[:5]
+ }
# Last 3 major versions (latest release per major).
last_3_major_latest = {best_per_major[major][2] for major in majors_desc[:3]}
- return _sort_tags_desc(current_major_last_5 | last_3_major_latest)
+ return _sort_tags_desc(current_major_selected | last_3_major_latest)
def tags_to_regex(tags: list[str]) -> str:
diff --git a/docs/scripts/write_redirects.py b/docs/scripts/write_redirects.py
new file mode 100644
index 0000000..c7c130c
--- /dev/null
+++ b/docs/scripts/write_redirects.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Write HTML redirect stubs from docs/redirects.yaml and docs/redirects/*.yaml."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+try:
+ import yaml
+except ImportError as exc: # pragma: no cover
+ raise SystemExit("PyYAML is required: pip install pyyaml") from exc
+
+DOCS_BASEURL = "https://databricks-solutions.github.io/lakeflow_framework/"
+
+_STUB_MARKER = "docs-redirect-stub"
+
+
+def _docs_dir() -> Path:
+ return Path(__file__).resolve().parent.parent
+
+
+def load_redirects(docs_dir: Path | None = None) -> list[dict[str, str]]:
+ """Load and merge redirect entries from the central manifest."""
+ docs_dir = docs_dir or _docs_dir()
+ entries: list[dict[str, str]] = []
+
+ single = docs_dir / "redirects.yaml"
+ if single.is_file():
+ data = yaml.safe_load(single.read_text(encoding="utf-8")) or {}
+ entries.extend(data.get("redirects", []))
+
+ redirects_dir = docs_dir / "redirects"
+ if redirects_dir.is_dir():
+ for path in sorted(redirects_dir.glob("*.yaml")):
+ data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+ entries.extend(data.get("redirects", []))
+
+ seen: set[str] = set()
+ deduped: list[dict[str, str]] = []
+ for entry in entries:
+ src = entry.get("from", "").strip()
+ dst = entry.get("to", "").strip()
+ if not src or not dst or src in seen:
+ continue
+ seen.add(src)
+ deduped.append({"from": src, "to": dst})
+ return deduped
+
+
+def _is_redirect_stub(path: Path) -> bool:
+ if not path.is_file():
+ return False
+ try:
+ return _STUB_MARKER in path.read_text(encoding="utf-8")
+ except OSError:
+ return False
+
+
+def _redirect_html(*, target: str, canonical: str) -> str:
+ return f"""<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta name="{_STUB_MARKER}" content="true">
+ <title>Page moved
+
+
+
+
+ This page has moved to {target}.
+
+