From b5bced68f7208507c819054e916daf5560dee780 Mon Sep 17 00:00:00 2001
From: rederik76 <127075662+rederik76@users.noreply.github.com>
Date: Sat, 11 Jul 2026 10:00:18 +1000
Subject: [PATCH 01/15] feat(docs): rebrand Sphinx site to immaterial and fix
multiversion builds
Switch the docs theme to sphinx-immaterial with Databricks styling, a
landing/quick-start IA, and the native version dropdown. Build each
version from that ref's own conf/source so historical RTD docs stay
intact, emit a shared superset versions.json for both selectors, and
extend 0.x selection back to 0.12 (latest patch per minor).
---
docs/README.md | 42 +-
docs/_templates/breadcrumbs.html | 15 -
docs/_templates/versions.html | 20 -
docs/conf.py | 373 ++++--
...07-scripted-versioned-docs-and-ui-scope.md | 12 +-
docs/scripts/build_versioned_docs.py | 82 +-
docs/scripts/select_versions.py | 23 +-
docs/source/_landing.rst | 78 ++
docs/source/_static/custom.css | 47 +-
docs/source/_static/databricks-theme.css | 1191 +++++++++++++++++
docs/source/_static/lff-logo.png | Bin 0 -> 1175051 bytes
docs/source/build_pipeline_bundle.rst | 4 +-
docs/source/build_pipeline_bundle_steps.rst | 2 +-
docs/source/deploy_framework.rst | 1 +
docs/source/get_started.rst | 9 +
docs/source/getting_started.rst | 40 +-
docs/source/index.rst | 41 +-
docs/source/quick_start.md | 150 +++
docs/source/what_is_lakeflow_framework.rst | 2 +-
requirements-docs.txt | 1 +
tests/test_select_versions.py | 91 ++
21 files changed, 1950 insertions(+), 274 deletions(-)
delete mode 100644 docs/_templates/breadcrumbs.html
delete mode 100644 docs/_templates/versions.html
create mode 100644 docs/source/_landing.rst
create mode 100644 docs/source/_static/databricks-theme.css
create mode 100644 docs/source/_static/lff-logo.png
create mode 100644 docs/source/get_started.rst
create mode 100644 docs/source/quick_start.md
create mode 100644 tests/test_select_versions.py
diff --git a/docs/README.md b/docs/README.md
index 56f94e1..23a7642 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -9,7 +9,7 @@ This folder contains the Sphinx documentation source and build tooling for local
- [Multiversion Implementation (No sphinx-multiversion)](#multiversion-implementation-no-sphinx-multiversion)
- [What Gets Built](#what-gets-built)
- [Version Selection Rules](#version-selection-rules)
- - [How the Sidebar Switcher Works](#how-the-sidebar-switcher-works)
+ - [How the Version Dropdown Works](#how-the-version-dropdown-works)
- [Purge and Rebuild Behavior](#purge-and-rebuild-behavior)
- [Local Build and Testing](#local-build-and-testing)
- [GitHub Pages Deployment](#github-pages-deployment)
@@ -59,6 +59,7 @@ Versioned docs are generated by:
- `main` branch is always built and published as `current`.
- Selected release tags are built from git tags.
+- Each version is built with **that ref's own** `docs/conf.py` and `docs/source` (no shared conf override). Historical tags keep their RTD theme; rebranded branches use immaterial.
- Output layout:
- `docs/build/html/current/index.html`
- `docs/build/html/vX.Y.Z/index.html`
@@ -70,9 +71,10 @@ Versioned docs are generated by:
`scripts/select_versions.py` applies these rules:
1. Identify the highest major version present in tags.
-2. Include the latest patch release for the last 5 minor series in that major.
-3. Include the latest available release for each of the last 3 majors.
-4. De-duplicate and sort descending (newest first).
+2. For major `0`, include the latest patch release for every minor from the current minor down to `0.12` (inclusive).
+3. For other majors, include the latest patch release for the last 5 minor series in that major.
+4. Include the latest available release for each of the last 3 majors.
+5. De-duplicate and sort descending (newest first).
Example ordering:
@@ -82,13 +84,27 @@ Example ordering:
- `v0.14.0`
- `v0.13.0`
- `v0.12.1`
+- … through `v0.12.x` when older minors exist
-### How the Sidebar Switcher Works
+### How the Version Dropdown Works
-- `scripts/build_versioned_docs.py` writes `versions.json` with `name`, `url`, and `is_latest`.
-- `docs/conf.py` loads this manifest via `DOCS_VERSIONS_FILE`.
-- `docs/_templates/versions.html` renders the version list in the RTD sidebar.
-- Links target `.../index.html` explicitly to avoid directory-listing behavior in local `file://` browsing.
+- `scripts/build_versioned_docs.py` writes a **superset** `versions.json` with:
+ - mike / sphinx-immaterial fields: `version`, `title`, `aliases`
+ - legacy RTD fields: `name`, `display_version`, `url`, `status`, `is_latest`, `release_date`
+- RTD-era refs load the manifest at build time via `DOCS_VERSIONS_FILE` into `versions.html`.
+- Immaterial refs enable `version_dropdown` and fetch the same parent `versions.json` over HTTP.
+- Use a local HTTP server to test the immaterial switcher (see below).
+
+### Preview builds
+
+```bash
+make -C docs html-multiversion-preview
+```
+
+- Still publishes `current` from `main`.
+- Also builds every local branch (except `main`) into the version menu.
+- To review this rebrand before merge, open the preview server and select `feature-docs-rebrand` (folder name uses `/` → `-`).
+- Preview uses **committed** branch tips only.
## Purge and Rebuild Behavior
@@ -122,7 +138,7 @@ make -C docs spelling
make -C docs html-multiversion
```
-1. If you need to test branch previews in the version menu:
+1. If you need to compare themes/IA across local branches (e.g. rebrand vs `main`):
```bash
make -C docs html-multiversion-preview
@@ -132,7 +148,7 @@ Practical rule of thumb:
- Doc text/layout change only: `html` + `spelling`
- Versioning/switcher/build-script change: `html` + `spelling` + `html-multiversion`
-- Branch preview verification: add `html-multiversion-preview`
+- Cross-branch theme/IA preview: add `html-multiversion-preview` and select the branch in the version menu
### Open docs
@@ -145,7 +161,9 @@ cd docs/build/html
python3 -m http.server 8000
```
-Then open [http://localhost:8000](http://localhost:8000).
+Then open [http://localhost:8000/current/index.html](http://localhost:8000/current/index.html) to exercise the version dropdown.
+
+Note: `make html` builds a single-version tree without a parent-level `versions.json`; use `html-multiversion` to test version switching.
## GitHub Pages Deployment
diff --git a/docs/_templates/breadcrumbs.html b/docs/_templates/breadcrumbs.html
deleted file mode 100644
index 52e83ea..0000000
--- a/docs/_templates/breadcrumbs.html
+++ /dev/null
@@ -1,15 +0,0 @@
-{% extends "!breadcrumbs.html" %}
-
-{% block breadcrumbs_aside %}
-
-{%- endif %}
diff --git a/docs/conf.py b/docs/conf.py
index afb2892..252f194 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -5,14 +5,20 @@
# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-import json
import os
-import subprocess
import sys
-from datetime import datetime
sys.path.append(os.path.abspath(".")) # Ensure the script is discoverable
from custom_markdown_builder import CustomMarkdownTranslator
+# Ensure sphinx-immaterial registers domain synopses before env init.
+from sphinx.domains.python import PythonDomain
+import sphinx.domains.std
+
+PythonDomain.initial_data.setdefault("synopses", {})
+sphinx.domains.std.StandardDomain.initial_data.setdefault("synopses", {})
+import sphinx_immaterial.apidoc.python.synopses # noqa: F401
+import sphinx_immaterial.apidoc.generic_synopses # noqa: F401
+
project = 'Lakeflow Framework'
copyright = '2026, Databricks'
author = 'Erik Seefeld, Haille Woldegebriel'
@@ -36,144 +42,299 @@
'sphinx_design',
'myst_parser',
'sphinx_tabs.tabs',
+ 'sphinx_copybutton',
'custom_markdown_builder',
"sphinxcontrib.spelling",
]
autosectionlabel_prefix_document = True
+# MyST — task-list checkboxes (Quick Start prerequisites) and fenced directives
+myst_enable_extensions = [
+ "colon_fence",
+ "tasklist",
+]
+myst_enable_checkboxes = True
+
templates_path = ['_templates', 'source/_templates']
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+# Code blocks: line numbers in a Cursor/VS Code-style gutter (table layout).
+html_codeblock_linenos_style = 'table'
+
+# Copy button — WAF-style command snippets only (not dark spec/code panels)
+copybutton_selector = "div.lf-command-block div.highlight pre"
+copybutton_prompt_is_regexp = True
+copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,6}\.\.\. | {5,8}: "
+copybutton_only_copy_prompt_lines = False
+copybutton_remove_prompts = True
+
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-import sphinx_rtd_theme
intersphinx_mapping = {
'rtd': ('https://docs.readthedocs.io/en/stable/', None),
'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
}
-html_theme = "sphinx_rtd_theme"
+html_theme = "sphinx_immaterial"
html_baseurl = "https://databricks-solutions.github.io/lakeflow_framework/"
+html_logo = "source/_static/lff-logo.png"
+html_favicon = "source/_static/lff-logo.png"
-# Inject the version switcher into the sidebar.
-html_sidebars = {
- '**': [
- 'versions.html',
- 'globaltoc.html',
- 'relations.html',
- 'sourcelink.html',
- 'searchbox.html',
+html_theme_options = {
+ "site_url": html_baseurl,
+ "repo_url": "https://github.com/databricks-solutions/lakeflow_framework",
+ "repo_name": "lakeflow_framework",
+ "icon": {
+ "repo": "fontawesome/brands/github",
+ },
+ "features": [
+ "navigation.instant",
+ "navigation.top",
+ "navigation.tabs",
+ "navigation.tabs.sticky",
+ "toc.integrate",
+ "toc.follow",
+ "search.suggest",
+ "content.code.copy",
],
+ "version_dropdown": True,
+ "version_json": "versions.json",
+ "palette": [
+ {
+ "media": "(prefers-color-scheme: light)",
+ "scheme": "default",
+ "primary": "red",
+ "accent": "blue-grey",
+ "toggle": {
+ "icon": "material/brightness-7",
+ "name": "Switch to dark mode",
+ },
+ },
+ {
+ "media": "(prefers-color-scheme: dark)",
+ "scheme": "slate",
+ "primary": "red",
+ "accent": "blue-grey",
+ "toggle": {
+ "icon": "material/brightness-4",
+ "name": "Switch to light mode",
+ },
+ },
+ ],
+ "font": {
+ "text": "Roboto",
+ "code": "Roboto Mono",
+ },
}
html_static_path = ['source/_static']
html_css_files = [
+ 'databricks-theme.css',
'custom.css',
]
-# Suppress generic Sphinx "Last updated" text; custom version metadata is shown.
+# Suppress generic Sphinx "Last updated" text.
html_last_updated_fmt = None
+html_context = {
+ "docs_current_version": os.environ.get("DOCS_CURRENT_VERSION", "current"),
+}
+
-def _head_release_date() -> str:
- try:
- return subprocess.run(
- ["git", "log", "-1", "--format=%cs"],
- cwd=_here,
- capture_output=True,
- text=True,
- check=True,
- ).stdout.strip()
- except Exception:
- return ""
-
-
-def _format_release_date(date_str: str) -> str:
- if not date_str:
- return ""
- try:
- dt = datetime.strptime(date_str, "%Y-%m-%d")
- return dt.strftime("%B %d, %Y")
- except ValueError:
- return date_str
-
-
-def _fallback_versions() -> list[dict[str, str]]:
- head_release_date = _head_release_date()
- return [
- {
- "name": "current",
- "display_version": release,
- "url": "../current/index.html",
- "is_latest": True,
- "status": "current",
- "release_date": head_release_date,
- "release_date_human": _format_release_date(head_release_date),
- }
+def _patch_landing_nav(app, pagename, templatename, context, doctree):
+ """Use index.html as the Home landing page; WAF-style section nav."""
+ nav = context.get("nav")
+ if nav is None:
+ return
+
+ from sphinx_immaterial.nav_adapt import MkdocsNavEntry
+
+ pathto = context["pathto"]
+ master = app.config.master_doc
+ home_url = pathto(master)
+ is_home = pagename == master
+
+ # Section index pages: non-selectable sidebar title, tab lands on first child.
+ section_index_pages = frozenset({
+ "features",
+ "deploy_framework",
+ "build_pipeline_bundle",
+ "dataflow_spec_reference",
+ "contributor",
+ })
+
+ def _first_leaf(entry):
+ current = entry
+ while current.children:
+ current = current.children[0]
+ return current
+
+ def _docname_from_url(url):
+ if not url or url == "#":
+ return None
+ path = url.split("#", 1)[0]
+ if path.endswith(".html"):
+ return path[:-5]
+ return path
+
+ def apply_section_captions(entries):
+ for entry in entries:
+ if entry.children:
+ apply_section_captions(entry.children)
+ docname = _docname_from_url(entry.url)
+ if docname in section_index_pages and entry.children:
+ leaf = _first_leaf(entry)
+ if leaf.url:
+ entry.caption_only = True
+ entry.url = leaf.url
+
+ apply_section_captions(nav)
+
+ def rewrite_home_urls(entries):
+ for entry in entries:
+ if entry.url:
+ path = entry.url.split("#", 1)[0]
+ if path == "home.html" or path.endswith("/home.html"):
+ fragment = entry.url[len(path):]
+ entry.url = home_url + fragment
+ rewrite_home_urls(entry.children)
+
+ rewrite_home_urls(nav)
+
+ nav[:] = [
+ entry
+ for entry in nav
+ if not (entry.url and entry.url.split("#", 1)[0] == home_url and "Home" in entry.title)
]
+ nav.insert(
+ 0,
+ MkdocsNavEntry(
+ title_text="Home",
+ url=home_url,
+ children=[],
+ active=is_home,
+ current=is_home,
+ active_or_section_within_active=is_home,
+ caption_only=False,
+ ),
+ )
-def _load_versions() -> list[dict[str, str]]:
- versions_file = os.environ.get("DOCS_VERSIONS_FILE")
- if not versions_file:
- default_versions_file = os.path.join(_here, "build", "html", "versions.json")
- if os.path.exists(default_versions_file):
- versions_file = default_versions_file
-
- if not versions_file:
- return _fallback_versions()
-
- try:
- with open(versions_file, encoding="utf-8") as f:
- versions = json.load(f)
- except FileNotFoundError:
- return _fallback_versions()
-
- # For per-version pages, switch from site-root links (e.g. "v1.2.3/") to
- # sibling paths (e.g. "../v1.2.3/"), which work under GitHub project pages.
- adapted = []
- for item in versions:
- adapted.append(
- {
- "name": item["name"],
- "display_version": item.get("display_version", item["name"].lstrip("v")),
- "url": f"../{item['name']}/index.html",
- "is_latest": item.get("is_latest", False),
- "status": item.get("status", "release"),
- "release_date": item.get("release_date", ""),
- "release_date_human": _format_release_date(item.get("release_date", "")),
- }
- )
- return adapted
-
-
-def _current_version_meta(versions: list[dict[str, str]], current: str) -> dict[str, str]:
- for item in versions:
- if item["name"] == current:
- return item
- return {
- "name": current,
- "display_version": current.lstrip("v"),
- "url": f"../{current}/index.html",
- "is_latest": current == "current",
- "status": "current" if current == "current" else "release",
- "release_date": "",
- "release_date_human": "",
- }
-
-
-_docs_current_version = os.environ.get("DOCS_CURRENT_VERSION", "current")
-_docs_versions = _load_versions()
+ if is_home:
+ for entry in nav[1:]:
+ entry.active = False
+ entry.current = False
+ entry.active_or_section_within_active = False
+
+ page = context.get("page")
+ if page is not None:
+ if "hide" not in page["meta"]:
+ page["meta"]["hide"] = []
+ page["meta"]["hide"].append("navigation")
+ page["meta"]["hide"].append("toc")
+
+ context["nav"] = nav
+
+
+def _enable_codeblock_linenos(app, doctree):
+ """Show line numbers on syntax-highlighted code blocks (except command snippets)."""
+ from docutils import nodes
+
+ for node in doctree.findall(nodes.literal_block):
+ classes = list(node.get('classes', []))
+ if 'lf-command-block' in classes:
+ continue
+ if node.rawsource == node.astext():
+ node['linenos'] = True
+
+
+def _tag_command_blocks(app, doctree):
+ """Quick Start console fences → light WAF-style command blocks with copy button."""
+ from docutils import nodes
+
+ if app.env.docname != 'quick_start':
+ return
+
+ for node in doctree.findall(nodes.literal_block):
+ language = node.get('language', '')
+ if language not in ('console', 'bash', 'shell', 'text', ''):
+ continue
+ classes = list(node.get('classes', []))
+ if 'lf-command-block' not in classes:
+ classes.append('lf-command-block')
+ node['classes'] = classes
+ node.attributes.pop('linenos', None)
+
+
+def _table_column_count(table) -> int:
+ from docutils import nodes
+
+ for tgroup in table.findall(nodes.tgroup):
+ cols = tgroup.get('cols')
+ if cols:
+ return int(cols)
+ return 0
+
+
+def _is_page_metadata_table(table) -> bool:
+ """Feature-page metadata strip: 2 cols, Applies To / Configuration Scope rows."""
+ from docutils import nodes
+
+ if _table_column_count(table) != 2:
+ return False
+
+ rows = list(table.findall(nodes.row))
+ if not rows:
+ return False
+
+ entries = list(rows[0].findall(nodes.entry))
+ if not entries:
+ return False
+
+ first_cell = entries[0].astext()
+ return (
+ 'Applies To' in first_cell
+ or 'Configuration Scope' in first_cell
+ or 'Databricks Docs' in first_cell
+ )
+
+
+def _mark_content_tables(app, doctree, docname=None):
+ """Tag body list-tables for theme CSS (excludes h1 metadata strips)."""
+ from docutils import nodes
+
+ for table in doctree.findall(nodes.table):
+ if _is_page_metadata_table(table):
+ continue
+
+ classes = list(table.get('classes', []))
+ if 'lf-content-table' not in classes:
+ classes.append('lf-content-table')
+ if 'data' not in classes:
+ classes.append('data')
+
+ col_count = _table_column_count(table)
+ col_class = f'lf-table-cols-{col_count}'
+ if 2 <= col_count <= 6 and col_class not in classes:
+ classes.append(col_class)
+
+ table['classes'] = classes
-html_context = {
- "docs_current_version": _docs_current_version,
- "docs_versions": _docs_versions,
- "docs_current_version_meta": _current_version_meta(_docs_versions, _docs_current_version),
-}
def setup(app):
app.set_translator("markdown", CustomMarkdownTranslator)
- app.add_css_file('custom.css')
+
+ def _init_domain_synopses(app):
+ for domain_name in ("py", "std"):
+ domain = app.env.get_domain(domain_name)
+ if "synopses" not in domain.data:
+ domain.data["synopses"] = {}
+
+ app.connect("builder-inited", _init_domain_synopses)
+ app.connect("doctree-read", _tag_command_blocks)
+ app.connect("doctree-read", _enable_codeblock_linenos)
+ app.connect("doctree-read", _mark_content_tables)
+ app.connect("html-page-context", _patch_landing_nav, priority=999)
#app.add_builder(MarkdownBuilder)
diff --git a/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md b/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md
index 13363cd..e4dfc9a 100644
--- a/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md
+++ b/docs/decisions/0007-scripted-versioned-docs-and-ui-scope.md
@@ -32,16 +32,18 @@ The pipeline builds:
- `main` as `current`
- selected release tags
+- optional local branches with `--preview`
- `docs/build/html/versions.json`
- root redirect `docs/build/html/index.html -> current/index.html`
-### 2) Keep selector UI scope minimal and theme-aligned
+Each version is built from that ref's own `docs/conf.py` + `docs/source` (worktree). Historical RTD docs keep RTD; rebranded refs use immaterial.
-- Keep the RTD selector at the bottom (`docs/_templates/versions.html`).
-- Keep breadcrumb version/release-date metadata (`docs/_templates/breadcrumbs.html`).
-- Do **not** keep a second custom header selector path.
+### 2) Shared superset `versions.json` + theme-native selectors
-This limits ongoing CSS/template overrides and reduces regressions from theme changes.
+- Write one `versions.json` with both mike fields (`version`, `title`, `aliases`) and legacy RTD fields (`name`, `display_version`, `url`, …).
+- RTD refs consume it at build time via `DOCS_VERSIONS_FILE` / `versions.html`.
+- Immaterial refs enable `version_dropdown` and load the same file over HTTP.
+- Do **not** force the building checkout's conf onto every historical version.
### 3) Place docs tooling under `docs/scripts/`
diff --git a/docs/scripts/build_versioned_docs.py b/docs/scripts/build_versioned_docs.py
index dc57dac..fe712cb 100755
--- a/docs/scripts/build_versioned_docs.py
+++ b/docs/scripts/build_versioned_docs.py
@@ -2,10 +2,14 @@
"""Build versioned docs without sphinx-multiversion.
Build strategy:
-- Always build `main` as the default development docs, published as `current`.
+- Always build ``main`` as ``current``.
- Build selected release tags from docs/scripts/select_versions.py.
+- Each version is built with **that ref's own** ``docs/conf.py`` and ``docs/source``
+ (so historical RTD docs stay RTD; rebranded branches use immaterial).
+- ``--preview`` also builds local branches into the version menu.
- Write per-version output under docs/build/html//.
-- Generate docs/build/html/versions.json for the sidebar switcher.
+- Generate docs/build/html/versions.json as a **superset** manifest consumed by
+ both sphinx-immaterial (mike fields) and legacy RTD ``versions.html``.
"""
from __future__ import annotations
@@ -40,6 +44,7 @@ def _selected_tags(repo_root: Path) -> list[str]:
def _list_preview_branches(repo_root: Path) -> list[str]:
+ """Local branches to include in preview builds (excludes main; current is main)."""
raw = _run_capture(
["git", "for-each-ref", "--format=%(refname:short)", "refs/heads"],
cwd=repo_root,
@@ -72,15 +77,48 @@ def _version_for_ref(repo_root: Path, ref: str, fallback: str) -> str:
return fallback
+def _superset_versions(links: list[dict[str, str]]) -> list[dict[str, object]]:
+ """Build a versions.json entry set readable by immaterial and legacy RTD.
+
+ Immaterial / mike fields: ``version``, ``title``, ``aliases``
+ Legacy RTD fields: ``name``, ``display_version``, ``url``, ``is_latest``,
+ ``status``, ``release_date``
+ """
+ payload: list[dict[str, object]] = []
+ for item in links:
+ name = item["name"]
+ display = item["display_version"]
+ is_current = item.get("status") == "current"
+ title = f"{display} (current)" if is_current else display
+ aliases: list[str] = ["latest"] if is_current else []
+ payload.append(
+ {
+ # mike / sphinx-immaterial
+ "version": name,
+ "title": title,
+ "aliases": aliases,
+ # legacy RTD versions.html (via conf.py _load_versions)
+ "name": name,
+ "display_version": display,
+ "url": f"{name}/index.html",
+ "is_latest": is_current,
+ "status": "current" if is_current else item.get("status", "release"),
+ "release_date": item.get("release_date", ""),
+ }
+ )
+ return payload
+
+
def _build_ref(
*,
repo_root: Path,
- docs_dir: Path,
worktrees_root: Path,
output_root: Path,
+ versions_file: Path,
ref: str,
version_name: str,
) -> None:
+ """Build from a git ref using that ref's own docs conf + source."""
worktree_path = worktrees_root / _normalize_name(version_name)
_run(["git", "worktree", "add", "--detach", str(worktree_path), ref], cwd=repo_root)
try:
@@ -88,15 +126,21 @@ def _build_ref(
out_dir.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env["DOCS_CURRENT_VERSION"] = version_name
- env["DOCS_VERSIONS_FILE"] = str(output_root / "versions.json")
+ # Legacy RTD conf.py loads the shared manifest at build time.
+ env["DOCS_VERSIONS_FILE"] = str(versions_file)
+ conf_dir = worktree_path / "docs"
+ source_dir = conf_dir / "source"
+ print(f"Building docs for {ref} -> {version_name}", file=sys.stderr)
_run(
[
- "sphinx-build",
+ sys.executable,
+ "-m",
+ "sphinx",
"-b",
"html",
"-c",
- str(docs_dir),
- str(worktree_path / "docs" / "source"),
+ str(conf_dir),
+ str(source_dir),
str(out_dir),
],
cwd=repo_root,
@@ -108,7 +152,14 @@ def _build_ref(
def main() -> None:
parser = argparse.ArgumentParser()
- parser.add_argument("--preview", action="store_true", help="Include local branches in build.")
+ parser.add_argument(
+ "--preview",
+ action="store_true",
+ help=(
+ "Also build local branches into the version menu. "
+ "``current`` remains ``main``; select a feature branch to preview its theme/IA."
+ ),
+ )
args = parser.parse_args()
docs_dir = Path(__file__).resolve().parent.parent
@@ -116,6 +167,7 @@ def main() -> None:
build_root = docs_dir / "build"
output_root = build_root / "html"
worktrees_root = build_root / ".worktrees"
+ versions_file = output_root / "versions.json"
_safe_remove(output_root)
_safe_remove(worktrees_root)
@@ -123,7 +175,8 @@ def main() -> None:
worktrees_root.mkdir(parents=True, exist_ok=True)
tags = _selected_tags(repo_root)
- versions = [{"name": "current", "ref": "main"}] + [{"name": t, "ref": t} for t in tags]
+ versions: list[dict[str, str]] = [{"name": "current", "ref": "main"}]
+ versions.extend({"name": t, "ref": t} for t in tags)
if args.preview:
for branch in _list_preview_branches(repo_root):
versions.append({"name": _normalize_name(branch), "ref": branch})
@@ -149,6 +202,7 @@ def main() -> None:
"is_latest": is_current,
"status": "current" if is_current else "release",
"release_date": _release_date_for_ref(repo_root, item["ref"]),
+ "ref": item["ref"],
}
)
@@ -162,15 +216,19 @@ def main() -> None:
if item["name"] == "current" or item["display_version"] != current_version
]
- (output_root / "versions.json").write_text(json.dumps(links, indent=2) + "\n", encoding="utf-8")
+ versions_file.write_text(
+ json.dumps(_superset_versions(links), indent=2) + "\n",
+ encoding="utf-8",
+ )
+ # Build every selected ref (including preview branches filtered out of the
+ # dropdown by the current-version de-dupe — still useful as named folders).
for item in deduped:
- print(f"Building docs for {item['ref']} -> {item['name']}", file=sys.stderr)
_build_ref(
repo_root=repo_root,
- docs_dir=docs_dir,
worktrees_root=worktrees_root,
output_root=output_root,
+ versions_file=versions_file,
ref=item["ref"],
version_name=item["name"],
)
diff --git a/docs/scripts/select_versions.py b/docs/scripts/select_versions.py
index 6a4ea33..03a5cd0 100755
--- a/docs/scripts/select_versions.py
+++ b/docs/scripts/select_versions.py
@@ -5,6 +5,8 @@
--------
* For the current release major (highest major present), include the latest
patch release for the last 5 minor series.
+* For major ``0``, include the latest patch for every minor from the current
+ minor down to ``0.12`` (inclusive), not capped at five minors.
* Also include the latest available release for each of the last 3 major
versions.
@@ -27,6 +29,9 @@
TAG_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
+# For 0.x releases, publish docs back to this minor (latest patch per minor).
+MIN_MINOR_MAJOR_0 = 12
+
def _get_tags() -> list[str]:
result = subprocess.run(
@@ -75,19 +80,27 @@ def select_versions(tags: list[str]) -> list[str]:
majors_desc = sorted(best_per_major.keys(), reverse=True)
current_major = majors_desc[0]
- # Last 5 minor series for current major (latest patch in each minor).
+ # Latest patch for each selected minor in the current major.
current_major_minors = sorted(
[minor for (major, minor) in best_per_minor if major == current_major],
reverse=True,
)
- current_major_last_5 = {
- best_per_minor[(current_major, minor)][1] for minor in current_major_minors[:5]
- }
+ if current_major == 0:
+ current_major_selected = {
+ best_per_minor[(0, minor)][1]
+ for minor in current_major_minors
+ if minor >= MIN_MINOR_MAJOR_0
+ }
+ else:
+ current_major_selected = {
+ best_per_minor[(current_major, minor)][1]
+ for minor in current_major_minors[:5]
+ }
# Last 3 major versions (latest release per major).
last_3_major_latest = {best_per_major[major][2] for major in majors_desc[:3]}
- return _sort_tags_desc(current_major_last_5 | last_3_major_latest)
+ return _sort_tags_desc(current_major_selected | last_3_major_latest)
def tags_to_regex(tags: list[str]) -> str:
diff --git a/docs/source/_landing.rst b/docs/source/_landing.rst
new file mode 100644
index 0000000..4b70685
--- /dev/null
+++ b/docs/source/_landing.rst
@@ -0,0 +1,78 @@
+.. raw:: html
+
+
+
The Lakeflow Framework
+
Metadata-driven pipelines for Databricks Lakeflow SDP
+
+ Configure once, deploy everywhere. The Lakeflow Framework accelerates medallion
+ pipelines with pattern-based data flow specs, DABs-native delivery, and
+ support for both centralized and domain-oriented operating models.
+
+
+Run one bundle. Get bronze, silver, and gold pipelines aligned to Databricks Lakeflow Spark Declarative Pipelines — with reusable patterns, data quality, CDC, and materialized views.
+
+New to the framework? Read :doc:`what_is_lakeflow_framework` for a product-level overview.
+
+.. raw:: html
+
+
+
+
+
+
Pattern-based design
+
+
+
Medallion, streaming, CDC, and gold-serving patterns out of the box.