docs: add versioned documentation (next, 1.6, 0.6)

Set up Docusaurus doc versioning with three versions:
- next (unreleased, from /docs directory)
- 1.6 (current release)
- 0.6 (last version of previous major)

Includes version dropdown in navbar, versioned API reference,
and automated version snapshotting in the release workflow.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: vdusek

.github/workflows/manual_release_stable.yaml

@@ -104,15 +104,83 @@ jobs:
       # TODO: add job for publish package to Conda
       # https://github.com/apify/crawlee-python/issues/104
 
+  version_docs:
+    name: Version docs
+    needs: [release_prepare, changelog_update, pypi_publish]
+    runs-on: ubuntu-latest
+    outputs:
+      version_docs_commitish: ${{ steps.commit_versioned_docs.outputs.commit_long_sha }}
+    permissions:
+      contents: write
+    env:
+      NODE_VERSION: 22
+      PYTHON_VERSION: 3.14
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Set up uv package manager
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Python dependencies
+        run: uv run poe install-dev
+
+      - name: Install website dependencies
+        run: |
+          cd website
+          corepack enable
+          yarn install
+
+      - name: Snapshot the current version
+        run: |
+          cd website
+          VERSION="$(python -c "import tomllib, pathlib; print(tomllib.loads(pathlib.Path('../pyproject.toml').read_text())['project']['version'])")"
+          MAJOR_MINOR="$(echo "$VERSION" | cut -d. -f1-2)"
+          export MAJOR_MINOR
+          # Remove existing version if present (patch releases override)
+          rm -rf "versioned_docs/version-${MAJOR_MINOR}"
+          rm -rf "versioned_sidebars/version-${MAJOR_MINOR}-sidebars.json"
+          jq 'map(select(. != env.MAJOR_MINOR))' versions.json > tmp.json && mv tmp.json versions.json
+          # Copy changelog
+          cp ../CHANGELOG.md ../docs/changelog.md
+          # Build API reference and create version snapshots
+          bash build_api_reference.sh
+          npx docusaurus docs:version "$MAJOR_MINOR"
+          npx docusaurus api:version "$MAJOR_MINOR"
+
+      - name: Commit and push versioned docs
+        id: commit_versioned_docs
+        uses: EndBug/add-and-commit@v10
+        with:
+          add: "website/versioned_docs website/versioned_sidebars website/versions.json"
+          message: "docs: version ${{ needs.release_prepare.outputs.version_number }} docs [skip ci]"
+          default_author: github_actions
+
   doc_release:
     name: Doc release
-    needs: [changelog_update, pypi_publish]
+    needs: [changelog_update, pypi_publish, version_docs]
     permissions:
       contents: write
       pages: write
       id-token: write
     uses: ./.github/workflows/_release_docs.yaml
     with:
-      # Use the ref from the changelog update to include the updated changelog.
-      ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
+      # Use the version_docs commit to include both changelog and versioned docs.
+      ref: ${{ needs.version_docs.outputs.version_docs_commitish }}
     secrets: inherit

pyproject.toml

@@ -140,6 +140,9 @@ packages = ["src/crawlee"]
 [tool.ruff]
 line-length = 120
 include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]
+exclude = [
+    "website/versioned_docs/**",
+]
 extend-exclude = ["src/crawlee/project_template"]
 
 [tool.ruff.lint]
@@ -251,6 +254,7 @@ include = ["src", "tests", "scripts", "docs", "website"]
 exclude = [
     "src/crawlee/project_template",
     "docs/guides/code_examples/storage_clients/custom_storage_client_example.py",
+    "website/versioned_docs",
 ]
 
 [[tool.ty.overrides]]

website/docusaurus.config.js

@@ -94,7 +94,6 @@ module.exports = {
                     path: '../docs',
                     sidebarPath: './sidebars.js',
                     rehypePlugins: [externalLinkProcessor],
-                    // disableVersioning: true,
                     editUrl: (doc) => {
                         return `https://github.com/apify/crawlee-python/edit/master/website/${doc.versionDocsDirPath}/${doc.docPath}`;
                     },
@@ -118,6 +117,7 @@ module.exports = {
                 },
                 sortSidebar: groupSort,
                 routeBasePath: 'api',
+                python: true,
                 pythonOptions: {
                     pythonModulePath: path.join(__dirname, '../src/crawlee'),
                     moduleShortcutsPath: path.join(__dirname, 'module_shortcuts.json'),
@@ -172,6 +172,12 @@ module.exports = {
                     includeVersionedDocs: false,
                     enableLlmsFullTxt: true,
                     relativePaths: false,
+                    excludeRoutes: [
+                        '/python/api/[0-9]*/**',
+                        '/python/api/[0-9]*',
+                        '/python/api/next/**',
+                        '/python/api/next',
+                    ],
                 },
             },
         ],
@@ -281,6 +287,12 @@ module.exports = {
                     label: 'Blog',
                     position: 'left',
                 },
+                {
+                    type: 'docsVersionDropdown',
+                    position: 'right',
+                    dropdownItemsBefore: [],
+                    dropdownItemsAfter: [],
+                },
             ],
         },
         colorMode: {

website/versioned_docs/version-0.6/api-packages.json

@@ -0,0 +1 @@
+[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"crawlee"}]