Set up Python monorepo with uv workspaces; migrate kolibri-sync-extras-plugin

.github/workflows/npm_publish.yml

@@ -79,7 +79,7 @@ jobs:
           run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           {
             echo "text<<SLACKEOF"
-            echo ":package: *npm packages published* (<${run_url}|workflow run>)"
+            echo ":package: *npm package published* (<${run_url}|workflow run>)"
             sed -n 's#^| *\[\([^]]*\)\](\([^)]*\)) *| *\([^ |]*\) *|$#• <\2|\1> v\3#p' publish_summary.md
             echo "SLACKEOF"
           } >> "$GITHUB_OUTPUT"

.github/workflows/pypi_packages_publish.yml

@@ -0,0 +1,82 @@
+name: Publish packages to PyPI
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'python_packages/kolibri-sync-extras-plugin/pyproject.toml'
+  workflow_dispatch:
+    inputs:
+      pypi_package:
+        description: 'Package to publish (or "all" to auto-detect)'
+        required: true
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - 'kolibri-sync-extras-plugin'
+      target:
+        description: 'Index to publish to'
+        required: true
+        type: choice
+        default: 'pypi'
+        options:
+          - 'pypi'
+          - 'testpypi'
+permissions:
+  contents: read
+  id-token: write
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      PYPI_TARGET: ${{ github.event.inputs.target }}
+    permissions:
+      contents: read
+      id-token: write # Required for PyPI OIDC trusted publishing
+    steps:
+      - uses: actions/checkout@v7.0.0
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v8.2.0
+        with:
+          enable-cache: true
+          cache-python: true
+      - name: Install Python
+        run: uv python install 3.10
+      - name: Publish
+        id: publish
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.pypi_package }}" != "all" ]; then
+            ./scripts/pypi_publish.sh "${{ github.event.inputs.pypi_package }}"
+          else
+            ./scripts/pypi_publish.sh
+          fi
+          if [ -f publish_summary.md ]; then
+            cat publish_summary.md >> "$GITHUB_STEP_SUMMARY"
+            echo "published=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No packages needed publishing — all versions match the target index." >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Build Slack message
+        if: steps.publish.outputs.published == 'true' && env.PYPI_TARGET != 'testpypi'
+        id: slack_message
+        run: |
+          run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          {
+            echo "text<<SLACKEOF"
+            echo ":package: *PyPI package published* (<${run_url}|workflow run>)"
+            sed -n 's#^| *\[\([^]]*\)\](\([^)]*\)) *| *\([^ |]*\) *|$#• <\2|\1> v\3#p' publish_summary.md
+            echo "SLACKEOF"
+          } >> "$GITHUB_OUTPUT"
+      - name: Notify Slack
+        if: steps.publish.outputs.published == 'true' && env.PYPI_TARGET != 'testpypi'
+        uses: slackapi/slack-github-action@v3
+        with:
+          # Intentionally shared with npm_publish.yml — one Slack channel for
+          # all package-publish notices; the message text disambiguates them.
+          webhook: ${{ secrets.SLACK_NPM_PACKAGES_WEBHOOK }}
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": ${{ toJSON(steps.slack_message.outputs.text) }}
+            }

.github/workflows/tox.yml

@@ -26,16 +26,42 @@ jobs:
         uses: fkirc/skip-duplicate-actions@master
         with:
           github_token: ${{ github.token }}
-          paths: '["**.py", ".github/workflows/tox.yml", "pyproject.toml", "uv.lock"]'
-  unit_test:
-    name: Python unit tests for uv-supported Python versions
+          paths: '["**.py", ".github/workflows/tox.yml", "pyproject.toml", "uv.lock", "python_packages/**"]'
+  unit_test_stage1:
+    name: Python unit tests (Python 3.10)
     needs: pre_job
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 0
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v8.2.0
+        with:
+          enable-cache: true
+          cache-python: true
+      - name: Install Python 3.10
+        run: uv python install 3.10
+      - name: Install dependencies
+        run: uv sync --group test --python 3.10
+      - name: Check migrations
+        run: uv run kolibri manage makemigrations --check
+        env:
+          KOLIBRI_HOME: ${{ runner.temp }}/.kolibri
+      - name: Run tests
+        run: |
+          uv run python -O -m pytest kolibri --color=no
+          uv run python -O -m pytest --color=no -p no:django test
+  unit_test_stage2:
+    name: Python unit tests for uv-supported Python versions
+    needs: [pre_job, stage1_required_checks]
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' && needs.stage1_required_checks.result == 'success' }}
+    runs-on: ubuntu-latest
     strategy:
       max-parallel: 5
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
+        python-version: ['3.8', '3.9', '3.11', '3.12', '3.13', '3.14']
     steps:
       - uses: actions/checkout@v7.0.0
         with:
@@ -59,8 +85,8 @@ jobs:
           uv run python -O -m pytest --color=no -p no:django test
   unit_test_no_uv:
     name: Python unit tests for Python versions without uv support
-    needs: pre_job
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: [pre_job, stage1_required_checks]
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' && needs.stage1_required_checks.result == 'success' }}
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 5
@@ -153,8 +179,8 @@ jobs:
           KOLIBRI_DATABASE_PORT: "5432"
   postgres_ssl:
     name: Python postgres SSL smoke tests
-    needs: pre_job
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: [pre_job, stage1_required_checks]
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' && needs.stage1_required_checks.result == 'success' }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -197,8 +223,8 @@ jobs:
           KOLIBRI_DATABASE_SSL_MODE: require
   macos:
     name: Python unit tests on Mac OS
-    needs: pre_job
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: [pre_job, stage1_required_checks]
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' && needs.stage1_required_checks.result == 'success' }}
     runs-on: macos-14
     strategy:
       max-parallel: 5
@@ -223,8 +249,8 @@ jobs:
           uv run python -O -m pytest --color=no -p no:django test
   windows:
     name: Python unit tests on Windows Server
-    needs: pre_job
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: [pre_job, stage1_required_checks]
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' && needs.stage1_required_checks.result == 'success' }}
     runs-on: windows-latest
     strategy:
       max-parallel: 5
@@ -249,19 +275,65 @@ jobs:
         run: |
           uv run python -O -m pytest kolibri --color=no
           uv run python -O -m pytest --color=no -p no:django test
+  sync_extras_plugin_tests:
+    name: kolibri-sync-extras-plugin tests
+    needs: [pre_job, stage1_required_checks]
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' && needs.stage1_required_checks.result == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7.0.0
+        with:
+          fetch-depth: 0
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v8.2.0
+        with:
+          enable-cache: true
+          cache-python: true
+      - name: Install Python 3.10
+        run: uv python install 3.10
+      - name: Install dependencies
+        # --all-packages is required: a plain `uv sync` scopes to this
+        # subproject alone and drops root kolibri's own runtime deps
+        # (Django, Click, etc.) from the shared venv.
+        working-directory: python_packages/kolibri-sync-extras-plugin
+        run: uv sync --group test --python 3.10 --all-packages
+      - name: Enable kolibri_sync_extras_plugin
+        working-directory: python_packages/kolibri-sync-extras-plugin
+        env:
+          KOLIBRI_HOME: .kolibri
+        run: uv run kolibri plugin enable kolibri_sync_extras_plugin
+      - name: Run tests
+        working-directory: python_packages/kolibri-sync-extras-plugin
+        env:
+          KOLIBRI_HOME: .kolibri
+        run: uv run python -O -m pytest --reuse-db
   # Single stable check for branch protection. Skipped matrix jobs don't
   # produce per-version check runs, so matrix-suffixed required checks hang
-  # as "Expected" on PRs that skip the tests; require this job instead.
-  required_checks:
-    name: Python tests
+  # as "Expected" on PRs that skip the tests; require these jobs instead.
+  # Stage 1: fast, blocking feedback.
+  stage1_required_checks:
+    name: Python tests (stage 1)
     needs:
       - pre_job
-      - unit_test
-      - unit_test_no_uv
+      - unit_test_stage1
       - postgres
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if any needed job failed or was cancelled
+        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: exit 1
+  # Stage 2: rest of the Python version matrix plus per-member-package tests.
+  stage2_required_checks:
+    name: Python tests
+    needs:
+      - stage1_required_checks
+      - unit_test_stage2
+      - unit_test_no_uv
       - postgres_ssl
       - macos
       - windows
+      - sync_extras_plugin_tests
     if: always()
     runs-on: ubuntu-latest
     steps:

docs/development_workflow.rst

@@ -95,6 +95,9 @@ First, all automated checks need to pass before merging. Then...
 * The reviewer might approve the PR, but also request minor changes such as a typo fix or variable name update. The submitter can then make the change and merge it themselves, with the understanding that the new changes will be limited in scope
 * Stale reviews should be dismissed by the PR submitter when the feedback has been addressed
 
+.. note::
+  Python tests run in a two-stage cascade (Stage 1: Python 3.10 + Postgres required before Stage 2). See :doc:`/howtos/python_monorepo`.
+
 
 Copyright and licensing
 ~~~~~~~~~~~~~~~~~~~~~~~

docs/howtos/index.rst

@@ -20,3 +20,4 @@ These guides are step by step guides for common tasks in getting started and wor
   working_with_urls_and_api_endpoints
   dev_data_setup
   multi_agent_setup
+  python_monorepo

docs/howtos/python_monorepo.md

@@ -0,0 +1,36 @@
+# Adding a Python member package
+
+Kolibri's Python code is organized as a [uv workspace](https://docs.astral.sh/uv/concepts/projects/workspaces/). The main `kolibri` package is the workspace root; additional packages live under `python_packages/`.
+
+## Adding a new package
+
+1. Create `python_packages/<package-name>/` with its own `pyproject.toml` — a normal, independently-buildable Python package (own `[build-system]`, own static `version`, own dependencies).
+2. If it depends on `kolibri` itself, resolve that dependency against this workspace instead of PyPI:
+   ```toml
+   [tool.uv.sources]
+   kolibri = { workspace = true }
+   ```
+3. No change is needed to the root `pyproject.toml` — its `[tool.uv.workspace]` `members` list already includes the glob `python_packages/*`, so any new package directory under it is picked up automatically.
+4. Run `uv sync --group dev` at the repo root to update the shared lockfile.
+5. Run `uv sync --group dev --all-packages` to install the new package into the shared workspace venv. `--all-packages` is required — a plain `uv sync` (or `uv sync --package <package-name>`) scopes the sync to a single project and drops root Kolibri's own runtime dependencies (Django, Click, etc.) from the shared venv.
+6. Add a test job for it in `.github/workflows/tox.yml`, in the Stage 2 section (see "CI cascade" below) — model it on the `sync_extras_plugin_tests` job, and add the new job's id to `stage2_required_checks`'s `needs:` list in the same file. A job left out of that list can fail without blocking merge, since branch protection only requires `stage2_required_checks` itself to pass.
+7. Add the package's import name to `known-first-party` in root `pyproject.toml`'s `[tool.ruff.lint.isort]` table, so ruff sorts its own imports as first-party rather than third-party.
+
+Member package versions are independent of each other and of the main `kolibri` package — there's no enforcement linking them. Use a static `version = "x.y.z"` field, not `setuptools-scm`-derived dynamic versioning: this repo's git tags are Kolibri's own release tags, so dynamic versioning inside the workspace would report Kolibri's version instead of the package's own.
+
+## Marking a package as publishable
+
+By default, a package under `python_packages/` is workspace-only — nothing publishes it. To publish it to PyPI:
+
+1. Add its `pyproject.toml` path to the `paths:` filter in `.github/workflows/pypi_packages_publish.yml`'s `push` trigger.
+2. Add its name to the `workflow_dispatch.inputs.pypi_package.options` list in the same file.
+3. Register a pending trusted publisher on PyPI and TestPyPI (see the "Python packages" section of [the release process docs](../release_process.rst)) before merging.
+
+## CI cascade
+
+Python tests run in two stages (`.github/workflows/tox.yml`):
+
+- **Stage 1** (blocking, fast feedback): core Kolibri tests on Python 3.10, and Postgres tests. Runs in parallel.
+- **Stage 2** (gated on Stage 1 via `needs:`): the rest of the Python version matrix, plus a test job per publishable member package. Acts as a broader safety net — it rarely fails once Stage 1 passes, but still gates merge.
+
+Both stages are required checks in branch protection. Lint, wheel build, and JS tests are unaffected — they run in their own workflows, in parallel with Stage 1.

docs/release_process.rst

@@ -57,3 +57,32 @@ Every publish includes an SLSA provenance attestation linking the npm version to
 .. code-block:: bash
 
    ./scripts/npm_provenance.sh <package-name> [version]
+
+
+Python packages
+===============
+
+Packages in ``python_packages/`` are published to PyPI independently of Kolibri releases and independently of each other. Only publishable packages are published; everything else is workspace-only. A package is publishable if it's listed in ``pypi_packages_publish.yml``'s ``paths:`` filter and ``workflow_dispatch`` options.
+
+Automatic publishing
+--------------------
+
+When code merging to ``develop`` changes a listed package's ``pyproject.toml``, the ``pypi_packages_publish.yml`` workflow compares that package's version against PyPI and publishes it if newer.
+
+Authentication uses PyPI OIDC trusted publishing (no API tokens).
+
+The workflow can also be triggered manually from the Actions tab — either for a specific package or for all packages — and can target TestPyPI instead of PyPI via the ``target`` input.
+
+Bumping a version
+-----------------
+
+.. code-block:: bash
+
+   uv version --package <package-name> --bump patch
+
+Or set an exact version: ``uv version --package <package-name> <new-version>``. Commit the resulting ``pyproject.toml``/``uv.lock`` change and merge to ``develop``.
+
+First publish of a new package
+------------------------------
+
+PyPI and TestPyPI support registering a *pending* trusted publisher for a project that doesn't exist yet. Before merging the PR that adds the package, register a pending publisher on both `pypi.org <https://pypi.org/manage/account/publishing/>`__ and `test.pypi.org <https://test.pypi.org/manage/account/publishing/>`__ for the new project name, with owner ``learningequality``, repository ``kolibri``, and workflow ``pypi_packages_publish.yml``.

pyproject.toml

@@ -148,7 +148,7 @@ exclude-newer = "7 days"
 exclude-newer-package = {le-utils = "0 seconds", morango = "0 seconds"}
 
 [tool.uv.workspace]
-members = ["."]
+members = [".", "python_packages/*"]
 
 [tool.setuptools]
 packages = ["kolibri"]
@@ -182,7 +182,7 @@ max-line-length = 160
 max-complexity = 10
 
 [tool.ruff.lint.isort]
-known-first-party = ["kolibri"]
+known-first-party = ["kolibri", "kolibri_sync_extras_plugin"]
 # Match the prior reorder-python-imports style: one import per line,
 # sorted case-insensitively rather than grouped by type.
 force-single-line = true

python_packages/README.md

@@ -0,0 +1,8 @@
+# python_packages/
+
+Member packages of the Kolibri Python monorepo, managed as a uv workspace.
+
+Each subdirectory is an independently-versioned, independently-publishable Python
+package with its own `pyproject.toml`. See
+[docs/howtos/python_monorepo.md](../docs/howtos/python_monorepo.md) for how to add
+one.

python_packages/kolibri-sync-extras-plugin/.gitignore

@@ -0,0 +1,4 @@
+# Kolibri's root .gitignore already covers standard Python/IDE artifacts;
+# this file only needs entries specific to this package.
+.kolibri/*
+!.kolibri/options.ini

python_packages/kolibri-sync-extras-plugin/.kolibri/options.ini

@@ -0,0 +1,3 @@
+[Sync]
+BACKGROUND_INITIALIZATION = True
+BACKGROUND_FINALIZATION = True

python_packages/kolibri-sync-extras-plugin/AUTHORS.rst

@@ -0,0 +1,15 @@
+
+Credits
+=======
+
+Development Lead and Copyright Holder
+-------------------------------------
+
+* Learning Equality – info@learningequality.org
+
+Community
+---------
+
+Please feel free to add your name on this list if you do a PR!
+
+* Blaine Jester