Merge pull request #112 from rtibblesbot/issue-108-b5f1c0

Add --dry-run flag and workflow_dispatch trigger for Launchpad script

Author: rtibbles

.github/workflows/build_debian.yml

@@ -2,6 +2,7 @@ name: Build Debian source package
 on:
   release:
     types: [published]
+  workflow_dispatch:
 jobs:
   check_version:
     runs-on: ubuntu-latest
@@ -10,17 +11,22 @@ jobs:
     steps:
       - name: Checkout codebase
         uses: actions/checkout@v4
-      - name: Extract version from release tag
-        id: version
-        run: |
-          VERSION=${GITHUB_REF#refs/tags/v}
-          echo "VERSION=${VERSION}" >> "$GITHUB_OUTPUT"
       - name: Extract version from changelog
         id: changelog_version
         run: |
           CHANGELOG_VERSION=$(dpkg-parsechangelog -S Version)
           echo "CHANGELOG_VERSION=${CHANGELOG_VERSION}" >> "$GITHUB_OUTPUT"
+      - name: Extract version from release tag
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "VERSION=${{ steps.changelog_version.outputs.CHANGELOG_VERSION }}" >> "$GITHUB_OUTPUT"
+          else
+            TAG_VERSION=${GITHUB_REF#refs/tags/v}
+            echo "VERSION=${TAG_VERSION}" >> "$GITHUB_OUTPUT"
+          fi
       - name: Validate Version consistency
+        if: ${{ github.event_name != 'workflow_dispatch' }}
         id: version_check
         env:
           VERSION: ${{ steps.version.outputs.VERSION }}
@@ -32,6 +38,7 @@ jobs:
           fi
           echo "Version check passed: ${VERSION}"
   build_package:
+    if: ${{ github.event_name != 'workflow_dispatch' }}
     runs-on: ubuntu-latest
     needs: check_version
     steps:
@@ -53,6 +60,7 @@ jobs:
           make sign-and-upload
           echo "upload completed successfully!"
   wait_for_source_builds:
+    if: ${{ github.event_name != 'workflow_dispatch' }}
     needs:
       - check_version
       - build_package
@@ -80,7 +88,13 @@ jobs:
         if: always()
         run: rm -f /tmp/lp-creds.txt
   copy_to_other_distributions:
-    needs: wait_for_source_builds
+    needs:
+      - check_version
+      - wait_for_source_builds
+    if: |
+      always() &&
+      needs.check_version.result == 'success' &&
+      (needs.wait_for_source_builds.result == 'success' || needs.wait_for_source_builds.result == 'skipped')
     runs-on: ubuntu-latest
     steps:
       - name: Checkout codebase
@@ -130,17 +144,25 @@ jobs:
         if: always()
         run: rm -f /tmp/lp-creds.txt
   block_release_step:
-    if: ${{ !github.event.release.prerelease }}
+    if: ${{ !github.event.release.prerelease && github.event_name != 'workflow_dispatch' }}
     name: Job to block publish of a release until it has been manually approved
     needs: wait_for_copy_builds
     runs-on: ubuntu-latest
     environment: release
     steps:
       - run: echo "Release approved — proceeding to promote to kolibri PPA."
   copy_package_from_proposed_to_ppa:
-    if: ${{ !github.event.release.prerelease }}
+    if: |
+      always() &&
+      needs.wait_for_copy_builds.result == 'success' &&
+      (
+        (!github.event.release.prerelease && needs.block_release_step.result == 'success') ||
+        (github.event_name == 'workflow_dispatch')
+      )
     name: Promote packages from kolibri-proposed to kolibri
-    needs: block_release_step
+    needs:
+      - wait_for_copy_builds
+      - block_release_step
     runs-on: ubuntu-latest
     steps:
       - name: Checkout codebase

README.rst

@@ -21,7 +21,7 @@ To fetch and build a new version of this package, the following workflow is sugg
 
 You can optimize this workflow according to your own needs.
 
-Changes can be built and released in ``kolibri-proposed`` by the `Learning Equality Launchpad team <https://launchpad.net/~learningequality/>`__.
+Changes can be built and released in ``kolibri-proposed`` by the `Learning Equality Launchpad team <https://launchpad.net/~learningequality>`__.
 
 Working in the repo
 -------------------
@@ -53,13 +53,82 @@ After this, pre-commit hooks will run automatically on ``git commit``. To run al
 Releasing
 ---------
 
-Push new changes to ``kolibri-proposed`` and test them there.
+Automated release workflow
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To build packages for all current Ubuntu release series:
+Publishing a GitHub release triggers the ``build_debian.yml`` workflow, which:
 
-#. Install Launchpadlib: ``sudo apt install python-launchpadlib``
-#. Run ``ppa-copy-packages.py`` script to copy the builds for Xenial to all other currently active and supported Ubuntu releases on Launchpad. The script is run from command line with ``python2 ppa-copy-packages.py``. After this, you should be prompted to create an API key for your Launchpad account.
-#. When a release in ``kolibri-proposed`` should be released as a stable release, use the binary copy function on Launchpad to copy builds from ``kolibri-proposed``.
+#. Validates the release tag version against ``debian/changelog``
+#. Builds, signs, and uploads the source package to the ``kolibri-proposed`` PPA via ``dput``
+#. Waits for Launchpad to build the source package
+#. Copies the built package to all supported Ubuntu series
+#. Waits for all copy builds to complete
+#. (Non-prerelease only) Requires manual approval via the ``release`` environment
+#. Promotes packages from ``kolibri-proposed`` to ``kolibri`` PPA
+
+Launchpad credentials setup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The workflow requires Launchpad API credentials stored as a GitHub Actions secret.
+
+To generate credentials:
+
+#. Install launchpadlib: ``pip install launchpadlib``
+#. Run the credentials helper script::
+
+     python3 scripts/create_lp_creds.py
+
+#. Approve the authorization request in your browser. This writes a credentials file (default: ``launchpad.credentials``).
+#. Copy the full content of the credentials file.
+#. In GitHub, go to the repository **Settings > Secrets and variables > Actions > New repository secret**.
+#. Create a secret named ``LP_CREDENTIALS`` and paste the credentials file content.
+
+The workflow writes this secret to a temporary file at runtime and cleans it up after each job.
+
+Manual workflow dispatch
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The workflow supports a ``workflow_dispatch`` trigger for manual reruns. This is useful when a release workflow fails partway through — you can fix the issue and rerun the workflow without it breaking because earlier steps already succeeded.
+
+To trigger from the GitHub UI:
+
+#. Go to **Actions > Build Debian source package > Run workflow**
+#. Click **Run workflow**
+
+To trigger from the command line::
+
+  gh workflow run build_debian.yml
+
+When triggered via ``workflow_dispatch``:
+
+- The ``build_package`` and ``wait_for_source_builds`` jobs are skipped (no release artifact to upload)
+- The version is read from ``debian/changelog`` instead of the release tag
+- The ``block_release_step`` manual approval gate is skipped
+- All copy and promote steps run normally — they are idempotent and safely handle packages that were already copied in a previous run
+
+Launchpad copy script
+~~~~~~~~~~~~~~~~~~~~~
+
+The ``scripts/launchpad_copy.py`` script manages Launchpad PPA operations with three subcommands:
+
+``copy-to-series``
+  Copies packages from the source Ubuntu series to all other supported series within the ``kolibri-proposed`` PPA::
+
+    python3 scripts/launchpad_copy.py copy-to-series
+
+``promote``
+  Promotes all published packages from ``kolibri-proposed`` to the ``kolibri`` PPA::
+
+    python3 scripts/launchpad_copy.py promote
+
+``wait-for-builds``
+  Polls Launchpad until all builds for a source package reach a terminal state::
+
+    python3 scripts/launchpad_copy.py wait-for-builds --package kolibri-server --version 1.0.0
+
+All subcommands are idempotent — rerunning them after a partial success safely skips packages that were already copied or promoted.
+
+Additional flags: ``-v`` / ``-vv`` for verbosity, ``-q`` for quiet mode, ``--debug`` for HTTP-level debugging.
 
 Overview
 --------

scripts/launchpad_copy.py

@@ -247,13 +247,19 @@ def perform_queued_copies(self, ppa):
                 log.info("")
                 first = False
             log.info("Copying %s to %s", ", ".join(sorted(names)), target_series)
-            ppa.syncSources(
-                from_archive=ppa,
-                to_series=target_series,
-                to_pocket=pocket,
-                include_binaries=True,
-                source_names=sorted(names),
-            )
+            try:
+                ppa.syncSources(
+                    from_archive=ppa,
+                    to_series=target_series,
+                    to_pocket=pocket,
+                    include_binaries=True,
+                    source_names=sorted(names),
+                )
+            except lre.BadRequest as e:
+                if "same version already published" in str(e):
+                    log.info("Already copied to %s — skipping", target_series)
+                else:
+                    raise
 
     def copy_to_series(self):
         """Copy packages from source series to all other supported Ubuntu series."""
@@ -410,12 +416,19 @@ def promote(self):
                 )
                 copied_any = True
             except lre.BadRequest as e:
-                if "is obsolete and will not accept new uploads" in str(e):
+                msg = str(e)
+                if "is obsolete and will not accept new uploads" in msg:
                     log.info(
                         "Skip obsolete series for %s %s",
                         pkg.source_package_name,
                         pkg.source_package_version,
                     )
+                elif "same version already published" in msg:
+                    log.info(
+                        "Already published %s %s — skipping",
+                        pkg.source_package_name,
+                        pkg.source_package_version,
+                    )
                 else:
                     raise
 
@@ -441,7 +454,6 @@ def build_parser():
     )
     parser.add_argument("-q", "--quiet", action="store_true", help="Suppress info output.")
     parser.add_argument("--debug", action="store_true", help="Enable HTTP debug output.")
-
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     subparsers.add_parser(

tests/test_launchpad_copy.py

@@ -215,6 +215,47 @@ def test_perform_queued_copies_skips_empty_queues(self):
 
         mock_ppa.syncSources.assert_not_called()
 
+    def test_perform_queued_copies_handles_already_synced(self):
+        """Idempotency: syncSources errors for already-copied packages are handled gracefully."""
+        wrapper = LaunchpadWrapper()
+        wrapper.queue_copy("kolibri-server", "jammy", "noble", "Release")
+
+        class MockBadRequest(Exception):
+            pass
+
+        mock_ppa = MagicMock()
+        mock_ppa.syncSources.side_effect = MockBadRequest(
+            "kolibri-server 0.9.0 in noble (same version already published)"
+        )
+
+        with patch("launchpad_copy.lre") as mock_lre:
+            mock_lre.BadRequest = MockBadRequest
+            wrapper.perform_queued_copies(mock_ppa)
+
+        # Should not raise — the error is handled gracefully
+
+    def test_perform_queued_copies_logs_already_synced(self, caplog):
+        """Idempotency: logs a message when syncSources finds package already exists."""
+        wrapper = LaunchpadWrapper()
+        wrapper.queue_copy("kolibri-server", "jammy", "noble", "Release")
+
+        class MockBadRequest(Exception):
+            pass
+
+        mock_ppa = MagicMock()
+        mock_ppa.syncSources.side_effect = MockBadRequest(
+            "kolibri-server 0.9.0 in noble (same version already published)"
+        )
+
+        with (
+            patch("launchpad_copy.lre") as mock_lre,
+            caplog.at_level(logging.INFO, logger=log.name),
+        ):
+            mock_lre.BadRequest = MockBadRequest
+            wrapper.perform_queued_copies(mock_ppa)
+
+        assert any("already" in r.message.lower() for r in caplog.records)
+
     def test_get_usable_sources_filters_by_whitelist(self):
         wrapper = LaunchpadWrapper()
         mock_ppa = MagicMock()
@@ -475,6 +516,85 @@ def test_skips_non_whitelisted_package(self):
         mock_dest_ppa.copyPackage.assert_not_called()
         assert result == 0
 
+    def test_handles_already_published_package_gracefully(self):
+        """Idempotency: promote skips packages already copied to dest PPA."""
+        wrapper = LaunchpadWrapper()
+        mock_source_ppa = MagicMock()
+        mock_dest_ppa = MagicMock()
+
+        mock_pkg = MagicMock()
+        mock_pkg.source_package_name = "kolibri-server"
+        mock_pkg.source_package_version = "0.9.0"
+        mock_pkg.distro_series_link = "https://lp/ubuntu/jammy"
+        mock_pkg.pocket = "Release"
+
+        mock_source_ppa.getPublishedSources.return_value = [mock_pkg]
+
+        class MockBadRequest(Exception):
+            pass
+
+        mock_dest_ppa.copyPackage.side_effect = MockBadRequest(
+            "kolibri-server 0.9.0 in jammy (same version already published in the target archive)"
+        )
+
+        with (
+            patch.object(
+                type(wrapper),
+                "proposed_ppa",
+                new_callable=lambda: property(lambda self: mock_source_ppa),
+            ),
+            patch.object(
+                type(wrapper),
+                "release_ppa",
+                new_callable=lambda: property(lambda self: mock_dest_ppa),
+            ),
+            patch("launchpad_copy.lre") as mock_lre,
+        ):
+            mock_lre.BadRequest = MockBadRequest
+            result = wrapper.promote()
+
+        assert result == 0
+
+    def test_already_published_logs_skip_message(self, caplog):
+        """Idempotency: promote logs that a package was already promoted."""
+        wrapper = LaunchpadWrapper()
+        mock_source_ppa = MagicMock()
+        mock_dest_ppa = MagicMock()
+
+        mock_pkg = MagicMock()
+        mock_pkg.source_package_name = "kolibri-server"
+        mock_pkg.source_package_version = "0.9.0"
+        mock_pkg.distro_series_link = "https://lp/ubuntu/jammy"
+        mock_pkg.pocket = "Release"
+
+        mock_source_ppa.getPublishedSources.return_value = [mock_pkg]
+
+        class MockBadRequest(Exception):
+            pass
+
+        mock_dest_ppa.copyPackage.side_effect = MockBadRequest(
+            "kolibri-server 0.9.0 in jammy (same version already published in the target archive)"
+        )
+
+        with (
+            patch.object(
+                type(wrapper),
+                "proposed_ppa",
+                new_callable=lambda: property(lambda self: mock_source_ppa),
+            ),
+            patch.object(
+                type(wrapper),
+                "release_ppa",
+                new_callable=lambda: property(lambda self: mock_dest_ppa),
+            ),
+            patch("launchpad_copy.lre") as mock_lre,
+            caplog.at_level(logging.INFO, logger=log.name),
+        ):
+            mock_lre.BadRequest = MockBadRequest
+            wrapper.promote()
+
+        assert any("already published" in r.message.lower() and "kolibri-server" in r.message for r in caplog.records)
+
 
 # --- wait-for-builds tests ---