Update CI files for branch main

.ci/ansible/Containerfile.j2

@@ -5,9 +5,9 @@ FROM {{ ci_base | default(pulp_default_container) }}
 ADD ./{{ item.name }} ./{{ item.name }}
 {% endfor %}
 
-# Install python packages
-# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
-# Hacking botocore (https://github.com/boto/botocore/pull/1990)
+{% for item in extra_files | default([]) %}
+ADD ./{{ item.origin }} {{ item.destination }}
+{% endfor %}
 
 # This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \

.ci/scripts/calc_constraints.py

@@ -53,6 +53,9 @@ def to_upper_bound(req):
         if requirement.name == "pulpcore":
             # An exception to allow for pulpcore deprecation policy.
             return fetch_pulpcore_upper_bound(requirement)
+        # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9'
+        if requirement.marker:
+            return f"# ENVIRONMENT IS UNTRACKABLE: {req}"
         for spec in requirement.specifier:
             if spec.operator == "~=":
                 return f"# NO BETTER CONSTRAINT: {req}"

.ci/scripts/collect_changes.py

@@ -7,16 +7,21 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 import itertools
+import json
 import os
 import re
 import tomllib
+import urllib.request
+from pathlib import Path
 
 from git import GitCommandError, Repo
 from packaging.version import parse as parse_version
 
+
+PYPI_PROJECT = "pulp_container"
+
 # Read Towncrier settings
-with open("pyproject.toml", "rb") as fp:
-    tc_settings = tomllib.load(fp)["tool"]["towncrier"]
+tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
@@ -35,7 +40,7 @@
 # see help(re.split) for more info.
 NAME_REGEX = r".*"
 VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*"
-VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})"
+VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})"
 DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
 TITLE_REGEX = (
     "("
@@ -75,6 +80,20 @@ def main():
     branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
     branches = [ref.name for ref in branches]
 
+    changed = False
+
+    try:
+        response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json")
+        pypi_record = json.loads(response.read())
+        yanked_versions = {
+            parse_version(version): release[0]["yanked_reason"]
+            for version, release in pypi_record["releases"].items()
+            if release[0]["yanked"] is True
+        }
+    except Exception:
+        # If something failed, just don't mark anything as yanked.
+        yanked_versions = {}
+
     with open(CHANGELOG_FILE, "r") as f:
         main_changelog = f.read()
     preamble, main_changes = split_changelog(main_changelog)
@@ -95,9 +114,19 @@ def main():
             if left[0] != right[0]:
                 main_changes.append(right)
 
+    if yanked_versions:
+        for change in main_changes:
+            if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]:
+                reason = yanked_versions[change[0]]
+                version = str(change[0])
+                change[1] = change[1].replace(version, "YANKED " + version, count=1)
+                if reason:
+                    change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1)
+                changed = True
+
     new_length = len(main_changes)
-    if old_length < new_length:
-        print(f"{new_length - old_length} new versions have been added.")
+    if old_length < new_length or changed:
+        print(f"{new_length - old_length} new versions have been added (or something has changed).")
         with open(CHANGELOG_FILE, "w") as fp:
             fp.write(preamble)
             for change in main_changes:

.github/workflows/create-branch.yml

@@ -20,6 +20,9 @@ jobs:
     strategy:
       fail-fast: false
 
+    permissions:
+      contents: write
+
     steps:
       - uses: "actions/checkout@v4"
         with:

.github/workflows/docs.yml

@@ -6,12 +6,12 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: "Docs"
+name: "Docs CI"
 on:
   workflow_call:
 
 jobs:
-  test:
+  changelog:
     if: "endsWith(github.base_ref, 'main')"
     runs-on: "ubuntu-latest"
     defaults:
@@ -22,28 +22,22 @@ jobs:
         with:
           fetch-depth: 1
           path: "pulp_container"
-      - uses: "actions/checkout@v4"
-        with:
-          fetch-depth: 0
-          repository: "pulp/pulp-docs"
-          path: "pulp-docs"
-          ref: "rewrite-as-mkdocs-plugin"
       - uses: "actions/setup-python@v5"
         with:
           python-version: "3.12"
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install ../pulp-docs towncrier
+          pip install towncrier
           echo ::endgroup::
       - name: "Build changelog"
         run: |
           towncrier build --yes --version 4.0.0.ci
-      - name: "Build docs"
-        working-directory: "pulp-docs"
-        run: |
-          pulp-docs fetch --dest ..
-          pulp-docs build
+  docs:
+    if: "endsWith(github.base_ref, 'main')"
+    uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@rewrite-as-mkdocs-plugin'
+    with:
+      pulpdocs_ref: 'rewrite-as-mkdocs-plugin'
 
   no-test:
     if: "!endsWith(github.base_ref, 'main')"

.github/workflows/nightly.yml

@@ -43,7 +43,7 @@ jobs:
 
       - uses: "actions/setup-python@v5"
         with:
-          python-version: "3.11"
+          python-version: "3.13"
 
       - name: "Install python dependencies"
         run: |

.github/workflows/publish.yml

@@ -146,11 +146,36 @@ jobs:
       - "publish-python-bindings"
       - "publish-ruby-bindings"
 
+    permissions:
+      contents: write
+
+    env:
+      TAG_NAME: "${{ github.ref_name }}"
+
     steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+          path: "pulp_container"
+
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.11"
+
+      - name: "Install towncrier"
+        run: |
+          pip install towncrier
+
+      - name: "Get release notes"
+        id: get_release_notes
+        run: |
+          NOTES=$(towncrier build --draft --version $TAG_NAME)
+          echo "body<<EOF" >> $GITHUB_OUTPUT
+          echo "$NOTES" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
       - name: "Create release on GitHub"
         uses: "actions/github-script@v7"
-        env:
-          TAG_NAME: "${{ github.ref_name }}"
         with:
           script: |
             const { TAG_NAME } = process.env;
@@ -159,5 +184,6 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               tag_name: TAG_NAME,
+              body: `${{ steps.get_release_notes.outputs.body }}`,
               make_latest: "legacy",
             });

doc_requirements.txt

@@ -5,4 +5,4 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 towncrier
-pulp-docs @ git+https://github.com/pulp/pulp-docs@main
+pulp-docs @ git+https://github.com/pulp/pulp-docs@rewrite-as-mkdocs-plugin

template_config.yml

@@ -20,6 +20,7 @@ deploy_client_to_rubygems: true
 deploy_to_pypi: true
 disabled_redis_runners: []
 docker_fixtures: false
+extra_files: []
 flake8: true
 flake8_ignore: []
 github_org: pulp