Update Azure and S3 storage settings for 2.16

Set storage settings in template_config.yml based on
pulpcore requirement (>=3.28.15,<3.40).

Storage formats: Azure (legacy), S3 (legacy)

This addresses the recent change where plugin-template stopped
automatically setting test scenario storage settings and now
requires plugins to define them in their template_config.yml.

Author: gerrod3

.ci/ansible/Containerfile.j2

@@ -5,9 +5,9 @@ FROM {{ ci_base | default(pulp_default_container) }}
 ADD ./{{ item.name }} ./{{ item.name }}
 {% endfor %}
 
-# Install python packages
-# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
-# Hacking botocore (https://github.com/boto/botocore/pull/1990)
+{% for item in extra_files | default([]) %}
+ADD ./{{ item.origin }} {{ item.destination }}
+{% endfor %}
 
 # This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \

.ci/ansible/settings.py.j2

@@ -26,56 +26,3 @@ API_ROOT = {{ api_root | repr }}
 {% endfor %}
 {% endif %}
 
-{% if s3_test | default(false) %}
-MEDIA_ROOT: ""
-S3_USE_SIGV4 = True
-{% if test_storages_compat_layer is defined and test_storages_compat_layer %}
-STORAGES = {
-    "default": {
-        "BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
-        "OPTIONS": {
-            "access_key": "{{ minio_access_key }}",
-            "secret_key": "{{ minio_secret_key }}",
-            "region_name": "eu-central-1",
-            "addressing_style": "path",
-            "signature_version": "s3v4",
-            "bucket_name": "pulp3",
-            "endpoint_url": "http://minio:9000",
-            "default_acl": "@none None",
-        },
-    },
-    "staticfiles": {
-      "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
-    },
-}
-{% else %}
-DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
-AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
-AWS_S3_REGION_NAME = "eu-central-1"
-AWS_S3_ADDRESSING_STYLE = "path"
-AWS_S3_SIGNATURE_VERSION = "s3v4"
-AWS_STORAGE_BUCKET_NAME = "pulp3"
-AWS_S3_ENDPOINT_URL = "http://minio:9000"
-AWS_DEFAULT_ACL = "@none None"
-{% endif %}
-{% endif %}
-
-{% if azure_test | default(false) %}
-DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
-MEDIA_ROOT = ""
-AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-AZURE_ACCOUNT_NAME = "devstoreaccount1"
-AZURE_CONTAINER =  "pulp-test"
-AZURE_LOCATION = "pulp3"
-AZURE_OVERWRITE_FILES = True
-AZURE_URL_EXPIRATION_SECS = 120
-AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
-{% endif %}
-
-{% if gcp_test | default(false) %}
-DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
-MEDIA_ROOT = ""
-GS_BUCKET_NAME = "gcppulp"
-GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
-{% endif %}

.ci/scripts/calc_constraints.py

@@ -53,6 +53,9 @@ def to_upper_bound(req):
         if requirement.name == "pulpcore":
             # An exception to allow for pulpcore deprecation policy.
             return fetch_pulpcore_upper_bound(requirement)
+        # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9'
+        if requirement.marker:
+            return f"# ENVIRONMENT IS UNTRACKABLE: {req}"
         for spec in requirement.specifier:
             if spec.operator == "~=":
                 return f"# NO BETTER CONSTRAINT: {req}"

.ci/scripts/check_release.py

@@ -10,8 +10,8 @@
 from git import Repo
 
 RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$"
-Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"]
-Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"]
+Y_CHANGELOG_EXTS = [".feature"]
+Z_CHANGELOG_EXTS = [".bugfix", ".misc"]
 
 
 def options():

.ci/scripts/collect_changes.py

@@ -7,16 +7,21 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 import itertools
+import json
 import os
 import re
 import tomllib
+import urllib.request
+from pathlib import Path
 
 from git import GitCommandError, Repo
 from packaging.version import parse as parse_version
 
+
+PYPI_PROJECT = "pulp_container"
+
 # Read Towncrier settings
-with open("pyproject.toml", "rb") as fp:
-    tc_settings = tomllib.load(fp)["tool"]["towncrier"]
+tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
@@ -35,7 +40,7 @@
 # see help(re.split) for more info.
 NAME_REGEX = r".*"
 VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*"
-VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})"
+VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})"
 DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
 TITLE_REGEX = (
     "("
@@ -75,6 +80,20 @@ def main():
     branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
     branches = [ref.name for ref in branches]
 
+    changed = False
+
+    try:
+        response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json")
+        pypi_record = json.loads(response.read())
+        yanked_versions = {
+            parse_version(version): release[0]["yanked_reason"]
+            for version, release in pypi_record["releases"].items()
+            if release[0]["yanked"] is True
+        }
+    except Exception:
+        # If something failed, just don't mark anything as yanked.
+        yanked_versions = {}
+
     with open(CHANGELOG_FILE, "r") as f:
         main_changelog = f.read()
     preamble, main_changes = split_changelog(main_changelog)
@@ -95,9 +114,19 @@ def main():
             if left[0] != right[0]:
                 main_changes.append(right)
 
+    if yanked_versions:
+        for change in main_changes:
+            if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]:
+                reason = yanked_versions[change[0]]
+                version = str(change[0])
+                change[1] = change[1].replace(version, "YANKED " + version, count=1)
+                if reason:
+                    change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1)
+                changed = True
+
     new_length = len(main_changes)
-    if old_length < new_length:
-        print(f"{new_length - old_length} new versions have been added.")
+    if old_length < new_length or changed:
+        print(f"{new_length - old_length} new versions have been added (or something has changed).")
         with open(CHANGELOG_FILE, "w") as fp:
             fp.write(preamble)
             for change in main_changes:

.github/workflows/create-branch.yml

@@ -20,6 +20,9 @@ jobs:
     strategy:
       fail-fast: false
 
+    permissions:
+      contents: write
+
     steps:
       - uses: "actions/checkout@v4"
         with:

.github/workflows/docs.yml

@@ -6,12 +6,12 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: "Docs"
+name: "Docs CI"
 on:
   workflow_call:
 
 jobs:
-  test:
+  changelog:
     if: "endsWith(github.base_ref, 'main')"
     runs-on: "ubuntu-latest"
     defaults:
@@ -22,28 +22,22 @@ jobs:
         with:
           fetch-depth: 1
           path: "pulp_container"
-      - uses: "actions/checkout@v4"
-        with:
-          fetch-depth: 0
-          repository: "pulp/pulp-docs"
-          path: "pulp-docs"
-          ref: "rewrite-as-mkdocs-plugin"
       - uses: "actions/setup-python@v5"
         with:
           python-version: "3.12"
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install ../pulp-docs towncrier
+          pip install towncrier
           echo ::endgroup::
       - name: "Build changelog"
         run: |
           towncrier build --yes --version 4.0.0.ci
-      - name: "Build docs"
-        working-directory: "pulp-docs"
-        run: |
-          pulp-docs fetch --dest ..
-          pulp-docs build
+  docs:
+    if: "endsWith(github.base_ref, 'main')"
+    uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@rewrite-as-mkdocs-plugin'
+    with:
+      pulpdocs_ref: 'rewrite-as-mkdocs-plugin'
 
   no-test:
     if: "!endsWith(github.base_ref, 'main')"

.github/workflows/nightly.yml

@@ -43,7 +43,7 @@ jobs:
 
       - uses: "actions/setup-python@v5"
         with:
-          python-version: "3.11"
+          python-version: "3.13"
 
       - name: "Install python dependencies"
         run: |

.github/workflows/publish.yml

@@ -146,11 +146,36 @@ jobs:
       - "publish-python-bindings"
       - "publish-ruby-bindings"
 
+    permissions:
+      contents: write
+
+    env:
+      TAG_NAME: "${{ github.ref_name }}"
+
     steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+          path: "pulp_container"
+
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.11"
+
+      - name: "Install towncrier"
+        run: |
+          pip install towncrier
+
+      - name: "Get release notes"
+        id: get_release_notes
+        run: |
+          NOTES=$(towncrier build --draft --version $TAG_NAME)
+          echo "body<<EOF" >> $GITHUB_OUTPUT
+          echo "$NOTES" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
       - name: "Create release on GitHub"
         uses: "actions/github-script@v7"
-        env:
-          TAG_NAME: "${{ github.ref_name }}"
         with:
           script: |
             const { TAG_NAME } = process.env;
@@ -159,5 +184,6 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               tag_name: TAG_NAME,
+              body: `${{ steps.get_release_notes.outputs.body }}`,
               make_latest: "legacy",
             });

.github/workflows/scripts/install.sh

@@ -95,9 +95,8 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: {"flatpak_index": false}\
+pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": "", "flatpak_index": false}\
 pulp_scenario_env: {}\
-test_storages_compat_layer: false\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
 fi
@@ -110,7 +109,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: {"flatpak_index": true}\
+pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": "", "flatpak_index": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi