Add catalog signing of PyManager index files

zooba · zooba · commit 89f84015f255 · 2026-04-13T16:10:24.000+01:00
diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py
@@ -20,6 +20,7 @@
 UPLOAD_USER = os.getenv("UPLOAD_USER", "")
 NO_UPLOAD = os.getenv("NO_UPLOAD", "no")[:1].lower() in "yt1"
 LOCAL_INDEX = os.getenv("LOCAL_INDEX", "no")[:1].lower() in "yt1"
+SIGN_COMMAND = os.getenv("SIGN_COMMAND", "")
 
 
 def find_cmd(env, exe):
@@ -40,6 +41,7 @@ def find_cmd(env, exe):
 
 PLINK = find_cmd("PLINK", "plink.exe")
 PSCP = find_cmd("PSCP", "pscp.exe")
+MAKECAT = find_cmd("MAKECAT", "makecat.exe")
 
 
 def _std_args(cmd):
@@ -193,6 +195,42 @@ def calculate_uploads():
         )
 
 
+def sign_json(cat_file, *files):
+    if not MAKECAT:
+        if not UPLOAD_HOST or NO_UPLOAD:
+            print("makecat.exe not found, but not uploading, so skip signing.")
+            return
+        raise RuntimeError("No makecat.exe found")
+    if not SIGN_COMMAND:
+        if not UPLOAD_HOST or NO_UPLOAD:
+            print("No signing command set, but not uploading, so skip signing.")
+            return
+        raise RuntimeError("No SIGN_COMMAND set")
+
+    cat = Path(cat_file).absolute()
+    cdf = cat.with_suffix(".cdf")
+    cdf.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(cdf, "w", encoding="ansi") as f:
+        print("[CatalogHeader]", file=f)
+        print("Name=", cat.name, sep="", file=f)
+        print("ResultDir=", cat.parent, sep="", file=f)
+        print("PublicVersion=0x00000001", file=f)
+        print("CatalogVersion=2", file=f)
+        print("HashAlgorithms=SHA256", file=f)
+        print("EncodingType=", file=f)
+        print(file=f)
+        print("[CatalogFiles]", file=f)
+        for a in map(Path, files):
+            print("<HASH>", a.name, "=", a.absolute(), sep="", file=f)
+
+    _run(MAKECAT, "-v", cdf)
+    if not cat.is_file():
+        raise FileNotFoundError(cat)
+    _run(SIGN_COMMAND, cat)
+    cdf.unlink()
+
+
 def remove_and_insert(index, new_installs):
     new = {(i["id"].casefold(), i["sort-version"].casefold()) for i in new_installs}
     to_remove = [
@@ -274,20 +312,28 @@ def find_missing_from_index(url, installs):
         except FileNotFoundError:
             pass
 
+
 print(INDEX_PATH, "mtime =", INDEX_MTIME)
 
 
+
 new_installs = [trim_install(i) for i, *_ in UPLOADS]
 validate_new_installs(new_installs)
 new_installs = sorted(new_installs, key=install_sortkey)
 remove_and_insert(index["versions"], new_installs)
 
 if INDEX_FILE:
     INDEX_FILE = Path(INDEX_FILE).absolute()
+    INDEX_CAT_FILE = INDEX_FILE.with_suffix(".cat")
     INDEX_FILE.parent.mkdir(parents=True, exist_ok=True)
     with open(INDEX_FILE, "w", encoding="utf-8") as f:
         json.dump(index, f)
 
+    sign_json(INDEX_CAT_FILE, INDEX_FILE)
+else:
+    INDEX_CAT_FILE = None
+
+
 if MANIFEST_FILE:
     # Use the sort-version so that the manifest name includes prerelease marks
     MANIFEST_FILE = Path(MANIFEST_FILE).absolute()
@@ -333,6 +379,10 @@ def find_missing_from_index(url, installs):
         print("Uploading", INDEX_FILE, "to", INDEX_URL)
         upload_ssh(INDEX_FILE, INDEX_PATH)
 
+    if INDEX_CAT_FILE:
+        print("Uploading", INDEX_CAT_FILE, "to", f"{INDEX_URL}.cat")
+        upload_ssh(INDEX_CAT_FILE, f"{INDEX_PATH}.cat")
+
     print("Purging", len(UPLOADS), "uploaded files")
     parents = set()
     for i, *_ in UPLOADS:
diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml
@@ -6,33 +6,83 @@ parameters:
   ExtractDir: ''
   SigningCertificate: ''
   ExportCommand: ''
+  ExportLegacyCommand: ''
   ContinueOnError: false
+  InstallTool: true
+  InstallLegacyTool: false
   AzureServiceConnectionName: 'Python Signing'
 
 steps:
-- ${{ if parameters.SigningCertificate }}:
-  - powershell: |
-      # Install sign tool
-      dotnet tool install --global --prerelease sign
-      $signtool = (gcm sign -EA SilentlyContinue).Source
-      if (-not $signtool) {
-        $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName
-      }
-      $signargs = 'code trusted-signing -v Information ' + `
-        '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + `
-        '-tse "$(TrustedSigningUri)" -tsa "$(TrustedSigningAccount)" -tscp "$(TrustedSigningCertificateName)" ' + `
-        '-d "$(SigningDescription)" '
+- ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
+  - ${{ if eq(parameters.InstallTool, 'true') }}:
+    - powershell: |
+        # Install sign tool
+        dotnet tool install --global --prerelease sign
+        $signtool = (gcm sign -EA SilentlyContinue).Source
+        if (-not $signtool) {
+          $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName
+        }
+        $signargs = 'code artifact-signing -v Information ' + `
+          '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + `
+          "-tse ""${env:TSE}"" -tsa ""${env:TSA}"" -tscp ""${env:TSCP}"" -d ""${env:DESCRIPTION}"""
+
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool"
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs"
+        if ($env:EXPORT_COMMAND) {
+          $signcmd = """$signtool"" $signargs"
+          Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd"
+        }
+      workingDirectory: $(Build.BinariesDirectory)
+      displayName: 'Install Azure Artifact Signing tools'
+      env:
+        TSE: $(TrustedSigningUri)
+        TSA: $(TrustedSigningAccount)
+        TSCP: $(TrustedSigningCertificateName)
+        DESCRIPTION: $(SigningDescription)
+        EXPORT_COMMAND: ${{ parameters.ExportCommand }}
+
+  - ${{ if eq(parameters.InstallLegacyTool, 'true') }}:
+    - powershell: |
+        git clone https://github.com/python/cpython-bin-deps --revision fb06137dccc43ed5b030cdd9e3560990b37f39da --depth 1 --progress -v "signtool"
+
+        $signtool = gi signtool\x64\signtool.exe
+        $dlib = gi signtool\azure_trusted_signing\x64\Azure.CodeSigning.Dlib.dll
+        Write-Host "##vso[task.setvariable variable=MAKECAT]$(gi signtool\x64\makecat.exe)"
+
+        ConvertTo-Json @{
+            Endpoint=$env:TSE;
+            CodeSigningAccountName=$env:TSA;
+            CertificateProfileName=$env:TSCP;
+            # Only allow Azure CLI credentials and environment credentials
+            ExcludeCredentials=@(
+                "ManagedIdentityCredential",
+                "WorkloadIdentityCredential",
+                "SharedTokenCacheCredential",
+                "VisualStudioCredential",
+                "VisualStudioCodeCredential",
+                "AzurePowerShellCredential",
+                "AzureDeveloperCliCredential",
+                "InteractiveBrowserCredential"
+            );
+        } | Out-File signtool\metadata.json -Encoding ascii
+        Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)"
 
-      Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool"
-      Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs"
-      if ($env:EXPORT_COMMAND) {
-        $signcmd = """$signtool"" $signargs"
-        Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd"
-      }
-    workingDirectory: $(Build.BinariesDirectory)
-    displayName: 'Install Trusted Signing tools'
-    env:
-      EXPORT_COMMAND: ${{ parameters.ExportCommand }}
+        $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td SHA256' + `
+                    "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)"""
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool"
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs"
+
+        if ($env:EXPORT_COMMAND) {
+          $signcmd = """$signtool"" $signargs"
+          Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd"
+        }
+      workingDirectory: $(Pipeline.Workspace)
+      displayName: 'Download signtool binaries'
+      env:
+        TSE: $(TrustedSigningUri)
+        TSA: $(TrustedSigningAccount)
+        TSCP: $(TrustedSigningCertificateName)
+        EXPORT_COMMAND: ${{ parameters.ExportLegacyCommand }}
 
   - ${{ if parameters.AzureServiceConnectionName }}:
     # We sign in once with the AzureCLI task, as it uses OIDC to obtain a
@@ -45,25 +95,25 @@ steps:
         scriptType: 'ps'
         scriptLocation: 'inlineScript'
         inlineScript: |
-          "##vso[task.setvariable variable=AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}"
-          "##vso[task.setvariable variable=AZURE_ID_TOKEN;issecret=true]${env:idToken}"
-          "##vso[task.setvariable variable=AZURE_TENANT_ID;issecret=true]${env:tenantId}"
+          "##vso[task.setvariable variable=__AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}"
+          "##vso[task.setvariable variable=__AZURE_ID_TOKEN;issecret=true]${env:idToken}"
+          "##vso[task.setvariable variable=__AZURE_TENANT_ID;issecret=true]${env:tenantId}"
         addSpnToEnvironment: true
 
     - powershell: >
         az login --service-principal
-        -u $(AZURE_CLIENT_ID)
-        --tenant $(AZURE_TENANT_ID)
+        -u $(__AZURE_CLIENT_ID)
+        --tenant $(__AZURE_TENANT_ID)
         --allow-no-subscriptions
-        --federated-token $(AZURE_ID_TOKEN)
+        --federated-token $(__AZURE_ID_TOKEN)
       displayName: 'Authenticate signing tools (2/2)'
 
   - ${{ if parameters.Include }}:
     - powershell: |
-        if ("${{ parameters.Exclude }}") {
-          $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File)
+        if ($env:EXCLUDE) {
+          $files = (dir $env:INCLUDE -Exclude $env:EXCLUDE -File)
         } else {
-          $files = (dir ${{ parameters.Include }} -File)
+          $files = (dir $env:INCLUDE -File)
         }
         if ($env:FILTER) {
           ($env:FILTER -split ';') -join "`n" | Out-File __filelist.txt -Encoding utf8
@@ -82,31 +132,37 @@ steps:
         continueOnError: true
       workingDirectory: ${{ parameters.WorkingDir }}
       env:
+        INCLUDE: ${{ parameters.Include }}
+        EXCLUDE: ${{ parameters.Exclude }}
         TRUSTED_SIGNING_CMD: $(__TrustedSigningCmd)
         TRUSTED_SIGNING_ARGS: $(__TrustedSigningArgs)
         ${{ if parameters.Filter }}:
           FILTER: ${{ parameters.Filter }}
 
 
-- ${{ if parameters.ExtractDir }}:
-  - powershell: |
-      if ("${{ parameters.Exclude }}") {
-        $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File)
-      } else {
-        $files = (dir ${{ parameters.Include }} -File)
-      }
-      $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1
-      if (-not $c) {
-        Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}"
-        exit
-      }
+  - ${{ if parameters.ExtractDir }}:
+    - powershell: |
+        if ($env:EXCLUDE) {
+          $files = (dir $env:INCLUDE -Exclude $env:EXCLUDE -File)
+        } else {
+          $files = (dir $env:INCLUDE -File)
+        }
+        $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1
+        if (-not $c) {
+          Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}"
+          exit
+        }
 
-      $d = mkdir "${{ parameters.ExtractDir }}" -Force
-      $cf = "$d\cert.cer"
-      [IO.File]::WriteAllBytes($cf, $c.RawData)
-      $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower()
+        $d = mkdir $env:EXTRACT_DIR -Force
+        $cf = "$d\cert.cer"
+        [IO.File]::WriteAllBytes($cf, $c.RawData)
+        $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower()
 
-      $info = @{ Subject=$c.Subject; SHA256=$csha; }
-      $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json"
-    displayName: "Extract certificate info"
-    workingDirectory: ${{ parameters.WorkingDir }}
+        $info = @{ Subject=$c.Subject; SHA256=$csha; }
+        $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json"
+      displayName: "Extract certificate info"
+      workingDirectory: ${{ parameters.WorkingDir }}
+      env:
+        INCLUDE: ${{ parameters.Include }}
+        EXCLUDE: ${{ parameters.Exclude }}
+        EXTRACT_DIR: ${{ parameters.ExtractDir }}
diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml
@@ -3,6 +3,7 @@ parameters:
   DoFreethreaded: false
   DoEmbed: false
   HashAlgorithms: ['SHA256']
+  SigningCertificate: ''
 
   Artifacts:
   - name: win32
@@ -31,6 +32,10 @@ jobs:
 
   variables:
   - group: PythonOrgPublish
+  - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}:
+    - group: CPythonSign
+  - ${{ if eq(parameters.SigningCertificate, 'TestSign') }}:
+    - group: CPythonTestSign
 
   workspace:
     clean: all
@@ -64,11 +69,20 @@ jobs:
     displayName: 'Download PuTTY key'
 
   - powershell: |
-      git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty"
+      git clone https://github.com/python/cpython-bin-deps --revision 9f9e6fc31a55406ee5ff0198ea47bbb445eeb942 --depth 1 --progress -v "putty"
       "##vso[task.prependpath]$(gi putty)"
     workingDirectory: $(Pipeline.Workspace)
     displayName: 'Download PuTTY binaries'
 
+  # Use the template just to configure the signing tool.
+  - template: sign-files.yml
+    parameters:
+      Include: ""
+      InstallTool: false
+      InstallLegacyTool: true
+      ExportLegacyCommand: SIGN_COMMAND
+      SigningCertificate: ${{ parameters.SigningCertificate }}
+
   - powershell: |
       if ($env:FILENAME) {
         "##vso[task.setvariable variable=_PyManagerIndexFilename]${env:FILENAME}"
@@ -113,6 +127,9 @@ jobs:
       UPLOAD_HOST_KEY: $(PyDotOrgHostKey)
       UPLOAD_USER: $(PyDotOrgUsername)
       UPLOAD_KEYFILE: $(sshkey.secureFilePath)
+      ${{ if variables['SIGN_COMMAND'] }}:
+        MAKECAT: $(MAKECAT)
+        SIGN_COMMAND: $(SIGN_COMMAND)
 
   - ${{ each alg in parameters.HashAlgorithms }}:
     - powershell: |
