refactor: split release pipeline into retryable jobs with idempotent publish flow

Hongwei-MB · Hongwei-MB · commit be223321e9c2 · 2026-03-16T22:26:37.000+08:00
Improve release consistency by isolating NuGet, attestation, tag, and GitHub release into dedicated jobs with gate logging, while moving NuGet push logic into a NUKE target for safer retries and clearer failure boundaries.

Made-with: Cursor
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -400,83 +400,190 @@ jobs:
           path: docs/.vitepress/dist
           if-no-files-found: error
 
-  release:
-    name: Release
+  release-gate:
+    name: Release Gate
     needs: [resolve-version, build-and-test]
     if: needs.resolve-version.outputs.is_release == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 10
     environment: release
     permissions:
-      contents: write
-      attestations: write
+      actions: read
+      contents: read
+    steps:
+      - name: Log environment and reviewer status
+        id: release-status
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          REPO="${{ github.repository }}"
+          RUN_ID="${{ github.run_id }}"
+          ENVIRONMENT_NAME="release"
+          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+          APPROVALS_JSON="$(gh api "repos/$REPO/actions/runs/$RUN_ID/approvals" 2>/dev/null || echo "[]")"
+          PENDING_JSON="$(gh api "repos/$REPO/actions/runs/$RUN_ID/pending_deployments" 2>/dev/null || echo "[]")"
+          APPROVAL_JSON="$(echo "$APPROVALS_JSON" | jq --arg env "$ENVIRONMENT_NAME" '[.[] | select(any(.environments[]?; .name == $env))] | first // {}')"
+
+          APPROVER="$(echo "$APPROVAL_JSON" | jq -r '.user.login // "N/A"')"
+          APPROVAL_STATE="$(echo "$APPROVAL_JSON" | jq -r '.state // "N/A"')"
+          APPROVED_AT="$(echo "$APPROVAL_JSON" | jq -r '.updated_at // "N/A"')"
+          APPROVAL_COMMENT="$(echo "$APPROVAL_JSON" | jq -r '.comment // "N/A"')"
+          APPROVAL_COMMENT="${APPROVAL_COMMENT//$'\n'/ }"
+          APPROVAL_COMMENT="${APPROVAL_COMMENT//|/\\|}"
+          PENDING_COUNT="$(echo "$PENDING_JSON" | jq 'length')"
+
+          mkdir -p artifacts/release
+          jq -n \
+            --arg event "release_environment_status" \
+            --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+            --arg run_id "$RUN_ID" \
+            --arg run_url "$RUN_URL" \
+            --arg env "$ENVIRONMENT_NAME" \
+            --arg approval_state "$APPROVAL_STATE" \
+            --arg approver "$APPROVER" \
+            --arg approval_comment "$APPROVAL_COMMENT" \
+            --arg approved_at "$APPROVED_AT" \
+            --arg tag "${{ needs.resolve-version.outputs.tag }}" \
+            --arg sha "${{ needs.resolve-version.outputs.sha }}" \
+            --argjson pending_count "$PENDING_COUNT" \
+            '{
+              event: $event,
+              timestamp: $timestamp,
+              run_id: $run_id,
+              run_url: $run_url,
+              environment: $env,
+              approval: {
+                state: $approval_state,
+                reviewer: $approver,
+                comment: $approval_comment,
+                approved_at: $approved_at
+              },
+              pending_deployments_count: $pending_count,
+              context: {
+                tag: $tag,
+                sha: $sha
+              }
+            }' > artifacts/release/release-status.json
+
+          echo "::group::Release Environment Status"
+          cat artifacts/release/release-status.json
+          echo "::endgroup::"
+
+          {
+            echo "## Environment and Reviewer Status"
+            echo ""
+            echo "| Property | Value |"
+            echo "|----------|-------|"
+            echo "| **Environment** | \`$ENVIRONMENT_NAME\` |"
+            echo "| **Approval State** | $APPROVAL_STATE |"
+            echo "| **Approved By** | @$APPROVER |"
+            echo "| **Approved At** | $APPROVED_AT |"
+            echo "| **Approval Comment** | $APPROVAL_COMMENT |"
+            echo "| **Pending Deployments** | $PENDING_COUNT |"
+            echo "| **Run** | [#$RUN_ID]($RUN_URL) |"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload release status artifact
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: release-status
+          path: artifacts/release/release-status.json
+          if-no-files-found: error
+
+  release-nuget:
+    name: Release NuGet
+    needs: [resolve-version, build-and-test, release-gate]
+    if: needs.resolve-version.outputs.is_release == 'true' && env.ENABLE_NUGET == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
       id-token: write
-    concurrency:
-      group: release-tag-${{ needs.resolve-version.outputs.tag }}
-      cancel-in-progress: false
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           ref: ${{ needs.resolve-version.outputs.sha }}
-          fetch-depth: 0
 
       - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
         with:
           global-json-file: global.json
 
       - name: Download packages
-        if: env.ENABLE_NUGET == 'true'
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: packages
           path: artifacts/packages
 
-      - name: Download installers
-        if: env.ENABLE_INSTALLERS == 'true'
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: artifacts/installers
-          pattern: installer-*
-          merge-multiple: true
-
       - name: Mark build script executable
         run: chmod +x build.sh
 
       - name: Verify release manifest
-        if: env.ENABLE_NUGET == 'true'
         shell: bash
         run: ./build.sh ValidateReleaseManifest
 
       - name: NuGet login (Trusted Publishing / OIDC)
-        if: env.ENABLE_NUGET == 'true' && env.NUGET_USE_OIDC == 'true'
+        if: env.NUGET_USE_OIDC == 'true'
         id: nuget-login
         uses: NuGet/login@d22cc5f58ff5b88bf9bd452535b4335137e24544 # v1.1.0
         with:
           user: ${{ vars.NUGET_USER || github.repository_owner }}
 
-      - name: Push to NuGet.org
-        if: env.ENABLE_NUGET == 'true'
+      - name: Resolve NuGet API key
         shell: bash
         env:
-          API_KEY: ${{ env.NUGET_USE_OIDC == 'true' && steps.nuget-login.outputs.NUGET_API_KEY || secrets.NUGET_API_KEY }}
+          OIDC_API_KEY: ${{ steps.nuget-login.outputs.NUGET_API_KEY }}
+          SECRET_API_KEY: ${{ secrets.NUGET_API_KEY }}
         run: |
+          set -euo pipefail
+          API_KEY="$SECRET_API_KEY"
+          if [ "$NUGET_USE_OIDC" = "true" ]; then
+            API_KEY="$OIDC_API_KEY"
+          fi
+
           if [ -z "$API_KEY" ]; then
             echo "::error::NUGET_API_KEY is not available. Set the secret or enable OIDC."
             exit 1
           fi
-          shopt -s nullglob
-          PACKAGES=(artifacts/packages/*.nupkg)
-          if [ ${#PACKAGES[@]} -eq 0 ]; then
-            echo "::error::No .nupkg files found"
-            exit 1
-          fi
-          for pkg in "${PACKAGES[@]}"; do
-            echo "Pushing $pkg"
-            dotnet nuget push "$pkg" \
-              --api-key "$API_KEY" \
-              --source https://api.nuget.org/v3/index.json \
-              --skip-duplicate
-          done
+
+          echo "::add-mask::$API_KEY"
+          echo "NUGET_API_KEY=$API_KEY" >> "$GITHUB_ENV"
+
+      - name: Push packages to NuGet.org
+        shell: bash
+        run: ./build.sh PushNuGetPackages
+
+  release-attest:
+    name: Release Attest
+    needs: [resolve-version, build-and-test, release-gate]
+    if: needs.resolve-version.outputs.is_release == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      attestations: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ needs.resolve-version.outputs.sha }}
+
+      - name: Download packages
+        if: env.ENABLE_NUGET == 'true'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: packages
+          path: artifacts/packages
+
+      - name: Download installers
+        if: env.ENABLE_INSTALLERS == 'true'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: artifacts/installers
+          pattern: installer-*
+          merge-multiple: true
 
       - name: Generate SBOM
         uses: anchore/sbom-action@57aae528053a48a3f6235f2d9461b05fbcb7366d # v0.23.1
@@ -489,6 +596,7 @@ jobs:
         id: attest-subjects
         shell: bash
         run: |
+          set -euo pipefail
           shopt -s nullglob
           SUBJECTS=""
           if [ "$ENABLE_NUGET" = "true" ]; then
@@ -519,39 +627,101 @@ jobs:
           subject-path: ${{ steps.attest-subjects.outputs.subjects }}
           sbom-path: sbom-spdx.json
 
-      - name: Create and push tag
+      - name: Upload SBOM artifact
+        if: hashFiles('sbom-spdx.json') != ''
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: sbom
+          path: sbom-spdx.json
+          if-no-files-found: error
+
+  release-tag:
+    name: Release Tag
+    needs: [resolve-version, release-gate, release-nuget]
+    if: needs.resolve-version.outputs.is_release == 'true' && (needs.release-nuget.result == 'success' || needs.release-nuget.result == 'skipped')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+    concurrency:
+      group: release-tag-${{ needs.resolve-version.outputs.tag }}
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ needs.resolve-version.outputs.sha }}
+          fetch-depth: 0
+
+      - name: Create and push tag (idempotent)
         shell: bash
         run: |
+          set -euo pipefail
           TAG="${{ needs.resolve-version.outputs.tag }}"
           SHA="${{ needs.resolve-version.outputs.sha }}"
 
           git fetch --tags --force
-          if git rev-parse "$TAG" >/dev/null 2>&1; then
-            echo "::error::Tag $TAG already exists locally"
+
+          LOCAL_SHA="$(git rev-parse -q --verify "refs/tags/$TAG^{commit}" 2>/dev/null || true)"
+          if [ -n "$LOCAL_SHA" ]; then
+            if [ "$LOCAL_SHA" = "$SHA" ]; then
+              echo "::notice::Tag $TAG already exists locally at expected SHA $SHA."
+              exit 0
+            fi
+            echo "::error::Tag $TAG exists locally at $LOCAL_SHA but expected $SHA."
             exit 1
           fi
-          if git ls-remote --tags origin "refs/tags/$TAG" | grep -q "$TAG"; then
-            echo "::error::Tag $TAG already exists on remote"
+
+          REMOTE_SHA="$(git ls-remote --tags origin "refs/tags/$TAG^{}" | awk '{print $1}' | head -n1)"
+          if [ -z "$REMOTE_SHA" ]; then
+            REMOTE_SHA="$(git ls-remote --tags origin "refs/tags/$TAG" | awk '{print $1}' | head -n1)"
+          fi
+
+          if [ -n "$REMOTE_SHA" ]; then
+            if [ "$REMOTE_SHA" = "$SHA" ]; then
+              echo "::notice::Tag $TAG already exists on remote at expected SHA $SHA."
+              exit 0
+            fi
+            echo "::error::Tag $TAG exists on remote at $REMOTE_SHA but expected $SHA."
             exit 1
           fi
 
           git tag "$TAG" "$SHA"
           git push origin "$TAG"
           echo "Created and pushed tag $TAG at $SHA"
 
-      - name: Create GitHub Release
+  release-github:
+    name: Release GitHub
+    needs: [resolve-version, release-tag, release-attest]
+    if: needs.resolve-version.outputs.is_release == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+    steps:
+      - name: Download installers
+        if: env.ENABLE_INSTALLERS == 'true'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: artifacts/installers
+          pattern: installer-*
+          merge-multiple: true
+
+      - name: Download SBOM artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        continue-on-error: true
+        with:
+          name: sbom
+          path: .
+
+      - name: Create or update GitHub Release
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
+          set -euo pipefail
           TAG="${{ needs.resolve-version.outputs.tag }}"
           RELEASE_NAME="dotnet.CI.template $TAG"
 
-          if gh release view "$TAG" >/dev/null 2>&1; then
-            echo "::error::Release already exists for tag $TAG"
-            exit 1
-          fi
-
           shopt -s nullglob
           RELEASE_ASSETS=()
 
@@ -563,14 +733,28 @@ jobs:
             RELEASE_ASSETS+=(sbom-spdx.json)
           fi
 
-          gh release create "$TAG" "${RELEASE_ASSETS[@]}" \
-            --title "$RELEASE_NAME" \
-            --generate-notes
+          if gh release view "$TAG" >/dev/null 2>&1; then
+            echo "::notice::Release already exists for $TAG. Uploading missing assets."
+            if [ ${#RELEASE_ASSETS[@]} -gt 0 ]; then
+              gh release upload "$TAG" "${RELEASE_ASSETS[@]}" --clobber
+            fi
+            exit 0
+          fi
+
+          if [ ${#RELEASE_ASSETS[@]} -gt 0 ]; then
+            gh release create "$TAG" "${RELEASE_ASSETS[@]}" \
+              --title "$RELEASE_NAME" \
+              --generate-notes
+          else
+            gh release create "$TAG" \
+              --title "$RELEASE_NAME" \
+              --generate-notes
+          fi
           echo "Created release $TAG with ${#RELEASE_ASSETS[@]} assets"
 
   deploy-docs:
     name: Deploy Documentation
-    needs: [resolve-version, release, build-docs]
+    needs: [resolve-version, release-github, build-docs]
     if: needs.resolve-version.outputs.is_release == 'true' && needs.build-docs.outputs.has_docs == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 10
diff --git a/build/BuildTask.Parameters.cs b/build/BuildTask.Parameters.cs
@@ -26,6 +26,9 @@ partial class BuildTask
     [Parameter("Publish self-contained output in Publish target")]
     readonly bool SelfContained;
 
+    [Parameter("NuGet API key for PushNuGetPackages target. Defaults to NUGET_API_KEY environment variable.")]
+    readonly string NuGetApiKey = string.Empty;
+
     [Parameter("Minimum line coverage percentage (0-100). CoverageReport fails if below this threshold.")]
     readonly int CoverageThreshold = 90;
 
diff --git a/build/BuildTask.Targets.PushNuGetPackages.cs b/build/BuildTask.Targets.PushNuGetPackages.cs
