From c8a695f89ddeb98459fae2960a5108330f036735 Mon Sep 17 00:00:00 2001 From: Christian Rackerseder Date: Tue, 30 Jun 2026 08:03:57 +0200 Subject: [PATCH] ci: add Production rollback workflow Add a first-class manual Production rollback workflow for ADR 0002. The workflow rolls back Production to an existing Production tag after explicit confirmation and a required rollback reason. It validates the selected tag, checks out the tag, builds the deployable artifact, and deploys it to wire-webapp-prod through the Production GitHub Environment. The workflow serializes with other Production deployments and does not create, move, delete, or rewrite release tags. For now the workflow rebuilds from the selected Production tag. This is an interim limitation until durable release artifact storage is available. --- .github/workflows/rollback-production.yml | 263 ++++++++++++++++++++++ bin/releaseMetadata.test.ts | 24 ++ bin/releaseMetadata.ts | 11 + bin/releaseMetadataCli.test.ts | 20 ++ bin/releaseMetadataCli.ts | 9 +- 5 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/rollback-production.yml diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml new file mode 100644 index 00000000000..e9f3dad3966 --- /dev/null +++ b/.github/workflows/rollback-production.yml @@ -0,0 +1,263 @@ +--- +name: Rollback Production + +on: + workflow_dispatch: + inputs: + production_tag: + description: 'Existing Production tag to roll back to, for example 2026-06-29.1-production' + required: true + type: string + reason: + description: 'Reason for the rollback' + required: true + type: string + incident_reference: + description: 'Optional incident reference' + required: false + type: string + confirm_rollback: + description: 'Confirm that this will deploy the selected Production tag to wire-webapp-prod' + required: true + default: false + type: boolean + +permissions: {} + +env: + ARTIFACT_PATH: apps/server/dist/s3/ebs.zip + AWS_REGION: eu-central-1 + PRODUCTION_ENVIRONMENT_NAME: wire-webapp-prod + +jobs: + validate_request: + name: Validate rollback request + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + production_tag: ${{ steps.validate_production_tag.outputs.production_tag }} + + steps: + - name: Validate confirmation and reason + env: + CONFIRM_ROLLBACK: ${{ inputs.confirm_rollback }} + ROLLBACK_REASON: ${{ inputs.reason }} + run: | + set -euo pipefail + + if [[ "${CONFIRM_ROLLBACK}" != 'true' ]]; then + echo 'confirm_rollback must be true to roll Production back.' >&2 + exit 1 + fi + + if [[ -z "${ROLLBACK_REASON//[[:space:]]/}" ]]; then + echo 'reason must not be empty.' >&2 + exit 1 + fi + + - name: Checkout workflow source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + + - name: Add repository Yarn wrapper to PATH + run: echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version-file: '.nvmrc' + cache: 'yarn' + + - name: Install dependencies + run: ./bin/yarn --immutable + + - name: Validate Production tag + id: validate_production_tag + env: + PRODUCTION_TAG: ${{ inputs.production_tag }} + run: | + set -euo pipefail + + validated_production_tag="$(./bin/yarn ts-node --project ./tsconfig.bin.json ./bin/releaseMetadataCli.ts validate-production-tag "${PRODUCTION_TAG}")" + echo "production_tag=${validated_production_tag}" >> "$GITHUB_OUTPUT" + + - name: Verify remote Production tag exists + env: + PRODUCTION_TAG: ${{ steps.validate_production_tag.outputs.production_tag }} + run: | + set -euo pipefail + + git ls-remote --exit-code --tags "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${PRODUCTION_TAG}" >/dev/null + + build_artifact: + name: Rebuild rollback artifact + runs-on: ubuntu-24.04 + needs: validate_request + permissions: + contents: read + outputs: + artifact_name: ${{ steps.artifact_metadata.outputs.artifact_name }} + rollback_commit_sha: ${{ steps.git_metadata.outputs.rollback_commit_sha }} + rollback_commit_subject: ${{ steps.git_metadata.outputs.rollback_commit_subject }} + rollback_commit_url: ${{ steps.git_metadata.outputs.rollback_commit_url }} + + steps: + - name: Checkout selected Production tag + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + ref: refs/tags/${{ needs.validate_request.outputs.production_tag }} + fetch-depth: 0 + + - name: Collect rollback git metadata + id: git_metadata + run: | + set -euo pipefail + + rollback_commit_sha="$(git rev-parse HEAD)" + rollback_commit_subject="$(git show -s --format=%s HEAD)" + rollback_commit_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${rollback_commit_sha}" + + { + echo "rollback_commit_sha=${rollback_commit_sha}" + echo "rollback_commit_subject=${rollback_commit_subject}" + echo "rollback_commit_url=${rollback_commit_url}" + } >> "$GITHUB_OUTPUT" + + - name: Add repository Yarn wrapper to PATH + run: echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version-file: '.nvmrc' + cache: 'yarn' + + - name: Install dependencies + run: ./bin/yarn --immutable + + - name: Update configuration + run: ./bin/yarn nx run webapp:configure + + # Interim limitation: until durable release artifact storage exists, this + # workflow rebuilds the deployable artifact from the selected Production tag. + - name: Build and package + run: ./bin/yarn nx run server:package + + - name: Compute artifact metadata + id: artifact_metadata + env: + PRODUCTION_TAG: ${{ needs.validate_request.outputs.production_tag }} + run: | + set -euo pipefail + + artifact_name="rollback-production-ebs-${PRODUCTION_TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "artifact_name=${artifact_name}" >> "$GITHUB_OUTPUT" + + - name: Upload rollback artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: ${{ steps.artifact_metadata.outputs.artifact_name }} + path: ${{ env.ARTIFACT_PATH }} + + deploy_to_production: + name: Deploy rollback to Production + runs-on: ubuntu-24.04 + needs: [validate_request, build_artifact] + permissions: {} + # Production rollback approval is enforced by the wire-webapp-prod GitHub Environment settings. + environment: wire-webapp-prod + concurrency: + group: release-cloud-production + cancel-in-progress: false + + steps: + - name: Download rollback artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: ${{ needs.build_artifact.outputs.artifact_name }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 + with: + aws-access-key-id: ${{ secrets.WEBTEAM_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.WEBTEAM_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Deploy rollback to Production + id: deploy + uses: aws-actions/aws-elasticbeanstalk-deploy@1f56e4e813ae4eb167e69ca324234c336c1df573 + with: + aws-region: ${{ env.AWS_REGION }} + application-name: Webapp + environment-name: ${{ env.PRODUCTION_ENVIRONMENT_NAME }} + version-label: ${{ github.run_id }}-${{ github.run_attempt }}-rollback-${{ needs.validate_request.outputs.production_tag }} + deployment-package-path: ebs.zip + wait-for-deployment: true + wait-for-environment-recovery: true + deployment-timeout: 150 + use-existing-application-version-if-available: true + + summarize_rollback: + name: Summarize rollback + runs-on: ubuntu-24.04 + needs: [validate_request, build_artifact, deploy_to_production] + if: ${{ always() }} + permissions: {} + + steps: + - name: Add rollback summary + env: + INCIDENT_REFERENCE: ${{ inputs.incident_reference }} + PRODUCTION_ENVIRONMENT_NAME: ${{ env.PRODUCTION_ENVIRONMENT_NAME }} + ROLLBACK_COMMIT_SHA: ${{ needs.build_artifact.outputs.rollback_commit_sha || 'unavailable' }} + ROLLBACK_COMMIT_SUBJECT: ${{ needs.build_artifact.outputs.rollback_commit_subject || 'unavailable' }} + ROLLBACK_COMMIT_URL: ${{ needs.build_artifact.outputs.rollback_commit_url || 'unavailable' }} + ROLLBACK_REASON: ${{ inputs.reason }} + ROLLBACK_TAG: ${{ needs.validate_request.outputs.production_tag || inputs.production_tag }} + WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + { + echo '## Production rollback' + echo "- Rollback tag: ${ROLLBACK_TAG}" + echo "- Rollback commit SHA: ${ROLLBACK_COMMIT_SHA}" + echo "- Rollback commit URL: ${ROLLBACK_COMMIT_URL}" + echo "- Rollback commit subject: ${ROLLBACK_COMMIT_SUBJECT}" + echo "- Production environment: ${PRODUCTION_ENVIRONMENT_NAME}" + echo "- Actor: ${GITHUB_ACTOR}" + echo "- Reason: ${ROLLBACK_REASON}" + + if [[ -n "${INCIDENT_REFERENCE}" ]]; then + echo "- Incident reference: ${INCIDENT_REFERENCE}" + fi + + echo "- Workflow run URL: ${WORKFLOW_RUN_URL}" + echo '- Tag mutation: no release tags were created, moved, deleted, or rewritten.' + echo '- Artifact source: this workflow rebuilds from the selected tag as an interim limitation until durable release artifact storage exists.' + } >> "$GITHUB_STEP_SUMMARY" + + notify_rollback_failure: + name: Notify rollback failure + runs-on: ubuntu-24.04 + needs: [validate_request, build_artifact, deploy_to_production] + if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }} + permissions: {} + + steps: + - name: Announce Production rollback failure + uses: 8398a7/action-slack@293f8dc0f9731ac35321056641cdef895f4f65f8 + env: + SLACK_WEBHOOK_URL: ${{ secrets.WIRE_DEPLOYOHOLICS_WEBHOOK_URL }} + with: + status: failure + text: | + Production rollback failed for tag ${{ needs.validate_request.outputs.production_tag || inputs.production_tag }} + Commit: ${{ needs.build_artifact.outputs.rollback_commit_sha || 'unavailable' }} + Triggered by: ${{ github.actor }} + Reason: ${{ inputs.reason }} + Incident reference: ${{ inputs.incident_reference || 'not provided' }} + Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/bin/releaseMetadata.test.ts b/bin/releaseMetadata.test.ts index 19cd098b531..fd39a19de21 100644 --- a/bin/releaseMetadata.test.ts +++ b/bin/releaseMetadata.test.ts @@ -27,6 +27,7 @@ import { isReleaseBranchName, productionTagExists, productionTagPointsToCommit, + validateProductionTagName, } from './releaseMetadata'; import type {CommitHash, ReleaseTagMetadata} from './releaseMetadata'; @@ -129,6 +130,29 @@ describe('releaseMetadata', () => { expect(actualProductionTagName.error.message).toBe('Invalid release identifier: 2026-06-19'); }); + it('validateProductionTagName() accepts a production tag name', () => { + const productionTagName = '2026-06-19.1-production'; + + const actualProductionTagName = validateProductionTagName(productionTagName); + + assert(actualProductionTagName.isOk === true); + + expect(actualProductionTagName.value).toBe(productionTagName); + }); + + it.each([ + '2026-06-19.0-production', + '2026-06-19-production.1', + '2026-06-19.1-beta.1', + 'release/2026-06-19.1-production', + ])('validateProductionTagName() rejects invalid production tag name "%s"', invalidProductionTagName => { + const actualProductionTagName = validateProductionTagName(invalidProductionTagName); + + assert(actualProductionTagName.isErr === true); + + expect(actualProductionTagName.error.message).toBe(`Invalid production tag name: ${invalidProductionTagName}`); + }); + it('createNextBetaTagName() increments the latest beta tag for the release identifier', () => { const releaseIdentifier = '2026-06-19.1'; const existingTagNames = ['2026-06-19.1-beta.1', '2026-06-19.1-beta.2']; diff --git a/bin/releaseMetadata.ts b/bin/releaseMetadata.ts index 120ea120d98..8206292f7e1 100644 --- a/bin/releaseMetadata.ts +++ b/bin/releaseMetadata.ts @@ -45,6 +45,7 @@ export type ProductionTagPointsToCommitParameters = { const releaseIdentifierPattern = String.raw`\d{4}-\d{2}-\d{2}\.[1-9]\d*`; const releaseBranchNamePattern = new RegExp(`^release/(${releaseIdentifierPattern})$`); +const productionTagNamePattern = new RegExp(`^(${releaseIdentifierPattern})-production$`); function escapeRegularExpression(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); @@ -94,6 +95,16 @@ export function createProductionTagName(releaseIdentifier: string): Result { + const productionTagNameMatches = productionTagNamePattern.test(productionTagName); + + if (!productionTagNameMatches) { + return Result.err(new Error(`Invalid production tag name: ${productionTagName}`)); + } + + return Result.ok(productionTagName as ProductionTagName); +} + export function createNextBetaTagName( releaseIdentifier: string, existingTagNames: readonly string[], diff --git a/bin/releaseMetadataCli.test.ts b/bin/releaseMetadataCli.test.ts index c9dd7030e58..e18408a0e0b 100644 --- a/bin/releaseMetadataCli.test.ts +++ b/bin/releaseMetadataCli.test.ts @@ -118,6 +118,26 @@ describe('releaseMetadataCli', () => { }); }); + it('validates a production tag name', () => { + const actualResult = runCommand(['validate-production-tag', '2026-06-19.1-production']); + + expect(actualResult).toEqual({ + errors: [], + exitCode: 0, + outputs: ['2026-06-19.1-production'], + }); + }); + + it('rejects an invalid production tag name', () => { + const actualResult = runCommand(['validate-production-tag', '2026-06-19.0-production']); + + expect(actualResult).toEqual({ + errors: ['Invalid production tag name: 2026-06-19.0-production'], + exitCode: 1, + outputs: [], + }); + }); + it('prints usage text for missing command arguments', () => { const actualResult = runCommand(['next-beta-tag']); diff --git a/bin/releaseMetadataCli.ts b/bin/releaseMetadataCli.ts index c4b54c651f5..ee143398b45 100644 --- a/bin/releaseMetadataCli.ts +++ b/bin/releaseMetadataCli.ts @@ -24,6 +24,7 @@ import { createProductionTagName, createReleaseBranchName, extractReleaseIdentifierFromBranchName, + validateProductionTagName, } from './releaseMetadata'; type ReleaseMetadataCliDependencies = { @@ -39,6 +40,7 @@ const usageText = [ ' releaseMetadataCli.ts release-branch ', ' releaseMetadataCli.ts next-beta-tag [existing-tag ...]', ' releaseMetadataCli.ts production-tag ', + ' releaseMetadataCli.ts validate-production-tag ', ].join('\n'); function writeResult( @@ -46,7 +48,8 @@ function writeResult( | ReturnType | ReturnType | ReturnType - | ReturnType, + | ReturnType + | ReturnType, dependencies: ReleaseMetadataCliDependencies, ): number { if (result.isErr) { @@ -80,6 +83,10 @@ export function runReleaseMetadataCli( return writeResult(createProductionTagName(primaryValue), dependencies); } + if (commandName === 'validate-production-tag' && primaryValue !== undefined) { + return writeResult(validateProductionTagName(primaryValue), dependencies); + } + dependencies.writeError(usageText); return 1; }