diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml new file mode 100644 index 00000000000..e9f3dad3966 --- /dev/null +++ b/.github/workflows/rollback-production.yml @@ -0,0 +1,263 @@ +--- +name: Rollback Production + +on: + workflow_dispatch: + inputs: + production_tag: + description: 'Existing Production tag to roll back to, for example 2026-06-29.1-production' + required: true + type: string + reason: + description: 'Reason for the rollback' + required: true + type: string + incident_reference: + description: 'Optional incident reference' + required: false + type: string + confirm_rollback: + description: 'Confirm that this will deploy the selected Production tag to wire-webapp-prod' + required: true + default: false + type: boolean + +permissions: {} + +env: + ARTIFACT_PATH: apps/server/dist/s3/ebs.zip + AWS_REGION: eu-central-1 + PRODUCTION_ENVIRONMENT_NAME: wire-webapp-prod + +jobs: + validate_request: + name: Validate rollback request + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + production_tag: ${{ steps.validate_production_tag.outputs.production_tag }} + + steps: + - name: Validate confirmation and reason + env: + CONFIRM_ROLLBACK: ${{ inputs.confirm_rollback }} + ROLLBACK_REASON: ${{ inputs.reason }} + run: | + set -euo pipefail + + if [[ "${CONFIRM_ROLLBACK}" != 'true' ]]; then + echo 'confirm_rollback must be true to roll Production back.' >&2 + exit 1 + fi + + if [[ -z "${ROLLBACK_REASON//[[:space:]]/}" ]]; then + echo 'reason must not be empty.' >&2 + exit 1 + fi + + - name: Checkout workflow source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + + - name: Add repository Yarn wrapper to PATH + run: echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version-file: '.nvmrc' + cache: 'yarn' + + - name: Install dependencies + run: ./bin/yarn --immutable + + - name: Validate Production tag + id: validate_production_tag + env: + PRODUCTION_TAG: ${{ inputs.production_tag }} + run: | + set -euo pipefail + + validated_production_tag="$(./bin/yarn ts-node --project ./tsconfig.bin.json ./bin/releaseMetadataCli.ts validate-production-tag "${PRODUCTION_TAG}")" + echo "production_tag=${validated_production_tag}" >> "$GITHUB_OUTPUT" + + - name: Verify remote Production tag exists + env: + PRODUCTION_TAG: ${{ steps.validate_production_tag.outputs.production_tag }} + run: | + set -euo pipefail + + git ls-remote --exit-code --tags "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${PRODUCTION_TAG}" >/dev/null + + build_artifact: + name: Rebuild rollback artifact + runs-on: ubuntu-24.04 + needs: validate_request + permissions: + contents: read + outputs: + artifact_name: ${{ steps.artifact_metadata.outputs.artifact_name }} + rollback_commit_sha: ${{ steps.git_metadata.outputs.rollback_commit_sha }} + rollback_commit_subject: ${{ steps.git_metadata.outputs.rollback_commit_subject }} + rollback_commit_url: ${{ steps.git_metadata.outputs.rollback_commit_url }} + + steps: + - name: Checkout selected Production tag + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + ref: refs/tags/${{ needs.validate_request.outputs.production_tag }} + fetch-depth: 0 + + - name: Collect rollback git metadata + id: git_metadata + run: | + set -euo pipefail + + rollback_commit_sha="$(git rev-parse HEAD)" + rollback_commit_subject="$(git show -s --format=%s HEAD)" + rollback_commit_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${rollback_commit_sha}" + + { + echo "rollback_commit_sha=${rollback_commit_sha}" + echo "rollback_commit_subject=${rollback_commit_subject}" + echo "rollback_commit_url=${rollback_commit_url}" + } >> "$GITHUB_OUTPUT" + + - name: Add repository Yarn wrapper to PATH + run: echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version-file: '.nvmrc' + cache: 'yarn' + + - name: Install dependencies + run: ./bin/yarn --immutable + + - name: Update configuration + run: ./bin/yarn nx run webapp:configure + + # Interim limitation: until durable release artifact storage exists, this + # workflow rebuilds the deployable artifact from the selected Production tag. + - name: Build and package + run: ./bin/yarn nx run server:package + + - name: Compute artifact metadata + id: artifact_metadata + env: + PRODUCTION_TAG: ${{ needs.validate_request.outputs.production_tag }} + run: | + set -euo pipefail + + artifact_name="rollback-production-ebs-${PRODUCTION_TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "artifact_name=${artifact_name}" >> "$GITHUB_OUTPUT" + + - name: Upload rollback artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: ${{ steps.artifact_metadata.outputs.artifact_name }} + path: ${{ env.ARTIFACT_PATH }} + + deploy_to_production: + name: Deploy rollback to Production + runs-on: ubuntu-24.04 + needs: [validate_request, build_artifact] + permissions: {} + # Production rollback approval is enforced by the wire-webapp-prod GitHub Environment settings. + environment: wire-webapp-prod + concurrency: + group: release-cloud-production + cancel-in-progress: false + + steps: + - name: Download rollback artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: ${{ needs.build_artifact.outputs.artifact_name }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 + with: + aws-access-key-id: ${{ secrets.WEBTEAM_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.WEBTEAM_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Deploy rollback to Production + id: deploy + uses: aws-actions/aws-elasticbeanstalk-deploy@1f56e4e813ae4eb167e69ca324234c336c1df573 + with: + aws-region: ${{ env.AWS_REGION }} + application-name: Webapp + environment-name: ${{ env.PRODUCTION_ENVIRONMENT_NAME }} + version-label: ${{ github.run_id }}-${{ github.run_attempt }}-rollback-${{ needs.validate_request.outputs.production_tag }} + deployment-package-path: ebs.zip + wait-for-deployment: true + wait-for-environment-recovery: true + deployment-timeout: 150 + use-existing-application-version-if-available: true + + summarize_rollback: + name: Summarize rollback + runs-on: ubuntu-24.04 + needs: [validate_request, build_artifact, deploy_to_production] + if: ${{ always() }} + permissions: {} + + steps: + - name: Add rollback summary + env: + INCIDENT_REFERENCE: ${{ inputs.incident_reference }} + PRODUCTION_ENVIRONMENT_NAME: ${{ env.PRODUCTION_ENVIRONMENT_NAME }} + ROLLBACK_COMMIT_SHA: ${{ needs.build_artifact.outputs.rollback_commit_sha || 'unavailable' }} + ROLLBACK_COMMIT_SUBJECT: ${{ needs.build_artifact.outputs.rollback_commit_subject || 'unavailable' }} + ROLLBACK_COMMIT_URL: ${{ needs.build_artifact.outputs.rollback_commit_url || 'unavailable' }} + ROLLBACK_REASON: ${{ inputs.reason }} + ROLLBACK_TAG: ${{ needs.validate_request.outputs.production_tag || inputs.production_tag }} + WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + { + echo '## Production rollback' + echo "- Rollback tag: ${ROLLBACK_TAG}" + echo "- Rollback commit SHA: ${ROLLBACK_COMMIT_SHA}" + echo "- Rollback commit URL: ${ROLLBACK_COMMIT_URL}" + echo "- Rollback commit subject: ${ROLLBACK_COMMIT_SUBJECT}" + echo "- Production environment: ${PRODUCTION_ENVIRONMENT_NAME}" + echo "- Actor: ${GITHUB_ACTOR}" + echo "- Reason: ${ROLLBACK_REASON}" + + if [[ -n "${INCIDENT_REFERENCE}" ]]; then + echo "- Incident reference: ${INCIDENT_REFERENCE}" + fi + + echo "- Workflow run URL: ${WORKFLOW_RUN_URL}" + echo '- Tag mutation: no release tags were created, moved, deleted, or rewritten.' + echo '- Artifact source: this workflow rebuilds from the selected tag as an interim limitation until durable release artifact storage exists.' + } >> "$GITHUB_STEP_SUMMARY" + + notify_rollback_failure: + name: Notify rollback failure + runs-on: ubuntu-24.04 + needs: [validate_request, build_artifact, deploy_to_production] + if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }} + permissions: {} + + steps: + - name: Announce Production rollback failure + uses: 8398a7/action-slack@293f8dc0f9731ac35321056641cdef895f4f65f8 + env: + SLACK_WEBHOOK_URL: ${{ secrets.WIRE_DEPLOYOHOLICS_WEBHOOK_URL }} + with: + status: failure + text: | + Production rollback failed for tag ${{ needs.validate_request.outputs.production_tag || inputs.production_tag }} + Commit: ${{ needs.build_artifact.outputs.rollback_commit_sha || 'unavailable' }} + Triggered by: ${{ github.actor }} + Reason: ${{ inputs.reason }} + Incident reference: ${{ inputs.incident_reference || 'not provided' }} + Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/bin/releaseMetadata.test.ts b/bin/releaseMetadata.test.ts index 19cd098b531..fd39a19de21 100644 --- a/bin/releaseMetadata.test.ts +++ b/bin/releaseMetadata.test.ts @@ -27,6 +27,7 @@ import { isReleaseBranchName, productionTagExists, productionTagPointsToCommit, + validateProductionTagName, } from './releaseMetadata'; import type {CommitHash, ReleaseTagMetadata} from './releaseMetadata'; @@ -129,6 +130,29 @@ describe('releaseMetadata', () => { expect(actualProductionTagName.error.message).toBe('Invalid release identifier: 2026-06-19'); }); + it('validateProductionTagName() accepts a production tag name', () => { + const productionTagName = '2026-06-19.1-production'; + + const actualProductionTagName = validateProductionTagName(productionTagName); + + assert(actualProductionTagName.isOk === true); + + expect(actualProductionTagName.value).toBe(productionTagName); + }); + + it.each([ + '2026-06-19.0-production', + '2026-06-19-production.1', + '2026-06-19.1-beta.1', + 'release/2026-06-19.1-production', + ])('validateProductionTagName() rejects invalid production tag name "%s"', invalidProductionTagName => { + const actualProductionTagName = validateProductionTagName(invalidProductionTagName); + + assert(actualProductionTagName.isErr === true); + + expect(actualProductionTagName.error.message).toBe(`Invalid production tag name: ${invalidProductionTagName}`); + }); + it('createNextBetaTagName() increments the latest beta tag for the release identifier', () => { const releaseIdentifier = '2026-06-19.1'; const existingTagNames = ['2026-06-19.1-beta.1', '2026-06-19.1-beta.2']; diff --git a/bin/releaseMetadata.ts b/bin/releaseMetadata.ts index 120ea120d98..8206292f7e1 100644 --- a/bin/releaseMetadata.ts +++ b/bin/releaseMetadata.ts @@ -45,6 +45,7 @@ export type ProductionTagPointsToCommitParameters = { const releaseIdentifierPattern = String.raw`\d{4}-\d{2}-\d{2}\.[1-9]\d*`; const releaseBranchNamePattern = new RegExp(`^release/(${releaseIdentifierPattern})$`); +const productionTagNamePattern = new RegExp(`^(${releaseIdentifierPattern})-production$`); function escapeRegularExpression(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); @@ -94,6 +95,16 @@ export function createProductionTagName(releaseIdentifier: string): Result { + const productionTagNameMatches = productionTagNamePattern.test(productionTagName); + + if (!productionTagNameMatches) { + return Result.err(new Error(`Invalid production tag name: ${productionTagName}`)); + } + + return Result.ok(productionTagName as ProductionTagName); +} + export function createNextBetaTagName( releaseIdentifier: string, existingTagNames: readonly string[], diff --git a/bin/releaseMetadataCli.test.ts b/bin/releaseMetadataCli.test.ts index c9dd7030e58..e18408a0e0b 100644 --- a/bin/releaseMetadataCli.test.ts +++ b/bin/releaseMetadataCli.test.ts @@ -118,6 +118,26 @@ describe('releaseMetadataCli', () => { }); }); + it('validates a production tag name', () => { + const actualResult = runCommand(['validate-production-tag', '2026-06-19.1-production']); + + expect(actualResult).toEqual({ + errors: [], + exitCode: 0, + outputs: ['2026-06-19.1-production'], + }); + }); + + it('rejects an invalid production tag name', () => { + const actualResult = runCommand(['validate-production-tag', '2026-06-19.0-production']); + + expect(actualResult).toEqual({ + errors: ['Invalid production tag name: 2026-06-19.0-production'], + exitCode: 1, + outputs: [], + }); + }); + it('prints usage text for missing command arguments', () => { const actualResult = runCommand(['next-beta-tag']); diff --git a/bin/releaseMetadataCli.ts b/bin/releaseMetadataCli.ts index c4b54c651f5..ee143398b45 100644 --- a/bin/releaseMetadataCli.ts +++ b/bin/releaseMetadataCli.ts @@ -24,6 +24,7 @@ import { createProductionTagName, createReleaseBranchName, extractReleaseIdentifierFromBranchName, + validateProductionTagName, } from './releaseMetadata'; type ReleaseMetadataCliDependencies = { @@ -39,6 +40,7 @@ const usageText = [ ' releaseMetadataCli.ts release-branch ', ' releaseMetadataCli.ts next-beta-tag [existing-tag ...]', ' releaseMetadataCli.ts production-tag ', + ' releaseMetadataCli.ts validate-production-tag ', ].join('\n'); function writeResult( @@ -46,7 +48,8 @@ function writeResult( | ReturnType | ReturnType | ReturnType - | ReturnType, + | ReturnType + | ReturnType, dependencies: ReleaseMetadataCliDependencies, ): number { if (result.isErr) { @@ -80,6 +83,10 @@ export function runReleaseMetadataCli( return writeResult(createProductionTagName(primaryValue), dependencies); } + if (commandName === 'validate-production-tag' && primaryValue !== undefined) { + return writeResult(validateProductionTagName(primaryValue), dependencies); + } + dependencies.writeError(usageText); return 1; }