fix: reset reserved blocks on the data disk after resize (#2123) #1059
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release AMI Nix | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| - release/* | |
| paths: | |
| - '.github/workflows/ami-release-nix.yml' | |
| - 'common-nix.vars.pkr.hcl' | |
| - 'ansible/vars.yml' | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| id-token: write | |
| actions: write | |
| jobs: | |
| prepare: | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| outputs: | |
| postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} | |
| steps: | |
| - name: Checkout Repo | |
| uses: supabase/postgres/.github/actions/shared-checkout@HEAD | |
| - name: Install nix | |
| uses: ./.github/actions/nix-install-ephemeral | |
| - name: Set PostgreSQL versions | |
| id: set-versions | |
| run: | | |
| VERSIONS=$(nix run nixpkgs#yq-go -- -o=json -I=0 '.postgres_major' ansible/vars.yml) | |
| echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT" | |
| build: | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} | |
| include: | |
| - runner: blacksmith-2vcpu-ubuntu-2404-arm | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 150 | |
| steps: | |
| - name: Checkout Repo | |
| uses: supabase/postgres/.github/actions/shared-checkout@HEAD | |
| - name: aws-creds | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.DEV_AWS_ROLE }} | |
| aws-region: "us-east-1" | |
| output-credentials: true | |
| role-duration-seconds: 7200 | |
| - name: Install nix | |
| uses: ./.github/actions/nix-install-ephemeral | |
| with: | |
| push-to-cache: 'true' | |
| env: | |
| DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} | |
| NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} | |
| - name: Set PostgreSQL version environment variable | |
| run: | | |
| echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV" | |
| echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> "$GITHUB_ENV" | |
| - name: Generate common-nix.vars.pkr.hcl | |
| run: | | |
| PG_VERSION="$(nix run nixpkgs#yq-go -- -r '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)" | |
| echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl | |
| - name: Build AMI stage 1 | |
| env: | |
| POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} | |
| run: | | |
| GIT_SHA=${{github.sha}} | |
| nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl | |
| # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined? | |
| nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl | |
| - name: Find stage 1 AMI | |
| run: | | |
| GIT_SHA=${{github.sha}} | |
| PG_VERSION=$(sed -n 's/postgres-version = "\(.*\)"/\1/p' common-nix.vars.pkr.hcl) | |
| REGION="us-east-1" | |
| echo "Looking for stage 1 AMI with postgresVersion=${PG_VERSION}-stage1 and sourceSha=${GIT_SHA} in region ${REGION}" | |
| STAGE1_AMI_ID=$(aws ec2 describe-images \ | |
| --region "$REGION" \ | |
| --owners self \ | |
| --filters \ | |
| "Name=tag:postgresVersion,Values=${PG_VERSION}-stage1" \ | |
| "Name=tag:sourceSha,Values=${GIT_SHA}" \ | |
| "Name=state,Values=available" \ | |
| --query 'Images[0].ImageId' \ | |
| --output text) | |
| if [ -z "$STAGE1_AMI_ID" ] || [ "$STAGE1_AMI_ID" = "None" ]; then | |
| echo "ERROR: Failed to find stage 1 AMI" | |
| exit 1 | |
| fi | |
| echo "Found stage 1 AMI: $STAGE1_AMI_ID" | |
| echo "STAGE1_AMI_ID=$STAGE1_AMI_ID" >> "$GITHUB_ENV" | |
| - name: Build AMI stage 2 | |
| env: | |
| POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} | |
| run: | | |
| GIT_SHA=${{github.sha}} | |
| nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl | |
| POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} | |
| nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" stage2-nix-psql.pkr.hcl | |
| - name: Grab release version | |
| id: process_release_version | |
| run: | | |
| VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "::notice title=AMI Published::Postgres AMI version: $VERSION" | |
| - name: Create nix flake revision tarball | |
| run: | | |
| GIT_SHA=${{github.sha}} | |
| MAJOR_VERSION=${{ matrix.postgres_version }} | |
| mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" | |
| echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" | |
| tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" . | |
| - name: configure aws credentials - staging | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.DEV_AWS_ROLE }} | |
| aws-region: "us-east-1" | |
| - name: Upload software manifest to s3 staging | |
| run: | | |
| cd ansible | |
| ansible-playbook -i localhost \ | |
| -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ | |
| -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ | |
| -e "postgres_major_version=${{ matrix.postgres_version }}" \ | |
| manifest-playbook.yml | |
| - name: Upload nix flake revision to s3 staging | |
| run: | | |
| aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz | |
| aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz | |
| aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz | |
| - name: configure aws credentials - prod | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.PROD_AWS_ROLE }} | |
| aws-region: "us-east-1" | |
| - name: Upload software manifest to s3 prod | |
| run: | | |
| cd ansible | |
| ansible-playbook -i localhost \ | |
| -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ | |
| -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ | |
| -e "postgres_major_version=${{ matrix.postgres_version }}" \ | |
| manifest-playbook.yml | |
| - name: Upload nix flake revision to s3 prod | |
| run: | | |
| aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz | |
| aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz | |
| aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz | |
| - name: GitHub OIDC Auth | |
| uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 | |
| with: | |
| aws-region: ap-southeast-1 | |
| role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role | |
| role-session-name: shared-services-jump | |
| - name: Assume destination role | |
| uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 | |
| with: | |
| aws-region: ap-southeast-1 | |
| role-to-assume: arn:aws:iam::279559813984:role/supabase-nix-catalog-artifacts-role-6387512 | |
| role-skip-session-tagging: true | |
| role-session-name: upload-assets | |
| role-chaining: true | |
| - name: Update nix store path catalog | |
| run: | | |
| VERSION="${{ steps.process_release_version.outputs.version }}" | |
| GIT_SHA="${{ github.sha }}" | |
| PG_VERSION="${{ matrix.postgres_version }}" | |
| SYSTEM="aarch64-linux" | |
| # Get store path for this build | |
| STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath") | |
| # Each postgres version gets its own catalog file (no race conditions) | |
| CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}.json" | |
| # Create catalog JSON for this version | |
| jq -n \ | |
| --arg ver "$VERSION" \ | |
| --arg sha "$GIT_SHA" \ | |
| --arg sys "$SYSTEM" \ | |
| --arg path "$STORE_PATH" \ | |
| '{version: $ver, git_sha: $sha, ($sys): $path}' > /tmp/catalog.json | |
| echo "Catalog for psql_${PG_VERSION}:" | |
| cat /tmp/catalog.json | |
| # Upload catalog | |
| aws s3 cp /tmp/catalog.json "$CATALOG_S3" \ | |
| --content-type "application/json" | |
| echo "Catalog uploaded to ${CATALOG_S3}" | |
| - name: Create release | |
| uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 | |
| with: | |
| name: ${{ steps.process_release_version.outputs.version }} | |
| tag_name: ${{ steps.process_release_version.outputs.version }} | |
| target_commitish: ${{github.sha}} | |
| - name: Create CLI tag for PG 17 | |
| if: matrix.postgres_version == '17' && github.event_name != 'workflow_dispatch' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| VERSION="${{ steps.process_release_version.outputs.version }}" | |
| CLI_TAG="v${VERSION}-cli" | |
| echo "Creating CLI tag: ${CLI_TAG}" | |
| git tag "${CLI_TAG}" "${{ github.sha }}" | |
| git push origin "${CLI_TAG}" | |
| - name: Trigger pg_upgrade_scripts workflow | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh workflow run publish-nix-pgupgrade-scripts.yml \ | |
| --ref "${{ github.ref_name }}" \ | |
| -f postgresVersion="${{ steps.process_release_version.outputs.version }}" | |
| - name: Trigger pg_upgrade_bin flake version workflow | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh workflow run publish-nix-pgupgrade-bin-flake-version.yml \ | |
| --ref "${{ github.ref_name }}" \ | |
| -f postgresVersion="${{ steps.process_release_version.outputs.version }}" | |
| - name: Slack Notification on Failure | |
| if: ${{ failure() }} | |
| uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3 | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} | |
| SLACK_USERNAME: 'gha-failures-notifier' | |
| SLACK_COLOR: 'danger' | |
| SLACK_MESSAGE: 'Building Postgres AMI failed' | |
| SLACK_FOOTER: '' | |
| - name: configure aws credentials for cleanup | |
| if: ${{ always() }} | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.DEV_AWS_ROLE }} | |
| aws-region: "us-east-1" | |
| - name: Cleanup resources after build | |
| if: ${{ always() }} | |
| run: | | |
| EXECUTION_ID="${{ env.EXECUTION_ID }}" | |
| aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids | |
| - name: Cleanup resources on build cancellation | |
| if: ${{ cancelled() }} | |
| run: | | |
| EXECUTION_ID="${{ env.EXECUTION_ID }}" | |
| aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids |