Skip to content
Draft

Pg x86 #2110

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion .github/actions/build-ami/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ inputs:
description: 'Prefix for the AMI name'
required: false
default: 'supabase-postgres'
packer_template:
description: 'Packer template for stage 1 (e.g., amazon-arm64-nix.pkr.hcl)'
required: false
default: 'amazon-arm64-nix.pkr.hcl'
instance_type:
description: 'EC2 instance type for the build'
required: false
default: 'c6g.4xlarge'

outputs:
stage2_ami_id:
Expand Down Expand Up @@ -65,7 +73,7 @@ runs:
-var "packer-execution-id=${{ env.EXECUTION_ID }}" \
-var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \
-var 'ami_regions=${{ inputs.ami_regions }}' \
amazon-arm64-nix.pkr.hcl
${{ inputs.packer_template }}

- name: Build AMI stage 2
id: build-stage2
Expand All @@ -84,4 +92,5 @@ runs:
-var "postgres_major_version=${{ inputs.postgres_version }}" \
-var "ami_name=${{ inputs.ami_name_prefix }}" \
-var "git_sha=${{ inputs.git_sha }}" \
-var "instance_type=${{ inputs.instance_type }}" \
stage2-nix-psql.pkr.hcl
55 changes: 44 additions & 11 deletions .github/workflows/ami-release-nix-single.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,22 @@ on:
required: true
type: string
default: 'main'
arch:
description: 'Architecture to build'
required: true
type: choice
options:
- arm64
- amd64
default: arm64

permissions:
contents: write
id-token: write

jobs:
build:
runs-on: large-linux-arm
runs-on: ${{ github.event.inputs.arch == 'amd64' && 'blacksmith-2vcpu-ubuntu-2404' || 'large-linux-arm' }}
timeout-minutes: 150

steps:
Expand All @@ -41,6 +49,26 @@ jobs:
run: |
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

- name: Set arch-specific variables
id: arch_vars
run: |
ARCH="${{ github.event.inputs.arch }}"
if [ "$ARCH" = "amd64" ]; then
{
echo "packer_template=amazon-amd64-nix.pkr.hcl"
echo "instance_type=c6i.4xlarge"
echo "ami_name_prefix=supabase-postgres-x86"
echo "version_suffix=-x86"
} >> "$GITHUB_OUTPUT"
else
{
echo "packer_template=amazon-arm64-nix.pkr.hcl"
echo "instance_type=c6g.4xlarge"
echo "ami_name_prefix=supabase-postgres"
echo "version_suffix="
} >> "$GITHUB_OUTPUT"
fi

- name: Install nix
uses: ./.github/actions/nix-install-ephemeral
with:
Expand All @@ -57,6 +85,9 @@ jobs:
region: us-east-1
ami_regions: '["us-east-1"]'
git_sha: ${{ steps.get_sha.outputs.sha }}
packer_template: ${{ steps.arch_vars.outputs.packer_template }}
instance_type: ${{ steps.arch_vars.outputs.instance_type }}
ami_name_prefix: ${{ steps.arch_vars.outputs.ami_name_prefix }}

- name: Grab release version
id: process_release_version
Expand All @@ -83,16 +114,17 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}" \
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
-e "arch=${{ github.event.inputs.arch }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 staging
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/upgrade_bundle.tar.gz

- name: configure aws credentials - prod
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
Expand All @@ -104,22 +136,23 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}" \
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
-e "arch=${{ github.event.inputs.arch }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 prod
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/upgrade_bundle.tar.gz

- name: Create release
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0
with:
name: ${{ steps.process_release_version.outputs.version }}
tag_name: ${{ steps.process_release_version.outputs.version }}
name: ${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}
tag_name: ${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}
target_commitish: ${{ steps.get_sha.outputs.sha }}

- name: Slack Notification on Failure
Expand Down
78 changes: 56 additions & 22 deletions .github/workflows/ami-release-nix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,24 @@ jobs:
strategy:
matrix:
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
include:
- runner: blacksmith-2vcpu-ubuntu-2404-arm
runs-on: ${{ matrix.runner }}
arch:
- name: arm64
runner: blacksmith-2vcpu-ubuntu-2404-arm
packer_template: amazon-arm64-nix.pkr.hcl
vars_file: development-arm.vars.pkr.hcl
instance_type: c6g.4xlarge
nix_system: aarch64-linux
ami_name: supabase-postgres
ami_arch_filter: arm64
- name: amd64
runner: blacksmith-2vcpu-ubuntu-2404
packer_template: amazon-amd64-nix.pkr.hcl
vars_file: development-x86.vars.pkr.hcl
instance_type: c6i.4xlarge
nix_system: x86_64-linux
ami_name: supabase-postgres-x86
ami_arch_filter: x86_64
runs-on: ${{ matrix.arch.runner }}
timeout-minutes: 150

steps:
Expand All @@ -67,7 +82,7 @@ jobs:
- name: Set PostgreSQL version environment variable
run: |
echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV"
echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> "$GITHUB_ENV"
echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}-${{ matrix.arch.name }}" >> "$GITHUB_ENV"

- name: Generate common-nix.vars.pkr.hcl
run: |
Expand All @@ -79,9 +94,8 @@ jobs:
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
run: |
GIT_SHA=${{github.sha}}
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
# why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
nix run github:supabase/postgres/${GIT_SHA}#packer -- init ${{ matrix.arch.packer_template }}
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "ami_name=${{ matrix.arch.ami_name }}" ${{ matrix.arch.packer_template }}

- name: Find stage 1 AMI
run: |
Expand All @@ -98,6 +112,7 @@ jobs:
"Name=tag:postgresVersion,Values=${PG_VERSION}-stage1" \
"Name=tag:sourceSha,Values=${GIT_SHA}" \
"Name=state,Values=available" \
"Name=architecture,Values=${{ matrix.arch.ami_arch_filter }}" \
--query 'Images[0].ImageId' \
--output text)

Expand All @@ -115,8 +130,7 @@ jobs:
run: |
GIT_SHA=${{github.sha}}
nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" stage2-nix-psql.pkr.hcl
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var "instance_type=${{ matrix.arch.instance_type }}" -var "ami_name=${{ matrix.arch.ami_name }}" stage2-nix-psql.pkr.hcl

- name: Grab release version
id: process_release_version
Expand All @@ -125,6 +139,22 @@ jobs:
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "::notice title=AMI Published::Postgres AMI version: $VERSION"

- name: Set arch-qualified version
id: arch_version
run: |
VERSION="${{ steps.process_release_version.outputs.version }}"
if [ "${{ matrix.arch.name }}" = "amd64" ]; then
{
echo "s3_version=${VERSION}-x86"
echo "release_tag=${VERSION}-x86"
} >> "$GITHUB_OUTPUT"
else
{
echo "s3_version=${VERSION}"
echo "release_tag=${VERSION}"
} >> "$GITHUB_OUTPUT"
fi

- name: Create nix flake revision tarball
run: |
GIT_SHA=${{github.sha}}
Expand All @@ -144,16 +174,17 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.arch_version.outputs.s3_version }}" \
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ matrix.postgres_version }}" \
-e "arch=${{ matrix.arch.name }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 staging
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/upgrade_bundle.tar.gz

- name: configure aws credentials - prod
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
Expand All @@ -165,16 +196,17 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.arch_version.outputs.s3_version }}" \
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ matrix.postgres_version }}" \
-e "arch=${{ matrix.arch.name }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 prod
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/upgrade_bundle.tar.gz

- name: GitHub OIDC Auth
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
Expand All @@ -197,13 +229,13 @@ jobs:
VERSION="${{ steps.process_release_version.outputs.version }}"
GIT_SHA="${{ github.sha }}"
PG_VERSION="${{ matrix.postgres_version }}"
SYSTEM="aarch64-linux"
SYSTEM="${{ matrix.arch.nix_system }}"

# Get store path for this build
STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath")

# Each postgres version gets its own catalog file (no race conditions)
CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}.json"
CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}-${SYSTEM}.json"

# Create catalog JSON for this version
jq -n \
Expand All @@ -225,12 +257,12 @@ jobs:
- name: Create release
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0
with:
name: ${{ steps.process_release_version.outputs.version }}
tag_name: ${{ steps.process_release_version.outputs.version }}
name: ${{ steps.arch_version.outputs.release_tag }}
tag_name: ${{ steps.arch_version.outputs.release_tag }}
target_commitish: ${{github.sha}}

- name: Create CLI tag for PG 17
if: matrix.postgres_version == '17' && github.event_name != 'workflow_dispatch'
if: matrix.postgres_version == '17' && matrix.arch.name == 'arm64' && github.event_name != 'workflow_dispatch'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand All @@ -241,6 +273,7 @@ jobs:
git push origin "${CLI_TAG}"

- name: Trigger pg_upgrade_scripts workflow
if: matrix.arch.name == 'arm64'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand All @@ -249,6 +282,7 @@ jobs:
-f postgresVersion="${{ steps.process_release_version.outputs.version }}"

- name: Trigger pg_upgrade_bin flake version workflow
if: matrix.arch.name == 'arm64'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand Down
Loading
Loading