Merge develop into INDATA-378-08 and resolve conflicts

Author: hunleyd

.github/actions/build-ami/action.yml

@@ -2,30 +2,27 @@ name: Build AMI
 description: Build both stage 1 and stage 2 AMIs
 
 inputs:
-  postgres_version:
-    description: 'PostgreSQL major version (e.g., 15)'
-    required: true
-  region:
-    description: 'AWS region'
+  ami_name_prefix:
+    description: 'Prefix for the AMI name'
     required: true
   ami_regions:
     description: 'AMI regions as JSON array (e.g., ["us-east-1"])'
     required: true
+  arch:
+    description: Architecture to build AMI for (amd64|arm64)
+    required: true
   git_sha:
     description: 'Git SHA for this build'
     required: true
-  ami_name_prefix:
-    description: 'Prefix for the AMI name'
-    required: false
-    default: 'supabase-postgres'
-  packer_template:
-    description: 'Packer template for stage 1 (e.g., amazon-arm64-nix.pkr.hcl)'
-    required: false
-    default: 'amazon-arm64-nix.pkr.hcl'
   instance_type:
     description: 'EC2 instance type for the build'
-    required: false
-    default: 'c6g.4xlarge'
+    required: true
+  postgres_version:
+    description: 'PostgreSQL major version (e.g., 15)'
+    required: true
+  region:
+    description: 'AWS region'
+    required: true
 
 outputs:
   stage2_ami_id:
@@ -41,6 +38,13 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Verify arch
+      shell: bash
+      run: |
+        case ${{ inputs.arch }} in
+        amd64 | arm64) ;;
+        *) echo "Unknown arch input, expected:(amd64|arm64) got:${{ inputs.arch }}" >&2 && exit 1 ;;
+        esac
     - name: Set execution ID
       id: set-execution-id
       shell: bash
@@ -68,12 +72,12 @@ runs:
         AWS_RETRY_MODE: adaptive
         AWS_REGION: ${{ inputs.region }}
       run: |
-        nix run .#build-ami -- stage1 \
+        nix run .#build-ami -- stage1 ${{ inputs.arch }} \
           -var "git-head-version=${{ inputs.git_sha }}" \
           -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
           -var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \
           -var 'ami_regions=${{ inputs.ami_regions }}' \
-          ${{ inputs.packer_template }}
+          amazon-${{ inputs.arch }}-nix.pkr.hcl
 
     - name: Build AMI stage 2
       id: build-stage2
@@ -86,7 +90,7 @@ runs:
         AWS_RETRY_MODE: adaptive
         AWS_REGION: ${{ inputs.region }}
       run: |
-        nix run .#build-ami -- stage2 \
+        nix run .#build-ami -- stage2 ${{ inputs.arch }} \
           -var "git-head-version=${{ inputs.git_sha }}" \
           -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
           -var "postgres_major_version=${{ inputs.postgres_version }}" \

.github/workflows/ami-release-nix-single.yml

@@ -19,7 +19,6 @@ on:
         options:
           - arm64
           - amd64
-        default: arm64
 
 permissions:
   contents: write
@@ -53,19 +52,18 @@ jobs:
         id: arch_vars
         run: |
           ARCH="${{ github.event.inputs.arch }}"
+          echo "arch=$ARCH" >>"$GITHUB_OUTPUT"
           if [ "$ARCH" = "amd64" ]; then
             {
-              echo "packer_template=amazon-amd64-nix.pkr.hcl"
-              echo "instance_type=c6i.4xlarge"
               echo "ami_name_prefix=supabase-postgres-x86"
               echo "arch_suffix=-x86"
+              echo "instance_type=c6i.4xlarge"
             } >> "$GITHUB_OUTPUT"
           else
             {
-              echo "packer_template=amazon-arm64-nix.pkr.hcl"
-              echo "instance_type=c6g.4xlarge"
               echo "ami_name_prefix=supabase-postgres"
               echo "arch_suffix="
+              echo "instance_type=c6g.4xlarge"
             } >> "$GITHUB_OUTPUT"
           fi
 
@@ -83,11 +81,11 @@ jobs:
         with:
           postgres_version: ${{ github.event.inputs.postgres_version }}
           region: us-east-1
+          ami_name_prefix: ${{ steps.arch_vars.outputs.ami_name_prefix }}
           ami_regions: '["us-east-1"]'
+          arch: ${{ steps.arch_vars.outputs.arch }}
           git_sha: ${{ steps.get_sha.outputs.sha }}
-          packer_template: ${{ steps.arch_vars.outputs.packer_template }}
           instance_type: ${{ steps.arch_vars.outputs.instance_type }}
-          ami_name_prefix: ${{ steps.arch_vars.outputs.ami_name_prefix }}
 
       - name: Grab release version
         id: process_release_version

.github/workflows/ami-release-nix.yml

@@ -44,14 +44,14 @@ jobs:
           - name: arm64
             runner: blacksmith-2vcpu-ubuntu-2404-arm
             packer_template: amazon-arm64-nix.pkr.hcl
-            vars_file: development-arm.vars.pkr.hcl
+            vars_file: development-arm64.vars.pkr.hcl
             instance_type: c6g.4xlarge
             nix_system: aarch64-linux
             ami_arch_filter: arm64
           - name: amd64
             runner: blacksmith-2vcpu-ubuntu-2404
             packer_template: amazon-amd64-nix.pkr.hcl
-            vars_file: development-x86.vars.pkr.hcl
+            vars_file: development-amd64.vars.pkr.hcl
             instance_type: c6i.4xlarge
             nix_system: x86_64-linux
             ami_arch_filter: x86_64

.github/workflows/base-image-nightly.yml

@@ -49,7 +49,7 @@ jobs:
           nix run github:supabase/postgres/${GIT_SHA}#packer -- build \
             -var "git-head-version=${GIT_SHA}" \
             -var "packer-execution-id=${EXECUTION_ID}" \
-            -var-file="development-arm.vars.pkr.hcl" \
+            -var-file="development-arm64.vars.pkr.hcl" \
             -var "base-image-nightly=true" \
             -var "build-timestamp=${BUILD_TIMESTAMP}" \
             -var "region=us-east-1" \

.github/workflows/testinfra-ami-build.yml

@@ -39,13 +39,14 @@ jobs:
       fail-fast: false
       matrix:
         postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
-        include:
-          - runner: blacksmith-2vcpu-ubuntu-2404-arm
-            arch: arm64
-            ubuntu_release: noble
-            ubuntu_version: 24.04
-            mcpu: neoverse-n1
-    runs-on: ${{ matrix.runner }}
+        target:
+          - arch: amd64
+            instance_type: c6i.4xlarge
+            runner: blacksmith-2vcpu-ubuntu-2404
+          - arch: arm64
+            instance_type: c6g.4xlarge
+            runner: blacksmith-2vcpu-ubuntu-2404-arm
+    runs-on: ${{ matrix.target.runner }}
     timeout-minutes: 150
 
     steps:
@@ -93,11 +94,13 @@ jobs:
         id: build-ami
         uses: ./.github/actions/build-ami
         with:
-          postgres_version: ${{ matrix.postgres_version }}
-          region: ap-southeast-1
+          ami_name_prefix: "supabase-postgres-${{ github.run_id }}-${{ matrix.target.arch }}"
           ami_regions: '["ap-southeast-1"]'
+          arch: ${{ matrix.target.arch }}
           git_sha: ${{ github.sha }}
-          ami_name_prefix: "supabase-postgres-${{ github.run_id }}"
+          instance_type: ${{ matrix.target.instance_type }}
+          postgres_version: ${{ matrix.postgres_version }}
+          region: ap-southeast-1
 
       - name: Run tests
         timeout-minutes: 10

README.md

@@ -87,7 +87,7 @@ Here's a comprehensive overview of the project's directory structure:
 | ansible.cfg | Ansible configuration |
 | amazon-arm64-nix.pkr.hcl | Packer configuration for AWS ARM64 builds |
 | common-nix.vars.pkr.hcl | Common Packer variables |
-| development-arm.vars.pkr.hcl | ARM development environment variables |
+| development-arm64.vars.pkr.hcl | ARM development environment variables |
 | CONTRIBUTING.md | Contribution guidelines |
 | README.md | Main project documentation |
 

ansible/files/gotrue.service.j2

@@ -17,6 +17,9 @@ After=apparmor.service
 # We want sysctl's to be applied
 After=systemd-sysctl.service
 
+# Ensure tuned is applied
+After=tuned.service
+
 # UFW Is modified by cloud init, but started non-blocking, so configuration
 # could be in-flight while gotrue is starting. I want to ensure future rules
 # that are relied on for security posture are applied before gotrue runs.

ansible/tasks/setup-system.yml

@@ -154,13 +154,6 @@
         owner: 'root'
         group: 'root'
 
-    # Set Sysctl params specific to keepalives
-    - name: Set net.ipv4.tcp_keepalive_time=1800
-      ansible.builtin.sysctl:
-        name: 'net.ipv4.tcp_keepalive_time'
-        value: 1800
-        state: 'present'
-
     - name: Set net.ipv4.tcp_keepalive_intvl=60
       ansible.builtin.sysctl:
         name: 'net.ipv4.tcp_keepalive_intvl'

ansible/tasks/setup-tuned.yml

@@ -112,18 +112,31 @@
             value: 10
 
 
-        - name: tuned - Set ip_local_port_range
+        - name: tuned - Set ip_local_reserved_ports
           become: true
           community.general.ini_file:
             create: true
             group: 'root'
             mode: '0644'
             no_extra_spaces: true
-            option: 'net.ipv4.ip_local_port_range'
+            option: 'net.ipv4.ip_local_reserved_ports'
             path: '/etc/tuned/profiles/postgresql/tuned.conf'
             section: 'sysctl'
             state: 'present'
-            value: '1025 65499'
+            value: '3000,3001,8085,9122,9187,9999'
+
+        - name: tuned - Set tcp_keepalive_time
+          become: true
+          community.general.ini_file:
+            create: true
+            group: 'root'
+            mode: '0644'
+            no_extra_spaces: true
+            option: 'net.ipv4.tcp_keepalive_time'
+            path: '/etc/tuned/profiles/postgresql/tuned.conf'
+            section: 'sysctl'
+            state: 'present'
+            value: 1800
         - name: tuned - Load zstd compressor module
           become: true
           community.general.modprobe:

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.6.0.090-orioledb"
-  postgres17: "17.6.1.133"
-  postgres15: "15.14.1.133"
+  postgresorioledb-17: "17.6.0.091-orioledb"
+  postgres17: "17.6.1.134"
+  postgres15: "15.14.1.134"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.25.1