feat: pin pgbouncer EC2 AMI tag, pin uv version in Lambda docker builds (#235)

hrodmn · web-flow · commit 9f138cf52032 · 2026-03-12T14:48:20.000-05:00
* fix: pin uv to v0.10.9 in all dockerfiles

* feat: pin pgbouncer EC2 AMI to 20260218 image instead of 'current'

* chore: clean up some unnecessary links between pgbouncer and the pgstac
bootstrapper

* fix: fix reference to wrong uv.lock file in stactools-item-generator

* chore(ci): add action to open issue for updating AMI tag
diff --git a/.github/workflows/update-pgbouncer-ami.yaml b/.github/workflows/update-pgbouncer-ami.yaml
@@ -0,0 +1,86 @@
+name: Remind to update PgBouncer AMI pin
+
+on:
+  schedule:
+    # Run quarterly: 1st of January, April, July, October
+    - cron: '0 9 1 1,4,7,10 *'
+  workflow_dispatch:
+
+jobs:
+  check-ami-pin:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Check age of pinned AMI date
+        id: check
+        run: |
+          # Extract the pinned date from PgBouncer.ts
+          PINNED=$(grep -oP '(?<=/noble/stable/)\d{8}(?=/)' lib/database/PgBouncer.ts | head -1)
+          if [ -z "$PINNED" ]; then
+            echo "No pinned date found (using 'current' or unrecognised format) — nothing to check."
+            echo "should_remind=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          PINNED_DATE=$(date -d "$PINNED" +%s)
+          NOW=$(date +%s)
+          AGE_DAYS=$(( (NOW - PINNED_DATE) / 86400 ))
+
+          echo "Pinned date: $PINNED ($AGE_DAYS days ago)"
+          echo "pinned=$PINNED" >> "$GITHUB_OUTPUT"
+          echo "age_days=$AGE_DAYS" >> "$GITHUB_OUTPUT"
+
+          if [ "$AGE_DAYS" -ge 90 ]; then
+            echo "should_remind=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "should_remind=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open reminder issue
+        if: steps.check.outputs.should_remind == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const pinned = '${{ steps.check.outputs.pinned }}';
+            const ageDays = '${{ steps.check.outputs.age_days }}';
+
+            // Check if a reminder issue is already open
+            const issues = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: 'maintenance',
+            });
+            const existing = issues.data.find(i => i.title.includes('PgBouncer AMI pin'));
+            if (existing) {
+              console.log(`Reminder issue already open: ${existing.html_url}`);
+              return;
+            }
+
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: `chore: update PgBouncer AMI pin (currently ${pinned}, ${ageDays} days old)`,
+              body: [
+                `The PgBouncer EC2 AMI is pinned to \`${pinned}\`, which is ${ageDays} days old.`,
+                '',
+                'To find a more recent date-versioned path in your region:',
+                '```',
+                'aws ssm get-parameters-by-path \\',
+                '  --path "/aws/service/canonical/ubuntu/server/noble/stable/" \\',
+                '  --recursive \\',
+                '  --query "Parameters[?ends_with(Name, \'amd64/hvm/ebs-gp3/ami-id\')].Name"',
+                '```',
+                '',
+                'Then update the `DEFAULT_AMI_SSM_PARAMETER` constant in `lib/database/PgBouncer.ts`',
+                'and the `@default` tag in both `PgBouncer.ts` and `lib/database/index.ts`.',
+                '',
+                'See: https://documentation.ubuntu.com/aws/aws-how-to/instances/find-ubuntu-images/',
+              ].join('\n'),
+              labels: ['maintenance'],
+            });
diff --git a/lib/database/PgBouncer.ts b/lib/database/PgBouncer.ts
@@ -25,6 +25,9 @@ export interface PgBouncerConfigProps {
   maxUserConnections?: number;
 }
 
+const DEFAULT_AMI_SSM_PARAMETER =
+  "/aws/service/canonical/ubuntu/server/noble/stable/20260218/amd64/hvm/ebs-gp3/ami-id";
+
 export interface PgBouncerProps {
   /**
    * VPC to deploy PgBouncer into
@@ -62,10 +65,24 @@ export interface PgBouncerProps {
   instanceProps?: Partial<ec2.InstanceProps>;
 
   /**
-   * Optional reference to the database bootstrapper CustomResource.
-   * When provided, the health check will re-trigger if the database setup changes.
+   * SSM parameter path for the PgBouncer EC2 instance machine image (AMI).
+   *
+   * Defaults to the latest Ubuntu Noble AMI. For stable deployments where
+   * EC2 replacement should only happen on explicit intent (not on Canonical
+   * AMI releases), pin this to a specific date-versioned path:
+   *   /aws/service/canonical/ubuntu/server/noble/stable/YYYYMMDD.X/amd64/hvm/ebs-gp3/ami-id
+   *
+   * To list available date-versioned paths in your region:
+   *   aws ssm get-parameters-by-path --path "/aws/service/canonical/ubuntu/server/noble/stable/" --recursive --query "Parameters[?ends_with(Name, 'amd64/hvm/ebs-gp3/ami-id')].Name"
+   *
+   * See: https://documentation.ubuntu.com/aws/aws-how-to/instances/find-ubuntu-images/
+   *
+   * With addPatchManager: true (default), SSM Patch Manager handles OS
+   * security updates without requiring instance replacement.
+   *
+   * @default /aws/service/canonical/ubuntu/server/noble/stable/20260218/amd64/hvm/ebs-gp3/ami-id
    */
-  databaseBootstrapper?: CustomResource;
+  readonly machineImageSsmParameter?: string;
 }
 
 export class PgBouncer extends Construct {
@@ -81,7 +98,7 @@ export class PgBouncer extends Construct {
   // so we perform this calculation.
 
   private getDefaultPgbouncerConfig(
-    dbMaxConnections: number
+    dbMaxConnections: number,
   ): Required<PgBouncerConfigProps> {
     // maxDbConnections (and maxUserConnections) are the only settings that need
     // to be responsive to the database size/max_connections setting
@@ -103,7 +120,7 @@ export class PgBouncer extends Construct {
     // Set defaults for optional props
 
     const defaultPgbouncerConfig = this.getDefaultPgbouncerConfig(
-      props.dbMaxConnections
+      props.dbMaxConnections,
     );
 
     // Merge provided config with defaults
@@ -119,10 +136,10 @@ export class PgBouncer extends Construct {
       assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"),
       managedPolicies: [
         iam.ManagedPolicy.fromAwsManagedPolicyName(
-          "AmazonSSMManagedInstanceCore"
+          "AmazonSSMManagedInstanceCore",
         ),
         iam.ManagedPolicy.fromAwsManagedPolicyName(
-          "CloudWatchAgentServerPolicy"
+          "CloudWatchAgentServerPolicy",
         ),
       ],
     });
@@ -132,7 +149,7 @@ export class PgBouncer extends Construct {
       new iam.PolicyStatement({
         actions: ["secretsmanager:GetSecretValue"],
         resources: [props.database.secret.secretArn],
-      })
+      }),
     );
 
     // Create a security group and allow connections from the Lambda IP ranges for this region
@@ -147,16 +164,16 @@ export class PgBouncer extends Construct {
       instanceName: "pgbouncer",
       instanceType: ec2.InstanceType.of(
         ec2.InstanceClass.T3,
-        ec2.InstanceSize.MICRO
+        ec2.InstanceSize.MICRO,
       ),
       vpcSubnets: {
         subnetType: props.usePublicSubnet
           ? ec2.SubnetType.PUBLIC
           : ec2.SubnetType.PRIVATE_WITH_EGRESS,
       },
       machineImage: ec2.MachineImage.fromSsmParameter(
-        "/aws/service/canonical/ubuntu/server/noble/stable/current/amd64/hvm/ebs-gp3/ami-id",
-        { os: ec2.OperatingSystemType.LINUX }
+        props.machineImageSsmParameter ?? DEFAULT_AMI_SSM_PARAMETER,
+        { os: ec2.OperatingSystemType.LINUX },
       ),
       blockDevices: [
         {
@@ -185,7 +202,7 @@ export class PgBouncer extends Construct {
     props.database.connections.allowFrom(
       this.instance,
       ec2.Port.tcp(5432),
-      "Allow PgBouncer to connect to RDS"
+      "Allow PgBouncer to connect to RDS",
     );
 
     // Create a new secret for pgbouncer connection credentials
@@ -205,7 +222,7 @@ export class PgBouncer extends Construct {
       runtime: lambda.Runtime.NODEJS_LATEST,
       handler: "index.handler",
       code: lambda.Code.fromAsset(
-        path.join(__dirname, "lambda/pgbouncer-secret-updater")
+        path.join(__dirname, "lambda/pgbouncer-secret-updater"),
       ),
       environment: {
         SOURCE_SECRET_ARN: props.database.secret.secretArn,
@@ -226,7 +243,7 @@ export class PgBouncer extends Construct {
             ? this.instance.instancePublicIp
             : this.instance.instancePrivateIp,
         },
-      }
+      },
     );
 
     // Add health check custom resource
@@ -238,10 +255,10 @@ export class PgBouncer extends Construct {
         handler: "index.handler",
         timeout: Duration.minutes(10),
         code: lambda.Code.fromAsset(
-          path.join(__dirname, "lambda/pgbouncer-health-check")
+          path.join(__dirname, "lambda/pgbouncer-health-check"),
         ),
         description: "PgBouncer health check function",
-      }
+      },
     );
 
     // Grant SSM permissions for health check
@@ -254,17 +271,13 @@ export class PgBouncer extends Construct {
           "ssm:ListCommandInvocations",
         ],
         resources: ["*"],
-      })
+      }),
     );
 
     this.healthCheck = new CustomResource(this, "PgBouncerHealthCheck", {
       serviceToken: healthCheckFunction.functionArn,
       properties: {
         InstanceId: this.instance.instanceId,
-        // Reference the database bootstrapper to re-trigger on database changes
-        ...(props.databaseBootstrapper && {
-          DatabaseBootstrapperRef: props.databaseBootstrapper.ref,
-        }),
       },
     });
 
@@ -275,7 +288,7 @@ export class PgBouncer extends Construct {
 
   private loadUserDataScript(
     pgBouncerConfig: Required<NonNullable<PgBouncerProps["pgBouncerConfig"]>>,
-    database: { secret: secretsmanager.ISecret }
+    database: { secret: secretsmanager.ISecret },
   ): ec2.UserData {
     const userDataScript = ec2.UserData.forLinux();
 
@@ -292,7 +305,9 @@ export class PgBouncer extends Construct {
         pgBouncerConfig.reservePoolTimeout +
         '"',
       'export MAX_DB_CONNECTIONS="' + pgBouncerConfig.maxDbConnections + '"',
-      'export MAX_USER_CONNECTIONS="' + pgBouncerConfig.maxUserConnections + '"'
+      'export MAX_USER_CONNECTIONS="' +
+        pgBouncerConfig.maxUserConnections +
+        '"',
     );
 
     // Load the startup script
diff --git a/lib/database/bootstrapper_runtime/handler.py b/lib/database/bootstrapper_runtime/handler.py
@@ -171,8 +171,15 @@ def handler(event, context):
     """Lambda Handler."""
     print(f"Handling {event}")
 
+    physicalResourceId = event.get("PhysicalResourceId") or context.log_stream_name
     if event["RequestType"] not in ["Create", "Update"]:
-        return send(event, context, "SUCCESS", {"msg": "No action to be taken"})
+        return send(
+            event,
+            context,
+            "SUCCESS",
+            {"msg": "No action to be taken"},
+            physicalResourceId=physicalResourceId,
+        )
 
     try:
         params = event["ResourceProperties"]
@@ -261,8 +268,14 @@ def handler(event, context):
 
     except Exception as e:
         print(f"Unable to bootstrap database with exception={e}")
-        send(event, context, "FAILED", {"message": str(e)})
+        send(
+            event,
+            context,
+            "FAILED",
+            {"message": str(e)},
+            physicalResourceId=physicalResourceId,
+        )
         raise e
 
     print("Complete.")
-    return send(event, context, "SUCCESS", {})
+    return send(event, context, "SUCCESS", {}, physicalResourceId=physicalResourceId)
diff --git a/lib/database/index.ts b/lib/database/index.ts
@@ -279,7 +279,7 @@ export class PgStacDatabase extends Construct {
           reservePoolSize: 5,
           reservePoolTimeout: 5,
         },
-        databaseBootstrapper: bootstrapper,
+        machineImageSsmParameter: props.pgbouncerAmiSsmParameter,
       });
 
       this._pgBouncerServer.node.addDependency(bootstrapper);
@@ -387,6 +387,26 @@ export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps {
    */
   readonly pgbouncerInstanceProps?: ec2.InstanceProps | any;
 
+  /**
+   * SSM parameter path for the PgBouncer EC2 instance machine image (AMI).
+   *
+   * Defaults to the latest Ubuntu Noble AMI (`current`). For stable deployments
+   * where EC2 replacement should only happen on explicit intent, pin this to a
+   * specific date-versioned path:
+   *   /aws/service/canonical/ubuntu/server/noble/stable/YYYYMMDD.X/amd64/hvm/ebs-gp3/ami-id
+   *
+   * To list available date-versioned paths in your region:
+   *   aws ssm get-parameters-by-path --path "/aws/service/canonical/ubuntu/server/noble/stable/" --recursive --query "Parameters[?ends_with(Name, 'amd64/hvm/ebs-gp3/ami-id')].Name"
+   *
+   * See: https://documentation.ubuntu.com/aws/aws-how-to/instances/find-ubuntu-images/
+   *
+   * With addPatchManager: true (default), SSM Patch Manager handles OS security
+   * updates without requiring instance replacement.
+   *
+   * @default /aws/service/canonical/ubuntu/server/noble/stable/20260218/amd64/hvm/ebs-gp3/ami-id
+   */
+  readonly pgbouncerAmiSsmParameter?: string;
+
   /**
    * Add patching system using AWS SSM for pgbouncer instance maintenance
    * `addPgbouncer` must be true for this to have an effect
diff --git a/lib/stac-api/runtime/Dockerfile b/lib/stac-api/runtime/Dockerfile
@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 WORKDIR /tmp
 COPY stac-api/runtime/uv.lock stac-api/runtime/pyproject.toml ./
diff --git a/lib/stac-auth-proxy/runtime/Dockerfile b/lib/stac-auth-proxy/runtime/Dockerfile
@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 WORKDIR /tmp
 COPY stac-auth-proxy/runtime/uv.lock stac-auth-proxy/runtime/pyproject.toml ./
diff --git a/lib/stac-loader/runtime/Dockerfile b/lib/stac-loader/runtime/Dockerfile
@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION=3.12
 FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 ENV UV_COMPILE_BYTECODE=1
 ENV PYTHONUNBUFFERED=1
diff --git a/lib/stactools-item-generator/runtime/Dockerfile b/lib/stactools-item-generator/runtime/Dockerfile
@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION=3.12
 FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 ENV UV_CACHE_DIR=/tmp/uv-cache/
 ENV UV_COMPILE_BYTECODE=1
@@ -10,7 +10,7 @@ ENV PATH=/tmp/.local/bin:$PATH
 
 WORKDIR /tmp
 
-COPY stactools-item-generator/runtime/pyproject.toml stac-loader/runtime/uv.lock ./
+COPY stactools-item-generator/runtime/pyproject.toml stactools-item-generator/runtime/uv.lock ./
 COPY stactools-item-generator/runtime/src/ ./src/
 
 RUN <<EOF
diff --git a/lib/tipg-api/runtime/Dockerfile b/lib/tipg-api/runtime/Dockerfile
@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 WORKDIR /tmp
 COPY tipg-api/runtime/uv.lock tipg-api/runtime/pyproject.toml ./
diff --git a/lib/titiler-pgstac-api/runtime/Dockerfile b/lib/titiler-pgstac-api/runtime/Dockerfile
@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 WORKDIR /tmp
 COPY titiler-pgstac-api/runtime/uv.lock titiler-pgstac-api/runtime/pyproject.toml ./
