diff --git a/Dockerfile.optimized b/Dockerfile.optimized index 06667b3fc..9d86edaaf 100644 --- a/Dockerfile.optimized +++ b/Dockerfile.optimized @@ -4,11 +4,22 @@ # checkov:skip=CKV_DOCKER_3: "The Dockerfile uses the official AWS Lambda Python base image (public.ecr.aws/lambda/python:3.12-arm64), which already configures the appropriate non-root user for Lambda execution" # checkov:skip=CKV_DOCKER_2: "The Dockerfile.optimized is specifically designed for AWS Lambda container images, which don't use Docker HEALTHCHECK instructions." +# ── Air-gapped / internal registry support ──────────────────────────────────── +# Override these ARGs to use images from an internal registry instead of public ones. +# Example (air-gapped): +# docker build \ +# --build-arg UV_IMAGE=123456789.dkr.ecr.us-east-1.amazonaws.com/idp-base:uv-0.9.6 \ +# --build-arg LAMBDA_BASE_IMAGE=123456789.dkr.ecr.us-east-1.amazonaws.com/idp-base:lambda-python-3.12-arm64 \ +# -f Dockerfile.optimized . +# ───────────────────────────────────────────────────────────────────────────── +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.9.6 +ARG LAMBDA_BASE_IMAGE=public.ecr.aws/lambda/python:3.12-arm64 + # Use specific version to avoid network issues -FROM ghcr.io/astral-sh/uv:0.9.6 AS uv +FROM ${UV_IMAGE} AS uv # Builder stage - bundle dependencies into Lambda task root -FROM public.ecr.aws/lambda/python:3.12-arm64 AS builder +FROM ${LAMBDA_BASE_IMAGE} AS builder # Enable bytecode compilation to improve cold-start performance ENV UV_COMPILE_BYTECODE=1 @@ -19,6 +30,11 @@ ENV UV_NO_INSTALLER_METADATA=1 # Enable copy mode to support bind mount caching ENV UV_LINK_MODE=copy +# Air-gapped PyPI mirror support: pass --build-arg UV_INDEX_URL=https://your-artifactory/pypi/simple/ +# When set, uv will use this instead of pypi.org to install Lambda requirements.txt packages. +ARG UV_INDEX_URL="" +ENV UV_INDEX_URL=${UV_INDEX_URL} + # Build argument for function path ARG FUNCTION_PATH ARG INSTALL_IDP_COMMON=true @@ -33,17 +49,24 @@ COPY ${FUNCTION_PATH}/requirements.txt* /build/ # Install all dependencies including idp_common_pkg in one step # Using mount from uv stage instead of COPY to avoid layer bloat +# When UV_INDEX_URL is set (air-gapped), pass --index-url explicitly rather than relying +# on the env var alone — this ensures the flag is always evaluated even when the uv +# cache layer was previously built against pypi.org. RUN --mount=from=uv,source=/uv,target=/bin/uv \ --mount=type=cache,target=/root/.cache/uv \ if [ -f /build/requirements.txt ]; then \ - sed 's|^\.\./\.\.\(/\.\.\)\?/lib/idp_common_pkg|/tmp/idp_common_pkg|' /build/requirements.txt > /tmp/requirements.txt && \ - uv pip install --python python3.12 --target "${LAMBDA_TASK_ROOT}" -r /tmp/requirements.txt && \ - rm /tmp/requirements.txt; \ + sed 's|^\.\./\.\.\(/\.\.\)\?/lib/idp_common_pkg|/tmp/idp_common_pkg|' /build/requirements.txt > /tmp/requirements.txt && \ + INDEX_ARG="" && \ + if [ -n "$UV_INDEX_URL" ]; then INDEX_ARG="--index-url $UV_INDEX_URL"; fi && \ + uv pip install --python python3.12 --target "${LAMBDA_TASK_ROOT}" $INDEX_ARG -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt; \ fi && \ rm -rf /tmp/idp_common_pkg # Final stage - minimal runtime -FROM public.ecr.aws/lambda/python:3.12-arm64 +# LAMBDA_BASE_IMAGE ARG must be re-declared after each FROM (Docker scoping rule) +ARG LAMBDA_BASE_IMAGE=public.ecr.aws/lambda/python:3.12-arm64 +FROM ${LAMBDA_BASE_IMAGE} # Conditionally install git (required for mlflow/gitpython) ARG INSTALL_GIT=false diff --git a/docs/deployment-private-network.md b/docs/deployment-private-network.md index 68f260f81..43f829407 100644 --- a/docs/deployment-private-network.md +++ b/docs/deployment-private-network.md @@ -129,6 +129,92 @@ idp-cli publish --source-dir . --bucket-basename my-idp-artifacts --prefix idp - --- +### Air-gapped CodeBuild — Internal Container Registries (optional) + +During stack deployment, CodeBuild builds the Lambda container images inside your account. In a **fully air-gapped VPC** where CodeBuild has no internet access, it cannot reach the two public registries used by the build: + +| Image | Public source | Used for | +|-------|--------------|----------| +| `ghcr.io/astral-sh/uv:0.9.6` | GitHub Container Registry | Python dependency installer inside Dockerfile | +| `public.ecr.aws/lambda/python:3.12-arm64` | Amazon Public ECR | Lambda base image | + +**Solution**: mirror both images to your internal ECR (or Artifactory), then pass their URIs as CloudFormation parameters. + +#### Step A: Mirror images to internal ECR + +Run the provided helper script — it pulls both images, re-tags them into the deployment account's ECR, and prints the ready-to-paste parameter string: + +```bash +./scripts/setup-airgapped-codebuild.sh \ + --region \ + --account +``` + +Example output: + +``` +✅ UV image pushed: 123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-build-tools:uv-0.9.6 +✅ Lambda base image pushed: 123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-build-tools:lambda-python-3.12-arm64 + +Add these parameters to your idp-cli deploy command: + UvImage=123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-build-tools:uv-0.9.6,LambdaBaseImage=123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-build-tools:lambda-python-3.12-arm64 +``` + +#### Step B: Add image parameters to the deploy command + +Append the `UvImage` and `LambdaBaseImage` parameters (from the script output) to your `idp-cli deploy --parameters` string (see Step 2 below). + +#### Internal PyPI mirror (optional) + +If CodeBuild also cannot reach PyPI (`pypi.org`) to install Lambda Python dependencies, pass `UvIndexUrl` pointing to your internal PyPI mirror (e.g. Artifactory): + +``` +UvIndexUrl=https://artifactory.company.com/artifactory/api/pypi/pypi-virtual/simple/ +``` + +#### Artifactory Docker registry (optional) + +If the images are stored in Artifactory rather than ECR, you need to provide credentials so CodeBuild can log in before pulling: + +1. Create a Secrets Manager secret with your Artifactory credentials (only needed once): + ```bash + aws secretsmanager create-secret \ + --name idp/artifactory-docker-creds \ + --secret-string '{"username":"svc-build","password":""}' \ + --region + ``` + +2. Add these parameters to your deploy command: + ``` + ArtifactoryDockerUrl=artifactory.company.com,ArtifactoryCredentialsSecretArn=arn:aws:secretsmanager:::secret:idp/artifactory-docker-creds- + ``` + +> **Security note**: credentials are fetched at build time via the IAM role — they are never stored in CloudFormation parameters or environment variable logs. + +#### CodeBuild VPC Placement (optional, for fully air-gapped subnets) + +If CodeBuild's subnets have no internet access at all (no NAT Gateway), you can place CodeBuild inside your VPC using the `CodeBuildVpcId`, `CodeBuildSubnetIds`, and `CodeBuildSecurityGroupId` parameters. When these are set: + +- CodeBuild runs entirely within your VPC with no public internet access +- The subnets need VPC Interface Endpoints for: `ecr.api`, `ecr.dkr`, `codebuild`, `logs`, `secretsmanager` — plus a free S3 **Gateway** endpoint +- Run the VPC endpoint deployment script with the `--codebuild-endpoints` flag to create them: + +```bash +python scripts/deploy-vpc-endpoints.py \ + --vpc-id \ + --stack-name IDP-PRIVATE \ + --security-group-id \ + --subnet-ids , \ + --codebuild-endpoints \ + --region +``` + +> **Note:** `CodeBuildVpcId` can be the same VPC as `ALBVpcId`, but `CodeBuildSubnetIds` should be private subnets that have the required VPC endpoints. These can be different from `LambdaSubnetIds`. + +> **See also**: [Artifactory Dependency Workaround](./artifactory-dependency-workaround.md) for a comprehensive guide covering all dependency resolution options. + +--- + ## Step 2: Deploy the IDP Stack Replace the placeholder values and run: @@ -155,7 +241,15 @@ idp-cli deploy \ | `LambdaSubnetIds` | subnet IDs | Subnets where Lambda functions run (can match ALBSubnetIds) | | `EnableMCP` | `false` | Disable Bedrock AgentCore Gateway (requires public endpoint) | | `DocumentKnowledgeBase` | `DISABLED` | Disable Knowledge Base (avoids extra VPC endpoints) | - +| `UvImage` | ECR URI | *(Air-gapped only)* Internal ECR URI for the `uv` build tool image. Replaces `ghcr.io/astral-sh/uv:0.9.6`. | +| `LambdaBaseImage` | ECR URI | *(Air-gapped only)* Internal ECR URI for the Lambda Python base image. Replaces `public.ecr.aws/lambda/python:3.12-arm64`. | +| `UvIndexUrl` | HTTPS URL | *(Air-gapped only)* Internal PyPI index URL (e.g. Artifactory). Replaces `pypi.org` for Lambda dependency installs. May contain auth credentials — stored with `NoEcho`. | +| `NpmRegistryUrl` | HTTPS URL | *(Air-gapped only)* Internal npm registry URL (e.g. Artifactory). Replaces `registry.npmjs.org` for the Web UI build. | +| `ArtifactoryDockerUrl` | hostname | *(Air-gapped only)* Artifactory Docker registry hostname. Required when images are stored in Artifactory instead of ECR. | +| `ArtifactoryCredentialsSecretArn` | secret ARN | *(Air-gapped only)* Secrets Manager secret ARN with Artifactory credentials. Required with `ArtifactoryDockerUrl`. | +| `CodeBuildVpcId` | VPC ID | *(Air-gapped only)* VPC ID to place CodeBuild in for network isolation. When set, both `DockerBuildProject` and `UICodeBuildProject` run inside this VPC. Requires `CodeBuildSubnetIds` and `CodeBuildSecurityGroupId`. | +| `CodeBuildSubnetIds` | subnet IDs | *(Air-gapped only)* Comma-separated private subnet IDs for CodeBuild VPC placement. Required with `CodeBuildVpcId`. | +| `CodeBuildSecurityGroupId` | SG ID | *(Air-gapped only)* Security group ID for CodeBuild VPC placement. Must allow outbound HTTPS (443) to VPC endpoints. Required with `CodeBuildVpcId`. | > **`AppSyncVisibility` is immutable** — it cannot be changed after the stack is created. To switch between GLOBAL and PRIVATE, delete and recreate the stack. ### Enterprise: deploying with a KMS-encrypted artifact bucket @@ -379,6 +473,7 @@ When `WebUIHosting=ALB` and `AppSyncVisibility=PRIVATE`, the following are handl | **`npm error engine Unsupported engine`** | Node.js 22.12+ required. `brew install node@22 && export PATH="/opt/homebrew/opt/node@22/bin:$PATH"` | | **Stack fails with `conflicting DNS domain`** | A VPC endpoint already exists for that service. Re-run `check-vpc-endpoints.sh` — it will detect this and set the right `Create*=false` flags. | | **UI loads but shows "network error"** | AppSync API is PRIVATE. From outside the VPC you need an SSM tunnel + `/etc/hosts` entry. From inside VPN/VPC it works automatically. | +| **CodeBuild fails: `pull access denied` / `name unknown` on image pull** | CodeBuild cannot reach public container registries (`ghcr.io`, `public.ecr.aws`). Run `scripts/setup-airgapped-codebuild.sh --region --account ` to mirror images to ECR, then pass `UvImage=` and `LambdaBaseImage=` to the deploy command. See [Artifactory Dependency Workaround](./artifactory-dependency-workaround.md). | | **CodeBuild fails: `AccessDenied: kms:Decrypt`** | The artifact bucket is KMS-encrypted but `ArtifactsBucketKmsKeyArn` was not passed to the deploy command. Redeploy with `--parameters "...ArtifactsBucketKmsKeyArn="`. | | **`UpdateDefaultConfig` custom resource fails with `NoSuchKey`** | Same root cause as above — `ConfigurationCopyFunction` silently skipped copying config files due to missing `kms:Decrypt`. Pass `ArtifactsBucketKmsKeyArn` and redeploy. | | **OCR Lambda times out / Textract calls hang** | Missing `textract` VPC endpoint. Lambda can't reach `com.amazonaws..textract` — security group only allows port 443 outbound to VPC endpoints. Run `deploy-vpc-endpoints.py` to add the missing endpoint. | diff --git a/patterns/unified/buildspec.yml b/patterns/unified/buildspec.yml index 8a74c4e3d..351bd265a 100644 --- a/patterns/unified/buildspec.yml +++ b/patterns/unified/buildspec.yml @@ -9,11 +9,61 @@ phases: - echo "Setting up Docker buildx for cross-platform builds..." - docker buildx create --use --name multiarch-builder --driver docker-container || docker buildx use multiarch-builder - docker buildx inspect --bootstrap + # ── Air-gapped: log into internal base image registry if provided ────── + # UV_IMAGE and LAMBDA_BASE_IMAGE env vars may point to an internal ECR + # repo instead of ghcr.io / public.ecr.aws. If they are in an ECR + # registry in the same account, the ECR login above already covers them. + # If they are in a different account, add an extra login here. + - | + if [ -n "$UV_IMAGE" ]; then + echo "Air-gapped mode: using custom UV image: $UV_IMAGE" + else + UV_IMAGE="ghcr.io/astral-sh/uv:0.9.6" + echo "Standard mode: using default UV image: $UV_IMAGE" + fi + - | + if [ -n "$LAMBDA_BASE_IMAGE" ]; then + echo "Air-gapped mode: using custom Lambda base image: $LAMBDA_BASE_IMAGE" + else + LAMBDA_BASE_IMAGE="public.ecr.aws/lambda/python:3.12-arm64" + echo "Standard mode: using default Lambda base image: $LAMBDA_BASE_IMAGE" + fi + - | + if [ -n "$UV_INDEX_URL" ]; then + echo "Air-gapped mode: uv will use custom PyPI index: $UV_INDEX_URL" + else + echo "Standard mode: uv will use default PyPI index (pypi.org)" + fi + # ── Air-gapped: Artifactory Docker registry login ────────────────────── + # When ARTIFACTORY_DOCKER_URL + ARTIFACTORY_CREDENTIALS_SECRET_ARN are + # set, CodeBuild fetches credentials from AWS Secrets Manager at runtime + # (credentials are NEVER hardcoded) and logs in so that docker buildx + # build can pull UV_IMAGE and LAMBDA_BASE_IMAGE from Artifactory. + # Customer admin creates the secret once: + # aws secretsmanager create-secret --name "idp/artifactory-docker-creds" + # --secret-string '{"username":"svc-build","password":""}' + - | + if [ -n "$ARTIFACTORY_DOCKER_URL" ] && [ -n "$ARTIFACTORY_CREDENTIALS_SECRET_ARN" ]; then + echo "Artifactory mode: logging into $ARTIFACTORY_DOCKER_URL via Secrets Manager" + SECRET=$(aws secretsmanager get-secret-value \ + --secret-id "$ARTIFACTORY_CREDENTIALS_SECRET_ARN" \ + --query SecretString --output text) + AF_USER=$(echo "$SECRET" | python3 -c "import sys,json; print(json.load(sys.stdin)['username'])") + AF_PASS=$(echo "$SECRET" | python3 -c "import sys,json; print(json.load(sys.stdin)['password'])") + echo "$AF_PASS" | docker login "$ARTIFACTORY_DOCKER_URL" \ + --username "$AF_USER" --password-stdin + echo "Artifactory Docker login successful" + unset AF_USER AF_PASS SECRET + else + echo "Standard mode: no Artifactory Docker registry configured" + fi build: commands: - echo "Building Unified Pattern Docker images (BDA + Pipeline)..." - echo "Using IMAGE_VERSION from environment (content-based hash)" - echo "Image tag will be $IMAGE_VERSION" + - echo "UV_IMAGE=${UV_IMAGE:-ghcr.io/astral-sh/uv:0.9.6}" + - echo "LAMBDA_BASE_IMAGE=${LAMBDA_BASE_IMAGE:-public.ecr.aws/lambda/python:3.12-arm64}" # BDA functions - export FUNCTION_bda_invoke="patterns/unified/src/bda_invoke_function" - export FUNCTION_bda_completion="patterns/unified/src/bda_completion_function" @@ -39,10 +89,18 @@ phases: if [ "$func_var" = "FUNCTION_mlflow_logger" ]; then EXTRA_ARGS="--build-arg INSTALL_GIT=true" fi + # Build UV_INDEX_URL arg string only when set (air-gapped PyPI mirror) + UV_INDEX_ARG="" + if [ -n "$UV_INDEX_URL" ]; then + UV_INDEX_ARG="--build-arg UV_INDEX_URL=${UV_INDEX_URL}" + fi docker buildx build \ --platform linux/arm64 \ -f Dockerfile.optimized \ --build-arg FUNCTION_PATH="${func_path}" \ + --build-arg UV_IMAGE="${UV_IMAGE}" \ + --build-arg LAMBDA_BASE_IMAGE="${LAMBDA_BASE_IMAGE}" \ + ${UV_INDEX_ARG} \ ${EXTRA_ARGS} \ --tag "${ECR_URI}:${func_name}-${IMAGE_VERSION}" \ --provenance=false \ diff --git a/patterns/unified/template.yaml b/patterns/unified/template.yaml index f33ef7927..91db9201f 100644 --- a/patterns/unified/template.yaml +++ b/patterns/unified/template.yaml @@ -196,6 +196,86 @@ Parameters: Default: "" Description: SageMaker MLflow tracking server ARN + # ── Air-gapped / private network parameters ─────────────────────────────── + # Leave these empty for standard internet-connected deployments. + # Set them when CodeBuild cannot reach public container registries or PyPI. + UvImage: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal ECR URI for the uv build tool image. + Replaces ghcr.io/astral-sh/uv:0.9.6 in Dockerfile.optimized. + Example: 123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-base:uv-0.9.6 + + LambdaBaseImage: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal ECR URI for the Lambda Python base image. + Replaces public.ecr.aws/lambda/python:3.12-arm64 in Dockerfile.optimized. + Example: 123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-base:lambda-python-3.12-arm64 + + UvIndexUrl: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal PyPI index URL for uv to install Lambda requirements.txt packages. + Replaces pypi.org when CodeBuild cannot reach the public internet. + Example: https://artifactory.company.com/artifactory/api/pypi/pypi-virtual/simple/ + + ArtifactoryDockerUrl: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Artifactory Docker registry hostname that CodeBuild logs into before + pulling UV_IMAGE and LAMBDA_BASE_IMAGE. Required when images are stored in Artifactory instead + of ECR. Must be used together with ArtifactoryCredentialsSecretArn. + Example: artifactory.company.com + + ArtifactoryCredentialsSecretArn: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) ARN of an AWS Secrets Manager secret containing Artifactory Docker + registry credentials. The secret must have JSON format: {"username":"...","password":"..."}. + Required when ArtifactoryDockerUrl is set. Create the secret once with: + aws secretsmanager create-secret --name idp/artifactory-docker-creds + --secret-string '{"username":"svc-build","password":""}' + AllowedPattern: "^(|arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+)$" + + NpmRegistryUrl: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal npm registry URL for any npm install steps in CodeBuild. + Replaces registry.npmjs.org when CodeBuild cannot reach the public internet. + Example: https://artifactory.company.com/artifactory/api/npm/npm-virtual/ + + CodeBuildVpcId: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) VPC ID to place CodeBuild in for network isolation. + When set, CodeBuild runs inside this VPC and cannot reach the public internet directly. + Requires CodeBuildSubnetIds and CodeBuildSecurityGroupId. + The subnets must have access to ECR, S3, Secrets Manager, Logs, and CodeBuild via + VPC endpoints — run scripts/deploy-vpc-endpoints.py --codebuild-endpoints to create them. + + CodeBuildSubnetIds: + Type: CommaDelimitedList + Default: "" + Description: >- + (Optional — air-gapped) Comma-separated subnet IDs for CodeBuild VPC placement. + Required when CodeBuildVpcId is set. Use private subnets that have VPC endpoints + for ECR (ecr.api, ecr.dkr), S3 (Gateway), Secrets Manager, Logs, and CodeBuild. + + CodeBuildSecurityGroupId: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Security group ID for CodeBuild VPC placement. + Required when CodeBuildVpcId is set. Must allow outbound HTTPS (443) to VPC endpoints. + Conditions: HasGuardrailConfig: !And [ @@ -212,6 +292,13 @@ Conditions: IsPrivateAppSync: !Equals [!Ref UsePrivateAppSync, "true"] IsMLflowEnabled: !Equals [!Ref EnableMLflow, "true"] HasArtifactsBucketKmsKey: !Not [!Equals [!Ref ArtifactsBucketKmsKeyArn, ""]] + # Air-gapped support conditions + HasUvImage: !Not [!Equals [!Ref UvImage, ""]] + HasLambdaBaseImage: !Not [!Equals [!Ref LambdaBaseImage, ""]] + HasUvIndexUrl: !Not [!Equals [!Ref UvIndexUrl, ""]] + HasArtifactoryDockerUrl: !Not [!Equals [!Ref ArtifactoryDockerUrl, ""]] + HasArtifactoryCredentials: !Not [!Equals [!Ref ArtifactoryCredentialsSecretArn, ""]] + UseCodeBuildVpc: !Not [!Equals [!Ref CodeBuildVpcId, ""]] Resources: @@ -2158,6 +2245,30 @@ Resources: - kms:DescribeKey Resource: !Ref ArtifactsBucketKmsKeyArn - !Ref AWS::NoValue + # Allow CodeBuild to read Artifactory credentials from Secrets Manager (air-gapped mode) + # Only applies when ArtifactoryCredentialsSecretArn is provided + - !If + - HasArtifactoryCredentials + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: !Ref ArtifactoryCredentialsSecretArn + - !Ref AWS::NoValue + # Required by CodeBuild when placed in a VPC (CodeBuildVpcId is set). + # CodeBuild needs to create/describe/delete ENIs to attach to the VPC. + - !If + - UseCodeBuildVpc + - Effect: Allow + Action: + - ec2:CreateNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DeleteNetworkInterface + - ec2:DescribeSubnets + - ec2:DescribeSecurityGroups + - ec2:DescribeVpcs + - ec2:CreateNetworkInterfacePermission + Resource: "*" + - !Ref AWS::NoValue DockerBuildProject: Type: AWS::CodeBuild::Project @@ -2186,6 +2297,34 @@ Resources: Value: !Ref ImageVersion - Name: AWS_REGION Value: !Ref AWS::Region + # ── Air-gapped support ───────────────────────────────────────────── + # Set these parameters when CodeBuild cannot reach public registries. + # UV_IMAGE: internal ECR URI for ghcr.io/astral-sh/uv:0.9.6 + # LAMBDA_BASE_IMAGE: internal ECR URI for public.ecr.aws/lambda/python:3.12-arm64 + # UV_INDEX_URL: Artifactory/internal PyPI URL for Lambda pip installs + # Leave empty (default) for standard internet-connected deployments. + - Name: UV_IMAGE + Value: !If [HasUvImage, !Ref UvImage, ""] + - Name: LAMBDA_BASE_IMAGE + Value: !If [HasLambdaBaseImage, !Ref LambdaBaseImage, ""] + - Name: UV_INDEX_URL + Value: !If [HasUvIndexUrl, !Ref UvIndexUrl, ""] + # ── Artifactory Docker registry (air-gapped) ─────────────────────── + # When set, buildspec.yml fetches credentials from Secrets Manager + # and logs into Artifactory before pulling UV_IMAGE/LAMBDA_BASE_IMAGE. + - Name: ARTIFACTORY_DOCKER_URL + Value: !If [HasArtifactoryDockerUrl, !Ref ArtifactoryDockerUrl, ""] + - Name: ARTIFACTORY_CREDENTIALS_SECRET_ARN + Value: !If [HasArtifactoryCredentials, !Ref ArtifactoryCredentialsSecretArn, ""] + - Name: NPM_REGISTRY_URL + Value: !Ref NpmRegistryUrl + VpcConfig: !If + - UseCodeBuildVpc + - VpcId: !Ref CodeBuildVpcId + Subnets: !Ref CodeBuildSubnetIds + SecurityGroupIds: + - !Ref CodeBuildSecurityGroupId + - !Ref AWS::NoValue TimeoutInMinutes: 90 CodeBuildExecutionRole: diff --git a/scripts/deploy-vpc-endpoints.py b/scripts/deploy-vpc-endpoints.py index 638e1f41b..864cbc717 100644 --- a/scripts/deploy-vpc-endpoints.py +++ b/scripts/deploy-vpc-endpoints.py @@ -31,7 +31,7 @@ import boto3 from botocore.exceptions import ClientError, NoCredentialsError -# The 14 Interface endpoint services IDP requires +# The core Interface endpoint services IDP requires (Lambda + AppSync deployment) # Maps CFN parameter name → AWS service suffix REQUIRED_ENDPOINTS = { "CreateAppSyncApiEndpoint": "appsync-api", @@ -54,6 +54,16 @@ "CreateStsEndpoint": "sts", } +# Additional endpoints required when CodeBuild is placed in a VPC (CodeBuildVpcId is set). +# These default to "false" in vpc-endpoints.yaml and must be explicitly enabled. +CODEBUILD_ENDPOINTS = { + "CreateEcrApiEndpoint": "ecr.api", # docker pull metadata / image manifests + "CreateEcrDkrEndpoint": "ecr.dkr", # docker image layer downloads from ECR + "CreateCodeBuildEndpoint": "codebuild", # CodeBuild agent ↔ service communication + # NOTE: S3 access for ECR layers and build artifacts uses a Gateway endpoint (free). + # Set --route-table-ids when running this script to auto-create the S3 Gateway endpoint. +} + def parse_args(): parser = argparse.ArgumentParser( @@ -71,6 +81,23 @@ def parse_args(): "--subnet-ids", help="Comma-separated subnet IDs (auto-read from IDP stack if omitted)", ) + parser.add_argument( + "--security-group-id", + help=( + "Security group ID to attach to VPC endpoints (auto-read from IDP stack output " + "'LambdaVpcSecurityGroupId' if omitted). Required when AppSyncVisibility=GLOBAL " + "or when using CodeBuild VPC endpoints without private AppSync." + ), + ) + parser.add_argument( + "--codebuild-endpoints", + action="store_true", + help=( + "Also create CodeBuild VPC endpoints (ecr.api, ecr.dkr, codebuild). " + "Required when CodeBuildVpcId is set and CodeBuild cannot reach the internet. " + "Ensure --route-table-ids is also set to create the required S3 Gateway endpoint." + ), + ) parser.add_argument("--region", default=None, help="AWS region (default: from AWS config)") parser.add_argument("--profile", default=None, help="AWS CLI profile name") parser.add_argument( @@ -166,20 +193,24 @@ def main(): print("❌ No AWS credentials found. Configure with 'aws configure' or set environment variables.") sys.exit(1) - # ── Read Lambda SG from IDP stack ──────────────────────────────────── + # ── Read Lambda SG from IDP stack (or use --security-group-id override) ─ print(f"🔍 Reading IDP stack outputs from: {args.stack_name}") - try: - lambda_sg = get_stack_output(cf, args.stack_name, "LambdaVpcSecurityGroupId") - except ClientError as e: - print(f"❌ Could not read stack '{args.stack_name}': {e}") - print(" Make sure the stack is CREATE_COMPLETE and AppSyncVisibility=PRIVATE.") - sys.exit(1) + lambda_sg = args.security_group_id + if not lambda_sg: + try: + lambda_sg = get_stack_output(cf, args.stack_name, "LambdaVpcSecurityGroupId") + except ClientError as e: + print(f"❌ Could not read stack '{args.stack_name}': {e}") + print(" Make sure the stack is CREATE_COMPLETE and the stack name is correct.") + sys.exit(1) if not lambda_sg: - print(f"❌ Output 'LambdaVpcSecurityGroupId' not found in stack '{args.stack_name}'.") - print(" Make sure AppSyncVisibility=PRIVATE was set when the stack was created.") + print(f"❌ Could not determine security group ID.") + print(" Either:") + print(" • Set AppSyncVisibility=PRIVATE when deploying the IDP stack, OR") + print(" • Pass --security-group-id explicitly") sys.exit(1) - print(f" Lambda SG: {lambda_sg}") + print(f" Security Group: {lambda_sg}") # ── Read subnet IDs ─────────────────────────────────────────────────── subnet_ids = args.subnet_ids @@ -191,6 +222,12 @@ def main(): sys.exit(1) print(f" Subnets: {subnet_ids}") + # ── Build endpoint map (core + optional CodeBuild endpoints) ───────── + endpoints_to_check = dict(REQUIRED_ENDPOINTS) + if args.codebuild_endpoints: + print(" CodeBuild VPC endpoints enabled (--codebuild-endpoints flag set)") + endpoints_to_check.update(CODEBUILD_ENDPOINTS) + # ── Check each endpoint ─────────────────────────────────────────────── print(f"\n🔍 Checking existing VPC endpoints in {args.vpc_id} (region: {region})...\n") @@ -198,7 +235,7 @@ def main(): create_list = [] # service suffixes to create skip_list = [] # service suffixes to skip - for param, service in sorted(REQUIRED_ENDPOINTS.items()): + for param, service in sorted(endpoints_to_check.items()): full = f"com.amazonaws.{region}.{service}" if endpoint_exists(ec2, args.vpc_id, region, service): print(f" ✅ {full:<50} already exists — will skip") @@ -208,6 +245,12 @@ def main(): print(f" ➕ {full:<50} missing — will create") create_list.append(service) + # For CodeBuild endpoints not in the check list, explicitly set them to "false" + # so the CFN template skips creating them (they default to "false" but be explicit) + if not args.codebuild_endpoints: + for param in CODEBUILD_ENDPOINTS: + skip_params[param] = "false" + print(f"\n📊 Summary: {len(create_list)} to create, {len(skip_list)} already exist") if not create_list: diff --git a/scripts/setup-airgapped-codebuild.sh b/scripts/setup-airgapped-codebuild.sh new file mode 100755 index 000000000..6c167d004 --- /dev/null +++ b/scripts/setup-airgapped-codebuild.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# ============================================================================= +# setup-airgapped-codebuild.sh +# +# Prepares an air-gapped AWS environment so that CodeBuild can build Lambda +# container images WITHOUT any outbound internet access. +# +# What this script does: +# 1. Creates an ECR repository for base images in the customer account +# 2. Pulls ghcr.io/astral-sh/uv:0.9.6 and re-tags + pushes it to ECR +# 3. Pulls public.ecr.aws/lambda/python:3.12-arm64 and re-tags + pushes to ECR +# 4. Prints the --parameters strings to pass to idp-cli deploy +# +# Run this on a machine that HAS internet access AND AWS credentials for the +# customer account. +# +# Usage: +# bash scripts/setup-airgapped-codebuild.sh \ +# --region \ +# --account \ +# [--repo-name ] # default: idp-base-images +# [--pypi-url ] # optional: your internal PyPI URL +# +# Requirements on this machine: +# - docker (running) +# - aws CLI v2 (configured with customer account credentials) +# ============================================================================= + +set -euo pipefail + +# ── Colours ────────────────────────────────────────────────────────────────── +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m' +CYAN='\033[0;36m'; BOLD='\033[1m'; NC='\033[0m' + +info() { echo -e "${CYAN}[INFO]${NC} $*"; } +success() { echo -e "${GREEN}[OK]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +die() { echo -e "${RED}[ERROR]${NC} $*" >&2; exit 1; } + +# ── Source image references (what we pull from public registries) ───────────── +UV_SOURCE_IMAGE="ghcr.io/astral-sh/uv:0.9.6" +LAMBDA_SOURCE_IMAGE="public.ecr.aws/lambda/python:3.12-arm64" + +# ── Defaults ───────────────────────────────────────────────────────────────── +REGION="" +ACCOUNT_ID="" +REPO_NAME="idp-base-images" +PYPI_URL="" + +# ── Argument parsing ────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --region) REGION="$2"; shift 2 ;; + --account) ACCOUNT_ID="$2"; shift 2 ;; + --repo-name) REPO_NAME="$2"; shift 2 ;; + --pypi-url) PYPI_URL="$2"; shift 2 ;; + -h|--help) + echo "Usage: $0 --region --account [--repo-name ] [--pypi-url ]" + echo "" + echo "Options:" + echo " --region AWS region (e.g. us-east-1)" + echo " --account AWS account ID (12-digit)" + echo " --repo-name ECR repo name for base images (default: idp-base-images)" + echo " --pypi-url Internal PyPI/Artifactory URL for Lambda pip installs (optional)" + exit 0 ;; + *) die "Unknown argument: $1" ;; + esac +done + +# ── Validate required args ──────────────────────────────────────────────────── +[[ -z "$REGION" ]] && die "--region is required. Example: --region us-east-1" +[[ -z "$ACCOUNT_ID" ]] && die "--account is required. Example: --account 123456789012" + +ECR_REGISTRY="${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com" +ECR_REPO="${ECR_REGISTRY}/${REPO_NAME}" + +UV_TARGET_TAG="${ECR_REPO}:uv-0.9.6" +LAMBDA_TARGET_TAG="${ECR_REPO}:lambda-python-3.12-arm64" + +echo "" +echo -e "${BOLD}============================================================" +echo -e " GenAI IDP Accelerator — Air-Gapped CodeBuild Setup" +echo -e "============================================================${NC}" +echo -e " AWS Region: ${REGION}" +echo -e " AWS Account: ${ACCOUNT_ID}" +echo -e " ECR Registry: ${ECR_REGISTRY}" +echo -e " ECR Repo: ${REPO_NAME}" +if [[ -n "$PYPI_URL" ]]; then + echo -e " PyPI Mirror: ${PYPI_URL}" +fi +echo "" + +# ── Step 1: Check prerequisites ─────────────────────────────────────────────── +info "Checking prerequisites..." +command -v docker &>/dev/null || die "Docker is not installed or not running" +command -v aws &>/dev/null || die "AWS CLI is not installed" +success "docker: $(docker --version | head -1)" +success "aws: $(aws --version)" + +# ── Step 2: Verify AWS credentials ─────────────────────────────────────────── +info "Verifying AWS credentials..." +CALLER=$(aws sts get-caller-identity --region "$REGION" --output json 2>/dev/null) \ + || die "AWS credentials are not configured. Run 'aws configure' first." +CALLER_ACCOUNT=$(echo "$CALLER" | python3 -c "import sys,json; print(json.load(sys.stdin)['Account'])") +CALLER_ARN=$(echo "$CALLER" | python3 -c "import sys,json; print(json.load(sys.stdin)['Arn'])") +if [[ "$CALLER_ACCOUNT" != "$ACCOUNT_ID" ]]; then + warn "AWS credentials are for account ${CALLER_ACCOUNT}, but --account is ${ACCOUNT_ID}" + warn "Make sure you're using the correct AWS profile for the customer account." +fi +success "Authenticated as: ${CALLER_ARN}" + +# ── Step 3: Create ECR repository ──────────────────────────────────────────── +info "Creating ECR repository: ${REPO_NAME}..." +aws ecr describe-repositories --repository-names "$REPO_NAME" --region "$REGION" &>/dev/null \ + && success "ECR repository '${REPO_NAME}' already exists" \ + || { + aws ecr create-repository \ + --repository-name "$REPO_NAME" \ + --region "$REGION" \ + --image-scanning-configuration scanOnPush=true \ + --output json > /dev/null + success "Created ECR repository: ${REPO_NAME}" + } + +# ── Step 4: Login to ECR ────────────────────────────────────────────────────── +info "Logging into ECR (${ECR_REGISTRY})..." +aws ecr get-login-password --region "$REGION" \ + | docker login --username AWS --password-stdin "$ECR_REGISTRY" 2>/dev/null +success "Logged into ECR" + +# ── Step 5: Login to AWS Public ECR (for Lambda base image) ────────────────── +info "Logging into AWS Public ECR (public.ecr.aws)..." +aws ecr-public get-login-password --region us-east-1 \ + | docker login --username AWS --password-stdin public.ecr.aws 2>/dev/null \ + && success "Logged into AWS Public ECR" \ + || warn "Could not log into public ECR — proceeding (may fail for Lambda image pull)" + +# ── Step 6: Pull, re-tag, and push uv image ────────────────────────────────── +echo "" +echo -e "${BOLD}── Processing uv image ────────────────────────────────────────${NC}" +info "Pulling ${UV_SOURCE_IMAGE}..." +docker pull "${UV_SOURCE_IMAGE}" +success "Pulled ${UV_SOURCE_IMAGE}" + +info "Re-tagging as ${UV_TARGET_TAG}..." +docker tag "${UV_SOURCE_IMAGE}" "${UV_TARGET_TAG}" + +info "Pushing ${UV_TARGET_TAG} to ECR..." +docker push "${UV_TARGET_TAG}" +success "Pushed ${UV_TARGET_TAG}" + +# ── Step 7: Pull, re-tag, and push Lambda base image ───────────────────────── +echo "" +echo -e "${BOLD}── Processing Lambda base image ───────────────────────────────${NC}" +info "Pulling ${LAMBDA_SOURCE_IMAGE}..." +docker pull "${LAMBDA_SOURCE_IMAGE}" +success "Pulled ${LAMBDA_SOURCE_IMAGE}" + +info "Re-tagging as ${LAMBDA_TARGET_TAG}..." +docker tag "${LAMBDA_SOURCE_IMAGE}" "${LAMBDA_TARGET_TAG}" + +info "Pushing ${LAMBDA_TARGET_TAG} to ECR..." +docker push "${LAMBDA_TARGET_TAG}" +success "Pushed ${LAMBDA_TARGET_TAG}" + +# ── Step 8: Clean up local images (optional) ───────────────────────────────── +info "Cleaning up local re-tagged images..." +docker rmi "${UV_TARGET_TAG}" "${LAMBDA_TARGET_TAG}" 2>/dev/null || true +success "Cleaned up" + +# ── Step 9: Print deployment parameters ────────────────────────────────────── +echo "" +echo -e "${BOLD}${GREEN}════════════════════════════════════════════════════════════════${NC}" +echo -e "${BOLD}${GREEN} ✅ Air-gapped base images pushed successfully!${NC}" +echo -e "${BOLD}${GREEN}════════════════════════════════════════════════════════════════${NC}" +echo "" +echo -e "${BOLD}Images pushed to ECR:${NC}" +echo -e " UV image: ${UV_TARGET_TAG}" +echo -e " Lambda base image: ${LAMBDA_TARGET_TAG}" +echo "" + +# Build the parameters string +EXTRA_PARAMS="UvImage=${UV_TARGET_TAG},LambdaBaseImage=${LAMBDA_TARGET_TAG}" +if [[ -n "$PYPI_URL" ]]; then + EXTRA_PARAMS="${EXTRA_PARAMS},UvIndexUrl=${PYPI_URL}" +fi + +echo -e "${BOLD}Use these parameters in your idp-cli deploy command:${NC}" +echo "" +echo -e " ${CYAN}idp-cli deploy \\${NC}" +echo -e " ${CYAN}--stack-name IDP-PRIVATE \\${NC}" +echo -e " ${CYAN}--template-url \\${NC}" +echo -e " ${CYAN}--admin-email admin@example.com \\${NC}" +echo -e " ${CYAN}--region ${REGION} --wait \\${NC}" +echo -e " ${CYAN}--parameters \"WebUIHosting=ALB,ALBVpcId=,ALBSubnetIds=,,ALBCertificateArn=,ALBScheme=internal,AppSyncVisibility=PRIVATE,LambdaSubnetIds=,,EnableMCP=false,DocumentKnowledgeBase=DISABLED,${EXTRA_PARAMS}\"${NC}" +echo "" + +if [[ -n "$PYPI_URL" ]]; then + echo -e "${BOLD}Air-gapped parameters breakdown:${NC}" + echo -e " ${YELLOW}UvImage${NC} = ${UV_TARGET_TAG}" + echo -e " ${YELLOW}LambdaBaseImage${NC} = ${LAMBDA_TARGET_TAG}" + echo -e " ${YELLOW}UvIndexUrl${NC} = ${PYPI_URL}" +else + echo -e "${BOLD}Air-gapped parameters breakdown:${NC}" + echo -e " ${YELLOW}UvImage${NC} = ${UV_TARGET_TAG}" + echo -e " ${YELLOW}LambdaBaseImage${NC} = ${LAMBDA_TARGET_TAG}" + echo "" + echo -e "${YELLOW}TIP:${NC} If Lambda requirements.txt packages also fail (uv pip install),${NC}" + echo -e " add ${YELLOW}--pypi-url ${NC} to this script and re-run." + echo -e " Then add ${YELLOW}UvIndexUrl=${NC} to the --parameters string above." +fi + +echo "" +echo -e "${BOLD}Also make sure the DockerBuildRole IAM role has ECR pull permissions${NC}" +echo -e "${BOLD}for ${ECR_REGISTRY}/${REPO_NAME} (already granted via AmazonEC2ContainerRegistryPowerUser).${NC}" +echo "" + +# ── Step 10: Save parameters to a file for convenience ─────────────────────── +PARAMS_FILE="airgapped-params-${REGION}.env" +cat > "${PARAMS_FILE}" << EOF +# Auto-generated by scripts/setup-airgapped-codebuild.sh +# Use these values in your idp-cli deploy --parameters string + +UV_IMAGE=${UV_TARGET_TAG} +LAMBDA_BASE_IMAGE=${LAMBDA_TARGET_TAG} +UV_INDEX_URL=${PYPI_URL} +REGION=${REGION} +ACCOUNT_ID=${ACCOUNT_ID} + +# Full parameter string for idp-cli deploy: +IDP_AIRGAPPED_PARAMS=UvImage=${UV_TARGET_TAG},LambdaBaseImage=${LAMBDA_TARGET_TAG}$([ -n "$PYPI_URL" ] && echo ",UvIndexUrl=${PYPI_URL}" || echo "") +EOF + +success "Parameters saved to: ${PARAMS_FILE}" +echo -e " Source this file to use the values: ${CYAN}source ${PARAMS_FILE}${NC}" +echo "" diff --git a/scripts/vpc-endpoints.yaml b/scripts/vpc-endpoints.yaml index 7898688e7..72d70b49e 100644 --- a/scripts/vpc-endpoints.yaml +++ b/scripts/vpc-endpoints.yaml @@ -57,6 +57,13 @@ Parameters: # (If you are accessing the UI via VPN/Direct Connect in production, # you do NOT need ssmmessages or ec2messages.) # + # REQUIRED only when CodeBuild is placed in a VPC (CodeBuildVpcId is set): + # ecr.api — ECR API endpoint (docker pull metadata / image manifest) + # ecr.dkr — ECR DKR endpoint (docker layer downloads) + # codebuild — CodeBuild service endpoint (build agent ↔ service communication) + # NOTE: S3 access for ECR layer data and build artifacts requires a free S3 Gateway + # endpoint — set RouteTableIds to automatically create it. + # # Set any flag to "false" if the endpoint already exists in the VPC. # Use scripts/check-vpc-endpoints.sh to auto-detect and generate the correct deploy command. # ────────────────────────────────────────────────────────── @@ -162,6 +169,37 @@ Parameters: that call STS AssumeRole (bda/bda_service.py, bda/blueprint_optimizer.py). Set to "false" if already exists. + # ── CodeBuild VPC endpoints (only needed when CodeBuildVpcId is set) ────── + CreateEcrApiEndpoint: + Type: String + Default: "false" + AllowedValues: ["true", "false"] + Description: >- + Create the ecr.api Interface endpoint. Required when CodeBuild is placed in a + VPC (CodeBuildVpcId is set) and needs to pull base images from ECR. + Handles docker pull metadata and image manifests. + Set to "false" if already exists or CodeBuild is not in a VPC. + + CreateEcrDkrEndpoint: + Type: String + Default: "false" + AllowedValues: ["true", "false"] + Description: >- + Create the ecr.dkr Interface endpoint. Required when CodeBuild is placed in a + VPC (CodeBuildVpcId is set) and needs to pull base images from ECR. + Handles docker image layer downloads. Must be used together with ecr.api endpoint. + Set to "false" if already exists or CodeBuild is not in a VPC. + + CreateCodeBuildEndpoint: + Type: String + Default: "false" + AllowedValues: ["true", "false"] + Description: >- + Create the codebuild Interface endpoint. Required when CodeBuild is placed in a + VPC (CodeBuildVpcId is set). Allows the CodeBuild agent to communicate with the + CodeBuild service without internet access. + Set to "false" if already exists or CodeBuild is not in a VPC. + Conditions: CreateGatewayEndpoints: !Not - !Equals [ !Join [ "", !Ref RouteTableIds ], "" ] @@ -181,6 +219,9 @@ Conditions: ShouldCreateAthenaEndpoint: !Equals [ !Ref CreateAthenaEndpoint, "true" ] ShouldCreateTextractEndpoint: !Equals [ !Ref CreateTextractEndpoint, "true" ] ShouldCreateStsEndpoint: !Equals [ !Ref CreateStsEndpoint, "true" ] + ShouldCreateEcrApiEndpoint: !Equals [ !Ref CreateEcrApiEndpoint, "true" ] + ShouldCreateEcrDkrEndpoint: !Equals [ !Ref CreateEcrDkrEndpoint, "true" ] + ShouldCreateCodeBuildEndpoint: !Equals [ !Ref CreateCodeBuildEndpoint, "true" ] Resources: @@ -522,6 +563,67 @@ Resources: - Key: Environment Value: !Ref EnvironmentTag + ########################################################################## + # CodeBuild VPC Endpoints (only needed when CodeBuildVpcId is set) + ########################################################################## + + EcrApiVpcEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: ShouldCreateEcrApiEndpoint + Properties: + VpcId: !Ref VpcId + ServiceName: !Sub "com.amazonaws.${AWS::Region}.ecr.api" + VpcEndpointType: Interface + SubnetIds: !Ref SubnetIds + SecurityGroupIds: + - !Ref VpcEndpointSecurityGroup + PrivateDnsEnabled: true + Tags: + - Key: Name + Value: !Sub "${IDPStackName}-ecr-api" + - Key: IDPStack + Value: !Ref IDPStackName + - Key: Environment + Value: !Ref EnvironmentTag + + EcrDkrVpcEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: ShouldCreateEcrDkrEndpoint + Properties: + VpcId: !Ref VpcId + ServiceName: !Sub "com.amazonaws.${AWS::Region}.ecr.dkr" + VpcEndpointType: Interface + SubnetIds: !Ref SubnetIds + SecurityGroupIds: + - !Ref VpcEndpointSecurityGroup + PrivateDnsEnabled: true + Tags: + - Key: Name + Value: !Sub "${IDPStackName}-ecr-dkr" + - Key: IDPStack + Value: !Ref IDPStackName + - Key: Environment + Value: !Ref EnvironmentTag + + CodeBuildVpcEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: ShouldCreateCodeBuildEndpoint + Properties: + VpcId: !Ref VpcId + ServiceName: !Sub "com.amazonaws.${AWS::Region}.codebuild" + VpcEndpointType: Interface + SubnetIds: !Ref SubnetIds + SecurityGroupIds: + - !Ref VpcEndpointSecurityGroup + PrivateDnsEnabled: true + Tags: + - Key: Name + Value: !Sub "${IDPStackName}-codebuild" + - Key: IDPStack + Value: !Ref IDPStackName + - Key: Environment + Value: !Ref EnvironmentTag + ########################################################################## # Gateway Endpoints (free — only created when RouteTableIds are provided) ########################################################################## diff --git a/template.yaml b/template.yaml index 4b2df9ea2..15838b5f6 100644 --- a/template.yaml +++ b/template.yaml @@ -557,6 +557,88 @@ Parameters: Default: "" Description: SageMaker MLflow tracking server ARN + # ── Air-gapped / private network parameters ─────────────────────────────── + # Leave these empty for standard internet-connected deployments. + # Set them when CodeBuild cannot reach public container registries or PyPI. + UvImage: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal ECR URI for the uv build tool image. + Replaces ghcr.io/astral-sh/uv:0.9.6 in Dockerfile.optimized. + Example: 123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-base:uv-0.9.6 + + LambdaBaseImage: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal ECR URI for the Lambda Python base image. + Replaces public.ecr.aws/lambda/python:3.12-arm64 in Dockerfile.optimized. + Example: 123456789012.dkr.ecr.us-east-1.amazonaws.com/idp-base:lambda-python-3.12-arm64 + + UvIndexUrl: + Type: String + Default: "" + NoEcho: true + Description: >- + (Optional — air-gapped) Internal PyPI index URL for uv to install Lambda requirements.txt packages. + Replaces pypi.org when CodeBuild cannot reach the public internet. + May contain authentication credentials in the URL — stored with NoEcho. + Example: https://artifactory.company.com/artifactory/api/pypi/pypi-virtual/simple/ + + ArtifactoryDockerUrl: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Artifactory Docker registry hostname that CodeBuild logs into before + pulling UV_IMAGE and LAMBDA_BASE_IMAGE. Required when images are stored in Artifactory instead + of ECR. Must be used together with ArtifactoryCredentialsSecretArn. + Example: artifactory.company.com + + ArtifactoryCredentialsSecretArn: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) ARN of an AWS Secrets Manager secret containing Artifactory Docker + registry credentials. The secret must have JSON format: {"username":"...","password":"..."}. + Required when ArtifactoryDockerUrl is set. Create the secret once with: + aws secretsmanager create-secret --name idp/artifactory-docker-creds + --secret-string '{"username":"svc-build","password":""}' + AllowedPattern: "^(|arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+)$" + + NpmRegistryUrl: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Internal npm registry URL for the Web UI CodeBuild build. + Replaces registry.npmjs.org when CodeBuild cannot reach the public internet. + Example: https://artifactory.company.com/artifactory/api/npm/npm-virtual/ + + CodeBuildVpcId: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) VPC ID to place CodeBuild in for network isolation. + When set, both DockerBuildProject and UICodeBuildProject run inside this VPC. + Requires CodeBuildSubnetIds and CodeBuildSecurityGroupId. + The subnets must have VPC endpoints for ECR (ecr.api, ecr.dkr), S3 (Gateway), + Secrets Manager, Logs, and CodeBuild — run scripts/deploy-vpc-endpoints.py + --codebuild-endpoints to create them. + + CodeBuildSubnetIds: + Type: CommaDelimitedList + Default: "" + Description: >- + (Optional — air-gapped) Comma-separated subnet IDs for CodeBuild VPC placement. + Required when CodeBuildVpcId is set. + + CodeBuildSecurityGroupId: + Type: String + Default: "" + Description: >- + (Optional — air-gapped) Security group ID for CodeBuild VPC placement. + Required when CodeBuildVpcId is set. Must allow outbound HTTPS (443) to VPC endpoints. + Rules: MLflowTrackingURIRequired: RuleCondition: !Equals [ !Ref EnableMLflow, "true" ] @@ -1527,6 +1609,19 @@ Resources: EnableMLflow: !Ref EnableMLflow MlflowTrackingURI: !Ref MlflowTrackingURI + # ── Air-gapped / private network parameters ──────────────────────── + # Passed through to the nested unified pattern stack so CodeBuild + # can use internal registries and PyPI mirrors when internet is blocked. + UvImage: !Ref UvImage + LambdaBaseImage: !Ref LambdaBaseImage + UvIndexUrl: !Ref UvIndexUrl + ArtifactoryDockerUrl: !Ref ArtifactoryDockerUrl + ArtifactoryCredentialsSecretArn: !Ref ArtifactoryCredentialsSecretArn + NpmRegistryUrl: !Ref NpmRegistryUrl + CodeBuildVpcId: !Ref CodeBuildVpcId + CodeBuildSubnetIds: !Join [",", !Ref CodeBuildSubnetIds] + CodeBuildSecurityGroupId: !Ref CodeBuildSecurityGroupId + ########################################################################## # Encryption key ##########################################################################