diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea1a96c3ffb..0f65cf4ab6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [main, staging, dev] + branches: ["**"] pull_request: branches: [main, staging, dev] @@ -14,17 +14,60 @@ permissions: contents: read jobs: + secret-scan: + name: Secret Scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Check committed secrets + run: bun run check:secrets + test-build: name: Test and Build - if: github.ref != 'refs/heads/dev' || github.event_name == 'pull_request' + needs: [secret-scan] + if: >- + github.event_name == 'pull_request' || + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/staging' uses: ./.github/workflows/test-build.yml secrets: inherit + # Resolve auto-incrementing GHCR tag for the production image branch (0.0.N) + resolve-ghcr-tag: + name: Resolve GHCR Tag + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/version-4.2-prod' + permissions: + contents: read + packages: read + outputs: + image_tag: ${{ steps.tag.outputs.image_tag }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Compute next GHCR tag + id: tag + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + TAG="$(bash scripts/ci/ghcr-next-branch-tag.sh)" + echo "image_tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "Computed GHCR tag: ${TAG}" + # Detect if this is a version release commit (e.g., "v0.5.24: ...") detect-version: name: Detect Version - runs-on: blacksmith-4vcpu-ubuntu-2404 - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/dev') + runs-on: ubuntu-latest + if: github.event_name == 'push' outputs: version: ${{ steps.extract.outputs.version }} is_release: ${{ steps.extract.outputs.is_release }} @@ -69,14 +112,84 @@ jobs: environment: dev secrets: inherit - # Dev: build all 3 images for ECR only (no GHCR, no ARM64) + # version-4.2-prod: build all 3 images for GHCR with auto-increment tag + build-ghcr-branch: + name: Build GHCR Branch Images + needs: [secret-scan, resolve-ghcr-tag] + if: github.event_name == 'push' && github.ref == 'refs/heads/version-4.2-prod' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - dockerfile: ./docker/app.Dockerfile + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-simstudio + - dockerfile: ./docker/db.Dockerfile + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-migrations + - dockerfile: ./docker/realtime.Dockerfile + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-realtime + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Generate tags + id: meta + run: | + IMAGE_TAG="${{ needs.resolve-ghcr-tag.outputs.image_tag }}" + GHCR_IMAGE="${{ matrix.ghcr_image }}" + echo "tags=${GHCR_IMAGE}:${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + + - name: Add swap + run: | + SWAPFILE=/mnt/gh-actions-swap + if sudo swapon --show=NAME | awk -v file="$SWAPFILE" '$1 == file { found = 1 } END { exit !found }'; then + echo "$SWAPFILE is already active" + free -h + exit 0 + fi + sudo swapoff "$SWAPFILE" 2>/dev/null || true + sudo rm -f "$SWAPFILE" + sudo dd if=/dev/zero of="$SWAPFILE" bs=1M count=10240 status=none + sudo chmod 600 "$SWAPFILE" + sudo mkswap "$SWAPFILE" + sudo swapon "$SWAPFILE" + free -h + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + build-args: | + NEXT_PUBLIC_APP_URL=${{ vars.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }} + provenance: false + sbom: false + + # Dev: build all 3 images for ECR only (no ARM64) build-dev: name: Build Dev ECR - needs: [detect-version, migrate-dev] + needs: [secret-scan, detect-version, migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest permissions: contents: read + packages: write id-token: write strategy: fail-fast: false @@ -90,26 +203,26 @@ jobs: ecr_repo_secret: ECR_REALTIME steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@v6 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }} aws-region: ${{ secrets.DEV_AWS_REGION }} - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2 + uses: aws-actions/amazon-ecr-login@v2 - name: Login to Docker Hub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 + uses: docker/setup-buildx-action@v3 - name: Resolve ECR repo name id: ecr-repo @@ -117,25 +230,49 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || '' }} + - name: Generate tags + id: meta + run: | + ECR_IMAGE="${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev" + echo "tags=${ECR_IMAGE}" >> "$GITHUB_OUTPUT" + + - name: Add swap + run: | + SWAPFILE=/mnt/gh-actions-swap + if sudo swapon --show=NAME | awk -v file="$SWAPFILE" '$1 == file { found = 1 } END { exit !found }'; then + echo "$SWAPFILE is already active" + free -h + exit 0 + fi + sudo swapoff "$SWAPFILE" 2>/dev/null || true + sudo rm -f "$SWAPFILE" + sudo dd if=/dev/zero of="$SWAPFILE" bs=1M count=10240 status=none + sudo chmod 600 "$SWAPFILE" + sudo mkswap "$SWAPFILE" + sudo swapon "$SWAPFILE" + free -h + - name: Build and push - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: docker/build-push-action@v6 with: context: . file: ${{ matrix.dockerfile }} platforms: linux/amd64 push: true - tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev + tags: ${{ steps.meta.outputs.tags }} + build-args: | + NEXT_PUBLIC_APP_URL=${{ vars.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }} provenance: false sbom: false # Main/staging: build AMD64 images and push to ECR + GHCR build-amd64: name: Build AMD64 - needs: [test-build, detect-version, migrate] + needs: [secret-scan, test-build, detect-version, migrate] if: >- github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest permissions: contents: read packages: write @@ -145,44 +282,43 @@ jobs: matrix: include: - dockerfile: ./docker/app.Dockerfile - ghcr_image: ghcr.io/simstudioai/simstudio + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-simstudio ecr_repo_secret: ECR_APP - dockerfile: ./docker/db.Dockerfile - ghcr_image: ghcr.io/simstudioai/migrations + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-migrations ecr_repo_secret: ECR_MIGRATIONS - dockerfile: ./docker/realtime.Dockerfile - ghcr_image: ghcr.io/simstudioai/realtime + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-realtime ecr_repo_secret: ECR_REALTIME steps: - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@v6 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2 + uses: aws-actions/amazon-ecr-login@v2 - name: Login to Docker Hub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GHCR - if: github.ref == 'refs/heads/main' - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 + uses: docker/setup-buildx-action@v3 - name: Resolve ECR repo name id: ecr-repo @@ -221,22 +357,40 @@ jobs: echo "tags=${TAGS}" >> $GITHUB_OUTPUT + - name: Add swap + run: | + SWAPFILE=/mnt/gh-actions-swap + if sudo swapon --show=NAME | awk -v file="$SWAPFILE" '$1 == file { found = 1 } END { exit !found }'; then + echo "$SWAPFILE is already active" + free -h + exit 0 + fi + sudo swapoff "$SWAPFILE" 2>/dev/null || true + sudo rm -f "$SWAPFILE" + sudo dd if=/dev/zero of="$SWAPFILE" bs=1M count=10240 status=none + sudo chmod 600 "$SWAPFILE" + sudo mkswap "$SWAPFILE" + sudo swapon "$SWAPFILE" + free -h + - name: Build and push images - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: docker/build-push-action@v6 with: context: . file: ${{ matrix.dockerfile }} platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} + build-args: | + NEXT_PUBLIC_APP_URL=${{ vars.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }} provenance: false sbom: false # Build ARM64 images for GHCR (main branch only, runs in parallel) build-ghcr-arm64: name: Build ARM64 (GHCR Only) - needs: [detect-version] - runs-on: blacksmith-8vcpu-ubuntu-2404-arm + needs: [secret-scan, detect-version] + runs-on: ubuntu-24.04-arm if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: read @@ -246,25 +400,25 @@ jobs: matrix: include: - dockerfile: ./docker/app.Dockerfile - image: ghcr.io/simstudioai/simstudio + image: ghcr.io/${{ github.repository_owner }}/p2-sim-simstudio - dockerfile: ./docker/db.Dockerfile - image: ghcr.io/simstudioai/migrations + image: ghcr.io/${{ github.repository_owner }}/p2-sim-migrations - dockerfile: ./docker/realtime.Dockerfile - image: ghcr.io/simstudioai/realtime + image: ghcr.io/${{ github.repository_owner }}/p2-sim-realtime steps: - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@v6 - name: Login to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 + uses: docker/setup-buildx-action@v3 - name: Generate ARM64 tags id: meta @@ -281,21 +435,39 @@ jobs: echo "tags=${TAGS}" >> $GITHUB_OUTPUT + - name: Add swap + run: | + SWAPFILE=/mnt/gh-actions-swap + if sudo swapon --show=NAME | awk -v file="$SWAPFILE" '$1 == file { found = 1 } END { exit !found }'; then + echo "$SWAPFILE is already active" + free -h + exit 0 + fi + sudo swapoff "$SWAPFILE" 2>/dev/null || true + sudo rm -f "$SWAPFILE" + sudo dd if=/dev/zero of="$SWAPFILE" bs=1M count=10240 status=none + sudo chmod 600 "$SWAPFILE" + sudo mkswap "$SWAPFILE" + sudo swapon "$SWAPFILE" + free -h + - name: Build and push ARM64 to GHCR - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: docker/build-push-action@v6 with: context: . file: ${{ matrix.dockerfile }} platforms: linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} + build-args: | + NEXT_PUBLIC_APP_URL=${{ vars.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }} provenance: false sbom: false # Create GHCR multi-arch manifests (only for main, after both builds) create-ghcr-manifests: name: Create GHCR Manifests - runs-on: blacksmith-2vcpu-ubuntu-2404 + runs-on: ubuntu-latest needs: [build-amd64, build-ghcr-arm64, detect-version] if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: @@ -303,13 +475,13 @@ jobs: strategy: matrix: include: - - image: ghcr.io/simstudioai/simstudio - - image: ghcr.io/simstudioai/migrations - - image: ghcr.io/simstudioai/realtime + - image: ghcr.io/${{ github.repository_owner }}/p2-sim-simstudio + - image: ghcr.io/${{ github.repository_owner }}/p2-sim-migrations + - image: ghcr.io/${{ github.repository_owner }}/p2-sim-realtime steps: - name: Login to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -344,15 +516,15 @@ jobs: # Check if docs changed check-docs-changes: name: Check Docs Changes - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' outputs: docs_changed: ${{ steps.filter.outputs.docs }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/checkout@v6 with: - fetch-depth: 2 # Need at least 2 commits to detect changes - - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4 + fetch-depth: 2 # Need at least 2 commits to detect changes + - uses: dorny/paths-filter@v4 id: filter with: filters: | @@ -372,21 +544,21 @@ jobs: # Create GitHub Release (only for version commits on main, after all builds complete) create-release: name: Create GitHub Release - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest needs: [create-ghcr-manifests, detect-version] if: needs.detect-version.outputs.is_release == 'true' permissions: contents: write steps: - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + uses: oven-sh/setup-bun@v2 with: - bun-version: 1.3.13 + bun-version: latest - name: Install dependencies run: bun install --frozen-lockfile @@ -394,4 +566,4 @@ jobs: - name: Create release env: GH_PAT: ${{ secrets.GITHUB_TOKEN }} - run: bun run scripts/create-single-release.ts ${{ needs.detect-version.outputs.version }} \ No newline at end of file + run: bun run scripts/create-single-release.ts ${{ needs.detect-version.outputs.version }} diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 54f8a2f47a8..1f1b1a9bb23 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -2,182 +2,108 @@ name: Build and Push Images on: workflow_call: + inputs: + image_tag: + description: Docker image tag to push to GHCR + required: true + type: string workflow_dispatch: + inputs: + image_tag: + description: Docker image tag to push to GHCR (for example, 0.0.7) + required: true + type: string permissions: contents: read packages: write - id-token: write jobs: - build-amd64: - name: Build AMD64 - runs-on: blacksmith-8vcpu-ubuntu-2404 + secret-scan: + name: Secret Scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Check committed secrets + run: bun run check:secrets + + build-ghcr-images: + name: Build GHCR Images + needs: [secret-scan] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: include: - dockerfile: ./docker/app.Dockerfile - ghcr_image: ghcr.io/simstudioai/simstudio - ecr_repo_secret: ECR_APP + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-simstudio - dockerfile: ./docker/db.Dockerfile - ghcr_image: ghcr.io/simstudioai/migrations - ecr_repo_secret: ECR_MIGRATIONS + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-migrations - dockerfile: ./docker/realtime.Dockerfile - ghcr_image: ghcr.io/simstudioai/realtime - ecr_repo_secret: ECR_REALTIME - outputs: - registry: ${{ steps.login-ecr.outputs.registry }} - + ghcr_image: ghcr.io/${{ github.repository_owner }}/p2-sim-realtime steps: - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} - aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_REGION || secrets.STAGING_AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2 - - - name: Login to Docker Hub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + uses: actions/checkout@v6 - name: Login to GHCR - if: github.ref == 'refs/heads/main' - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 + uses: docker/setup-buildx-action@v3 + + - name: Validate image tag + run: | + if [[ ! "${{ inputs.image_tag }}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Invalid Docker tag: ${{ inputs.image_tag }}" >&2 + exit 1 + fi - name: Generate tags id: meta run: | - ECR_REGISTRY="${{ steps.login-ecr.outputs.registry }}" - ECR_REPO="${{ secrets[matrix.ecr_repo_secret] }}" + IMAGE_TAG="${{ inputs.image_tag }}" GHCR_IMAGE="${{ matrix.ghcr_image }}" + echo "tags=${GHCR_IMAGE}:${IMAGE_TAG}" >> "$GITHUB_OUTPUT" - # ECR tags (always build for ECR) - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - ECR_TAG="latest" - elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then - ECR_TAG="dev" - else - ECR_TAG="staging" - fi - ECR_IMAGE="${ECR_REGISTRY}/${ECR_REPO}:${ECR_TAG}" - - # Build tags list - TAGS="${ECR_IMAGE}" - - # Add GHCR tags only for main branch - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - GHCR_AMD64="${GHCR_IMAGE}:latest-amd64" - GHCR_SHA="${GHCR_IMAGE}:${{ github.sha }}-amd64" - TAGS="${TAGS},$GHCR_AMD64,$GHCR_SHA" + - name: Add swap + run: | + SWAPFILE=/mnt/gh-actions-swap + if sudo swapon --show=NAME | awk -v file="$SWAPFILE" '$1 == file { found = 1 } END { exit !found }'; then + echo "$SWAPFILE is already active" + free -h + exit 0 fi - - echo "tags=${TAGS}" >> $GITHUB_OUTPUT + sudo swapoff "$SWAPFILE" 2>/dev/null || true + sudo rm -f "$SWAPFILE" + sudo dd if=/dev/zero of="$SWAPFILE" bs=1M count=10240 status=none + sudo chmod 600 "$SWAPFILE" + sudo mkswap "$SWAPFILE" + sudo swapon "$SWAPFILE" + free -h - name: Build and push images - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 + uses: docker/build-push-action@v6 with: context: . file: ${{ matrix.dockerfile }} platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} + build-args: | + NEXT_PUBLIC_APP_URL=${{ vars.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }} provenance: false sbom: false - - build-ghcr-arm64: - name: Build ARM64 (GHCR Only) - runs-on: blacksmith-8vcpu-ubuntu-2404-arm - if: github.ref == 'refs/heads/main' - strategy: - fail-fast: false - matrix: - include: - - dockerfile: ./docker/app.Dockerfile - image: ghcr.io/simstudioai/simstudio - - dockerfile: ./docker/db.Dockerfile - image: ghcr.io/simstudioai/migrations - - dockerfile: ./docker/realtime.Dockerfile - image: ghcr.io/simstudioai/realtime - - steps: - - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Login to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 - - - name: Generate ARM64 tags - id: meta - run: | - IMAGE="${{ matrix.image }}" - echo "tags=${IMAGE}:latest-arm64,${IMAGE}:${{ github.sha }}-arm64" >> $GITHUB_OUTPUT - - - name: Build and push ARM64 to GHCR - uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 - with: - context: . - file: ${{ matrix.dockerfile }} - platforms: linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - provenance: false - sbom: false - - create-ghcr-manifests: - name: Create GHCR Manifests - runs-on: blacksmith-2vcpu-ubuntu-2404 - needs: [build-amd64, build-ghcr-arm64] - if: github.ref == 'refs/heads/main' - strategy: - matrix: - include: - - image: ghcr.io/simstudioai/simstudio - - image: ghcr.io/simstudioai/migrations - - image: ghcr.io/simstudioai/realtime - - steps: - - name: Login to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create and push manifests - run: | - IMAGE_BASE="${{ matrix.image }}" - - # Create latest manifest - docker manifest create "${IMAGE_BASE}:latest" \ - "${IMAGE_BASE}:latest-amd64" \ - "${IMAGE_BASE}:latest-arm64" - docker manifest push "${IMAGE_BASE}:latest" - - # Create SHA manifest - docker manifest create "${IMAGE_BASE}:${{ github.sha }}" \ - "${IMAGE_BASE}:${{ github.sha }}-amd64" \ - "${IMAGE_BASE}:${{ github.sha }}-arm64" - docker manifest push "${IMAGE_BASE}:${{ github.sha }}" \ No newline at end of file diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 25a133f44b3..1d19f18f1fe 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -5,8 +5,10 @@ DATABASE_URL="postgresql://postgres:your_password@localhost:5432/simstudio" # (logs listing, audit logs, dashboard aggregations). Reads fall back to # DATABASE_URL when unset. # DATABASE_REPLICA_URL="" - -# PostgreSQL Port (Optional) - defaults to 5432 if not specified +# Docker Compose (p2prod / test-env): set these so db + migrations use the same credentials. +# POSTGRES_USER=postgres +# POSTGRES_PASSWORD=your_password +# POSTGRES_DB=simstudio # POSTGRES_PORT=5432 # Authentication (Required unless DISABLE_AUTH=true) diff --git a/docker-compose.dev-env.yml b/docker-compose.dev-env.yml index 73667d6eb2a..3d4cdc08bc3 100644 --- a/docker-compose.dev-env.yml +++ b/docker-compose.dev-env.yml @@ -5,61 +5,29 @@ services: dockerfile: docker/app.Dockerfile restart: unless-stopped ports: - - '3000:3000' + - "3000:3000" deploy: resources: limits: memory: 8G + env_file: + - apps/sim/.env environment: - # Core - NODE_ENV=production - DATABASE_URL=${DATABASE_URL:?DATABASE_URL is required} + - SOCKET_SERVER_URL=http://realtime:3002 - BETTER_AUTH_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-} - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} - ENCRYPTION_KEY=${ENCRYPTION_KEY:-your_encryption_key_here} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} - - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} - - CRON_SECRET=${CRON_SECRET:-} - # Socket / Realtime - - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002} - - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} - - REDIS_URL=${REDIS_URL:-} - # Copilot / Agent - - COPILOT_API_KEY=${COPILOT_API_KEY} - - SIM_AGENT_API_URL=${SIM_AGENT_API_URL} - # OAuth - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-placeholder} - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET:-placeholder} - GITHUB_CLIENT_ID=${GITHUB_CLIENT_ID:-placeholder} - GITHUB_CLIENT_SECRET=${GITHUB_CLIENT_SECRET:-placeholder} - - SLACK_CLIENT_ID=${SLACK_CLIENT_ID:-placeholder} - - SLACK_CLIENT_SECRET=${SLACK_CLIENT_SECRET:-placeholder} - - FB_CLIENT_ID=${FB_CLIENT_ID:-placeholder} - - FB_CLIENT_SECRET=${FB_CLIENT_SECRET:-placeholder} - - FB_ACCESS_TOKEN=${FB_ACCESS_TOKEN:-} - - SHOPIFY_CLIENT_ID=${SHOPIFY_CLIENT_ID:-placeholder} - - SHOPIFY_CLIENT_SECRET=${SHOPIFY_CLIENT_SECRET:-placeholder} - - GOOGLE_ADS_DEVELOPER_TOKEN=${GOOGLE_ADS_DEVELOPER_TOKEN:-} - - GOOGLE_ADS_CLIENT_ID=${GOOGLE_ADS_CLIENT_ID:-placeholder} - - GOOGLE_ADS_CLIENT_SECRET=${GOOGLE_ADS_CLIENT_SECRET:-placeholder} - - GOOGLE_ADS_REFRESH_TOKEN=${GOOGLE_ADS_REFRESH_TOKEN:-} - # AWS / S3 - - AWS_REGION=${AWS_REGION:-} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} - - S3_BUCKET_NAME=${S3_BUCKET_NAME:-} - - S3_LOGS_BUCKET_NAME=${S3_LOGS_BUCKET_NAME:-} - - S3_KB_BUCKET_NAME=${S3_KB_BUCKET_NAME:-} - - S3_EXECUTION_FILES_BUCKET_NAME=${S3_EXECUTION_FILES_BUCKET_NAME:-} - - S3_CHAT_BUCKET_NAME=${S3_CHAT_BUCKET_NAME:-} - - S3_COPILOT_BUCKET_NAME=${S3_COPILOT_BUCKET_NAME:-} - - S3_PROFILE_PICTURES_BUCKET_NAME=${S3_PROFILE_PICTURES_BUCKET_NAME:-} - # Email - RESEND_API_KEY=${RESEND_API_KEY:-placeholder} - - FROM_EMAIL_ADDRESS=${FROM_EMAIL_ADDRESS:-} - # LLM / AI keys + - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} + - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002} + - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://realtime:3002} - OPENAI_API_KEY=${OPENAI_API_KEY:-} - OPENAI_API_KEY_1=${OPENAI_API_KEY_1:-} - OPENAI_API_KEY_2=${OPENAI_API_KEY_2:-} @@ -72,39 +40,20 @@ services: - XAI_API_KEY_1=${XAI_API_KEY_1:-} - XAI_API_KEY_2=${XAI_API_KEY_2:-} - XAI_API_KEY_3=${XAI_API_KEY_3:-} - - SAMBANOVA_API_KEY=${SAMBANOVA_API_KEY:-} - - SAMBANOVA_API_KEY_1=${SAMBANOVA_API_KEY_1:-} - - SAMBANOVA_API_KEY_2=${SAMBANOVA_API_KEY_2:-} - - SAMBANOVA_API_KEY_3=${SAMBANOVA_API_KEY_3:-} - - GEMINI_API_KEY=${GEMINI_API_KEY:-} - - IDEOGRAM_API_KEY=${IDEOGRAM_API_KEY:-} - # Tool / integration keys - - FIGMA_API_KEY=${FIGMA_API_KEY:-} - - BROWSER_USE_API_KEY=${BROWSER_USE_API_KEY:-} - - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY:-} - - SEMRUSH_API_KEY=${SEMRUSH_API_KEY:-} - - FREESTYLE_API_KEY=${FREESTYLE_API_KEY:-placeholder} - # Google public - - NEXT_PUBLIC_GOOGLE_API_KEY=${NEXT_PUBLIC_GOOGLE_API_KEY:-} - - NEXT_PUBLIC_GOOGLE_PROJECT_NUMBER=${NEXT_PUBLIC_GOOGLE_PROJECT_NUMBER:-} - # Arena - - ARENA_BACKEND_BASE_URL=${ARENA_BACKEND_BASE_URL:-} - - ARENA_FRONTEND_APP_URL=${ARENA_FRONTEND_APP_URL:-} - - NEXT_PUBLIC_ARENA_BACKEND_BASE_URL=${NEXT_PUBLIC_ARENA_BACKEND_BASE_URL:-} - - NEXT_PUBLIC_ARENA_FRONTEND_APP_URL=${NEXT_PUBLIC_ARENA_FRONTEND_APP_URL:-} - # Analytics / admin - - NEXT_PUBLIC_MIX_PANEL_TOKEN=${NEXT_PUBLIC_MIX_PANEL_TOKEN:-} - - NEXT_PUBLIC_PLATFORM_ADMIN_EMAILS=${NEXT_PUBLIC_PLATFORM_ADMIN_EMAILS:-} - # Misc - - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - - CHROMEDRIVER_PATH=${CHROMEDRIVER_PATH:-} + - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} + - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} + - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} + - REDIS_URL=${REDIS_URL:-} + - COPILOT_API_KEY=${COPILOT_API_KEY} + - SIM_AGENT_API_URL=${SIM_AGENT_API_URL} + depends_on: db: condition: service_healthy realtime: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3000'] + test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3000"] interval: 90s timeout: 5s retries: 3 @@ -116,55 +65,25 @@ services: dockerfile: docker/realtime.Dockerfile restart: unless-stopped ports: - - '3002:3002' + - "3002:3002" deploy: resources: limits: memory: 4G + env_file: + - apps/realtime/.env environment: - - DATABASE_URL=${DATABASE_URL:?DATABASE_URL is required} - - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000} - - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} - - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} - - REDIS_URL=${REDIS_URL:-} + - NODE_ENV=production + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} depends_on: db: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3002/health'] + test: + ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3002/health"] interval: 90s timeout: 5s retries: 3 - start_period: 10s - - migrations: - image: ghcr.io/simstudioai/migrations:latest - environment: - - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} - depends_on: - db: - condition: service_healthy - command: ['bun', 'run', 'db:migrate'] - restart: 'no' - - db: - image: pgvector/pgvector:pg17 - restart: unless-stopped - ports: - - '5432:5432' - environment: - - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - - POSTGRES_DB=${POSTGRES_DB:-simstudio} - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U postgres'] - interval: 5s - timeout: 5s - retries: 5 volumes: - postgres_data: \ No newline at end of file + postgres_data: diff --git a/docker-compose.local-build.yml b/docker-compose.local-build.yml index b5cf734b891..ccd874d59a7 100644 --- a/docker-compose.local-build.yml +++ b/docker-compose.local-build.yml @@ -12,9 +12,3 @@ services: build: context: . dockerfile: docker/realtime.Dockerfile - - migrations: - image: p2-sim-migrations - build: - context: . - dockerfile: docker/db.Dockerfile diff --git a/docker-compose.p2prod.yml b/docker-compose.p2prod.yml index 9df9e771207..3064fa7ca84 100644 --- a/docker-compose.p2prod.yml +++ b/docker-compose.p2prod.yml @@ -5,13 +5,17 @@ services: dockerfile: docker/app.Dockerfile restart: unless-stopped ports: - - '3000:3000' + - "3000:3000" deploy: resources: limits: memory: 12G + env_file: + - apps/sim/.env environment: - - DATABASE_URL=postgresql://p2agent:37zJ905LhNrI@prod-agents.cyva1vnsabpo.us-west-2.rds.amazonaws.com:5432/simstudio?sslmode=require + - NODE_ENV=production + - DATABASE_URL=${DATABASE_URL:?DATABASE_URL is required} + - SOCKET_SERVER_URL=http://realtime:3002 - BETTER_AUTH_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} @@ -50,7 +54,14 @@ services: realtime: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3000/api/health'] + test: + [ + "CMD", + "wget", + "--spider", + "--quiet", + "http://127.0.0.1:3000/api/health", + ] interval: 90s timeout: 5s retries: 3 @@ -62,23 +73,22 @@ services: dockerfile: docker/realtime.Dockerfile restart: unless-stopped ports: - - '3002:3002' + - "3002:3002" deploy: resources: limits: memory: 6G + env_file: + - apps/realtime/.env environment: - - DATABASE_URL=postgresql://p2agent:37zJ905LhNrI@prod-agents.cyva1vnsabpo.us-west-2.rds.amazonaws.com:5432/simstudio?sslmode=require - - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000} - - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} - - REDIS_URL=${REDIS_URL:-} + - NODE_ENV=production + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} depends_on: db: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3002/health'] + test: + ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3002/health"] interval: 90s timeout: 5s retries: 3 @@ -91,14 +101,14 @@ services: depends_on: db: condition: service_healthy - command: ['bun', 'run', 'db:migrate'] - restart: 'no' + command: ["bun", "run", "db:migrate"] + restart: "no" db: image: pgvector/pgvector:pg17 restart: unless-stopped ports: - - '${POSTGRES_PORT:-5432}:5432' + - "${POSTGRES_PORT:-5432}:5432" environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} @@ -106,10 +116,10 @@ services: volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ['CMD-SHELL', 'pg_isready -U postgres'] + test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 5s retries: 5 volumes: - postgres_data: \ No newline at end of file + postgres_data: diff --git a/docker-compose.sandbox.yml b/docker-compose.sandbox.yml index a74202fe6af..a949e43812a 100644 --- a/docker-compose.sandbox.yml +++ b/docker-compose.sandbox.yml @@ -8,61 +8,29 @@ services: NEXT_PUBLIC_APP_URL: ${NEXT_PUBLIC_APP_URL:-http://localhost:3000} restart: unless-stopped ports: - - '3000:3000' + - "3000:3000" deploy: resources: limits: memory: 12G + env_file: + - apps/sim/.env environment: - # Core - NODE_ENV=production - DATABASE_URL=${DATABASE_URL:?DATABASE_URL is required} + - SOCKET_SERVER_URL=http://realtime:3002 - BETTER_AUTH_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-} - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} - ENCRYPTION_KEY=${ENCRYPTION_KEY:-your_encryption_key_here} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} - - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} - - CRON_SECRET=${CRON_SECRET:-} - # Socket / Realtime - - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://localhost:3002} - - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} - - REDIS_URL=${REDIS_URL:-} - # Copilot / Agent - - COPILOT_API_KEY=${COPILOT_API_KEY} - - SIM_AGENT_API_URL=${SIM_AGENT_API_URL} - # OAuth - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-placeholder} - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET:-placeholder} - GITHUB_CLIENT_ID=${GITHUB_CLIENT_ID:-placeholder} - GITHUB_CLIENT_SECRET=${GITHUB_CLIENT_SECRET:-placeholder} - - SLACK_CLIENT_ID=${SLACK_CLIENT_ID:-placeholder} - - SLACK_CLIENT_SECRET=${SLACK_CLIENT_SECRET:-placeholder} - - FB_CLIENT_ID=${FB_CLIENT_ID:-placeholder} - - FB_CLIENT_SECRET=${FB_CLIENT_SECRET:-placeholder} - - FB_ACCESS_TOKEN=${FB_ACCESS_TOKEN:-} - - SHOPIFY_CLIENT_ID=${SHOPIFY_CLIENT_ID:-placeholder} - - SHOPIFY_CLIENT_SECRET=${SHOPIFY_CLIENT_SECRET:-placeholder} - - GOOGLE_ADS_DEVELOPER_TOKEN=${GOOGLE_ADS_DEVELOPER_TOKEN:-} - - GOOGLE_ADS_CLIENT_ID=${GOOGLE_ADS_CLIENT_ID:-placeholder} - - GOOGLE_ADS_CLIENT_SECRET=${GOOGLE_ADS_CLIENT_SECRET:-placeholder} - - GOOGLE_ADS_REFRESH_TOKEN=${GOOGLE_ADS_REFRESH_TOKEN:-} - # AWS / S3 - - AWS_REGION=${AWS_REGION:-} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} - - S3_BUCKET_NAME=${S3_BUCKET_NAME:-} - - S3_LOGS_BUCKET_NAME=${S3_LOGS_BUCKET_NAME:-} - - S3_KB_BUCKET_NAME=${S3_KB_BUCKET_NAME:-} - - S3_EXECUTION_FILES_BUCKET_NAME=${S3_EXECUTION_FILES_BUCKET_NAME:-} - - S3_CHAT_BUCKET_NAME=${S3_CHAT_BUCKET_NAME:-} - - S3_COPILOT_BUCKET_NAME=${S3_COPILOT_BUCKET_NAME:-} - - S3_PROFILE_PICTURES_BUCKET_NAME=${S3_PROFILE_PICTURES_BUCKET_NAME:-} - # Email - RESEND_API_KEY=${RESEND_API_KEY:-placeholder} - - FROM_EMAIL_ADDRESS=${FROM_EMAIL_ADDRESS:-} - # LLM / AI keys + - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} + - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002} + - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://realtime:3002} - OPENAI_API_KEY=${OPENAI_API_KEY:-} - OPENAI_API_KEY_1=${OPENAI_API_KEY_1:-} - OPENAI_API_KEY_2=${OPENAI_API_KEY_2:-} @@ -75,32 +43,12 @@ services: - XAI_API_KEY_1=${XAI_API_KEY_1:-} - XAI_API_KEY_2=${XAI_API_KEY_2:-} - XAI_API_KEY_3=${XAI_API_KEY_3:-} - - SAMBANOVA_API_KEY=${SAMBANOVA_API_KEY:-} - - SAMBANOVA_API_KEY_1=${SAMBANOVA_API_KEY_1:-} - - SAMBANOVA_API_KEY_2=${SAMBANOVA_API_KEY_2:-} - - SAMBANOVA_API_KEY_3=${SAMBANOVA_API_KEY_3:-} - - GEMINI_API_KEY=${GEMINI_API_KEY:-} - - IDEOGRAM_API_KEY=${IDEOGRAM_API_KEY:-} - # Tool / integration keys - - FIGMA_API_KEY=${FIGMA_API_KEY:-} - - BROWSER_USE_API_KEY=${BROWSER_USE_API_KEY:-} - - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY:-} - - SEMRUSH_API_KEY=${SEMRUSH_API_KEY:-} - - FREESTYLE_API_KEY=${FREESTYLE_API_KEY:-placeholder} - # Google public - - NEXT_PUBLIC_GOOGLE_API_KEY=${NEXT_PUBLIC_GOOGLE_API_KEY:-} - - NEXT_PUBLIC_GOOGLE_PROJECT_NUMBER=${NEXT_PUBLIC_GOOGLE_PROJECT_NUMBER:-} - # Arena - - ARENA_BACKEND_BASE_URL=${ARENA_BACKEND_BASE_URL:-} - - ARENA_FRONTEND_APP_URL=${ARENA_FRONTEND_APP_URL:-} - - NEXT_PUBLIC_ARENA_BACKEND_BASE_URL=${NEXT_PUBLIC_ARENA_BACKEND_BASE_URL:-} - - NEXT_PUBLIC_ARENA_FRONTEND_APP_URL=${NEXT_PUBLIC_ARENA_FRONTEND_APP_URL:-} - # Analytics / admin - - NEXT_PUBLIC_MIX_PANEL_TOKEN=${NEXT_PUBLIC_MIX_PANEL_TOKEN:-} - - NEXT_PUBLIC_PLATFORM_ADMIN_EMAILS=${NEXT_PUBLIC_PLATFORM_ADMIN_EMAILS:-} - # Misc - - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - - CHROMEDRIVER_PATH=${CHROMEDRIVER_PATH:-} + - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} + - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} + - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} + - REDIS_URL=${REDIS_URL:-} + - COPILOT_API_KEY=${COPILOT_API_KEY} + - SIM_AGENT_API_URL=${SIM_AGENT_API_URL} depends_on: db: condition: service_healthy @@ -109,7 +57,14 @@ services: realtime: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3000/api/health'] + test: + [ + "CMD", + "wget", + "--spider", + "--quiet", + "http://127.0.0.1:3000/api/health", + ] interval: 90s timeout: 5s retries: 3 @@ -121,21 +76,22 @@ services: dockerfile: docker/realtime.Dockerfile restart: unless-stopped ports: - - '3002:3002' + - "3002:3002" deploy: resources: limits: memory: 6G + env_file: + - apps/realtime/.env environment: - - DATABASE_URL=${DATABASE_URL:?DATABASE_URL is required} - - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000} - - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} + - NODE_ENV=production + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} depends_on: db: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3002/health'] + test: + ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3002/health"] interval: 90s timeout: 5s retries: 3 @@ -148,25 +104,4 @@ services: depends_on: db: condition: service_healthy - command: ['bun', 'run', 'db:migrate'] - restart: 'no' - - db: - image: pgvector/pgvector:pg17 - restart: unless-stopped - ports: - - '${POSTGRES_PORT:-5432}:5432' - environment: - - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - - POSTGRES_DB=${POSTGRES_DB:-simstudio} - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U postgres'] - interval: 5s - timeout: 5s - retries: 5 - -volumes: - postgres_data: + command: ["bun", "run", "db:migrate"] diff --git a/docker-compose.test-env.yml b/docker-compose.test-env.yml index cb4f6d3b837..beb25dd07fb 100644 --- a/docker-compose.test-env.yml +++ b/docker-compose.test-env.yml @@ -1,23 +1,23 @@ services: simstudio: - build: - context: . - dockerfile: docker/app.Dockerfile + image: ghcr.io/arenadeveloper02/p2-sim-simstudio:${IMAGE_TAG:-dev} restart: unless-stopped ports: - - '3000:3000' + - "3000:3000" deploy: resources: limits: memory: 8G + env_file: + - apps/sim/.env environment: - NODE_ENV=production - - DATABASE_URL=postgresql://p2agent:Td3%23Mp6%26Vr8%21ThR@p2-agents-dev-v2.cyva1vnsabpo.us-west-2.rds.amazonaws.com:5432/simstudio?sslmode=require + - DATABASE_URL=${DATABASE_URL:?DATABASE_URL is required} + - SOCKET_SERVER_URL=http://realtime:3002 - BETTER_AUTH_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} - ENCRYPTION_KEY=${ENCRYPTION_KEY:-your_encryption_key_here} - - FREESTYLE_API_KEY=${FREESTYLE_API_KEY:-placeholder} - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-placeholder} - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET:-placeholder} - GITHUB_CLIENT_ID=${GITHUB_CLIENT_ID:-placeholder} @@ -25,11 +25,12 @@ services: - RESEND_API_KEY=${RESEND_API_KEY:-placeholder} - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - SOCKET_SERVER_URL=${SOCKET_SERVER_URL:-http://realtime:3002} - - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} + - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://realtime:3002} - OPENAI_API_KEY=${OPENAI_API_KEY:-} - OPENAI_API_KEY_1=${OPENAI_API_KEY_1:-} - OPENAI_API_KEY_2=${OPENAI_API_KEY_2:-} - OPENAI_API_KEY_3=${OPENAI_API_KEY_3:-} + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} - ANTHROPIC_API_KEY_1=${ANTHROPIC_API_KEY_1:-} - ANTHROPIC_API_KEY_2=${ANTHROPIC_API_KEY_2:-} - ANTHROPIC_API_KEY_3=${ANTHROPIC_API_KEY_3:-} @@ -49,67 +50,45 @@ services: realtime: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3000'] + test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3000"] interval: 90s timeout: 5s retries: 3 start_period: 10s realtime: - build: - context: . - dockerfile: docker/realtime.Dockerfile + image: ghcr.io/arenadeveloper02/p2-sim-realtime:${IMAGE_TAG:-dev} restart: unless-stopped ports: - - '3002:3002' + - "3002:3002" deploy: resources: limits: memory: 4G + env_file: + - apps/realtime/.env environment: - - DATABASE_URL=postgresql://p2agent:Td3%23Mp6%26Vr8%21ThR@p2-agents-dev-v2.cyva1vnsabpo.us-west-2.rds.amazonaws.com:5432/simstudio?sslmode=require - - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} - - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000} - - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} - - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-} - - INTERNAL_API_SECRET=${INTERNAL_API_SECRET} - - REDIS_URL=${REDIS_URL:-} + - NODE_ENV=production + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} depends_on: db: condition: service_healthy healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3002/health'] + test: + ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3002/health"] interval: 90s timeout: 5s retries: 3 start_period: 10s migrations: - image: ghcr.io/simstudioai/migrations:latest + image: ghcr.io/arenadeveloper02/p2-sim-migrations:${IMAGE_TAG:-dev} environment: - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} depends_on: db: condition: service_healthy - command: ['bun', 'run', 'db:migrate'] - restart: 'no' - - db: - image: pgvector/pgvector:pg17 - restart: unless-stopped - ports: - - '5432:5432' - environment: - - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - - POSTGRES_DB=${POSTGRES_DB:-simstudio} - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U postgres'] - interval: 5s - timeout: 5s - retries: 5 + command: ["bun", "run", "db:migrate"] volumes: postgres_data: diff --git a/scripts/deploy-ec2-ghcr.sh b/scripts/deploy-ec2-ghcr.sh index 841566d0f33..7d8ce625335 100755 --- a/scripts/deploy-ec2-ghcr.sh +++ b/scripts/deploy-ec2-ghcr.sh @@ -1,6 +1,3 @@ -#!/usr/bin/env bash -set -euo pipefail - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_DEPLOY_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" DEPLOY_ROOT="${DEPLOY_ROOT:-}" @@ -48,6 +45,23 @@ fi export IMAGE_TAG="${IMAGE_TAG:-$IMAGE_TAG_ARG}" +SIM_ENV_FILE="${DEPLOY_ROOT}/apps/sim/.env" +REALTIME_ENV_FILE="${DEPLOY_ROOT}/apps/realtime/.env" + +if [ ! -f "$SIM_ENV_FILE" ]; then + echo "Missing env file: $SIM_ENV_FILE" >&2 + echo "Copy apps/sim/.env.example to apps/sim/.env and configure it before deploying." >&2 + exit 1 +fi + +if [ ! -f "$REALTIME_ENV_FILE" ]; then + echo "Missing env file: $REALTIME_ENV_FILE" >&2 + echo "Copy apps/realtime/.env.example to apps/realtime/.env and configure it before deploying." >&2 + exit 1 +fi + +COMPOSE_ENV_ARGS=(--env-file "$SIM_ENV_FILE") + GHCR_PASSWORD="${GHCR_TOKEN:-${CR_PAT:-}}" if [ -n "$GHCR_PASSWORD" ]; then echo "$GHCR_PASSWORD" | docker login ghcr.io \ @@ -59,15 +73,15 @@ echo "Deploying IMAGE_TAG=${IMAGE_TAG}" echo "Using compose file: ${COMPOSE_FILE}" cd "$DEPLOY_ROOT" -docker compose -f "$COMPOSE_FILE" pull simstudio realtime migrations -docker compose -f "$COMPOSE_FILE" up -d --remove-orphans +docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" pull simstudio realtime migrations +docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" up -d --remove-orphans deadline=$((SECONDS + ${DEPLOY_HEALTH_TIMEOUT_SECONDS:-240})) until curl -fsS http://127.0.0.1:3002/health >/dev/null; do if [ "$SECONDS" -ge "$deadline" ]; then - docker compose -f "$COMPOSE_FILE" ps - docker compose -f "$COMPOSE_FILE" logs --tail=100 realtime + docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" ps + docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" logs --tail=100 realtime exit 1 fi sleep 5 @@ -75,12 +89,12 @@ done until curl -fsS http://127.0.0.1:3000 >/dev/null; do if [ "$SECONDS" -ge "$deadline" ]; then - docker compose -f "$COMPOSE_FILE" ps - docker compose -f "$COMPOSE_FILE" logs --tail=100 simstudio + docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" ps + docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" logs --tail=100 simstudio exit 1 fi sleep 5 done docker image prune -f -docker compose -f "$COMPOSE_FILE" ps +docker compose "${COMPOSE_ENV_ARGS[@]}" -f "$COMPOSE_FILE" ps diff --git a/scripts/deploy-ec2-local-build.sh b/scripts/deploy-ec2-local-build.sh index 2c1f4c13879..683cf8081ab 100755 --- a/scripts/deploy-ec2-local-build.sh +++ b/scripts/deploy-ec2-local-build.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -set -euo pipefail BRANCH="${1:-}" DEPLOY_ROOT="${DEPLOY_ROOT:-/root/git/p2-sim}" @@ -12,16 +11,32 @@ if [ -z "$BRANCH" ]; then exit 1 fi +CONTAINERS=$(docker ps -a -q --filter ancestor="p2-sim-simstudio") +docker stop $CONTAINERS +docker rm $CONTAINERS + +CONTAINERS=$(docker ps -a -q --filter ancestor="p2-sim-realtime") +docker stop $CONTAINERS +docker rm $CONTAINERS + +docker rmi -f "p2-sim-simstudio" +docker rmi -f "p2-sim-realtime" + cd "$DEPLOY_ROOT" +SIM_ENV_FILE="${DEPLOY_ROOT}/apps/sim/.env" +if [ ! -f "$SIM_ENV_FILE" ]; then + echo "Missing env file: $SIM_ENV_FILE" >&2 + exit 1 +fi + +COMPOSE_ENV_ARGS=(--env-file "$SIM_ENV_FILE") + git stash -git fetch origin "$BRANCH" +git fetch git checkout "$BRANCH" git pull --rebase origin "$BRANCH" -docker compose -f "$COMPOSE_FILE" -f "$LOCAL_BUILD_FILE" down --remove-orphans -docker image rm -f p2-sim-simstudio p2-sim-realtime p2-sim-migrations 2>/dev/null || true -docker compose -f "$COMPOSE_FILE" -f "$LOCAL_BUILD_FILE" up -d --build --remove-orphans -docker image prune -f +docker compose -f "$COMPOSE_FILE" -f "$LOCAL_BUILD_FILE" up -d --build cd ~