prod-release-deploy.yaml

name: Release Pipeline

on:
  workflow_dispatch:
    inputs:
      bump_type:
        description: 'Version bump type'
        required: true
        default: 'patch'
        type: choice
        options:
          - major
          - minor
          - patch
      force_build_all:
        description: 'Force rebuild all components'
        required: false
        type: boolean
        default: true
      components:
        description: 'Components to build (comma-separated: frontend,backend,operator,ambient-runner,state-sync,public-api,ambient-api-server) - leave empty for all'
        required: false
        type: string
        default: ''
concurrency:
  group: prod-release-deploy
  cancel-in-progress: false

jobs:
  release:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    outputs:
      new_tag: ${{ steps.next_version.outputs.new_tag }}
      build-matrix: ${{ steps.matrix.outputs.build-matrix }}
      merge-matrix: ${{ steps.matrix.outputs.merge-matrix }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0  # Fetch all history for changelog generation

      - name: Get Latest Tag
        id: get_latest_tag
        run: |
          # List all existing tags for debugging
          echo "All existing tags:"
          git tag --list 'v*.*.*' --sort=-version:refname

          # Get the latest tag using version sort, or use v0.0.0 if no tags exist
          LATEST_TAG=$(git tag --list 'v*.*.*' --sort=-version:refname | head -n 1)
          if [ -z "$LATEST_TAG" ]; then
            exit 1
          fi
          echo "latest_tag=$LATEST_TAG" >> $GITHUB_OUTPUT
          echo "Latest tag: $LATEST_TAG"

      - name: Calculate Next Version
        id: next_version
        run: |
          LATEST_TAG="${{ steps.get_latest_tag.outputs.latest_tag }}"
          # Remove 'v' prefix for calculation
          VERSION=${LATEST_TAG#v}

          # Split version into components
          IFS='.' read -r MAJOR MINOR PATCH <<< "$VERSION"

          # Bump version based on input
          case "${{ github.event.inputs.bump_type }}" in
            major)
              MAJOR=$((MAJOR + 1))
              MINOR=0
              PATCH=0
              ;;
            minor)
              MINOR=$((MINOR + 1))
              PATCH=0
              ;;
            patch)
              PATCH=$((PATCH + 1))
              ;;
          esac

          NEW_VERSION="v${MAJOR}.${MINOR}.${PATCH}"
          echo "new_tag=$NEW_VERSION" >> $GITHUB_OUTPUT
          echo "New version: $NEW_VERSION"

      - name: Generate Changelog
        id: changelog
        run: |
          LATEST_TAG="${{ steps.get_latest_tag.outputs.latest_tag }}"
          NEW_TAG="${{ steps.next_version.outputs.new_tag }}"

          # Use Python for reliable changelog generation with author grouping
          python3 -c "
          import subprocess, sys

          latest_tag = sys.argv[1]
          new_tag = sys.argv[2]
          repo = sys.argv[3]

          commit_range = 'HEAD' if latest_tag == 'v0.0.0' else f'{latest_tag}..HEAD'

          result = subprocess.run(
              ['git', 'log', commit_range, '--format=%an<<<<DELIM>>>>%s (%h)'],
              capture_output=True, text=True
          )

          if result.returncode != 0:
              print(f'Error: git log failed: {result.stderr}', file=sys.stderr)
              sys.exit(1)

          commits_by_author = {}
          count_by_author = {}

          for line in result.stdout.strip().split('\n'):
              if line and '<<<<DELIM>>>>' in line:
                  author, commit = line.split('<<<<DELIM>>>>', 1)
                  if author not in commits_by_author:
                      commits_by_author[author] = []
                      count_by_author[author] = 0
                  commits_by_author[author].append(commit)
                  count_by_author[author] += 1

          sorted_authors = sorted(count_by_author.items(), key=lambda x: x[1], reverse=True)

          # Detect first-time contributors
          first_timers = []
          if latest_tag != 'v0.0.0':
              # Resolve tag to ISO date — --before requires a date, not a ref name
              tag_date_result = subprocess.run(
                  ['git', 'log', '-1', '--format=%ci', latest_tag],
                  capture_output=True, text=True
              )
              tag_date = tag_date_result.stdout.strip()
              if tag_date_result.returncode == 0 and tag_date:
                  # Get all unique author names before the tag date in one call
                  prior = subprocess.run(
                      ['git', 'log', '--all', f'--before={tag_date}', '--format=%an'],
                      capture_output=True, text=True
                  )
                  prior_authors = set()
                  if prior.returncode == 0 and prior.stdout.strip():
                      prior_authors = set(prior.stdout.strip().split('\n'))
                  for author, _ in sorted_authors:
                      if author not in prior_authors:
                          first_timers.append(author)

          print(f'# Release {new_tag}')
          print()
          print(f'## Changes since {latest_tag}')
          print()

          if first_timers:
              print('## 🎉 First-Time Contributors')
              print()
              for author in sorted(first_timers):
                  print(f'- {author}')
              print()

          for author, count in sorted_authors:
              print(f'### {author} ({count})')
              for commit in commits_by_author[author]:
                  print(f'- {commit}')
              print()

          print(f'**Full Changelog**: https://github.com/{repo}/compare/{latest_tag}...{new_tag}')
          " "$LATEST_TAG" "$NEW_TAG" "${{ github.repository }}" > RELEASE_CHANGELOG.md

          cat RELEASE_CHANGELOG.md

      - name: Create Tag
        id: create_tag
        uses: rickstaa/action-create-tag@v1
        with:
          tag: ${{ steps.next_version.outputs.new_tag }}
          message: "Release ${{ steps.next_version.outputs.new_tag }}"
          force_push_tag: false
          github_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Create Release Archive
        id: create_archive
        run: |
          NEW_TAG="${{ steps.next_version.outputs.new_tag }}"
          ARCHIVE_NAME="vteam-${NEW_TAG}.tar.gz"

          # Create archive of entire repository at this tag
          git archive --format=tar.gz --prefix=vteam-${NEW_TAG}/ HEAD > $ARCHIVE_NAME

          echo "archive_name=$ARCHIVE_NAME" >> $GITHUB_OUTPUT

      - name: Create Release
        id: create_release
        uses: softprops/action-gh-release@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ steps.next_version.outputs.new_tag }}
          name: "Release ${{ steps.next_version.outputs.new_tag }}"
          body_path: RELEASE_CHANGELOG.md
          draft: false
          prerelease: false
          files: |
            ${{ steps.create_archive.outputs.archive_name }}
            RELEASE_CHANGELOG.md

      - name: Build component matrices
        id: matrix
        run: |
          ALL_COMPONENTS='[
            {"name":"frontend","context":"./components/frontend","image":"quay.io/ambient_code/vteam_frontend","dockerfile":"./components/frontend/Dockerfile"},
            {"name":"backend","context":"./components/backend","image":"quay.io/ambient_code/vteam_backend","dockerfile":"./components/backend/Dockerfile"},
            {"name":"operator","context":"./components/operator","image":"quay.io/ambient_code/vteam_operator","dockerfile":"./components/operator/Dockerfile"},
            {"name":"ambient-runner","context":"./components/runners","image":"quay.io/ambient_code/vteam_claude_runner","dockerfile":"./components/runners/ambient-runner/Dockerfile"},
            {"name":"state-sync","context":"./components/runners/state-sync","image":"quay.io/ambient_code/vteam_state_sync","dockerfile":"./components/runners/state-sync/Dockerfile"},
            {"name":"public-api","context":"./components/public-api","image":"quay.io/ambient_code/vteam_public_api","dockerfile":"./components/public-api/Dockerfile"},
            {"name":"ambient-api-server","context":"./components/ambient-api-server","image":"quay.io/ambient_code/vteam_api_server","dockerfile":"./components/ambient-api-server/Dockerfile"}
          ]'

          FORCE_ALL="${{ github.event.inputs.force_build_all }}"
          SELECTED="${{ github.event.inputs.components }}"

          if [ "$FORCE_ALL" == "true" ] || [ -z "$SELECTED" ]; then
            FILTERED="$ALL_COMPONENTS"
          else
            FILTERED=$(echo "$ALL_COMPONENTS" | jq -c --arg sel "$SELECTED" '[.[] | select(.name as $n | $sel | split(",") | map(gsub("^\\s+|\\s+$";"")) | index($n))]')
          fi

          if [ "$(echo "$FILTERED" | jq 'length')" -eq 0 ]; then
            echo "::error::No components matched the selection. Aborting release."
            exit 1
          fi

          BUILD_MATRIX=$(echo "$FILTERED" | jq -c '.')
          MERGE_MATRIX=$(echo "$FILTERED" | jq -c '[.[] | {name, image}]')

          echo "build-matrix=$BUILD_MATRIX" >> $GITHUB_OUTPUT
          echo "merge-matrix=$MERGE_MATRIX" >> $GITHUB_OUTPUT

          echo "Components to build:"
          echo "$FILTERED" | jq -r '.[].name'

  build:
    needs: release
    permissions:
      contents: read
      pull-requests: read
      issues: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        # IMPORTANT: suffix values must match the hardcoded suffixes in
        # merge-manifests. Update both together if arches change.
        arch:
          - runner: ubuntu-latest
            platform: linux/amd64
            suffix: amd64
          - runner: ubuntu-24.04-arm
            platform: linux/arm64
            suffix: arm64
        component: ${{ fromJSON(needs.release.outputs.build-matrix) }}
    runs-on: ${{ matrix.arch.runner }}
    steps:
      - name: Checkout code from the tag generated above
        uses: actions/checkout@v6
        with:
            ref: ${{ needs.release.outputs.new_tag }}
            fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to Quay.io
        uses: docker/login-action@v3
        with:
          registry: quay.io
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}

      - name: Log in to Red Hat Container Registry
        uses: docker/login-action@v3
        with:
          registry: registry.redhat.io
          username: ${{ secrets.REDHAT_USERNAME }}
          password: ${{ secrets.REDHAT_PASSWORD }}

      - name: Build and push ${{ matrix.component.name }} (${{ matrix.arch.suffix }})
        uses: docker/build-push-action@v7
        with:
          context: ${{ matrix.component.context }}
          file: ${{ matrix.component.dockerfile }}
          platforms: ${{ matrix.arch.platform }}
          push: true
          tags: ${{ matrix.component.image }}:${{ needs.release.outputs.new_tag }}-${{ matrix.arch.suffix }}
          build-args: AMBIENT_VERSION=${{ needs.release.outputs.new_tag }}
          cache-from: type=gha,scope=${{ matrix.component.name }}-${{ matrix.arch.suffix }}
          cache-to: type=gha,mode=max,scope=${{ matrix.component.name }}-${{ matrix.arch.suffix }}

  merge-manifests:
    needs: [release, build]
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        component: ${{ fromJSON(needs.release.outputs.merge-matrix) }}
    steps:
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to Quay.io
        uses: docker/login-action@v3
        with:
          registry: quay.io
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}

      - name: Create multi-arch manifest for ${{ matrix.component.name }}
        # Suffixes (-amd64, -arm64) must match the arch matrix in the build job above.
        # Arch-suffixed tags remain in the registry after merging. Clean these up
        # via Quay tag expiration policies or a periodic job.
        run: |
          docker buildx imagetools create \
            -t ${{ matrix.component.image }}:${{ needs.release.outputs.new_tag }} \
            ${{ matrix.component.image }}:${{ needs.release.outputs.new_tag }}-amd64 \
            ${{ matrix.component.image }}:${{ needs.release.outputs.new_tag }}-arm64

  deploy-rhoai-mlflow:
    runs-on: ubuntu-latest
    needs: [release]
    steps:
      - name: Checkout code from release tag
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release.outputs.new_tag }}

      - name: Install oc
        uses: redhat-actions/oc-installer@v1
        with:
          oc_version: 'latest'

      - name: Log in to OpenShift Cluster
        run: |
          oc login ${{ secrets.PROD_OPENSHIFT_SERVER }} --token=${{ secrets.PROD_OPENSHIFT_TOKEN }} --insecure-skip-tls-verify

      - name: Deploy RHOAI operator via OLM
        run: |
          oc apply -f components/manifests/components/openshift-ai/namespace.yaml
          oc apply -f components/manifests/components/openshift-ai/operatorgroup.yaml
          oc apply -f components/manifests/components/openshift-ai/subscription.yaml

      - name: Wait for RHOAI operator to be ready
        run: |
          echo "Waiting for RHOAI operator CSV to appear..."
          for i in $(seq 1 60); do
            CSV=$(oc get subscription rhods-operator -n redhat-ods-operator \
              -o jsonpath='{.status.installedCSV}' 2>/dev/null)
            if [ -n "$CSV" ]; then
              echo "Found CSV: $CSV"
              break
            fi
            if [ "$i" -eq 60 ]; then
              echo "::error::RHOAI operator CSV did not appear within timeout"
              exit 1
            fi
            echo "Attempt $i/60 - CSV not yet available, waiting 10s..."
            sleep 10
          done
          echo "Waiting for CSV $CSV to succeed..."
          oc wait csv "$CSV" -n redhat-ods-operator \
            --for=jsonpath='{.status.phase}'=Succeeded --timeout=600s

      - name: Wait for DataScienceCluster v2 API to be available
        run: |
          echo "Waiting for DataScienceCluster v2 API to be served..."
          for i in $(seq 1 60); do
            if oc api-resources --api-group=datasciencecluster.opendatahub.io 2>/dev/null | grep -q v2; then
              echo "DataScienceCluster v2 API is available"
              break
            fi
            if [ "$i" -eq 60 ]; then
              echo "::error::DataScienceCluster v2 API did not become available within timeout"
              exit 1
            fi
            echo "Attempt $i/60 - v2 API not yet available, waiting 10s..."
            sleep 10
          done

      - name: Apply DSCInitialization and DataScienceCluster
        run: |
          oc apply -f components/manifests/components/openshift-ai/dsci.yaml
          oc apply -f components/manifests/components/openshift-ai/datasciencecluster.yaml

      - name: Wait for MLflow Operator CRD to be available
        run: |
          echo "Waiting for MLflow CRD to be registered..."
          for i in $(seq 1 60); do
            if oc get crd mlflows.mlflow.opendatahub.io &>/dev/null; then
              echo "MLflow CRD is available"
              break
            fi
            if [ "$i" -eq 60 ]; then
              echo "::error::MLflow CRD did not become available within timeout"
              exit 1
            fi
            echo "Attempt $i/60 - MLflow CRD not yet available, waiting 10s..."
            sleep 10
          done

      - name: Ensure mlflow database exists in PostgreSQL
        run: |
          oc exec -n ambient-code deploy/postgresql -- \
            psql -U postgres -tAc \
            "SELECT 1 FROM pg_database WHERE datname = 'mlflow'" | grep -q 1 \
          || oc exec -n ambient-code deploy/postgresql -- \
            psql -U postgres -c "CREATE DATABASE mlflow"

      - name: Ensure mlflow-db-credentials secret exists
        run: |
          if oc get secret mlflow-db-credentials -n redhat-ods-applications &>/dev/null; then
            echo "Secret already exists"
          else
            echo "Creating mlflow-db-credentials from postgresql-credentials..."
            DB_USER=$(oc get secret postgresql-credentials -n ambient-code -o jsonpath='{.data.db\.user}' | base64 -d)
            DB_PASS=$(oc get secret postgresql-credentials -n ambient-code -o jsonpath='{.data.db\.password}' | base64 -d)
            DB_HOST=$(oc get secret postgresql-credentials -n ambient-code -o jsonpath='{.data.db\.host}' | base64 -d)
            DB_PORT=$(oc get secret postgresql-credentials -n ambient-code -o jsonpath='{.data.db\.port}' | base64 -d)
            oc create namespace redhat-ods-applications --dry-run=client -o yaml | oc apply -f -
            oc create secret generic mlflow-db-credentials \
              -n redhat-ods-applications \
              --from-literal=uri="postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}.ambient-code.svc.cluster.local:${DB_PORT}/mlflow?sslmode=disable"
          fi

      - name: Deploy MLflow instance
        run: |
          oc apply -f components/manifests/components/openshift-ai/mlflow.yaml

  deploy-to-openshift:
    runs-on: ubuntu-latest
    needs: [release, merge-manifests]
    steps:
      - name: Checkout code from release tag
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release.outputs.new_tag }}

      - name: Install oc
        uses: redhat-actions/oc-installer@v1
        with:
          oc_version: 'latest'

      - name: Install kustomize
        uses: imranismail/setup-kustomize@v2
        with:
          kustomize-version: '5.4.3'

      - name: Log in to OpenShift Cluster
        run: |
          oc login ${{ secrets.PROD_OPENSHIFT_SERVER }} --token=${{ secrets.PROD_OPENSHIFT_TOKEN }} --insecure-skip-tls-verify

      - name: Deploy observability stack
        run: |
          oc apply -k components/manifests/observability/

      - name: Determine which components were built
        id: built
        run: |
          MATRIX='${{ needs.release.outputs.build-matrix }}'
          NAMES=$(echo "$MATRIX" | jq -r '[.[].name] | join(",")')
          echo "names=$NAMES" >> $GITHUB_OUTPUT
          echo "Built components: $NAMES"

      - name: Update kustomization with release image tags
        working-directory: components/manifests/overlays/production
        run: |
          RELEASE_TAG="${{ needs.release.outputs.new_tag }}"
          BUILT="${{ steps.built.outputs.names }}"

          # Map component names to their deployment names and container names
          # runner and state-sync are Jobs (no deployment); their tags come from the operator env
          declare -A DEPLOY_MAP=(
            ["frontend"]="frontend:frontend"
            ["backend"]="backend-api:backend-api"
            ["operator"]="agentic-operator:agentic-operator"
            ["public-api"]="public-api:public-api"
            ["ambient-api-server"]="ambient-api-server:ambient-api-server"
          )

          for comp_image in \
            "frontend:quay.io/ambient_code/vteam_frontend" \
            "backend:quay.io/ambient_code/vteam_backend" \
            "operator:quay.io/ambient_code/vteam_operator" \
            "ambient-runner:quay.io/ambient_code/vteam_claude_runner" \
            "state-sync:quay.io/ambient_code/vteam_state_sync" \
            "public-api:quay.io/ambient_code/vteam_public_api" \
            "ambient-api-server:quay.io/ambient_code/vteam_api_server"; do
            COMP="${comp_image%%:*}"
            IMAGE="${comp_image#*:}"

            # Seed kustomize with the currently deployed tag so unbuilt components
            # don't fall back to the repo's ":latest" placeholder
            DEPLOY_INFO="${DEPLOY_MAP[$COMP]:-}"
            if [ -n "$DEPLOY_INFO" ]; then
              DEPLOY_NAME="${DEPLOY_INFO%%:*}"
              CONTAINER_NAME="${DEPLOY_INFO#*:}"
              CURRENT_TAG=$(oc get deployment "$DEPLOY_NAME" -n ambient-code \
                -o jsonpath="{.spec.template.spec.containers[?(@.name=='${CONTAINER_NAME}')].image}" 2>/dev/null \
                | grep -oP ':\K[^"]+$' || true)
              if [ -n "$CURRENT_TAG" ]; then
                kustomize edit set image ${IMAGE}:latest=${IMAGE}:${CURRENT_TAG}
              fi
            elif [ "$COMP" = "ambient-runner" ]; then
              CURRENT_TAG=$(oc get deployment agentic-operator -n ambient-code \
                -o jsonpath='{.spec.template.spec.containers[?(@.name=="agentic-operator")].env[?(@.name=="AMBIENT_CODE_RUNNER_IMAGE")].value}' 2>/dev/null \
                | grep -oP ':\K[^"]+$' || true)
              if [ -n "$CURRENT_TAG" ]; then
                kustomize edit set image ${IMAGE}:latest=${IMAGE}:${CURRENT_TAG}
              fi
            elif [ "$COMP" = "state-sync" ]; then
              CURRENT_TAG=$(oc get deployment agentic-operator -n ambient-code \
                -o jsonpath='{.spec.template.spec.containers[?(@.name=="agentic-operator")].env[?(@.name=="STATE_SYNC_IMAGE")].value}' 2>/dev/null \
                | grep -oP ':\K[^"]+$' || true)
              if [ -n "$CURRENT_TAG" ]; then
                kustomize edit set image ${IMAGE}:latest=${IMAGE}:${CURRENT_TAG}
              fi
            fi

            # Override with the new release tag if this component was built
            if echo ",$BUILT," | grep -q ",$COMP,"; then
              kustomize edit set image ${IMAGE}:latest=${IMAGE}:${RELEASE_TAG}
            fi
          done

      - name: Validate kustomization
        working-directory: components/manifests/overlays/production
        run: |
          kustomize build . > /dev/null
          echo "✅ Kustomization validation passed"

      - name: Apply production overlay with kustomize
        working-directory: components/manifests/overlays/production
        run: |
          oc apply -k . -n ambient-code

      - name: Update frontend environment variables
        run: |
          oc set env deployment/frontend -n ambient-code -c frontend \
            GITHUB_APP_SLUG="ambient-code" \
            FEEDBACK_URL="https://forms.gle/7XiWrvo6No922DUz6"

      - name: Update operator environment variables
        run: |
          RELEASE_TAG="${{ needs.release.outputs.new_tag }}"
          BUILT="${{ steps.built.outputs.names }}"
          ARGS=""
          if echo ",$BUILT," | grep -q ",ambient-runner,"; then
            ARGS="$ARGS AMBIENT_CODE_RUNNER_IMAGE=quay.io/ambient_code/vteam_claude_runner:${RELEASE_TAG}"
          fi
          if echo ",$BUILT," | grep -q ",state-sync,"; then
            ARGS="$ARGS STATE_SYNC_IMAGE=quay.io/ambient_code/vteam_state_sync:${RELEASE_TAG}"
          fi
          if [ -n "$ARGS" ]; then
            oc set env deployment/agentic-operator -n ambient-code -c agentic-operator $ARGS
          fi

      - name: Pin OPERATOR_IMAGE in operator-config ConfigMap
        run: |
          RELEASE_TAG="${{ needs.release.outputs.new_tag }}"
          BUILT="${{ steps.built.outputs.names }}"
          if echo ",$BUILT," | grep -q ",operator,"; then
            oc patch configmap operator-config -n ambient-code --type=merge \
              -p "{\"data\":{\"OPERATOR_IMAGE\":\"quay.io/ambient_code/vteam_operator:${RELEASE_TAG}\"}}"
          fi

      - name: Update agent registry ConfigMap with release image tags
        run: |
          RELEASE_TAG="${{ needs.release.outputs.new_tag }}"
          BUILT="${{ steps.built.outputs.names }}"
          REGISTRY=$(oc get configmap ambient-agent-registry -n ambient-code \
            -o jsonpath='{.data.agent-registry\.json}')
          if echo ",$BUILT," | grep -q ",ambient-runner,"; then
            REGISTRY=$(echo "$REGISTRY" | sed \
              "s|quay.io/ambient_code/vteam_claude_runner[@:][^\"]*|quay.io/ambient_code/vteam_claude_runner:${RELEASE_TAG}|g")
          fi
          if echo ",$BUILT," | grep -q ",state-sync,"; then
            REGISTRY=$(echo "$REGISTRY" | sed \
              "s|quay.io/ambient_code/vteam_state_sync[@:][^\"]*|quay.io/ambient_code/vteam_state_sync:${RELEASE_TAG}|g")
          fi
          oc patch configmap ambient-agent-registry -n ambient-code --type=merge \
            -p "{\"data\":{\"agent-registry.json\":$(echo "$REGISTRY" | jq -Rs .)}}"