build-and-push.yml

name: Docker Image Build and Push

on:
  workflow_dispatch:
    inputs:
      skip_tests:
        description: "Skip tests and push image directly (for debugging image build issues)"
        required: false
        type: boolean
        default: false
  push:
    branches:
      - main
    paths:
      - ".github/actions/build-and-push/action.yml"
      - ".github/actions/run-notebook-test/action.yml"
      - ".github/actions/validate-packages/action.yml"
      - ".github/workflows/build-and-push.yml"
      - "Dockerfile"
      - "conda-env/env-*.yml"
      - "install.R"
      - "apt.txt"
      - "Desktop/**"
      - "tests/**"
      - ".github/scripts/**"

permissions:
  contents: write
  packages: write
  pull-requests: write

jobs:
  build-test-push:
    runs-on: ubuntu-latest
    outputs:
      image_tag: ${{ steps.setup.outputs.tag }}
      image_name: ${{ steps.setup.outputs.full_name }}
      image_pushed: ${{ steps.push_image.outputs.pushed }}
      validation_status: ${{ steps.validation_status.outputs.status }}
      tests_skipped: ${{ steps.check_skip.outputs.skip_tests }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Check if tests should be skipped
        id: check_skip
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.skip_tests }}" = "true" ]; then
            echo "skip_tests=true" >> $GITHUB_OUTPUT
            echo "SKIP_TESTS=true" >> $GITHUB_ENV
          else
            echo "skip_tests=false" >> $GITHUB_OUTPUT
            echo "SKIP_TESTS=false" >> $GITHUB_ENV
          fi

      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set image metadata (name, tag, version)
        id: setup
        run: |
          set -euo pipefail

          echo "name=container-images/${{ github.event.repository.name }}" >> $GITHUB_ENV
          echo "full_name=ghcr.io/${{ github.repository_owner }}/container-images/${{ github.event.repository.name }}" >> $GITHUB_OUTPUT

          # Extract VERSION from Dockerfile (strip whitespace + quotes)
          if grep -q "LABEL org.opencontainers.image.version=" Dockerfile; then
            version=$(grep "LABEL org.opencontainers.image.version=" Dockerfile | head -1 | cut -d '=' -f2- | tr -d ' "')
          elif grep -q "LABEL VERSION=" Dockerfile; then
            version=$(grep "LABEL VERSION=" Dockerfile | head -1 | cut -d '=' -f2- | tr -d ' "')
          else
            echo "::error::No VERSION label found in Dockerfile"
            exit 1
          fi

          # Enforce YYYY.MM.DD (fail hard if bogus)
          if ! echo "$version" | grep -Eq '^[0-9]{4}\.[0-9]{2}\.[0-9]{2}$'; then
            echo "::error::Invalid VERSION format '$version' (expected YYYY.MM.DD)"
            exit 1
          fi

          echo "version=${version}" >> $GITHUB_ENV
          echo "version=${version}" >> $GITHUB_OUTPUT

          short_sha=$(echo "${{ github.sha }}" | cut -c1-7)
          echo "tag=${short_sha}" >> $GITHUB_ENV
          echo "tag=${short_sha}" >> $GITHUB_OUTPUT

          echo "IMAGE_NAME=ghcr.io/${{ github.repository_owner }}/container-images/${{ github.event.repository.name }}:${short_sha}" >> $GITHUB_ENV

      - name: Build the Docker image
        run: |
          set -euo pipefail

          docker build . \
            -f Dockerfile \
            --build-arg GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} \
            --tag ghcr.io/${{ github.repository_owner }}/${{ env.name }}:${{ env.tag }} \
            --tag ghcr.io/${{ github.repository_owner }}/${{ env.name }}:latest

          # Deterministic tag for version
          docker tag ghcr.io/${{ github.repository_owner }}/${{ env.name }}:${{ env.tag }} \
                    ghcr.io/${{ github.repository_owner }}/${{ env.name }}:${{ env.version }}

      # ============================================================================
      # TEST NOTEBOOKS (composite action)
      # ============================================================================
      - name: Run notebook tests (all test-*.ipynb)
        if: env.SKIP_TESTS == 'false'
        uses: ./.github/actions/run-notebook-test
        with:
          image: ${{ env.IMAGE_NAME }}
          tests_dir: tests
          pattern: "test-*.ipynb"
          exclude_pattern: "*-output.ipynb"
          output_suffix: "-output"
          fail_fast: "true"
          earthdata_user: ${{ secrets.EARTHDATA_USER }}
          earthdata_pass: ${{ secrets.EARTHDATA_PASS }}

      - name: Upload notebook outputs
        if: always() && env.SKIP_TESTS == 'false'
        uses: actions/upload-artifact@v4
        with:
          name: test-results
          path: tests/*-output.ipynb
          if-no-files-found: warn
          retention-days: 7

      # ============================================================================
      # VALIDATE PACKAGES (composite action)
      # ============================================================================
      - name: Validate packages
        if: env.SKIP_TESTS == 'false'
        uses: ./.github/actions/validate-packages
        with:
          image: ${{ env.IMAGE_NAME }}
          python_version: "3.11"

      - name: Set validation status
        if: env.SKIP_TESTS == 'false'
        id: validation_status
        run: |
          set -euo pipefail
          success_count=$(grep -c "STATUS: SUCCESS" reproducibility/build.log 2>/dev/null || true)
          if [ "$success_count" -eq 2 ]; then
            echo "status=success" >> $GITHUB_OUTPUT
          else
            echo "status=failed" >> $GITHUB_OUTPUT
          fi

      - name: Upload validation results
        if: always() && env.SKIP_TESTS == 'false'
        uses: actions/upload-artifact@v4
        with:
          name: validation-results
          path: |
            reproducibility/packages-python-pinned.yaml
            reproducibility/packages-r-pinned.R
            reproducibility/build.log
          if-no-files-found: warn
          retention-days: 7

      # ============================================================================
      # PUSH IMAGE (only if tests passed or were skipped)
      # ============================================================================
      - name: Push the Docker image
        id: push_image
        if: env.SKIP_TESTS == 'true' || success()
        run: |
          set -euo pipefail

          docker push ghcr.io/${{ github.repository_owner }}/${{ env.name }}:${{ env.tag }}
          docker push ghcr.io/${{ github.repository_owner }}/${{ env.name }}:latest
          docker push ghcr.io/${{ github.repository_owner }}/${{ env.name }}:${{ env.version }}

          echo "pushed=true" >> $GITHUB_OUTPUT
          echo "✓ Docker image pushed successfully"

      # ============================================================================
      # ALSO PUSH IMAGE TO NEW LOCATION (backwards compatibility)
      # ============================================================================
      - name: Push new tags (back-compat location)
        if: steps.push_image.outputs.pushed == 'true'
        run: |
          set -euo pipefail

          SRC="ghcr.io/${{ github.repository_owner }}/${{ env.name }}"
          DST="ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}"

          # Retag the already-built local image (no rebuild, no pull)
          docker tag ${SRC}:${{ env.tag }}     ${DST}:${{ env.tag }}
          docker tag ${SRC}:latest            ${DST}:latest
          docker tag ${SRC}:${{ env.version }} ${DST}:${{ env.version }}

          # Push all three tags to new location
          docker push ${DST}:${{ env.tag }}
          docker push ${DST}:latest
          docker push ${DST}:${{ env.version }}

      # ============================================================================
      # UPLOAD RELEASE METADATA (consumed by draft-release workflow)
      # ============================================================================
      - name: Write release metadata
        if: steps.push_image.outputs.pushed == 'true'
        run: |
          set -euo pipefail
          dockerfile_sha256=$(sha256sum Dockerfile | awk '{print $1}')
          jq -n \
            --arg sha "${{ github.sha }}" \
            --arg tests_skipped "${{ steps.check_skip.outputs.skip_tests }}" \
            --arg tag "${{ env.tag }}" \
            --arg version "${{ env.version }}" \
            --arg dockerfile_sha256 "$dockerfile_sha256" \
            '{sha: $sha, tests_skipped: $tests_skipped, tag: $tag, version: $version, dockerfile_sha256: $dockerfile_sha256}' \
            > release-metadata.json

      - name: Upload release metadata artifact
        if: steps.push_image.outputs.pushed == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: release-metadata
          path: release-metadata.json
          retention-days: 7

  create-release-pr:
    runs-on: ubuntu-latest
    needs: build-test-push
    if: |
      needs.build-test-push.outputs.image_pushed == 'true' &&
      needs.build-test-push.result == 'success' &&
      needs.build-test-push.outputs.tests_skipped == 'false'
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Download validation results
        uses: actions/download-artifact@v4
        with:
          name: validation-results
          path: reproducibility

      - name: Set PR body based on validation results
        id: pr_body
        run: |
          success_count=$(grep -c "STATUS: SUCCESS" reproducibility/build.log 2>/dev/null || true)

          test_status="✅ All tests passed"
          if [ "${{ needs.build-test-push.outputs.validation_status }}" != "success" ]; then
            test_status="⚠️ Package validation had warnings (see build.log)"
          fi

          if [ "$success_count" -eq 2 ]; then
            cat > reproducibility/pr_body.txt << EOF
          ## Package Validation: ✅ SUCCESS

          **Docker Image:** \`${{ needs.build-test-push.outputs.image_name }}:${{ needs.build-test-push.outputs.image_tag }}\`

          **Test Status:** ${test_status}

          All Python packages from env-*.yml files and all R packages from install.R,
          install_geospatial.sh, and install_tidyverse.sh are present in the container image.

          ### Changes
          - **reproducibility/packages-python-pinned.yaml**: Updated with packages from env files only
          - **reproducibility/packages-r-pinned.R**: Updated with latest R package versions from site-library
          - **reproducibility/build.log**: Validation report for both Python and R packages

          Auto-generated from latest container image build.
          EOF
          else
            cat > reproducibility/pr_body.txt << EOF
          ## Package Validation: ⚠️ FAILED

          **Docker Image:** \`${{ needs.build-test-push.outputs.image_name }}:${{ needs.build-test-push.outputs.image_tag }}\`

          **Test Status:** ${test_status}

          Some packages were NOT found in the container image. Check reproducibility/build.log for details.

          ### Changes
          - **reproducibility/packages-python-pinned.yaml**: Updated with subset of packages that were successfully installed
          - **reproducibility/packages-r-pinned.R**: Updated with latest R package versions
          - **reproducibility/build.log**: Detailed report of missing packages (both Python and R)

          **Action Required:** Review reproducibility/build.log for missing packages and investigate installation issues.

          Auto-generated from latest container image build.
          EOF
          fi

      - name: Commit changes
        run: |
          git config --local user.email "github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

          git add reproducibility/packages-python-pinned.yaml reproducibility/packages-r-pinned.R reproducibility/build.log

          if ! git diff --staged --quiet; then
            git commit -m "Update pinned package versions from container image"
          else
            echo "No changes to commit"
          fi

      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          branch: ci/update-pins-${{ needs.build-test-push.outputs.image_tag }}
          delete-branch: true
          title: "Update pinned package versions - Build ${{ needs.build-test-push.outputs.image_tag }}"
          body-path: reproducibility/pr_body.txt
          assignees: eeholmes
          reviewers: eeholmes
          commit-message: "Update pinned package versions from container image ${{ needs.build-test-push.outputs.image_tag }}"