syncgcp.yaml

name: Upload to GCP Bucket 
on:
  push:
    branches: [main, citest]
  workflow_dispatch:

jobs:
  upload_and_dockerize:
    runs-on: ubuntu-22.04

    steps:
      # Step 1: Checkout the repository
      - name: Checkout repository
        uses: actions/checkout@v2

      # Step 2: Set up Python environment
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.10'

      # Step 3: Install Google Cloud CLI
      - name: Install Google Cloud CLI
        run: |
          sudo apt-get update
          sudo apt-get install -y google-cloud-cli

      # Step 4: Decode and Write GCP Service Account Key
      - name: Decode and Write GCP Service Account Key
        run: |
          echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" | base64 -d > /tmp/gcp-key.json
        shell: bash

      # Step 5: Authenticate using the Temporary File
      - name: Authenticate with GCP
        env:
          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp-key.json
        run: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}

      # Step 6: Set GCP Project ID
      - name: Set GCP Project ID
        run: gcloud config set project ${{ secrets.GCP_PROJECT_ID }}

      # Step 7: Upload Artifacts to GCP Bucket
      - name: Upload Artifacts
        env:
          GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
        run: |
          gsutil -m cp -r artifacts/* gs://${GCS_BUCKET_NAME}/artifacts/
          gsutil -m cp -r pipeline/airflow/dags/* gs://${GCS_BUCKET_NAME}/airflow-dags/

      # Step 8: Remove Broken Symbolic Links
      - name: Remove Broken Symbolic Links
        run: |
          find . -xtype l -delete

      # Step 9: Upload All Files to GCP Bucket
      - name: Upload All Files to GCP Bucket
        env:
          GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
        run: |
          gsutil -m rsync -r . gs://${GCS_BUCKET_NAME}/GitHub/

      # Step 10: Verify Upload
      - name: Verify GCP Bucket Upload
        env:
          GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
        run: |
          echo "Listing files in GCS bucket:"
          gsutil ls gs://${GCS_BUCKET_NAME}/

      # Step 11: Build Docker Image
      - name: Build Docker Image
        run: |
          gcloud auth configure-docker us-east1-docker.pkg.dev
          docker build -t us-east1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/gcf-artifacts/upload-workflow:${{ github.run_id }} .

      # Step 12: Push Docker Image
      - name: Push Docker Image
        run: |
          docker push us-east1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/gcf-artifacts/upload-workflow:${{ github.run_id }}