stackhpc-ci-cleanup.yml

---
name: Clean up stale CI resources
on:
  schedule:
    # Every 2 hours at quarter past
    - cron: '15 0/2 * * *'

jobs:
  ci-cleanup:
    name: Clean up stale CI resources
    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
    runs-on: ubuntu-latest
    permissions: {}
    strategy:
      matrix:
        environment: [SMS Lab, Leafcloud]
    environment: ${{ matrix.environment }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          path: src/kayobe-config

      - name: Setup Python
        uses: actions/setup-python@v6

      - name: Generate clouds.yaml
        run: |
          cat << EOF > clouds.yaml
          ${{ secrets.CLOUDS_YAML }}
          EOF

      - name: Determine OpenStack release
        id: openstack_release
        run: |
          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT

      - name: Install OpenStack client
        run: |
          pip install python-openstackclient -c https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt

      - name: Clean up aio instances over 3 hours old
        run: |
          result=0
          changes_before=$(date -Imin -d -3hours)
          for status in ACTIVE BUILD ERROR SHUTOFF; do
              for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
                  echo "Cleaning up $status instance $instance"
                  openstack server show $instance
                  if ! openstack server delete $instance; then
                      echo "Failed to delete $status instance $instance"
                      result=1
                  fi
              done
          done
          exit $result
        env:
          OS_CLOUD: ${{ vars.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Clean up DOCA builder instances over 5 hours old
        run: |
          result=0
          changes_before=$(date -Imin -d -5hours)
          for status in ACTIVE BUILD ERROR SHUTOFF; do
              for instance in $(openstack server list --tags skc-ci-doca-builder --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
                  echo "Cleaning up $status instance $instance"
                  openstack server show $instance
                  if ! openstack server delete $instance; then
                      echo "Failed to delete $status instance $instance"
                      result=1
                  fi
              done
          done
          exit $result
        env:
          OS_CLOUD: ${{ vars.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Clean up host image builder instances over 5 hours old
        run: |
          result=0
          changes_before=$(date -Imin -d -5hours)
          for status in ACTIVE BUILD ERROR SHUTOFF; do
              for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
                  echo "Cleaning up $status instance $instance"
                  openstack server show $instance
                  if ! openstack server delete $instance; then
                      echo "Failed to delete $status instance $instance"
                      result=1
                  fi
              done
          done
          exit $result
        env:
          OS_CLOUD: ${{ vars.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Clean up GH actions multinode instances over 12 hours old
        run: |
          result=0
          changes_before=$(date -Imin -d -12hours)
          for status in ACTIVE BUILD ERROR SHUTOFF; do
              for instance in $(openstack server list --tags gh-actions-multinode --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
                  echo "Cleaning up $status instance $instance"
                  openstack server show $instance
                  if ! openstack server delete $instance; then
                      echo "Failed to delete $status instance $instance"
                      result=1
                  fi
              done
          done
          exit $result
        env:
          OS_CLOUD: ${{ vars.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Send message to Slack via Workflow Builder
        uses: slackapi/slack-github-action@v1.26.0
        with:
          payload: |
            {
              "channel-id": "${{ env.SLACK_CHANNEL_ID }}",
              "inputs": "${{ env.INPUTS }}",
              "message": "${{ env.MESSAGE }}",
              "results-url": "${{ env.RESULTS_URL }}",
              "workflow-url": "${{ env.WORKFLOW_URL }}"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          # #release-train-alerts
          SLACK_CHANNEL_ID: C03B28HRP53
          INPUTS: "N/A"
          MESSAGE: "SKC CI Cleanup workflow failed :sob:"
          RESULTS_URL: "N/A"
          WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
        if: failure()