release-pipeline.yml

name: OpenCue Release Pipeline

# Trigger this pipeline when a commit is tagged with a version number, e.g. "v0.4.32".
on:
  push:
    tags:
      - "v*"

jobs:
  check_opencue_licenses:
    name: Check Licenses of OpenCue Dependencies
    runs-on: ubuntu-22.04
    container: python:3.9
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Run license check
        shell: bash
        run: ci/check_licenses.sh

  build_opencue_packages:
    name: Build Python Packages
    runs-on: ubuntu-22.04
    container: python:3.7
    outputs:
      opencue_proto_path: ${{ steps.package_outputs.outputs.opencue_proto_path }}
      opencue_rqd_path: ${{ steps.package_outputs.outputs.opencue_rqd_path }}
      opencue_pycue_path: ${{ steps.package_outputs.outputs.opencue_pycue_path }}
      opencue_pyoutline_path: ${{ steps.package_outputs.outputs.opencue_pyoutline_path }}
      opencue_cueadmin_path: ${{ steps.package_outputs.outputs.opencue_cueadmin_path }}
      opencue_cueman_path: ${{ steps.package_outputs.outputs.opencue_cueman_path }}
      opencue_cuesubmit_path: ${{ steps.package_outputs.outputs.opencue_cuesubmit_path }}
      opencue_cuegui_path: ${{ steps.package_outputs.outputs.opencue_cuegui_path }}
      opencue_cuenimby_path: ${{ steps.package_outputs.outputs.opencue_cuenimby_path }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-tags: true
          fetch-depth: 0

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Get current tag name
        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV}

      - name: Verify tag name and version match
        run: |
          set -e
          if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then
            echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}"
            echo "Original GITHUB_REF: ${GITHUB_REF}"
            exit 1
          fi

      - uses: ./.github/actions/build-python-packages

      - name: Gather package paths
        id: package_outputs
        run: |
          echo "opencue_proto_path=$(find ./packages -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_rqd_path=$(find ./packages -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_pycue_path=$(find ./packages -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_pyoutline_path=$(find ./packages -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuesubmit_path=$(find ./packages -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cueadmin_path=$(find ./packages -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuenimby_path=$(find ./packages -name 'opencue_cuenimby-*.whl' -print -quit)" >> $GITHUB_OUTPUT

  build_rust_binaries:
    name: Build Rust Binaries
    strategy:
      matrix:
        include:
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-22.04
          - target: x86_64-unknown-linux-musl
            os: ubuntu-22.04
          - target: x86_64-apple-darwin
            os: macos-14
          - target: aarch64-apple-darwin
            os: macos-14
          - target: x86_64-pc-windows-msvc
            os: windows-2022
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-tags: true
          fetch-depth: 0

      - name: Mark repository as safe
        run: git config --global --add safe.directory $GITHUB_WORKSPACE
        shell: bash

      - name: Get current tag name
        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV}
        shell: bash

      - name: Verify tag name and version match
        run: |
          set -e
          if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then
            echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}"
            echo "Original GITHUB_REF: ${GITHUB_REF}"
            exit 1
          fi
        shell: bash

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}

      - name: Install dependencies (Ubuntu)
        if: matrix.os == 'ubuntu-22.04'
        run: |
          sudo apt-get update
          sudo apt-get install -y libx11-dev protobuf-compiler
          if [ "${{ matrix.target }}" = "x86_64-unknown-linux-musl" ]; then
            sudo apt-get install -y musl-tools
          fi

      - name: Install dependencies (macOS)
        if: matrix.os == 'macos-14'
        run: |
          brew install protobuf

      - name: Install dependencies (Windows)
        if: matrix.os == 'windows-2022'
        run: choco install protoc -y

      - name: Set build ID
        run: |
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> $GITHUB_ENV
        shell: bash

      - name: Build Rust binaries
        run: |
          cd rust
          cargo build --release --target ${{ matrix.target }} --no-default-features -p rqd
        shell: bash

      - name: Create release directory
        run: mkdir -p release
        shell: bash

      - name: Copy binaries (Linux)
        if: matrix.os == 'ubuntu-22.04'
        run: |
          cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}

      - name: Copy binaries (macOS)
        if: matrix.os == 'macos-14'
        run: |
          cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}

      - name: Copy binaries (Windows)
        if: matrix.os == 'windows-2022'
        run: |
          copy rust\target\${{ matrix.target }}\release\openrqd.exe release\openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}.exe
        shell: cmd

      - name: Upload binaries
        uses: actions/upload-artifact@v4
        with:
          name: rust-binaries-${{ matrix.target }}
          path: release/

  upload_python_packages_pypi:
    needs: build_opencue_packages
    runs-on: ubuntu-22.04
    container: python:3.11
    name: "Upload python packages to pypi repo"
    environment: release
    env:
      TWINE_USERNAME: "__token__"
      TWINE_PASSWORD: "${{ secrets.PYPI_API_TOKEN }}"

    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Upload packages
        run: |
          python3 -m pip install --upgrade twine
          python3 -m twine upload --verbose packages/opencue_*

  release_docker_images:
    # needs: preflight
    runs-on: ubuntu-22.04
    environment: release
    strategy:
      matrix:
        component: [cuebot, rqd]

    name: Release ${{ matrix.component }} Docker image
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-tags: true
          fetch-depth: 0

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Set build ID
        run: |
          set -e
          echo "Build ID: $(ci/generate_version_number.py)"
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}

      - name: Get current tag name
        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV}

      - name: Verify tag name and version match
        run: |
          set -e
          if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then
            echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}"
            echo "Original GITHUB_REF: ${GITHUB_REF}"
            exit 1
          fi

      - name: Pull Docker image from staging
        run: |
          set -e
          docker pull opencuebuild/${{ matrix.component }}:${BUILD_ID}

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USER }}
          password: ${{ secrets.DOCKER_PASS }}

      - name: Rebuild and push Docker image
        run: |
          set -e
          docker tag opencuebuild/${{ matrix.component }}:${BUILD_ID} opencue/${{ matrix.component }}:${{ env.BUILD_ID }}
          docker push opencue/${{ matrix.component }}:${{ env.BUILD_ID }}
          docker tag opencuebuild/${{ matrix.component }}:${BUILD_ID} opencue/${{ matrix.component }}:latest
          docker push opencue/${{ matrix.component }}:latest

      # This step has been failing with permission issues.
      # Commenting this out temporarily to unblock the release of v1.4
      # - name: Docker Hub Description
      #   uses: peter-evans/dockerhub-description@v4
      #   with:
      #     username: ${{ secrets.DOCKER_USER }}
      #     password: ${{ secrets.DOCKER_PASS }}
      #     repository: opencue/${{ matrix.component }}
      #     readme-filepath: ./${{ matrix.component }}/README.md

  create_release:
    needs: build_rust_binaries
    name: Create Release
    environment: release
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.S3_REGION }}
          role-to-assume: ${{ secrets.AWS_S3_ROLE }}
          role-duration-seconds: 1800

      - name: Set build ID
        run: |
          set -e
          ci/generate_version_number.sh > VERSION
          echo "Build ID: $(ci/generate_version_number.py)"
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}

      - name: Download Rust binaries artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: rust-binaries-*
          path: rust-binaries
          merge-multiple: true

      - name: Fetch artifacts
        id: fetch_artifacts
        env:
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
        run: |
          mkdir -p "${GITHUB_WORKSPACE}/artifacts/"
          aws s3 sync "s3://${S3_BUCKET}/opencue/${BUILD_ID}/" "${GITHUB_WORKSPACE}/artifacts/"
          echo "filenames=$(ls "${GITHUB_WORKSPACE}/artifacts/" | xargs)" >> ${GITHUB_OUTPUT}

      - name: List artifacts
        run: |
          echo ${{ steps.fetch_artifacts.outputs.filenames }}

      - name: Generate release notes
        id: release_notes
        run: |
          last_tagged_version=$(git describe --tags --abbrev=0 $(git rev-list --tags --skip=1 --max-count=1))
          commits_since_last_release=$(git log --reverse --pretty="* %H %s" ${last_tagged_version}..HEAD)
          # Use a delimiter to preserve the multiline string.
          # See https://github.community/t/set-output-truncates-multiline-strings/16852
          delimiter="$(openssl rand -hex 8)"
          echo "commits<<${delimiter}" >> ${GITHUB_OUTPUT}
          echo "${commits_since_last_release}" >> ${GITHUB_OUTPUT}
          echo "${delimiter}" >> ${GITHUB_OUTPUT}

      - name: Create release
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: v${{ env.BUILD_ID }}
          release_name: v${{ env.BUILD_ID }}
          body: |
            To learn how to install and configure OpenCue, see our [Getting Started guide](https://www.opencue.io/docs/getting-started/).

            ## Changes:

            ${{ steps.release_notes.outputs.commits }}
          draft: true
          prerelease: false

      - name: Upload License
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/LICENSE
          asset_name: LICENSE
          asset_content_type: application/octet-stream

      - name: Upload Database Schema
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/schema-${{ env.BUILD_ID }}.sql
          asset_name: schema-${{ env.BUILD_ID }}.sql
          asset_content_type: application/octet-stream

      - name: Upload Demo Data
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/seed_data-${{ env.BUILD_ID }}.sql
          asset_name: seed_data-${{ env.BUILD_ID }}.sql
          asset_content_type: application/octet-stream

      - name: Upload Cuebot JAR
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/cuebot-${{ env.BUILD_ID }}-all.jar
          asset_name: cuebot-${{ env.BUILD_ID }}-all.jar
          asset_content_type: application/octet-stream

      - name: Upload Cuebot RPM
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/opencue-cuebot-${{ env.BUILD_ID }}-1.noarch.rpm
          asset_name: opencue-cuebot-${{ env.BUILD_ID }}-1.noarch.rpm
          asset_content_type: application/octet-stream

      # Pause python artifact releases for now while ASWF discusses the implications of publishing
      # binary artifacts
      #
      # - name: Upload RQD package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_rqd-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: rqd-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload CueGUI package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_cuegui-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: cuegui-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload PyCue package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_pycue-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: pycue-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload PyOutline package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_pyoutline-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: pyoutline-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload CueSubmit package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_cuesubmit-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: cuesubmit-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload CueAdmin package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_cueadmin-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: cueadmin-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload CueMan package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_cueman-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: cueman-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      # - name: Upload CueNIMBY package
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }}
      #     asset_path: ${{ github.workspace }}/artifacts/opencue_cuenimby-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_name: cuenimby-${{ env.BUILD_ID }}-py2.py3-none-any.whl
      #     asset_content_type: application/octet-stream

      - name: Upload openrqd (Linux GNU)
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-gnu
          asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-gnu
          asset_content_type: application/octet-stream

      - name: Upload openrqd (Linux MUSL)
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-musl
          asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-musl
          asset_content_type: application/octet-stream

      - name: Upload openrqd (macOS Intel)
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-apple-darwin
          asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-apple-darwin
          asset_content_type: application/octet-stream

      - name: Upload openrqd (macOS Apple Silicon)
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-aarch64-apple-darwin
          asset_name: openrqd-${{ env.BUILD_ID }}-aarch64-apple-darwin
          asset_content_type: application/octet-stream

      - name: Upload openrqd (Windows)
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-pc-windows-msvc.exe
          asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-pc-windows-msvc.exe
          asset_content_type: application/octet-stream