build_and_test_and_release.yml

name: build_all
# Build, test and release Splitgraph/sgr binary/engine

on: push

jobs:
  build_and_test:
    runs-on: ubuntu-20.04
    if: "!contains(github.event.head_commit.message, '[skip ci]')"
    env:
      COMPOSE_VERSION: "1.25.4"
      POETRY_VERSION: "1.3.2"
      DOCKER_REPO: splitgraph
      DOCKER_ENGINE_IMAGE: engine
      DOCKER_TAG: development
      DBT_VENV: dbt_venv
      TAP_MYSQL_VENV: tap_mysql_venv

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: "recursive"
      - name: Setup Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip_cache
      - name: "Run pre-commit hooks (typechecks and the formatter)"
        run: |
          pip install -U pip
          pip install pre-commit
          # NB this uses mypy in .pre-commit-config.yaml
          pre-commit run --all-files
      - name: "Install prerequisites"
        run: "./.ci/install_prereqs.sh"
      - name: "Install package"
        run: ./.ci/install.sh
      - uses: actions/cache@v3
        with:
          path: ~/docker_images
          key: docker_cache
      - name: "Build the Splitgraph engine"
        run: |
          docker load -i /home/runner/docker_images/engine.tar || true
          docker images
          cd engine && make with_postgis=1
      - name: "Save Docker cache"
        run: |
          mkdir -p ~/docker_images
          docker images
          docker save -o ~/docker_images/engine.tar $DOCKER_REPO/$DOCKER_ENGINE_IMAGE:$DOCKER_TAG
          echo "Saved engine $DOCKER_REPO/$DOCKER_ENGINE_IMAGE:$DOCKER_TAG to cache"
          ls -lah ~/docker_images
      - name: "Start the test Compose stack"
        if: "!contains(github.event.head_commit.message, '[skip test]')"
        run: ./.ci/up_architecture.sh
      - name: "Run the tests"
        if: "!contains(github.event.head_commit.message, '[skip test]')"
        run: |
          . "$(poetry env info --path)/bin/activate"
          export PATH=$PATH:"$PWD"/bin
          poetry run pytest test/ -v -m "not mounting and not example"
          ./wait-for-test-architecture.sh --mounting
          export PATH=$PATH:$TAP_MYSQL_VENV/bin
          poetry run pytest test/ -v -m "mounting and not example" --cov-append
          ./.ci/down_architecture.sh
      - name: "Run example tests"
        # Don't run example tests if we're doing a release -- we'll be
        # recording all of them as Asciinemas anyway, so this is redundant.
        if: "!startsWith(github.ref, 'refs/tags/') && !contains(github.event.head_commit.message, '[skip test]')"
        run: |
          . "$(poetry env info --path)/bin/activate"
          export PATH=$PATH:"$PWD"/bin
          export PATH=$PATH:$(readlink -f $DBT_VENV/bin)
          poetry run pytest examples/ -sv -m "example" --cov-append
      - name: "Submit coverage to Coveralls"
        if: "!contains(github.event.head_commit.message, '[skip test]')"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          poetry run coveralls --service=github
      - name: "Push out Docker engine (tags only)"
        if: "startsWith(github.ref, 'refs/tags/') && !contains(github.event.head_commit.message, '[skip engine]')"
        env:
          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
        run: |
          echo "Uploading engine image to Docker Hub"
          ./.ci/push_engine.sh
      - name: "Build doc / Asciicast bundle"
        if: "startsWith(github.ref, 'refs/tags/') && !contains(github.event.head_commit.message, '[skip docs]')"
        env:
          SG_DEMO_KEY: ${{ secrets.SG_DEMO_KEY }}
          SG_DEMO_SECRET: ${{ secrets.SG_DEMO_SECRET }}
        run: |
          echo "Building Asciicast/doc bundle"
          . "$(poetry env info --path)/bin/activate"
          export PATH=$PATH:"$PWD"/bin
          export PATH=$PATH:$(readlink -f $DBT_VENV/bin)
          ./.ci/build_docs.sh
          cp ./install.sh dist
          # TODO figure out if we want to do poetry upload here (can only do once, so will fail
          # if we're retrying an upload)
          # "$HOME"/.poetry/bin/poetry build
      - name: "Build and publish wheel"
        if: "startsWith(github.ref, 'refs/tags/') && !contains(github.event.head_commit.message, '[skip wheel]')"
        env:
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        # We could split this out into a separate stage but it would mean installing Poetry, Compose and this package all over again.
        run: |
          ./.ci/build_wheel.sh
          cp ./install.sh dist
          # TODO figure out if we want to do poetry upload here (can only do once, so will fail
          # if we're retrying an upload)
          # "$HOME"/.poetry/bin/poetry build
      - name: "Build wheel only (do not configure publish)"
        # The {windows,linux,osx}_binary stage will run if ref is tag, or msg contains "[artifacts]""
        # If no tag, but [artifacts], we still need to build the wheel, but with NO_PUBLISH=1
        # But if tag _and_ [artifacts], we want to skip this stage, to not build the wheel twice
        if: "!startsWith(github.ref, 'refs/tags/') && contains(github.event.head_commit.message, '[artifacts]')"
        env:
          NO_PUBLISH: "1"
        run: |
          ./.ci/build_wheel.sh
      - name: "Upload release artifacts"
        uses: actions/upload-artifact@v2
        with:
          name: dist
          path: dist

  windows_binary:
    runs-on: windows-latest
    if: "startsWith(github.ref, 'refs/tags/') || contains(github.event.head_commit.message, '[artifacts]')"
    needs: build_and_test
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Get artifacts
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist
      - name: Build the binary
        # default is powershell that doesn't do inline asterisk expansion.
        shell: bash
        run: |
          pip install dist/splitgraph-*-py3-none-any.whl
          pip install pyinstaller==4.10
          pyinstaller -F splitgraph.spec
          dist/sgr.exe --version
      - name: Upload binary as artifact
        uses: actions/upload-artifact@v2
        with:
          name: sgr-windows
          path: dist/sgr.exe

  linux_binary:
    runs-on: ubuntu-18.04
    if: "startsWith(github.ref, 'refs/tags/') || contains(github.event.head_commit.message, '[artifacts]')"
    needs: build_and_test
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Get artifacts
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist
      - name: Build the binary
        run: |
          pip install dist/splitgraph-*-py3-none-any.whl
          pip install pyinstaller==4.10
          pyinstaller -F splitgraph.spec
          dist/sgr --version
      - name: Smoke test the binary
        env:
          SG_DEMO_KEY: ${{ secrets.SG_DEMO_KEY }}
          SG_DEMO_SECRET: ${{ secrets.SG_DEMO_SECRET }}
        run: |
          dist/sgr --verbosity DEBUG engine add --password password \
            --image splitgraph/engine:latest
          dist/sgr status
          dist/sgr cloud login-api --api-key "$SG_DEMO_KEY" --api-secret "$SG_DEMO_SECRET"
          dist/sgr clone --download-all splitgraph/census
          dist/sgr checkout splitgraph/census:latest
      - name: Upload binary as artifact
        uses: actions/upload-artifact@v2
        with:
          name: sgr-linux
          path: dist/sgr

  osx_binary:
    runs-on: macOS-latest
    if: "startsWith(github.ref, 'refs/tags/') || contains(github.event.head_commit.message, '[artifacts]')"
    needs: build_and_test
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Get artifacts
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist
      - name: Build the single-file binary
        run: |
          pip install dist/splitgraph-*-py3-none-any.whl
          pip install pyinstaller==4.10
          pyinstaller -F splitgraph.spec
          dist/sgr --version
      - name: Upload single-file binary as artifact
        uses: actions/upload-artifact@v2
        with:
          name: sgr-osx
          path: dist/sgr
      - name: Build the multi-file binary.gz
        run: |
          pyinstaller --clean --noconfirm --onedir splitgraph.spec
          dist/sgr-pkg/sgr --version
          cd dist/sgr-pkg && tar zcvf ../sgr.tgz .
      - name: Upload multi-file binary.gz as artifact
        uses: actions/upload-artifact@v2
        with:
          name: sgr-osx
          path: dist/sgr.tgz

  upload_release:
    runs-on: ubuntu-18.04
    if: "startsWith(github.ref, 'refs/tags/')"
    needs:
      - windows_binary
      - linux_binary
      - osx_binary
    steps:
      - name: Get artifacts
        uses: actions/download-artifact@v2
        with:
          path: artifacts
      - name: Rename artifacts
        run: |
          mv artifacts/sgr-windows/sgr.exe artifacts/sgr-windows-x86_64.exe
          mv artifacts/sgr-linux/sgr artifacts/sgr-linux-x86_64
          mv artifacts/sgr-osx/sgr artifacts/sgr-osx-x86_64
          mv artifacts/sgr-osx/sgr.tgz artifacts/sgr-osx-x86_64.tgz
      - name: Release artifacts
        uses: softprops/action-gh-release@v1
        with:
          files: |
            artifacts/sgr-windows-x86_64.exe
            artifacts/sgr-linux-x86_64
            artifacts/sgr-osx-x86_64
            artifacts/sgr-osx-x86_64.tgz
            artifacts/dist/sgr-docs-bin.tar.gz
            artifacts/dist/install.sh
          draft: true
          fail_on_unmatched_files: false