pro-integration.yml

name: integration-tests-against-pro
on:
  workflow_dispatch:
    inputs:
      targetRef:
        description: 'LocalStack Pro Ref to test with'
        required: false
  pull_request:
    paths:
      - ".github/workflows/pro-integration.yml"
      - "localstack/**"
      - "tests/**"
      - "setup.py"
      - "pyproject.toml"
      - "setup.cfg"
      - "Dockerfile"
      - "Dockerfile.rh"
      - "docker-compose.yml"
      - "bin/**"
    branches:
      - master
      - 'v[0-9]+'
  push:
    paths:
      - ".github/workflows/pro-integration.yml"
      - "localstack/**"
      - "tests/**"
      - "setup.py"
      - "pyproject.toml"
      - "setup.cfg"
      - "Dockerfile"
      - "Dockerfile.rh"
      - "docker-compose.yml"
      - "bin/**"
    branches:
      - master

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  run-integration-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 100
    defaults:
      run:
        working-directory: localstack-ext
    environment: localstack-ext-tests
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository  # skip job if fork PR
    steps:
      - name: Determine companion-ref
        id: determine-companion-ref
        uses: actions/github-script@v6
        with:
          github-token: ${{ secrets.PRO_ACCESS_TOKEN }}
          result-encoding: string
          script: |
            if (context.payload.inputs && context.payload.inputs.targetRef) {
              console.log("Using manually set target reference: ", context.payload.inputs.targetRef)
              return context.payload.inputs.targetRef
            }

            const DEFAULT_REF = "refs/heads/master"

            async function isCompanionRefExisting(refName) {
              try {
                // strip the leading "refs/" for the API call
                const apiRef = refName.substr(5)
                console.log("Checking if companion repo has ref: ", apiRef)
                await github.rest.git.getRef({owner: "localstack", repo: "localstack-ext", ref: apiRef})
                return true
              } catch (error) {
                  if (error.status == 404) {
                    return false
                  } else {
                    // another (unexpected) error ocurred, raise the error
                    throw new Error(`Fetching companion refs failed: ${JSON.stringify(error)}`)
                  }
              }
            }

            let ref = context.ref
            let baseRef = null
            if (context.payload.pull_request) {
              // pull requests have their own refs (f.e. 'refs/pull/1/merge')
              // use the PR head ref instead
              ref = `refs/heads/${context.payload.pull_request.head.ref}`
              baseRef = `refs/heads/${context.payload.pull_request.base.ref}`
            }

            if (ref == DEFAULT_REF) {
              console.log("Current ref is default ref. Using the same for ext repo: ", DEFAULT_REF)
              return DEFAULT_REF
            }

            if (await isCompanionRefExisting(ref)) {
              console.log("Using companion ref in ext repo: ", ref)
              return ref
            } else if (baseRef && baseRef != DEFAULT_REF && (await isCompanionRefExisting(baseRef))) {
              console.log("Using PR base companion ref in ext repo: ", baseRef)
              return baseRef
            }

            // the companion repo does not have a companion ref, use the default
            console.log("Ext repo does not have a companion ref. Using default: ", DEFAULT_REF)
            return DEFAULT_REF
      - name: Checkout Pro
        uses: actions/checkout@v2
        with:
          repository: localstack/localstack-ext
          ref: ${{steps.determine-companion-ref.outputs.result}}
          token: ${{ secrets.PRO_ACCESS_TOKEN }}
          path: localstack-ext
      - name: Checkout Open Source
        uses: actions/checkout@v2
        with:
          path: localstack
      - name: Set up Python 3.10
        id: setup-python
        uses: actions/setup-python@v2
        with:
          python-version: '3.10'
      - name: Set up Node 14.x
        uses: actions/setup-node@v2
        with:
          node-version: 14.x
      - name: Set up JDK 11
        uses: actions/setup-java@v2
        with:
          java-version: '11'
          distribution: 'temurin'
      - uses: hashicorp/setup-terraform@v1
        with:
          terraform_version: 0.13.7
      - name: Set up system wide dependencies
        run: |
          sudo apt-get install libsasl2-dev jq
      - name: Cache LocalStack-ext dependencies (venv)
        uses: actions/cache@v2
        id: ext-cache
        with:
          path: localstack-ext/.venv
          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }}
      - name: Install Python Dependencies for LocalStack Pro
        run: make install
      - name: Cache LocalStack community dependencies (venv, static libs)
        uses: actions/cache@v2
        id: os-cache
        with:
          path: |
            localstack/.venv
            localstack/.filesystem/usr/lib/localstack
          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }}
      - name: Install Dependencies for LocalStack Community # lambda tests look for libraries in this virtualenv
        working-directory: localstack
        run: |
          make install
          make init
      - name: Link community LocalStack into Pro venv
        run: |
          source .venv/bin/activate
          pip install -e ../localstack[runtime,test]
      - name: Create Pro Entrypoints
        # Entrypoints need to be generated _after_ the community edition has been linked into the venv
        run: make entrypoints
      - name: Test LocalStack Pro startup
        env:
          DEBUG: 1
          DNS_ADDRESS: 0
          LOCALSTACK_API_KEY: "test"
        run: |
          source .venv/bin/activate
          bin/test_localstack_pro.sh
      - name: Run community integration tests
        env:
          DEBUG: 1
          DNS_ADDRESS: 0
          LAMBDA_EXECUTOR: "local"
          LOCALSTACK_API_KEY: "test"
          AWS_SECRET_ACCESS_KEY: "test"
          AWS_ACCESS_KEY_ID: "test"
          AWS_DEFAULT_REGION: "us-east-1"
          HOST_TMP_FOLDER: /tmp/localstack
          PYTEST_LOGLEVEL: debug
        run: |
          # Remove the host tmp folder (might contain remnant files with different permissions)
          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
          source .venv/bin/activate
          python -m pytest --capture=no --reruns 2 --durations=10 --junitxml=target/reports/pytest.xml ../localstack/tests/integration/
      - name: Run Lambda Tests for lambda executor docker
        env:
          DEBUG: 0
          DNS_ADDRESS: 0
          LAMBDA_EXECUTOR: "docker"
          LOCALSTACK_API_KEY: "test"
          HOST_TMP_FOLDER: /tmp/localstack
        run: |
          # Remove the host tmp folder (might contain remnant files with different permissions)
          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
          source .venv/bin/activate
          python -m pytest --reruns 2 --durations=10 --show-capture=no --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='lambda-docker' \
            ../localstack/tests/integration/awslambda/ \
            ../localstack/tests/integration/test_integration.py \
            ../localstack/tests/integration/test_apigateway.py
      - name: Run Lambda Tests for lambda executor docker-reuse
        env:
          DEBUG: 1
          DNS_ADDRESS: 0
          LAMBDA_EXECUTOR: "docker-reuse"
          LOCALSTACK_API_KEY: "test"
          HOST_TMP_FOLDER: /tmp/localstack
        run: |
          # Remove the host tmp folder (might contain remnant files with different permissions)
          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
          source .venv/bin/activate
          python -m pytest --reruns 2 --durations=10 --show-capture=no --junitxml=target/reports/lambda-docker-reuse.xml -o junit_suite_name='lambda-docker-reuse' \
            ../localstack/tests/integration/awslambda/ \
            ../localstack/tests/integration/test_integration.py \
            ../localstack/tests/integration/test_apigateway.py
      - name: Publish LocalStack Community Integration Test Results
        uses: EnricoMi/publish-unit-test-result-action@v1
        if: always()
        with:
          files: localstack-ext/target/reports/*.xml
          check_name: LocalStack integration with Pro