build.yml

name: Build and test DataSQRL Examples

run-name: examples build [${{ inputs.correlation_id || github.run_number }}]

on:
  push:
    branches: [ "main" ]
  pull_request:
  workflow_dispatch:
    inputs:
      sqrl_image:
        description: "Full SQRL CLI image reference (e.g. ghcr.io/datasqrl/cmd:pr-123 or datasqrl/cmd:dev). When set, overrides SQRL_VERSION and skips Docker Hub publishing."
        required: false
        default: ''
      correlation_id:
        description: "Optional correlation ID used by upstream dispatchers to locate this run."
        required: false
        default: ''

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.correlation_id || github.run_id }}
  cancel-in-progress: true

env:
  TZ: 'America/Los_Angeles'
  SQRL_VERSION: '0.10.8'

jobs:
  build-and-push:
    runs-on: ubuntu-latest-4-cores
    timeout-minutes: 30
    permissions:
      id-token: write
      contents: read

    strategy:
      fail-fast: false
      matrix:
        include:
          - example: recommendation
            path: clickstream-ai-recommendation
            test_commands: |
              compile recommendation_package_api.json
              compile recommendation_package_kafka.json

          - example: credit-card-analytics
            path: finance-credit-card-chatbot
            tag: credit-card-analytics
            test_commands: |
              compile credit-card-analytics/creditcard_analytics_package_test.json
              compile credit-card-analytics/creditcard_analytics_package_kafka.json
              test creditcard_analytics_package_test.json creditcard_analytics_package_jwt.json -r credit-card-analytics
          - example: credit-card-rewards
            path: finance-credit-card-chatbot
            tag: credit-card-rewards
            test_commands: |
              compile credit-card-rewards/creditcard_rewards_package_test.json
              compile credit-card-rewards/creditcard_rewards_package_kafka.json
              test creditcard_rewards_package_test.json creditcard_rewards_package_jwt.json -r credit-card-rewards
          - example: credit-card-views
            path: finance-credit-card-chatbot
            tag: credit-card-views
            test_commands: |
              compile credit-card-views/creditcard_views_package_test.json
              test credit-card-views/creditcard_views_package_test.json

          - example: healthcare-study-analytics
            path: healthcare-study
            tag: healthcare-study-analytics
            test_commands: |
              compile healthcare-study-analytics/study_analytics_package_test.json
              test healthcare-study-analytics/study_analytics_package_test.json
          - example: healthcare-study-api
            path: healthcare-study
            tag: healthcare-study-api
            test_commands: |
              compile healthcare-study-api/study_api_package_api.json
              compile healthcare-study-api/study_api_package_test.json
              test healthcare-study-api/study_api_package_test.json
          - example: healthcare-study-stream
            path: healthcare-study
            tag: healthcare-study-stream
            test_commands: |
              compile healthcare-study-stream/study_stream_package_kafka.json
              compile healthcare-study-stream/study_stream_package_test.json
              test healthcare-study-stream/study_stream_package_test.json

          - example: iceberg-dedup
            path: iceberg-data-deduplication
            test_commands: |
              compile datagen_package_local.json
              compile data_compaction_package_local.json
              compile data_delete_package_local.json

          - example: iot-sensor
            path: iot-sensor-metrics
            test_commands: |
              compile sensors_package_api.json
              compile sensors_package_test.json
              test sensors_package_test.json
              test sensors_package_api.json

          - example: law-enforcement
            path: law-enforcement
            test_commands: |
              compile baseball_card_package_test.json
              test baseball_card_package_test.json

          - example: logistics
            path: logistics-shipping-geodata
            test_commands: |
              compile logistics_package_test.json
              test logistics_package_test.json

          - example: oil-gas-monitoring
            path: oil-gas-agent-automation
            tag: oil-gas-agent-monitoring
            test_commands: |
              compile oil-gas-agent-monitoring/monitoring_agent_package_test.json
          - example: oil-gas-operations
            path: oil-gas-agent-automation
            tag: oil-gas-agent-operations
            test_commands: |
              compile oil-gas-agent-operations/operations_agent_package_test.json
              test oil-gas-agent-operations/operations_agent_package_test.json

          - example: usage-analytics
            path: usage-analytics
            test_commands: |
              compile usertokens_package_test.json
              test usertokens_package_test.json

          - example: canary-aggregation
            path: test-jobs/aggregation-tests
            tag: canary
            test_commands: |
              compile aggregation-test-package.json
              compile aggregation-query-test-package.json
              compile aggregation-query-test-iceberg-package.json
              compile aggregation-query-subscription-test-package.json

          - example: canary-mutation
            path: test-jobs/mutation-tests
            tag: canary
            test_commands: |
              compile mutation-test-package.json

          - example: canary-failures
            path: test-jobs/failure-tests
            tag: canary
            test_commands: |
              compile runtime-failing-connector-package.json
              compile runtime-failing-udf-package.json

          - example: canary-batch
            path: test-jobs/batch-test
            tag: canary
            test_commands: |
              compile simple-batch-test-package.json

          - example: canary-scheduled-batch
            path: test-jobs/scheduled-batch-test
            tag: canary
            test_commands: |
              compile scheduled-batch-package.json
              compile simple-scheduled-batch-package.json

          - example: udf-jbang
            path: user-defined-function/jbang
            tag: udf-jbang
            test_commands: |
              compile udf_jbang_package.json

          - example: udf-maven
            path: user-defined-function/maven-project
            tag: udf-maven
            test_commands: |
              compile udf_maven_package.json

    steps:
    - uses: actions/checkout@v4

    - name: Resolve SQRL image
      id: resolve-image
      run: |
        if [ -n "${{ inputs.sqrl_image }}" ]; then
          IMAGE="${{ inputs.sqrl_image }}"
          IS_SNAPSHOT=true
        else
          IMAGE="datasqrl/cmd:${SQRL_VERSION}"
          IS_SNAPSHOT=false
        fi
        echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
        echo "is_snapshot=$IS_SNAPSHOT" >> "$GITHUB_OUTPUT"
        echo "Using SQRL image: $IMAGE (snapshot=$IS_SNAPSHOT)"

    - name: Setup Timezone
      uses: szenius/set-timezone@v1.1
      with:
        timezoneLinux: "America/Los_Angeles"

    - name: Cache Docker images
      uses: ScribeMD/docker-cache@0.5.0
      with:
        # one cache entry per SQRL image ref & OS
        key: sqrl-${{ runner.os }}-${{ steps.resolve-image.outputs.image }}

    - name: Pull SQRL Docker image
      run: docker pull "${{ steps.resolve-image.outputs.image }}"

    - name: Configure AWS credentials via OIDC
      # Tolerated: PRs from forks cannot mint an OIDC token for this repo's role.
      # Such PRs fall back to running without AWS credentials, which is fine for
      # every matrix entry except the S3-backed iceberg canary.
      continue-on-error: true
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ secrets.AWS_DEV_GITHUB_ACTION_ROLE }}
        aws-region: us-east-1

    - name: Run ${{ matrix.example }} Tests
      working-directory: ${{ matrix.path }}
      env:
        SQRL_IMAGE: ${{ steps.resolve-image.outputs.image }}
      run: |
        # Read raw lines from the matrix value
        mapfile -t raw_cmds <<< "${{ matrix.test_commands }}"

        # Keep only non‑empty lines
        cmds=()
        for c in "${raw_cmds[@]}"; do
          [[ -z "${c//[[:space:]]/}" ]] && continue   # skip blanks
          cmds+=("$c")
        done

        if (( ${#cmds[@]} == 0 )); then
          echo "ERROR: No test commands specified for ${{ matrix.example }}"
          exit 1
        fi

        for cmd in "${cmds[@]}"; do
          echo "::group::Running: $cmd"
          docker run -i -p 8888:8888 -p 8081:8081 -p 9092:9092 \
            -e AWS_ACCESS_KEY_ID \
            -e AWS_SECRET_ACCESS_KEY \
            -e AWS_SESSION_TOKEN \
            -e AWS_REGION \
            -e AWS_DEFAULT_REGION \
            --rm -v "$PWD":/workspace "$SQRL_IMAGE" $cmd
          echo "::endgroup::"
        done

    - name: Write Dockerfile
      if: steps.resolve-image.outputs.is_snapshot != 'true'
      env:
        SQRL_IMAGE: ${{ steps.resolve-image.outputs.image }}
      run: |
        cat <<EOF > ${{ matrix.path }}/Dockerfile
        FROM ${SQRL_IMAGE}

        COPY . /workspace
        WORKDIR /workspace
        EOF

    - name: Set up QEMU
      if: steps.resolve-image.outputs.is_snapshot != 'true'
      uses: docker/setup-qemu-action@v2

    - name: Set up Docker Buildx
      if: steps.resolve-image.outputs.is_snapshot != 'true'
      uses: docker/setup-buildx-action@v2

    - name: Login to Docker Hub
      if: steps.resolve-image.outputs.is_snapshot != 'true'
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKER_USERNAME }}
        password: ${{ secrets.DOCKER_PASSWORD }}

    - name: Publish ${{ matrix.example }}
      if: steps.resolve-image.outputs.is_snapshot != 'true'
      uses: docker/build-push-action@v3
      with:
        context: ./${{ matrix.path }}
        file: ./${{ matrix.path }}/Dockerfile
        push: true
        tags: datasqrl/examples:${{ matrix.tag && matrix.tag != '' && matrix.tag || matrix.path }}
        platforms: linux/amd64,linux/arm64

  lint-graphql:
    name: Lint GraphQL Schemas
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Lint GraphQL schemas
        run: |
          SCHEMAS=$(find . -name "*.graphqls" -type f)
          if [ -z "$SCHEMAS" ]; then
            echo "No GraphQL schemas found"
            exit 0
          fi

          LINT_OPTS="--rules fields-are-camel-cased,types-are-capitalized,enum-values-all-caps,input-object-values-are-camel-cased"
          IGNORE='{"fields-are-camel-cased":["Query.Shipment","Query.Vehicle","Query.Customer"]}'
          FAILED=0

          for schema in $SCHEMAS; do
            echo "::group::Checking $schema"
            output=$(docker run --rm -v "$PWD":/app -w /app node:18-slim npx graphql-schema-linter $LINT_OPTS --ignore "$IGNORE" "$schema" 2>&1)
            result=$?
            echo "$output" | grep -v "^npm"
            echo "::endgroup::"
            if [ $result -ne 0 ]; then
              FAILED=1
            fi
          done

          exit $FAILED

  build:
    needs: [build-and-push, lint-graphql]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Fail if any build job failed
        run: |
          failed=$(jq -r '.[] | select(.result=="failure")' <<< '${{ toJson(needs) }}' | wc -l)
          if [[ "$failed" -gt 0 ]]; then
            echo "::error ::${failed} matrix job(s) failed"
            exit 1
          fi