ci: migrate to shared Docker image for integration tests

Author: radugheo

.github/workflows/integration_tests.yml

@@ -6,31 +6,76 @@ on:
   pull_request:
     branches: [ main ]
 
+permissions:
+  contents: read
+  pull-requests: write
+  actions: read
+
 jobs:
-  integration-tests:
+  discover-testcases:
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: write
+    outputs:
+      testcases: ${{ steps.discover.outputs.testcases }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Discover testcases
+      id: discover
+      run: |
+        # Find all testcase folders (excluding common folders like README, etc.)
+        testcase_dirs=$(find testcases -maxdepth 1 -type d -name "*-*" | sed 's|testcases/||' | sort)
+
+        echo "Found testcase directories:"
+        echo "$testcase_dirs"
 
+        # Convert to JSON array for matrix
+        testcases_json=$(echo "$testcase_dirs" | jq -R -s -c 'split("\n")[:-1]')
+        echo "testcases=$testcases_json" >> $GITHUB_OUTPUT
+
+  integration-tests:
+    needs: [discover-testcases]
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        include:
-          - build-dir: quickstart-agent
-          - build-dir: simple-hitl-agent
-        
+        testcase: ${{ fromJson(needs.discover-testcases.outputs.testcases) }}
+        environment: [alpha, cloud]
+        use_azure_chat: [true, false]
+
+    name: "${{ matrix.testcase }} / ${{ matrix.environment }} / ${{ matrix.use_azure_chat && 'UiPathAzureChatOpenAI' || 'UiPathChat' }}"
+
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
 
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Build Docker image (${{ matrix.build-dir }})
+    - name: Build Docker image
+      run: |
+        echo "Building Docker image for ${{ matrix.testcase }} at $(date)"
+        docker build -f testcases/Dockerfile \
+          -t uipath-llamaindex-testbase:latest \
+          .
+        echo "Docker image built at $(date)"
+
+    - name: Run testcase
+      env:
+        CLIENT_ID:      ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_ID || secrets.CLOUD_TEST_CLIENT_ID }}
+        CLIENT_SECRET:  ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_SECRET || secrets.CLOUD_TEST_CLIENT_SECRET }}
+        BASE_URL:       ${{ matrix.environment == 'alpha' && secrets.ALPHA_BASE_URL || secrets.CLOUD_BASE_URL }}
+        USE_AZURE_CHAT: ${{ matrix.use_azure_chat }}
       run: |
-        docker build -f testcases/${{ matrix.build-dir }}/Dockerfile \
-          -t ${{ matrix.build-dir }}:test \
-          --build-arg CLIENT_ID="${{ secrets.ALPHA_TEST_CLIENT_ID }}" \
-          --build-arg CLIENT_SECRET="${{ secrets.ALPHA_TEST_CLIENT_SECRET }}" \
-          --build-arg BASE_URL="${{ secrets.ALPHA_BASE_URL }}" \
-          .
+        echo "Running testcase: ${{ matrix.testcase }}"
+        echo "Environment: ${{ matrix.environment }}"
+        echo "LLM: ${{ matrix.use_azure_chat && 'UiPathAzureChatOpenAI' || 'UiPathChat' }}"
+        echo "USE_AZURE_CHAT: ${{ matrix.use_azure_chat }}"
+
+        docker run --rm \
+          -e CLIENT_ID="$CLIENT_ID" \
+          -e CLIENT_SECRET="$CLIENT_SECRET" \
+          -e BASE_URL="$BASE_URL" \
+          -e USE_AZURE_CHAT="$USE_AZURE_CHAT" \
+          uipath-llamaindex-testbase:latest \
+          bash /app/testcases/${{ matrix.testcase }}/run.sh

testcases/Dockerfile

@@ -0,0 +1,15 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm
+
+WORKDIR /app
+
+COPY . .
+
+# Install dependencies for all testcases
+RUN uv sync
+
+
+# Set environment variables for runtime
+ENV UIPATH_TENANT_ID=${UIPATH_TENANT_ID:-""}
+ENV UIPATH_JOB_KEY=8c6a342e-036e-492c-a3d2-99e66f6554ce
+
+CMD ["bash"]

testcases/quickstart-agent/Dockerfile

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+cd /app/testcases/quickstart-agent
+
+# Sync dependencies for this specific testcase
+echo "Syncing dependencies..."
+uv sync
+
+# Authenticate with UiPath
+echo "Authenticating with UiPath..."
+uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL"
+
+# Pack the agent
+echo "Packing agent..."
+uv run uipath pack
+
+# Run the agent
+echo "Running agent..."
+echo "Input from input.json file"
+uv run uipath run agent --file input.json
+
+# Validate output
+echo "Validating output..."
+python src/assert.py || { echo "Validation failed!"; exit 1; }
+
+echo "Testcase completed successfully."

testcases/quickstart-agent/run.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+cd /app/testcases/simple-hitl-agent
+
+# Sync dependencies for this specific testcase
+echo "Syncing dependencies..."
+uv sync
+
+# Authenticate with UiPath
+echo "Authenticating with UiPath..."
+uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL"
+
+# Pack the agent
+echo "Packing agent..."
+uv run uipath pack
+
+# Run the agent
+echo "Running agent..."
+echo "Input from input.json file"
+uv run uipath run agent --file input.json
+
+echo "Resuming agent run with human response..."
+uv run uipath run agent --file human_response.json --resume
+
+# Validate output
+echo "Validating output..."
+python src/assert.py || { echo "Validation failed!"; exit 1; }
+
+echo "Testcase completed successfully."