From 2241702db0f60de3e2175b6e5407b21273dc4d84 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:11:39 +0000 Subject: [PATCH 1/9] add language filtering for codeql --- .github/workflows/codeql.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0a7aa09530a8..01b7044650e4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,8 +27,49 @@ on: - cron: '25 10 * * 6' jobs: + detect-changes: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + actions: ${{ steps.filter.outputs.actions }} + go: ${{ steps.filter.outputs.go }} + java-kotlin: ${{ steps.filter.outputs['java-kotlin'] }} + javascript-typescript: ${{ steps.filter.outputs['javascript-typescript'] }} + python: ${{ steps.filter.outputs.python }} + rust: ${{ steps.filter.outputs.rust }} + steps: + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + actions: + - '.github/**' + go: + - 'sdks/go/**' + - 'playground/backend/**' + - 'learning/katas/go/**' + - 'learning/tour-of-beam/backend/**' + java-kotlin: + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + javascript-typescript: + - 'sdks/typescript/**' + python: + - 'sdks/python/**' + rust: + - '**/*.rs' + analyze: name: Analyze (${{ matrix.language }}) + needs: detect-changes + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.detect-changes.outputs[matrix.language] == 'true' # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources From 5d20cac0d42cc5324eed11e9d2044872aa00d974 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:16:20 +0000 Subject: [PATCH 2/9] fix workflow file --- .github/workflows/codeql.yml | 53 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 01b7044650e4..1564809720f4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,12 +33,7 @@ jobs: contents: read pull-requests: read outputs: - actions: ${{ steps.filter.outputs.actions }} - go: ${{ steps.filter.outputs.go }} - java-kotlin: ${{ steps.filter.outputs['java-kotlin'] }} - javascript-typescript: ${{ steps.filter.outputs['javascript-typescript'] }} - python: ${{ steps.filter.outputs.python }} - rust: ${{ steps.filter.outputs.rust }} + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: dorny/paths-filter@v3 id: filter @@ -62,14 +57,36 @@ jobs: - 'sdks/python/**' rust: - '**/*.rs' + - name: Generate Matrix + id: set-matrix + run: | + ALL_MATRIX='[ + {"language": "actions", "build-mode": "none"}, + {"language": "go", "build-mode": "manual"}, + {"language": "java-kotlin", "build-mode": "autobuild"}, + {"language": "javascript-typescript", "build-mode": "none"}, + {"language": "python", "build-mode": "none"}, + {"language": "rust", "build-mode": "none"} + ]' + + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT + else + FILTERED=$(echo "$ALL_MATRIX" | jq -c '[.[] | select( + (.language == "actions" and "${{ steps.filter.outputs.actions }}" == "true") or + (.language == "go" and "${{ steps.filter.outputs.go }}" == "true") or + (.language == "java-kotlin" and "${{ steps.filter.outputs['java-kotlin'] }}" == "true") or + (.language == "javascript-typescript" and "${{ steps.filter.outputs['javascript-typescript'] }}" == "true") or + (.language == "python" and "${{ steps.filter.outputs.python }}" == "true") or + (.language == "rust" and "${{ steps.filter.outputs.rust }}" == "true") + )]') + echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + fi analyze: name: Analyze (${{ matrix.language }}) needs: detect-changes - if: | - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - needs.detect-changes.outputs[matrix.language] == 'true' + if: needs.detect-changes.outputs.matrix != '[]' # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources @@ -90,21 +107,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - language: actions - build-mode: none - # - language: c-cpp - # build-mode: autobuild - - language: go - build-mode: manual - - language: java-kotlin - build-mode: autobuild - - language: javascript-typescript - build-mode: none - - language: python - build-mode: none - - language: rust - build-mode: none + include: ${{ fromJSON(needs.detect-changes.outputs.matrix) }} # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both From 9e21adf27b508618f63ce89598788e3758723841 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:19:32 +0000 Subject: [PATCH 3/9] remove third party dorny action --- .github/workflows/codeql.yml | 81 +++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1564809720f4..a029e67f10d3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,28 +35,11 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: dorny/paths-filter@v3 - id: filter + - name: Checkout repository history + uses: actions/checkout@v4 with: - filters: | - actions: - - '.github/**' - go: - - 'sdks/go/**' - - 'playground/backend/**' - - 'learning/katas/go/**' - - 'learning/tour-of-beam/backend/**' - java-kotlin: - - 'sdks/java/**' - - 'runners/**' - - 'examples/java/**' - - 'examples/kotlin/**' - javascript-typescript: - - 'sdks/typescript/**' - python: - - 'sdks/python/**' - rust: - - '**/*.rs' + fetch-depth: 0 + - name: Generate Matrix id: set-matrix run: | @@ -71,18 +54,56 @@ jobs: if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT + exit 0 + fi + + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" else - FILTERED=$(echo "$ALL_MATRIX" | jq -c '[.[] | select( - (.language == "actions" and "${{ steps.filter.outputs.actions }}" == "true") or - (.language == "go" and "${{ steps.filter.outputs.go }}" == "true") or - (.language == "java-kotlin" and "${{ steps.filter.outputs['java-kotlin'] }}" == "true") or - (.language == "javascript-typescript" and "${{ steps.filter.outputs['javascript-typescript'] }}" == "true") or - (.language == "python" and "${{ steps.filter.outputs.python }}" == "true") or - (.language == "rust" and "${{ steps.filter.outputs.rust }}" == "true") - )]') - echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.event.after }}" + fi + + if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") + else + CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "") fi + check_path() { + local pattern="$1" + if echo "$CHANGED_FILES" | grep -q -E "$pattern"; then + echo "true" + else + echo "false" + fi + } + + ACTIONS_CHANGED=$(check_path "^\.github/") + GO_CHANGED=$(check_path "^sdks/go/|^playground/backend/|^learning/katas/go/|^learning/tour-of-beam/backend/") + JAVA_CHANGED=$(check_path "^sdks/java/|^runners/|^examples/java/|^examples/kotlin/|\.java$") + JS_CHANGED=$(check_path "^sdks/typescript/|\.js$|\.ts$") + PYTHON_CHANGED=$(check_path "^sdks/python/|\.py$") + RUST_CHANGED=$(check_path "\.rs$") + + FILTERED=$(echo "$ALL_MATRIX" | jq -c --arg actions "$ACTIONS_CHANGED" \ + --arg go "$GO_CHANGED" \ + --arg java "$JAVA_CHANGED" \ + --arg js "$JS_CHANGED" \ + --arg python "$PYTHON_CHANGED" \ + --arg rust "$RUST_CHANGED" \ + '[.[] | select( + (.language == "actions" and $actions == "true") or + (.language == "go" and $go == "true") or + (.language == "java-kotlin" and $java == "true") or + (.language == "javascript-typescript" and $js == "true") or + (.language == "python" and $python == "true") or + (.language == "rust" and $rust == "true") + )]') + + echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + analyze: name: Analyze (${{ matrix.language }}) needs: detect-changes From eaef601a22ecbd62a653540ec25af80034c18f0a Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:25:33 +0000 Subject: [PATCH 4/9] add comments --- .github/workflows/codeql.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a029e67f10d3..7922f7b2e857 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -43,6 +43,7 @@ jobs: - name: Generate Matrix id: set-matrix run: | + # Full matrix of all CodeQL supported languages and their build modes in Apache Beam. ALL_MATRIX='[ {"language": "actions", "build-mode": "none"}, {"language": "go", "build-mode": "manual"}, @@ -52,11 +53,13 @@ jobs: {"language": "rust", "build-mode": "none"} ]' + # On periodic scheduled runs (cron) or manual dispatches, scan all languages. if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT exit 0 fi + # Determine the commit range to compare for changed files. if [ "${{ github.event_name }}" = "pull_request" ]; then BASE_SHA="${{ github.event.pull_request.base.sha }}" HEAD_SHA="${{ github.event.pull_request.head.sha }}" @@ -65,12 +68,14 @@ jobs: HEAD_SHA="${{ github.event.after }}" fi + # Fall back to comparing against HEAD~1 if base SHA is empty or zeroed. if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") else CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "") fi + # Helper function to check if any changed file matches a given regex pattern. check_path() { local pattern="$1" if echo "$CHANGED_FILES" | grep -q -E "$pattern"; then @@ -80,6 +85,7 @@ jobs: fi } + # Check path changes for each language/module context. ACTIONS_CHANGED=$(check_path "^\.github/") GO_CHANGED=$(check_path "^sdks/go/|^playground/backend/|^learning/katas/go/|^learning/tour-of-beam/backend/") JAVA_CHANGED=$(check_path "^sdks/java/|^runners/|^examples/java/|^examples/kotlin/|\.java$") @@ -87,6 +93,7 @@ jobs: PYTHON_CHANGED=$(check_path "^sdks/python/|\.py$") RUST_CHANGED=$(check_path "\.rs$") + # Filter ALL_MATRIX entries using jq, keeping only entries for languages with path changes. FILTERED=$(echo "$ALL_MATRIX" | jq -c --arg actions "$ACTIONS_CHANGED" \ --arg go "$GO_CHANGED" \ --arg java "$JAVA_CHANGED" \ From 8fc4719bc54d7208d999e6af83b0fd17199e2497 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 17 Jul 2026 16:09:49 +0000 Subject: [PATCH 5/9] use approved SHA of dorny/paths-filter to dynamically filter matrix --- .github/workflows/codeql.yml | 86 +++++++++++++----------------------- 1 file changed, 31 insertions(+), 55 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7922f7b2e857..663e65af46cf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,10 +35,28 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - name: Checkout repository history - uses: actions/checkout@v4 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d + id: filter with: - fetch-depth: 0 + filters: | + actions: + - '.github/**' + go: + - 'sdks/go/**' + - 'playground/backend/**' + - 'learning/katas/go/**' + - 'learning/tour-of-beam/backend/**' + java-kotlin: + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + javascript-typescript: + - 'sdks/typescript/**' + python: + - 'sdks/python/**' + rust: + - '**/*.rs' - name: Generate Matrix id: set-matrix @@ -56,60 +74,18 @@ jobs: # On periodic scheduled runs (cron) or manual dispatches, scan all languages. if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT - exit 0 - fi - - # Determine the commit range to compare for changed files. - if [ "${{ github.event_name }}" = "pull_request" ]; then - BASE_SHA="${{ github.event.pull_request.base.sha }}" - HEAD_SHA="${{ github.event.pull_request.head.sha }}" - else - BASE_SHA="${{ github.event.before }}" - HEAD_SHA="${{ github.event.after }}" - fi - - # Fall back to comparing against HEAD~1 if base SHA is empty or zeroed. - if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then - CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") else - CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "") - fi - - # Helper function to check if any changed file matches a given regex pattern. - check_path() { - local pattern="$1" - if echo "$CHANGED_FILES" | grep -q -E "$pattern"; then - echo "true" - else - echo "false" - fi - } - - # Check path changes for each language/module context. - ACTIONS_CHANGED=$(check_path "^\.github/") - GO_CHANGED=$(check_path "^sdks/go/|^playground/backend/|^learning/katas/go/|^learning/tour-of-beam/backend/") - JAVA_CHANGED=$(check_path "^sdks/java/|^runners/|^examples/java/|^examples/kotlin/|\.java$") - JS_CHANGED=$(check_path "^sdks/typescript/|\.js$|\.ts$") - PYTHON_CHANGED=$(check_path "^sdks/python/|\.py$") - RUST_CHANGED=$(check_path "\.rs$") - - # Filter ALL_MATRIX entries using jq, keeping only entries for languages with path changes. - FILTERED=$(echo "$ALL_MATRIX" | jq -c --arg actions "$ACTIONS_CHANGED" \ - --arg go "$GO_CHANGED" \ - --arg java "$JAVA_CHANGED" \ - --arg js "$JS_CHANGED" \ - --arg python "$PYTHON_CHANGED" \ - --arg rust "$RUST_CHANGED" \ - '[.[] | select( - (.language == "actions" and $actions == "true") or - (.language == "go" and $go == "true") or - (.language == "java-kotlin" and $java == "true") or - (.language == "javascript-typescript" and $js == "true") or - (.language == "python" and $python == "true") or - (.language == "rust" and $rust == "true") + # Filter ALL_MATRIX entries keeping only entries for languages with path changes. + FILTERED=$(echo "$ALL_MATRIX" | jq -c '[.[] | select( + (.language == "actions" and "${{ steps.filter.outputs.actions }}" == "true") or + (.language == "go" and "${{ steps.filter.outputs.go }}" == "true") or + (.language == "java-kotlin" and "${{ steps.filter.outputs['java-kotlin'] }}" == "true") or + (.language == "javascript-typescript" and "${{ steps.filter.outputs['javascript-typescript'] }}" == "true") or + (.language == "python" and "${{ steps.filter.outputs.python }}" == "true") or + (.language == "rust" and "${{ steps.filter.outputs.rust }}" == "true") )]') - - echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + fi analyze: name: Analyze (${{ matrix.language }}) From 1fcdbfe847d064c4ac1ac83c74374186a796fa72 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 17 Jul 2026 16:17:25 +0000 Subject: [PATCH 6/9] add checkout step before paths-filter --- .github/workflows/codeql.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 663e65af46cf..c97f8e1d6283 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,6 +35,8 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: + - name: Checkout repository + uses: actions/checkout@v4 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d id: filter with: From eb5eb1e4df164dbcf5e171eaeb12be610bc6d232 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 17 Jul 2026 16:21:28 +0000 Subject: [PATCH 7/9] compact ALL_MATRIX JSON to a single line before writing to GITHUB_OUTPUT --- .github/workflows/codeql.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c97f8e1d6283..c1a420482d48 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -75,7 +75,8 @@ jobs: # On periodic scheduled runs (cron) or manual dispatches, scan all languages. if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT + COMPACT_ALL=$(echo "$ALL_MATRIX" | jq -c .) + echo "matrix=$COMPACT_ALL" >> $GITHUB_OUTPUT else # Filter ALL_MATRIX entries keeping only entries for languages with path changes. FILTERED=$(echo "$ALL_MATRIX" | jq -c '[.[] | select( From 61af1793a97caa1c5d3006dc34eed7a75d638005 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 17 Jul 2026 17:27:49 +0000 Subject: [PATCH 8/9] update path filters to use file extensions and build configuration files --- .github/workflows/codeql.yml | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c1a420482d48..eecc8fe26869 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -44,21 +44,35 @@ jobs: actions: - '.github/**' go: - - 'sdks/go/**' - - 'playground/backend/**' - - 'learning/katas/go/**' - - 'learning/tour-of-beam/backend/**' + - '**/*.go' + - '**/go.mod' + - '**/go.sum' java-kotlin: - - 'sdks/java/**' - - 'runners/**' - - 'examples/java/**' - - 'examples/kotlin/**' + - '**/*.java' + - '**/*.kt' + - '**/*.gradle' + - '**/gradle.properties' + - '**/gradlew' + - '**/gradlew.bat' javascript-typescript: - - 'sdks/typescript/**' + - '**/*.js' + - '**/*.ts' + - '**/*.tsx' + - '**/*.jsx' + - '**/package.json' + - '**/package-lock.json' + - '**/tsconfig.json' python: - - 'sdks/python/**' + - '**/*.py' + - '**/*.pyx' + - '**/setup.py' + - '**/setup.cfg' + - '**/pyproject.toml' + - '**/requirements.txt' rust: - '**/*.rs' + - '**/Cargo.toml' + - '**/Cargo.lock' - name: Generate Matrix id: set-matrix From 5d5a8acfec13b01499b77527e18db60ef9aaede8 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 17 Jul 2026 17:38:39 +0000 Subject: [PATCH 9/9] remove uncessary checks --- .github/workflows/codeql.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index eecc8fe26869..25962dac3b99 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -57,8 +57,6 @@ jobs: javascript-typescript: - '**/*.js' - '**/*.ts' - - '**/*.tsx' - - '**/*.jsx' - '**/package.json' - '**/package-lock.json' - '**/tsconfig.json' @@ -68,11 +66,9 @@ jobs: - '**/setup.py' - '**/setup.cfg' - '**/pyproject.toml' - - '**/requirements.txt' + - '**/*requirements.txt' rust: - '**/*.rs' - - '**/Cargo.toml' - - '**/Cargo.lock' - name: Generate Matrix id: set-matrix