Merge branch 'main' into feat/dsql-system-diagnostics

Signed-off-by: James Morle <7480305+Morlej@users.noreply.github.com>

Author: Morlej

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
       self_mutation_happened: ${{ steps.self_mutation.outputs.self_mutation_happened }}
     steps:
       - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0  # full history
 

.github/workflows/codeql.yml

@@ -73,7 +73,7 @@ jobs:
         # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
     # Add any setup steps before running the `github/codeql-action/init` action.
     # This includes steps like installing compilers or runtimes (`actions/setup-node`

.github/workflows/dependency-review.yml

@@ -8,7 +8,7 @@ jobs:
   dependency-review:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
       - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
         with:
           config-file: amazon-ospo/dependency-review-config/default/dependency-review-config.yml@8e4c9fdde54d2b7c6a3a28b97eddd26c4cd90a66 # main

.github/workflows/merge-prevention.yml

@@ -45,18 +45,19 @@ jobs:
     env:
       GH_TOKEN: ${{ github.token }}
       PR_LABELS_JSON: ${{ toJson(github.event.pull_request.labels.*.name) }}
+      PR_NUMBER_EVENT: ${{ github.event.pull_request.number }}
     steps:
       - name: Get PR info
         id: get-pr
         run: |
-          if [ "${{ github.event_name }}" == "merge_group" ]; then
-            PR_NUMBER=$(echo "${{ github.ref }}" | grep -oP '(?<=/pr-)\d+' || echo "")
-            PR_LABELS=$(gh api repos/${{ github.repository }}/pulls/$PR_NUMBER | jq -c '[.labels[].name] // []')
+          if [ "$GITHUB_EVENT_NAME" == "merge_group" ]; then
+            PR_NUMBER=$(echo "$GITHUB_REF" | grep -oP '(?<=/pr-)\d+' || echo "")
+            PR_LABELS=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" | jq -c '[.labels[].name] // []')
             echo "::group::Getting Information"
-            gh api repos/${{ github.repository }}/pulls/$PR_NUMBER
+            gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER"
             echo "::endgroup::"
-          elif [ "${{ github.event_name }}" == "pull_request" ]; then
-            PR_NUMBER="${{ github.event.pull_request.number }}"
+          elif [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
+            PR_NUMBER="$PR_NUMBER_EVENT"
             PR_LABELS=$(echo "$PR_LABELS_JSON" | jq -c '.')
           fi
           echo "::group::Debug Output Values"
@@ -70,8 +71,9 @@ jobs:
     needs: get-pr-info
     if: always()
     steps:
-      - run: |
-          PR_NUMBER="${{ needs.get-pr-info.outputs.pr_number }}"
+      - env:
+          PR_NUMBER: ${{ needs.get-pr-info.outputs.pr_number }}
+        run: |
           # Default to 0 (allow all) if not set
           if [ -z "$HALT_MERGES" ]; then
             HALT_MERGES=0
@@ -98,19 +100,21 @@ jobs:
     needs: get-pr-info
     if: always()
     steps:
-      - run: |
+      - env:
+          PR_LABELS: ${{ needs.get-pr-info.outputs.pr_labels }}
+        run: |
           echo "::group::Debug Output Values"
-          echo "PR_LABELS: ${{ needs.get-pr-info.outputs.pr_labels }}"
+          echo "PR_LABELS: $PR_LABELS"
           echo "::endgroup::"
-      - name: When PR has the "${{ env.DO_NOT_MERGE_LABEL }}" label
+      - name: When PR has the do-not-merge label
         id: pr-has-label
         if: contains(needs.get-pr-info.outputs.pr_labels, env.DO_NOT_MERGE_LABEL)
         run: |
-          echo "::error::❌ The label \"${{ env.DO_NOT_MERGE_LABEL }}\" is used to prevent merging."
+          echo "::error::❌ The label \"$DO_NOT_MERGE_LABEL\" is used to prevent merging."
           exit 1
-      - name: When PR does not have the "${{ env.DO_NOT_MERGE_LABEL }}" label
+      - name: When PR does not have the do-not-merge label
         id: pr-missing-label
-        if: ! contains(needs.get-pr-info.outputs.pr_labels, env.DO_NOT_MERGE_LABEL)
+        if: ${{ !contains(needs.get-pr-info.outputs.pr_labels, env.DO_NOT_MERGE_LABEL) }}
         run: |
-          echo "::debug::✅ The label \"${{ env.DO_NOT_MERGE_LABEL }}\" is absent"
+          echo "::debug::✅ The label \"$DO_NOT_MERGE_LABEL\" is absent"
           exit 0

.github/workflows/scorecard-analysis.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/security-scanners.yml

@@ -34,19 +34,32 @@ jobs:
     env:
       GITLEAKS_VERSION: "8.30.0"
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0
       - name: Install gitleaks
         run: |
           curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" | tar -xz;
           sudo mv gitleaks /usr/local/bin/;
           gitleaks --version;
-      - name: Run gitleaks (full history)
+      - name: Run gitleaks
         id: gitleaks
+        env:
+          # On pull_request, restrict the scan to commits the PR introduces so
+          # it cannot fail on secrets that live only on other, unmerged branches.
+          # On push/schedule/dispatch, scan the full history reachable from HEAD.
+          PR_BASE_REF: ${{ github.base_ref }}
         run: |
           set +e
-          gitleaks git --config=.gitleaks.toml --baseline-path=.gitleaks-baseline.json --report-path=gitleaks-report_sarif.json --report-format=sarif .
+          # checkout uses fetch-depth: 0, so the base branch is already present
+          # as origin/<base_ref>; no extra fetch is needed.
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            LOG_OPTS="origin/$PR_BASE_REF..HEAD"
+          else
+            LOG_OPTS="HEAD"
+          fi
+          echo "::notice::gitleaks scanning range: $LOG_OPTS"
+          gitleaks git --config=.gitleaks.toml --baseline-path=.gitleaks-baseline.json --log-opts="$LOG_OPTS" --report-path=gitleaks-report_sarif.json --report-format=sarif .
           GITLEAKS_EXIT=$?
           set -e
           echo "exit_code=$GITLEAKS_EXIT" >> "$GITHUB_OUTPUT"
@@ -72,10 +85,10 @@ jobs:
       security-events: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
       - run: |
           echo "bandit[sarif]==1.9.3" > requirements.txt
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
           python-version: '3.x'
           cache: 'pip'
@@ -111,7 +124,7 @@ jobs:
       security-events: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0
       - run: |
           curl -sSfL https://raw.githubusercontent.com/anchore/grype/43e7e3246ed01b1ec0ff54f9b054201ccbe78e3a/install.sh | sh -s -- -b /usr/local/bin v0.104.3
           grype --version
@@ -145,10 +158,10 @@ jobs:
       security-events: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #v7.0.0
       - run: |
           echo "checkov==3.2.500" > requirements.txt
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
           python-version: '3.x'
           cache: 'pip'
@@ -186,12 +199,12 @@ jobs:
       security-events: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0 # Required for baseline comparison
       - run: |
           echo "semgrep==1.151.0" > requirements.txt
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
           python-version: '3.x'
           cache: 'pip'
@@ -209,33 +222,15 @@ jobs:
               BASELINE_ARGS="--baseline-commit $BASELINE_SHA"
             fi
 
-            # Run semgrep, capture exit code
-            # Rule exclusions documented in .semgrep.yaml
+            # Run semgrep against the vendored ruleset (RFC #211).
+            # Exclusions are baked into tools/semgrep/rules-vendored.yaml
+            # (source of truth: tools/semgrep/rules-status.toml), so no
+            # --exclude-rule flags are needed. --config is explicit because
+            # CI does not inherit mise's SEMGREP_RULES env.
             set +e
-            semgrep scan --oss-only --verbose --metrics=off --config=r/all \
+            semgrep scan --oss-only --verbose --metrics=off \
+              --config=tools/semgrep/rules-vendored.yaml \
               --max-log-list-entries=0 \
-              --exclude-rule="bbp-pattern-inject" \
-              --exclude-rule="ai.generic.detect-generic-ai-anthprop.detect-generic-ai-anthprop" \
-              --exclude-rule="generic.secrets.security.detected-sonarqube-docs-api-key.detected-sonarqube-docs-api-key" \
-              --exclude-rule="apex.lang.best-practice.ncino.accessmodifiers.globalaccessmodifiers.global-access-modifiers" \
-              --exclude-rule="apex.lang.best-practice.ncino.urls.absoluteurls.absolute-urls" \
-              --exclude-rule="apex.lang.security.ncino.dml.apexcsrfconstructor.apex-csrf-constructor" \
-              --exclude-rule="apex.lang.security.ncino.dml.dmlnativestatements.dml-native-statements" \
-              --exclude-rule="apex.lang.security.ncino.encryption.badcrypto.bad-crypto" \
-              --exclude-rule="apex.lang.security.ncino.endpoints.insecurehttprequest.insecure-http-request" \
-              --exclude-rule="apex.lang.security.ncino.endpoints.namedcredentialsconstantmatch.named-credentials-constant-match" \
-              --exclude-rule="apex.lang.security.ncino.endpoints.namedcredentialsstringmatch.named-credentials-string-match" \
-              --exclude-rule="apex.lang.security.ncino.injection.apexsoqlinjectionfromunescapedurlparam.soql-injection-unescaped-url-param" \
-              --exclude-rule="apex.lang.security.ncino.injection.apexsoqlinjectionunescapedparam.soql-injection-unescaped-param" \
-              --exclude-rule="apex.lang.security.ncino.sharing.specifysharinglevel.specify-sharing-level" \
-              --exclude-rule="apex.lang.security.ncino.system.systemdebug.system-debug" \
-              --exclude-rule="elixir.lang.best-practice.deprecated-bnot-operator.deprecated_bnot_operator" \
-              --exclude-rule="elixir.lang.best-practice.deprecated-bxor-operator.deprecated_bxor_operator" \
-              --exclude-rule="elixir.lang.best-practice.deprecated-calendar-iso-day-of-week-3.deprecated_calendar_iso_day_of_week_3" \
-              --exclude-rule="elixir.lang.best-practice.deprecated-use-bitwise.deprecated_use_bitwise" \
-              --exclude-rule="elixir.lang.best-practice.enum-map-into.enum_map_into" \
-              --exclude-rule="elixir.lang.best-practice.enum-map-join.enum_map_join" \
-              --exclude-rule="elixir.lang.correctness.atom-exhaustion.atom_exhaustion" \
               --sarif-output semgrep-report_sarif.json $BASELINE_ARGS
             SEMGREP_EXIT=$?
             set -e
@@ -314,7 +309,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
       - name: Wait for ClamAV service
         run: timeout 300 bash -c 'until echo > /dev/tcp/localhost/3310; do sleep 5; done' 2>/dev/null
       - run: |
@@ -359,7 +354,7 @@ jobs:
     # - if: always()
     #   run: |
     #     docker inspect $(docker ps -qf "ancestor=sonarqube:community@sha256:48dd0e946ad6481dde43bb31d1a7af09c22f59be6399b195dcce7b87d82c5f40") | jq '.[] | {"test": .Config.Healthcheck.Test, "state": .State.Health}'
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
       with:
         fetch-depth: 0
     - name: Wait for SonarQube to be ready

.github/workflows/upgrade-main.yml

@@ -32,7 +32,7 @@ jobs:
       patch_created: ${{ steps.create_patch.outputs.patch_created }}
     steps:
       - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           ref: main
 
@@ -73,7 +73,7 @@ jobs:
     if: ${{ needs.upgrade.outputs.patch_created }}
     steps:
       - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           ref: main
 

.gitleaks.toml

@@ -13,6 +13,12 @@ description = "Global allowlist"
 paths = [
     '''\.gitleaksignore$''',
     '''\.gitleaks-baseline\.json$''',
+    # Vendored Semgrep ruleset (RFC #211): contains secret-detection regexes
+    # that gitleaks would false-positive on. Generated by tools/semgrep/rules-update.py.
+    # gitleaks scans full history, so we allowlist BOTH the current filename and
+    # the former name (r-all.active.yaml) it carried in earlier branch commits.
+    '''tools/semgrep/rules-vendored\.yaml$''',
+    '''tools/semgrep/r-all\.active\.yaml$''',
 ]
 
 # Example: Add custom rules or override defaults

.pre-commit-config.yaml

@@ -1,3 +1,14 @@
+# The generated, byte-for-byte vendored Semgrep ruleset (RFC #211) is produced
+# by tools/semgrep/rules-update.py and must stay identical to the upstream
+# download. It necessarily contains detection patterns (private-key markers,
+# example secrets, non-permalink URLs) and exceeds size limits, so ALL content
+# hooks must skip it; any autofix hook would corrupt the ruleset. The
+# hand-authored rules-update.py and rules-status.toml stay in scope on purpose.
+exclude: |
+    (?x)^(
+        tools/semgrep/rules-vendored\.yaml
+    )$
+
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
@@ -56,7 +67,7 @@ repos:
     rev: v1.151.0
     hooks:
     -   id: semgrep
-        args: [ '--oss-only', '--verbose', '--metrics=off', '--baseline-commit', 'main', '--config=r/all', '--sarif-output', '.tmp/semgrep.sarif.json', '--error' ]
+        args: [ '--oss-only', '--verbose', '--metrics=off', '--baseline-commit', 'main', '--config=tools/semgrep/rules-vendored.yaml', '--max-log-list-entries=0', '--sarif-output', '.tmp/semgrep.sarif.json', '--error' ]
         stages: [pre-push]
 -   repo: https://github.com/bridgecrewio/checkov
     rev: 3.2.500