diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..1b06f3ebf --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.github/workflows/*.lock.yml linguist-generated=true merge=ours diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 763b14ada..5581591e0 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -14,7 +14,7 @@ runs: using: composite steps: - name: Build container image and push to Azure - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: file: ${{ inputs.DOCKER_FILE }} tags: ${{ inputs.DOCKER_REPO }}:latest diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 000000000..2fc91e5a3 --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,9 @@ +{ + "entries": { + "actions/github-script@v9.0.0": { + "repo": "actions/github-script", + "version": "v9.0.0", + "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3f4b1b563..8548a9d3f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,60 +1,66 @@ -version: 2 updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" - # Maintain dependencies for GitHub composite Actions (/.github/actions) - # Waiting for supporting wildcards see https://github.com/dependabot/dependabot-core/issues/5137 - - package-ecosystem: "github-actions" - directory: "/.github/actions/azure-login" - schedule: - interval: "daily" - - package-ecosystem: "github-actions" - directory: "/.github/actions/do-login" - schedule: - interval: "daily" - - package-ecosystem: "github-actions" - directory: "/.github/actions/docker-build" - schedule: - interval: "daily" - - package-ecosystem: "maven" - directory: "/" - schedule: - interval: "daily" - groups: - graphql-kotlin: - patterns: - - "com.expediagroup:graphql-kotlin-*" - maven-core: - patterns: - - "org.apache.maven:maven-*" - jackson: - patterns: - - "com.fasterxml.jackson.*:jackson-*" - jetty: - patterns: - - "org.eclipse.jetty:*" - jwt: - patterns: - - "io.jsonwebtoken:jjwt-*" - logback: - patterns: - - "ch.qos.logback:*" - jakarta-apis: - patterns: - - "jakarta.*:*" - rest-assured: - patterns: - - "io.rest-assured:*" - quarkus-logging: - patterns: - - "io.quarkiverse.logging.logback:*" - - package-ecosystem: docker - directory: "/" - schedule: - interval: "daily" - - package-ecosystem: docker - directory: "/deploy" - schedule: - interval: "daily" +- directory: / + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. + package-ecosystem: github-actions + schedule: + interval: daily +- directory: /.github/actions/azure-login + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. + package-ecosystem: github-actions + schedule: + interval: daily +- directory: /.github/actions/do-login + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. + package-ecosystem: github-actions + schedule: + interval: daily +- directory: /.github/actions/docker-build + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. + package-ecosystem: github-actions + schedule: + interval: daily +- directory: / + groups: + graphql-kotlin: + patterns: + - com.expediagroup:graphql-kotlin-* + jackson: + patterns: + - com.fasterxml.jackson.*:jackson-* + jakarta-apis: + patterns: + - jakarta.*:* + jetty: + patterns: + - org.eclipse.jetty:* + jwt: + patterns: + - io.jsonwebtoken:jjwt-* + logback: + patterns: + - ch.qos.logback:* + maven-core: + patterns: + - org.apache.maven:maven-* + quarkus-logging: + patterns: + - io.quarkiverse.logging.logback:* + rest-assured: + patterns: + - io.rest-assured:* + package-ecosystem: maven + schedule: + interval: daily +- directory: / + package-ecosystem: docker + schedule: + interval: daily +- directory: /deploy + package-ecosystem: docker + schedule: + interval: daily +version: 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 339b9c8ab..158e57897 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,11 @@ jobs: adoptopenjdk) echo "ARGS=-Padoptopenjdk,-adoptium" >> $GITHUB_ENV ;; esac - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: java-version: '21' distribution: 'temurin' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e8995cae5..18cef5dc7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -36,11 +36,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 + uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -48,7 +48,7 @@ jobs: # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: java-version: '21' distribution: 'temurin' @@ -59,6 +59,6 @@ jobs: ./mvnw --batch-mode clean install -Padoptium,-adoptopenjdk - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 + uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml new file mode 100644 index 000000000..810811480 --- /dev/null +++ b/.github/workflows/dependabot-auto-merge.yml @@ -0,0 +1,14 @@ +# This is a templated file from https://github.com/adoptium/.eclipsefdn/tree/main/otterdog/blueprints/require_dependabot_auto_merge.yml +name: Dependabot auto-merge +on: pull_request_target + +permissions: read-all + +jobs: + dependabot: + permissions: + contents: write + pull-requests: write + uses: adoptium/.github/.github/workflows/dependabot-auto-merge.yml@main + with: + merge_method: merge diff --git a/.github/workflows/deploy-adoptium.yml b/.github/workflows/deploy-adoptium.yml index 408447548..af6df4bec 100644 --- a/.github/workflows/deploy-adoptium.yml +++ b/.github/workflows/deploy-adoptium.yml @@ -14,7 +14,7 @@ jobs: if: startsWith(github.repository, 'adoptium/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Login to Azure uses: ./.github/actions/azure-login diff --git a/.github/workflows/deploy-adoptopenjdk.yml b/.github/workflows/deploy-adoptopenjdk.yml index a434e819e..b5fee5d15 100644 --- a/.github/workflows/deploy-adoptopenjdk.yml +++ b/.github/workflows/deploy-adoptopenjdk.yml @@ -14,7 +14,7 @@ jobs: if: startsWith(github.repository, 'adoptium/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Login to Azure uses: ./.github/actions/azure-login diff --git a/.github/workflows/production-branch-checker.lock.yml b/.github/workflows/production-branch-checker.lock.yml new file mode 100644 index 000000000..e33c0541d --- /dev/null +++ b/.github/workflows/production-branch-checker.lock.yml @@ -0,0 +1,1412 @@ +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"8d97e8feb9eaad736d25881a2f1e567b4e806bd396fc967b822dba8ffad92821","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"v0.76.1","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# +# Secrets used: +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@v0.76.1 +# +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55 +# - ghcr.io/github/gh-aw-mcpg:v0.3.19 +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + +name: "Production Branch Merge Checker" +on: + pull_request: + branches: + - production + +permissions: {} + +concurrency: + cancel-in-progress: true + group: production-branch-checker-${{ github.ref }} + +run-name: "Production Branch Merge Checker" + +jobs: + activation: + needs: pre_activation + if: > + needs.pre_activation.outputs.activated == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id) + runs-on: ubuntu-slim + permissions: + actions: read + contents: read + outputs: + body: ${{ steps.sanitized.outputs.body }} + comment_id: "" + comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} + text: ${{ steps.sanitized.outputs.text }} + title: ${{ steps.sanitized.outputs.title }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.pre_activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.pre_activation.outputs.setup-parent-span-id || needs.pre_activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/production-branch-checker.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","*.adoptium.net"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Checkout .github and .agents folders + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_WORKFLOW_FILE: "production-branch-checker.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + await main(); + - name: Compute current body text + id: sanitized + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_ALLOWED_DOMAINS: "*.adoptium.net,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,localhost,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/compute_text.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_436c82b8abddf95e_EOF' + + GH_AW_PROMPT_436c82b8abddf95e_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_436c82b8abddf95e_EOF' + + Tools: add_comment, missing_tool, missing_data, noop + + GH_AW_PROMPT_436c82b8abddf95e_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_436c82b8abddf95e_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + - **checkouts**: The following repositories have been checked out and are available in the workspace: + - `$GITHUB_WORKSPACE` → `__GH_AW_GITHUB_REPOSITORY__` (cwd) [full history, all branches available as remote-tracking refs] + - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). + + + GH_AW_PROMPT_436c82b8abddf95e_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_436c82b8abddf95e_EOF' + + {{#runtime-import .github/workflows/production-branch-checker.md}} + GH_AW_PROMPT_436c82b8abddf95e_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST, + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + copilot-requests: write + pull-requests: read + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_WORKFLOW_ID_SANITIZED: productionbranchchecker + outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/production-branch-checker.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + fetch-depth: 0 + - name: Create gh-aw temp directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request || github.event.issue.pull_request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Restore agent config folders from base branch + if: steps.checkout-pr.outcome == 'success' + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55 ghcr.io/github/gh-aw-firewall/squid:0.25.55 ghcr.io/github/gh-aw-mcpg:v0.3.19 ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config + run: | + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_c3fa6940be766066_EOF' + {"add_comment":{"hide_older_comments":true,"max":1},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_c3fa6940be766066_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | + { + "description_suffixes": { + "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added. Supports reply_to_id for discussion threading." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "add_comment": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "item_number": { + "issueOrPRNumber": true + }, + "reply_to_id": { + "type": "string", + "maxLength": 256 + }, + "repo": { + "type": "string", + "maxLength": 256 + } + } + }, + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + }, + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 + } + } + } + } + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" + + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} + GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="8080" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' + + mkdir -p /home/runner/.copilot + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_77a465c4e1147b33_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", + "env": { + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", + "repos": "$GITHUB_MCP_GUARD_REPOS" + } + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_77a465c4e1147b33_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true + env: + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); + await main(); + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 30 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["*.adoptium.net","api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ github.token }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + S2STOKENS: true + XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Append agent step summary + if: always() + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs + if: always() + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "*.adoptium.net,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,localhost,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true + # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) + if command -v awf &> /dev/null; then + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + else + echo 'AWF binary not installed, skipping firewall log summary' + fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/agent_usage.json + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt + /tmp/gh-aw/agent/ + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - safe_outputs + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + pull-requests: write + concurrency: + group: "gh-aw-conclusion-production-branch-checker" + cancel-in-progress: false + queue: max + outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/production-branch-checker.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/production-branch-checker.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/production-branch-checker.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); + await main(); + - name: Record missing tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/production-branch-checker.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/production-branch-checker.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); + await main(); + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/production-branch-checker.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "production-branch-checker" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "30" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + + detection: + needs: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') + runs-on: ubuntu-latest + permissions: + contents: read + copilot-requests: write + outputs: + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/production-branch-checker.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55 ghcr.io/github/gh-aw-firewall/squid:0.25.55 + - name: Check if detection needed + id: detection_guard + if: always() + env: + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + WORKFLOW_NAME: "Production Branch Merge Checker" + WORKFLOW_DESCRIPTION: "No description provided" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ github.token }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + S2STOKENS: true + XDG_CONFIG_HOME: /home/runner + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: detection + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + with: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } + + pre_activation: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id + runs-on: ubuntu-slim + outputs: + activated: ${{ steps.check_membership.outputs.is_team_member == 'true' }} + matched_command: '' + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/production-branch-checker.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Check team membership for workflow + id: check_membership + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_REQUIRED_ROLES: "admin,maintainer,write" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_membership.cjs'); + await main(); + + safe_outputs: + needs: + - activation + - agent + - detection + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + pull-requests: write + timeout-minutes: 15 + env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/production-branch-checker" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} + GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" + GH_AW_WORKFLOW_ID: "production-branch-checker" + GH_AW_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/production-branch-checker.md" + outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + comment_id: ${{ steps.process_safe_outputs.outputs.comment_id }} + comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Production Branch Merge Checker" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/production-branch-checker.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash + run: | + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "*.adoptium.net,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,localhost,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore + diff --git a/.github/workflows/production-branch-checker.md b/.github/workflows/production-branch-checker.md new file mode 100644 index 000000000..782a091e9 --- /dev/null +++ b/.github/workflows/production-branch-checker.md @@ -0,0 +1,355 @@ +--- +on: + pull_request: + branches: [ production ] + +permissions: + contents: read + pull-requests: read + +features: + copilot-requests: true + +safe-outputs: + add-comment: + hide-older-comments: true + allowed-domains: + - "*.adoptium.net" + +network: + allowed: + - defaults + - "*.adoptium.net" + +checkout: + fetch-depth: 0 + +concurrency: + group: production-branch-checker-${{ github.ref }} + cancel-in-progress: true + +timeout-minutes: 30 +--- + +# Production Branch Merge Checker + +You are a production deployment risk-analysis agent for the **Adoptium API** +(`api.adoptium.net`). Your job is to analyse pull requests that merge `main` +into the `production` branch, produce a concise summary of **meaningful +changes**, assign a **merge risk level**, and provide **post-merge validation +guidance** — all posted as a single PR comment. + +## Context + +This repository contains the Adoptium API v3 — a Kotlin / Maven / Quarkus +application that serves binary metadata for Eclipse Temurin OpenJDK builds. + +The deployment model is: + +| Branch | Environment | Kubernetes Namespace | URL | +|--------|-------------|---------------------|-----| +| `main` | Staging | `api-staging` | `https://staging-api.adoptium.net` | +| `production` | **Production** | `api` | `https://api.adoptium.net` | + +Merging `main` → `production` triggers a Docker image build and a Kubernetes +redeployment of both the **Frontend** (REST API) and the **Updater** (data +sync) into the production namespace. + +### Key modules + +| Module | Role | +|--------|------| +| `adoptium-frontend-parent/adoptium-api-v3-frontend` | REST API frontend (Quarkus) | +| `adoptium-api-v3-persistence` | MongoDB persistence layer | +| `adoptium-updater-parent/adoptium-api-v3-updater` | Periodic data updater (GitHub → DB) | +| `adoptium-models-parent/*` | Shared data models | +| `adoptium-api-v3-telemetry` | Application Insights telemetry | +| `adoptium-frontend-parent/adoptium-frontend-assets` | OpenAPI specs & static assets | +| `deploy/` | Dockerfile, Kubernetes & MongoDB configs | + +### What matters vs what doesn't + +**High-signal changes** (always include in summary): +- REST API endpoint changes (new routes, changed request/response shapes, removed endpoints) +- Database schema or query changes in persistence layer +- Updater logic changes (data source parsing, refresh intervals, new data sources) +- Deployment configuration changes (`deploy/`, Dockerfiles, Kubernetes manifests) +- MongoDB configuration or migration changes +- Authentication / security changes +- OpenAPI spec changes +- Application configuration changes (`application.properties`, `application.yaml`) +- Feature flag or environment variable changes + +**Low-signal changes** (exclude from the summary or mention only as a one-line count): +- Dependency version bumps in `pom.xml` (unless a major version change or known-breaking) +- Maven plugin version updates +- CI workflow changes (`.github/workflows/`) — unless they affect deployment +- Code style / formatting changes +- Test-only changes (mention count but don't detail) +- Documentation-only changes (`*.md`, `*.adoc`) +- `.gitignore`, `.editorconfig`, and other tooling config + +## Instructions + +### Step 1: Gather the diff + +Examine all files changed in this PR. Use the diff between the `production` +branch (base) and the PR head (which is `main`) to identify every changed file. + +Categorise each changed file into one of these buckets: + +| Category | Examples | Signal | +|----------|----------|--------| +| **API / Frontend** | `adoptium-frontend-parent/**/*.kt`, OpenAPI specs | High | +| **Persistence** | `adoptium-api-v3-persistence/**/*.kt` | High | +| **Updater** | `adoptium-updater-parent/**/*.kt` | High | +| **Models** | `adoptium-models-parent/**/*.kt` | High | +| **Deployment** | `deploy/**`, `Dockerfile`, `docker-compose.yml` | High | +| **Telemetry** | `adoptium-api-v3-telemetry/**` | Medium | +| **Config** | `application.properties`, `application.yaml`, env vars | High | +| **Dependencies** | `pom.xml` (version bumps only) | Low — skip | +| **CI/Workflows** | `.github/workflows/**` (non-deploy) | Low — skip | +| **Tests only** | `**/test/**` | Low — count only | +| **Docs** | `*.md`, `*.adoc`, `docs/**` | Low — skip | +| **Tooling** | `.gitignore`, `.editorconfig`, `mvnw*` | Low — skip | + +### Step 2: Summarise meaningful changes + +For every file in a **High** or **Medium** signal category, read the diff and +write a concise, human-readable summary of what changed. Group changes by +module. Focus on *behaviour* changes, not line-by-line diffs: + +- "Added new `/v3/assets/latest` endpoint that returns the latest release per platform" +- "Changed MongoDB query in `ReleaseRepository` to filter by vendor field" +- "Updated updater refresh interval from 6 minutes to 3 minutes" +- "Modified Dockerfile to use multi-stage build with JDK 21" + +For **Low** signal files, emit a single line like: +> 📦 12 dependency version bumps in pom.xml files (no major version changes) +> 🧪 8 test files changed +> 📝 3 documentation files updated + +If a `pom.xml` change includes a **major version bump** of a runtime dependency +(not a Maven plugin), or adds/removes a dependency, promote it to High signal +and include it in the detailed summary. + +### Step 3: Live endpoint comparison (staging vs production) + +When the diff includes changes to **API / Frontend**, **Models**, or +**Persistence** code, probe the live APIs to show reviewers the concrete +effect of the changes. This makes the report far more useful than a +code-only diff. + +#### Which endpoints to check + +Map changed source files to API routes: + +| Source file pattern | API route to probe | +|--------------------|-----------------------| +| `routes/info/AvailableReleasesResource.kt` | `/v3/info/available_releases` | +| `routes/info/ReleaseListResource.kt` | `/v3/info/release_versions` | +| `routes/info/ReleaseNotesResource.kt` | `/v3/info/release_notes/{feature_version}` | +| `routes/info/TypesResource.kt` | `/v3/types/*` | +| `routes/AssetsResource.kt` | `/v3/assets/feature_releases/{version}/ga` | +| `routes/VersionResource.kt` | `/v3/version/*` | +| `routes/packages/BinaryResource.kt` | `/v3/binary/latest/{version}/ga/linux/x64/jdk/hotspot/normal/eclipse` | +| `routes/packages/InstallerResource.kt` | `/v3/installer/latest/{version}/ga/windows/x64/jdk/hotspot/normal/eclipse` | +| `routes/packages/ChecksumResource.kt` | `/v3/checksum/latest/{version}/ga/linux/x64/jdk/hotspot/normal/eclipse` | +| `routes/packages/SignatureResource.kt` | `/v3/signature/latest/{version}/ga/linux/x64/jdk/hotspot/normal/eclipse` | +| `routes/CdxaResource.kt` | `/v3/cdxas/{feature_version}` | +| `routes/AttestationAliasResource.kt` | `/v3/attestations/{feature_version}` | +| `routes/stats/DownloadStatsResource.kt` | `/v3/stats/downloads/total` | +| Model classes (`adoptium-models-parent/**`) | All of the above (models are shared) | +| Persistence classes | All of the above (persistence is shared) | + +For `{version}` / `{feature_version}` placeholders, use the **latest GA +feature version** obtained from +`/v3/info/available_releases` → `most_recent_feature_version`. + +#### How to probe + +For each affected route: + +1. **Fetch from staging:** `https://staging-api.adoptium.net{route}` +2. **Fetch from production:** `https://api.adoptium.net{route}` +3. Compare the JSON responses structurally (ignore ordering of array elements + and ephemeral fields like `updated_at` timestamps). + +#### What to report + +For each probed endpoint, report one of: + +| Result | Meaning | +|--------|---------| +| ✅ **Identical** | Staging and production return the same response | +| 🆕 **New endpoint** | Staging returns 200, production returns 404 | +| ⚠️ **Response differs** | Structural diff — show the key differences | +| 🔴 **Staging error** | Staging returns 4xx/5xx — flag for investigation | +| ⏭️ **Skipped** | Endpoint requires path params that can't be auto-resolved | + +When responses differ, summarise the differences concisely: +- New fields added to the response +- Fields removed from the response +- Changed field types or value patterns +- Different array lengths (e.g., "staging returns 12 releases, production returns 11") +- Different HTTP status codes + +Do **not** dump full JSON payloads. Show only the structural differences. + +#### When to skip + +- If no API / Frontend / Models / Persistence files changed, skip this step + entirely and note: "No API-affecting changes detected — live comparison skipped." +- If staging is unreachable (connection timeout or 5xx on health check), note + the failure and skip live comparison. Do not let this block the report. +- Binary download endpoints (`/v3/binary/`) — only check the redirect URL and + HTTP status, do not download the actual binary. + +### Step 4: Assess merge risk + +Assign an overall risk level based on the following criteria: + +#### 🔴 HIGH RISK +Assign HIGH if **any** of the following are true: +- REST API endpoint signatures changed (path, method, request/response types) +- REST API endpoints removed or renamed +- Database schema changes or new MongoDB indexes +- Deployment configuration changes (Dockerfile base image, K8s manifests, resource limits) +- MongoDB connection or authentication changes +- Changes to the data updater that alter what data is stored or how it is parsed +- OpenAPI spec breaking changes (removed fields, changed types) +- Security-related changes (auth, CORS, TLS) +- Environment variable additions/removals that require infra changes +- Quarkus **major** version upgrade (e.g. 3.x → 4.x) + +#### 🟡 MEDIUM RISK +Assign MEDIUM if **none** of the HIGH triggers apply, but **any** of these do: +- Quarkus minor/patch version upgrade (e.g. 3.35 → 3.36) — these are frequent + and low-risk in practice +- New API endpoints added (additive, non-breaking) +- Changes to `/v3/stats/**` endpoints — stats routes are only consumed + internally, so changes here do not affect external consumers +- Query logic changes in persistence layer (same schema, different queries) +- Updater timing or scheduling changes +- Telemetry configuration changes +- New or modified application config properties +- Addition of new data sources to the updater +- Non-trivial refactoring of shared model classes + +#### 🟢 LOW RISK +Assign LOW if **only** these types of changes are present: +- Dependency version bumps (patch/minor, no known breaking changes) +- Test-only changes +- Documentation updates +- Code style / formatting +- CI workflow changes that don't affect deployment +- Maven plugin updates + +### Step 5: Generate post-merge validation checklist + +Based on the changes detected, produce a targeted checklist of things to +verify after merging to production. Only include checks relevant to what +actually changed — don't dump a generic checklist. + +**Always include** (for any non-trivial merge): +- [ ] Verify pods restarted successfully in `api` namespace +- [ ] Check `https://api.adoptium.net/v3/info/available_releases` returns 200 +- [ ] Confirm no error spike in Application Insights (first 15 minutes) + +**If API endpoints changed:** +- [ ] Verify affected endpoints return expected responses +- [ ] Check OpenAPI spec at `https://api.adoptium.net/q/openapi` is updated +- [ ] Verify no 404s or 500s on changed routes + +**If persistence/database changed:** +- [ ] Monitor MongoDB query performance (slow query log) +- [ ] Verify data integrity — spot-check a few releases via API + +**If updater changed:** +- [ ] Verify updater pod is running and scheduling correctly +- [ ] Check that next incremental update completes without errors +- [ ] Confirm release data is being refreshed (compare timestamps) + +**If deployment config changed:** +- [ ] Verify container image was built and pushed correctly +- [ ] Check resource utilisation (CPU/memory) is within expected bounds +- [ ] Confirm health check endpoints are passing + +**If telemetry changed:** +- [ ] Verify telemetry data is flowing to Application Insights +- [ ] Check for any new custom metrics/events appearing + +### Step 6: Produce the report + +Output a single markdown PR comment with this structure: + +```markdown +## 🚀 Production Merge Report + +### Risk Level: 🔴 HIGH / 🟡 MEDIUM / 🟢 LOW + +> One-sentence summary of why this risk level was assigned. + +### Changes Summary + +#### API / Frontend +- Description of each meaningful change + +#### Persistence +- Description of each meaningful change + +#### Updater +- Description of each meaningful change + +(only include sections for modules that have high/medium signal changes) + +#### Low-Signal Changes +- 📦 N dependency version bumps +- 🧪 N test files changed +- 📝 N documentation files updated + +### Live Endpoint Comparison (Staging vs Production) + +| Endpoint | Status | Details | +|----------|--------|---------| +| `/v3/info/available_releases` | ✅ Identical | — | +| `/v3/assets/feature_releases/21/ga` | ⚠️ Differs | Staging returns 12 releases (new: 21.0.7_6), prod has 11 | +| `/v3/binary/latest/21/ga/linux/x64/jdk/hotspot/normal/eclipse` | ✅ Identical | Same redirect URL | +| `/v3/stats/downloads/total` | 🆕 New | Returns 200 on staging, 404 on production | + +(only include this section when API-affecting changes are detected) + +### Post-Merge Validation Checklist + +- [ ] Verify pods restarted successfully in `api` namespace +- [ ] Check `https://api.adoptium.net/v3/info/available_releases` returns 200 +- [ ] Confirm no error spike in Application Insights (first 15 minutes) +- [ ] (additional checks based on what changed) + +### Staging Verification + +> Before merging, confirm these changes have been validated on staging: +> `https://staging-api.adoptium.net` +> +> - [ ] Staging API is healthy and serving requests +> - [ ] Changed endpoints tested on staging +> - [ ] No new errors in staging Application Insights +``` + +### Important rules + +- **Be concise.** Reviewers want a quick read, not a novel. Aim for one line + per change in the summary. +- **Focus on behaviour, not code.** "Added rate limiting to `/v3/assets`" is + useful. "Modified line 47 of AssetsResource.kt" is not. +- **Never list individual dependency bumps** unless they are major version + changes of runtime dependencies. Just give a count. +- **Risk assessment must be conservative.** If in doubt between two levels, + pick the higher one. It's better to over-warn than under-warn. +- **The checklist must be actionable.** Every item should be something a human + can actually do and verify. Include specific URLs where possible. +- **Always mention staging.** Remind reviewers to verify on + `https://staging-api.adoptium.net` before merging to production. +- **Don't invent changes.** Only report what is actually in the diff. If the + diff is entirely dependency bumps, say so and assign LOW risk. +- Post the report as a PR comment. \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 69711c35d..19f45e124 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,9 +14,9 @@ jobs: contents: write # for Git to git push steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: java-version: '21' distribution: 'temurin' diff --git a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/DownloadStatsInterface.kt b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/DownloadStatsInterface.kt index 95904bc6b..eda0426a7 100644 --- a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/DownloadStatsInterface.kt +++ b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/DownloadStatsInterface.kt @@ -31,6 +31,32 @@ class StatEntry( @ApplicationScoped class DownloadStatsInterface { + companion object { + // Placeholder for the cumulative package download baseline that existed prior to + // daily tracking of Cloudflare package downloads. The Cloudflare API only exposes + // downloads for the last 24h, so this baseline (manually entered into the data + // store / configured here) is added to the running total of daily snapshots to + // produce a cumulative package download count over time. The variable is dated + // with the day the baseline was captured; create a new dated baseline when the + // baseline is next refreshed. + // Map: feature_version -> baseline downloads at the date encoded in the name. + val PACKAGE_DOWNLOAD_BASELINE_2026_05_28: Map = mapOf( + 8 to 635751L, + 11 to 8029903L, + 16 to 20L, + 17 to 531547L, + 18 to 3851L, + 19 to 9614L, + 20 to 5285L, + 21 to 666282L, + 22 to 10417L, + 23 to 16341L, + 24 to 21104L, + 25 to 287069L, + 26 to 18710L + ) + } + @Schema(hidden = true) private var versionSupplier: VersionSupplier @@ -267,14 +293,24 @@ class DownloadStatsInterface { val githubDownloads = githubStats.sumOf { it.downloads } - val packageDownloads = packageStats.sumOf { it.downloads } + val trackedPackageDownloadsByVersion: Map = + packageStats.associate { it.feature_version to it.downloads } + + // Cumulative package downloads = manually-captured baseline + running total of + // daily Cloudflare snapshots stored since baseline date. + val packageBreakdown: Map = + (PACKAGE_DOWNLOAD_BASELINE_2026_05_28.keys + trackedPackageDownloadsByVersion.keys) + .associateWith { featureVersion -> + (PACKAGE_DOWNLOAD_BASELINE_2026_05_28[featureVersion] ?: 0L) + + (trackedPackageDownloadsByVersion[featureVersion] ?: 0L) + } + + val packageDownloads = packageBreakdown.values.sum() val dockerBreakdown = dockerStats.associate { Pair(it.repo, it.pulls) } val githubBreakdown = githubStats.associate { Pair(it.feature_version, it.downloads) } - val packageBreakdown = packageStats.associate { Pair(it.feature_version, it.downloads) } - val totalStats = TotalStats(dockerPulls, githubDownloads, packageDownloads, dockerPulls + githubDownloads + packageDownloads) return DownloadStats(TimeSource.now(), totalStats, githubBreakdown, dockerBreakdown, packageBreakdown) } diff --git a/adoptium-api-v3-staging-checker/README.md b/adoptium-api-v3-staging-checker/README.md new file mode 100644 index 000000000..329acd81e --- /dev/null +++ b/adoptium-api-v3-staging-checker/README.md @@ -0,0 +1,69 @@ +# Staging Live Checker + +Compares responses from the staging API against the live API to verify they are in sync. + +## Build + +From the project root (the staging checker requires the `staging-checker` and `adoptium` profiles): + +```bash +./mvnw package -Pstaging-checker,adoptium -pl adoptium-api-v3-staging-checker -am -DskipTests +``` + +## Run + +```bash +java -cp adoptium-api-v3-staging-checker/target/adoptium-api-v3-staging-checker-*-jar-with-dependencies.jar \ + net.adoptium.api.v3.checker.StagingLiveChecker +``` + +## Usage + +``` +Usage: StagingLiveChecker [vendor] [options] + +Vendor (first positional argument): + adoptium Use Adoptium staging/live URLs (default) + adoptopenjdk Use AdoptOpenJDK staging/live URLs + +Options: + --staging-url Override the staging API base URL + --live-url Override the live API base URL + --versions Comma-separated list of Java versions to check + (default: 8,11,17,21,22,23,24,25) + --help, -h Show this help message and exit +``` + +## Examples + +Check Adoptium with default versions: + +```bash +java -cp ...jar net.adoptium.api.v3.checker.StagingLiveChecker +``` + +Check AdoptOpenJDK: + +```bash +java -cp ...jar net.adoptium.api.v3.checker.StagingLiveChecker adoptopenjdk +``` + +Check specific versions only: + +```bash +java -cp ...jar net.adoptium.api.v3.checker.StagingLiveChecker adoptium --versions 11,17,21 +``` + +Use custom staging and live URLs: + +```bash +java -cp ...jar net.adoptium.api.v3.checker.StagingLiveChecker \ + --staging-url https://my-staging.example.com \ + --live-url https://my-live.example.com +``` + +## Exit Codes + +- `0` — All URL checks passed. +- `1` — One or more URL checks failed. Failed URLs are listed in the summary output. + diff --git a/adoptium-api-v3-staging-checker/pom.xml b/adoptium-api-v3-staging-checker/pom.xml new file mode 100644 index 000000000..321a13c3d --- /dev/null +++ b/adoptium-api-v3-staging-checker/pom.xml @@ -0,0 +1,195 @@ + + + 4.0.0 + + + net.adoptium.api + adoptium-api-v3 + 3.0.1-SNAPSHOT + ../pom.xml + + + adoptium-api-v3-staging-checker + + + net.adoptium.api.v3.checker.StagingLiveChecker + uber-jar + -jar-with-dependencies + + + + + io.quarkus + quarkus-rest + + + io.quarkus + quarkus-kotlin + + + org.jetbrains.kotlinx + kotlinx-coroutines-core + + + org.jboss.weld.se + weld-se-core + + + org.awaitility + awaitility + test + + + org.jboss.weld + weld-junit5 + test + + + org.assertj + assertj-core + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.platform + junit-platform-engine + test + + + org.junit.platform + junit-platform-commons + test + + + io.quarkus + quarkus-junit5 + test + + + org.jboss.logging + jboss-logging + + + + + io.rest-assured + rest-assured + test + + + io.quarkus + quarkus-elytron-security-properties-file + + + org.skyscreamer + jsonassert + compile + + + com.google.code.gson + gson + + + net.adoptium.api + adoptium-http-client-datasource + ${project.version} + compile + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-test + ${kotlin.version} + test + + + + + src/main/kotlin + + + org.apache.maven.plugins + maven-jar-plugin + + + test + + test-jar + + + + jar + + jar + + + classes + + **/V3Updater** + **/KickOffUpdate** + **/UpdateTrigger** + + + + + + + io.quarkus + quarkus-maven-plugin + true + + + + build + generate-code + generate-code-tests + + + + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + compile + + compile + + + + src/main/kotlin + target/generated-sources/annotations + + + + + test-compile + test-compile + + test-compile + + + + + 1.8 + + + + + diff --git a/adoptium-api-v3-staging-checker/src/main/kotlin/net/adoptium/api/v3/checker/StagingLiveChecker.kt b/adoptium-api-v3-staging-checker/src/main/kotlin/net/adoptium/api/v3/checker/StagingLiveChecker.kt new file mode 100644 index 000000000..20e2e0d53 --- /dev/null +++ b/adoptium-api-v3-staging-checker/src/main/kotlin/net/adoptium/api/v3/checker/StagingLiveChecker.kt @@ -0,0 +1,594 @@ +package net.adoptium.api.v3.checker + +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.google.gson.JsonPrimitive +import kotlinx.coroutines.runBlocking +import net.adoptium.api.v3.dataSources.DefaultUpdaterHtmlClient +import net.adoptium.api.v3.dataSources.HttpClientFactory +import org.apache.http.HttpResponse +import org.apache.http.concurrent.FutureCallback +import org.skyscreamer.jsonassert.JSONCompare +import org.skyscreamer.jsonassert.JSONCompareMode +import org.skyscreamer.jsonassert.JSONCompareResult +import java.net.URL +import java.net.URLEncoder +import kotlin.coroutines.suspendCoroutine + +class StagingLiveChecker( + val stagingUrl: String, + val liveUrl: String, + val vendor: String +) { + companion object { + + const val STAGING_URL = "https://staging-api.adoptium.net" + const val LIVE_URL = "https://api.adoptium.net" + + const val STAGING_ADOPTOPENJDK_URL = "https://staging-api.adoptopenjdk.net" + const val LIVE_ADOPTOPENJDK_URL = "https://api.adoptopenjdk.net" + + val TEMPLATED: Array = arrayOf( + + "/v3/assets/feature_releases/{version}/ea", + "/v3/assets/feature_releases/{version}/ga", + "/v3/assets/feature_releases/{version}/ea?architecture=x64", + "/v3/assets/feature_releases/{version}/ea?before=2020-12-21T10:15:30Z", + "/v3/assets/feature_releases/{version}/ea?heap_size=large", + "/v3/assets/feature_releases/{version}/ea?image_type=jdk", + "/v3/assets/feature_releases/{version}/ea?jvm_impl=hotspot", + "/v3/assets/feature_releases/{version}/ea?jvm_impl=openj9", + "/v3/assets/feature_releases/{version}/ea?os=linux", + "/v3/assets/feature_releases/{version}/ea&page=1", + "/v3/assets/feature_releases/{version}/ea&page=1", + "/v3/assets/feature_releases/{version}/ea?page=1", + "/v3/assets/feature_releases/{version}/ea?page=2", + "/v3/assets/feature_releases/{version}/ea?sort_method=DATE", + "/v3/assets/feature_releases/{version}/ea?sort_order=ASC", + "/v3/assets/feature_releases/{version}/ea?vendor=openjdk", + "/v3/assets/feature_releases/{version}/ga?jvm_impl=hotspot", + "/v3/assets/feature_releases/{version}/ga&page=1", + "/v3/assets/feature_releases/{version}/ga&page=1", + "/v3/assets/feature_releases/{version}/ga?page=1", + "/v3/assets/feature_releases/{version}/ea?vendor=eclipse", + + "/v3/assets/latest/{version}/hotspot", + "/v3/assets/latest/{version}/openj9", + + "/v3/binary/latest/{version}/ea/linux/x64/jdk/hotspot/normal/adoptium", + "/v3/binary/latest/{version}/ea/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ea/linux/x64/jdk/hotspot/normal/eclipse", + "/v3/binary/latest/{version}/ea/linux/x64/jdk/hotspot/normal/ibm", + "/v3/binary/latest/{version}/ea/mac/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ea/windows/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/aix/ppc64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/alpine-linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/aarch64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/aarch64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/arm/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/ppc64le/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/ppc64le/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/s390x/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/s390x/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/linux/x64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/mac/aarch64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/mac/aarch64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/mac/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/mac/x64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/windows/x32/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/windows/x32/jre/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/windows/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/windows/x64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/latest/{version}/ga/windows/x64/jre/openj9/normal/adoptopenjdk", + + + "/v3/installer/latest/{version}/ea/linux/s390x/jre/openj9/large/adoptopenjdk", + "/v3/installer/latest/{version}/ga/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/mac/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/mac/x64/jre/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x32/jdk/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x32/jre/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x64/jdk/hotspot/normal/openjdk", + "/v3/installer/latest/{version}/ga/windows/x64/jre/hotspot/large/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x64/jre/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x86/jdk/hotspot/normal/adoptopenjdk", + "/v3/installer/latest/{version}/ga/windows/x86/jre/hotspot/normal/adoptopenjdk", + + "/v3/assets/version/(,{version}]", + "/v3/assets/version/({version},{version+1}]", + "/v3/assets/version/({version},{version+1})?image_type=sbom&release_type=ea", + "/v3/assets/version/[{version},{version+1})", + "/v3/assets/version/({version},{version+1})?architecture=x64", + "/v3/assets/version/({version},{version+1})?heap_size=large", + "/v3/assets/version/({version},{version+1})?image_type=jdk", + "/v3/assets/version/({version},{version+1})?jvm_impl=openj9", + "/v3/assets/version/({version},{version+1})?lts=true", + "/v3/assets/version/({version},{version+1})?os=linux", + "/v3/assets/version/({version},{version+1})?page=2", + "/v3/assets/version/({version},{version+1})?page_size=2", + "/v3/assets/version/({version},{version+1})?project=jdk", + "/v3/assets/version/({version},{version+1})?release_type=ea", + "/v3/assets/version/({version},{version+1})?sort_method=DATE", + "/v3/assets/version/({version},{version+1})?sort_order=ASC", + "/v3/assets/version/({version},{version+1})?vendor=openjdk", + ) + + val RANGE_OF_CALLS: Array = arrayOf( + + "/v3/assets/version/[1.0,100.0]", + "/v3/assets/version/[1.0,100.0]?jvm_impl=hotspot", + "/v3/assets/version/(10,11)", + "/v3/assets/version/[11,)", + "/v3/assets/version/11.0.10+9", + "/v3/assets/version/(11.0,12.0)", + "/v3/assets/version/11.0.2+9", + "/v3/assets/version/[11.0.5,11.0.6)", + "/v3/assets/version/11.0.8+10", + "/v3/assets/version/(11,12)", + "/v3/assets/version/[11,12)", + "/v3/assets/version/(11,12)?architecture=x64", + "/v3/assets/version/(11,12)?heap_size=large", + "/v3/assets/version/(11,12)?image_type=jdk", + "/v3/assets/version/(11,12)?image_type=&release_type=ea", + "/v3/assets/version/(11,12)?image_type=sbom&release_type=ea", + "/v3/assets/version/(11,12)?jvm_impl=openj9", + "/v3/assets/version/(11,12)?lts=true", + "/v3/assets/version/(11,12)?os=linux", + "/v3/assets/version/(11,12)?page=2", + "/v3/assets/version/(11,12)?page_size=2", + "/v3/assets/version/(11,12)?project=jdk", + "/v3/assets/version/(11,12)?release_type=ea", + "/v3/assets/version/(11,12)?sort_method=DATE", + "/v3/assets/version/(11,12)?sort_order=ASC", + "/v3/assets/version/(11,12)?vendor=openjdk", + "/v3/assets/version/(11,99)", + "/v3/assets/version/(11,99)?architecture=x64", + "/v3/assets/version/(11,99)?heap_size=large", + "/v3/assets/version/(11,99)?image_type=jdk", + "/v3/assets/version/(11,99)?jvm_impl=openj9", + "/v3/assets/version/(11,99)?lts=true", + "/v3/assets/version/(11,99)?os=linux", + "/v3/assets/version/(11,99)?page=2", + "/v3/assets/version/(11,99)?page_size=2", + "/v3/assets/version/(11,99)?project=jdk", + "/v3/assets/version/(11,99)?release_type=ea", + "/v3/assets/version/(11,99)?sort_method=DATE", + "/v3/assets/version/(11,99)?sort_order=ASC", + "/v3/assets/version/(11,99)?vendor=openjdk", + "/v3/assets/version/(12,13)", + "/v3/assets/version/[12,13)", + "/v3/assets/version/(13,14)", + "/v3/assets/version/[13,14)", + "/v3/assets/version/(14,15)", + "/v3/assets/version/[14,15)", + "/v3/assets/version/(15,16)", + "/v3/assets/version/[15,16)", + "/v3/assets/version/[15,99)", + "/v3/assets/version/(1.8.0,9)", + "/v3/assets/version/[8,)", + "/v3/assets/version/8.0.212+4", + "/v3/assets/version/(8,12]", + "/v3/assets/version/(8,18)?image_type=sbom&release_type=ea", + "/v3/assets/version/[8,9)", + "/v3/assets/version/(,9.0]", + + + "/v3/binary/version/jdk-11.0.11+9_openj9-0.26.0/linux/x64/jdk/openj9/normal/adoptopenjdk", + "/v3/binary/version/jdk-11.0.17+8/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/version/jdk-11.0.17+8/linux/x64/jre/hotspot/normal/adoptopenjdk", + "/v3/binary/version/jdk-15.0.1+9/linux/aarch64/jdk/hotspot/normal/adoptopenjdk.sha1", + "/v3/binary/version/jdk-15.0.1+9/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/version/jdk-15.0.1+9/linux/x64/jdk/hotspot/normal/adoptopenjdk.sha1", + "/v3/binary/version/jdk-15.0.1+9/mac/x64/jdk/hotspot/normal/adoptopenjdk.sha1", + "/v3/binary/version/jdk-15.0.1+9/windows/x64/jdk/hotspot/normal/adoptopenjdk.sha1", + "/v3/binary/version/jdk-17+35/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/version/jdk-457889/windows/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/version/jdk8u242-b08/linux/x64/jdk/hotspot/normal/adoptopenjdk", + "/v3/binary/version/jdk8u242-b08/linux/x64/jdk/hotspot/normal/adoptopenjdk.sha1", + "/v3/binary/version/jdk8u242-b08/windows/x64/jdk/hotspot/normal/adoptopenjdk.sha1", + "/v3/binary/version/jdk8u292-b10/linux/x64/jdk/hotspot/normal/adoptopenjdk", + + + "/v3/info/available_releases", + "/v3/info/release_names", + "/v3/info/release_versions", + + + "/v3/version/jdk-11.0.6+10", + "/v3/version/jdk8u212-b04", + ) + + @JvmStatic + fun main(args: Array) { + if (args.contains("--help") || args.contains("-h")) { + println("Usage: StagingLiveChecker [vendor] [options]") + println() + println("Vendor (first positional argument):") + println(" adoptium Use Adoptium staging/live URLs (default)") + println(" adoptopenjdk Use AdoptOpenJDK staging/live URLs") + println() + println("Options:") + println(" --staging-url Override the staging API base URL") + println(" --live-url Override the live API base URL") + println(" --versions Comma-separated list of Java versions to check") + println(" (default: 8,11,17,21,22,23,24,25)") + println(" --help, -h Show this help message and exit") + println() + println("Examples:") + println(" StagingLiveChecker") + println(" StagingLiveChecker adoptopenjdk") + println(" StagingLiveChecker adoptium --versions 11,17,21") + println(" StagingLiveChecker --staging-url https://my-staging.example.com --live-url https://my-live.example.com") + kotlin.system.exitProcess(0) + } + + val argsList = args.toMutableList() + val vendor = if (argsList.isNotEmpty() && argsList[0] == "adoptopenjdk") { + argsList.removeAt(0) + "adoptopenjdk" + } else { + if (argsList.isNotEmpty() && argsList[0] == "adoptium") { + argsList.removeAt(0) + } + "adoptium" + } + + var stagingUrl: String? = null + var liveUrl: String? = null + var versions: IntArray? = null + val remaining = mutableListOf() + + val iter = argsList.iterator() + while (iter.hasNext()) { + val arg = iter.next() + when { + arg == "--staging-url" -> stagingUrl = if (iter.hasNext()) iter.next() else error("--staging-url requires a value") + arg == "--live-url" -> liveUrl = if (iter.hasNext()) iter.next() else error("--live-url requires a value") + arg == "--versions" -> versions = if (iter.hasNext()) iter.next().split(",").map { it.trim().toInt() }.toIntArray() else error("--versions requires a value") + else -> remaining.add(arg) + } + } + + val resolvedStagingUrl = stagingUrl ?: if (vendor == "adoptopenjdk") STAGING_ADOPTOPENJDK_URL else STAGING_URL + val resolvedLiveUrl = liveUrl ?: if (vendor == "adoptopenjdk") LIVE_ADOPTOPENJDK_URL else LIVE_URL + + val resolvedVersions = versions ?: if (remaining.isNotEmpty()) { + remaining.map { it.toInt() }.toIntArray() + } else { + intArrayOf(8, 11, 17, 21, 22, 23, 24, 25) + } + + val failures = runBlocking { + StagingLiveChecker(resolvedStagingUrl, resolvedLiveUrl, vendor).compareAll(resolvedVersions) + } + println() + println("==== Summary ====") + if (failures.isEmpty()) { + println("StagingLiveChecker completed successfully. All URLs passed.") + kotlin.system.exitProcess(0) + } else { + println("StagingLiveChecker completed with ${failures.size} failure(s):") + failures.forEach { println(" FAILED: $it") } + kotlin.system.exitProcess(1) + } + } + } + + suspend fun compareAll(versions: IntArray = intArrayOf(8, 11, 17, 21, 22, 23, 24, 25)): List { + val failures = mutableListOf() + + versions.forEach { version -> + TEMPLATED.forEach { url -> + val resolvedUrl = url + .replace("{version}", version.toString()) + .replace("{version+1}", (version + 1).toString()) + val result = compare(resolvedUrl) + if (result != null) failures.add(result) + } + } + + RANGE_OF_CALLS + .forEach { url -> + val result = compare(url) + if (result != null) failures.add(result) + } + + return failures + } + + private suspend fun StagingLiveChecker.compare(url: String): String? { + try { + val l = formUrl(liveUrl, url) + + val redirect = !l.contains("binary") && !l.contains("installer") + + val staging = get(URL(formUrl(stagingUrl, url)), redirect) + val live = get(URL(formUrl(liveUrl, url)), redirect) + + when { + live.statusLine.statusCode != staging.statusLine.statusCode -> { + println("Bad code $url ${live.statusLine.statusCode} ${staging.statusLine.statusCode}") + return "$url (status code mismatch: live=${live.statusLine.statusCode}, staging=${staging.statusLine.statusCode})" + } + + live.statusLine.statusCode == 200 -> { + return compareJsonData(live, staging, url, formUrl(stagingUrl, url), formUrl(liveUrl, url)) + } + + live.statusLine.statusCode == 307 -> { + val liveLocation = live.getHeaders("location")[0].getValue() + val stagingLocation = staging.getHeaders("location")[0].getValue() + if (liveLocation != stagingLocation) { + println("Different redirect: $liveLocation $stagingLocation") + return "$url (different redirect: live=$liveLocation, staging=$stagingLocation)" + } else { + println("good $url ${live.statusLine.statusCode}") + } + } + + else -> { + println("good $url ${live.statusLine.statusCode}") + } + } + } catch (e: Exception) { + e.printStackTrace() + return "$url (exception: ${e.message})" + } + return null + } + + private fun formUrl(host: String, url: String): String { + val finalSection = url.indexOfLast { c -> c == '/' } + 1 + + val paramsIndex = if (url.contains('?')) { + url.indexOfLast { c -> c == '?' } + } else { + url.length + } + + val prefix = url.substring(0, finalSection) + val suffix = url.substring(finalSection, paramsIndex) + val query = url.substring(paramsIndex) + + val query2 = if (query.contains("vendor")) { + query + } else if (query.isEmpty()) { + "?vendor=${vendor}" + } else { + "$query&vendor=${vendor}" + } + + val urlEncoded = URLEncoder.encode(suffix, "UTF-8") + + val url2 = if (host.contains("adoptium") && urlEncoded.contains("adoptopenjdk")) { + urlEncoded.replace("adoptopenjdk", "eclipse") + } else { + urlEncoded + } + val normalizedHost = host.removeSuffix("/") + return "$normalizedHost$prefix$url2$query2" + } + + private fun compareJsonData( + live: HttpResponse, + staging: HttpResponse, + url: String, + stagingUrl: String, + liveUrl: String + ): String? { + val liveJsonCleaned = getData(live) + val stagingJsonCleaned = getData(staging) + + removeOutOfSyncData(liveJsonCleaned, stagingJsonCleaned) + + try { + val result: JSONCompareResult = JSONCompare.compareJSON( + liveJsonCleaned?.toString(), + stagingJsonCleaned?.toString(), + JSONCompareMode.NON_EXTENSIBLE + ) + if (result.failed()) { + println("====================") + println("Failed url curl -L -o - \"$stagingUrl\" | grep -v download_count > /tmp/a && curl -L -o - \"$liveUrl\" | grep -v download_count > /tmp/b && diff /tmp/a /tmp/b") + println(result.message) + println("====================") + return "$url (JSON mismatch)" + } else { + println("good $url ${live.statusLine.statusCode}") + } + } catch (e: Exception) { + println("Exception comparing JSON for $url: ${e.message}") + return "$url (JSON comparison exception: ${e.message})" + } + return null + } + + private fun removeOutOfSyncData( + liveJsonCleaned: JsonElement?, + stagingJsonCleaned: JsonElement? + ) { + // if array sizes differ it is likely that staging and live are slightly out of sync, remove elements that are out of sync + if (liveJsonCleaned?.isJsonArray == true && stagingJsonCleaned?.isJsonArray == true) { + while (liveJsonCleaned.asJsonArray.size() > 0 && stagingJsonCleaned.asJsonArray.size() > 0) { + // probably due to updates out of sync, pop off the newer entries + val liveId = liveJsonCleaned.asJsonArray?.get(0)?.asJsonObject?.get("id") + val stagingId = stagingJsonCleaned.asJsonArray?.get(0)?.asJsonObject?.get("id") + + if (liveId != stagingId) { + stagingJsonCleaned.asJsonArray?.remove(0) + liveJsonCleaned.asJsonArray?.size()?.minus(1)?.let { liveJsonCleaned.asJsonArray?.remove(it) } + } else { + break + } + } + } + } + + private fun getData(live: HttpResponse): JsonElement? { + val liveBody = DefaultUpdaterHtmlClient.extractBody(live) + var liveJson = JsonParser.parseString(liveBody) + + // Ignore download counts as they will probably always differ + liveJson = removeFieldElement(liveJson) { key, value -> + key == "download_count" || + key == "aqavit_results_link" || + value.isJsonObject && + value.asJsonObject.getAsJsonPrimitive("vendor")?.asString == "adoptium" || + key == "release_notes" + } + return liveJson + } + + private fun removeFieldElement(jsonValue: JsonElement?, predicate: (String?, JsonElement) -> Boolean): JsonElement? { + return when { + jsonValue == null -> { + null + } + + jsonValue.isJsonNull -> { + jsonValue.asJsonNull + } + + jsonValue.isJsonObject -> { + removeField(jsonValue.asJsonObject, predicate) + } + + jsonValue.isJsonArray -> { + removeField(jsonValue.asJsonArray, predicate) + } + + else -> { + jsonValue.asJsonPrimitive + } + } + } + + private fun removeField(jsonObject: JsonObject?, predicate: (String?, JsonElement) -> Boolean): JsonElement? { + if (jsonObject == null) return null + + val result = JsonObject() + + if (jsonObject.has("dateTime")) { + return JsonPrimitive(jsonObject.get("dateTime").asString) + } + + jsonObject + .entrySet() + .filter { !predicate(it.key, it.value) } + .forEach { + val v = removeFieldElement(it.value, predicate) + if (v != null) result.add(it.key, v) + } + + return result + } + + private fun removeField(array: JsonArray?, predicate: (String?, JsonElement) -> Boolean): JsonArray? { + if (array == null) return null + + val jsonArray = JsonArray() + + array + .filter { !predicate(null, it) } + .forEach { + jsonArray.add(removeFieldElement(it, predicate)) + } + + if (jsonArray.size() == 0) { + return null + } + + return jsonArray + } + + private fun removeFieldElement(s: String, jsonValue: JsonElement?): JsonElement? { + return when { + jsonValue == null -> { + null + } + + jsonValue.isJsonNull -> { + jsonValue.asJsonNull + } + + jsonValue.isJsonObject -> { + removeField(s, jsonValue.asJsonObject) + } + + jsonValue.isJsonArray -> { + removeField(s, jsonValue.asJsonArray) + } + + else -> { + jsonValue.asJsonPrimitive + } + } + } + + private fun removeField(s: String, jsonObject: JsonObject?): JsonElement? { + if (jsonObject == null) return null + + val result = JsonObject() + + if (jsonObject.has("dateTime")) { + return JsonPrimitive(jsonObject.get("dateTime").asString) + } + + jsonObject + .entrySet() + .filter { it.key != s } + .forEach { + result.add(it.key, removeFieldElement(s, it.value)) + } + + return result + } + + private fun removeField(s: String, array: JsonArray?): JsonArray? { + if (array == null) return null + + val jsonArray = JsonArray() + + array.forEach { + jsonArray.add(removeFieldElement(s, it)) + } + + return jsonArray + } + + private val httpClient = HttpClientFactory().getNonRedirectHttpClient() + private val httpClientRedirect = HttpClientFactory().getHttpClient() + + private suspend fun get(url: URL, redirect: Boolean = false): HttpResponse { + val request = org.apache.http.client.methods.RequestBuilder + .get(url.toURI()) + .setConfig(HttpClientFactory.REQUEST_CONFIG) + .build() + + var client = if (redirect) httpClientRedirect else httpClient + + return suspendCoroutine { continuation -> + client.execute( + request, + object : FutureCallback { + override fun completed(p0: HttpResponse) { + continuation.resumeWith(Result.success(p0)) + } + + override fun failed(p0: Exception) { + continuation.resumeWith(Result.failure(p0)) + } + + override fun cancelled() { + continuation.resumeWith(Result.failure(java.util.concurrent.CancellationException("Request cancelled"))) + } + } + ) + } + } +} diff --git a/adoptium-api-versions/pom.xml b/adoptium-api-versions/pom.xml index d85729edd..e82618dcf 100644 --- a/adoptium-api-versions/pom.xml +++ b/adoptium-api-versions/pom.xml @@ -9,20 +9,20 @@ 1.11.0 - 2.21.3 - 2.21 + 2.22.1 + 2.22 17 5.1.2 17 - 2.3.21 - 1.5.32 + 2.4.0 + 1.5.38 17 3.9.16 3.9.0 UTF-8 UTF-8 - 3.35.3 - 6.0.0 + 3.37.3 + 6.0.1 @@ -38,7 +38,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.6.2 + 3.6.3 enforce @@ -74,7 +74,7 @@ io.ktor ktor-bom - 3.5.0 + 3.5.1 pom import @@ -122,7 +122,7 @@ io.mockk mockk-jvm - 1.14.9 + 1.14.11 test @@ -187,7 +187,7 @@ org.mongodb mongodb-driver-kotlin-coroutine - 5.7.0 + 5.9.0 @@ -198,7 +198,7 @@ de.flapdoodle.embed de.flapdoodle.embed.mongo - 4.24.0 + 4.33.0 test @@ -245,7 +245,7 @@ org.jooq jooq - 3.21.4 + 3.21.6 org.slf4j @@ -261,7 +261,7 @@ com.microsoft.azure applicationinsights-runtime-attach - 3.7.8 + 3.7.9 io.opentelemetry.javaagent.instrumentation @@ -271,7 +271,7 @@ io.opentelemetry.javaagent.instrumentation opentelemetry-javaagent-netty-4.1 - 2.27.0-alpha + 2.29.0-alpha org.jboss.weld @@ -298,12 +298,12 @@ io.quarkiverse.logging.logback quarkus-logging-logback - 1.1.2 + 1.2.0 io.quarkiverse.logging.logback quarkus-logging-logback-impl - 1.1.2 + 1.2.0 io.quarkus @@ -335,19 +335,19 @@ org.junit.jupiter junit-jupiter - 6.0.3 + 6.1.2 test org.eclipse.jetty jetty-server - 12.1.9 + 12.1.11 test org.eclipse.jetty jetty-client - 12.1.9 + 12.1.11 org.bouncycastle @@ -367,7 +367,7 @@ jakarta.json.bind jakarta.json.bind-api - 3.0.1 + 3.0.2 jakarta.ws.rs diff --git a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt index 6fa987750..7a1ccac4e 100644 --- a/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt +++ b/adoptium-frontend-parent/adoptium-api-v3-frontend/src/test/kotlin/net/adoptium/api/DownloadStatsPathTest.kt @@ -149,9 +149,11 @@ class DownloadStatsPathTest : FrontendTest() { .toCompletableFuture() .get().entity as DownloadStats - assertEquals(170L, stats.total_downloads.total) + val packageBaselineTotal = DownloadStatsInterface.PACKAGE_DOWNLOAD_BASELINE_2026_05_28.values.sum() + assertEquals(170L + packageBaselineTotal, stats.total_downloads.total) assertEquals(100L, stats.total_downloads.docker_pulls) assertEquals(70L, stats.total_downloads.github_downloads) + assertEquals(packageBaselineTotal, stats.total_downloads.package_downloads) assertEquals(30L, stats.github_downloads[8]) assertEquals(40L, stats.github_downloads[11]) assertEquals(60L, stats.docker_pulls["a-repo-name"]) diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClient.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClient.kt index 4576fc474..e4815b4cc 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClient.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClient.kt @@ -51,13 +51,18 @@ class CloudflareClient @Inject constructor( // GraphQL query with variables - static constant for reuse // Pagination uses clientRequestPath_gt to filter out first page + // datetime is NOT included in dimensions so each unique path appears only once + // (counts aggregated across the entire date range), making clientRequestPath_gt + // pagination correct. Previously, ordering by [datetime_ASC, clientRequestPath_ASC] + // with clientRequestPath_gt pagination caused data loss for entries with later + // datetimes but paths <= the last path from the previous page. private const val GRAPHQL_QUERY = """ query (${'$'}zoneTag: String!, ${'$'}limit: Int!, ${'$'}startDate: Time!, ${'$'}endDate: Time!, ${'$'}lastPath: String) { viewer { zones(filter: { zoneTag: ${'$'}zoneTag }) { httpRequestsAdaptiveGroups( limit: ${'$'}limit, - orderBy: [datetime_ASC, clientRequestPath_ASC], + orderBy: [clientRequestPath_ASC], filter: { datetime_geq: ${'$'}startDate, datetime_lt: ${'$'}endDate, @@ -78,7 +83,6 @@ class CloudflareClient @Inject constructor( ) { count dimensions { - datetime clientRequestPath } } diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponse.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponse.kt index 928261f89..d183bb7b2 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponse.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponse.kt @@ -2,19 +2,16 @@ package net.adoptium.api.v3.stats.cloudflare import com.fasterxml.jackson.databind.ObjectMapper import org.slf4j.LoggerFactory -import java.time.Instant -import java.time.temporal.ChronoUnit data class CloudflarePackageStats( - val datetime: Instant, val count: Long, val path: String ) : Comparable { companion object { private val COMPARATOR: Comparator = compareBy( - { it.datetime }, { it.path }, { it.count } + { it.path }, { it.count } ) } @@ -164,15 +161,14 @@ data class CloudflareResponse( val datetimeStr = dimensions.path(ResponseKey.DATETIME).asText() val path = dimensions.path(ResponseKey.PATH).asText() - if (datetimeStr.isNullOrBlank() || path.isNullOrBlank()) { + if (path.isNullOrBlank()) { LOGGER.warn("There is a group with missing information $group") continue } val count = countNode.asLong() if (count > 0) { - val datetime = Instant.parse(datetimeStr).truncatedTo(ChronoUnit.MINUTES) - dataList.add(CloudflarePackageStats(datetime, count, path)) + dataList.add(CloudflarePackageStats(count, path)) } } } @@ -186,12 +182,11 @@ data class CloudflareResponse( */ fun merge(other: CloudflareResponse): CloudflareResponse { val mergedData = (this.data + other.data) - .groupBy { it.path to it.datetime } - .map { (key, entries) -> + .groupBy { it.path } + .map { (path, entries) -> CloudflarePackageStats( - datetime = key.second, count = entries.sumOf { it.count }, - path = key.first + path = path ) } .toSet() diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculator.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculator.kt index 798fd83eb..c46b01872 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculator.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculator.kt @@ -6,7 +6,6 @@ import net.adoptium.api.v3.TimeSource import net.adoptium.api.v3.dataSources.persitence.ApiPersistence import net.adoptium.api.v3.models.CloudflarePackageDownloadStatsDbEntry import org.slf4j.LoggerFactory -import java.time.ZoneId @ApplicationScoped open class CloudflareStatsCalculator @Inject constructor( @@ -34,29 +33,30 @@ open class CloudflareStatsCalculator @Inject constructor( suspend fun updateDb() { try { - val endDate = TimeSource.now() + // Fetch complete previous day data (midnight yesterday to midnight today) + // This ensures we always get a full day of data and avoids partial/duplicate entries + val endDate = TimeSource.now().toLocalDate().atStartOfDay(TimeSource.ZONE) val startDate = endDate.minusDays(1) LOGGER.info("Fetching CloudFlare package stats from $startDate to $endDate") val response = cloudFlareClient.fetchDownloadStats(startDate, endDate) - + // Group by version only (datetime is no longer part of the Cloudflare response + // since it's been removed from the query dimensions for correct pagination) val stats = response.data .mapNotNull { stats -> val version = extractVersionFromPath(stats.path) version?.let { stats to version } } - .groupingBy { (stats, version) -> - stats.datetime.atZone(TimeSource.ZONE) to version - } + .groupingBy { (_, version) -> version } .fold(0L) { currentTotal, element -> currentTotal + element.first.count } - .map { (key, totalDownloads) -> + .map { (version, totalDownloads) -> CloudflarePackageDownloadStatsDbEntry( - date = key.first, - feature_version = key.second, + date = endDate, + feature_version = version, downloads = totalDownloads ) } diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/DownloadStatsInterfaceTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/DownloadStatsInterfaceTest.kt index bb31d8e4c..0d8486f68 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/DownloadStatsInterfaceTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/DownloadStatsInterfaceTest.kt @@ -33,9 +33,11 @@ class DownloadStatsInterfaceTest { val downloadStatsInterface = DownloadStatsInterface(apiPersistenceMock, UpdatableVersionSupplierStub()) val stats = downloadStatsInterface.getTotalDownloadStats() - assertEquals(1500L, stats.total_downloads.package_downloads) - assertEquals(1000L, stats.package_downloads[17]) - assertEquals(500L, stats.package_downloads[21]) + val baseline = DownloadStatsInterface.PACKAGE_DOWNLOAD_BASELINE_2026_05_28 + val baselineTotal = baseline.values.sum() + assertEquals(1500L + baselineTotal, stats.total_downloads.package_downloads) + assertEquals(1000L + (baseline[17] ?: 0L), stats.package_downloads[17]) + assertEquals(500L + (baseline[21] ?: 0L), stats.package_downloads[21]) } } @@ -62,18 +64,20 @@ class DownloadStatsInterfaceTest { val downloadStatsInterface = DownloadStatsInterface(apiPersistenceMock, UpdatableVersionSupplierStub()) val stats = downloadStatsInterface.getTotalDownloadStats() - // Package: 1000(v17) + 100(v21) = 1100 - assertEquals(1000L, stats.package_downloads[17]) - assertEquals(100L, stats.package_downloads[21]) + val baseline = DownloadStatsInterface.PACKAGE_DOWNLOAD_BASELINE_2026_05_28 + val baselineTotal = baseline.values.sum() + // Package: 1000(v17) + 100(v21) + baseline + assertEquals(1000L + (baseline[17] ?: 0L), stats.package_downloads[17]) + assertEquals(100L + (baseline[21] ?: 0L), stats.package_downloads[21]) // GitHub: 200(v8) + 100(v11) = 300 // Docker: 500 - // Total: 300 + 500 + 1100 = 1900 - assertEquals(1900L, stats.total_downloads.total) + // Total: 300 + 500 + 1100 + baselineTotal + assertEquals(1900L + baselineTotal, stats.total_downloads.total) } } @Test - fun `package stats should return zero when no entries exist`() { + fun `package stats should return baseline-only total when no tracked entries exist`() { runBlocking { val apiPersistenceMock = mockk() @@ -84,7 +88,38 @@ class DownloadStatsInterfaceTest { val downloadStatsInterface = DownloadStatsInterface(apiPersistenceMock, UpdatableVersionSupplierStub()) val stats = downloadStatsInterface.getTotalDownloadStats() - assertEquals(0L, stats.total_downloads.package_downloads) + val baselineTotal = DownloadStatsInterface.PACKAGE_DOWNLOAD_BASELINE_2026_05_28.values.sum() + assertEquals(baselineTotal, stats.total_downloads.package_downloads) + } + } + + @Test + fun `package stats should include cumulative baseline added to tracked downloads`() { + runBlocking { + val apiPersistenceMock = mockk() + val baseTime = TimeSource.now() + + coEvery { apiPersistenceMock.getLatestAllDockerStats() } returns emptyList() + coEvery { apiPersistenceMock.getLatestGithubStatsForFeatureVersion(any()) } returns null + coEvery { apiPersistenceMock.getAggregatedPackageStats(any(), any()) } returns listOf( + CloudflarePackageDownloadStatsDbEntry(baseTime.minusMinutes(1), 1000, 17), + CloudflarePackageDownloadStatsDbEntry(baseTime.minusMinutes(1), 500, 21) + ) + + // Currently the baseline placeholder is empty; verify breakdown matches tracked stats. + // This guards the cumulative-sum semantics (baseline + tracked) and will continue to + // pass once a non-empty baseline is populated for additional feature versions. + val downloadStatsInterface = DownloadStatsInterface(apiPersistenceMock, UpdatableVersionSupplierStub()) + val stats = downloadStatsInterface.getTotalDownloadStats() + + val baseline = DownloadStatsInterface.PACKAGE_DOWNLOAD_BASELINE_2026_05_28 + val expected17 = 1000L + (baseline[17] ?: 0L) + val expected21 = 500L + (baseline[21] ?: 0L) + val expectedTotal = expected17 + expected21 + baseline.filterKeys { it != 17 && it != 21 }.values.sum() + + assertEquals(expected17, stats.package_downloads[17]) + assertEquals(expected21, stats.package_downloads[21]) + assertEquals(expectedTotal, stats.total_downloads.package_downloads) } } } diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/V3UpdaterEndToEndTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/V3UpdaterEndToEndTest.kt index 2ded667a7..dde2d3306 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/V3UpdaterEndToEndTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/V3UpdaterEndToEndTest.kt @@ -464,6 +464,29 @@ class V3UpdaterEndToEndTest { } } + @Test + fun `cdxa summary query failure does not delete CDXAs`() { + runBlocking { + val cdxaRepo = AdoptCdxaReposTestDataGenerator.generate() + + // Simulate a failure in getCdxaSummary (e.g., spurious HTTP 401) + val getCdxaSummary: ((String, String, String) -> GHCdxaRepoSummaryData?) = { org, repo, directory -> + LOGGER.info("getCdxaSummary: simulating failure for $org/$repo/$directory") + throw Exception("Simulated HTTP 401 Unauthorized error for $org/$repo/$directory") + } + + val getCdxaByName: ((String, String, String) -> GHCdxa?) = { _, _, _ -> + null + } + + val updatedRepo = runCdxaUpdateTest(cdxaRepo, getCdxaSummary, getCdxaByName) + + // CDXAs must NOT be deleted when the summary query fails + assertTrue(updatedRepo.repos.size == cdxaRepo.repos.size) + assertTrue(updatedRepo == cdxaRepo) + } + } + @Test fun `removed release disappears`() { runBlocking { diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClientTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClientTest.kt index 1f2c0e818..77044c64d 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClientTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareClientTest.kt @@ -11,18 +11,16 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.time.Instant -import java.time.ZoneId import java.time.temporal.ChronoUnit class CloudflareClientTest { @Test fun `should successfully fetch and aggregate paginated results`() = runBlocking { - val testDateTime = Instant.parse("2024-01-15T00:00:00Z") val response = CloudflareResponse( data = setOf( - CloudflarePackageStats(testDateTime, 100, "/artifactory/deb/pool/main/t/temurin-17/temurin-17-jdk_17.0.10_amd64.deb"), - CloudflarePackageStats(testDateTime, 50, "/artifactory/rpm/centos/7/x86_64/Packages/temurin-17-jdk-17.0.10.x86_64.rpm") + CloudflarePackageStats(100, "/artifactory/deb/pool/main/t/temurin-17/temurin-17-jdk_17.0.10_amd64.deb"), + CloudflarePackageStats(50, "/artifactory/rpm/centos/7/x86_64/Packages/temurin-17-jdk-17.0.10.x86_64.rpm") ) ) @@ -34,8 +32,8 @@ class CloudflareClientTest { calculator.updateDb() - val startTime = testDateTime.minus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) - val endTime = testDateTime.plus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) + val startTime = Instant.EPOCH.atZone(TimeSource.ZONE) + val endTime = Instant.now().plus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) val savedStats = database.getPackageStats(startTime, endTime) assertEquals(1, savedStats.size) @@ -58,16 +56,15 @@ class CloudflareClientTest { @Test fun `CloudflareResponse merge should keep unique entries separate and keep latest copy`() { - val datetime = Instant.parse("2024-01-15T00:00:00Z") val response1 = CloudflareResponse( setOf( - CloudflarePackageStats(datetime, 50, "/path/duplicated.deb"), - CloudflarePackageStats(datetime, 100, "/path/one.deb")) + CloudflarePackageStats(50, "/path/duplicated.deb"), + CloudflarePackageStats(100, "/path/one.deb")) ) val response2 = CloudflareResponse( setOf( - CloudflarePackageStats(datetime, 50, "/path/duplicated.deb"), - CloudflarePackageStats(datetime, 200, "/path/two.deb") + CloudflarePackageStats(50, "/path/duplicated.deb"), + CloudflarePackageStats(200, "/path/two.deb") ) ) diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponseTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponseTest.kt index d07f70260..4d5842c74 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponseTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareResponseTest.kt @@ -4,12 +4,10 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import java.time.Instant class CloudflareResponseTest { companion object { - private val TEST_DATETIME = Instant.parse("2025-03-24T00:00:00Z") private val SAMPLE_RESPONSE = """ { "data": { @@ -20,15 +18,13 @@ class CloudflareResponseTest { { "count": 84, "dimensions": { - "clientRequestPath": "/artifactory/apk/alpine/main/x86_64/temurin-21-jdk-21.0.10_p7-r0.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/artifactory/apk/alpine/main/x86_64/temurin-21-jdk-21.0.10_p7-r0.apk" } }, { "count": 58, "dimensions": { - "clientRequestPath": "/artifactory/apk/alpine/main/x86_64/temurin-21-jre-21.0.10_p7-r0.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/artifactory/apk/alpine/main/x86_64/temurin-21-jre-21.0.10_p7-r0.apk" } } ] @@ -100,8 +96,7 @@ class CloudflareResponseTest { "httpRequestsAdaptiveGroups": [ { "dimensions": { - "clientRequestPath": "/test.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/test.apk" } } ] @@ -152,15 +147,13 @@ class CloudflareResponseTest { { "count": 0, "dimensions": { - "clientRequestPath": "/zero.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/zero.apk" } }, { "count": 100, "dimensions": { - "clientRequestPath": "/valid.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/valid.apk" } } ] @@ -189,15 +182,13 @@ class CloudflareResponseTest { { "count": 84, "dimensions": { - "clientRequestPath": "", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "" } }, { "count": 100, "dimensions": { - "clientRequestPath": "/valid.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/valid.apk" } } ] @@ -214,7 +205,7 @@ class CloudflareResponseTest { } @Test - fun `from json should handle entries with blank datetime`() { + fun `from json should handle entries without datetime dimension`() { val json = """ { "data": { @@ -225,15 +216,13 @@ class CloudflareResponseTest { { "count": 84, "dimensions": { - "clientRequestPath": "/test.apk", - "datetime": "" + "clientRequestPath": "/test.apk" } }, { "count": 100, "dimensions": { - "clientRequestPath": "/valid.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/valid.apk" } } ] @@ -245,8 +234,9 @@ class CloudflareResponseTest { """.trimIndent() val response = CloudflareResponse.fromJson(json) - assertEquals(1, response.data.size) - assertEquals("/valid.apk", response.data.first().path) + assertEquals(2, response.data.size) + assertTrue(response.data.any { it.path == "/test.apk" && it.count == 84L }) + assertTrue(response.data.any { it.path == "/valid.apk" && it.count == 100L }) } @Test @@ -261,8 +251,7 @@ class CloudflareResponseTest { { "count": 50, "dimensions": { - "clientRequestPath": "/zone1.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/zone1.apk" } } ] @@ -272,8 +261,7 @@ class CloudflareResponseTest { { "count": 75, "dimensions": { - "clientRequestPath": "/zone2.apk", - "datetime": "2026-02-25T00:00:00Z" + "clientRequestPath": "/zone2.apk" } } ] @@ -292,17 +280,17 @@ class CloudflareResponseTest { @Test fun `merge two pages keep only unique elements`() { - val commonItem = CloudflarePackageStats(TEST_DATETIME, 150, "/path/one.apk") + val commonItem = CloudflarePackageStats(150, "/path/one.apk") val firstPage = CloudflareResponse( setOf( commonItem, - CloudflarePackageStats(TEST_DATETIME, 200, "/path/two.deb") + CloudflarePackageStats(200, "/path/two.deb") ) ) val secondPage = CloudflareResponse( setOf( commonItem, - CloudflarePackageStats(TEST_DATETIME, 300, "/path/three.rpm") + CloudflarePackageStats(300, "/path/three.rpm") ) ) @@ -320,7 +308,7 @@ class CloudflareResponseTest { fun `merge should handle empty responses`() { val empty = CloudflareResponse(emptySet()) val nonEmpty = CloudflareResponse( - setOf(CloudflarePackageStats(TEST_DATETIME, 100, "/test.apk")) + setOf(CloudflarePackageStats(100, "/test.apk")) ) assertEquals(0, empty.merge(empty).data.size) @@ -329,7 +317,7 @@ class CloudflareResponseTest { } @Test - fun `from json should truncate datetime to minute precision`() { + fun `from json should parse entries without datetime dimension`() { val json = """ { "data": { @@ -340,15 +328,13 @@ class CloudflareResponseTest { { "count": 5, "dimensions": { - "clientRequestPath": "/test.apk", - "datetime": "2026-03-11T00:00:11Z" + "clientRequestPath": "/test1.apk" } }, { "count": 10, "dimensions": { - "clientRequestPath": "/test.apk", - "datetime": "2026-03-11T00:00:12Z" + "clientRequestPath": "/test2.apk" } } ] @@ -361,9 +347,8 @@ class CloudflareResponseTest { val response = CloudflareResponse.fromJson(json) assertEquals(2, response.data.size) - // Both should be truncated to the same minute - val expectedDateTime = Instant.parse("2026-03-11T00:00:00Z") - assertTrue(response.data.all { it.datetime == expectedDateTime }) + assertTrue(response.data.any { it.path == "/test1.apk" && it.count == 5L }) + assertTrue(response.data.any { it.path == "/test2.apk" && it.count == 10L }) } } diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculatorTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculatorTest.kt index 7a2dc6d39..049fa5f04 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculatorTest.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/v3/stats/cloudflare/CloudflareStatsCalculatorTest.kt @@ -14,7 +14,6 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource import org.junit.jupiter.params.provider.ValueSource import java.time.Instant -import java.time.ZoneId import java.time.temporal.ChronoUnit class CloudflareStatsCalculatorTest { @@ -64,25 +63,22 @@ class CloudflareStatsCalculatorTest { } @Test - fun `updateDb should aggregate downloads by date and version`() = runBlocking { + fun `updateDb should aggregate downloads by version`() = runBlocking { val mockClient = mockk() val database = InMemoryApiPersistence(AdoptRepos(emptyList()), AdoptCdxaRepos(emptyList())) val calculator = CloudflareStatsCalculator(database, mockClient) - val testDateTime = Instant.parse("2024-01-15T00:00:00Z") val response = CloudflareResponse( data = setOf( // Version 17 entries - should aggregate to 150 - CloudflarePackageStats(testDateTime, 100, "/artifactory/deb/pool/main/t/temurin-17/temurin-17-jdk_17.0.10_amd64.deb"), - CloudflarePackageStats(testDateTime, 50, "/artifactory/rpm/centos/7/x86_64/Packages/temurin-17-jdk-17.0.10.x86_64.rpm"), + CloudflarePackageStats(100, "/artifactory/deb/pool/main/t/temurin-17/temurin-17-jdk_17.0.10_amd64.deb"), + CloudflarePackageStats(50, "/artifactory/rpm/centos/7/x86_64/Packages/temurin-17-jdk-17.0.10.x86_64.rpm"), // Version 21 entries - should aggregate to 300 - CloudflarePackageStats(testDateTime, 200, "/artifactory/deb/pool/main/t/temurin-21/temurin-21-jdk_21.0.2_amd64.deb"), - CloudflarePackageStats(testDateTime, 100, "/artifactory/apk/alpine/main/x86_64/temurin-21-jdk-21.0.2.apk"), + CloudflarePackageStats(200, "/artifactory/deb/pool/main/t/temurin-21/temurin-21-jdk_21.0.2_amd64.deb"), + CloudflarePackageStats(100, "/artifactory/apk/alpine/main/x86_64/temurin-21-jdk-21.0.2.apk"), // Invalid entry - should be skipped - CloudflarePackageStats(testDateTime, 999, "/invalid/path"), - // Different date - should be separate entry - CloudflarePackageStats(testDateTime.plus(1, ChronoUnit.DAYS), 75, "/artifactory/deb/pool/main/t/temurin-17/temurin-17-jdk_17.0.10_amd64.deb") + CloudflarePackageStats(999, "/invalid/path") ) ) @@ -90,27 +86,22 @@ class CloudflareStatsCalculatorTest { calculator.updateDb() - val startTime = testDateTime.minus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) - val endTime = testDateTime.plus(2, ChronoUnit.DAYS).atZone(TimeSource.ZONE) + val startTime = Instant.EPOCH.atZone(TimeSource.ZONE) + val endTime = Instant.now().plus(2, ChronoUnit.DAYS).atZone(TimeSource.ZONE) val savedStats = database.getPackageStats(startTime, endTime) - assertEquals(3, savedStats.size) + assertEquals(2, savedStats.size) - // Find stats for version 17 on first datetime - val version17Stats = savedStats.find { it.feature_version == 17 && it.date.toInstant() == testDateTime } + // Find stats for version 17 + val version17Stats = savedStats.find { it.feature_version == 17 } assertNotNull(version17Stats) assertEquals(150, version17Stats!!.downloads) - // Find stats for version 21 on first datetime - val version21Stats = savedStats.find { it.feature_version == 21 && it.date.toInstant() == testDateTime } + // Find stats for version 21 + val version21Stats = savedStats.find { it.feature_version == 21 } assertNotNull(version21Stats) assertEquals(300, version21Stats!!.downloads) - // Find stats for version 17 on second datetime - val version17NextDayStats = savedStats.find { it.feature_version == 17 && it.date.toInstant() == testDateTime.plus(1, ChronoUnit.DAYS) } - assertNotNull(version17NextDayStats) - assertEquals(75, version17NextDayStats!!.downloads) - coVerify(exactly = 1) { mockClient.fetchDownloadStats(any(), any()) } } @@ -128,7 +119,7 @@ class CloudflareStatsCalculatorTest { val savedStats = database.getPackageStats( - Instant.now().minus(7, ChronoUnit.DAYS).atZone(TimeSource.ZONE), + Instant.EPOCH.atZone(TimeSource.ZONE), Instant.now().plus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) ) assertTrue(savedStats.isEmpty()) @@ -141,14 +132,13 @@ class CloudflareStatsCalculatorTest { val database = InMemoryApiPersistence(AdoptRepos(emptyList()), AdoptCdxaRepos(emptyList())) val calculator = CloudflareStatsCalculator(database, mockClient) - val testDateTime = Instant.parse("2024-01-15T00:00:00Z") val response = CloudflareResponse( data = setOf( - CloudflarePackageStats(testDateTime, 100, "/artifactory/rpm/centos/7/x86_64/Packages/temurin-17-jdk-17.0.18.0.0.8-1.x86_64.rpm"), - CloudflarePackageStats(testDateTime, 50, "/invalid/path1"), - CloudflarePackageStats(testDateTime, 200, "/artifactory/deb/pool/main/t/temurin-11/temurin-11-jdk_11.0.30.0.0%2b7-0_amd64.deb"), - CloudflarePackageStats(testDateTime, 75, "/another/invalid"), - CloudflarePackageStats(testDateTime, 150, "/artifactory/apk/alpine/main/x86_64/temurin-8-jdk-8.482.08-r0.apk") + CloudflarePackageStats(100, "/artifactory/rpm/centos/7/x86_64/Packages/temurin-17-jdk-17.0.18.0.0.8-1.x86_64.rpm"), + CloudflarePackageStats(50, "/invalid/path1"), + CloudflarePackageStats(200, "/artifactory/deb/pool/main/t/temurin-11/temurin-11-jdk_11.0.30.0.0%2b7-0_amd64.deb"), + CloudflarePackageStats(75, "/another/invalid"), + CloudflarePackageStats(150, "/artifactory/apk/alpine/main/x86_64/temurin-8-jdk-8.482.08-r0.apk") ) ) @@ -159,8 +149,8 @@ class CloudflareStatsCalculatorTest { val savedStats = database.getPackageStats( - testDateTime.minus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE), - testDateTime.plus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) + Instant.EPOCH.atZone(TimeSource.ZONE), + Instant.now().plus(1, ChronoUnit.DAYS).atZone(TimeSource.ZONE) ) assertEquals(3, savedStats.size) diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubCdxaSummaryClient.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubCdxaSummaryClient.kt index 8a1c09f96..c4df6211f 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubCdxaSummaryClient.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubCdxaSummaryClient.kt @@ -28,18 +28,33 @@ open class GraphQLGitHubCdxaSummaryClient @Inject constructor( graphQLGitHubInterface.queryApi(query::withCursor, null) } catch (e: java.lang.Exception) { LOGGER.error("Exception on cdxa summary query $org/$repo/$directory :"+e) - return null + /* + * Re-throwing here makes transient GraphQL/API failures visible to the caller instead of collapsing them + * into null. That matters because callers were interpreting null as “no CDXAs exist” and deleting persisted + * records, whereas an exception preserves the current DB state and allows retry/recovery higher up. + */ + throw e } val ghCdxaRepoSummary: GHCdxaRepoSummaryData? = try { if (result == null || result?.data == null) { - return null + /* + * This keeps the valid “no summary / folder missing” path returning null, while separating it from actual + * request failures. In other words, empty data is still treated as an empty result, but transport/query errors + * are no longer mistaken for absence of CDXAs. + */ + null } else { result.data } } catch (e: java.lang.Exception) { LOGGER.error("Exception mapping cdxa summary query response $org/$repo/$directory :"+e+" query result: "+result) - return null + /* + * Same rationale as above: if response parsing/mapping fails, that is an operational error, not a + * legitimate empty result. Re-throwing prevents malformed or partial responses from triggering the + * same deletion path as a real “nothing found” case. + */ + throw e } return ghCdxaRepoSummary diff --git a/pom.xml b/pom.xml index c9e740e54..200c54a66 100644 --- a/pom.xml +++ b/pom.xml @@ -16,10 +16,10 @@ 1.7.1 - 0.8.14 + 0.8.15 17 17 - 2.3.21 + 2.4.0 3.2.0 3.8.0 3.6.1 @@ -32,14 +32,14 @@ 17 17 17 - 3.10.0 + 3.11.0 2.0.4 3.1.4 1.0 - 3.6.2 + 3.6.3 3.6.3 - 3.5.0 - 3.5.1 + 3.5.1 + 3.5.2 3.1.4 0.8.10 3.5.0 @@ -49,8 +49,8 @@ 3.2.0 3.6.0 2.7.1 - 7.22.0 - 1.23.1 + 7.23.0 + 1.25.7 3.28.0 3.9.0 0.18 @@ -59,8 +59,8 @@ 2.2.1 4.0.0-M16 3.4.0 - 4.9.8.3 - 3.5.5 + 4.10.3.0 + 3.5.6 1.0.10 1.4.0 3.2.0 @@ -74,7 +74,7 @@ 4.7.3 UTF-8 UTF-8 - 3.35.3 + 3.37.3 @@ -660,4 +660,13 @@ + + + + staging-checker + + adoptium-api-v3-staging-checker + + +