fix(ci): create buildx builder in the build step's DOCKER_CONFIG #5323
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pull Request | |
| env: | |
| # TODO: unhardcode this | |
| REGISTRY: iad.ocir.io/idyksih5sir9/cozystack | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| # Cancel in‑flight runs for the same PR when a new push arrives. | |
| concurrency: | |
| group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| detect-changes: | |
| name: Detect changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| code: ${{ steps.filter.outputs.code }} | |
| steps: | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| code: | |
| - '!docs/**' | |
| build: | |
| name: Build | |
| # Run each Build on its own ephemeral VM instead of the shared self-hosted | |
| # runner. The shared host runs 4 runner agents against one dockerd-embedded | |
| # buildkit; concurrent build jobs serialize on buildkit's single-writer bbolt | |
| # cache lock + exporter mutex and stall to the 30-min timeout (proven via a | |
| # live SIGUSR1 goroutine dump). One VM per job => one buildkit per job => that | |
| # contention is severed by construction. `make build` is serial (one image at | |
| # a time), so a small shape suffices -- 4cpu/16gb, not the 24cpu the e2e job | |
| # uses -- and it leans on the warm mode=max cache (#2938) to stay under the | |
| # 30-min wall. The `debug` label still routes to self-hosted for the | |
| # breakpoint path. Phase 0.5 of #2937. | |
| runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-4cpu-16gb-x86-64' }} | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| packages: write | |
| needs: ["detect-changes"] | |
| # Never run when the PR carries the "release" label or only docs changed. | |
| if: | | |
| needs.detect-changes.outputs.code == 'true' | |
| && !contains(github.event.pull_request.labels.*.name, 'release') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| # Ephemeral runners don't carry two tools the persistent self-hosted | |
| # runner accreted: the helm-unittest plugin (make unit-tests) and the | |
| # flux CLI (the installer's `flux push artifact` publishes the | |
| # cozystack-packages OCI artifact the operator pulls). Install if absent | |
| # (idempotent; the debug/self-hosted path already has them). Durable fix: | |
| # bake both into the oracle-vm runner image, then drop this step. | |
| - name: Set up build toolchain | |
| run: | | |
| helm plugin list 2>/dev/null | grep -q unittest \ | |
| || helm plugin install https://github.com/helm-unittest/helm-unittest | |
| command -v flux >/dev/null \ | |
| || curl -s https://fluxcd.io/install.sh | sudo bash | |
| - name: Run unit tests | |
| run: make unit-tests | |
| - name: Run controller Go tests | |
| run: make test-controllers | |
| - name: Set up Docker config | |
| run: | | |
| if [ -d ~/.docker ]; then | |
| cp -r ~/.docker "${{ runner.temp }}/.docker" | |
| fi | |
| - name: Login to GitHub Container Registry | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.OCIR_USER}} | |
| password: ${{ secrets.OCIR_TOKEN }} | |
| registry: iad.ocir.io | |
| env: | |
| DOCKER_CONFIG: ${{ runner.temp }}/.docker | |
| - name: Build | |
| run: make build | |
| env: | |
| DOCKER_CONFIG: ${{ runner.temp }}/.docker | |
| # Unique per (PR, commit) so concurrent PR builds never push the same | |
| # tag. PUBLISH_VERSIONED/PUBLISH_FLOATING stay off — PR images are | |
| # consumed by digest, and :latest belongs to release builds. | |
| IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }} | |
| PUBLISH_VERSIONED: '0' | |
| PUBLISH_FLOATING: '0' | |
| - name: Build Talos image | |
| run: make -C packages/core/talos talos-nocloud | |
| - name: Save git diff as patch | |
| if: "!contains(github.event.pull_request.labels.*.name, 'release')" | |
| run: git diff HEAD > _out/assets/pr.patch | |
| - name: Upload git diff patch | |
| if: "!contains(github.event.pull_request.labels.*.name, 'release')" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-patch | |
| path: _out/assets/pr.patch | |
| - name: Upload Talos image | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: talos-image | |
| path: _out/assets/nocloud-amd64.raw.xz | |
| resolve_assets: | |
| name: "Resolve assets" | |
| runs-on: ubuntu-latest | |
| if: contains(github.event.pull_request.labels.*.name, 'release') | |
| outputs: | |
| disk_id: ${{ steps.fetch_assets.outputs.disk_id }} | |
| steps: | |
| - name: Generate GitHub App token | |
| id: app-token | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} | |
| private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} | |
| owner: cozystack | |
| - name: Checkout code | |
| if: contains(github.event.pull_request.labels.*.name, 'release') | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Extract tag from PR branch (release PR) | |
| if: contains(github.event.pull_request.labels.*.name, 'release') | |
| id: get_tag | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const branch = context.payload.pull_request.head.ref; | |
| const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/); | |
| if (!m) { | |
| core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`); | |
| return; | |
| } | |
| core.setOutput('tag', `v${m[1]}`); | |
| - name: Find draft release & asset IDs (release PR) | |
| if: contains(github.event.pull_request.labels.*.name, 'release') | |
| id: fetch_assets | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ steps.app-token.outputs.token }} | |
| script: | | |
| const tag = '${{ steps.get_tag.outputs.tag }}'; | |
| const releases = await github.rest.repos.listReleases({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| per_page: 100 | |
| }); | |
| const draft = releases.data.find(r => r.tag_name === tag && r.draft); | |
| if (!draft) { | |
| core.setFailed(`Draft release '${tag}' not found`); | |
| return; | |
| } | |
| const find = (n) => draft.assets.find(a => a.name === n)?.id; | |
| const diskId = find('nocloud-amd64.raw.xz'); | |
| if (!diskId) { | |
| core.setFailed('Required assets missing in draft release'); | |
| return; | |
| } | |
| core.setOutput('disk_id', diskId); | |
| e2e: | |
| name: "E2E Tests" | |
| runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }} | |
| #runs-on: [oracle-vm-32cpu-128gb-x86-64] | |
| timeout-minutes: 180 | |
| permissions: | |
| contents: read | |
| packages: read | |
| checks: write | |
| needs: ["build", "resolve_assets"] | |
| if: ${{ always() && (needs.build.result == 'success' || needs.resolve_assets.result == 'success') }} | |
| steps: | |
| - name: Generate GitHub App token | |
| if: contains(github.event.pull_request.labels.*.name, 'release') | |
| id: app-token | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} | |
| private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} | |
| owner: cozystack | |
| # ▸ Checkout and prepare the codebase | |
| # fetch-depth: 0 is required by the "Select E2E tests" step so that | |
| # `git diff origin/${BASE_REF}...HEAD` can resolve the base ref. | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # ▸ Regular PR path – download artefacts produced by the *build* job | |
| - name: "Download Talos image (regular PR)" | |
| if: "!contains(github.event.pull_request.labels.*.name, 'release')" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: talos-image | |
| path: _out/assets | |
| - name: Download PR patch | |
| if: "!contains(github.event.pull_request.labels.*.name, 'release')" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pr-patch | |
| path: _out/assets | |
| - name: Apply patch | |
| if: "!contains(github.event.pull_request.labels.*.name, 'release')" | |
| run: | | |
| git apply _out/assets/pr.patch | |
| # ▸ Release PR path – fetch artefacts from the corresponding draft release | |
| - name: Download assets from draft release (release PR) | |
| if: contains(github.event.pull_request.labels.*.name, 'release') | |
| run: | | |
| mkdir -p _out/assets | |
| curl -sSL -H "Authorization: token ${APP_TOKEN}" -H "Accept: application/octet-stream" \ | |
| -o _out/assets/nocloud-amd64.raw.xz \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ needs.resolve_assets.outputs.disk_id }}" | |
| env: | |
| APP_TOKEN: ${{ steps.app-token.outputs.token }} | |
| - name: Set sandbox ID | |
| run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV | |
| # ▸ Prepare environment | |
| - name: Prepare workspace | |
| run: | | |
| rm -rf /tmp/$SANDBOX_NAME | |
| cp -r ${{ github.workspace }} /tmp/$SANDBOX_NAME | |
| - name: Prepare environment | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| attempt=0 | |
| until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do | |
| attempt=$((attempt + 1)) | |
| if [ $attempt -ge 3 ]; then | |
| echo "Attempt $attempt failed, exiting..." | |
| exit 1 | |
| fi | |
| echo "Attempt $attempt failed, retrying..." | |
| done | |
| echo "Prepare environment completed after $((attempt + 1)) attempts" | |
| # ▸ Install Cozystack | |
| - name: Install Cozystack into sandbox | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then | |
| echo "❌ Install failed (no retry — see diagnostics below)" | |
| echo "::group::Diagnostics: HelmRelease status" | |
| docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true | |
| echo "::endgroup::" | |
| echo "::group::Diagnostics: not-Ready HelmReleases (describe)" | |
| docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \ | |
| | while read -r ns name; do | |
| echo "--- $ns/$name ---" | |
| docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true | |
| done | |
| echo "::endgroup::" | |
| echo "::group::Diagnostics: recent events" | |
| docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true | |
| echo "::endgroup::" | |
| exit 1 | |
| fi | |
| # Test Impact Analysis (TIA): walk the PackageSource graph for the PR | |
| # diff and select only the bats files affected. Runs by default; opt | |
| # out via the `full-e2e` label to run the full suite. | |
| # `skip=true` means nothing to test (docs-only PR). | |
| - name: Select E2E tests | |
| id: select | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }} | |
| env: | |
| BASE_REF: ${{ github.base_ref }} | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt | |
| apps=$(./hack/select-e2e.sh /tmp/changed.txt) | |
| echo "Selected apps: ${apps:-<none>}" | |
| echo "apps=${apps}" >> $GITHUB_OUTPUT | |
| if [ -z "${apps}" ]; then | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run OpenAPI tests | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi | |
| # Run E2E tests. With `full-e2e` label the selector is skipped, its | |
| # outputs are empty, the != 'true' guard lets this step run, and the | |
| # full bats list is used. Without the label, the selector decides; | |
| # an empty selection (docs-only) skips this step entirely. | |
| - name: Run E2E tests | |
| id: e2e_tests | |
| if: ${{ steps.select.outputs.skip != 'true' }} | |
| env: | |
| SELECTED_APPS: ${{ steps.select.outputs.apps }} | |
| FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }} | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| if [ "$FULL_E2E" = "true" ] || [ -z "$SELECTED_APPS" ]; then | |
| apps_list=$(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1) | |
| else | |
| apps_list="$SELECTED_APPS" | |
| fi | |
| failed_tests="" | |
| for app in $apps_list; do | |
| echo "::group::Testing $app" | |
| if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then | |
| echo "::endgroup::" | |
| echo "✅ Test $app completed successfully" | |
| else | |
| echo "::endgroup::" | |
| echo "❌ Test $app failed (no retry — see diagnostics below)" | |
| failed_tests="$failed_tests $app" | |
| echo "::group::Diagnostics for $app" | |
| docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true | |
| docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true | |
| # COSI state at the moment of failure — the CRDs ship no printer | |
| # columns, so pull the readiness fields explicitly (cozyreport | |
| # captures the same at end-of-job, but bucket states move between | |
| # a test failing and the report being taken). | |
| docker exec "$SANDBOX_NAME" sh -c ' | |
| kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1 | |
| kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1 | |
| kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1 | |
| ' || true | |
| echo "::endgroup::" | |
| fi | |
| done | |
| if [ -n "$failed_tests" ]; then | |
| echo "❌ Failed tests:$failed_tests" | |
| exit 1 | |
| fi | |
| echo "✅ All E2E tests passed" | |
| # ▸ Collect debug information (always runs) | |
| - name: Collect report | |
| if: always() | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true | |
| - name: Upload cozyreport.tgz | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cozyreport | |
| path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz | |
| - name: Collect images list | |
| if: always() | |
| run: | | |
| cd /tmp/$SANDBOX_NAME | |
| make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true | |
| - name: Upload image list | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: image-list | |
| path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt | |
| # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach, | |
| # inspect Talos/Cozystack state and resume with `breakpoint resume`. | |
| # | |
| # Gated by the `debug` or `release` label (release PRs are bot-created, | |
| # so their presence implies maintainer authorship). authorized-users is | |
| # kept as a second defense layer at the breakpoint level. | |
| # | |
| # Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action) | |
| # pinned by SHA. The fork adds: pause-idle mode (initial grace period for | |
| # the first SSH connection, idle-aware exit afterwards), endpoint output | |
| # and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open" | |
| # that carries the SSH endpoint in output.summary while the breakpoint is | |
| # paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to | |
| # conclusion=success when the breakpoint exits). | |
| # | |
| # Configured via the BREAKPOINT_ENDPOINT repository variable — in fork | |
| # PRs repository variables are not exposed, so the step is skipped and | |
| # forks cannot reach the self-hosted rendezvous server. | |
| - name: Breakpoint on E2E failure | |
| if: | | |
| failure() && | |
| vars.BREAKPOINT_ENDPOINT != '' && | |
| (contains(github.event.pull_request.labels.*.name, 'debug') | |
| || contains(github.event.pull_request.labels.*.name, 'release')) | |
| # cozystack/breakpoint-action v2-cozy.1 | |
| # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m | |
| uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4 | |
| with: | |
| mode: pause-idle | |
| endpoint: ${{ vars.BREAKPOINT_ENDPOINT }} | |
| authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83 | |
| check-run-name: "Breakpoint Open" | |
| github-token: ${{ github.token }} | |
| check-run-summary-template: | | |
| ## 🔴 SSH breakpoint open — paused for debug | |
| ``` | |
| {endpoint} | |
| ``` | |
| Enter the e2e sandbox after SSH: | |
| ``` | |
| docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash | |
| export KUBECONFIG=/workspace/kubeconfig | |
| ``` | |
| Resume from inside: `breakpoint resume`. Otherwise the breakpoint | |
| exits 10 minutes after the last SSH session disconnects. | |
| # ▸ Tear down environment (always runs) | |
| - name: Tear down sandbox | |
| if: always() | |
| run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true | |
| - name: Remove workspace | |
| if: always() | |
| run: rm -rf /tmp/$SANDBOX_NAME |