test: JUnit XML test results → Codecov Test Analytics + EnricoMi check (#21)

Author: markovejnovic

.github/workflows/ci.yml

@@ -104,6 +104,31 @@ jobs:
         # under /srv/hyper, unavailable on a CI runner.
         run: mix coveralls.json --no-start --warnings-as-errors
 
+      # !cancelled() (not always()) uploads results on test PASS or FAIL -- the
+      # point of Test Analytics is failure/flake history -- while still skipping
+      # if the workflow was cancelled. junit_formatter writes the XML at end of
+      # run regardless of pass/fail. report_type: test_results sends JUnit to
+      # Test Analytics via codecov-action (the standalone test-results-action is
+      # deprecated and pinned to Node 20).
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: _build/test/junit.xml
+          flags: elixir
+          report_type: test_results
+
+      # The artifact feeds the workflow_run publisher (test-results.yml), which
+      # runs EnricoMi with a write token -- so the GitHub Check works even on
+      # fork PRs, where this job's token is read-only.
+      - name: Upload Elixir test results artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: elixir-test-results
+          path: _build/test/junit.xml
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v7
         with:
@@ -139,13 +164,38 @@ jobs:
       - name: clippy
         run: cargo clippy --all-targets --all-features -- -D warnings
 
-      - name: Install cargo-llvm-cov
+      - name: Install cargo-llvm-cov and nextest
         uses: taiki-e/install-action@v2
         with:
-          tool: cargo-llvm-cov
+          tool: cargo-llvm-cov,nextest
 
       - name: test + coverage
-        run: cargo llvm-cov --all-features --lcov --output-path lcov.info
+        # `nextest` runs the suite while llvm-cov collects coverage AND nextest
+        # writes target/nextest/ci/junit.xml (see .config/nextest.toml). One run,
+        # two artifacts: lcov.info for coverage, junit.xml for Test Analytics.
+        run: cargo llvm-cov nextest --profile ci --all-features --lcov --output-path lcov.info
+
+      # !cancelled() uploads on pass OR fail (see elixir job). The action runs at
+      # repo root, so the path is fully qualified -- working-directory only
+      # affects `run:` steps, not `uses:` actions. report_type: test_results
+      # sends JUnit via codecov-action (test-results-action is deprecated/Node 20).
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: native/suidhelper/target/nextest/ci/junit.xml
+          flags: rust
+          report_type: test_results
+
+      # Repo-relative path: upload-artifact is an action, so the rust job's
+      # `working-directory: native/suidhelper` default does NOT apply.
+      - name: Upload Rust test results artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-test-results
+          path: native/suidhelper/target/nextest/ci/junit.xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v7
@@ -155,3 +205,16 @@ jobs:
           files: native/suidhelper/lcov.info
           flags: rust
           fail_ci_if_error: true
+
+  # Uploads the triggering event payload so the workflow_run publisher
+  # (test-results.yml) can map results back to the originating PR -- required
+  # for PR comments on fork PRs. Tiny job, always runs.
+  event_file:
+    name: Upload event file
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v7
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

.github/workflows/test-results.yml

@@ -0,0 +1,52 @@
+# Publishes a unified GitHub Check + PR comment from the JUnit XML produced by
+# the CI workflow. Runs via workflow_run (not in CI itself) so it executes in
+# the base-repo context with a write token -- making test annotations work on
+# fork PRs and Dependabot branches, where CI's own token is read-only.
+name: Test Results
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+
+# Least privilege: only this workflow gets write scopes, and it never checks
+# out untrusted PR code -- it only reads artifacts the CI run produced.
+permissions: {}
+
+jobs:
+  publish:
+    name: Publish test results
+    runs-on: ubuntu-latest
+    # Publish on success OR failure (failing tests are the whole point); skip
+    # when CI was cancelled or skipped.
+    if: >-
+      github.event.workflow_run.conclusion == 'success' ||
+      github.event.workflow_run.conclusion == 'failure'
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      actions: read
+    steps:
+      # download-artifact@v8 fetches artifacts from ANOTHER run via run-id +
+      # github-token (no third-party action needed). Each artifact lands in its
+      # own subdir: artifacts/<name>/...
+      - name: Download CI artifacts
+        uses: actions/download-artifact@v8
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+          path: artifacts
+
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          # head_sha attaches the check to the PR's commit (not the default
+          # branch). event_file/event_name let the action resolve the PR for
+          # commenting -- both come from the workflow_run payload.
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/**/*.xml"
+          check_name: Test Results

config/config.exs

@@ -34,6 +34,18 @@ if config_env() == :test do
   config :opentelemetry, traces_exporter: :none
   # No cluster formation during tests.
   config :libcluster, topologies: []
+
+  # JUnit XML for Codecov Test Analytics. A fixed report_dir (not the default
+  # app_path) gives CI a stable path; automatic_create_dir? mkdirs it since
+  # the formatter uses File.write! which won't create parents. include_* embed
+  # the source file + line so Codecov can link failures back to the test.
+  config :junit_formatter,
+    report_dir: "_build/test",
+    report_file: "junit.xml",
+    print_report_file: true,
+    include_filename?: true,
+    include_file_line?: true,
+    automatic_create_dir?: true
 end
 
 config :hyper, Hyper.Img.Db.Repo,

mix.exs

@@ -58,6 +58,7 @@ defmodule Hyper.MixProject do
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
       {:stream_data, "~> 1.0", only: [:dev, :test], runtime: false},
       {:excoveralls, "~> 0.18", only: :test, runtime: false},
+      {:junit_formatter, "~> 3.4", only: :test, runtime: false},
       {:dialyxir, "~> 1.4", only: [:dev], runtime: false},
       {:ex_doc, "~> 0.34", only: :dev, runtime: false},
       {:ecto_sql, "~> 3.13"},

mix.lock

@@ -32,6 +32,7 @@
   "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"},
   "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"},
   "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
+  "junit_formatter": {:hex, :junit_formatter, "3.4.0", "d0e8db6c34dab6d3c4154c3b46b21540db1109ae709d6cf99ba7e7a2ce4b1ac2", [:mix], [], "hexpm", "bb36e2ae83f1ced6ab931c4ce51dd3dbef1ef61bb4932412e173b0cfa259dacd"},
   "libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"},
   "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"},
   "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},

native/suidhelper/.config/nextest.toml

@@ -0,0 +1,12 @@
+# nextest CI profile. The [profile.ci.junit] table makes `--profile ci` write
+# JUnit XML to target/nextest/ci/junit.xml, which Codecov Test Analytics ingests.
+# store-success-output/-failure-output keep captured stdio in the report so a
+# failing test's output is visible in Codecov.
+[profile.ci]
+# Retry once to dampen rare flakes without masking genuine failures.
+retries = 1
+
+[profile.ci.junit]
+path = "junit.xml"
+store-success-output = false
+store-failure-output = true

test/test_helper.exs

@@ -1 +1,4 @@
-ExUnit.start()
+# JUnitFormatter writes JUnit XML (consumed by Codecov Test Analytics) as a side
+# effect of the normal test run. Listing formatters explicitly REPLACES the
+# defaults, so ExUnit.CLIFormatter must be named here to keep console output.
+ExUnit.start(formatters: [ExUnit.CLIFormatter, JUnitFormatter])