[DX-3769] CLI Helper to Run Unit Tests Easier

.gitignore

@@ -59,6 +59,7 @@ race.*
 core/services/job/testdata/wasm/testmodule.wasm
 core/services/job/testdata/wasm/testmodule.br
 temp-repo
+diagnose-*/
 
 # DB state
 ./db/

GNUmakefile

@@ -17,6 +17,8 @@ LOOPINSTALL_PUBLIC_ARGS  := $(if $(strip $(CL_LOOPINSTALL_OUTPUT_DIR)),--output-
 LOOPINSTALL_PRIVATE_ARGS := $(if $(strip $(CL_LOOPINSTALL_OUTPUT_DIR)),--output-installation-artifacts $(CL_LOOPINSTALL_OUTPUT_DIR)/private.json)
 LOOPINSTALL_TESTING_ARGS := $(if $(strip $(CL_LOOPINSTALL_OUTPUT_DIR)),--output-installation-artifacts $(CL_LOOPINSTALL_OUTPUT_DIR)/testing.json)
 GOLANGCI_LINT_VERSION = "v2.11.4"
+# Pin path so `make generate` does not pick up a different mockery (e.g. v3) from PATH.
+MOCKERY_BIN ?= $(shell GOBIN="$$(go env GOBIN)"; if [ -n "$$GOBIN" ]; then echo "$$GOBIN/mockery"; else echo "$$(go env GOPATH)/bin/mockery"; fi)
 
 .PHONY: install
 install: install-chainlink-autoinstall ## Install chainlink and all its dependencies.
@@ -175,7 +177,7 @@ operator-ui: ## Fetch the frontend
 generate: codecgen mockery protoc gomods modgraph ## Execute all go:generate commands.
 	## Updating PATH makes sure that go:generate uses the version of protoc installed by the protoc make command.
 	export PATH="$(HOME)/.local/bin:$(PATH)"; gomods -w go generate -x ./...
-	find . -type f -name .mockery.yaml -execdir mockery \; ## Execute mockery for all .mockery.yaml files
+	find . -type f -name .mockery.yaml -execdir $(MOCKERY_BIN) \; ## Execute mockery for all .mockery.yaml files (see mockery target: v2)
 
 .PHONY: rm-mocked
 rm-mocked:
@@ -270,6 +272,22 @@ modgraph:
 test-short: ## Run 'go test -short' and suppress uninteresting output
 	go test -short ./... | grep -v "\[no test files\]" | grep -v "\(cached\)"
 
+# Chainlink tools/test harness (Postgres setup + optional diagnose). Uses the
+# nested module directly so its dependencies stay out of the root module. Pass
+# flags and packages via ARGS (quoted), e.g. make new_test ARGS="-v -p 4 ./core/..."
+# Note: do not use "make target -p 4 ..." — -p is a make flag; use ARGS= instead.
+.PHONY: new_test
+new_test: ## tools/test: passthrough go test. Usage: make new_test ARGS="-v -p 4 ./core/..."
+	go -C tools/test run . run $(ARGS)
+
+.PHONY: new_gotestsum
+new_gotestsum: ## tools/test: gotestsum. Usage: make new_gotestsum ARGS="--format=dots -- -count=1 ./core/..."
+	go -C tools/test run . gotestsum $(ARGS)
+
+.PHONY: new_test_diagnose
+new_test_diagnose: ## tools/test: diagnose (flakes/slow). Usage: make new_test_diagnose ARGS="--iterations 5 -- --timeout 9m ./core/..."
+	go -C tools/test run . diagnose $(ARGS)
+
 .PHONY: gocs
 gocs: ## Run gocs to generate changeset markdown files.
 	go run github.com/smartcontractkit/gocs/cmd/gocs@v0.2.0

tools/README.md

@@ -3,3 +3,7 @@
 ## [Docker](./docker)
 
 Manage Docker for development and testing
+
+## [test](./test/)
+
+A harness for running /chainlink tests. From the repo root use **`make new_test`** / **`make new_gotestsum`** / **`make new_test_diagnose`**; details in [tools/test/README.md](./test/README.md).

tools/test/AGENTS.md

@@ -0,0 +1,34 @@
+A test runner harness for the /chainlink repo.
+
+<goals>
+- Provide a single, easy command to setup and run tests in /chainlink repo, eliminating `make` command chaining.
+- Enable automatically re-running tests and analyzing results to catch and diagnose flakes and slow tests
+- Provide an AI skill for the process in `.agents/skills/diagnose-tests/SKILL.md`
+</goals>
+
+<rules>
+- From /chainlink root, document `make new_test`, `make new_gotestsum`, and `make new_test_diagnose`. When working only inside this module, `go run . …` is fine.
+- Each output should account for a pretty, human-readable terminal experience, and a minimal version meant for AI ingestion
+</rules>
+
+<modes>
+<mode name="go test" subcommand="run"> 
+Run tests using vanilla `go test` command and arguments
+</mode>
+<mode name="gotestsum" subcommand="gotestsum"> 
+Run tests using gotestsum for those that prefer its output and tools
+</mode>
+<mode name="diagnose" subcommand="diagnose"> 
+Opinionated flow to re-run tests and identify flakes, races, timeouts, and test runtimes.
+</mode>
+</modes>
+
+<commands>
+Run these commands to validate any changes you make
+```sh
+golangci-lint run ./... --fix
+go test ./...
+```
+
+DO NOT use other commands like `goimports`, `gofmt`, or `go vet` for formatting and lint checks.
+</commands>

tools/test/README.md

@@ -0,0 +1,41 @@
+# /chainlink Test Runner
+
+A Go harness to run unit tests in /chainlink with a simpler flow and control scheme. Plus a mode to help you hunt down flakes, races, and timeouts.
+
+## Run
+
+You can run using `go -C tools/test run .` or through make targets.
+
+```sh
+go -C tools/test run . -h # Help menu
+
+# Use plain go test
+go -C tools/test run . run -count=1 ./core/... 
+make new_test ARGS="-count=1 ./core/..."
+
+# Use gotestsum
+go -C tools/test run . gotestsum --format=testname -- -count=1 ./core/...
+make new_gotestsum ARGS="--format=testname -- -count=1 ./core/..."
+
+# Diagnose and fix flaky tests
+go -C tools/test run . diagnose --iterations 5 -- --timeout=9m ./core/...
+make new_test_diagnose ARGS="--iterations 5 -- --timeout=9m ./core/..."
+```
+
+When **developing only inside this directory** (nested module), use `go run .` instead of `go -C tools/test`:
+
+```sh
+go run . -h
+go run . run -count=1 ./core/...
+go run . diagnose --iterations 5 -- ./core/...
+```
+
+### AI Skill
+
+Use the [/diagnose-tests](/.agents/skills/diagnose-tests/SKILL.md) ai skill with your favorite agent to run a `diagnose` loop.
+
+## Why not just `go test`?
+
+There is no way to tell `go test` about some universal, one-time setup step (like creating a Postgres DB), so we need a light wrapper to take care of this.
+
+We could make just `go test` work if we have each test package that needs a DB launch their own using [testcontainers-go](https://github.com/testcontainers/testcontainers-go), but performance implications of that are still unknown.

tools/test/fixing-flaky-tests.md

@@ -0,0 +1,135 @@
+# Finding the Root Cause of Test Flakes in Go
+
+Flaky tests can arise from many sources and can be frustrating to fix. Here's a non-exhaustive guide to help you find and resolve common causes for flakes in Go. But first, to answer a common question...
+
+## The Test Only Flakes 0.xx% of the Time, Why Bother Fixing It?
+
+You bother fixing it because of **MATH!**
+
+Let's imagine a large repo with 10,000 tests, and let's imagine only 100 (1%) of them are flaky. Let's further imagine that each of those flaky tests has a chance of flaking 1% of the time. If you are a responsible dev that requires all of your tests to pass in CI before you merge, flaky tests have now become a massive headache.
+
+$$P(\text{at least one flaky test}) = 1 - (1 - 0.01)^{100}$$
+
+$$P(\text{at least one flaky test}) \approx 63.40\%$$
+
+Even a small percentage of tests with a small chance of flaking can cause massive damage to dev velocity.
+
+
+## Tips on Finding and Fixing Flakes
+
+Ideally, if you're dealing with a flaky test, you'll already have some examples of it flaking in front of you so you can dig through logs and stack traces and figure it out that way. If that's not the case, or you'd like some more evidence, or you're just stumped, try reproducing the flake. How you reproduce the flake is often the best clue as to why it's flaking.
+
+You can also try some more precise configurations below.
+
+### 1. Run the Test in Isolation
+
+As we saw above, flaky tests become issues even when their chance of flaking is tiny. You might be hunting down a flake that only happens 0.5% of the time, so your only real solution is to run the test over and over.
+
+```sh
+# Run just that test 1,000 times, stopping after the first failure
+go test ./package -run TestName -count 1000 -failfast
+```
+
+### 2. Run the Test Package
+
+Tests rarely run in isolation in the real world. If you can't get the flake to happen when isolated, try running the whole package on repeat.
+
+```sh
+# Run all tests in the package over and over.
+go test ./package -count 1000 -failfast
+```
+
+If you get the test to fail here, but not independently, it's likely that it depends on the execution of other tests in the package. Look for global resources your test could be sharing with others, and do your best to isolate all of your unit tests.
+
+### 3. Randomize Test Order
+
+If that's still not doing the job, or you're still scratching your head, try randomizing the test order. Go runs tests in a deterministic order by default.
+
+```sh
+# -shuffle randomizes test order
+go test ./package -shuffle on -count 1000 -failfast
+# You can supply your own int value to shuffle as a seed
+go test ./package -shuffle 15 -count 1000 -failfast
+```
+
+### 4. Check for Races
+
+If your test is failing in a situation like this, it's possible a race condition is causing the failure. Go's `-race` flag isn't guaranteed to catch all races every time. Just like flakes, you sometimes just need to get lucky (unlucky?).
+
+```sh
+# Tests with -race detection take longer to run, and aren't always going to catch issues, especially in large test suites.
+go test ./package -race -shuffle on -count 100 -failfast
+```
+
+### 5. Emulate Your Target System
+
+Tests will often fail in CI, but not locally. You can try re-running the test in CI, but this might take a long time, cost a lot of money, or generally be annoying. There are a few tricks you can do to emulate CI environments locally.
+
+#### 5.1 Play with -cpu and -parallel
+
+You can artificially constrain or expand parallel execution directly in Go. [GOMAXPROCS](https://pkg.go.dev/runtime#hdr-Environment_Variables) is set to the number of CPUs your system has by default, and controls how many OS threads can run Go code at once. You can manipulate this value, or otherwise play with how many tests can run at once easily. This can help you figure out if resource constraints are hurting your tests.
+
+```sh
+# Use -cpu to change GOMAXPROCS. You can supply a list of values to try out different values at once
+go test ./package -shuffle 15 -count 1000 -failfast -cpu 1,2,4
+# Use -parallel to set the max amount of tests allowed to run in parallel at once
+go test ./package -shuffle 15 -count 1000 -failfast -parallel 4
+```
+
+#### 5.2 Use Docker
+
+Docker can help you emulate your CI environment a little better. You can lookup what type of GitHub Actions runner your CI workflow uses by matching to the lists [here](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories) and [here](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners/about-larger-runners#specifications-for-general-larger-runners). You can then package your Go tests in a Docker container, and run them with varying resources.
+
+```sh
+# Run a basic Ubuntu container with resources matching a standard GitHub-hosted runner
+docker run -it --cpus=4 --memory="16g" ubuntu-24.04
+```
+
+You can also try using [dockexec](https://github.com/mvdan/dockexec) for convenience, but I've never personally tried it.
+
+#### 5.3 Use act
+
+[act](https://github.com/nektos/act) is a project that lets you emulate your GitHub Actions workflows locally. It's not perfect, and can be tricky to setup for more complex workflows, but it is a nice option if you suspect issues are further back in the workflow, and don't want to run the full CI process.
+
+### 6. Use Your Target System
+
+Sometimes you can only discover the truth by going directly to the source. Before you do so, please double check what `runs-on` systems your workflows use. If you're only using `ubuntu-latest` runners, these runs should be free, or at least very cheap. `8-core`, `16-core`, and `32-core` workflows can become very expensive, very quickly. Please use caution and discretion when running these workflows repeatedly.
+
+#### 6.1 CI Resource Constraints
+
+It is sometimes the case that tests only fail in CI environments because those environments are underpowered. **This is more rare than you think, be cautious of [System 1 thinking](https://en.wikipedia.org/wiki/Thinking,_Fast_and_Slow) here.** You can diagnose this with [this excellent GitHub workflow telemetry action](https://github.com/catchpoint/workflow-telemetry-action) that can give you detailed stats on how many resources your tests are consuming. (This is also handy if you're looking to optimize your CI runtimes or costs.) If your tests are flaking due to low resources, consider other options before just increasing the power of the CI runners. [Increasing the power of a GitHub Actions workflow by a single tier doubles its cost](https://docs.github.com/en/billing/managing-billing-for-your-products/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates-for-x64-powered-larger-runners). If your workflow runs often, you can burn a lot of cash quickly. You can otherwise try strategies like:
+
+* Splitting the tests into different workflows, each running on `ubuntu-latest`
+* Moving more resource-hungry tests to run only on nightly cadences
+* Try removing `t.Parallel()` from subtests, as too many tests trying to run at once will often hurt stability and runtimes on smaller machines
+
+### 7. Fix It!
+
+Maybe you've found the source of the flake and are now drilling down into the reasons why. Whatever those reasons might be, I urge you to, at least briefly, reframe the problem and ask if the test is actually working as intended, and that it is revealing flaky behavior in your application instead. Consider that you might have found a rare bug, rather than a rare flake.
+
+### 8. Give Up
+
+It's not my favorite answer, but sometimes this truly is the solution. It's hard to know exactly when you should abandon hope, but maybe the below steps can help you figure it out.
+
+#### 8.1 Evaluate the Importance of the Test
+
+Ask yourself these questions to help figure out if it's worth working on this flake further, and to help you figure out what to do next.
+
+* What does the test actually check? Is it a critical path?
+* Is the test flaking because it's a bad test? Is it trying to test behavior that shouldn't or can't be tested?
+* Can you write a new test that checks the same behavior, but doesn't fall to the same issues?
+* Can you come back to this later? Maybe in a week or two you'll have new ideas, or maybe the underlying system will change in ways that make this flake no longer an issue?
+
+#### 8.2 Turn it Off
+
+Assuming you're ready to declare defeat, it's time to turn off the test. How you do this depends on the test, your team, and the answers to the questions above. If you've determined the test isn't particularly important and isn't worth running anymore, you should just delete it.
+
+## Chainlink `tools/test` harness
+
+For repeated runs with Postgres setup, `go test -json` capture, and machine-readable reports under `diagnose-*` directories, use the harness from the **repository root** (`go -C tools/test run .`, declared in the root `go.mod`):
+
+```sh
+go -C tools/test run . diagnose --iterations 50 -- --failfast ./path/to/package
+```
+
+See [README.md](./README.md), root `GNUmakefile` targets `new_test` / `new_gotestsum` / `new_test_diagnose`, and the agent playbook [`.agents/skills/diagnose-tests/SKILL.md`](../../.agents/skills/diagnose-tests/SKILL.md).

tools/test/go.mod

@@ -0,0 +1,95 @@
+module github.com/smartcontractkit/chainlink/v2/tools/test
+
+go 1.26.2
+
+require (
+	charm.land/fang/v2 v2.0.1
+	charm.land/lipgloss/v2 v2.0.3
+	github.com/charmbracelet/x/term v0.2.2
+	github.com/spf13/cobra v1.10.2
+	github.com/spf13/pflag v1.0.10
+	github.com/spf13/viper v1.21.0
+	github.com/stretchr/testify v1.11.1
+	github.com/testcontainers/testcontainers-go v0.42.0
+	github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
+)
+
+require (
+	dario.cat/mergo v1.0.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/charmbracelet/colorprofile v0.4.3 // indirect
+	github.com/charmbracelet/ultraviolet v0.0.0-20260422141423-a0f1f21775f7 // indirect
+	github.com/charmbracelet/x/ansi v0.11.7 // indirect
+	github.com/charmbracelet/x/exp/charmtone v0.0.0-20260422141420-a6cbdff8a7e2 // indirect
+	github.com/charmbracelet/x/termios v0.1.1 // indirect
+	github.com/charmbracelet/x/windows v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.11.0 // indirect
+	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-connections v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/ebitengine/purego v0.10.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
+	github.com/magiconair/properties v1.8.10 // indirect
+	github.com/mattn/go-runewidth v0.0.23 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/go-archive v0.2.0 // indirect
+	github.com/moby/moby/api v1.54.1 // indirect
+	github.com/moby/moby/client v0.4.0 // indirect
+	github.com/moby/patternmatcher v0.6.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/mango v0.2.0 // indirect
+	github.com/muesli/mango-cobra v1.3.0 // indirect
+	github.com/muesli/mango-pflag v0.2.0 // indirect
+	github.com/muesli/roff v0.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.3.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/sagikazarmark/locafero v0.12.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.26.3 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.50.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
+	golang.org/x/text v0.36.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)