|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "testing" |
| 5 | + |
| 6 | + "github.com/kosli-dev/cli/internal/docker" |
| 7 | + "github.com/kosli-dev/cli/internal/version" |
| 8 | + "github.com/stretchr/testify/require" |
| 9 | + "github.com/stretchr/testify/suite" |
| 10 | +) |
| 11 | + |
| 12 | +// FingerprintCaptureTestSuite asserts the customer-facing contract that |
| 13 | +// `kosli fingerprint` produces shell-capturable output: stdout is exactly |
| 14 | +// the fingerprint, stderr is exactly empty. This is the contract Kosli |
| 15 | +// users rely on when they write `FP=$(kosli fingerprint ... 2>&1)` in CI. |
| 16 | +// |
| 17 | +// The contract must hold even when the CLI has internal reasons to want |
| 18 | +// to write to stderr (e.g. an outdated-version notice). Any future code |
| 19 | +// path that writes to stderr from a fingerprint invocation breaks this |
| 20 | +// contract and breaks customer pipelines, regardless of cause. |
| 21 | +type FingerprintCaptureTestSuite struct { |
| 22 | + suite.Suite |
| 23 | + dockerImage string |
| 24 | +} |
| 25 | + |
| 26 | +const ( |
| 27 | + // SHA256 of cmd/kosli/testdata/file1, which contains "hello world!". |
| 28 | + file1Fingerprint = "7509e5bda0c762d2bac7f90d758b5b2263fa01ccbc542ab5e3df163be08e6ca9" |
| 29 | + |
| 30 | + // SHA256 of cmd/kosli/testdata/folder1, pinned in fingerprint_test.go. |
| 31 | + folder1Fingerprint = "c43808cb04c6e66c4c6fc1f972dd67c3b9b71c81e0a0c78730da3699922d17be" |
| 32 | + |
| 33 | + // Realistic notice the version-check goroutine emits when a newer |
| 34 | + // release exists. Stubbed in via SetCheckForUpdateOverride. |
| 35 | + fakeUpdateNotice = "\nA new version of the Kosli CLI is available: v9.99.0 (you have v0.0.1)\nUpgrade: https://docs.kosli.com/getting_started/install/\n" |
| 36 | +) |
| 37 | + |
| 38 | +// SetupSuite pulls the alpine test image used by the docker variant. Same |
| 39 | +// pattern as FingerprintTestSuite in fingerprint_test.go — the image is |
| 40 | +// pinned by digest so the assertion can be exact. |
| 41 | +func (suite *FingerprintCaptureTestSuite) SetupSuite() { |
| 42 | + suite.dockerImage = "library/alpine@sha256:e15947432b813e8ffa90165da919953e2ce850bef511a0ad1287d7cb86de84b5" |
| 43 | + err := docker.PullDockerImage(suite.dockerImage) |
| 44 | + require.NoError(suite.T(), err) |
| 45 | +} |
| 46 | + |
| 47 | +// TestFingerprintFile_CaptureCleanliness pins the contract for the |
| 48 | +// most common shell-capture pattern: `kosli fingerprint <file> --artifact-type=file`. |
| 49 | +// stdout must be exactly the fingerprint + newline; stderr must be exactly empty. |
| 50 | +func (suite *FingerprintCaptureTestSuite) TestFingerprintFile_CaptureCleanliness() { |
| 51 | + // Stub the update check to deterministically return a notice, so the |
| 52 | + // test fails if any code path forwards that notice to stderr — which |
| 53 | + // is exactly what was breaking customer CI pipelines. |
| 54 | + defer version.SetCheckForUpdateOverride(func(string) (string, error) { |
| 55 | + return fakeUpdateNotice, nil |
| 56 | + })() |
| 57 | + |
| 58 | + _, combined, stdout, stderr, err := executeCommandC( |
| 59 | + "fingerprint --artifact-type file testdata/file1") |
| 60 | + suite.Require().NoError(err) |
| 61 | + |
| 62 | + // Contract 1: stdout is exactly the fingerprint and nothing else. |
| 63 | + suite.Equal(file1Fingerprint+"\n", stdout, |
| 64 | + "stdout must contain only the fingerprint — anything else breaks shell capture") |
| 65 | + |
| 66 | + // Contract 2: stderr is exactly empty on the success path. |
| 67 | + // Stricter than NotContains because the threat is general — any new |
| 68 | + // stderr writer (deprecation notice, telemetry warning, framework |
| 69 | + // log) would silently break `2>&1` capture in customer CI. |
| 70 | + suite.Equal("", stderr, |
| 71 | + "stderr must be empty — any output here pollutes 2>&1 capture pipelines") |
| 72 | + |
| 73 | + // Contract 3: combined stdout+stderr (what `2>&1` capture sees) parses |
| 74 | + // as a fingerprint. This is the customer's actual usage: |
| 75 | + // FP=$(kosli fingerprint ... 2>&1) |
| 76 | + // If this fails, customer CI fails. |
| 77 | + suite.Equal(file1Fingerprint+"\n", combined, |
| 78 | + "combined output (the 2>&1 capture pattern) must be exactly the fingerprint") |
| 79 | +} |
| 80 | + |
| 81 | +// TestFingerprintDir_CaptureCleanliness covers the directory variant. The |
| 82 | +// original cyber-dojo bug fired here because the dir path runs long enough |
| 83 | +// for the background version-check goroutine to complete and write to |
| 84 | +// stderr before the command exits. Same three contracts as the file variant. |
| 85 | +func (suite *FingerprintCaptureTestSuite) TestFingerprintDir_CaptureCleanliness() { |
| 86 | + defer version.SetCheckForUpdateOverride(func(string) (string, error) { |
| 87 | + return fakeUpdateNotice, nil |
| 88 | + })() |
| 89 | + |
| 90 | + _, combined, stdout, stderr, err := executeCommandC( |
| 91 | + "fingerprint --artifact-type dir testdata/folder1") |
| 92 | + suite.Require().NoError(err) |
| 93 | + |
| 94 | + suite.Equal(folder1Fingerprint+"\n", stdout, |
| 95 | + "stdout must contain only the fingerprint — anything else breaks shell capture") |
| 96 | + |
| 97 | + suite.Equal("", stderr, |
| 98 | + "stderr must be empty — any output here pollutes 2>&1 capture pipelines") |
| 99 | + |
| 100 | + suite.Equal(folder1Fingerprint+"\n", combined, |
| 101 | + "combined output (the 2>&1 capture pattern) must be exactly the fingerprint") |
| 102 | +} |
| 103 | + |
| 104 | +// TestFingerprintDocker_CaptureCleanliness covers the docker variant. The |
| 105 | +// docker code path goes through internal/docker.GetImageFingerprint, which |
| 106 | +// hits the local Docker daemon and is entirely separate from the file/dir |
| 107 | +// hashing path — so it could legitimately introduce its own stderr writers |
| 108 | +// (Docker API warnings, daemon connection logs, etc.) that the file/dir |
| 109 | +// tests would not catch. Pinning the contract here protects that surface. |
| 110 | +// |
| 111 | +// Mirrors the docker test in fingerprint_test.go: alpine pinned by digest, |
| 112 | +// so the resulting fingerprint is stable and the assertion can be exact. |
| 113 | +func (suite *FingerprintCaptureTestSuite) TestFingerprintDocker_CaptureCleanliness() { |
| 114 | + defer version.SetCheckForUpdateOverride(func(string) (string, error) { |
| 115 | + return fakeUpdateNotice, nil |
| 116 | + })() |
| 117 | + |
| 118 | + const alpineFingerprint = "e15947432b813e8ffa90165da919953e2ce850bef511a0ad1287d7cb86de84b5" |
| 119 | + |
| 120 | + _, combined, stdout, stderr, err := executeCommandC( |
| 121 | + "fingerprint --artifact-type docker " + suite.dockerImage) |
| 122 | + suite.Require().NoError(err) |
| 123 | + |
| 124 | + suite.Equal(alpineFingerprint+"\n", stdout, |
| 125 | + "stdout must contain only the fingerprint — anything else breaks shell capture") |
| 126 | + |
| 127 | + suite.Equal("", stderr, |
| 128 | + "stderr must be empty — any output here pollutes 2>&1 capture pipelines") |
| 129 | + |
| 130 | + suite.Equal(alpineFingerprint+"\n", combined, |
| 131 | + "combined output (the 2>&1 capture pattern) must be exactly the fingerprint") |
| 132 | +} |
| 133 | + |
| 134 | +// TestFingerprintFile_DebugModeIsAllowedToWriteStderr pins the *other* |
| 135 | +// half of the contract: stderr is an opt-in channel for debug output, not |
| 136 | +// silent across the board. This stops a future contributor from "fixing" |
| 137 | +// a TestFingerprintFile_CaptureCleanliness failure by silencing all stderr |
| 138 | +// — they need to keep the debug channel working. |
| 139 | +// |
| 140 | +// stdout MUST still be exactly the fingerprint, even in debug mode, because |
| 141 | +// `FP=$(kosli fingerprint --debug=true ...)` (without 2>&1) is still a |
| 142 | +// supported pattern that the customer might use when troubleshooting. |
| 143 | +func (suite *FingerprintCaptureTestSuite) TestFingerprintFile_DebugModeIsAllowedToWriteStderr() { |
| 144 | + defer version.SetCheckForUpdateOverride(func(string) (string, error) { |
| 145 | + return fakeUpdateNotice, nil |
| 146 | + })() |
| 147 | + |
| 148 | + _, _, stdout, stderr, err := executeCommandC( |
| 149 | + "fingerprint --artifact-type file testdata/file1 --debug=true") |
| 150 | + suite.Require().NoError(err) |
| 151 | + |
| 152 | + // stdout invariant holds even under --debug: the fingerprint and only |
| 153 | + // the fingerprint. This protects `$(...)` capture (which doesn't |
| 154 | + // include stderr) from being broken by debug output. |
| 155 | + suite.Equal(file1Fingerprint+"\n", stdout, |
| 156 | + "stdout must be the fingerprint even in debug mode — protects $(...) capture") |
| 157 | + |
| 158 | + // In debug mode the fingerprint command's own debug output MUST reach |
| 159 | + // stderr. Asserting on the fingerprint-specific log line (not just any |
| 160 | + // debug output) ensures this catches a regression where the logger is |
| 161 | + // silenced *during* the fingerprint operation — e.g. someone "fixing" |
| 162 | + // a CaptureCleanliness failure by routing logger.ErrOut to io.Discard |
| 163 | + // in the fingerprint code path. Earlier framework-level debug logs |
| 164 | + // from PreRunE would mask that, so we assert on the fingerprint marker. |
| 165 | + suite.Contains(stderr, "calculated fingerprint", |
| 166 | + "fingerprint-specific debug output must reach stderr in --debug mode — "+ |
| 167 | + "if this fails, someone has silenced the logger inside the fingerprint code path") |
| 168 | +} |
| 169 | + |
| 170 | +func TestFingerprintCaptureTestSuite(t *testing.T) { |
| 171 | + suite.Run(t, new(FingerprintCaptureTestSuite)) |
| 172 | +} |
0 commit comments