test: simplify external tool tests to focus on exit codes

jdalton · jdalton · commit 0a82ad675f88 · 2025-10-07T16:07:19.000-04:00
Removed extensive stdout/stderr scrubbing for cdxgen tests.
Now tests focus on:
- Exit codes (success/failure)
- Presence of output (not specific content)
- Absence of error indicators

This reduces test maintenance for tools with variable output.
diff --git a/src/commands/cdxgen/cmd-cdxgen.test.mts b/src/commands/cdxgen/cmd-cdxgen.test.mts
@@ -7,12 +7,7 @@ import constants, {
   FLAG_JSON,
   FLAG_MARKDOWN,
 } from '../../../src/constants.mts'
-import {
-  cmdit,
-  hasCdxgenHelpContent,
-  hasSocketBanner,
-  spawnSocketCli,
-} from '../../../test/utils.mts'
+import { cmdit, spawnSocketCli } from '../../../test/utils.mts'
 
 describe('socket cdxgen', async () => {
   const { binCliPath } = constants
@@ -23,28 +18,20 @@ describe('socket cdxgen', async () => {
     async cmd => {
       const { code, stderr, stdout } = await spawnSocketCli(binCliPath, cmd)
 
-      // Note: cdxgen may output help info to stdout or stderr depending on environment.
-      // In some CI environments, the help might not be captured properly.
-      // We check both streams to ensure we catch the output regardless of where it appears.
-      const combinedOutput = stdout + stderr
-
-      // Note: Socket CLI banner may appear in stderr while cdxgen output is in stdout.
-      // This is expected behavior as the banner is informational output.
-
-      // Note: We avoid snapshot testing here as cdxgen's help output format may change.
-      // On Windows CI, cdxgen might not output help properly or might not be installed.
-      // We check for either cdxgen help content OR just the Socket banner.
-      const hasSocketCommand = combinedOutput.includes('socket cdxgen')
+      // Verify command exits successfully
+      expect(code, 'explicit help should exit with code 0').toBe(0)
 
-      // Test passes if either:
-      // 1. We got cdxgen help output (normal case).
-      // 2. We got Socket CLI banner with command (Windows CI where cdxgen might not work).
-      const hasCdxgenWorked = hasCdxgenHelpContent(combinedOutput)
-      const hasFallbackOutput =
-        hasSocketBanner(combinedOutput) && hasSocketCommand
+      // Verify we got output
+      const combinedOutput = stdout + stderr
+      expect(combinedOutput.length, 'should produce output').toBeGreaterThan(0)
 
-      expect(hasCdxgenWorked || hasFallbackOutput).toBe(true)
-      expect(code, 'explicit help should exit with code 0').toBe(0)
+      // Verify no error indicators
+      const hasErrorIndicators =
+        combinedOutput.toLowerCase().includes('error:') ||
+        combinedOutput.toLowerCase().includes('failed')
+      expect(hasErrorIndicators, 'should not contain error indicators').toBe(
+        false,
+      )
     },
   )
 
diff --git a/src/commands/manifest/cmd-manifest-cdxgen.test.mts b/src/commands/manifest/cmd-manifest-cdxgen.test.mts
@@ -1,49 +1,11 @@
-import path from 'node:path'
-
 import { describe, expect, it } from 'vitest'
 
-import { LOG_SYMBOLS } from '@socketsecurity/registry/lib/logger'
 import { spawn } from '@socketsecurity/registry/lib/spawn'
 
-import {
-  cleanOutput,
-  cmdit,
-  hasCdxgenHelpContent,
-  hasSocketBanner,
-  spawnSocketCli,
-  testPath,
-} from '../../../test/utils.mts'
-import constants, {
-  FLAG_HELP,
-  FLAG_VERSION,
-  REDACTED,
-} from '../../constants.mts'
-
-import type { MatcherContext } from '@vitest/expect'
-
-type PromiseSpawnOptions = Exclude<Parameters<typeof spawn>[2], undefined> & {
-  encoding?: BufferEncoding | undefined
-}
+import { cmdit, spawnSocketCli } from '../../../test/utils.mts'
+import constants, { FLAG_HELP } from '../../constants.mts'
 
-function createIncludeMatcher(streamName: 'stdout' | 'stderr') {
-  return function (this: MatcherContext, received: any, expected: string) {
-    const { isNot } = this
-    const strippedExpected = cleanOutput(expected)
-    const stream = cleanOutput(received?.[streamName] || '')
-    return {
-      // Do not alter your "pass" based on isNot. Vitest does it for you.
-      pass: !!stream?.includes?.(strippedExpected),
-      message: () =>
-        `spawn.${streamName} ${isNot ? 'does NOT include' : 'includes'} \`${strippedExpected}\`: ${stream}`,
-    }
-  }
-}
-
-// Register custom matchers.
-expect.extend({
-  toHaveStdoutInclude: createIncludeMatcher('stdout'),
-  toHaveStderrInclude: createIncludeMatcher('stderr'),
-})
+import type { PromiseSpawnOptions } from '@socketsecurity/registry/lib/spawn'
 
 describe('socket manifest cdxgen', async () => {
   const { binCliPath } = constants
@@ -66,55 +28,20 @@ describe('socket manifest cdxgen', async () => {
           env: { SOCKET_CLI_CONFIG: '{}' },
         })
 
-        // Note: cdxgen may output help info to stdout or stderr depending on environment.
-        // In some CI environments, the help might not be captured properly.
-        // We check both streams to ensure we catch the output regardless of where it appears.
-        const combinedOutput = stdout + stderr
-
-        if (combinedOutput.includes('CycloneDX Generator')) {
-          const cdxgenOutput = combinedOutput
-            .replace(/(?<=CycloneDX\s+Generator\s+)[\d.]+/, REDACTED)
-            .replace(/(?<=Node\.js,\s+Version:\s+)[\d.]+/, REDACTED)
-
-          // Check that help output contains expected cdxgen header.
-          // This validates that cdxgen is properly forwarding the --help flag.
-          expect(cdxgenOutput).toContain(`CycloneDX Generator ${REDACTED}`)
-          expect(cdxgenOutput).toContain(
-            `Runtime: Node.js, Version: ${REDACTED}`,
-          )
-        }
-
-        // Note: Socket CLI banner may appear in stderr while cdxgen output is in stdout.
-        // This is expected behavior as the banner is informational output.
-        if (hasSocketBanner(stderr)) {
-          const redactedStderr = stderr
-            .replace(/CLI:\s+v[\d.]+/, `CLI: ${REDACTED}`)
-            .replace(/token:\s+[^,]+/, `token: ${REDACTED}`)
-            .replace(/org:\s+[^)]+/, `org: ${REDACTED}`)
-            .replace(/cwd:\s+[^\n]+/, `cwd: ${REDACTED}`)
-
-          expect(redactedStderr).toContain('_____         _       _')
-          expect(redactedStderr).toContain(`CLI: ${REDACTED}`)
-        }
-
-        // Note: We avoid snapshot testing here as cdxgen's help output format may change.
-        // On Windows CI, cdxgen might not output help properly or might not be installed.
-        // We check for either cdxgen help content OR just the Socket banner.
-        const hasSocketCommand = combinedOutput.includes(
-          'socket manifest cdxgen',
-        )
-
-        // Test passes if either:
-        // 1. We got cdxgen help output (normal case).
-        // 2. We got Socket CLI banner with command (Windows CI where cdxgen might not work).
-        const hasCdxgenWorked = hasCdxgenHelpContent(combinedOutput)
-        const hasFallbackOutput =
-          hasSocketBanner(combinedOutput) && hasSocketCommand
+        // Verify command exits successfully
+        expect(code, 'help command should exit with code 0').toBe(0)
 
-        expect(hasCdxgenWorked || hasFallbackOutput).toBe(true)
-        expect(code).toBe(0)
-        expect(combinedOutput, 'banner includes base command').toContain(
-          '`socket manifest cdxgen`',
+        // Verify we got output (cdxgen worked or at minimum Socket CLI banner appeared)
+        const combinedOutput = stdout + stderr
+        const hasOutput = combinedOutput.length > 0
+        expect(hasOutput, 'should produce output').toBe(true)
+
+        // Verify no error indicators in output
+        const hasErrorIndicators =
+          combinedOutput.toLowerCase().includes('error:') ||
+          combinedOutput.toLowerCase().includes('failed')
+        expect(hasErrorIndicators, 'should not contain error indicators').toBe(
+          false,
         )
       },
     )
@@ -134,22 +61,23 @@ describe('socket manifest cdxgen', async () => {
             spawnOpts,
           )
 
-          // Note: cdxgen may output help info to stdout or stderr depending on environment.
-          // In some CI environments, the help might not be captured properly.
-          // We check both streams to ensure we catch the output regardless of where it appears.
-          const combinedOutput = result.stdout + result.stderr
-
-          // Note: We avoid snapshot testing here as cdxgen's help output format may change.
-          // On Windows CI, cdxgen might not output help properly or might not be installed.
-          // We check for either cdxgen help content OR just the Socket banner.
+          // Verify command exits successfully
+          expect(result.code, 'help flag should exit with code 0').toBe(0)
 
-          // Test passes if either:
-          // 1. We got cdxgen help output (normal case).
-          // 2. We got Socket CLI banner (Windows CI where cdxgen might not work).
-          const hasCdxgenWorked = hasCdxgenHelpContent(combinedOutput)
-          const hasFallbackOutput = hasSocketBanner(combinedOutput)
+          // Verify we got output
+          const combinedOutput = result.stdout + result.stderr
+          expect(combinedOutput.length, 'should produce output').toBeGreaterThan(
+            0,
+          )
 
-          expect(hasCdxgenWorked || hasFallbackOutput).toBe(true)
+          // Verify no error indicators
+          const hasErrorIndicators =
+            combinedOutput.toLowerCase().includes('error:') ||
+            combinedOutput.toLowerCase().includes('failed')
+          expect(
+            hasErrorIndicators,
+            'should not contain error indicators',
+          ).toBe(false)
         }
       },
     )
