integration-progress.mjs

#!/usr/bin/env node

import fs from 'node:fs';
import path from 'node:path';

const ROOT = process.cwd();
const CHECK_MODE = process.argv.includes('--check');
const HANDLER_TEST_DIR = path.join(ROOT, 'src/daemon/handlers/__tests__');
const PROVIDER_SCENARIO_DIR = path.join(ROOT, 'test/integration/provider-scenarios');
const COVERAGE_SUMMARY = path.join(ROOT, 'coverage/coverage-summary.json');
const COMMAND_CATALOG = path.join(ROOT, 'src/command-catalog.ts');
const COMMAND_CONTRACT_FILES = [
  path.join(ROOT, 'src/commands/client-command-contracts.ts'),
  path.join(ROOT, 'src/commands/interaction-command-contracts.ts'),
];
const COMMAND_CATALOG_SOURCE = fs.readFileSync(COMMAND_CATALOG, 'utf8');
const clientCommandMethods = readClientCommandMethods();

const handlerTests = listFiles(HANDLER_TEST_DIR, (file) => file.endsWith('.test.ts'));
const providerScenarioTests = listFiles(PROVIDER_SCENARIO_DIR, (file) => file.endsWith('.test.ts'));
const providerScenarioSources = listFiles(PROVIDER_SCENARIO_DIR, (file) => file.endsWith('.ts'));
const providerScenarioSupportSources = providerScenarioSources.filter((file) => !file.endsWith('.test.ts'));
const handlerStats = summarizeFiles(handlerTests);
const providerScenarioStats = summarizeFiles(providerScenarioTests);
const providerScenarioSupportStats = summarizeFiles(providerScenarioSupportSources);
const mockHeavyHandlerFiles = handlerTests.filter((file) =>
  fs.readFileSync(file, 'utf8').includes('vi.mock('),
);
const mockHeavyHandlerRows = summarizeMockHeavyHandlerFiles(mockHeavyHandlerFiles);
const providerPressureRows = summarizeProviderPressure(providerScenarioSources);
const publicCommandRows = summarizePublicCommandCoverage(providerScenarioTests);
const missingPublicCommands = publicCommandRows.filter((command) => command.references === 0);
const flagCoverageRows = summarizeProviderScenarioFlagCoverage(providerScenarioTests);
const missingFlagRows = flagCoverageRows.filter((flag) => flag.references === 0);
const excludedFlagRows = summarizeProviderScenarioFlagExclusions();
const publicCliFlagKeys = readPublicCliFlagKeys();
const classifiedFlagKeys = new Set([
  ...flagCoverageRows.map((flag) => flag.key),
  ...excludedFlagRows.flatMap((group) => group.keys),
]);
const unclassifiedFlagKeys = [...publicCliFlagKeys].filter((key) => !classifiedFlagKeys.has(key));
const coverage = readCoverageSummary();
const lowCoverageFiles = readLowCoverageFiles();

const rows = [
  ['Handler unit test files', String(handlerStats.files)],
  ['Handler unit test LOC', String(handlerStats.lines)],
  ['Handler unit tests', String(handlerStats.tests)],
  ['Handler files with vi.mock', String(mockHeavyHandlerFiles.length)],
  ['Provider scenario files', String(providerScenarioStats.files)],
  ['Provider scenario LOC', String(providerScenarioStats.lines)],
  ['Provider scenario tests', String(providerScenarioStats.tests)],
  ['Provider scenario support files', String(providerScenarioSupportStats.files)],
  ['Provider scenario support LOC', String(providerScenarioSupportStats.lines)],
  ['Provider scenario / handler LOC', ratio(providerScenarioStats.lines, handlerStats.lines)],
  [
    'Public commands covered by provider-backed integration',
    `${publicCommandRows.length - missingPublicCommands.length}/${publicCommandRows.length}`,
  ],
  ['Public commands missing provider-backed integration coverage', String(missingPublicCommands.length)],
  [
    'Device-observable workflow flags covered by provider-backed integration',
    `${flagCoverageRows.length - missingFlagRows.length}/${flagCoverageRows.length}`,
  ],
  ['Device-observable workflow flags missing provider-backed integration coverage', String(missingFlagRows.length)],
  [
    'Public CLI flags intentionally outside provider-backed integration',
    String(excludedFlagRows.reduce((sum, group) => sum + group.keys.length, 0)),
  ],
  ['Public CLI flags unclassified by progress script', String(unclassifiedFlagKeys.length)],
];

if (coverage) {
  rows.push(
    ['Coverage statements', formatPercent(coverage.statements)],
    ['Coverage branches', formatPercent(coverage.branches)],
    ['Coverage functions', formatPercent(coverage.functions)],
    ['Coverage lines', formatPercent(coverage.lines)],
  );
} else {
  rows.push(['Coverage summary', 'not available; run pnpm test:coverage first']);
}

console.log('Provider-backed integration status');
console.log('');
console.log('| Measure | Value |');
console.log('| --- | ---: |');
for (const [name, value] of rows) {
  console.log(`| ${name} | ${value} |`);
}

if (mockHeavyHandlerRows.length > 0) {
  console.log('');
  console.log('Mock-heavy handler unit tests');
  console.log('');
  console.log('| Tests | LOC | File |');
  console.log('| ---: | ---: | --- |');
  for (const file of mockHeavyHandlerRows) {
    console.log(`| ${file.tests} | ${file.lines} | ${file.file} |`);
  }
}

if (missingPublicCommands.length > 0) {
  console.log('');
  console.log('Public command coverage gaps');
  console.log('');
  console.log('| Command |');
  console.log('| --- |');
  for (const command of missingPublicCommands) {
    console.log(`| ${command.command} |`);
  }
}

if (missingFlagRows.length > 0) {
  console.log('');
  console.log('Device-observable workflow flag coverage gaps');
  console.log('');
  console.log('| Flag | Intended integration coverage |');
  console.log('| --- | --- |');
  for (const flag of missingFlagRows) {
    console.log(`| ${flag.key} | ${flag.reason} |`);
  }
}

if (excludedFlagRows.length > 0) {
  console.log('');
  console.log('Public CLI flag coverage outside provider-backed integration');
  console.log('');
  console.log('| Bucket | Flags | Coverage owner |');
  console.log('| --- | --- | --- |');
  for (const group of excludedFlagRows) {
    console.log(`| ${group.name} | ${group.keys.join(', ')} | ${group.owner} |`);
  }
}

if (unclassifiedFlagKeys.length > 0) {
  console.log('');
  console.log('Unclassified public CLI flags');
  console.log('');
  console.log('| Flag |');
  console.log('| --- |');
  for (const key of unclassifiedFlagKeys) {
    console.log(`| ${key} |`);
  }
}

if (providerPressureRows.length > 0) {
  console.log('');
  console.log('Provider transcript pressure');
  console.log('');
  console.log('| Contract surface | References | Files |');
  console.log('| --- | ---: | ---: |');
  for (const pressure of providerPressureRows) {
    console.log(`| ${pressure.name} | ${pressure.references} | ${pressure.files} |`);
  }
}

if (CHECK_MODE) {
  const failures = [];
  if (missingPublicCommands.length > 0) {
    failures.push(
      `missing Provider-backed integration command coverage: ${missingPublicCommands.map((row) => row.command).join(', ')}`,
    );
  }
  if (missingFlagRows.length > 0) {
    failures.push(
      `missing Provider-backed integration workflow flag coverage: ${missingFlagRows.map((row) => row.key).join(', ')}`,
    );
  }
  if (unclassifiedFlagKeys.length > 0) {
    failures.push(`unclassified public CLI flags: ${unclassifiedFlagKeys.join(', ')}`);
  }
  if (failures.length > 0) {
    console.error('');
    console.error(`provider-backed integration progress check failed: ${failures.join('; ')}`);
    process.exit(1);
  }
}

function summarizeProviderScenarioFlagCoverage(files) {
  const flagTargets = [
    ['platform', 'selection across platform-specific provider-backed integration flows'],
    ['target', 'target-class routing such as tv/mobile/desktop'],
    ['device', 'human-readable device selection'],
    ['udid', 'Apple device selection'],
    ['serial', 'Android device selection'],
    ['iosSimulatorDeviceSet', 'iOS simulator-set scoping reaches inventory resolution'],
    ['androidDeviceAllowlist', 'Android serial allowlist reaches inventory resolution'],
    ['session', 'named session routing'],
    ['surface', 'macOS app/frontmost/desktop/menubar surfaces'],
    ['activity', 'Android explicit launch activity'],
    ['launchConsole', 'iOS simulator launch console capture'],
    ['saveScript', 'open/close replay recording output'],
    ['relaunch', 'open terminates before launch'],
    ['shutdown', 'close/disconnect shutdown behavior'],
    ['appsFilter', 'apps --all vs default filtering'],
    ['header', 'install-from-source URL headers', ['headers']],
    ['retainPaths', 'retained install-source materialization'],
    ['retentionMs', 'install-source materialization TTL'],
    ['count', 'repeated press/click/swipe input'],
    ['fps', 'recording frame-rate request'],
    ['quality', 'recording quality scaling'],
    ['hideTouches', 'recording without touch overlays'],
    ['intervalMs', 'repeated press interval'],
    ['delayMs', 'typing/fill delay'],
    ['holdMs', 'press hold duration'],
    ['jitterPx', 'press jitter'],
    ['pixels', 'scroll distance'],
    ['doubleTap', 'double tap gesture'],
    ['clickButton', 'desktop mouse button selection', ['button']],
    ['backMode', 'explicit app/system back behavior', ['mode']],
    ['pauseMs', 'swipe repeat pause'],
    ['pattern', 'swipe repeat pattern'],
    ['snapshotInteractiveOnly', 'interactive snapshot/ref refresh', ['interactiveOnly']],
    ['snapshotCompact', 'compact snapshot output', ['compact']],
    ['snapshotDepth', 'scoped snapshot depth', ['depth']],
    ['snapshotScope', 'scoped snapshot capture', ['scope']],
    ['snapshotRaw', 'raw snapshot node output', ['raw']],
    ['out', 'artifact output path plumbing'],
    ['overlayRefs', 'screenshot ref overlay annotation'],
    ['screenshotFullscreen', 'screenshot full-screen capture mode'],
    ['screenshotMaxSize', 'screenshot max-size post-processing'],
    ['screenshotNoStabilize', 'screenshot stabilization opt-out', ['stabilize']],
    ['restart', 'logs clear --restart workflow'],
    ['networkInclude', 'network dump include modes', ['include']],
    ['noRecord', 'action recording suppression'],
    ['replayUpdate', 'selector-healing replay update', ['update']],
    ['replayEnv', 'replay/test variable injection', ['env']],
    ['failFast', 'test suite stops after first failure'],
    ['timeoutMs', 'wait/test timeout flags'],
    ['retries', 'test suite retry budget flows through request path'],
    ['artifactsDir', 'test artifact root'],
    ['steps', 'batch inline steps'],
    ['batchOnError', 'batch stop-on-error policy', ['onError']],
    ['batchMaxSteps', 'batch max-step guard', ['maxSteps']],
    ['findFirst', 'find first disambiguation'],
    ['findLast', 'find last disambiguation'],
  ];
  const sources = files.map((file) => fs.readFileSync(file, 'utf8')).join('\n');
  return flagTargets.map(([key, reason, aliases = []]) => {
    const references = [key, ...aliases].reduce(
      (count, candidate) => count + countFlagReferences(sources, candidate),
      0,
    );
    return { key, reason, references };
  });
}

function countFlagReferences(text, key) {
  const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
  return text.match(new RegExp(`\\b${escaped}\\s*:`, 'g'))?.length ?? 0;
}

function summarizeProviderScenarioFlagExclusions() {
  return [
    {
      name: 'config, output, diagnostics, and transport',
      owner: 'args/CLI transport/auth tests',
      keys: [
        'config',
        'remoteConfig',
        'stateDir',
        'daemonBaseUrl',
        'daemonAuthToken',
        'daemonTransport',
        'daemonServerMode',
        'tenant',
        'sessionIsolation',
        'runId',
        'leaseId',
        'leaseBackend',
        'json',
        'help',
        'version',
        'verbose',
      ],
    },
    {
      name: 'remote connection and session-lock policy',
      owner: 'connection/runtime/request policy tests',
      keys: ['force', 'noLogin', 'sessionLock', 'sessionLocked', 'sessionLockConflicts'],
    },
    {
      name: 'Metro and React Native runtime preparation',
      owner: 'Metro companion integration and parser tests',
      keys: [
        'metroHost',
        'metroPort',
        'metroProjectRoot',
        'metroKind',
        'metroPublicBaseUrl',
        'metroProxyBaseUrl',
        'metroBearerToken',
        'metroPreparePort',
        'metroListenHost',
        'metroStatusHost',
        'metroStartupTimeoutMs',
        'metroProbeTimeoutMs',
        'metroRuntimeFile',
        'metroNoReuseExisting',
        'metroNoInstallDeps',
        'bundleUrl',
        'launchUrl',
      ],
    },
    {
      name: 'parser/client-only command flags',
      owner: 'args, CLI, screenshot-diff, and batch tests',
      keys: [
        'githubActionsArtifact',
        'snapshotDiff',
        'snapshotForceFull',
        'baseline',
        'threshold',
        'reportJunit',
        'replayMaestro',
        'stepsFile',
      ],
    },
    {
      name: 'platform boot fallback without provider seam',
      owner: 'handler and Android platform unit tests',
      keys: ['headless'],
    },
  ];
}

function readPublicCliFlagKeys() {
  const sources = [
    path.join(ROOT, 'src/utils/command-schema.ts'),
    path.join(ROOT, 'src/commands/capture-screenshot-options.ts'),
  ];
  const keys = new Set();
  for (const source of sources) {
    const text = fs.readFileSync(source, 'utf8');
    for (const match of text.matchAll(/\{\s*key: '([^']+)'[\s\S]*?names:\s*\[([^\]]*)\]/g)) {
      const key = match[1];
      const names = match[2] ?? '';
      if (names.includes("'--") || names.includes("'-")) {
        keys.add(key);
      }
    }
  }
  return keys;
}

if (lowCoverageFiles.length > 0) {
  console.log('');
  console.log('Lowest covered implementation files');
  console.log('');
  console.log('| Missing statements | Statements | Statement coverage | File |');
  console.log('| ---: | ---: | ---: | --- |');
  for (const file of lowCoverageFiles) {
    console.log(
      `| ${file.missingStatements} | ${file.statementTotal} | ${formatPercent(file.statementPercent)} | ${file.file} |`,
    );
  }
}

function listFiles(dir, predicate) {
  if (!fs.existsSync(dir)) return [];
  const entries = fs.readdirSync(dir, { withFileTypes: true });
  return entries.flatMap((entry) => {
    const fullPath = path.join(dir, entry.name);
    if (entry.isDirectory()) return listFiles(fullPath, predicate);
    return predicate(fullPath) ? [fullPath] : [];
  });
}

function summarizeFiles(files) {
  let lines = 0;
  let tests = 0;
  for (const file of files) {
    const text = fs.readFileSync(file, 'utf8');
    lines += text.split('\n').length;
    tests += countTestDeclarations(text);
  }
  return { files: files.length, lines, tests };
}

function summarizeMockHeavyHandlerFiles(files) {
  return files
    .map((file) => {
      const text = fs.readFileSync(file, 'utf8');
      return {
        file: path.relative(ROOT, file),
        lines: text.split('\n').length,
        tests: countTestDeclarations(text),
      };
    })
    .sort((a, b) => b.lines - a.lines)
    .slice(0, 12);
}

function summarizeProviderPressure(files) {
  const surfaces = [
    {
      name: 'Android ADB provider',
      pattern: /\bAndroidAdbProvider\b|\bandroidAdbProvider\b|\badbProvider\b|\badb\.(?:exec|installer|puller|portReverse)\b/g,
    },
    {
      name: 'Apple runner provider',
      pattern: /\bAppleRunnerProvider\b|\bappleRunnerProvider\b|\b(?:ios|macos|tvos)\.runner\b/g,
    },
    {
      name: 'Apple simctl/devicectl provider',
      pattern:
        /\bsimctl\b|\bdevicectl\b|\brunXcrun\b|\bsimctl\s*:|\bdevicectl\s*:/g,
    },
    {
      name: 'Apple macOS helper provider',
      pattern: /\bmacos-helper\b|\bagent-device-macos-helper\b|\bmacosHelper\s*:/g,
    },
    {
      name: 'Apple macOS host provider',
      pattern:
        /\bmacos-host\b|\bmacosHost\s*:|\bAppleMacOsHostProvider\b|\bopenBundle\b|\bopenTarget\b|\breadClipboard\b|\bwriteClipboard\b|\breadDarkMode\b|\bsetDarkMode\b|\blistApps\b/g,
    },
    {
      name: 'Apple generic host-tool provider',
      pattern:
        /\bxcrun\b|['"](?:open|pbcopy|pbpaste|plutil|osascript|swift|codesign|mdfind|ps|pkill)['"]/g,
    },
    {
      name: 'Linux semantic desktop provider',
      pattern: /\bdesktop\b|\bopenTarget\b|\bcloseApp\b/g,
    },
    {
      name: 'Linux semantic accessibility/clipboard/screenshot provider',
      pattern:
        /\baccessibility\b|\bcaptureTree\b|\bclipboard\b|\breadText\b|\bwriteText\b|\bscreenshot\b|\bcapture\s*:/g,
    },
    {
      name: 'Linux semantic input provider',
      pattern: /\bLinuxInputProvider\b|\bprovider\.input\b|\binput\s*:|\['input'/g,
    },
    {
      name: 'Linux generic tool provider',
      pattern:
        /\bLinuxToolProvider\b|\blinuxToolProvider\b|\brunCommand\b|\bwhichCommand\b|\bxdotool\b|\bydotool\b|\bxclip\b|\bscrot\b|\bgrim\b|\bwmctrl\b|\bpkill\b/g,
    },
    {
      name: 'Recording provider',
      pattern: /\bRecordingProvider\b|\brecordingProvider\b|\bstartRecording\b/g,
    },
  ];

  return surfaces
    .map((surface) => {
      let references = 0;
      let filesWithReferences = 0;
      for (const file of files) {
        const text = fs.readFileSync(file, 'utf8');
        const matches = text.match(surface.pattern)?.length ?? 0;
        references += matches;
        if (matches > 0) filesWithReferences += 1;
      }
      return {
        name: surface.name,
        references,
        files: filesWithReferences,
      };
    })
    .filter((surface) => surface.references > 0);
}

function summarizePublicCommandCoverage(files) {
  const publicCommands = readPublicCommands();
  const commandRefsByFile = files.map((file) => ({
    file,
    commands: extractProviderScenarioCommandReferences(fs.readFileSync(file, 'utf8')),
  }));

  return publicCommands.map((command) => {
    let references = 0;
    let filesWithReferences = 0;
    for (const file of commandRefsByFile) {
      const count = file.commands.filter((candidate) => candidate === command).length;
      references += count;
      if (count > 0) filesWithReferences += 1;
    }
    return { command, references, files: filesWithReferences };
  });
}

function readPublicCommands() {
  return [...readPublicCommandEntries().values()].sort();
}

function readPublicCommandEntries() {
  const match = COMMAND_CATALOG_SOURCE.match(/export const PUBLIC_COMMANDS = \{([\s\S]*?)\} as const;/);
  if (!match) {
    throw new Error('Unable to find PUBLIC_COMMANDS in src/command-catalog.ts');
  }
  const commands = new Map();
  for (const command of match[1].matchAll(/\b([A-Za-z0-9_]+):\s*'([^']+)'/g)) {
    commands.set(command[1], command[2]);
  }
  return commands;
}

function readClientCommandMethods() {
  const commands = new Map();
  for (const file of COMMAND_CONTRACT_FILES) {
    const text = fs.readFileSync(file, 'utf8');
    for (const block of readCommandContractBlocks(text)) {
      for (const method of block.source.matchAll(/\bclient\.([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)\s*\(/g)) {
        commands.set(`${method[1]}.${method[2]}`, block.name);
      }
    }
  }
  return commands;
}

function readCommandContractBlocks(text) {
  const starts = [
    ...text.matchAll(/defineExecutableCommand\(\s*metadata\(\s*['"]([^'"]+)['"]\s*\)/g),
    ...text.matchAll(/defineFieldCommand\(\s*['"]([^'"]+)['"]/g),
    ...text.matchAll(/defineCommand\(\s*\{[\s\S]*?\bname:\s*['"]([^'"]+)['"]/g),
  ]
    .map((match) => ({
      index: match.index ?? 0,
      name: match[1],
    }))
    .sort((a, b) => a.index - b.index);

  return starts.map((start, index) => {
    const end = starts[index + 1]?.index ?? text.length;
    return {
      name: start.name,
      source: text.slice(start.index, end),
    };
  });
}

function extractProviderScenarioCommandReferences(text) {
  const commands = [];
  for (const match of text.matchAll(/\bcommand:\s*['"]([^'"]+)['"]|\.callCommand\(\s*['"]([^'"]+)['"]/g)) {
    commands.push(match[1] ?? match[2]);
  }
  for (const [method, command] of clientCommandMethods) {
    const escapedMethod = method.replace('.', '\\.');
    const matches = text.match(new RegExp(`\\.${escapedMethod}\\s*\\(`, 'g'))?.length ?? 0;
    for (let index = 0; index < matches; index += 1) commands.push(command);
  }
  return commands;
}

function countTestDeclarations(text) {
  return [...text.matchAll(/(?:^|[^\w.])test\(/g)].length;
}

function readCoverageSummary() {
  if (!fs.existsSync(COVERAGE_SUMMARY)) return null;
  const summary = JSON.parse(fs.readFileSync(COVERAGE_SUMMARY, 'utf8'));
  const total = summary.total;
  if (!total) return null;
  return {
    statements: Number(total.statements?.pct ?? 0),
    branches: Number(total.branches?.pct ?? 0),
    functions: Number(total.functions?.pct ?? 0),
    lines: Number(total.lines?.pct ?? 0),
  };
}

function readLowCoverageFiles() {
  if (!fs.existsSync(COVERAGE_SUMMARY)) return [];
  const summary = JSON.parse(fs.readFileSync(COVERAGE_SUMMARY, 'utf8'));
  return Object.entries(summary)
    .filter(([file]) => file !== 'total')
    .map(([file, value]) => {
      const statements = value.statements ?? {};
      const statementTotal = Number(statements.total ?? 0);
      const statementCovered = Number(statements.covered ?? 0);
      return {
        file: path.relative(ROOT, file),
        statementPercent: Number(statements.pct ?? 0),
        statementTotal,
        missingStatements: statementTotal - statementCovered,
      };
    })
    .filter((file) => file.statementTotal >= 10 && file.statementPercent < 60)
    .sort((a, b) => b.missingStatements - a.missingStatements)
    .slice(0, 10);
}

function ratio(numerator, denominator) {
  if (denominator === 0) return 'n/a';
  return `${((numerator / denominator) * 100).toFixed(1)}%`;
}

function formatPercent(value) {
  return `${value.toFixed(2)}%`;
}