Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 49 additions & 1 deletion bin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import { hideBin } from 'yargs/helpers';
import { VERSION } from './src/lib/version.js';

const WIZARD_VERSION = VERSION;

Check warning on line 9 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe assignment of an `any` value

const NODE_VERSION_RANGE = '>=18.17.0';

Expand Down Expand Up @@ -279,7 +279,7 @@
const { startPlayground } = await import(
'./src/ui/tui/playground/start-playground.js'
);
(startPlayground as (version: string) => void)(WIZARD_VERSION);

Check warning on line 282 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe argument of type `any` assigned to a parameter of type `string`
})();
} else if (options.skill) {
// Run a specific skill by ID
Expand Down Expand Up @@ -359,7 +359,7 @@
);

const { Flow } = await import('./src/ui/tui/router.js');
const tui = startTUI(WIZARD_VERSION, Flow.McpAdd);

Check warning on line 362 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe argument of type `any` assigned to a parameter of type `string`
const session = buildSession({
debug: options.debug,
localMcp: options.local,
Expand Down Expand Up @@ -405,7 +405,7 @@
);

const { Flow } = await import('./src/ui/tui/router.js');
const tui = startTUI(WIZARD_VERSION, Flow.McpRemove);

Check warning on line 408 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe argument of type `any` assigned to a parameter of type `string`
const session = buildSession({
debug: options.debug,
localMcp: options.local,
Expand All @@ -428,12 +428,27 @@
.help();
});

// Audit-only options. `--areas` constrains the run to a subset of audit
// areas (Installation, Identification, Web Analytics, …). When omitted
// the agent runs everything its discovery determines applies.
const auditSubcommandOptions = {
areas: {
describe:
'Comma-separated audit areas to constrain the run to (e.g. "Web Analytics, Feature Flags"). See --help for the full list.',
type: 'string' as const,
},
};

// ── Skill-based workflow subcommands (derived from registry) ─────────
for (const wfConfig of getSubcommandWorkflows()) {
const isAudit = wfConfig.command === 'audit';
cli.command(
wfConfig.command!,

Check warning on line 446 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Forbidden non-null assertion
wfConfig.description,
(y) => y.options(skillSubcommandOptions),
(y) =>
isAudit
? y.options({ ...skillSubcommandOptions, ...auditSubcommandOptions })
: y.options(skillSubcommandOptions),
(argv) => {
const options = { ...argv };
if (options.ci) {
Expand Down Expand Up @@ -476,8 +491,10 @@
);
const { analytics } = await import('./src/utils/analytics.js');

const tui = startTUI(WIZARD_VERSION, config.flowKey as any);

Check warning on line 494 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unexpected any. Specify a different type

Check warning on line 494 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe argument of type `any` assigned to a parameter of type `Flow`

Check warning on line 494 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe argument of type `any` assigned to a parameter of type `string`

const auditAreas = await maybeParseAuditAreas(config, options);

const session = buildSession({
debug: options.debug as boolean | undefined,
forceInstall: options.forceInstall as boolean | undefined,
Expand All @@ -489,9 +506,10 @@
projectId: options.projectId as string | undefined,
email: options.email as string | undefined,
menu: options.menu as boolean | undefined,
integration: options.integration as any,

Check warning on line 509 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unexpected any. Specify a different type

Check warning on line 509 in bin.ts

View workflow job for this annotation

GitHub Actions / Lint

Unsafe assignment of an `any` value
benchmark: options.benchmark as boolean | undefined,
yaraReport: options.yaraReport as boolean | undefined,
auditAreas,
});
session.workflowLabel = config.flowKey;
if (options.skillId) {
Expand Down Expand Up @@ -618,6 +636,8 @@
? (options.installDir as string)
: path.join(process.cwd(), options.installDir as string);

const auditAreas = await maybeParseAuditAreas(config, options);

const session = buildSession({
debug: options.debug as boolean | undefined,
forceInstall: options.forceInstall as boolean | undefined,
Expand All @@ -632,6 +652,7 @@
projectId: options.projectId as string | undefined,
benchmark: options.benchmark as boolean | undefined,
yaraReport: options.yaraReport as boolean | undefined,
auditAreas,
...env,
});
session.workflowLabel = config.flowKey;
Expand Down Expand Up @@ -704,3 +725,30 @@
process.exit(1);
});
}

/**
* Parse `--areas` into a typed AuditArea[]. Returns `undefined` when the
* workflow isn't audit or no flag was passed. Logs a hint listing every
* allowed area when unknown values are passed (and ignores them).
*/
async function maybeParseAuditAreas(
config: WorkflowConfig,
options: Record<string, unknown>,
): Promise<import('./src/lib/workflows/audit/areas').AuditArea[] | undefined> {
if (config.command !== 'audit') return undefined;
const areasArg = options.areas as string | undefined;
if (!areasArg) return undefined;

const { parseAuditAreas, formatAreasHint } = await import(
'./src/lib/workflows/audit/areas.js'
);
const { areas, unknown } = parseAuditAreas(areasArg);
if (unknown.length > 0) {
getUI().log.warn(
`Ignoring unknown audit area(s): ${unknown.join(
', ',
)}. Allowed areas: ${formatAreasHint()}.`,
);
}
return areas.length > 0 ? areas : undefined;
}
4 changes: 4 additions & 0 deletions src/lib/__tests__/wizard-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -298,4 +298,8 @@ describe('WIZARD_TOOL_NAMES', () => {
it('exposes audit_add_checks so future workflows can append checks through the MCP server', () => {
expect(WIZARD_TOOL_NAMES).toContain('wizard-tools:audit_add_checks');
});

it('exposes audit_get_areas so the agent can fetch the wizard-supplied area constraint at runtime', () => {
expect(WIZARD_TOOL_NAMES).toContain('wizard-tools:audit_get_areas');
});
});
6 changes: 6 additions & 0 deletions src/lib/agent/agent-interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,11 @@ export type AgentConfig = {
/** Feature flag key -> variant (evaluated at start of run). */
wizardFlags?: Record<string, string>;
wizardMetadata?: Record<string, string>;
/**
* Audit-only: areas the wizard is constraining the run to. Surfaced
* to the agent via the `audit_get_areas` MCP tool.
*/
auditAreas?: ReadonlyArray<import('../workflows/audit/areas').AuditArea>;
};

/**
Expand Down Expand Up @@ -657,6 +662,7 @@ export async function initializeAgent(
workingDirectory: config.workingDirectory,
detectPackageManager: config.detectPackageManager,
skillsBaseUrl: config.skillsBaseUrl,
auditAreas: config.auditAreas,
});
mcpServers['wizard-tools'] = wizardToolsServer;

Expand Down
1 change: 1 addition & 0 deletions src/lib/agent/agent-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ export async function runWorkflow(
skillsBaseUrl,
wizardFlags,
wizardMetadata,
auditAreas: session.auditAreas,
},
sessionToOptions(session),
);
Expand Down
10 changes: 10 additions & 0 deletions src/lib/wizard-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type { Integration } from './constants';
import type { FrameworkConfig } from './framework-config';
import type { WizardReadinessResult } from './health-checks/readiness';
import type { SettingsConflict } from './agent/agent-interface';
import type { AuditArea } from './workflows/audit/areas';

export interface Credentials {
accessToken: string;
Expand Down Expand Up @@ -168,6 +169,13 @@ export interface WizardSession {

// Resolved framework config (set after integration is known)
frameworkConfig: FrameworkConfig | null;

/**
* Audit-only: area-level constraint from `--areas`. Empty / undefined
* means the run is unconstrained. Surfaced to the agent via the
* `audit_get_areas` MCP tool.
*/
auditAreas?: AuditArea[];
}

/**
Expand All @@ -189,6 +197,7 @@ export function buildSession(args: {
benchmark?: boolean;
yaraReport?: boolean;
projectId?: string;
auditAreas?: AuditArea[];
}): WizardSession {
return {
debug: args.debug ?? false,
Expand Down Expand Up @@ -234,5 +243,6 @@ export function buildSession(args: {
workflowLabel: null,
skillId: null,
frameworkConfig: null,
auditAreas: args.auditAreas,
};
}
40 changes: 40 additions & 0 deletions src/lib/wizard-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
type AuditCheck,
type AuditStatus,
} from './workflows/audit/types';
import { ALL_AUDIT_AREAS, type AuditArea } from './workflows/audit/areas';

// ---------------------------------------------------------------------------
// SDK dynamic import (ESM module loaded once, cached)
Expand Down Expand Up @@ -175,6 +176,13 @@ export interface WizardToolsOptions {

/** Base URL for the skills server (e.g. http://localhost:8765 or GitHub releases URL) */
skillsBaseUrl: string;

/**
* Audit-only: subset of audit areas the run is constrained to.
* Surfaced via the `audit_get_areas` MCP tool. Omit / pass `[]` for
* an unconstrained run.
*/
auditAreas?: ReadonlyArray<AuditArea>;
}

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -432,6 +440,7 @@ const SERVER_NAME = 'wizard-tools';
*/
export async function createWizardToolsServer(options: WizardToolsOptions) {
const { workingDirectory, detectPackageManager, skillsBaseUrl } = options;
const auditAreas: ReadonlyArray<AuditArea> = options.auditAreas ?? [];
const sdk = await getSDKModule();
const { tool, createSdkMcpServer } = sdk;

Expand Down Expand Up @@ -811,6 +820,35 @@ export async function createWizardToolsServer(options: WizardToolsOptions) {
},
);

// -- audit_get_areas ------------------------------------------------------

const auditGetAreas = tool(
'audit_get_areas',
`Return the audit areas the wizard is constraining this run to. An empty array means the run is unconstrained — the agent should run every area its discovery determines applies. When non-empty, the agent must skip any area not in the list. Allowed values (canonical capitalization): ${ALL_AUDIT_AREAS.join(
', ',
)}.`,
{},
() => {
logToFile(
`audit_get_areas: returning ${auditAreas.length} area(s)${
auditAreas.length > 0 ? ` [${auditAreas.join(', ')}]` : ''
}`,
);
return Promise.resolve({
content: [
{
type: 'text' as const,
text: JSON.stringify({
areas: [...auditAreas],
allowed: [...ALL_AUDIT_AREAS],
constrained: auditAreas.length > 0,
}),
},
],
});
},
);

// -- Assemble server ------------------------------------------------------

return createSdkMcpServer({
Expand All @@ -825,6 +863,7 @@ export async function createWizardToolsServer(options: WizardToolsOptions) {
auditSeedChecks,
auditAddChecks,
auditResolveChecks,
auditGetAreas,
],
});
}
Expand All @@ -839,6 +878,7 @@ export const WIZARD_TOOL_NAMES = [
`${SERVER_NAME}:audit_seed_checks`,
`${SERVER_NAME}:audit_add_checks`,
`${SERVER_NAME}:audit_resolve_checks`,
`${SERVER_NAME}:audit_get_areas`,
];

// ---------------------------------------------------------------------------
Expand Down
67 changes: 67 additions & 0 deletions src/lib/workflows/audit/__tests__/selection.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { AUDIT_SEED_CHECKS, AUDIT_CORE_CHECKS } from '../seed';
import { AUDIT_SPECIALISTS } from '../specialists';
import { parseAuditAreas, normalizeAuditArea, ALL_AUDIT_AREAS } from '../areas';

describe('audit basic specialist registry', () => {
it('exposes identification + event-capture as the basic specialists', () => {
expect(AUDIT_SPECIALISTS.map((s) => s.area)).toEqual([
'Identification',
'Event Capture',
]);
});
});

describe('audit seed', () => {
it('seeds 3 core + 4 identification + 3 event-capture (10 entries) unconditionally', () => {
expect(AUDIT_SEED_CHECKS).toHaveLength(10);
expect(AUDIT_SEED_CHECKS.slice(0, 3)).toEqual(AUDIT_CORE_CHECKS);
expect(
AUDIT_SEED_CHECKS.filter((c) => c.area === 'Identification'),
).toHaveLength(4);
expect(
AUDIT_SEED_CHECKS.filter((c) => c.area === 'Event Capture'),
).toHaveLength(3);
});

it('every seeded check starts as pending', () => {
for (const check of AUDIT_SEED_CHECKS) {
expect(check.status).toBe('pending');
}
});
});

describe('audit areas', () => {
it('exposes the canonical 8-area enum', () => {
expect(ALL_AUDIT_AREAS).toEqual([
'Installation',
'Identification',
'Event Capture',
'Web Analytics',
'Feature Flags',
'Experiments',
'LLM Analytics',
'Error Tracking',
]);
});

it('normalizes case-insensitively to canonical capitalization', () => {
expect(normalizeAuditArea('web analytics')).toBe('Web Analytics');
expect(normalizeAuditArea('LLM ANALYTICS')).toBe('LLM Analytics');
expect(normalizeAuditArea(' feature flags ')).toBe('Feature Flags');
expect(normalizeAuditArea('not-a-real-area')).toBeNull();
});

it('parses --areas, dedupes, and surfaces unknown values for hinting', () => {
const { areas, unknown } = parseAuditAreas(
'Web Analytics, llm analytics, web analytics, bogus, feature flags',
);
expect(areas).toEqual(['Web Analytics', 'LLM Analytics', 'Feature Flags']);
expect(unknown).toEqual(['bogus']);
});

it('returns empty when no input is provided', () => {
const { areas, unknown } = parseAuditAreas(undefined);
expect(areas).toEqual([]);
expect(unknown).toEqual([]);
});
});
73 changes: 73 additions & 0 deletions src/lib/workflows/audit/areas.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/**
* Audit areas — the canonical, wizard-side enum of areas the audit runner
* may produce.
*
* The wizard exposes the user-supplied subset to the agent at runtime via
* the `audit_get_areas` MCP tool, so the agent (and its discovery subagent)
* can constrain dispatch to those areas only. An empty list means no
* constraint — the agent runs everything.
*
* Adding a new area: append it here AND add a row to context-mill's
* audit description.md "Discoverable specialists" table whose specialist
* produces findings under that area.
*/

export const AUDIT_AREAS = [
'Installation',
'Identification',
'Event Capture',
'Web Analytics',
'Feature Flags',
'Experiments',
'LLM Analytics',
'Error Tracking',
] as const;

export type AuditArea = (typeof AUDIT_AREAS)[number];

export const ALL_AUDIT_AREAS: ReadonlyArray<AuditArea> = AUDIT_AREAS;

/** Case-insensitive lookup. Returns the canonical capitalization on hit. */
export function normalizeAuditArea(value: string): AuditArea | null {
const folded = value.trim().toLowerCase();
return AUDIT_AREAS.find((a) => a.toLowerCase() === folded) ?? null;
}

export function isAuditArea(value: string): value is AuditArea {
return AUDIT_AREAS.includes(value as AuditArea);
}

export interface ParsedAreas {
areas: AuditArea[];
unknown: string[];
}

/**
* Parse a comma-separated `--areas` value. Tokens are matched
* case-insensitively against the canonical enum. Duplicates are dropped.
* Unknown tokens are returned separately so callers can surface a hint
* to the user.
*/
export function parseAuditAreas(input: string | undefined): ParsedAreas {
if (!input) return { areas: [], unknown: [] };
const tokens = input
.split(',')
.map((s) => s.trim())
.filter(Boolean);
const areas: AuditArea[] = [];
const unknown: string[] = [];
for (const token of tokens) {
const canonical = normalizeAuditArea(token);
if (canonical) {
if (!areas.includes(canonical)) areas.push(canonical);
} else {
unknown.push(token);
}
}
return { areas, unknown };
}

/** Human-readable hint listing every allowed area, comma-separated. */
export function formatAreasHint(): string {
return AUDIT_AREAS.join(', ');
}
Loading
Loading