diff --git a/README.md b/README.md index d756ba6..695ca77 100644 --- a/README.md +++ b/README.md @@ -5,29 +5,27 @@ ## Getting Started 1. Install the package `npm i -D apify-test-tools` - - because it uses [annotate](https://vitest.dev/guide/test-context.html#annotate), `vitest` version to be at least `3.2.0` - - make sure that `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022` -2. create test directories: `mkdir -p test/platform/core` - - core (hourly) tests should go to `test/platform/core` - - daily tests should go to `test/platform` -3. setup github worklows TODO + - requires [`vitest`](https://vitest.dev/) `>= 3.2.0` (uses [`annotate`](https://vitest.dev/guide/test-context.html#annotate)) + - set `target` and `module` to `ES2022` in your `tsconfig.json` `compilerOptions` +2. Create a test directory: `mkdir -p test/platform` +3. Set up GitHub workflows (see below) File structure: ``` google-maps ├── actors -└── src +├── src └── test ├── unit └── platform - ├── core <- Core tests need to be inside core directory + ├── core <- Legacy: hourly tests (see "Hourly tests" section) │ └── core.test.ts - ├── some.test.ts <- Other tests can be defined anywhere inside platform directory + ├── some.test.ts └── some-other.test.ts ``` -## Github worklows +## GitHub Workflows There should be 4 GH workflow files in `.github/workflows`. @@ -45,8 +43,6 @@ on: jobs: platformTestsCore: uses: apify-store/github-actions-source/.github/workflows/platform-tests.yaml@new_master - with: - subtest: core secrets: inherit ``` @@ -97,153 +93,139 @@ jobs: secrets: inherit ``` -## Differences in writing tests +## Writing Tests ---- +### Basic usage -### Test structure - -To run the tests concurrently, we had to start the run outside of `it` and then call `await` inside. This is now no longer needed and everything can be inside `it` aka `testActor`. +```ts +import { describe, testActor } from 'apify-test-tools'; -Before: +describe('google-maps', () => { + testActor(actorId, 'smoke test', async ({ run, expect }) => { + const result = await run({ input: { query: 'London Eye' } }); -```ts -({ it, xit, run, expect, expectAsync, input, describe }: TestSpecInputs) => { - describe('test', () => { - { - const runPromise = run({ actorId, input }) - it('actor test 1', async () => { - const runResult = await runPromise; - - // your checks - }); - } - - { - const runPromise = run({ actorId, input }) - it('actor test 2', async () => { - const runResult = await runPromise; - - // your checks - }); - } - }); -}) + await expect(result).toFinishWith({ + datasetItemCount: { min: 1, max: 10 }, + }); + }); +}); ``` -After: +`testActor` provides a `run` function that calls the actor built in the current CI run, and extends `expect` with custom matchers (e.g. `toFinishWith`). -```ts -import { describe, testActor } from 'apify-test-tools'; - -describe('test', () => { - testActor(actorId, 'actor test 1', async ({ expect, run }) => { - const runResult = await run({ input }) +### Validating run results - // your checks - )}; +```ts +await expect(result).toFinishWith({ + // all fields below are optional and have sensible defaults + status: 'SUCCEEDED', + duration: { min: 600, max: 600_000 }, // ms + failedRequests: 0, + requestsRetries: { max: 3 }, + forbiddenLogs: ['ReferenceError', 'TypeError'], - testActor(actorId, 'actor test 2', async ({ expect, run }) => { - const runResult = await run({ input }) + // required — exact number or range + datasetItemCount: { min: 80, max: 120 }, - // your checks - )}; -}) + // optional — PPE event counts; any omitted event is expected to be 0 + chargedEventCounts: { + 'actor-start': 1, + 'place-scraped': { min: 9 }, + }, +}); ``` -`testActor` extends `expect` with couple of custom matchers (e.g. `toFinishWith`) and provides `run` function call the correct actor, based on it’s first parameter - ---- - -### Validating basic run attributes +### Shared validation helpers -Before: +Create reusable helpers in e.g. `test/platform/utils.ts` and import them in test files: ```ts -await expectAsync(runResult).toHaveStatus('SUCCEEDED'); +import { ExpectStatic } from 'apify-test-tools'; -await expectAsync(runResult).withLog((log) => { - expect(log).not.toContain('ReferenceError'); - expect(log).not.toContain('TypeError'); -}); +export const validatePlace = (expect: ExpectStatic, place: unknown) => { + expect(place.title, 'place title').toBeNonEmptyString(); + expect(place.url, 'place url').toBeNonEmptyString(); +}; +``` -await expectAsync(runResult).withStatistics((stats) => { - expect(stats.requestsRetries).withContext(runResult.format('Request retries')).toBeLessThan(3); - expect(stats.crawlerRuntimeMillis).withContext(runResult.format('Run time')).toBeWithinRange(600, 600_000); -}); +## Trigger & Alert Configuration -await expectAsync(runResult).withDataset(({ dataset }) => { - expect(dataset.items?.length).withContext(runResult.format('Dataset cleanItemCount')).toBe(100); -}); -``` +Tests default to running on `daily` and `pullRequest` triggers, with Slack alerts enabled. Use the `triggers` option on `describe` or `testActor` to override. -After: +### Opting out of a trigger ```ts -await expect(runResult).toFinishWith({ - datasetItemCount: 100, -}); +// This suite only runs on daily and hourly, never on pull requests +describe({ + name: 'google-maps', + triggers: { runWhen: { pullRequest: false } }, +}, () => { ... }); ``` -You can also specify a range: +### Running only on specific triggers ```ts -await expect(runResult).toFinishWith({ - datasetItemCount: { min: 80, max: 120 }, -}); +// This test only runs hourly (opt out of daily and pullRequest) +testActor(actorId, { + name: 'extended smoke', + triggers: { runWhen: { daily: false, pullRequest: false } }, +}, async ({ run, expect }) => { ... }); ``` -Here is full example of what you can validate with `toFinishWith` +### Inheriting and overriding through the describe hierarchy + +`triggers` are merged field-by-field from outer to inner — children only need to override what they want to change: ```ts -await expect(runResult).toFinishWith({ - // These are default - status: 'SUCCEEDED', - duration: { - min: 600, // 0.6 sec - max: 600_000, // 10 min +describe( + { + name: 'google-maps', + triggers: { runWhen: { pullRequest: false } }, // disable PR runs for the whole suite }, - failedRequests: 0, - requestsRetries: { max: 3 }, - forbiddenLogs: ['ReferenceError', 'TypeError'], - - // only datasetItemCount is required - datasetItemCount: { min: 80, max: 120 }, - - // optional - chargedEventCounts: { - 'actor-start': 1, - 'place-scraped': 9, + () => { + testActor(actorId, 'smoke', async ({ run, expect }) => { + // inherits pullRequest: false from the describe above + }); + + testActor( + actorId, + { + name: 'extended', + triggers: { runWhen: { daily: false } }, // additionally disable daily + }, + async ({ run, expect }) => { + // effective: pullRequest: false, daily: false → runs hourly only + }, + ); }, -}); +); ``` ---- - -### Custom validations - -Before: +### Disabling Slack alerts ```ts -expect(place.title).withContext(runResult.format(`London Eye's title`)).toEqual('lastminute.com London Eye'); +describe({ + name: 'experimental', + triggers: { alerts: { slack: false } }, // failures won't ping Slack +}, () => { ... }); ``` -After: +### Hourly tests (core directory) + +> **Legacy:** Tests inside `core/` were historically run hourly. This is supported for backward compatibility but new tests should opt in explicitly instead: ```ts -expect(place.title, `London Eye's title`).toEqual('lastminute.com London Eye'); +testActor(actorId, { + name: 'smoke', + triggers: { runWhen: { hourly: true } }, +}, async ({ run, expect }) => { ... }); ``` ---- - -### Custom validation functions - -You can now create your own functions wrapping a common validation logic in e.g. `test/platform/utils.ts` and import it in test files. +### Reading the current trigger at runtime ```ts -import { ExpectStatic } from 'apify-test-tools' +import { getCurrentTrigger, TRIGGER_ENV_VAR } from 'apify-test-tools'; -export const validateItem = (expect: ExpectStatic, item: any) { - expect(item.title, 'Item title').toBeString(); -} +// Returns 'hourly' | 'daily' | 'pullRequest' | undefined +const trigger = getCurrentTrigger(); ``` diff --git a/bin/test-report.ts b/bin/test-report.ts index 257f356..eaea1f9 100644 --- a/bin/test-report.ts +++ b/bin/test-report.ts @@ -36,7 +36,12 @@ export const reportTestResults = async ({ } } - const failedAssertions: { message: string; runLink: string; actorName: string }[] = []; + const failedAssertions: { + message: string; + runLink: string; + actorName: string; + alerts: JsonAssertionResult['meta']['alerts']; + }[] = []; console.error(); console.error(`PASSED: ${passed.length}, FAILED: ${failed.length}`); @@ -63,6 +68,7 @@ export const reportTestResults = async ({ message: message.split('\n')?.[0], runLink: meta.runLink, actorName: meta.actorName, + alerts: meta.alerts, })), ); } @@ -80,7 +86,10 @@ export const reportTestResults = async ({ return; } - if (failedAssertions.length === 0) { + // Default to true when alerts is not configured — backward-compatible + const slackAssertions = failedAssertions.filter(({ alerts }) => alerts?.slack !== false); + + if (slackAssertions.length === 0) { return; } @@ -88,19 +97,15 @@ export const reportTestResults = async ({ const total = failed.length + passed.length; const jobLink = jobUrl ? ` Check <${jobUrl}|the job>.` : ''; let slackMessage = `\`${workflowName ?? '-'}\``; - slackMessage += `: has ${failedAssertions.length} failed assertions. Failing test suites: ${failed.length}/${total}.${jobLink}`; - slackMessage += `\n\n${failedAssertions[0].message} --- <${failedAssertions[0].runLink}|${failedAssertions[0].actorName}>`; - const blocks = failedAssertions + slackMessage += `: has ${slackAssertions.length} failed assertions. Failing test suites: ${failed.length}/${total}.${jobLink}`; + slackMessage += `\n\n${slackAssertions[0].message} --- <${slackAssertions[0].runLink}|${slackAssertions[0].actorName}>`; + const blocks = slackAssertions .slice(1) .map(({ message, runLink, actorName }) => `• ${message} --- <${runLink}|${actorName}>`); console.error('SLACK:', slackMessage); console.error('\tblocks:', blocks.join('\n\t\t')); - if (!reportSlackChannel) { - return; - } - if (!dryRun) { const slackToken = getEnvVar('SLACK_TOKEN_TESTS_BOT'); await sendSlackMessage(reportSlackChannel, slackMessage, blocks, slackToken); @@ -123,6 +128,13 @@ interface JsonAssertionResult { runId: string; runLink: string; actorName: string; + /** + * Alerting config set by the test via `alerts` in `testActor`/`describe`. + * `undefined` means the test didn't opt in or out — treat as "notify" for + * backward compatibility. + * `slack: false` explicitly disables the Slack notification for that test. + */ + alerts?: { slack?: boolean }; }; duration?: Milliseconds | null; failureMessages: string[] | null; diff --git a/index.ts b/index.ts index 255c0fa..35edbd8 100644 --- a/index.ts +++ b/index.ts @@ -1,2 +1,12 @@ -export { describe, testActor, testStandbyActor, ExpectStatic } from './lib/lib.js'; +export { describe, testActor, testStandbyActor, ExpectStatic, getCurrentTrigger, TRIGGER_ENV_VAR } from './lib/lib.js'; export { RunTestResult } from './lib/run-test-result.js'; +export type { + TriggerType, + RunWhenConfig, + AlertsConfig, + TriggerConfig, + DescribeConfig, + DescribeOptions, + TestActorConfig, + ActorOptions, +} from './lib/types.js'; diff --git a/lib/consts.ts b/lib/consts.ts index e092b73..208a888 100644 --- a/lib/consts.ts +++ b/lib/consts.ts @@ -1,4 +1,30 @@ -import type { ToFinishWithOptionsWithDefaults } from './types.js'; +import type { AlertsConfig, RunWhenConfig, ToFinishWithOptionsWithDefaults } from './types.js'; + +/** + * Both the test runs and the vitest test specs should finish in this time - 1 hour + */ +export const DEFAULT_TEST_RUN_DURATION_MS = 60 * 60 * 1000; // 1 hour + +// Default trigger config. +// `Required<...>` on both runWhen and alerts ensures a compile error when a new field is +// added, forcing an explicit opt-in/opt-out decision for existing tests. +// hourly is false by default — only specific directories (e.g. core/) run hourly, +// controlled via BACKWARD_COMPATIBLE_HOURLY_DIR. +export const DEFAULT_TRIGGERS: { runWhen: Required; alerts: Required } = { + runWhen: { hourly: false, daily: true, pullRequest: true }, + alerts: { slack: true }, +}; + +export const DEFAULT_DESCRIBE_OPTIONS = { + concurrent: true, + timeout: DEFAULT_TEST_RUN_DURATION_MS, +}; + +export const DEFAULT_TEST_ACTOR_OPTIONS = { + retry: 1, + // prevent orphaned runs + timeout: DEFAULT_TEST_RUN_DURATION_MS, +}; export const TO_FINISH_WITH_OPTIONS: ToFinishWithOptionsWithDefaults = { status: 'SUCCEEDED', @@ -11,8 +37,3 @@ export const TO_FINISH_WITH_OPTIONS: ToFinishWithOptionsWithDefaults = { maxRetriesPerRequest: null, forbiddenLogs: ['ReferenceError', 'TypeError'], }; - -/** - * Both the test runs and the vitest test specs should finish in this time - 1 hour - */ -export const DEFAULT_TEST_RUN_DURATION_MS = 60 * 60 * 1000; // 1 hour diff --git a/lib/lib.ts b/lib/lib.ts index c4590b7..d60ec77 100644 --- a/lib/lib.ts +++ b/lib/lib.ts @@ -1,22 +1,38 @@ +import { fileURLToPath } from 'node:url'; + import type { Actor, ActorRun, ActorRunListItem, ActorStandby, Task } from 'apify-client'; import { ApifyClient } from 'apify-client'; import type { SuiteFactory, TestContext, TestFunction } from 'vitest'; import { describe as vitestDescribe, ExpectStatic, test as vitestTest } from 'vitest'; -import { DEFAULT_TEST_RUN_DURATION_MS } from './consts.js'; +import { DEFAULT_DESCRIBE_OPTIONS, DEFAULT_TEST_ACTOR_OPTIONS, DEFAULT_TRIGGERS } from './consts.js'; import { extendExpect } from './extend-expect.js'; import { RunTestResult } from './run-test-result.js'; -import type { ActorBuild, ActorTestOptions, RunOptions } from './types.js'; +import { shouldRunForTrigger } from './trigger.js'; +import type { + ActorBuild, + ActorTestOptions, + DescribeConfig, + RunOptions, + TestActorConfig, + TriggerConfig, +} from './types.js'; import { getActorPrefilledInput, sleep } from './utils.js'; -const ACTOR_BUILDS = 'ACTOR_BUILDS'; +export { getCurrentTrigger, TRIGGER_ENV_VAR } from './trigger.js'; +export { ExpectStatic }; + +// --------------------------------------------------------------------------- +// Actor builds — loaded once at startup from the ACTOR_BUILDS env var +// --------------------------------------------------------------------------- + let actorBuilds: ActorBuild[] = []; try { - const actorBuildsEnv = process.env[ACTOR_BUILDS]; - if (actorBuildsEnv) { - actorBuilds = JSON.parse(actorBuildsEnv); + const raw = process.env.ACTOR_BUILDS; + if (raw) { + actorBuilds = JSON.parse(raw); if (!Array.isArray(actorBuilds)) { - throw new Error(`${ACTOR_BUILDS} env variable should contain a JSON array, got ${typeof actorBuilds}`); + throw new Error(`ACTOR_BUILDS env variable should contain a JSON array, got ${typeof actorBuilds}`); } } } catch (err) { @@ -27,43 +43,177 @@ const config = actorBuilds.reduce>((map, cfg) => { map.set(cfg.actorName, cfg); map.set(cfg.actorId, cfg); return map; -}, new Map()); - -export { ExpectStatic }; +}, new Map()); const { TESTER_APIFY_TOKEN, RUN_PLATFORM_TESTS, RUN_ALL_PLATFORM_TESTS } = process.env; const apifyClient = new ApifyClient({ token: TESTER_APIFY_TOKEN }); -const DEFAULT_TEST_OPTIONS: ActorTestOptions = { - // we want to run tests concurrently - concurrent: true, - // test should finish within 1 hour - timeout: DEFAULT_TEST_RUN_DURATION_MS, -}; +// Strip the extension so the comparison works for both .ts (source maps) and .js (compiled). +const THIS_FILE_BASE = fileURLToPath(import.meta.url).replace(/\.[jt]s$/, ''); -export const describe = (name: string, fn?: SuiteFactory, options: ActorTestOptions = DEFAULT_TEST_OPTIONS) => { - vitestDescribe.runIf(!!RUN_PLATFORM_TESTS || !!RUN_ALL_PLATFORM_TESTS)(name, options, fn); -}; +/** + * Returns the file path of the first call-stack frame outside this library file. + * Used at describe/testActor registration time to detect which test file is calling. + */ +function getCallerFile(): string | undefined { + for (const line of (new Error().stack ?? '').split('\n').slice(1)) { + const match = line.match(/\((.+?):\d+:\d+\)/) ?? line.match(/^\s+at (.+?):\d+:\d+\s*$/); + if (!match) continue; + const filePath = match[1].replace(/^file:\/\//, ''); + if (filePath.replace(/\.[jt]s$/, '') === THIS_FILE_BASE) continue; + if (filePath.startsWith('node:') || filePath.includes('node_modules')) continue; + return filePath; + } + return undefined; +} -const DEFAULT_TEST_ACTOR_OPTIONS: ActorTestOptions = { - retry: 1, - // prevent orphaned runs - timeout: DEFAULT_TEST_RUN_DURATION_MS, +/** + * Returns DEFAULT_TRIGGERS, with hourly promoted to true when the caller file is + * under BACKWARD_COMPATIBLE_HOURLY_DIR. This allows a specific directory (e.g. core/) + * to retain pre-config-system hourly behaviour without touching individual test files. + */ +function getEffectiveDefaults(): TriggerConfig { + const hourlyDir = process.env.BACKWARD_COMPATIBLE_HOURLY_DIR; + if (hourlyDir) { + const callerFile = getCallerFile(); + if (callerFile && (callerFile.includes(`/${hourlyDir}/`) || callerFile.includes(`\\${hourlyDir}\\`))) { + return { ...DEFAULT_TRIGGERS, runWhen: { ...DEFAULT_TRIGGERS.runWhen, hourly: true } }; + } + } + return DEFAULT_TRIGGERS; +} + +// --------------------------------------------------------------------------- +// Hierarchical trigger stack +// Describes push their triggers onto the stack before collecting their +// children; testActor reads the merged result at registration time. +// Since vitest calls the suite factory synchronously, the stack is always +// consistent during collection. +// --------------------------------------------------------------------------- +const triggersStack: TriggerConfig[] = []; + +/** + * Merges a sequence of trigger layers left-to-right (outermost → innermost). + * Both `runWhen` and `alerts` are shallow-merged field-by-field so children only + * need to override the specific keys they want to change. + * + * Exported for unit testing. + */ +export function mergeInheritedTriggers(layers: TriggerConfig[]): TriggerConfig { + return layers.reduce( + (merged, layer) => ({ + runWhen: layer.runWhen !== undefined ? { ...merged.runWhen, ...layer.runWhen } : merged.runWhen, + alerts: layer.alerts !== undefined ? { ...merged.alerts, ...layer.alerts } : merged.alerts, + }), + {}, + ); +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Wraps a suite of actor tests. Conditionally runs based on environment flags + * and the `runWhen` trigger config. + * + * Preferred (new) style — config object with `name`: + * ```ts + * // All triggers enabled by default — opt out of specific ones: + * describe({ name: 'my-actor', triggers: { runWhen: { pullRequest: false }, alerts: { slack: false } } }, () => { ... }); + * ``` + * + * Legacy style — still supported: + * ```ts + * describe('my-actor', () => { ... }); + * describe('my-actor', () => { ... }, { timeout: 120_000 }); + * ``` + */ +export const describe = ( + configOrName: DescribeConfig | string, + fn?: SuiteFactory, + legacyOptions?: ActorTestOptions, +) => { + const resolved: DescribeConfig = + typeof configOrName === 'string' + ? { name: configOrName, options: { ...DEFAULT_DESCRIBE_OPTIONS, ...legacyOptions } } + : { options: DEFAULT_DESCRIBE_OPTIONS, ...configOrName }; + + const { name, triggers, options } = resolved; + + triggersStack.push(triggers ?? {}); + const merged = mergeInheritedTriggers([getEffectiveDefaults(), ...triggersStack]); + const shouldRun = (!!RUN_PLATFORM_TESTS || !!RUN_ALL_PLATFORM_TESTS) && shouldRunForTrigger(merged.runWhen); + + vitestDescribe.runIf(shouldRun)(name, options ?? {}, (test) => { + fn?.(test); + }); + + triggersStack.pop(); }; +/** + * Resolves and merges config for testActor / testStandbyActor. + * Handles both the new config-object style and the legacy string style. + */ +function resolveActorTestConfig( + actorName: string, + configOrName: TestActorConfig | string, + legacyOptions?: ActorTestOptions, +) { + const resolved: TestActorConfig = + typeof configOrName === 'string' + ? { name: configOrName, options: { ...DEFAULT_TEST_ACTOR_OPTIONS, ...legacyOptions } } + : { ...configOrName, options: { ...DEFAULT_TEST_ACTOR_OPTIONS, ...configOrName.options } }; + + const { name, triggers, options } = resolved; + + const effectiveTriggers = mergeInheritedTriggers([ + getEffectiveDefaults(), + ...triggersStack, + ...(triggers !== undefined ? [triggers] : []), + ]); + const shouldRun = + (!!RUN_ALL_PLATFORM_TESTS || config.has(actorName)) && shouldRunForTrigger(effectiveTriggers.runWhen); + + return { fullName: `${actorName}: ${name}`, effectiveTriggers, vitestOptions: options ?? {}, shouldRun }; +} + +/** + * Registers a platform actor test. Conditionally runs based on whether the + * actor was built and the `runWhen` trigger config (inherited from enclosing + * `describe` blocks and optionally overridden per test). + * + * Preferred (new) style — config object with `name`: + * ```ts + * // Inherits triggers from enclosing describe; override only what differs: + * testActor(actorId, { name: 'smoke', triggers: { runWhen: { pullRequest: false } } }, async ({ run, expect }) => { ... }); + * ``` + * + * Legacy style — still supported: + * ```ts + * testActor(actorId, 'smoke', async ({ run, expect }) => { ... }); + * testActor(actorId, 'smoke', async ({ run, expect }) => { ... }, { retry: 2 }); + * ``` + */ export const testActor = ( actorName: string, - testName: string, + configOrName: TestActorConfig | string, fn: TestFunction<{ run: ReturnType> }>, - testOptions?: ActorTestOptions, + legacyOptions?: ActorTestOptions, ) => { - const options = { - ...DEFAULT_TEST_ACTOR_OPTIONS, - ...testOptions, - }; - const name = `${actorName}: ${testName}`; - const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorName); - vitestTest.runIf(shouldRun)(name, options, async (context: TYPE) => { + const { fullName, effectiveTriggers, vitestOptions, shouldRun } = resolveActorTestConfig( + actorName, + configOrName, + legacyOptions, + ); + + vitestTest.runIf(shouldRun)(fullName, vitestOptions, async (context: TYPE) => { + // Embed alerts config in task.meta so the JSON reporter serializes it + // and report-tests can read it alongside runLink / actorName. + // @ts-expect-error: `TaskMeta` cannot be retyped + context.task.meta = { ...context.task.meta, alerts: effectiveTriggers.alerts }; + const { expect, ...rest } = context; await fn({ expect: extendExpect(expect), @@ -74,26 +224,36 @@ export const testActor = ( }; /** - * This wrapper creates a new task with specific `build` of the standby actor and provides - * `callStandby` function, which calls the task's `standbyUrl`. + * Creates a new task with a specific build of the standby actor and provides + * a `callStandby` function that calls the task's standby URL. + * + * Preferred (new) style — config object with `name`: + * ```ts + * testStandbyActor(actorId, { name: 'CDS standby', triggers: { runWhen: { pullRequest: false } } }, async ({ callStandby }) => { ... }); + * ``` * - * Using task is just current shortcoming of standby feature but ideally we would use Actor directly + * Legacy style — still supported: + * ```ts + * testStandbyActor(actorId, 'CDS standby', async ({ callStandby }) => { ... }); + * ``` */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export const testStandbyActor = ( actorName: string, - testName: string, + configOrName: TestActorConfig | string, fn: TestFunction<{ callStandby: ReturnType> }>, - testOptions?: ActorTestOptions, + legacyOptions?: ActorTestOptions, ) => { - const options = { - ...DEFAULT_TEST_ACTOR_OPTIONS, - ...testOptions, - }; - const name = `${actorName}: ${testName}`; - const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorName); + const { fullName, effectiveTriggers, vitestOptions, shouldRun } = resolveActorTestConfig( + actorName, + configOrName, + legacyOptions, + ); + + vitestTest.runIf(shouldRun)(fullName, vitestOptions, async (context: T) => { + // @ts-expect-error: `TaskMeta` cannot be retyped + context.task.meta = { ...context.task.meta, alerts: effectiveTriggers.alerts }; - vitestTest.runIf(shouldRun)(name, options, async (context: T) => { const standbyTask = await createStandbyTask(actorName, config.get(actorName)?.buildNumber); const { expect, ...rest } = context; @@ -101,7 +261,7 @@ export const testStandbyActor = ( try { await fn({ expect: extendExpect(expect), - callStandby: createStartStandbyFn(standbyTask, context, name), + callStandby: createStartStandbyFn(standbyTask, context, fullName), ...rest, }); } finally { diff --git a/lib/trigger.ts b/lib/trigger.ts new file mode 100644 index 0000000..529cdae --- /dev/null +++ b/lib/trigger.ts @@ -0,0 +1,41 @@ +import type { RunWhenConfig, TriggerType } from './types.js'; + +export const TRIGGER_ENV_VAR = 'TEST_TRIGGER'; + +const VALID_TRIGGERS: readonly TriggerType[] = ['hourly', 'daily', 'pullRequest']; + +/** + * Returns the current trigger type from the `TEST_TRIGGER` environment variable, + * or `undefined` when the variable is absent or unrecognized. + * When `undefined`, tests run unconditionally (no trigger-based filtering). + * + * In GitHub Actions workflows, set the env var before running tests: + * ```yaml + * env: + * TEST_TRIGGER: pullRequest # or: hourly | daily + * ``` + */ +export function getCurrentTrigger(): TriggerType | undefined { + const raw = process.env[TRIGGER_ENV_VAR]; + if (raw && (VALID_TRIGGERS as readonly string[]).includes(raw)) { + return raw as TriggerType; + } + return undefined; +} + +/** + * Returns `true` when the test should run for the current trigger. + * + * - If `runWhen` is omitted the test always runs (backwards-compatible default). + * - If `TEST_TRIGGER` is not set, the test always runs (no filtering). + * - Otherwise runs only when `runWhen[currentTrigger] === true`. + * + * Note: callers are expected to pass the fully-merged `runWhen` (including + * inherited defaults), so all trigger keys should already be explicitly set. + */ +export function shouldRunForTrigger(runWhen: RunWhenConfig | undefined): boolean { + if (!runWhen) return true; + const trigger = getCurrentTrigger(); + if (!trigger) return true; + return runWhen[trigger] === true; +} diff --git a/lib/types.ts b/lib/types.ts index 6b301f3..ebe763e 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -98,6 +98,105 @@ export type RunStatus = | 'TIMING-OUT' | 'TIMED-OUT'; +/** + * Named trigger types that can gate a test or suite. + * The active trigger is supplied via the `TEST_TRIGGER` environment variable. + */ +export type TriggerType = 'hourly' | 'daily' | 'pullRequest'; + +/** + * Controls which trigger types cause the test/suite to be included in the run. + * When `TEST_TRIGGER` is not set, all tests run regardless of `runWhen`. + * + * Each field defaults to the value set in `DEFAULT_TRIGGERS` (currently `daily` and + * `pullRequest` are true, `hourly` is false). Set a field to `false` to opt out of + * a trigger, or `true` to opt in. + * + * Keys are merged field-by-field through the describe hierarchy, so a child + * only needs to override the specific trigger it wants to change. + * + * Example — disable only PR runs: + * ```ts + * runWhen: { pullRequest: false } + * ``` + */ +export type RunWhenConfig = Partial>; + +/** + * Defines which alerting channels fire when this test/suite fails. + * Evaluated by the `report-tests` command after the run. + * Keys are merged field-by-field through the describe hierarchy. + * + * Example: + * ```ts + * alerts: { slack: true } + * ``` + */ +export type AlertsConfig = { + slack?: boolean; +}; + +/** + * Trigger and alerting configuration, shared between `DescribeConfig` and + * `TestActorConfig`. Inherited and merged field-by-field down the describe + * hierarchy so children only need to override what they want to change. + */ +export type TriggerConfig = { + runWhen?: RunWhenConfig; + alerts?: AlertsConfig; +}; + +/** Vitest-level options for a `describe` suite. */ +export type DescribeOptions = { + timeout?: number; + concurrent?: boolean; + sequential?: boolean; +}; + +/** Vitest-level options for an individual `testActor` / `testStandbyActor`. */ +export type ActorOptions = { + retry?: number; + timeout?: number; +}; + +/** + * Config object passed as the first argument to `describe`. + * `triggers` is inherited and merged with nested describes / testActors. + * + * Example: + * ```ts + * describe({ + * name: 'my-actor', + * triggers: { runWhen: { daily: true }, alerts: { slack: true } }, + * options: { concurrent: false }, + * }, () => { ... }); + * ``` + */ +export type DescribeConfig = { + name: string; + triggers?: TriggerConfig; + options?: DescribeOptions; +}; + +/** + * Config object passed as the second argument to `testActor` / `testStandbyActor`. + * `triggers` merges field-by-field with whatever was set on enclosing describes. + * + * Example: + * ```ts + * testActor(actorId, { + * name: 'smoke', + * triggers: { runWhen: { hourly: true } }, + * options: { retry: 2 }, + * }, async ({ run }) => { ... }); + * ``` + */ +export type TestActorConfig = { + name: string; + triggers?: TriggerConfig; + options?: ActorOptions; +}; + export interface ActorMatchers { toBeArray: () => R; toBeBoolean: () => R; diff --git a/test/unit/trigger.test.ts b/test/unit/trigger.test.ts new file mode 100644 index 0000000..cf0cdd9 --- /dev/null +++ b/test/unit/trigger.test.ts @@ -0,0 +1,89 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { getCurrentTrigger, shouldRunForTrigger, TRIGGER_ENV_VAR } from '../../lib/trigger.js'; + +describe('getCurrentTrigger', () => { + const saved = process.env[TRIGGER_ENV_VAR]; + + afterEach(() => { + if (saved === undefined) { + delete process.env[TRIGGER_ENV_VAR]; + } else { + process.env[TRIGGER_ENV_VAR] = saved; + } + }); + + it('returns undefined when TEST_TRIGGER is not set', () => { + delete process.env[TRIGGER_ENV_VAR]; + expect(getCurrentTrigger()).toBeUndefined(); + }); + + it.each(['hourly', 'daily', 'pullRequest'] as const)('returns $0 when TEST_TRIGGER=$0', (trigger) => { + process.env[TRIGGER_ENV_VAR] = trigger; + expect(getCurrentTrigger()).toBe(trigger); + }); + + it.each(['weekly', 'locally', 'on-call', '', ' '])('returns undefined for unrecognized value "%s"', (value) => { + process.env[TRIGGER_ENV_VAR] = value; + expect(getCurrentTrigger()).toBeUndefined(); + }); +}); + +describe('shouldRunForTrigger', () => { + beforeEach(() => { + delete process.env[TRIGGER_ENV_VAR]; + }); + + afterEach(() => { + delete process.env[TRIGGER_ENV_VAR]; + }); + + describe('no runWhen config', () => { + it('always runs when runWhen is undefined', () => { + expect(shouldRunForTrigger(undefined)).toBe(true); + }); + }); + + describe('TEST_TRIGGER not set', () => { + it('runs regardless of runWhen when no trigger is set', () => { + expect(shouldRunForTrigger({ daily: true })).toBe(true); + expect(shouldRunForTrigger({ hourly: false })).toBe(true); + expect(shouldRunForTrigger({})).toBe(true); + }); + }); + + describe('trigger matches runWhen', () => { + it('runs when the current trigger is explicitly true', () => { + process.env[TRIGGER_ENV_VAR] = 'daily'; + expect(shouldRunForTrigger({ daily: true })).toBe(true); + }); + + it('runs when multiple triggers are enabled and current one is among them', () => { + process.env[TRIGGER_ENV_VAR] = 'pullRequest'; + expect(shouldRunForTrigger({ daily: true, pullRequest: true })).toBe(true); + }); + }); + + describe('trigger not in runWhen', () => { + it('does not run when the current trigger is explicitly false', () => { + process.env[TRIGGER_ENV_VAR] = 'hourly'; + expect(shouldRunForTrigger({ hourly: false })).toBe(false); + }); + + it('does not run when the current trigger is absent (undefined !== true)', () => { + process.env[TRIGGER_ENV_VAR] = 'hourly'; + // Note: in practice the merge stack always prepends DEFAULT_TRIGGERS so the + // merged runWhen passed here will have all triggers explicitly set. + expect(shouldRunForTrigger({})).toBe(false); + expect(shouldRunForTrigger({ daily: true })).toBe(false); + }); + + it.each(['hourly', 'daily', 'pullRequest'] as const)( + 'does not run for %s when that trigger is explicitly false', + (trigger) => { + process.env[TRIGGER_ENV_VAR] = trigger; + expect(shouldRunForTrigger({ [trigger]: false })).toBe(false); + }, + ); + }); +}); diff --git a/test/unit/triggers-merging.test.ts b/test/unit/triggers-merging.test.ts new file mode 100644 index 0000000..5507fc8 --- /dev/null +++ b/test/unit/triggers-merging.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from 'vitest'; + +import { mergeInheritedTriggers } from '../../lib/lib.js'; + +describe('mergeInheritedTriggers', () => { + describe('empty / no layers', () => { + it('returns empty config for an empty stack', () => { + expect(mergeInheritedTriggers([])).toEqual({}); + }); + + it('returns the single layer as-is', () => { + expect(mergeInheritedTriggers([{ runWhen: { daily: true } }])).toEqual({ + runWhen: { daily: true }, + }); + }); + }); + + describe('runWhen — shallow merge, child keys override', () => { + it('child runWhen is merged with parent runWhen field-by-field', () => { + const result = mergeInheritedTriggers([{ runWhen: { daily: true } }, { runWhen: { hourly: true } }]); + expect(result.runWhen).toEqual({ daily: true, hourly: true }); + }); + + it('child can override a specific trigger set by parent', () => { + const result = mergeInheritedTriggers([{ runWhen: { daily: true } }, { runWhen: { daily: false } }]); + expect(result.runWhen).toEqual({ daily: false }); + }); + + it('grandchild merges across all ancestors', () => { + const result = mergeInheritedTriggers([ + { runWhen: { daily: true } }, + { runWhen: { hourly: true } }, + { runWhen: { pullRequest: true } }, + ]); + expect(result.runWhen).toEqual({ daily: true, hourly: true, pullRequest: true }); + }); + + it('inherits parent runWhen when child has none', () => { + const result = mergeInheritedTriggers([ + { runWhen: { daily: true } }, + { alerts: { slack: true } }, // no runWhen + ]); + expect(result.runWhen).toEqual({ daily: true }); + }); + + it('child can disable a trigger set by grandparent via intermediate layer', () => { + const result = mergeInheritedTriggers([ + { runWhen: { daily: true, hourly: true } }, + {}, // intermediate — no runWhen + { runWhen: { hourly: false } }, + ]); + expect(result.runWhen).toEqual({ daily: true, hourly: false }); + }); + }); + + describe('alerts — shallow merge, child keys override', () => { + it('inherits parent alerts when child has none', () => { + const result = mergeInheritedTriggers([{ alerts: { slack: true } }, {}]); + expect(result.alerts).toEqual({ slack: true }); + }); + + it('child alerts override parent keys', () => { + const result = mergeInheritedTriggers([{ alerts: { slack: true } }, { alerts: { slack: false } }]); + expect(result.alerts).toEqual({ slack: false }); + }); + + it('accumulates alerts across layers when keys are disjoint', () => { + // If we add more alert keys in future, they should accumulate + const result = mergeInheritedTriggers([ + { alerts: { slack: true } }, + { alerts: {} }, // empty override still triggers shallow merge + ]); + // empty override produces { ...{ slack: true }, ...{} } = { slack: true } + expect(result.alerts).toEqual({ slack: true }); + }); + + it('three layers — deepest alerts win per key', () => { + const result = mergeInheritedTriggers([ + { alerts: { slack: true } }, + { alerts: { slack: false } }, + { alerts: { slack: true } }, + ]); + expect(result.alerts).toEqual({ slack: true }); + }); + }); + + describe('combined runWhen + alerts across layers', () => { + it('resolves both independently across a realistic describe → testActor stack', () => { + // Outer describe: daily + slack + // Inner describe: no extra config + // testActor: add hourly without losing daily + const result = mergeInheritedTriggers([ + { runWhen: { daily: true }, alerts: { slack: true } }, + {}, + { runWhen: { hourly: true } }, + ]); + expect(result).toEqual({ + runWhen: { daily: true, hourly: true }, // merged field-by-field + alerts: { slack: true }, // inherited from outer describe + }); + }); + + it('testActor can disable a trigger from enclosing describe', () => { + const result = mergeInheritedTriggers([ + { runWhen: { daily: true, pullRequest: true }, alerts: { slack: true } }, + { runWhen: { pullRequest: false } }, // testActor opts out of pullRequest + ]); + expect(result).toEqual({ + runWhen: { daily: true, pullRequest: false }, + alerts: { slack: true }, + }); + }); + + it('testActor with no config inherits everything from enclosing describe', () => { + const result = mergeInheritedTriggers([ + { runWhen: { pullRequest: true }, alerts: { slack: true } }, + {}, // testActor with no overrides + ]); + expect(result).toEqual({ + runWhen: { pullRequest: true }, + alerts: { slack: true }, + }); + }); + }); +});