From 5f411300700f26e4caa308d5c1213af9aa9a30e1 Mon Sep 17 00:00:00 2001 From: PredictabilityAtScale <131020168+PredictabilityAtScale@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:47:32 -0700 Subject: [PATCH] Add policy lint checks to validate command --- docs/validation.md | 13 +++- src/validation/validate.ts | 146 ++++++++++++++++++++++++++++++++++++- tests/validation.test.ts | 59 +++++++++++++++ 3 files changed, 215 insertions(+), 3 deletions(-) diff --git a/docs/validation.md b/docs/validation.md index 489d026..d91f663 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -36,6 +36,14 @@ const result = await kit.validatePrompt('support/reply'); | `POK012` | Warning | Variable declared in `context.inputs` but never used | | `POK013` | Error | Invalid context regex pattern (`allow_regex` or `deny_regex`), including prompt id, variable name, field name, and raw configured value | | `POK014` | Warning | `trim` configured without `max_size` (trim-to-budget skipped) | +| `POK040` | Warning | Risky context input appears unbounded (`max_size` missing) | +| `POK041` | Warning | Context input has no hardening validators (`allow/deny regex`, `non_empty`, `reject_secrets`) | +| `POK042` | Warning | Provider has no provider-specific cache config | +| `POK043` | Warning | `cache.gemini.cached_content` and `cache.google.cached_content` conflict | +| `POK044` | Warning | Provider configured without an explicit `model` | +| `POK045` | Warning | Environment/tier cache override may be missing while base cache is defined | +| `POK046` | Warning | Template uses variables but `context.inputs` is not declared | +| `POK047` | Warning | Inline tool definition missing `description` or `input_schema` | | `POK033` | Runtime error | `non_empty` validation failed | | `POK034` | Runtime error | `reject_secrets` validation matched | | `POK020` | Error | Include resolution failed (missing file) | @@ -49,7 +57,7 @@ Unknown front matter keys are checked against known keys using Levenshtein dista ⚠ POK010: Unknown front matter field: "tempreature" (Did you mean "temperature"?) ``` -Known front matter keys: `id`, `schema_version`, `description`, `provider`, `model`, `fallback_models`, `reasoning`, `sampling`, `response`, `tools`, `mcp`, `context`, `includes`, `environments`, `tiers`, `metadata`. +Known front matter keys: `id`, `schema_version`, `description`, `provider`, `model`, `fallback_models`, `reasoning`, `sampling`, `response`, `tools`, `mcp`, `context`, `includes`, `environments`, `tiers`, `metadata`, `cache`, `provider_options`. ## Variable validation @@ -114,6 +122,9 @@ context: - `reject_secrets` rejects common secret-like strings with `POK034`, unless `return_message` is configured. - During static validation and compilation, malformed `allow_regex` or `deny_regex` patterns are reported as `POK013`. - During static validation, `trim` without `max_size` returns a `POK014` warning. +- During static validation, risky unbounded inputs and missing hardening are flagged as `POK040` and `POK041`. +- During static validation, provider/cache hygiene checks can emit `POK042`–`POK045`. +- During static validation, inline tool quality checks can emit `POK047`. Regex compilation errors include the prompt id, variable name, field name, and raw configured value to make bad prompt definitions easy to locate. diff --git a/src/validation/validate.ts b/src/validation/validate.ts index aedc34a..c0dca31 100644 --- a/src/validation/validate.ts +++ b/src/validation/validate.ts @@ -4,7 +4,6 @@ import { extractVariables } from '../renderer/index.js'; import { resolveIncludes } from '../composition/index.js'; import { compileContextRegex, - formatInvalidContextRegexMessage, getContextInputs, getContextInputNames, } from '../context.js'; @@ -26,9 +25,21 @@ export interface PromptValidationResult { const KNOWN_FRONT_MATTER_KEYS = new Set([ 'id', 'schema_version', 'description', 'provider', 'model', 'fallback_models', 'reasoning', 'sampling', 'response', 'tools', 'mcp', 'context', 'includes', - 'environments', 'tiers', 'metadata', 'cache', + 'environments', 'tiers', 'metadata', 'cache', 'provider_options', ]); +const RISKY_UNBOUNDED_INPUT_NAMES = [ + 'message', + 'prompt', + 'history', + 'transcript', + 'document', + 'content', + 'input', + 'body', + 'context', +]; + /** * Validate a parsed prompt asset, returning all errors and warnings. */ @@ -121,8 +132,42 @@ export function validateAsset( } } + if (usedVars.size > 0 && (!asset.context?.inputs || asset.context.inputs.length === 0)) { + warnings.push({ + code: 'POK046', + message: `Template uses ${usedVars.size === 1 ? 'a variable' : 'variables'} but context.inputs is not declared.`, + filePath, + suggestion: 'Declare context.inputs to enable input policy validation.', + }); + } + // Context regex definitions compile successfully for (const input of getContextInputs(asset)) { + const lowerName = input.name.toLowerCase(); + + if (input.max_size === undefined && RISKY_UNBOUNDED_INPUT_NAMES.some((needle) => lowerName.includes(needle))) { + warnings.push({ + code: 'POK040', + message: `Context input "${input.name}" has no max_size and appears unbounded.`, + filePath, + suggestion: 'Add max_size to constrain prompt payload growth.', + }); + } + + if ( + input.allow_regex === undefined + && input.deny_regex === undefined + && input.non_empty === undefined + && input.reject_secrets === undefined + ) { + warnings.push({ + code: 'POK041', + message: `Context input "${input.name}" has no input hardening validators.`, + filePath, + suggestion: 'Consider non_empty/reject_secrets and allow/deny regex validators.', + }); + } + if (input.trim !== undefined && input.trim !== false && input.max_size === undefined) { warnings.push({ code: 'POK014', @@ -157,6 +202,103 @@ export function validateAsset( } } + if (asset.provider) { + let providerCache: unknown; + let cacheSuggestionField: string | undefined; + + switch (asset.provider) { + case 'openai': + providerCache = asset.cache?.openai; + cacheSuggestionField = 'cache.openai'; + break; + case 'anthropic': + providerCache = asset.cache?.anthropic; + cacheSuggestionField = 'cache.anthropic'; + break; + case 'gemini': + case 'google': + providerCache = asset.cache?.gemini ?? asset.cache?.google; + cacheSuggestionField = 'cache.gemini'; + break; + default: + break; + } + + if (cacheSuggestionField && providerCache === undefined) { + warnings.push({ + code: 'POK042', + message: `Provider "${asset.provider}" has no provider-specific cache settings.`, + filePath, + suggestion: `Consider configuring ${cacheSuggestionField} for better cache-hit behavior.`, + }); + } + + if (!asset.model) { + warnings.push({ + code: 'POK044', + message: `Provider "${asset.provider}" is configured without a model.`, + filePath, + suggestion: 'Set model in prompt or defaults to avoid adapter-time errors.', + }); + } + } + + if ( + asset.cache?.gemini?.cached_content + && asset.cache.google?.cached_content + && asset.cache.gemini.cached_content !== asset.cache.google.cached_content + ) { + warnings.push({ + code: 'POK043', + message: 'cache.gemini.cached_content and cache.google.cached_content are both set to different values.', + filePath, + suggestion: 'Use one canonical value; Gemini prefers cache.gemini.cached_content.', + }); + } + + for (const [envName, overrides] of Object.entries(asset.environments ?? {})) { + if (asset.cache && !overrides.cache) { + warnings.push({ + code: 'POK045', + message: `Environment "${envName}" does not override cache while prompt-level cache is defined.`, + filePath, + suggestion: 'Confirm cache strategy is intentionally shared across environments.', + }); + } + } + + for (const [tierName, overrides] of Object.entries(asset.tiers ?? {})) { + if (asset.cache && !overrides.cache) { + warnings.push({ + code: 'POK045', + message: `Tier "${tierName}" does not override cache while prompt-level cache is defined.`, + filePath, + suggestion: 'Confirm cache strategy is intentionally shared across tiers.', + }); + } + } + + for (const tool of asset.tools ?? []) { + if (typeof tool !== 'string') { + if (!tool.description) { + warnings.push({ + code: 'POK047', + message: `Inline tool "${tool.name}" is missing a description.`, + filePath, + suggestion: 'Add description to improve model tool-selection quality.', + }); + } + if (!tool.input_schema) { + warnings.push({ + code: 'POK047', + message: `Inline tool "${tool.name}" is missing input_schema.`, + filePath, + suggestion: 'Add input_schema so tool inputs are strongly typed.', + }); + } + } + } + return { valid: errors.length === 0, errors, diff --git a/tests/validation.test.ts b/tests/validation.test.ts index d559e08..80e4f48 100644 --- a/tests/validation.test.ts +++ b/tests/validation.test.ts @@ -66,7 +66,9 @@ describe('validateAsset', () => { sections: { prompt_template: '{{ pull_request }}' }, }); const warning = result.warnings.find((w) => w.code === 'POK011'); + const policyWarning = result.warnings.find((w) => w.code === 'POK046'); expect(warning).toBeDefined(); + expect(policyWarning).toBeDefined(); expect(warning?.message).toContain('pull_request'); }); @@ -175,6 +177,63 @@ describe('validateAsset', () => { expect(result.warnings.some((warning) => warning.code === 'POK014')).toBe(true); }); + it('warns on risky unbounded context inputs and missing hardening', () => { + const result = validateAsset({ + id: 'test', + schema_version: 1, + context: { + inputs: [{ name: 'user_message' }], + }, + sections: { prompt_template: '{{ user_message }}' }, + }); + + expect(result.valid).toBe(true); + expect(result.warnings.some((warning) => warning.code === 'POK040')).toBe(true); + expect(result.warnings.some((warning) => warning.code === 'POK041')).toBe(true); + }); + + it('warns when provider cache/model guidance is missing', () => { + const result = validateAsset({ + id: 'test', + schema_version: 1, + provider: 'openai', + sections: { prompt_template: 'Hello' }, + }); + + expect(result.valid).toBe(true); + expect(result.warnings.some((warning) => warning.code === 'POK042')).toBe(true); + expect(result.warnings.some((warning) => warning.code === 'POK044')).toBe(true); + }); + + it('warns on conflicting gemini/google cache entries', () => { + const result = validateAsset({ + id: 'test', + schema_version: 1, + provider: 'gemini', + model: 'gemini-2.5-pro', + cache: { + gemini: { cached_content: 'cachedContents/abc' }, + google: { cached_content: 'cachedContents/xyz' }, + }, + sections: { prompt_template: 'Hello' }, + }); + + expect(result.valid).toBe(true); + expect(result.warnings.some((warning) => warning.code === 'POK043')).toBe(true); + }); + + it('warns on inline tools missing schema metadata', () => { + const result = validateAsset({ + id: 'test', + schema_version: 1, + tools: [{ name: 'lookup_customer' }], + sections: { prompt_template: 'Hello' }, + }); + + expect(result.valid).toBe(true); + expect(result.warnings.filter((warning) => warning.code === 'POK047')).toHaveLength(2); + }); + it('does not warn when trim is explicitly false without max_size', () => { const result = validateAsset({ id: 'test',