Merge pull request #11 from PredictabilityAtScale:codex/review-prompt-templates-for-validation-issues

Improve prompt validation with new static warnings for inputs, cache, provider, and tools

Author: PredictabilityAtScale

docs/validation.md

@@ -36,6 +36,14 @@ const result = await kit.validatePrompt('support/reply');
 | `POK012` | Warning | Variable declared in `context.inputs` but never used |
 | `POK013` | Error | Invalid context regex pattern (`allow_regex` or `deny_regex`), including prompt id, variable name, field name, and raw configured value |
 | `POK014` | Warning | `trim` configured without `max_size` (trim-to-budget skipped) |
+| `POK040` | Warning | Risky context input appears unbounded (`max_size` missing) |
+| `POK041` | Warning | Context input has no hardening validators (`allow/deny regex`, `non_empty`, `reject_secrets`) |
+| `POK042` | Warning | Provider has no provider-specific cache config |
+| `POK043` | Warning | `cache.gemini.cached_content` and `cache.google.cached_content` conflict |
+| `POK044` | Warning | Provider configured without an explicit `model` |
+| `POK045` | Warning | Environment/tier cache override may be missing while base cache is defined |
+| `POK046` | Warning | Template uses variables but `context.inputs` is not declared |
+| `POK047` | Warning | Inline tool definition missing `description` or `input_schema` |
 | `POK033` | Runtime error | `non_empty` validation failed |
 | `POK034` | Runtime error | `reject_secrets` validation matched |
 | `POK020` | Error | Include resolution failed (missing file) |
@@ -49,7 +57,7 @@ Unknown front matter keys are checked against known keys using Levenshtein dista
 ⚠ POK010: Unknown front matter field: "tempreature" (Did you mean "temperature"?)
 ```
 
-Known front matter keys: `id`, `schema_version`, `description`, `provider`, `model`, `fallback_models`, `reasoning`, `sampling`, `response`, `tools`, `mcp`, `context`, `includes`, `environments`, `tiers`, `metadata`.
+Known front matter keys: `id`, `schema_version`, `description`, `provider`, `model`, `fallback_models`, `reasoning`, `sampling`, `response`, `tools`, `mcp`, `context`, `includes`, `environments`, `tiers`, `metadata`, `cache`, `provider_options`.
 
 ## Variable validation
 
@@ -114,6 +122,9 @@ context:
 - `reject_secrets` rejects common secret-like strings with `POK034`, unless `return_message` is configured.
 - During static validation and compilation, malformed `allow_regex` or `deny_regex` patterns are reported as `POK013`.
 - During static validation, `trim` without `max_size` returns a `POK014` warning.
+- During static validation, risky unbounded inputs and missing hardening are flagged as `POK040` and `POK041`.
+- During static validation, provider/cache hygiene checks can emit `POK042`–`POK045`.
+- During static validation, inline tool quality checks can emit `POK047`.
 
 Regex compilation errors include the prompt id, variable name, field name, and raw configured value to make bad prompt definitions easy to locate.
 

src/validation/validate.ts

@@ -4,7 +4,6 @@ import { extractVariables } from '../renderer/index.js';
 import { resolveIncludes } from '../composition/index.js';
 import {
   compileContextRegex,
-  formatInvalidContextRegexMessage,
   getContextInputs,
   getContextInputNames,
 } from '../context.js';
@@ -26,9 +25,21 @@ export interface PromptValidationResult {
 const KNOWN_FRONT_MATTER_KEYS = new Set([
   'id', 'schema_version', 'description', 'provider', 'model', 'fallback_models',
   'reasoning', 'sampling', 'response', 'tools', 'mcp', 'context', 'includes',
-  'environments', 'tiers', 'metadata', 'cache',
+  'environments', 'tiers', 'metadata', 'cache', 'provider_options',
 ]);
 
+const RISKY_UNBOUNDED_INPUT_NAMES = [
+  'message',
+  'prompt',
+  'history',
+  'transcript',
+  'document',
+  'content',
+  'input',
+  'body',
+  'context',
+];
+
 /**
  * Validate a parsed prompt asset, returning all errors and warnings.
  */
@@ -121,8 +132,42 @@ export function validateAsset(
     }
   }
 
+  if (usedVars.size > 0 && (!asset.context?.inputs || asset.context.inputs.length === 0)) {
+    warnings.push({
+      code: 'POK046',
+      message: `Template uses ${usedVars.size === 1 ? 'a variable' : 'variables'} but context.inputs is not declared.`,
+      filePath,
+      suggestion: 'Declare context.inputs to enable input policy validation.',
+    });
+  }
+
   // Context regex definitions compile successfully
   for (const input of getContextInputs(asset)) {
+    const lowerName = input.name.toLowerCase();
+
+    if (input.max_size === undefined && RISKY_UNBOUNDED_INPUT_NAMES.some((needle) => lowerName.includes(needle))) {
+      warnings.push({
+        code: 'POK040',
+        message: `Context input "${input.name}" has no max_size and appears unbounded.`,
+        filePath,
+        suggestion: 'Add max_size to constrain prompt payload growth.',
+      });
+    }
+
+    if (
+      input.allow_regex === undefined
+      && input.deny_regex === undefined
+      && input.non_empty === undefined
+      && input.reject_secrets === undefined
+    ) {
+      warnings.push({
+        code: 'POK041',
+        message: `Context input "${input.name}" has no input hardening validators.`,
+        filePath,
+        suggestion: 'Consider non_empty/reject_secrets and allow/deny regex validators.',
+      });
+    }
+
     if (input.trim !== undefined && input.trim !== false && input.max_size === undefined) {
       warnings.push({
         code: 'POK014',
@@ -157,6 +202,103 @@ export function validateAsset(
     }
   }
 
+  if (asset.provider) {
+    let providerCache: unknown;
+    let cacheSuggestionField: string | undefined;
+
+    switch (asset.provider) {
+      case 'openai':
+        providerCache = asset.cache?.openai;
+        cacheSuggestionField = 'cache.openai';
+        break;
+      case 'anthropic':
+        providerCache = asset.cache?.anthropic;
+        cacheSuggestionField = 'cache.anthropic';
+        break;
+      case 'gemini':
+      case 'google':
+        providerCache = asset.cache?.gemini ?? asset.cache?.google;
+        cacheSuggestionField = 'cache.gemini';
+        break;
+      default:
+        break;
+    }
+
+    if (cacheSuggestionField && providerCache === undefined) {
+      warnings.push({
+        code: 'POK042',
+        message: `Provider "${asset.provider}" has no provider-specific cache settings.`,
+        filePath,
+        suggestion: `Consider configuring ${cacheSuggestionField} for better cache-hit behavior.`,
+      });
+    }
+
+    if (!asset.model) {
+      warnings.push({
+        code: 'POK044',
+        message: `Provider "${asset.provider}" is configured without a model.`,
+        filePath,
+        suggestion: 'Set model in prompt or defaults to avoid adapter-time errors.',
+      });
+    }
+  }
+
+  if (
+    asset.cache?.gemini?.cached_content
+    && asset.cache.google?.cached_content
+    && asset.cache.gemini.cached_content !== asset.cache.google.cached_content
+  ) {
+    warnings.push({
+      code: 'POK043',
+      message: 'cache.gemini.cached_content and cache.google.cached_content are both set to different values.',
+      filePath,
+      suggestion: 'Use one canonical value; Gemini prefers cache.gemini.cached_content.',
+    });
+  }
+
+  for (const [envName, overrides] of Object.entries(asset.environments ?? {})) {
+    if (asset.cache && !overrides.cache) {
+      warnings.push({
+        code: 'POK045',
+        message: `Environment "${envName}" does not override cache while prompt-level cache is defined.`,
+        filePath,
+        suggestion: 'Confirm cache strategy is intentionally shared across environments.',
+      });
+    }
+  }
+
+  for (const [tierName, overrides] of Object.entries(asset.tiers ?? {})) {
+    if (asset.cache && !overrides.cache) {
+      warnings.push({
+        code: 'POK045',
+        message: `Tier "${tierName}" does not override cache while prompt-level cache is defined.`,
+        filePath,
+        suggestion: 'Confirm cache strategy is intentionally shared across tiers.',
+      });
+    }
+  }
+
+  for (const tool of asset.tools ?? []) {
+    if (typeof tool !== 'string') {
+      if (!tool.description) {
+        warnings.push({
+          code: 'POK047',
+          message: `Inline tool "${tool.name}" is missing a description.`,
+          filePath,
+          suggestion: 'Add description to improve model tool-selection quality.',
+        });
+      }
+      if (!tool.input_schema) {
+        warnings.push({
+          code: 'POK047',
+          message: `Inline tool "${tool.name}" is missing input_schema.`,
+          filePath,
+          suggestion: 'Add input_schema so tool inputs are strongly typed.',
+        });
+      }
+    }
+  }
+
   return {
     valid: errors.length === 0,
     errors,

tests/validation.test.ts

@@ -66,7 +66,9 @@ describe('validateAsset', () => {
       sections: { prompt_template: '{{ pull_request }}' },
     });
     const warning = result.warnings.find((w) => w.code === 'POK011');
+    const policyWarning = result.warnings.find((w) => w.code === 'POK046');
     expect(warning).toBeDefined();
+    expect(policyWarning).toBeDefined();
     expect(warning?.message).toContain('pull_request');
   });
 
@@ -175,6 +177,63 @@ describe('validateAsset', () => {
     expect(result.warnings.some((warning) => warning.code === 'POK014')).toBe(true);
   });
 
+  it('warns on risky unbounded context inputs and missing hardening', () => {
+    const result = validateAsset({
+      id: 'test',
+      schema_version: 1,
+      context: {
+        inputs: [{ name: 'user_message' }],
+      },
+      sections: { prompt_template: '{{ user_message }}' },
+    });
+
+    expect(result.valid).toBe(true);
+    expect(result.warnings.some((warning) => warning.code === 'POK040')).toBe(true);
+    expect(result.warnings.some((warning) => warning.code === 'POK041')).toBe(true);
+  });
+
+  it('warns when provider cache/model guidance is missing', () => {
+    const result = validateAsset({
+      id: 'test',
+      schema_version: 1,
+      provider: 'openai',
+      sections: { prompt_template: 'Hello' },
+    });
+
+    expect(result.valid).toBe(true);
+    expect(result.warnings.some((warning) => warning.code === 'POK042')).toBe(true);
+    expect(result.warnings.some((warning) => warning.code === 'POK044')).toBe(true);
+  });
+
+  it('warns on conflicting gemini/google cache entries', () => {
+    const result = validateAsset({
+      id: 'test',
+      schema_version: 1,
+      provider: 'gemini',
+      model: 'gemini-2.5-pro',
+      cache: {
+        gemini: { cached_content: 'cachedContents/abc' },
+        google: { cached_content: 'cachedContents/xyz' },
+      },
+      sections: { prompt_template: 'Hello' },
+    });
+
+    expect(result.valid).toBe(true);
+    expect(result.warnings.some((warning) => warning.code === 'POK043')).toBe(true);
+  });
+
+  it('warns on inline tools missing schema metadata', () => {
+    const result = validateAsset({
+      id: 'test',
+      schema_version: 1,
+      tools: [{ name: 'lookup_customer' }],
+      sections: { prompt_template: 'Hello' },
+    });
+
+    expect(result.valid).toBe(true);
+    expect(result.warnings.filter((warning) => warning.code === 'POK047')).toHaveLength(2);
+  });
+
   it('does not warn when trim is explicitly false without max_size', () => {
     const result = validateAsset({
       id: 'test',