Merge pull request #9 from PredictabilityAtScale/codex/add-support-for-new-conversation-body-format

# Conflicts:
#	README.md
#	docs/providers.md
#	src/providers/gemini.ts
#	tests/providers.test.ts

Author: PredictabilityAtScale

README.md

@@ -135,7 +135,7 @@ Supported values for `warnings.contextSize` are `auto`, `off`, `result-only`, `c
 - **Composition** — `includes` to share system instructions across prompts, with circular detection
 - **Folder defaults** — `defaults.md` inheritance for shared provider, model, metadata, and system instructions
 - **Overrides** — Environment and tier-based overrides (base → env → tier → runtime)
-- **4 provider adapters** — OpenAI, Anthropic, Gemini, OpenRouter — body-only output
+- **5 provider adapters** — OpenAI (Chat), OpenAI (Responses), Anthropic, Gemini, OpenRouter — body-only output
 - **Provider-aware input caching controls** — optional `cache` front matter maps to OpenAI prompt cache hints, Anthropic `cache_control`, and Gemini `cachedContent`
 - **Validation** — Zod schema validation, Levenshtein-based "did you mean?" for typos, variable usage checks
 - **Context hardening** — structured regexes with flags, `/pattern/i` convenience syntax, and built-in `non_empty` / `reject_secrets` validators
@@ -194,6 +194,7 @@ Provider adapters are also available as direct imports:
 
 ```typescript
 import { openaiAdapter } from 'promptopskit/openai';
+import { openaiResponsesAdapter } from 'promptopskit/openai-responses';
 import { anthropicAdapter } from 'promptopskit/anthropic';
 import { geminiAdapter } from 'promptopskit/gemini';
 import { openrouterAdapter } from 'promptopskit/openrouter';
@@ -502,14 +503,15 @@ Renders a prompt for a specific provider. Returns `{ resolved, request?, returnM
 |--------|------|-------------|
 | `path` | `string` | Prompt path (no extension), e.g. `'support/reply'` |
 | `source` | `string` | Inline prompt source (alternative to path) |
-| `provider` | `string` | `'openai'`, `'anthropic'`, `'gemini'`, `'openrouter'` |
+| `provider` | `string` | `'openai'`, `'openai-responses'`, `'anthropic'`, `'gemini'`, `'openrouter'` |
 | `variables` | `Record<string, string>` | Template variables |
 | `onContextOverflow` | `(info) => string` | Optional callback to transform oversized context values before rendering |
 | `environment` | `string` | Environment override name |
 | `tier` | `string` | Tier override name |
 | `history` | `Array<{ role, content }>` | Conversation history |
 | `toolRegistry` | `Record<string, unknown>` | Tool definitions for resolving string tool references |
 | `strict` | `boolean` | Fail on missing variables |
+| `openaiResponses` | `object` | Optional Responses API extras (`previous_response_id`, `conversation`, `instructions`, `parallel_tool_calls`, `max_tool_calls`, `store`, `metadata`, `include`, `background`) |
 
 ### `kit.loadPrompt(path)` / `kit.resolvePrompt(path, options)` / `kit.validatePrompt(path)`
 
@@ -529,7 +531,7 @@ Prompt files use YAML front matter with these fields:
 |-------|------|-------------|
 | `id` | `string` | Unique prompt identifier (required) |
 | `schema_version` | `number` | Schema version, currently `1` |
-| `provider` | `string` | `openai`, `anthropic`, `gemini` (or `google`), `openrouter`, `any` |
+| `provider` | `string` | `openai`, `openai-responses`, `anthropic`, `gemini` (or `google`), `openrouter`, `any` |
 | `model` | `string` | Model name |
 | `fallback_models` | `string[]` | Fallback model list |
 | `reasoning` | `object` | `{ effort, budget_tokens }` |

docs/api-reference.md

@@ -63,14 +63,15 @@ const result = await kit.renderPrompt({
 |--------|------|-------------|
 | `path` | `string` | Prompt path (no extension), e.g. `'support/reply'` |
 | `source` | `string` | Inline prompt source (alternative to `path`) |
-| `provider` | `string` | `'openai'`, `'anthropic'`, `'gemini'`, `'openrouter'` (required) |
+| `provider` | `string` | `'openai'`, `'openai-responses'`, `'anthropic'`, `'gemini'`, `'openrouter'` (required) |
 | `variables` | `Record<string, string>` | Template variables |
 | `onContextOverflow` | `(info) => string` | Optional callback to transform an oversized context value before rendering |
 | `environment` | `string` | Environment override name |
 | `tier` | `string` | Tier override name |
 | `history` | `Array<{ role, content }>` | Conversation history |
 | `toolRegistry` | `Record<string, unknown>` | Tool definitions for resolving string tool references |
 | `strict` | `boolean` | Fail on missing variables (default `false`) |
+| `openaiResponses` | `object` | Optional Responses API extras (`previous_response_id`, `conversation`, `instructions`, `parallel_tool_calls`, `max_tool_calls`, `store`, `metadata`, `include`, `background`) |
 
 Either `path` or `source` must be provided.
 

docs/providers.md

@@ -1,16 +1,53 @@
 # Provider Adapters
 
-PromptOpsKit ships four provider adapters. Direct `render()` calls always produce a `{ body, provider, model }` object shaped for the target API. Async `renderPrompt()` helpers may instead return `{ provider, model, returnMessage }` when context validation is configured to short-circuit before request shaping. You handle the HTTP call — no auth, no headers, no HTTP client opinions.
+PromptOpsKit ships five provider adapters. Direct `render()` calls always produce a `{ body, provider, model }` object shaped for the target API. Async `renderPrompt()` helpers may instead return `{ provider, model, returnMessage }` when context validation is configured to short-circuit before request shaping. You handle the HTTP call — no auth, no headers, no HTTP client opinions.
 
 ## Supported providers
 
 | Provider | Front matter value | Adapter |
 |----------|-------------------|---------|
-| OpenAI | `openai` | `openaiAdapter` |
+| OpenAI (Chat Completions) | `openai` | `openaiAdapter` |
+| OpenAI (Responses API) | `openai-responses` | `openaiResponsesAdapter` |
 | Anthropic | `anthropic` | `anthropicAdapter` |
 | Google Gemini | `gemini` or `google` | `geminiAdapter` |
 | OpenRouter | `openrouter` | `openrouterAdapter` |
 
+
+## Normalized front matter vs provider-specific options
+
+PromptOpsKit already normalizes common settings across providers via front matter fields like `sampling`, `reasoning`, `response`, and `tools`.
+
+When a provider has extra knobs with no clean cross-provider equivalent, use `provider_options`:
+
+```yaml
+provider_options:
+  anthropic:
+    top_k: 50
+    tool_choice:
+      type: auto
+  gemini:
+    candidate_count: 2
+    top_k: 20
+    seed: 42
+    response_modalities: ["TEXT"]
+    thinking_budget_tokens: 2048
+```
+
+This keeps portable settings in normalized fields, while still exposing advanced provider-specific controls.
+
+
+## Streaming support
+
+`response.stream` support differs by provider:
+
+| Provider | `response.stream` behavior |
+|----------|----------------------------|
+| `openai` | Mapped to body `stream` |
+| `openai-responses` | Mapped to body `stream` |
+| `anthropic` | Mapped to body `stream` |
+| `openrouter` | Mapped to body `stream` (same as OpenAI) |
+| `gemini` | **Not body-mapped**; Gemini streaming is endpoint-based (`streamGenerateContent`) |
+
 ## Usage via `renderPrompt`
 
 ```typescript
@@ -42,6 +79,7 @@ When a prompt includes multiple cache blocks (for example `cache.openai` + `cach
 
 ```typescript
 import { openaiAdapter } from 'promptopskit/openai';
+import { openaiResponsesAdapter } from 'promptopskit/openai-responses';
 import { anthropicAdapter } from 'promptopskit/anthropic';
 import { geminiAdapter } from 'promptopskit/gemini';
 import { openrouterAdapter } from 'promptopskit/openrouter';
@@ -63,7 +101,7 @@ interface ProviderAdapter {
 }
 ```
 
-Direct adapter rendering accepts the same `environment` and `tier` selectors as `kit.renderPrompt()`. Use the synchronous `validate()` and `render()` methods when you already have a compiled `ResolvedPromptAsset`, and use the async `validatePrompt()` and `renderPrompt()` helpers when you want the adapter to resolve either markdown source or a compiled artifact from disk. Context input validation runs through the same shared prompt-input wrapper for OpenAI, Anthropic, Gemini, and OpenRouter, so `allow_regex`, `deny_regex`, `non_empty`, `reject_secrets`, and `return_message` behave consistently across all four.
+Direct adapter rendering accepts the same `environment` and `tier` selectors as `kit.renderPrompt()`. Use the synchronous `validate()` and `render()` methods when you already have a compiled `ResolvedPromptAsset`, and use the async `validatePrompt()` and `renderPrompt()` helpers when you want the adapter to resolve either markdown source or a compiled artifact from disk. Context input validation runs through the same shared prompt-input wrapper for OpenAI, OpenAI Responses, Anthropic, Gemini, and OpenRouter, so `allow_regex`, `deny_regex`, `non_empty`, `reject_secrets`, and `return_message` behave consistently across all five.
 
 Server-side example:
 
@@ -179,7 +217,7 @@ If you want UsageTap begin/end tracking around a provider call, use the optional
 
 See [UsageTap](./usagetap.md) for setup, lifecycle helpers, entitlement behavior, tool gating, standalone usage extractors, and provider examples.
 
-## OpenAI
+## OpenAI (`openai`)
 
 Body shape: [Chat Completions API](https://platform.openai.com/docs/api-reference/chat)
 
@@ -195,6 +233,44 @@ Body shape: [Chat Completions API](https://platform.openai.com/docs/api-referenc
 }
 ```
 
+## OpenAI Responses (`openai-responses`)
+
+Body shape: [Responses API](https://platform.openai.com/docs/api-reference/responses)
+
+```json
+{
+  "model": "gpt-5.4",
+  "instructions": "...",
+  "input": [
+    { "role": "user", "content": "..." }
+  ],
+  "temperature": 0.7,
+  "reasoning": { "effort": "medium" }
+}
+```
+
+Field mapping (differences from `openai`):
+
+| Front matter | Body field (`openai-responses`) |
+|-------------|----------------------------------|
+| `sampling.max_output_tokens` | `max_output_tokens` |
+| `reasoning.effort` | `reasoning: { effort }` |
+| `response.format: json` | `text: { format: { type: "json_object" } }` |
+| `response.schema` | `text: { format: { type: "json_schema", name, schema, strict } }` |
+| `sections.system_instructions` | `instructions` (top-level) |
+| `history + prompt_template` | `input` items instead of `messages` |
+| `tools` | Responses function tools (`{ type, name, description, parameters }`) |
+
+Warnings:
+- `reasoning.budget_tokens` is ignored (Responses uses `reasoning.effort`).
+
+Extra supported options via `renderPrompt(..., { openaiResponses: { ... } })` or direct adapter runtime:
+- `previous_response_id` (conversation chaining)
+- `conversation` (mutually exclusive with `previous_response_id`)
+- `parallel_tool_calls`, `max_tool_calls`
+- `store`, `metadata`, `include`, `background`
+- `instructions` override (runtime override for top-level instructions)
+
 Field mapping:
 
 | Front matter | Body field |
@@ -208,6 +284,7 @@ Field mapping:
 | `sampling.max_output_tokens` | `max_tokens` |
 | `reasoning.effort` | `reasoning_effort` |
 | `response.format: json` | `response_format: { type: "json_object" }` |
+| `response.schema` | `response_format: { type: "json_schema", json_schema: { name, schema, strict } }` |
 | `response.stream` | `stream` |
 | `cache.openai.prompt_cache_key` | `prompt_cache_key` |
 | `cache.openai.retention` | `prompt_cache_retention` |
@@ -244,6 +321,8 @@ Key differences from OpenAI:
 - `cache.anthropic.mode: automatic` maps to top-level `cache_control`.
 - `cache.anthropic.mode: explicit` applies `cache_control` at block level for selected sections/tools.
 - `cache.anthropic.ttl` supports `5m` (default) or `1h`.
+- `provider_options.anthropic.top_k` maps to `top_k`.
+- `provider_options.anthropic.tool_choice` maps to `tool_choice`.
 
 Warnings:
 - `frequency_penalty` and `presence_penalty` are not supported — ignored with a warning.
@@ -276,8 +355,16 @@ Key differences:
 - Sampling parameters are nested under `generationConfig`.
 - `top_p` maps to `topP`, `max_output_tokens` maps to `maxOutputTokens`, `stop` maps to `stopSequences`.
 - `response.format: json` maps to `generationConfig.responseMimeType: "application/json"`.
+- `response.schema` maps to `generationConfig.responseSchema` (portable normalized schema shape).
+- `response.stream` is not body-mapped for Gemini; use the streaming endpoint (`streamGenerateContent`).
 - `reasoning.effort` maps to `thinkingConfig.thinkingBudget` (high=8192, medium=4096, low=1024).
 - `cache.gemini.cached_content` (or `cache.google.cached_content`) maps to top-level `cachedContent`.
+- `provider_options.gemini.candidate_count` maps to `generationConfig.candidateCount`.
+- `provider_options.gemini.top_k` maps to `generationConfig.topK`.
+- `provider_options.gemini.seed` maps to `generationConfig.seed`.
+- `provider_options.gemini.response_schema` maps to `generationConfig.responseSchema`.
+- `provider_options.gemini.response_modalities` maps to `generationConfig.responseModalities`.
+- `provider_options.gemini.thinking_budget_tokens` overrides effort-derived thinking budget.
 
 Warnings:
 - `frequency_penalty` and `presence_penalty` are not supported — ignored with a warning.

package.json

@@ -79,6 +79,16 @@
         "types": "./dist/providers/openrouter.d.cts",
         "default": "./dist/providers/openrouter.cjs"
       }
+    },
+    "./openai-responses": {
+      "import": {
+        "types": "./dist/providers/openai-responses.d.ts",
+        "default": "./dist/providers/openai-responses.js"
+      },
+      "require": {
+        "types": "./dist/providers/openai-responses.d.cts",
+        "default": "./dist/providers/openai-responses.cjs"
+      }
     }
   },
   "files": [

src/index.ts

@@ -63,7 +63,7 @@ export { interpolate, extractVariables } from './renderer/index.js';
 export { resolveIncludes } from './composition/index.js';
 export { applyOverrides } from './overrides/index.js';
 export { validateAsset, validateAssetWithIncludes } from './validation/index.js';
-export { getAdapter, openaiAdapter } from './providers/index.js';
+export { getAdapter, openaiAdapter, openaiResponsesAdapter } from './providers/index.js';
 export { anthropicAdapter } from './providers/anthropic.js';
 export { geminiAdapter } from './providers/gemini.js';
 export { openrouterAdapter } from './providers/openrouter.js';
@@ -121,6 +121,8 @@ export interface RenderPromptOptions {
   toolRegistry?: Record<string, unknown>;
   /** Strict mode — fail on missing variables */
   strict?: boolean;
+  /** OpenAI Responses API-specific request options */
+  openaiResponses?: RuntimeRenderOptions['openaiResponses'];
 }
 
 // --- Result ---
@@ -263,6 +265,7 @@ export class PromptOpsKit {
       history: options.history,
       toolRegistry: options.toolRegistry,
       strict: options.strict,
+      openaiResponses: options.openaiResponses,
     });
 
     return {

src/overrides/apply-overrides.ts

@@ -56,5 +56,20 @@ function mergeOverride(
     result.response = { ...result.response, ...override.response };
   }
 
+  if (override.provider_options !== undefined) {
+    result.provider_options = {
+      ...result.provider_options,
+      ...override.provider_options,
+      anthropic: {
+        ...result.provider_options?.anthropic,
+        ...override.provider_options.anthropic,
+      },
+      gemini: {
+        ...result.provider_options?.gemini,
+        ...override.provider_options.gemini,
+      },
+    };
+  }
+
   return result;
 }

src/providers/anthropic.ts

@@ -34,6 +34,13 @@ export const anthropicAdapter: ProviderAdapter = withPromptInputSupport({
     if (resolvedAsset.reasoning?.effort !== undefined) {
       warnings.push('Anthropic uses budget_tokens for thinking, not effort. effort will be mapped approximately.');
     }
+    if (resolvedAsset.response?.schema !== undefined) {
+      warnings.push('Anthropic does not support response.schema structured output in this adapter. It will be ignored.');
+    }
+
+    if (resolvedAsset.provider_options?.anthropic?.top_k !== undefined && resolvedAsset.provider_options.anthropic.top_k < 0) {
+      errors.push('Anthropic provider_options.top_k must be >= 0.');
+    }
 
     return { valid: errors.length === 0, errors, warnings };
   },
@@ -108,6 +115,11 @@ export const anthropicAdapter: ProviderAdapter = withPromptInputSupport({
       };
     }
 
+    // Provider-specific options
+    if (resolvedAsset.provider_options?.anthropic?.top_k !== undefined) {
+      body.top_k = resolvedAsset.provider_options.anthropic.top_k;
+    }
+
     // Streaming
     if (resolvedAsset.response?.stream !== undefined) {
       body.stream = resolvedAsset.response.stream;
@@ -146,6 +158,10 @@ export const anthropicAdapter: ProviderAdapter = withPromptInputSupport({
       });
     }
 
+    if (resolvedAsset.provider_options?.anthropic?.tool_choice !== undefined) {
+      body.tool_choice = resolvedAsset.provider_options.anthropic.tool_choice;
+    }
+
     return {
       body,
       provider: 'anthropic',

src/providers/gemini.ts

@@ -36,6 +36,9 @@ export const geminiAdapter: ProviderAdapter = withPromptInputSupport({
     if (geminiCache && googleCache && geminiCache !== googleCache) {
       warnings.push('Both cache.gemini.cached_content and cache.google.cached_content are set. Gemini uses cache.gemini.cached_content.');
     }
+    if (resolvedAsset.response?.stream !== undefined) {
+      warnings.push('Gemini streaming is endpoint-based (streamGenerateContent), not body-based. response.stream will be ignored.');
+    }
 
     return { valid: errors.length === 0, errors, warnings };
   },
@@ -82,17 +85,31 @@ export const geminiAdapter: ProviderAdapter = withPromptInputSupport({
     // Generation config
     const generationConfig: Record<string, unknown> = {};
 
+    const geminiOptions = resolvedAsset.provider_options?.gemini;
+
     if (resolvedAsset.sampling?.temperature !== undefined) generationConfig.temperature = resolvedAsset.sampling.temperature;
     if (resolvedAsset.sampling?.top_p !== undefined) generationConfig.topP = resolvedAsset.sampling.top_p;
     if (resolvedAsset.sampling?.max_output_tokens !== undefined) generationConfig.maxOutputTokens = resolvedAsset.sampling.max_output_tokens;
     if (resolvedAsset.sampling?.stop !== undefined) generationConfig.stopSequences = resolvedAsset.sampling.stop;
 
+    if (geminiOptions?.candidate_count !== undefined) generationConfig.candidateCount = geminiOptions.candidate_count;
+    if (geminiOptions?.top_k !== undefined) generationConfig.topK = geminiOptions.top_k;
+    if (geminiOptions?.seed !== undefined) generationConfig.seed = geminiOptions.seed;
+    if (geminiOptions?.response_schema !== undefined) generationConfig.responseSchema = geminiOptions.response_schema;
+    if (geminiOptions?.response_modalities !== undefined) generationConfig.responseModalities = geminiOptions.response_modalities;
+
+    if (resolvedAsset.response?.schema !== undefined) generationConfig.responseSchema = resolvedAsset.response.schema;
+
     if (resolvedAsset.response?.format === 'json') {
       generationConfig.responseMimeType = 'application/json';
     }
 
     // Thinking config
-    if (resolvedAsset.reasoning?.effort) {
+    if (geminiOptions?.thinking_budget_tokens !== undefined) {
+      body.thinkingConfig = {
+        thinkingBudget: geminiOptions.thinking_budget_tokens,
+      };
+    } else if (resolvedAsset.reasoning?.effort) {
       body.thinkingConfig = {
         thinkingBudget: resolvedAsset.reasoning.effort === 'high' ? 8192 : resolvedAsset.reasoning.effort === 'medium' ? 4096 : 1024,
       };

src/providers/index.ts

@@ -8,18 +8,21 @@ export type {
   RuntimeRenderOptions,
 } from './types.js';
 export { openaiAdapter } from './openai.js';
+export { openaiResponsesAdapter } from './openai-responses.js';
 export { anthropicAdapter } from './anthropic.js';
 export { geminiAdapter } from './gemini.js';
 export { openrouterAdapter } from './openrouter.js';
 
 import type { ProviderAdapter } from './types.js';
 import { openaiAdapter } from './openai.js';
+import { openaiResponsesAdapter } from './openai-responses.js';
 import { anthropicAdapter } from './anthropic.js';
 import { geminiAdapter } from './gemini.js';
 import { openrouterAdapter } from './openrouter.js';
 
 const adapters: Record<string, ProviderAdapter> = {
   openai: openaiAdapter,
+  'openai-responses': openaiResponsesAdapter,
   anthropic: anthropicAdapter,
   google: geminiAdapter,
   gemini: geminiAdapter,