Merge pull request #68 from LAA-Software-Engineering/feature/examples-example1-openai-cost-docs

Examples, docs, and OpenAI run cost estimation

Author: leo-aa88

.gitignore

@@ -33,3 +33,6 @@ go.work.sum
 # Editor/IDE
 # .idea/
 # .vscode/
+
+tmp/
+examples/**/.agentic/

docs/EXAMPLES.md

@@ -1,6 +1,8 @@
 # Examples
 
-Short, runnable patterns for **`apiVersion: agentic.dev/v0`**. For the full YAML spec, CLI behaviour, and field semantics, see [**`design_doc.md`**](design_doc.md).
+Short, runnable patterns for **`apiVersion: agentic.dev/v0`**. For the full YAML spec, CLI behaviour, and field semantics, see [**`DESIGN_DOC.md`**](DESIGN_DOC.md).
+
+A checked-in copy of the **OpenAI `support_snippet`** project from **section 4** lives under [**`examples/example1/`**](../examples/example1/). Its **`metadata.name`** is **`example1`**, matching that folder. From the repository root, pass **`--project examples/example1`** to **`agentctl`** (or **`cd` there** and use **`--project .`**).
 
 ---
 
@@ -20,7 +22,7 @@ my-agent-system/
   workflows/hello.yaml
 ```
 
-The generated files match the snippets in sections 2–4 below (with `metadata.name` set from the argument you pass to `init`).
+Sections **2–3** mirror what `init` creates. **Section 4** is a separate **`gpt-4o-mini`** project layout you can copy beside or instead of the scaffold.
 
 ---
 
@@ -102,23 +104,42 @@ agentctl run    workflow/hello --project my-agent-system
 
 ---
 
-## 4. OpenAI chat (real model)
+## 4. Real OpenAI example (`gpt-4o-mini`)
+
+This is a small but **end-to-end** project: a **native echo** step supplies fixed “policy” text, then **`gpt-4o-mini`** drafts a one-line customer reply. You need a valid **[OpenAI API key](https://platform.openai.com/api-keys)** and outbound **HTTPS** to `api.openai.com`.
+
+**Repo copy:** [**`examples/example1/`**](../examples/example1/) — **`agentctl validate --project examples/example1`** from the repo root, or **`agentctl validate --project .`** after **`cd examples/example1`**.
+
+The runtime calls OpenAI’s **`/v1/chat/completions`** endpoint. The agent **must** answer with a **single JSON object** (no markdown fences); the engine parses that object and exposes its fields to **`spec.output`**.
+
+**`totalCostUsd` on runs** is accumulated from each step’s reported cost. Native tools report **0**. For **OpenAI**, the client estimates USD from the API **`usage`** token counts × approximate per-million rates for known models (**`gpt-4o-mini`**, **`gpt-4o`**, and dated variants such as **`gpt-4o-mini-…`**). Other model ids stay at **0** until their rates are added in code; see **`internal/models/openai_cost.go`** and verify against [OpenAI pricing](https://openai.com/api/pricing/).
+
+### Layout
 
-The control plane currently wires **`type: openai`** to the OpenAI **`/v1/chat/completions`** API. Set a key via **`apiKeyFrom`** (MVP: **`env:VAR`** only).
+```text
+example1/
+  project.yaml
+  policies/default.yaml
+  tools/helper.yaml
+  agents/support_writer.yaml
+  workflows/support_snippet.yaml
+```
 
-**`project.yaml`** (add an `openai` provider and point defaults at an OpenAI model id):
+Reuse **`policies/default.yaml`** and **`tools/helper.yaml`** from **section 3** unchanged.
+
+### `project.yaml`
 
 ```yaml
 apiVersion: agentic.dev/v0
 kind: Project
 metadata:
-  name: my-agent-system
+  name: example1
 spec:
   imports:
     - ./policies/default.yaml
     - ./tools/helper.yaml
-    - ./agents/assistant.yaml
-    - ./workflows/chat.yaml
+    - ./agents/support_writer.yaml
+    - ./workflows/support_snippet.yaml
   defaults:
     policy: default
     model: openai/gpt-4o-mini
@@ -131,46 +152,112 @@ spec:
         apiKeyFrom: env:OPENAI_API_KEY
 ```
 
-```bash
-export OPENAI_API_KEY="sk-..."   # required before validate/plan/apply/run
-```
+### `agents/support_writer.yaml`
 
-**`agents/assistant.yaml`** — `metadata.name` is what workflow steps reference in **`agent:`**. The executor expects the model’s reply to be a **JSON object** (plain text, not fenced code blocks).
+`metadata.name` is the value you use in **`agent:`** on the workflow step.
 
 ```yaml
 apiVersion: agentic.dev/v0
 kind: Agent
 metadata:
-  name: assistant
+  name: support_writer
 spec:
   model: openai/gpt-4o-mini
   policy: default
+  constraints:
+    timeoutSeconds: 60
   instructions: |
-    You are a concise assistant. Respond with a single JSON object only, no markdown.
-    Shape: {"message": "<your reply>"}
+    You draft short customer-facing email lines for a storefront.
+    You receive JSON in the user message: product name and a return-policy line from internal systems.
+    Respond with one JSON object only (no markdown, no code fences).
+    Use exactly this shape: {"subject": "<=8 words>", "line": "<=25 words, friendly>"}
 ```
 
-**`workflows/chat.yaml`**
+### `workflows/support_snippet.yaml`
+
+The compose step passes the echo step’s payload into the model via **`${steps.context.output.echo...}`** (see §13.1 in **`DESIGN_DOC.md`**).
+
+**CLI-driven product (requires `--input`).** If you use **`${input.product}`** anywhere in the workflow, you **must** pass **`--input product=...`** on **`run`**. Otherwise interpolation fails with **`undefined path "input.product"`** because the run input object is empty.
 
 ```yaml
 apiVersion: agentic.dev/v0
 kind: Workflow
 metadata:
-  name: chat
+  name: support_snippet
 spec:
   policy: default
   steps:
-    - id: reply
-      agent: assistant
+    - id: context
+      uses: tool.helper.echo
+      with:
+        product: "${input.product}"
+        policy_line: "30-day returns on all SKUs; free outbound shipping on defects."
+    - id: compose
+      agent: support_writer
       with:
-        topic: "Say hello in one short sentence."
+        product: "${input.product}"
+        return_policy: "${steps.context.output.echo.policy_line}"
+  output:
+    value:
+      product: ${input.product}
+      subject: ${steps.compose.output.subject}
+      line: ${steps.compose.output.line}
 ```
 
+**Zero-argument demo.** To run **`agentctl run workflow/support_snippet`** with no **`--input`**, put a literal product on the first step and thread it through **`steps.context.output.echo`** (the checked-in [**`examples/example1/`**](../examples/example1/) tree uses **`${input.product}`** instead, so it **requires** **`--input product=...`** unless you edit the YAML):
+
+```yaml
+    - id: context
+      uses: tool.helper.echo
+      with:
+        product: "ACME USB-C hub" # literal default; or "${input.product}" + --input product=...
+        policy_line: "30-day returns on all SKUs; free outbound shipping on defects."
+    - id: compose
+      agent: support_writer
+      with:
+        product: "${steps.context.output.echo.product}"
+        return_policy: "${steps.context.output.echo.policy_line}"
+  output:
+    value:
+      product: ${steps.context.output.echo.product}
+      subject: ${steps.compose.output.subject}
+      line: ${steps.compose.output.line}
+```
+
+### Commands
+
+If you copied the files to another folder, point **`--project`** at that path instead. For the [**in-repo example**](../examples/example1/), from the **repository root** use **`examples/example1`** (the directory path), not only the project name **`example1`**.
+
+```bash
+export OPENAI_API_KEY="sk-..."   # required for any step that calls the model
+
+agentctl validate --project examples/example1
+agentctl plan   --project examples/example1
+agentctl apply  --project examples/example1 --auto-approve
+
+# Checked-in example1 workflow uses ${input.product} on the context step:
+agentctl run workflow/support_snippet --project examples/example1 --input product="ACME USB-C hub"
+
+# After switching the workflow to a literal product + steps.context... (see above), you can omit --input.
+```
+
+Default **`run`** output is still **Run ID + status**. To see the workflow **`spec.output`** object ( **`product`**, **`subject`**, **`line`**, etc.):
+
+```bash
+agentctl logs --run <run-id> --project examples/example1
+```
+
+After the trace table, the CLI prints **Workflow output (from spec.output)** as indented JSON when the run succeeded and **`output_json`** is non-empty.
+
+Or list recent runs as JSON (includes **`output`** on each run):
+
 ```bash
-agentctl run workflow/chat --project my-agent-system
+agentctl logs -o json --project examples/example1
 ```
 
-Optional: add **`spec.output.schema`** on the agent (path relative to project root) to validate the JSON against JSON Schema; see test fixtures under `internal/engine/testdata/` and **`design_doc.md`**.
+**`agentctl logs --run <id> -o json`** also includes top-level **`input`**, **`output`**, and **`workflowName`** alongside **`events`**.
+
+Optional: add **`spec.output.schema`** on the agent (path relative to the project root) so replies are validated with JSON Schema; see `internal/engine/testdata/wfproj/schemas/` and **`DESIGN_DOC.md`**.
 
 ---
 

examples/example1/agents/support_writer.yaml

@@ -0,0 +1,14 @@
+apiVersion: agentic.dev/v0
+kind: Agent
+metadata:
+  name: support_writer
+spec:
+  model: openai/gpt-4o-mini
+  policy: default
+  constraints:
+    timeoutSeconds: 60
+  instructions: |
+    You draft short customer-facing email lines for a storefront.
+    You receive JSON in the user message: product name and a return-policy line from internal systems.
+    Respond with one JSON object only (no markdown, no code fences).
+    Use exactly this shape: {"subject": "<=8 words>", "line": "<=25 words, friendly>"}

examples/example1/policies/default.yaml

@@ -0,0 +1,8 @@
+apiVersion: agentic.dev/v0
+kind: Policy
+metadata:
+  name: default
+spec:
+  execution:
+    maxWallClockSeconds: 300
+    maxTotalCostUsd: 5

examples/example1/project.yaml

@@ -0,0 +1,20 @@
+apiVersion: agentic.dev/v0
+kind: Project
+metadata:
+  name: example1
+spec:
+  imports:
+    - ./policies/default.yaml
+    - ./tools/helper.yaml
+    - ./agents/support_writer.yaml
+    - ./workflows/support_snippet.yaml
+  defaults:
+    policy: default
+    model: openai/gpt-4o-mini
+  providers:
+    models:
+      mock:
+        type: mock
+      openai:
+        type: openai
+        apiKeyFrom: env:OPENAI_API_KEY

examples/example1/tools/helper.yaml

@@ -0,0 +1,6 @@
+apiVersion: agentic.dev/v0
+kind: Tool
+metadata:
+  name: helper
+spec:
+  type: native

examples/example1/workflows/support_snippet.yaml

@@ -0,0 +1,22 @@
+apiVersion: agentic.dev/v0
+kind: Workflow
+metadata:
+  name: support_snippet
+spec:
+  policy: default
+  steps:
+    - id: context
+      uses: tool.helper.echo
+      with:
+        product: "${input.product}"
+        policy_line: "30-day returns on all SKUs; free outbound shipping on defects."
+    - id: compose
+      agent: support_writer
+      with:
+        product: "${steps.context.output.echo.product}"
+        return_policy: "${steps.context.output.echo.policy_line}"
+  output:
+    value:
+      product: ${steps.context.output.echo.product}
+      subject: ${steps.compose.output.subject}
+      line: ${steps.compose.output.line}

internal/models/models_test.go

@@ -3,6 +3,7 @@ package models
 import (
 	"context"
 	"encoding/json"
+	"math"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -118,13 +119,13 @@ func TestOpenAIClient_Generate_usesChatCompletions(t *testing.T) {
 			t.Errorf("Authorization %q", auth)
 		}
 		w.Header().Set("Content-Type", "application/json")
-		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"hello"}}]}`))
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"hello"}}],"usage":{"prompt_tokens":1000,"completion_tokens":500}}`))
 	}))
 	defer srv.Close()
 
 	c := &OpenAIClient{APIKey: "sk-mock", BaseURL: srv.URL + "/v1", HTTPClient: srv.Client()}
 	resp, err := c.Generate(context.Background(), GenerateRequest{
-		Model: "gpt-4.1",
+		Model: "gpt-4o-mini",
 		Messages: []ChatMessage{
 			{Role: "user", Content: "hi"},
 		},
@@ -135,6 +136,33 @@ func TestOpenAIClient_Generate_usesChatCompletions(t *testing.T) {
 	if resp.Content != "hello" {
 		t.Fatalf("content %q", resp.Content)
 	}
+	// 1000/1e6*0.15 + 500/1e6*0.60 = 0.00045
+	want := 1000.0/1e6*0.15 + 500.0/1e6*0.60
+	if math.Abs(resp.Meta.CostUSD-want) > 1e-9 {
+		t.Fatalf("CostUSD got %v want %v", resp.Meta.CostUSD, want)
+	}
+}
+
+func TestOpenAIClient_Generate_unknownModel_zeroCost(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"x"}}],"usage":{"prompt_tokens":100,"completion_tokens":100}}`))
+	}))
+	defer srv.Close()
+
+	c := &OpenAIClient{APIKey: "sk-mock", BaseURL: srv.URL + "/v1", HTTPClient: srv.Client()}
+	resp, err := c.Generate(context.Background(), GenerateRequest{
+		Model: "unknown-model-xyz",
+		Messages: []ChatMessage{
+			{Role: "user", Content: "hi"},
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.Meta.CostUSD != 0 {
+		t.Fatalf("CostUSD %v", resp.Meta.CostUSD)
+	}
 }
 
 func TestResolveAPIKeyFrom_env(t *testing.T) {

internal/models/openai.go

@@ -93,16 +93,26 @@ func (c *OpenAIClient) Generate(ctx context.Context, req GenerateRequest) (Gener
 				Content string `json:"content"`
 			} `json:"message"`
 		} `json:"choices"`
+		Usage *struct {
+			PromptTokens     int `json:"prompt_tokens"`
+			CompletionTokens int `json:"completion_tokens"`
+		} `json:"usage"`
 	}
 	if err := json.Unmarshal(b, &out); err != nil {
 		return GenerateResponse{}, fmt.Errorf("models: decode openai response: %w", err)
 	}
 	if len(out.Choices) == 0 {
 		return GenerateResponse{}, fmt.Errorf("models: openai returned no choices")
 	}
+	var pt, ct int
+	if out.Usage != nil {
+		pt = out.Usage.PromptTokens
+		ct = out.Usage.CompletionTokens
+	}
+	cost := estimateOpenAIChatCostUSD(req.Model, pt, ct)
 	return GenerateResponse{
 		Content: out.Choices[0].Message.Content,
-		Meta:    GenerateMeta{DurationMs: time.Since(start).Milliseconds(), CostUSD: 0},
+		Meta:    GenerateMeta{DurationMs: time.Since(start).Milliseconds(), CostUSD: cost},
 	}, nil
 }