Skip to content

Commit 656df8b

Browse files
gltanakagltanaka
authored
Merge pull request #1056 from promptdriven/change/issue-897
chore: PDD sync for #897
2 parents 2973a5e + dfb0c81 commit 656df8b

20 files changed

Lines changed: 1964 additions & 215 deletions

‎.pdd/meta/llm_invoke_python.json‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
2-
"pdd_version": "0.0.241",
3-
"timestamp": "2026-05-17T18:33:34.290918+00:00",
4-
"command": "example",
5-
"prompt_hash": "08f20f2e33886688d14103186bc55760a745b23e9bbe2626a0a7a6c4e1c1528c",
6-
"code_hash": "42370b9d59b0caca264991021fb612f02bdae906c8043451288a64697af894ae",
2+
"pdd_version": "0.0.246.dev34",
3+
"timestamp": "2026-05-21T02:39:13.009516+00:00",
4+
"command": "update",
5+
"prompt_hash": "b409c7b353bbce6bbe8c0d2e6badaf35acdc80b33eb22846844903caf9bf6b53",
6+
"code_hash": "f3401967632bcd3cc3ed00a74d82f958218df1811a3d422a2daf6f248dea16a4",
77
"example_hash": "48c5aece0ddd153f95ec8a53802d2173d4da12a920b398c5feebb307b9958417",
8-
"test_hash": "630ae15410752b1bda2608d72ab104add66b1f9cbbd134c003183339b46353f9",
8+
"test_hash": "c02ea5f55cb6da638618711aaf89bc8e4e44609608066385a8c7965feec5a820",
99
"test_files": {
10-
"test_llm_invoke.py": "630ae15410752b1bda2608d72ab104add66b1f9cbbd134c003183339b46353f9",
10+
"test_llm_invoke.py": "c02ea5f55cb6da638618711aaf89bc8e4e44609608066385a8c7965feec5a820",
1111
"test_llm_invoke_csv_model_registration.py": "1583b5e076fe5227a11f3d1079034ab4a21bc6131a3433f37bd68e35a99ba49e",
1212
"test_llm_invoke_integration.py": "2eb4bd2565761a4148762c6fb73c887cb6e12ff962742e05f5c2d4d62b47aaf9",
1313
"test_llm_invoke_nested_schema.py": "c983a19874abacc3e0ea9d6ca2ec87495960d2dd96d4e93be4039ed1bc995b9b",
@@ -21,4 +21,4 @@
2121
"pdd/core/cloud.py": "0487c0b989996af144df3af1c818a6eeafe52fd209710e5a54eb43990c89ae97",
2222
"pdd/server/token_counter.py": "3391a7c708713e3d370bf9e981a837f1f7ade08af75d69f402301dbf65a79c9a"
2323
}
24-
}
24+
}

‎.pdd/meta/llm_invoke_python_run.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"timestamp": "2026-05-21T04:24:21.384951+00:00",
3+
"exit_code": 0,
4+
"tests_passed": 293,
5+
"tests_failed": 0,
6+
"coverage": 100.0,
7+
"test_hash": "c02ea5f55cb6da638618711aaf89bc8e4e44609608066385a8c7965feec5a820",
8+
"test_files": {
9+
"test_llm_invoke.py": "c02ea5f55cb6da638618711aaf89bc8e4e44609608066385a8c7965feec5a820",
10+
"test_llm_invoke_csv_model_registration.py": "1583b5e076fe5227a11f3d1079034ab4a21bc6131a3433f37bd68e35a99ba49e",
11+
"test_llm_invoke_integration.py": "2eb4bd2565761a4148762c6fb73c887cb6e12ff962742e05f5c2d4d62b47aaf9",
12+
"test_llm_invoke_nested_schema.py": "c983a19874abacc3e0ea9d6ca2ec87495960d2dd96d4e93be4039ed1bc995b9b",
13+
"test_llm_invoke_retry_cost.py": "bfdfe7b814b78813f86b8bc6d081831daf531187feff66358260f6282925958c",
14+
"test_llm_invoke_vertex_retry.py": "eafe91ba9376dc7e2da36faf3cd2de3cef50f70cf1c4fc5655e373deb7f50c26"
15+
}
16+
}

‎.pdd/meta/summarize_directory_python.json‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
{
2-
"pdd_version": "0.0.228",
3-
"timestamp": "2026-05-06T03:28:21.684198+00:00",
4-
"command": "regenerate-public",
5-
"prompt_hash": "ea40b7699d4cf1ab98a7eed076aea7117a1c175ae16956afcfc7f06856837b17",
6-
"code_hash": "a96e3b5ddb13717395fbc046cf0279486e1225d63deb51657f1ddb4eaf80c0af",
7-
"example_hash": null,
8-
"test_hash": "6198ba0d33bcb2d06b6a90106dd2ae7adfc112a1ca8de62c58b23b8f9c2502a9",
2+
"pdd_version": "0.0.246.dev34",
3+
"timestamp": "2026-05-21T02:39:13.009516+00:00",
4+
"command": "update",
5+
"prompt_hash": "d4bb457d5fbf85648e7c2222affd99928729947aa2a46936b7ae2c855bf34d80",
6+
"code_hash": "e715f8c9f90dd9cb6b656ef83df4a09388edd97451e0706e964fe713eecf6aaf",
7+
"example_hash": "99f9b729f3b148891d471e79e5a29e67e39c30aad8e9922ced2982ca02b158ea",
8+
"test_hash": "049ed6ddc2cd8bd3915eeb79f0b87524eef136157f684cfd7453d35d0f799425",
99
"test_files": {
10-
"test_summarize_directory.py": "6198ba0d33bcb2d06b6a90106dd2ae7adfc112a1ca8de62c58b23b8f9c2502a9"
10+
"test_summarize_directory.py": "049ed6ddc2cd8bd3915eeb79f0b87524eef136157f684cfd7453d35d0f799425"
1111
},
1212
"include_deps": {
13-
"context/llm_invoke_example.py": "d749aa00a35c88c254c4aaa9e1280d85342d71214710e2f3f7a2bd9388a92168",
13+
"context/llm_invoke_example.py": "48c5aece0ddd153f95ec8a53802d2173d4da12a920b398c5feebb307b9958417",
1414
"context/load_prompt_template_example.py": "a1cd6619182c6c951f5856dda4070e202875a5884bbfab9cc191d24de2f4951f",
15-
"context/python_preamble.prompt": "57a3e51f529024ec0cb9658cd6ac61a7c8051ba0c8e887b31cf00b2e78a07d83"
15+
"context/python_preamble.prompt": "0388ed131bf986f8752e1bc4c81e4da0460cfe2908ec8c60b1314edbab768254"
1616
}
17-
}
17+
}

‎.pdd/meta/summarize_directory_python_run.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"timestamp": "2026-05-21T04:24:21.384951+00:00",
3+
"exit_code": 0,
4+
"tests_passed": 81,
5+
"tests_failed": 0,
6+
"coverage": 100.0,
7+
"test_hash": "049ed6ddc2cd8bd3915eeb79f0b87524eef136157f684cfd7453d35d0f799425",
8+
"test_files": {
9+
"test_summarize_directory.py": "049ed6ddc2cd8bd3915eeb79f0b87524eef136157f684cfd7453d35d0f799425"
10+
}
11+
}

‎.pdd/meta/track_cost_python.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"pdd_version": "0.0.246.dev34",
3+
"timestamp": "2026-05-21T04:24:21.384951+00:00",
4+
"command": "update",
5+
"prompt_hash": "413975fcaf5d900d1c7edd631ad843a00c1ebe983e70812f86e689821c6218fa",
6+
"code_hash": "3a1282ced73cc96c0428a006db619282c23c31aed6a50e2d427f106505ebd0ce",
7+
"example_hash": "bd60efaa7d6274b0e4d1743ac05785461af65ec785b254730288d08774f72ed6",
8+
"test_hash": "2d272934a7ce4f40ffb1555451ed71fb09ff5fb451b7fd5a6893374968e99d0f",
9+
"test_files": {
10+
"test_track_cost.py": "2d272934a7ce4f40ffb1555451ed71fb09ff5fb451b7fd5a6893374968e99d0f"
11+
},
12+
"include_deps": {}
13+
}

‎.pdd/meta/track_cost_python_run.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"timestamp": "2026-05-21T04:24:21.384951+00:00",
3+
"exit_code": 0,
4+
"tests_passed": 29,
5+
"tests_failed": 0,
6+
"coverage": 100.0,
7+
"test_hash": "2d272934a7ce4f40ffb1555451ed71fb09ff5fb451b7fd5a6893374968e99d0f",
8+
"test_files": {
9+
"test_track_cost.py": "2d272934a7ce4f40ffb1555451ed71fb09ff5fb451b7fd5a6893374968e99d0f"
10+
}
11+
}

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -804,6 +804,7 @@ The generated CSV file includes the following columns:
804804
- cost: The estimated cost of the operation in USD (e.g., 0.05 for 5 cents). This will be zero for local models or operations that do not use a LLM.
805805
- input_files: A list of input files involved in the operation
806806
- output_files: A list of output files generated or modified by the operation
807+
- attempted_models: Semicolon-delimited audit log of every model PDD attempted for the command, across all LLM calls the command made (e.g. `generate` runs code-generation followed by postprocess code extraction — both contribute). When PDD's default model fails and the run falls back to another provider (for example Vertex AI → DeepSeek), each attempted model appears here so users can see the full fallback history rather than only the final successful model. The `model` column above names the model that actually produced the command's output; `attempted_models` is the complete record of what was tried. For commands that catch a substep failure and recover with a different model, the list may contain entries that came AFTER the model named in `model` — those represent attempts that were tried but didn't produce the final output. For a single-attempt successful command this column contains just the successful model. Semicolons inside model names are sanitized to preserve the delimiter. **Ordering:** sequential (single-thread) command paths produce a list in wall-clock attempt order; concurrent paths (e.g. `auto-deps --concurrency > 1`, which fans summarization across worker threads) sort their per-file contributions by file-submission index — a deterministic alternative to wall-clock ordering, which would otherwise depend on thread-scheduler timing.
807808

808809
This comprehensive output allows for detailed tracking of not only the cost and type of operations but also the specific files involved in each PDD command execution.
809810

‎architecture.json‎

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1139,9 +1139,11 @@
11391139
}
11401140
},
11411141
{
1142-
"reason": "Tracks LLM usage costs across operations.",
1143-
"description": "Records token usage and calculates costs. Aggregates costs across workflow steps.",
1144-
"dependencies": [],
1142+
"reason": "Click decorator that tracks per-command LLM cost, model, and attempted-models history to a CSV file.",
1143+
"description": "Records token usage and calculates costs. Aggregates costs across workflow steps. Captures chronological attempted-models history (including failed cloud attempts that fell back to local) via ctx.obj['attempted_models'].",
1144+
"dependencies": [
1145+
"llm_invoke_python.prompt"
1146+
],
11451147
"priority": 33,
11461148
"filename": "track_cost_python.prompt",
11471149
"filepath": "pdd/track_cost.py",
@@ -1156,7 +1158,7 @@
11561158
{
11571159
"name": "track_cost",
11581160
"signature": "(func)",
1159-
"returns": "None",
1161+
"returns": "Callable",
11601162
"sideEffects": [
11611163
"None"
11621164
]
@@ -1171,24 +1173,16 @@
11711173
},
11721174
{
11731175
"name": "collect_files",
1174-
"signature": "(args, kwargs)",
1175-
"returns": "None",
1176-
"sideEffects": [
1177-
"None"
1178-
]
1179-
},
1180-
{
1181-
"name": "wrapper",
1182-
"signature": "()",
1183-
"returns": "None",
1176+
"signature": "(args, kwargs) -> Tuple[List[str], List[str]]",
1177+
"returns": "Tuple[List[str], List[str]]",
11841178
"sideEffects": [
11851179
"None"
11861180
]
11871181
},
11881182
{
11891183
"name": "looks_like_file",
1190-
"signature": "(path_str)",
1191-
"returns": "None",
1184+
"signature": "(path_str) -> bool",
1185+
"returns": "bool",
11921186
"sideEffects": [
11931187
"None"
11941188
]
@@ -8984,5 +8978,47 @@
89848978
]
89858979
}
89868980
}
8981+
},
8982+
{
8983+
"reason": "Provides unified LLM invocation across all PDD operations with provider abstraction, retries, and context window validation.",
8984+
"description": "Provides unified LLM invocation across all PDD operations with provider abstraction, retries, and context window validation.",
8985+
"dependencies": [
8986+
"path_resolution_python.prompt",
8987+
"server/token_counter_python.prompt"
8988+
],
8989+
"priority": 236,
8990+
"filename": "llm_invoke_python.prompt",
8991+
"filepath": "pdd/llm_invoke.py",
8992+
"tags": [
8993+
"module",
8994+
"python"
8995+
],
8996+
"interface": {
8997+
"type": "module",
8998+
"module": {
8999+
"functions": [
9000+
{
9001+
"name": "llm_invoke",
9002+
"signature": "(prompt, input_json, strength, temperature, verbose, output_pydantic, output_schema, time, use_batch_mode, messages, language, use_cloud)",
9003+
"returns": "Dict[str, Any]"
9004+
},
9005+
{
9006+
"name": "setup_file_logging",
9007+
"signature": "(log_file_path=None)",
9008+
"returns": "None"
9009+
},
9010+
{
9011+
"name": "set_verbose_logging",
9012+
"signature": "(verbose=False)",
9013+
"returns": "None"
9014+
},
9015+
{
9016+
"name": "set_quiet_logging",
9017+
"signature": "()",
9018+
"returns": "None"
9019+
}
9020+
]
9021+
}
9022+
}
89879023
}
89889024
]

‎context/cli_example.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,18 @@ def example_cost_tracking_setup():
181181
- cost: Estimated cost in USD (e.g., 0.05 for 5 cents)
182182
- input_files: List of input files involved
183183
- output_files: List of output files generated/modified
184+
- attempted_models: Semicolon-delimited audit log of every model
185+
PDD attempted across all LLM calls the command made (multi-
186+
substep commands like generate run code-gen + postprocess and
187+
both substeps contribute). The `model` column names the model
188+
that produced the operation's output; this column is the full
189+
record of what was tried, including substeps that failed and
190+
were recovered. The two may legitimately differ. Shows fallback
191+
history when a default model fails and PDD switches providers
192+
(e.g. Vertex AI -> DeepSeek). Ordering: sequential paths emit
193+
attempts in wall-clock order; concurrent paths (e.g. auto-deps
194+
--concurrency > 1) sort by file-submission index for
195+
deterministic output across runs.
184196
185197
Args:
186198
None

‎context/core/cli_example.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,18 @@ def example_cost_tracking_setup():
181181
- cost: Estimated cost in USD (e.g., 0.05 for 5 cents)
182182
- input_files: List of input files involved
183183
- output_files: List of output files generated/modified
184+
- attempted_models: Semicolon-delimited audit log of every model
185+
PDD attempted across all LLM calls the command made (multi-
186+
substep commands like generate run code-gen + postprocess and
187+
both substeps contribute). The `model` column names the model
188+
that produced the operation's output; this column is the full
189+
record of what was tried, including substeps that failed and
190+
were recovered. The two may legitimately differ. Shows fallback
191+
history when a default model fails and PDD switches providers
192+
(e.g. Vertex AI -> DeepSeek). Ordering: sequential paths emit
193+
attempts in wall-clock order; concurrent paths (e.g. auto-deps
194+
--concurrency > 1) sort by file-submission index for
195+
deterministic output across runs.
184196
185197
Args:
186198
None

0 commit comments

Comments
 (0)