Merge branch 'feature/avg-cost-per-page' into 'develop'

feat: add avg cost per page to test results, comparison, and downloads

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!606

Author: rstrahan

CHANGELOG.md

@@ -7,6 +7,12 @@ SPDX-License-Identifier: MIT-0
 
 ### Added
 
+- **Average Cost Per Page Metric** — Test results and test comparison views now display an "Avg Cost/Page" metric, calculated from total cost and page counts in the cost breakdown. Also included in CSV and JSON exports from the comparison view.
+
+- **Prompt Preview** — New "Prompt Preview" tab in the Configuration page lets you preview the actual prompts sent to the LLM for each processing step (Classification, Extraction, Assessment, Summarization). Config-derived placeholders are filled in with real values (class names, cleaned JSON Schema), while document-specific placeholders are shown as highlighted markers. Includes token estimates, copy-to-clipboard, and a substitution details panel showing the exact schema sent to the LLM. Helps optimize document class schemas and prompt templates.
+
+- **IDP CLI `chat` Command & SDK `ChatOperation`** — Interactive Agent Companion Chat from the terminal and programmatic SDK access. Runs the same multi-agent orchestrator as the Web UI locally, with real-time streaming and multi-turn conversation support. Includes Analytics Agent, Error Analyzer Agent, and optionally Code Intelligence Agent (`--enable-code-intelligence`). Available as `idp-cli chat --stack-name <stack>` for interactive use, `--prompt` flag for single-shot scripting, and `client.chat.send_message()` in the Python SDK. See `docs/idp-cli.md#chat`.
+
 - **Multi-Document Discovery** — New capability to automatically discover document classes from a collection of documents. Instead of manually defining document schemas one at a time, users point to a folder of mixed documents and the system automatically identifies document types, clusters similar documents, generates JSON Schemas with field definitions for each type, and saves them to a configuration version — ready for immediate use in the processing pipeline. Available from the Web UI, CLI (`idp-cli discover-multidoc`), and SDK (`client.discovery.run_multi_doc()`).
   - **Web UI**: New "Multi-Document" tab on the Discovery page with job submission form (config version selector, bucket selector, S3 prefix input, zip upload), jobs table with search/filter/sort/pagination, and detailed job results page with pipeline progress, expandable JSON schemas, config deep-links, and Quality Review Report
   - **CLI**: `idp-cli discover-multidoc --dir ./samples/ -o ./schemas/` with Rich progress bars, results table, and reflection report

docs/test-studio.md

@@ -484,6 +484,7 @@ For full details on configuration versioning, see [configuration-versions.md](co
 - Comprehensive metrics display including:
   - **Test run metadata**: Configuration version, duration, context, file counts
   - **Overall accuracy and confidence metrics**
+  - **Cost metrics**: Total cost and average cost per page
   - **Accuracy breakdown** (precision, recall, F1-score, false alarm rate, false discovery rate)
   - **Field-Level Metrics**: Per-field extraction performance table with columns: Field Name, Accuracy, Precision, Recall, TP, FP, TN, FN
   - **Average Document Split Classification Metrics**:

src/ui/src/components/test-studio/TestComparison.tsx

@@ -20,7 +20,13 @@ import TestStudioHeader from './TestStudioHeader';
 import useLocalStorage from '../common/local-storage';
 import useConfigurationVersions from '../../hooks/use-configuration-versions';
 import { formatConfigVersionLink, formatConfigVersionText, type ConfigVersion as UtilsConfigVersion } from './utils/configVersionUtils';
-import { parseComparisonMetrics, parseWeightedOverallScores, parseConfigSettingValues } from '../../graphql/awsjson-parsers';
+import {
+  parseComparisonMetrics,
+  parseWeightedOverallScores,
+  parseConfigSettingValues,
+  calculateAvgCostPerPage,
+} from '../../graphql/awsjson-parsers';
+import type { CostBreakdown } from '../../graphql/awsjson-types';
 
 const client = generateClient();
 
@@ -302,6 +308,13 @@ const TestComparison = ({ preSelectedTestRunIds = [] }: TestComparisonProps): Re
           run.totalCost !== null && run.totalCost !== undefined ? `$${Number(run.totalCost).toFixed(4)}` : 'N/A',
         ),
       ],
+      [
+        'Avg Cost/Page',
+        ...Object.values(completeTestRuns).map((run) => {
+          const avg = calculateAvgCostPerPage(run.totalCost as number, run.costBreakdown as CostBreakdown);
+          return avg !== null ? `$${avg.toFixed(4)}` : 'N/A';
+        }),
+      ],
       [
         'Average Accuracy',
         ...Object.values(completeTestRuns).map((run) =>
@@ -624,6 +637,7 @@ const TestComparison = ({ preSelectedTestRunIds = [] }: TestComparisonProps): Re
             completedFiles: testRun.completedFiles,
             failedFiles: testRun.failedFiles,
             totalCost: testRun.totalCost,
+            avgCostPerPage: calculateAvgCostPerPage(testRun.totalCost as number, testRun.costBreakdown as CostBreakdown),
             averageAccuracy: testRun.overallAccuracy,
             averageConfidence: testRun.averageConfidence,
             averageWeightedOverallScore: (() => {
@@ -818,6 +832,15 @@ const TestComparison = ({ preSelectedTestRunIds = [] }: TestComparisonProps): Re
                     ]),
                   ),
                 },
+                {
+                  metric: 'Avg Cost/Page',
+                  ...Object.fromEntries(
+                    Object.entries(completeTestRuns).map(([testRunId, testRun]) => {
+                      const avg = calculateAvgCostPerPage(testRun.totalCost as number, testRun.costBreakdown as CostBreakdown);
+                      return [testRunId, avg !== null ? `$${avg.toFixed(4)}` : 'N/A'];
+                    }),
+                  ),
+                },
                 {
                   metric: 'Average Accuracy',
                   ...Object.fromEntries(

src/ui/src/components/test-studio/TestResults.tsx

@@ -35,6 +35,7 @@ import useAppContext from '../../contexts/app';
 import { formatConfigVersionLink } from './utils/configVersionUtils';
 import {
   parseCostBreakdown,
+  calculateAvgCostPerPage,
   parseAccuracyBreakdown,
   parseSplitClassificationMetrics,
   parseFieldMetrics,
@@ -751,6 +752,9 @@ const TestResults = ({ testRunId, setSelectedTestRunId }: TestResultsProps): Rea
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const costBreakdown: any = results.costBreakdown ? parseCostBreakdown(results.costBreakdown as string) : null;
+
+  // Calculate avg cost per page from costBreakdown page counts
+  const avgCostPerPage = calculateAvgCostPerPage(results.totalCost as number, costBreakdown);
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const accuracyBreakdown: any = results.accuracyBreakdown ? parseAccuracyBreakdown(results.accuracyBreakdown as string) : null;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -1012,6 +1016,10 @@ const TestResults = ({ testRunId, setSelectedTestRunId }: TestResultsProps): Rea
               {results.totalCost !== null && results.totalCost !== undefined ? `$${(results.totalCost as number).toFixed(4)}` : 'N/A'}
             </Box>
           </Box>
+          <Box>
+            <Box variant="awsui-key-label">Avg Cost/Page</Box>
+            <Box fontSize="heading-l">{avgCostPerPage !== null ? `$${avgCostPerPage.toFixed(4)}` : 'N/A'}</Box>
+          </Box>
           <Box>
             <Box variant="awsui-key-label">Avg Confidence</Box>
             <Box fontSize="heading-l">

src/ui/src/graphql/awsjson-parsers.ts

@@ -58,6 +58,22 @@ export function parseCostBreakdown(json: unknown): CostBreakdown {
   return safeParse<CostBreakdown>(json, {});
 }
 
+export function calculateTotalPages(costBreakdown: CostBreakdown): number {
+  let totalPages = 0;
+  Object.values(costBreakdown).forEach((services) => {
+    Object.values(services).forEach((details) => {
+      if (details.unit === 'pages') totalPages += Number(details.value) || 0;
+    });
+  });
+  return totalPages;
+}
+
+export function calculateAvgCostPerPage(totalCost: number | null | undefined, costBreakdown: CostBreakdown | null): number | null {
+  if (totalCost == null || !costBreakdown) return null;
+  const totalPages = calculateTotalPages(costBreakdown);
+  return totalPages > 0 ? totalCost / totalPages : null;
+}
+
 export function parseTestRunConfig(json: unknown): TestRunConfig | null {
   return safeParse<TestRunConfig | null>(json, null);
 }

src/ui/src/graphql/awsjson-types.ts

@@ -30,6 +30,7 @@ export interface CostBreakdownServiceDetail {
   estimated_cost?: number;
   value?: number;
   unit_cost?: number;
+  unit?: string;
   [key: string]: unknown;
 }