Merge branch 'dev' into feature/vincilb/json-export

Author: vawsgit

.editorconfig

@@ -0,0 +1,21 @@
+# This file is for EditorConfig and helps maintain consistent coding styles.
+
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab
+indent_size = 4
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true

.github/workflows/run_pytest.yaml

@@ -22,6 +22,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[dev]"
+          pip install -e ".[llm]"
       - name: Test with pytest
         run: |
           coverage run -m pytest  -v -s

.gitignore

@@ -2,11 +2,14 @@
 .vscode/settings.json
 **/__pycache__/
 **/*.egg-info/
+.venv/
 .coverage
 .ash/
+.temp/
+uv.lock
 
 .kiro/*
 .kiro/**/*
 
 build/
-.temp/
+.temp/

.vscode/settings.json.example

@@ -0,0 +1,18 @@
+{
+  "git.openRepositoryInParentFolders": "always",
+  "python.analysis.extraPaths": [],
+  "notebook.formatOnSave.enabled": true,
+  "notebook.codeActionsOnSave": {
+    "notebook.source.fixAll": "explicit",
+    "notebook.source.organizeImports": "explicit"
+  },
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "charliermarsh.ruff",
+    "editor.codeActionsOnSave": {
+      "source.fixAll": "explicit",
+      "source.organizeImports": "explicit"
+    },
+  },
+  "makefile.configureOnOpen": false
+}

AGENTS.md

@@ -0,0 +1,16 @@
+# General Guidance for Agentic Coding Assistants
+
+## Documentation
+- When writing documentation, always try to say more with less.
+- Documentation should be minimal, and only say what needs to be said to communicate how to work with and extend the system.
+
+## MKDocs Documentation Project
+- An MKDocs documentation project exists at [docs/](./docs/). More information can be found in the [docs/README.md](./docs/README.md) and [docs/AGENTS.md](./docs/AGENTS.md) files.
+
+## Testing
+- Test documentation and guidelines can be found in [tests/README.md](./tests/README.md).
+
+## README.md 
+- In addition to MKDocs, The project strives to maintain developer documentation distributed in README.md files throughout the codebase. This documentation exists to help human and AI coding assistants when working with the codebase.
+- When creating directories or working in a directory that does not have a README.md create a README.md file and document the final state of the code and logic that's in the directory. Do this in language that an AI coding assistant trying to understand the implementation and codebas would understand.
+- When working in a directory, always look for a README.md and/or AGENTS.md file for important context about the directory/code contained within.

Makefile

@@ -1,5 +1,11 @@
 # Simple Makefile for stickler
 
+# Define color codes for terminal output
+RED := \033[0;31m
+GREEN := \033[0;32m
+YELLOW := \033[1;33m
+NC := \033[0m  # No Color
+
 install:
 	pip install -e .
 
@@ -12,3 +18,23 @@ test:
 clean:
 	find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
 	find . -type f -name "*.pyc" -delete
+
+# Run linting checks and automatically fix issues
+lint:
+	ruff check --fix
+
+# CI/CD version of lint that only checks but doesn't modify files
+# Used in CI pipelines to verify code quality without making changes
+lint-cicd:
+	@echo "Running code quality checks..."
+	@if ! ruff check; then \
+		echo -e "$(RED)ERROR: Ruff linting failed!$(NC)"; \
+		echo -e "$(YELLOW)Please run 'make ruff-lint' locally to fix these issues.$(NC)"; \
+		exit 1; \
+	fi
+	@if ! ruff format --check; then \
+		echo -e "$(RED)ERROR: Code formatting check failed!$(NC)"; \
+		echo -e "$(YELLOW)Please run 'make format' locally to fix these issues.$(NC)"; \
+		exit 1; \
+	fi
+	@echo -e "$(GREEN)All code quality checks passed!$(NC)"

README.md

@@ -1,5 +1,7 @@
 # Stickler: Structured Object Evaluation for GenAI
 
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/awslabs/stickler)
+
 When in the course of human events, it becomes necessary to evaluate structured outputs from generative AI systems, we must acknowledge that traditional evaluation treats all fields equally. But **not all fields are created equal**.
 
 **Stickler is a Python library that enables complex structured JSON comparison and evaluation** that lets you focus on the fields your customer actually cares about, to answer the question: "Is it doing a good job?" 
@@ -117,6 +119,28 @@ result = ground_truth.compare_with(prediction, evaluator_format=True)
 print(f"Overall Score: {result['overall']['anls_score']:.3f}")
 ```
 
+### Confidence Evaluation
+
+Evaluate prediction confidence calibration with AUROC metrics:
+
+```python
+# Prediction with confidence scores
+prediction = Invoice.from_json({
+    "invoice_number": {"value": "INV-2024-001", "confidence": 0.95},
+    "total": {"value": 1247.50, "confidence": 0.8}
+})
+
+# Enable confidence metrics
+result = ground_truth.compare_with(
+    prediction,
+    add_confidence_metrics=True,
+    document_field_comparisons=True
+)
+
+print(f"Overall Score: {result['overall_score']:.3f}")
+print(f"Confidence AUROC: {result['auroc_confidence_metric']:.3f}")
+```
+
 ### Dynamic Model Creation (New!)
 
 Create models from JSON configuration for maximum flexibility:
@@ -654,3 +678,8 @@ print(f"Line Items: {result['field_scores']['line_items']:.3f}")  # ~1.0 - match
 ## Examples
 
 Check out the `examples/` directory for more detailed usage examples and notebooks.
+
+# A note for AI assisted coding agents
+- The project uses coding assistant agnostic context files, like README.md and AGENTS.md
+- When working in a directory, always look for a README.md and/or AGENTS.md file for important context about the directory/code contained within.
+- Read this [AGENTS.md](./AGENTS.md) file for more information

docs/docs/Contributing/code-style.md

@@ -26,6 +26,23 @@ ruff check --fix .
 ruff check src/stickler/comparators/
 ```
 
+### Automatic Formatting in VS Code
+
+To enable automatic code formatting and linting in VS Code:
+
+1. Copy the example settings file:
+   ```bash
+   cp .vscode/settings.json.example .vscode/settings.json
+   ```
+
+2. Install the Ruff extension for VS Code (if not already installed):
+   - Open VS Code
+   - Go to Extensions (Cmd+Shift+X on macOS, Ctrl+Shift+X on Windows/Linux)
+   - Search for "Ruff" by Charlie Marsh
+   - Click Install
+
+The settings file configures VS Code to automatically format Python files on save and organize imports using Ruff.
+
 ### CI Integration
 
 Linting runs automatically on every push and pull request via GitHub Actions. While currently non-blocking, you should address linting issues before submitting PRs.

docs/docs/Core-Concepts/Confidence_Integration.md

@@ -0,0 +1,191 @@
+---
+title: Confidence Integration
+---
+
+# Confidence Integration
+
+Stickler supports confidence scores alongside field values to evaluate how well prediction confidence correlates with actual accuracy.
+
+## AUROC for Confidence Calibration: Theoretical Foundation
+
+### What is Confidence Estimation?
+
+Confidence estimation in machine learning refers to a model's ability to assess the reliability of its own predictions. Rather than simply outputting a prediction, a well-calibrated model also provides a confidence score indicating how likely that prediction is to be correct. This is crucial for production systems where understanding prediction uncertainty enables better decision-making, error handling, and human-AI collaboration.
+
+In structured data extraction tasks, confidence estimation allows systems to:
+- Flag uncertain extractions for human review
+- Prioritize high-confidence predictions for automated processing  
+- Provide transparency about model reliability to end users
+- Enable adaptive thresholding based on use case requirements
+
+### Research Foundation and Motivation
+
+The choice of AUROC for confidence evaluation is grounded in research on model calibration and confidence assessment. Other approaches to evaluating confidence, such as Expected Calibration Error (ECE), have significant limitations that motivated our selection of AUROC.
+
+#### Limitations of Expected Calibration Error (ECE)
+
+As noted in recent research (Jiang et al., 2025), Expected Calibration Error has a fundamental flaw:
+
+> "Expected calibration error (ECE) (Guo et al., 2017a; Naeini et al., 2015) is also a standard metric to measure how closely a model's confidence matches its accuracy. However, ECE does not assess a model's ability to discriminate between correct and incorrect answers—a model with accuracy 0.5 can achieve perfect ECE by outputting a confidence of 0.5 for all of its answers. Therefore, we focus our results on the AUC."
+
+This limitation means ECE can be "gamed" by models that output uniform confidence scores, making it unsuitable for evaluating whether confidence scores actually help distinguish between correct and incorrect predictions.
+
+#### Why AUROC for Confidence Evaluation?
+
+AUROC addresses the core question: **"Do higher confidence scores correspond to more accurate predictions?"** This discrimination ability is essential for practical applications where confidence scores guide decision-making.
+
+**Mathematical Definition:**
+AUROC measures the probability that a randomly chosen correct prediction has higher confidence than a randomly chosen incorrect prediction. It treats confidence estimation as a binary classification problem where:
+- **Positive class**: Correct predictions (field matches ground truth)
+- **Negative class**: Incorrect predictions (field doesn't match ground truth)  
+- **Classifier score**: Model's confidence score
+
+**Advantages over alternatives:**
+1. **Discrimination focus**: Directly measures whether confidence correlates with accuracy
+2. **Threshold independence**: Evaluates performance across all possible confidence thresholds
+3. **Intuitive interpretation**: Values closer to 1.0 indicate better calibration
+4. **Robust to class imbalance**: Works well even when correct/incorrect predictions are unbalanced
+
+### AUROC Calculation in Stickler
+
+**AUROC (Area Under ROC Curve)** measures confidence calibration quality:
+
+- **1.0**: Perfect calibration (high confidence = correct, low confidence = incorrect)
+- **0.5**: Random calibration (confidence doesn't correlate with accuracy)  
+- **0.0**: Inverse calibration (high confidence = incorrect, low confidence = correct)
+
+#### Implementation Details
+
+1. For each field with confidence, determine if it matches ground truth
+2. Create binary classification: match (1) or no-match (0)
+3. Use confidence scores as prediction probabilities
+4. Calculate ROC curve and area under it
+
+#### Requirements
+
+- At least one field with confidence scores
+- Both matches and non-matches in the comparison
+- `document_field_comparisons=True` must be enabled
+
+### Interpretation Guidelines
+
+**Well-Calibrated Models (AUROC ≈ 0.8-1.0):**
+- High confidence predictions are predominantly correct
+- Low confidence predictions are predominantly incorrect
+- Confidence scores provide meaningful signal for decision-making
+
+**Poorly-Calibrated Models (AUROC ≈ 0.0-0.3):**
+- High confidence predictions are predominantly incorrect
+- May indicate systematic overconfidence or model issues
+- Confidence scores mislead rather than inform
+
+**Random Calibration (AUROC ≈ 0.5):**
+- Confidence scores don't correlate with accuracy
+- Model may need confidence calibration techniques
+- Consider post-processing methods like Platt scaling
+
+### Limitations and Considerations
+
+**Edge Cases:**
+- **All predictions correct**: AUROC undefined (no negative class)
+- **All predictions incorrect**: AUROC undefined (no positive class)
+- **Uniform confidence**: AUROC = 0.5 regardless of accuracy
+
+**Sample Size Requirements:**
+- Reliable AUROC calculation requires sufficient examples of both correct and incorrect predictions
+- Very small datasets may produce unstable AUROC estimates
+
+**Interpretation Context:**
+- AUROC should be interpreted alongside overall accuracy
+- High AUROC with low accuracy may indicate good uncertainty awareness
+- Low AUROC with high accuracy suggests overconfident correct predictions
+
+### References and Further Reading
+
+**Key Papers:**
+- Jiang, Z., et al. (2025). "Confidence Estimation in Large Language Models." [arXiv:2502.01126](https://arxiv.org/pdf/2502.01126)
+- Guo, C., et al. (2017). "On Calibration of Modern Neural Networks." ICML 2017
+- Naeini, M. P., et al. (2015). "Obtaining Well Calibrated Probabilities Using Bayesian Binning." AAAI 2015
+
+**Additional Resources:**
+- Scikit-learn ROC AUC documentation: [sklearn.metrics.roc_auc_score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html)
+
+## JSON Structure
+
+### Standard Format
+```json
+{
+  "name": "Widget",
+  "price": 29.99
+}
+```
+
+### Confidence Format
+```json
+{
+  "name": {"value": "Widget", "confidence": 0.95},
+  "price": {"value": 29.99, "confidence": 0.8}
+}
+```
+
+### Mixed Format
+```json
+{
+  "name": {"value": "Widget", "confidence": 0.95},
+  "price": 29.99,
+  "sku": {"value": "ABC123", "confidence": 0.7}
+}
+```
+
+## Confidence Structure Rules
+
+A valid confidence structure must have exactly two keys:
+- `"value"`: The actual field value
+- `"confidence"`: Float between 0.0 and 1.0
+
+## Nested Structures
+
+Confidence works with nested objects and arrays:
+
+```json
+{
+  "customer": {
+    "name": {"value": "John Doe", "confidence": 0.92},
+    "address": {
+      "street": {"value": "123 Main St", "confidence": 0.85},
+      "city": "New York"
+    }
+  },
+  "items": [
+    {
+      "product": {"value": "Laptop", "confidence": 0.89},
+      "price": {"value": 1299.99, "confidence": 0.76}
+    }
+  ]
+}
+```
+
+## Confidence Access API
+
+```python
+# Get individual field confidence
+confidence = model.get_field_confidence("name")  # Returns float or None
+
+# Get all confidences
+all_confidences = model.get_all_confidences()  # Returns dict
+
+# Nested field access
+street_conf = model.get_field_confidence("address.street")
+
+# Array field access  
+item_conf = model.get_field_confidence("items[0].product")
+```
+
+## Field Path Format
+
+Confidence paths use dot notation for nested fields and bracket notation for arrays:
+
+- Simple field: `"name"`
+- Nested field: `"address.street"`
+- Array element: `"items[0].product"`
+- Nested in array: `"orders[1].customer.name"`

docs/docs/Guides/.nav.yml

@@ -3,4 +3,5 @@ nav:
   - StructuredModel_Dynamic_Creation.md
   - StructuredModel_Advanced_Functionality.md
   - Universal_Aggregate_Field_Feature.md
-  - ['*']
+  - ['*']
+  - Confidence_Evaluation_Guide.md