feat: Removed python schema files & support JSON Schema uploads and updates to related deployment scripts

docs/CustomizeSchemaData.md

@@ -15,9 +15,9 @@ Before processing documents, schemas must be **registered** in the system and gr
 ```mermaid
 flowchart TB
     subgraph Step1["<b>Step 1: Register Schemas</b> (one per document type)<br/>POST /schemavault/ × N"]
-        S1["🗎 AutoInsuranceClaimForm<br/><i>autoclaim.py</i><br/>Schema ID: abc123"]
-        S2["🗎 PoliceReportDocument<br/><i>policereport.py</i><br/>Schema ID: def456"]
-        S3["🗎 RepairEstimateDocument<br/><i>repairestimate.py</i><br/>Schema ID: ghi789"]
+        S1["🗎 AutoInsuranceClaimForm<br/><i>autoclaim.json</i><br/>Schema ID: abc123"]
+        S2["🗎 PoliceReportDocument<br/><i>policereport.json</i><br/>Schema ID: def456"]
+        S3["🗎 RepairEstimateDocument<br/><i>repairestimate.json</i><br/>Schema ID: ghi789"]
         S4["🗎 ...<br/><i>more schemas</i>"]
     end
 
@@ -79,10 +79,10 @@ A new class needs to be created that defines the schema as a strongly typed Pyth
 
 | Schema                    | File                                                                              | Class Name                      | Auto-registered |
 | ------------------------- | --------------------------------------------------------------------------------- | ------------------------------- | --------------- |
-| Auto Insurance Claim Form | [autoclaim.py](/src/ContentProcessorAPI/samples/schemas/autoclaim.py)             | `AutoInsuranceClaimForm`        | ✅               |
-| Police Report             | [policereport.py](/src/ContentProcessorAPI/samples/schemas/policereport.py)       | `PoliceReportDocument`          | ✅               |
-| Repair Estimate           | [repairestimate.py](/src/ContentProcessorAPI/samples/schemas/repairestimate.py)   | `RepairEstimateDocument`        | ✅               |
-| Damaged Vehicle Image     | [damagedcarimage.py](/src/ContentProcessorAPI/samples/schemas/damagedcarimage.py) | `DamagedVehicleImageAssessment` | ✅               |
+| Auto Insurance Claim Form | [autoclaim.json](/src/ContentProcessorAPI/samples/schemas/autoclaim.json)             | `AutoInsuranceClaimForm`        | ✅               |
+| Police Report             | [policereport.json](/src/ContentProcessorAPI/samples/schemas/policereport.json)       | `PoliceReportDocument`          | ✅               |
+| Repair Estimate           | [repairestimate.json](/src/ContentProcessorAPI/samples/schemas/repairestimate.json)   | `RepairEstimateDocument`        | ✅               |
+| Damaged Vehicle Image     | [damagedcarimage.json](/src/ContentProcessorAPI/samples/schemas/damagedcarimage.json) | `DamagedVehicleImageAssessment` | ✅               |
 
 > **Note:** All 4 schemas are automatically registered during deployment (via `azd up` or the `register_schema.py` script) and grouped into the **"Auto Claim"** schema set.
 
@@ -92,58 +92,58 @@ Duplicate one of these files and update with a class definition that represents
 > 
 > *Generate a Schema Class based on the following autoclaim.py schema definition, which has been built and derived from Pydantic BaseModel class. The generated Schema Class should be called "Freight Shipment Bill of Lading" schema file. Please define the entities based on standard bill of lading documents in the logistics industry.*
 
-### Class Structure
-
-Each schema `.py` file must include:
-
-```python
-from pydantic import BaseModel, Field
-from typing import List, Optional
-
-class SubModel(BaseModel):
-    """Description of this sub-entity — used as LLM context."""
-    
-    field_name: Optional[str] = Field(
-        description="What this field represents, e.g. Consignee company name"
-    )
-
-class MyDocumentSchema(BaseModel):
-    """Top-level description of the document type."""
-
-    some_field: Optional[str] = Field(description="...")
-    sub_entity: Optional[SubModel] = Field(description="...")
-    
-    @staticmethod
-    def example() -> "MyDocumentSchema":
-        """Returns an empty instance of this schema."""
-        return MyDocumentSchema(some_field="", sub_entity=SubModel.example())
-
-    @staticmethod
-    def from_json(json_str: str) -> "MyDocumentSchema":
-        """Creates an instance from a JSON string."""
-        return MyDocumentSchema.model_validate_json(json_str)
-
-    def to_dict(self) -> dict:
-        """Converts this instance to a dictionary."""
-        return self.model_dump()
+### Schema Document Structure
+
+Each schema `.json` file must be a JSON Schema (Draft 2020-12) with
+`"type": "object"` at the root and a `"properties"` block. Example:
+
+```json
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "MyDocumentSchema",
+  "description": "Top-level description of the document type.",
+  "type": "object",
+  "properties": {
+    "some_field": {
+      "type": ["string", "null"],
+      "description": "What this field represents, e.g. policy number"
+    },
+    "sub_entity": {
+      "$ref": "#/$defs/SubModel"
+    }
+  },
+  "$defs": {
+    "SubModel": {
+      "title": "SubModel",
+      "description": "Description of this sub-entity — used as LLM context.",
+      "type": "object",
+      "properties": {
+        "field_name": {
+          "type": ["string", "null"],
+          "description": "What this field represents, e.g. Consignee company name"
+        }
+      }
+    }
+  }
+}
 ```
 
 ### Key Rules
 
 | Element                  | Requirement                                                                                                                                                                  |
 | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Inheritance**          | All classes must inherit from `pydantic.BaseModel`                                                                                                                           |
-| **Field descriptions**   | Every field must have a `description=` — this is the prompt text the LLM uses for extraction. Include examples for better accuracy (e.g., `"Date of loss, e.g. 01/15/2026"`) |
-| **Optional vs Required** | Use `Optional[str]` for fields that may not be present in every document                                                                                                     |
-| **Subclasses**           | Use nested `BaseModel` classes for complex entities (address, line items, etc.)                                                                                              |
-| **Required methods**     | `example()`, `from_json()`, `to_dict()` — all three must be present                                                                                                          |
-| **Class docstring**      | Include a description — it's used as context during mapping                                                                                                                  |
+| **Root type**            | Must be `"type": "object"` with a `"properties"` block                                                                                                                       |
+| **Field descriptions**   | Every property must have a `"description"` — this is the prompt text the LLM uses for extraction. Include examples for better accuracy (e.g., `"Date of loss, e.g. 01/15/2026"`) |
+| **Optional vs Required** | Use `["string", "null"]` for fields that may not be present in every document; list required keys in the root `"required"` array if any                                       |
+| **Sub-objects**          | Define reusable nested types under `"$defs"` and reference them via `"$ref": "#/$defs/<Name>"`                                                                                |
+| **Class name**           | Use a top-level `"title"` field; this becomes `ClassName` in the Schema Vault. If absent, the request body's `ClassName` (or filename) is used                                |
+| **Top-level description**| Include a `"description"` — it's used as context during mapping                                                                                                              |
 
 ---
 
 ## Step 2: Register Schemas
 
-After creating your `.py` class files, register each schema in the system. Registration uploads the class file to Blob Storage and stores metadata in Cosmos DB.
+After creating your `.json` schema files, register each schema in the system. Registration uploads the file to Blob Storage and stores metadata in Cosmos DB.
 
 ### Option A: Register via API (individual)
 
@@ -152,7 +152,7 @@ After creating your `.py` class files, register each schema in the system. Regis
 | Part          | Type        | Description                                                       |
 | ------------- | ----------- | ----------------------------------------------------------------- |
 | `schema_info` | JSON string | `{"ClassName": "MyDocumentSchema", "Description": "My Document"}` |
-| `file`        | File upload | The `.py` class file (max 1 MB)                                   |
+| `file`        | File upload | The `.json` JSON Schema file (max 1 MB)                           |
 
 Example using the REST Client extension:
 
@@ -177,10 +177,10 @@ For bulk registration, use the provided script with a JSON manifest. The script
 ```json
 {
   "schemas": [
-    { "File": "autoclaim.py",       "ClassName": "AutoInsuranceClaimForm",       "Description": "Auto Insurance Claim Form" },
-    { "File": "damagedcarimage.py", "ClassName": "DamagedVehicleImageAssessment","Description": "Damaged Vehicle Image Assessment" },
-    { "File": "policereport.py",    "ClassName": "PoliceReportDocument",         "Description": "Police Report Document" },
-    { "File": "repairestimate.py",  "ClassName": "RepairEstimateDocument",       "Description": "Repair Estimate Document" }
+    { "File": "autoclaim.json",       "ClassName": "AutoInsuranceClaimForm",       "Description": "Auto Insurance Claim Form" },
+    { "File": "damagedcarimage.json", "ClassName": "DamagedVehicleImageAssessment","Description": "Damaged Vehicle Image Assessment" },
+    { "File": "policereport.json",    "ClassName": "PoliceReportDocument",         "Description": "Police Report Document" },
+    { "File": "repairestimate.json",  "ClassName": "RepairEstimateDocument",       "Description": "Repair Estimate Document" }
   ],
   "schemaset": {
     "Name": "Auto Claim",
@@ -259,16 +259,90 @@ Repeat for each schema. The SchemaSet now holds references to all your document
 Once schemas are registered and grouped into a SchemaSet, the pipeline uses them automatically during the **Map** step:
 
 1. **Schema lookup** — The Map handler reads the `Schema_Id` from the processing queue message, then fetches metadata from Cosmos DB
-2. **Dynamic class loading** — Downloads the `.py` file from Blob Storage and dynamically loads the Pydantic class
-3. **JSON Schema generation** — Calls `model_json_schema()` on the class to produce a full JSON Schema with all field descriptions
+2. **Schema materialisation** — Downloads the JSON Schema document from Blob Storage and builds a Pydantic model from it in memory (no code execution)
+3. **JSON Schema generation** — Calls `model_json_schema()` on the materialised model to produce the schema with all field descriptions
 4. **LLM extraction** — Embeds the JSON Schema into the GPT-5.1 system prompt with `response_format` for structured JSON output (temperature=0.1 for deterministic results)
-5. **Validation & scoring** — Parses the GPT response back into the Pydantic class, then computes per-field confidence scores using log-probabilities
+5. **Validation & scoring** — Parses the GPT response back into the Pydantic model, then computes per-field confidence scores using log-probabilities
 
 This means your field descriptions in the schema class **directly influence extraction quality** — write clear, specific descriptions with examples for best results.
 
 ---
 
-## Related Documentation
+## Authoring Schemas as JSON
+
+The schema vault accepts **JSON Schema** documents (Draft 2020-12) only.
+JSON schemas are treated strictly as data: the worker parses them and
+materialises a Pydantic model in memory without executing any uploaded
+code, eliminating an entire class of remote-code-execution risk in the
+schema-management path. The legacy executable `.py` format has been
+removed; uploads of `.py` files are rejected with HTTP 415.
+
+### Format requirements
+
+| | JSON Schema |
+| --- | --- |
+| Format | Declarative JSON document |
+| Worker behaviour | Parses JSON, builds model in memory |
+| Authoring | Pydantic-compatible JSON |
+| Side-effects on import | Impossible |
+
+### Authoring with the conversion helper
+
+If you have an existing Pydantic-based `.py` schema, the repo ships a
+helper that emits the equivalent JSON Schema:
+
+```bash
+python scripts/py_schema_to_json.py \
+    src/ContentProcessorAPI/samples/schemas/autoclaim.py \
+    AutoInsuranceClaimForm
+```
+
+This writes `autoclaim.json` next to the source file. Under the hood it
+calls `Model.model_json_schema()` from Pydantic v2 — the same call the
+worker uses today to build the LLM prompt. The output is therefore
+already aligned with the contract the pipeline expects.
+
+The accelerator ships a golden conversion of the auto-claim sample at
+[/src/ContentProcessorAPI/samples/schemas/autoclaim.json](/src/ContentProcessorAPI/samples/schemas/autoclaim.json)
+that you can reference.
+
+### Upload via API
+
+`POST /schemavault/` accepts JSON Schema documents. Send the file with
+`Content-Type: application/json`:
+
+```http
+POST /schemavault/
+Content-Type: multipart/form-data
+- data: { "ClassName": "InvoiceSchema", "Description": "Invoice extraction" }
+- file: invoice.json   (application/json)
+```
+
+When uploading JSON:
+
+- The schema must be a JSON object with `"type": "object"` and a
+  `"properties"` block.
+- The schema's `title` (if present) becomes the `ClassName` recorded in
+  Cosmos. If the JSON has no `title`, the request body's `ClassName` is
+  used as a fallback.
+- Two project-specific extension keywords are accepted:
+  - `x-cps-extract-prompt` — optional override for the LLM extraction
+    prompt for that field.
+  - `x-cps-required-on-save` — marks a field that must be present in
+    the LLM output before persistence.
+  Any other `x-…` keyword is rejected.
+- The schema must be ≤ 1 MB.
+
+### Constraints relative to the legacy Python schemas
+
+JSON schemas are pure data. They cannot carry custom validation logic
+written in Python (e.g. `field_validator`). For most extraction
+schemas this is not a limitation — the existing samples don't use
+custom validators — but if you depend on imperative validation, keep
+authoring those schemas in Python locally and run the resulting JSON
+through the API.
+
+
 
 - [Modifying System Processing Prompts](./CustomizeSystemPrompts.md) — Customize extraction and mapping prompts
 - [Gap Analysis Ruleset Guide](./GapAnalysisRulesetGuide.md) — Define gap rules that reference your document types

infra/scripts/post_deployment.ps1

@@ -124,6 +124,15 @@ if (-not $ApiReady) {
 
         Write-Host "  Registering new schema '$ClassName'..."
 
+        # Only JSON Schema descriptors are accepted. The legacy .py format
+        # was removed as part of the schemavault RCE remediation.
+        $extension = [System.IO.Path]::GetExtension($SchemaFile).ToLowerInvariant()
+        if ($extension -ne '.json') {
+            Write-Host "  Unsupported schema extension '$extension' for '$SchemaFile'. Only .json is accepted. Skipping..."
+            continue
+        }
+        $contentType = 'application/json'
+
         # Build multipart form data
         $dataPayload = @{ ClassName = $ClassName; Description = $Description } | ConvertTo-Json -Compress
         $fileBytes   = [System.IO.File]::ReadAllBytes($SchemaFile)
@@ -137,7 +146,7 @@ if (-not $ApiReady) {
             $dataPayload,
             "--$boundary",
             "Content-Disposition: form-data; name=`"file`"; filename=`"$fileName`"",
-            "Content-Type: text/x-python$LF",
+            "Content-Type: $contentType$LF",
             [System.Text.Encoding]::UTF8.GetString($fileBytes),
             "--$boundary--$LF"
         ) -join $LF

infra/scripts/post_deployment.sh

@@ -136,10 +136,19 @@ else
     echo "  Registering new schema '$CLASS_NAME'..."
     DATA_PAYLOAD="{\"ClassName\": \"$CLASS_NAME\", \"Description\": \"$DESCRIPTION\"}"
 
+    # Only JSON Schema descriptors are accepted. The legacy .py format
+    # was removed as part of the schemavault RCE remediation.
+    EXT="${FILE_NAME##*.}"
+    if [ "$EXT" != "json" ]; then
+      echo "  Unsupported schema extension '.$EXT' for '$FILE_NAME'. Only .json is accepted. Skipping..."
+      continue
+    fi
+    CONTENT_TYPE="application/json"
+
     RESPONSE=$(curl -s -w "\n%{http_code}" \
       -X POST "$SCHEMAVAULT_URL" \
       -F "data=$DATA_PAYLOAD" \
-      -F "file=@$SCHEMA_FILE;type=text/x-python" \
+      -F "file=@$SCHEMA_FILE;type=$CONTENT_TYPE" \
       --connect-timeout 60)
 
     HTTP_CODE=$(echo "$RESPONSE" | tail -1)

scripts/py_schema_to_json.py

@@ -0,0 +1,76 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Convert a legacy Pydantic ``.py`` schema into a declarative ``.json`` schema.
+
+This helper is part of the migration away from executable Python schemas.
+It imports a Pydantic model from a ``.py`` file *in a trusted local
+context* (the developer's machine), reads its
+:py:meth:`pydantic.BaseModel.model_json_schema` output, and writes the
+result to a ``.json`` file alongside.
+
+Usage:
+
+    python scripts/py_schema_to_json.py \
+        src/ContentProcessorAPI/samples/schemas/autoclaim.py \
+        AutoInsuranceClaimForm
+
+The generated JSON is what should be uploaded to the schema vault going
+forward; it is data only and never executed by the worker.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+def convert(py_path: Path, class_name: str, out_path: Path | None = None) -> Path:
+    """Load *class_name* from *py_path* and write its JSON schema next to it."""
+    spec = importlib.util.spec_from_file_location(py_path.stem, py_path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Cannot import schema module from {py_path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)  # noqa: S102 - trusted local conversion only
+
+    cls = getattr(module, class_name, None)
+    if cls is None or not isinstance(cls, type) or not issubclass(cls, BaseModel):
+        raise RuntimeError(
+            f"'{class_name}' is not a Pydantic BaseModel in {py_path}"
+        )
+
+    schema = cls.model_json_schema()
+    # Pydantic emits "title" at the root; ensure it matches the requested
+    # class name so the worker's ``derive_class_name`` picks it up.
+    schema["title"] = class_name
+
+    target = out_path or py_path.with_suffix(".json")
+    target.write_text(json.dumps(schema, indent=2) + "\n", encoding="utf-8")
+    return target
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("py_path", type=Path, help="Path to the .py schema file.")
+    parser.add_argument("class_name", help="BaseModel class to export.")
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=None,
+        help="Output .json path (defaults to alongside the input).",
+    )
+    args = parser.parse_args()
+
+    target = convert(args.py_path, args.class_name, args.out)
+    print(f"Wrote {target}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/ContentProcessor/pyproject.toml

@@ -25,6 +25,7 @@ dependencies = [
     "protobuf==6.33.6",
     "pyjwt==2.12.1",
     "pyasn1==0.6.3",
+    "jsonschema==4.25.1",
 ]
 
 

src/ContentProcessor/requirements.txt

@@ -16,6 +16,7 @@ dnspython==2.8.0
 idna==3.11
 iniconfig==2.3.0
 isodate==0.7.2
+jsonschema==4.25.1
 mongomock==4.3.0
 msal==1.34.0
 msal-extensions==1.3.1