Merge remote-tracking branch 'origin/main' into jleinas-biacore-insight-kinetics-affinity

# Conflicts:
#	src/allotropy/allotrope/schema_mappers/adm/binding_affinity_analyzer/benchling/_2024/_12/binding_affinity_analyzer.py
#	src/allotropy/parsers/cytiva_biacore_insight/cytiva_biacore_insight_data_creator.py

Author: james-leinas

.github/workflows/test.yml

@@ -37,13 +37,12 @@ jobs:
             - '.github/**'
           schema-generation:
             - 'src/allotropy/allotrope/schemas/**'
-            - 'src/allotropy/allotrope/schema_parser/**'
             - 'src/allotropy/allotrope/models/**'
             - 'src/allotropy/allotrope/schemas.py'
+            - 'src/allotropy/schema_gen/**'
             - 'src/allotropy/exceptions.py'
-            - 'scripts/generate_schemas.py'
             - 'scripts/download_schema.py'
-            - 'tests/allotrope/schema_parser/**'
+            - 'tests/schema_gen/**'
             - 'pyproject.toml'
     - name: Detect single parser changes
       id: detect-parser
@@ -109,8 +108,7 @@ jobs:
       run: |
         if [[ "${{ needs.detect-changes.outputs.run-all }}" == "true" ]]; then
           echo "Running all tests"
-          hatch run test_all.py3.10:pytest -n 2 tests --ignore=tests/allotrope/schema_parser/generate_schemas_test.py
-        else
+          hatch run test_all.py3.10:pytest -n 2 tests        else
           echo "Running tests for parser: ${{ needs.detect-changes.outputs.single-parser }}"
           hatch run test_all.py3.10:pytest -n 2 tests/parsers/${{ needs.detect-changes.outputs.single-parser }} tests/parser_factory_test.py::test_table_contents
         fi
@@ -137,8 +135,7 @@ jobs:
       run: |
         if [[ "${{ needs.detect-changes.outputs.run-all }}" == "true" ]]; then
           echo "Running all tests"
-          hatch run test_all.py3.11:pytest -n 2 tests --ignore=tests/allotrope/schema_parser/generate_schemas_test.py
-        else
+          hatch run test_all.py3.11:pytest -n 2 tests        else
           echo "Running tests for parser: ${{ needs.detect-changes.outputs.single-parser }}"
           hatch run test_all.py3.11:pytest -n 2 tests/parsers/${{ needs.detect-changes.outputs.single-parser }} tests/parser_factory_test.py::test_table_contents
         fi
@@ -164,8 +161,7 @@ jobs:
       run: |
         if [[ "${{ needs.detect-changes.outputs.run-all }}" == "true" ]]; then
           echo "Running all tests"
-          hatch run test_all.py3.12:pytest -n 2 tests --ignore=tests/allotrope/schema_parser/generate_schemas_test.py
-        else
+          hatch run test_all.py3.12:pytest -n 2 tests        else
           echo "Running tests for parser: ${{ needs.detect-changes.outputs.single-parser }}"
           hatch run test_all.py3.12:pytest -n 2 tests/parsers/${{ needs.detect-changes.outputs.single-parser }} tests/parser_factory_test.py::test_table_contents
         fi
@@ -194,7 +190,7 @@ jobs:
     - name: Install click
       run: pip install click!=8.3.0
     - name: Run Schema Generation Test
-      run: hatch run test_all.py${{ matrix.python-version }}:pytest tests/allotrope/schema_parser/generate_schemas_test.py -v
+      run: hatch run test_all.py${{ matrix.python-version }}:pytest tests/schema_gen/ -v
       timeout-minutes: 5
 
   lint:

CLAUDE.md

@@ -0,0 +1,74 @@
+# Allotropy
+
+Lab instrument data parser library. Converts vendor file formats to Allotrope Simple Model (ASM) JSON format using generated Python dataclass models.
+
+## Quick Reference
+
+```bash
+# Run tests
+hatch run test_all.py3.10:pytest tests/ -x -q
+
+# Run linting
+hatch run lint
+
+# Auto-fix lint
+hatch run lint:fmt
+
+# Generate models from schemas
+hatch run scripts:generate-schemas \
+  "http://purl.allotrope.org/json-schemas/adm/cell-counting/REC/2024/09/cell-counting.schema"
+```
+
+## Architecture Overview
+
+```
+src/allotropy/
+├── allotrope/
+│   ├── schemas/              # Cached JSON schemas (fetched from upstream or manually placed)
+│   ├── models/               # Python dataclasses representing ASM schema types
+│   │   ├── adm/              # Generated technique + core models (NEVER manually edit)
+│   │   ├── qudt/             # Generated unit models (NEVER manually edit)
+│   │   └── shared/           # Shared base types, enums, and components (manually maintained)
+│   ├── schema_mappers/      # Map intermediate Data objects → Model instances
+│   ├── schema_gen/          # Code generation pipeline (see schema_gen/CLAUDE.md for details)
+│   │   ├── generate.py      # Entry point / orchestrator
+│   │   ├── codegen/         # JSON schema → Python dataclass generator (package)
+│   │   ├── fetcher.py       # Schema fetching + caching + dependency resolution
+│   │   ├── naming.py        # URL ↔ Python identifier transformations
+│   │   └── serializer.py    # Dataclass ↔ JSON dict round-tripping
+│   └── converter.py         # Dispatches parser output through schema mapper to JSON
+├── parsers/                  # Vendor-specific file parsers (one package per instrument)
+└── types.py                  # Shared type aliases
+```
+
+## Schema Mapper Layer
+
+Schema mappers sit between parsers and generated models:
+
+```
+Parser (reads vendor file) → intermediate Data dataclass → Mapper.map_model() → Model instance
+```
+
+Each mapper defines:
+- Intermediate dataclasses (`Data`, `Metadata`, `Measurement`, `MeasurementGroup`, etc.)
+- A `Mapper` class extending `SchemaMapper[Data, Model]` with `map_model()` method
+- The `MANIFEST` URL string for the `field_asm_manifest` field
+
+## Field Naming Conventions
+
+Generated models use these naming patterns:
+
+| JSON Name | Python Field |
+|-----------|-------------|
+| `$asm.manifest` | `field_asm_manifest` |
+| `ASM file identifier` | `asm_file_identifier` |
+| `UNC path` | `unc_path` |
+| `pH` | `p_h` |
+| `pO2` | `p_o2` |
+| `pCO2` | `p_co2` |
+
+Quantity value types use abbreviated unit names: `TQuantityValueDegC`, `TQuantityValueMAU`, `TQuantityValueMmHg`, `TQuantityValueMicroLPermin`, `TQuantityValueM`, `TQuantityValueNM`, `TQuantityValueM1s1`, `TQuantityValueS1`, `TQuantityValueRU`, `TQuantityValueS`.
+
+## Code Rules
+
+- **No runtime imports.** All imports must be at module level. Never put `import` or `from ... import` inside a function body to work around circular dependencies — fix the dependency instead.

mypy.ini

@@ -8,3 +8,6 @@ disallow_untyped_decorators = False
 
 [mypy-deepdiff.*]
 ignore_missing_imports = True
+
+[mypy-allotropy.allotrope.models.*]
+disable_error_code = assignment, attr-defined

pyproject.toml

@@ -48,7 +48,6 @@ classifiers = [
 ]
 dependencies = [
   "babel >= 2.9.0",
-  "cattrs >= 23.2.0",
   "chardet >= 5.2.0",
   "defusedxml >= 0.7.1",
   # NOTE: jsonschema 4.18.0 introduces a serious performance regression, seemingly due to use of new
@@ -117,16 +116,11 @@ path = "src/allotropy/__about__.py"
 # 3.11. Use hatch run test_all to tests against 3.10-3.12
 # Note: if your hatch env is not picking up this setting, run hatch env prune.
 python = "3.11.9"
-# We define dependencies for linting and datamodel codegen in the default environment because we lint in
+# We define dependencies for linting and codegen in the default environment because we lint in
 # the generation script, and we test generation in tests, so we need them everywhere except the actual project.
 dependencies = [
   "autoflake == 2.2.0",
-  # black pinned for compatability with datamodel-code-generator
   "black == 22.3.0",
-  "datamodel-code-generator == 0.25.2",
-  # Without jsonschema-spec and pydantic, pydantic spews a bunch of errors when parsing schemas
-  "jsonschema-specifications == 2023.7.1",
-  "pydantic == 1.10.17",
   "ruff == 0.0.289",
   "semantic-version",
 ]
@@ -170,8 +164,6 @@ run = "scripts/visualization.py {args:}"
 
 [tool.hatch.envs.lint]
 extra-dependencies = [
-  # mypy sometimes can't find cattrs even though it's a project dependency, so add explicitly here.
-  "cattrs >= 23.2.0",
   "more-itertools >= 10.1.0",
   "mypy == 1.8.0",
   "pandas-stubs == 2.1.4.231227",
@@ -290,11 +282,15 @@ ban-relative-imports = "all"
   "N815",
 ]
 # Generated models can have class fields that shadow python builtins
-"src/allotropy/allotrope/models/**/*" = ["A003"]
+# PLC0414: "import X as X" used for explicit mypy re-exports of shared types
+"src/allotropy/allotrope/models/**/*" = ["A003", "N801", "PLC0414", "RUF001"]
 # Scripts can print
 "scripts/**/*" = ["T201"]
 # update_version can use subprocess
 "scripts/update_version.py" = ["S603", "S607"]
+# schema_gen is a script/CLI library — print() is the intended output mechanism
+# S603/S607: subprocess calls to ruff/black in _lint_file are safe known tools
+"src/allotropy/schema_gen/**/*" = ["T201", "S603", "S607"]
 
 [tool.ruff.pyupgrade]
 # Preserve types, even if a file imports `from __future__ import annotations`.

scripts/create_parser.py

@@ -3,13 +3,17 @@
 
 import click
 
-from allotropy.allotrope.schema_parser.path_util import (
+from allotropy.allotrope.path_util import (
     get_import_path_from_path,
     get_manifest_from_schema_path,
     get_model_path_from_schema_path,
     SCHEMA_DIR_PATH,
 )
-from allotropy.allotrope.schema_parser.schema_model import snake_to_upper_camel
+
+
+def snake_to_upper_camel(name: str) -> str:
+    return "".join(word.capitalize() for word in name.split("_"))
+
 
 ALLOTROPY_DIR = Path(Path(__file__).parent.parent, "src/allotropy")
 TEMPLATE_DIR = Path(Path(__file__).parent, "templates")

scripts/download_schema.py

@@ -1,15 +1,41 @@
 #!/usr/bin/env python3
 import click
 
-from allotropy.allotrope.schema_parser.reference_resolver import download_schema
+from allotropy.schema_gen.fetcher import SchemaFetcher
+from allotropy.schema_gen.technique_resolver import (
+    is_shorthand,
+    resolve_shorthand_to_urls,
+)
 
 
 @click.command()
-@click.argument("schema_url")
-def _download_schema(schema_url: str) -> None:
-    print(f"Downloading schema at {schema_url}...")
-    schema_path = download_schema(schema_url)
-    print(f"Downloaded to {schema_path}")
+@click.argument("schema_input")
+def _download_schema(schema_input: str) -> None:
+    """Download an Allotrope schema and its dependencies.
+
+    SCHEMA_INPUT can be a full purl URL or a shorthand like "plate-reader 2026/03".
+
+    \b
+    Examples:
+      plate-reader 2026/03
+      pcr WD/2025/06
+      http://purl.allotrope.org/json-schemas/adm/pcr/REC/2024/09/qpcr.schema
+    """
+    if is_shorthand(schema_input):
+        urls = resolve_shorthand_to_urls(schema_input)
+        click.echo(f"Resolved to {len(urls)} schema(s):")
+        for url in urls:
+            click.echo(f"  {url}")
+    else:
+        urls = [schema_input]
+
+    fetcher = SchemaFetcher()
+    total = 0
+    for url in urls:
+        click.echo(f"Downloading schema at {url}...")
+        schemas = fetcher.fetch_with_dependencies(url)
+        total += len(schemas)
+    click.echo(f"Downloaded {total} schema(s) to cache")
 
 
 if __name__ == "__main__":

scripts/generate_schemas.py

@@ -1,13 +1,57 @@
 #!/usr/bin/env python3
 import click
 
-from allotropy.allotrope.schema_parser.generate_schemas import generate_schemas
+from allotropy.schema_gen.generate import (
+    _discover_cached_technique_urls,
+    generate_models,
+)
+from allotropy.schema_gen.technique_resolver import (
+    is_shorthand,
+    resolve_shorthand_to_urls,
+)
 
 
 @click.command()
-@click.option("-r", "--regex", help="Regex to determine which schemas to generate.")
-def _generate_schemas(regex: str | None = None) -> None:
-    generate_schemas(schema_regex=regex)
+@click.argument("schema_urls", nargs=-1, required=False)
+@click.option(
+    "--all",
+    "regenerate_all",
+    is_flag=True,
+    help="Regenerate all cached technique schemas.",
+)
+def _generate_schemas(
+    schema_urls: tuple[str, ...], *, regenerate_all: bool = False
+) -> None:
+    """Generate Python models from one or more Allotrope schema URLs.
+
+    Each argument can be a full purl URL or a shorthand like "plate-reader 2026/03".
+    Use --all to regenerate every cached technique schema.
+
+    \b
+    Examples:
+      plate-reader 2026/03
+      pcr WD/2025/06
+      http://purl.allotrope.org/json-schemas/adm/pcr/REC/2024/09/qpcr.schema
+    """
+    urls: list[str]
+    if regenerate_all:
+        urls = _discover_cached_technique_urls()
+        click.echo(f"Discovered {len(urls)} cached technique schema(s)")
+    elif schema_urls:
+        urls = []
+        for arg in schema_urls:
+            if is_shorthand(arg):
+                resolved = resolve_shorthand_to_urls(arg)
+                click.echo(f"Resolved '{arg}' to {len(resolved)} schema(s):")
+                for u in resolved:
+                    click.echo(f"  {u}")
+                urls.extend(resolved)
+            else:
+                urls.append(arg)
+    else:
+        msg = "Provide schema URLs/shorthands or use --all."
+        raise click.UsageError(msg)
+    generate_models(urls)
 
 
 if __name__ == "__main__":

scripts/update_version.py

@@ -10,7 +10,7 @@
 import semantic_version  # type: ignore
 
 from allotropy.__about__ import __version__
-from allotropy.allotrope.schema_parser.path_util import ALLOTROPY_DIR, ROOT_DIR
+from allotropy.allotrope.path_util import ALLOTROPY_DIR, ROOT_DIR
 
 SECTION_TO_PREFIX = {
     "feat": "Added",

src/allotropy/allotrope/allotrope.py

@@ -9,7 +9,7 @@
 
 def serialize_and_validate_allotrope(model: Any) -> dict[str, Any]:
     try:
-        allotrope_dict = unstructure(model)
+        allotrope_dict: dict[str, Any] = unstructure(model)
     except Exception as e:
         msg = f"Failed to serialize allotrope model: {e}"
         raise AllotropeSerializationError(msg) from e