Update default values in data models to use PydanticUndefined and adjust validation tests for required fields

Author: Zalfsten

middleware/sql_to_arc/src/middleware/sql_to_arc/models.py

@@ -8,6 +8,7 @@
 from typing import Any, get_args, get_origin
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
+from pydantic_core import PydanticUndefined
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +17,7 @@ def spec_field(
     *,
     required: bool | None = None,
     allow_spec_override: bool = False,
-    default: Any = None,
+    default: Any = PydanticUndefined,
     **kwargs: Any,
 ) -> Any:
     """Define database-mapped fields with ARC spec metadata."""
@@ -66,7 +67,8 @@ def _field_metadata(field_info: Any) -> dict[str, Any]:
     @staticmethod
     def _field_default(field_info: Any) -> Any:
         """Return default value for a field, including default_factory values."""
-        return field_info.get_default(call_default_factory=True)
+        val = field_info.get_default(call_default_factory=True)
+        return None if val is PydanticUndefined else val
 
     @staticmethod
     def _field_accepts_string(annotation: Any) -> bool:
@@ -252,8 +254,8 @@ class InvestigationRow(BaseRow):
     identifier: str = spec_field()
     title: str = spec_field()
     description_text: str = spec_field(default="", allow_spec_override=True)
-    submission_date: datetime | None = spec_field()
-    public_release_date: datetime | None = spec_field()
+    submission_date: datetime | None = spec_field(default=None)
+    public_release_date: datetime | None = spec_field(default=None)
 
 
 class StudyRow(BaseRow):
@@ -262,55 +264,55 @@ class StudyRow(BaseRow):
     identifier: str = spec_field()
     investigation_ref: str = spec_field()
     title: str = spec_field()
-    description_text: str | None = spec_field()
-    submission_date: datetime | None = spec_field()
-    public_release_date: datetime | None = spec_field()
+    description_text: str | None = spec_field(default=None)
+    submission_date: datetime | None = spec_field(default=None)
+    public_release_date: datetime | None = spec_field(default=None)
 
 
 class AssayRow(BaseRow):
     """Pydantic model for assay database rows."""
 
     identifier: str = spec_field()
     investigation_ref: str = spec_field()
-    study_ref: str | None = spec_field()
-    title: str | None = spec_field()
-    description_text: str | None = spec_field()
-    measurement_type_term: str | None = spec_field()
-    measurement_type_uri: str | None = spec_field()
-    measurement_type_version: str | None = spec_field()
-    technology_type_term: str | None = spec_field()
-    technology_type_uri: str | None = spec_field()
-    technology_type_version: str | None = spec_field()
-    technology_platform: str | None = spec_field()
+    study_ref: str | None = spec_field(default=None)
+    title: str | None = spec_field(default=None)
+    description_text: str | None = spec_field(default=None)
+    measurement_type_term: str | None = spec_field(default=None)
+    measurement_type_uri: str | None = spec_field(default=None)
+    measurement_type_version: str | None = spec_field(default=None)
+    technology_type_term: str | None = spec_field(default=None)
+    technology_type_uri: str | None = spec_field(default=None)
+    technology_type_version: str | None = spec_field(default=None)
+    technology_platform: str | None = spec_field(default=None)
 
 
 class PublicationRow(BaseRow):
     """Pydantic model for publication database rows."""
 
     investigation_ref: str = spec_field()
     target_type: str = spec_field()
-    pubmed_id: str | None = spec_field()
-    doi: str | None = spec_field()
-    authors: str | None = spec_field()
-    title: str | None = spec_field()
-    status_term: str | None = spec_field()
-    status_uri: str | None = spec_field()
-    status_version: str | None = spec_field()
-    target_ref: str | None = spec_field()
+    pubmed_id: str | None = spec_field(default=None)
+    doi: str | None = spec_field(default=None)
+    authors: str | None = spec_field(default=None)
+    title: str | None = spec_field(default=None)
+    status_term: str | None = spec_field(default=None)
+    status_uri: str | None = spec_field(default=None)
+    status_version: str | None = spec_field(default=None)
+    target_ref: str | None = spec_field(default=None)
 
 
 class ContactRow(BaseRow):
     """Pydantic model for contact database rows."""
 
     investigation_ref: str = spec_field()
     target_type: str = spec_field()
-    last_name: str | None = spec_field()
-    first_name: str | None = spec_field()
-    mid_initials: str | None = spec_field()
-    email: str | None = spec_field()
-    phone: str | None = spec_field()
-    fax: str | None = spec_field()
-    postal_address: str | None = spec_field()
-    affiliation: str | None = spec_field()
-    roles: str | None = spec_field()  # JSON string
-    target_ref: str | None = spec_field()
+    last_name: str | None = spec_field(default=None)
+    first_name: str | None = spec_field(default=None)
+    mid_initials: str | None = spec_field(default=None)
+    email: str | None = spec_field(default=None)
+    phone: str | None = spec_field(default=None)
+    fax: str | None = spec_field(default=None)
+    postal_address: str | None = spec_field(default=None)
+    affiliation: str | None = spec_field(default=None)
+    roles: str | None = spec_field(default=None)  # JSON string
+    target_ref: str | None = spec_field(default=None)

middleware/sql_to_arc/tests/unit/test_database_validation.py

@@ -50,14 +50,14 @@ async def test_validate_row_missing_optional_warns(caplog: pytest.LogCaptureFixt
 @pytest.mark.asyncio
 async def test_validate_row_description_text_exception() -> None:
     """Test that missing required columns raise Exception."""
-    # description_text is required and must exist.
-    row = {"identifier": "1", "title": "Test"}
+    # title is required and must exist.
+    row = {"identifier": "1", "description_text": "Desc"}
 
     with pytest.raises(Exception) as excinfo:
         InvestigationRow.model_validate(row)
 
     assert "Missing required columns" in str(excinfo.value)
-    assert "description_text" in str(excinfo.value)
+    assert "title" in str(excinfo.value)
 
 
 @pytest.mark.asyncio

middleware/sql_to_arc/tests/unit/test_main.py

@@ -89,14 +89,18 @@ async def test_process_investigations_flow(monkeypatch: pytest.MonkeyPatch) -> N
     mock_db = MagicMock()
 
     # Mock DB stream methods
-    async def mock_gen(data: list[Any]) -> AsyncGenerator[Any, None]:
+    async def mock_gen(**kwargs: Any) -> AsyncGenerator[Any, None]:
+        stats = kwargs.get("stats")
+        data = [
+            InvestigationRow(identifier="1", title="T1", description_text="D1"),
+            InvestigationRow(identifier="2", title="T2", description_text="D2"),
+        ]
+        if stats:
+            stats.found_datasets += len(data)
         for item in data:
             yield item
 
-    mock_db.stream_investigations.side_effect = lambda **_: mock_gen([
-        InvestigationRow(identifier="1", title="T1", description_text="D1"),
-        InvestigationRow(identifier="2", title="T2", description_text="D2"),
-    ])
+    mock_db.stream_investigations.side_effect = mock_gen
 
     # Mock related data fetch
     async def mock_fetch_related(*_args: Any, **_kwargs: Any) -> RelatedDataBatch:
@@ -133,4 +137,4 @@ async def mock_process_inv(*args: Any, **kwargs: Any) -> None:
 
     assert stats.found_datasets == 2  # noqa: PLR2004
     assert stats.total_studies == 1
-    mock_db.stream_investigations.assert_called_with(limit=10)
+    mock_db.stream_investigations.assert_called_with(stats=stats, limit=10)