Merge pull request #84 from datakind/Validation-Errors

adjusted validation files

Author: Mesh-ach

src/webapp/routers/data.py

@@ -886,8 +886,8 @@ def validation_helper(
     if not allowed_schemas:
         allowed_schemas = infer_models_from_filename(file_name, "pdp")
 
-    inferred_schemas = set()
-    # TODO:
+    inferred_schemas: list[str] = []
+
     try:
         inferred_schemas = storage_control.validate_file(
             get_external_bucket_name(inst_id),
@@ -914,7 +914,7 @@ def validation_helper(
         local_session.get().add(new_file_record)
     except Exception as e:
         logging.error(f"Error message: {str(e)}")
-
+    print(f"Inferred Schemas {list(inferred_schemas)}")
     return {
         "name": file_name,
         "inst_id": inst_id,
@@ -923,7 +923,9 @@ def validation_helper(
     }
 
 
-@router.post("/{inst_id}/input/validate-sftp/{file_name:path}")
+@router.post(
+    "/{inst_id}/input/validate-sftp/{file_name:path}", response_model=ValidationResult
+)
 def validate_file_sftp(
     inst_id: str,
     file_name: str,

src/webapp/validation.py

@@ -174,11 +174,11 @@ def validate_dataset(
 
     # 2) merge requested models
     if models is None:
-        model_list: List[str] = []
+        model_list = []
     elif isinstance(models, str):
         model_list = [models]
     else:
-        model_list = models
+        model_list = list(models)  # <- ensures it's not a set
 
     merged_specs: Dict[str, dict] = {}
     for m in model_list:

src/webapp/validation_test.py

@@ -0,0 +1,69 @@
+import pytest
+import pandas as pd
+from pathlib import Path
+from unittest.mock import patch
+from src.webapp.validation import validate_file_reader, HardValidationError
+
+# Minimal schema for testing
+MOCK_BASE_SCHEMA = {
+    "base": {
+        "data_models": {
+            "test_model": {
+                "columns": {
+                    "foo_col": {
+                        "dtype": "int",
+                        "nullable": False,
+                        "required": True,
+                        "aliases": ["foo"],
+                    },
+                    "bar_col": {
+                        "dtype": "str",
+                        "nullable": True,
+                        "required": False,
+                        "aliases": ["bar"],
+                    },
+                }
+            }
+        }
+    }
+}
+
+MOCK_EXT_SCHEMA = {"institutions": {"pdp": {"data_models": {}}}}
+
+
+@pytest.fixture
+def tmp_csv_file(tmp_path: Path):
+    df = pd.DataFrame({"foo_col": [1, 2], "bar_col": ["a", "b"]})
+    file_path = tmp_path / "test.csv"
+    df.to_csv(file_path, index=False)
+    return str(file_path)
+
+
+def test_validate_file_reader_passes(tmp_csv_file):
+    with (
+        patch("src.webapp.validation.load_json") as mock_load,
+        patch("os.path.exists", return_value=True),
+    ):
+        mock_load.side_effect = (
+            lambda path: MOCK_BASE_SCHEMA if "base" in path else MOCK_EXT_SCHEMA
+        )
+        result = validate_file_reader(tmp_csv_file, ["test_model"])
+        assert result["validation_status"] == "passed"
+        assert result["schemas"] == ["test_model"]
+
+
+def test_validate_file_reader_fails_missing_required(tmp_path):
+    df = pd.DataFrame({"bar_col": ["x", "y"]})  # Missing "foo_col"
+    file_path = tmp_path / "invalid.csv"
+    df.to_csv(file_path, index=False)
+
+    with (
+        patch("src.webapp.validation.load_json") as mock_load,
+        patch("os.path.exists", return_value=True),
+    ):
+        mock_load.side_effect = (
+            lambda path: MOCK_BASE_SCHEMA if "base" in path else MOCK_EXT_SCHEMA
+        )
+        with pytest.raises(HardValidationError) as exc_info:
+            validate_file_reader(str(file_path), ["test_model"])
+        assert "Missing required columns" in str(exc_info.value)