Merge pull request #226 from datakind/develop

Merge develop into staging

Author: vishpillai123

pyproject.toml

@@ -26,12 +26,12 @@ dependencies = [
     "pandas~=2.0",
     "six~=1.16.0",
     "thefuzz[speedup]~=0.22.1",
-    "databricks-sql-connector~=3.5.0",
+    "databricks-sql-connector[pyarrow]~=4.2.0",
     "pandera~=0.13",
     "mlflow~=2.22",
     "cachetools",
     "types-cachetools",
-    "edvise~=0.1.12",
+    "edvise~=0.2.0",
 ]
 
 [project.urls]

src/webapp/database.py

@@ -218,6 +218,10 @@ class InstTable(Base):
     edvise_id: Mapped[str | None] = mapped_column(
         String(VAR_CHAR_LENGTH), nullable=True
     )
+    # Only populated for Legacy schools (any-format uploads).
+    legacy_id: Mapped[str | None] = mapped_column(
+        String(VAR_CHAR_LENGTH), nullable=True
+    )
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True),
         server_default=func.now(),

src/webapp/fixtures/validation_error_snapshots/mixed_types_multiple_columns.txt

@@ -8,4 +8,4 @@ Column 'Score' has validation errors:
   • Row 3: Validation failed for greater_than(0) check. Current value: found '-5'
 
 Column 'Student ID' has validation errors:
-  • Row 1: Value must be at least 3 characters long. Current value: found '****' (value masked for privacy)
+  • Row 1: Value must be at least 3 characters long. Current value: found 'AB'

src/webapp/gcsutil.py

@@ -377,8 +377,8 @@ def validate_file(
             allowed_schemas: List of schema/model names allowed.
             base_schema: Base schema dict.
             inst_schema: Optional extension schema with institutions.* blocks.
-            institution_id: Key into inst_schema["institutions"]: "edvise", "pdp", or
-                institution UUID for custom. Default "pdp" for backward compatibility.
+            institution_id: Key into inst_schema["institutions"]: "edvise", "pdp",
+                "legacy" (any-format uploads), or institution UUID for custom. Default "pdp".
             institution_identifier: Optional institution ID (e.g. UUID). Reserved for
                 future use; Edvise uses JSON-based validation only (different shape).