Merge pull request #229 from datakind/feat/local_inst_testing

feat: Optional local institution seed from `config/local_inst_data.json`

Author: mrmaloof

.gitignore

@@ -97,6 +97,9 @@ ENV/
 env.bak/
 venv.bak/
 
+# Local config / dev data
+config/local_inst_data.json
+
 # mkdocs documentation
 /site
 

README.md

@@ -13,14 +13,3 @@ This repo contains:
 
 
 NOTE: this repo was forked from the https://github.com/datakind/student-success-tool repo, which means some of the static files (e.g. CONTRIBUTING.md) may be outdated or may include irrelevant information from that repo. Please update those as you see fit. For information about the specific items listed above, defer to the specific readmes in the relevant directory.
-
-## Local edvise development override
-
-Production uses a pinned Git reference for `edvise`. For local development, use an
-editable install after syncing the environment.
-
-1. Clone `edvise` alongside `edvise-api` (so `../edvise` exists).
-2. Run `uv sync`.
-3. Override locally: `uv pip install -e ../edvise`
-
-To revert back to the pinned Git dependency, run `uv sync --reinstall-package edvise`.

config/local_inst_data.example.json

@@ -0,0 +1,60 @@
+[
+  {
+    "inst_id": "inst-uuid-here",
+    "name": "Example institution",
+    "state": "XX",
+    "retention_days": null,
+    "pdp_id": "",
+    "edvise_id": null,
+    "batches": [
+      {
+        "batch_id": "batch-uuid-here",
+        "inst_id": "inst-uuid-here",
+        "file_names_to_ids": {
+          "example_course.csv": "file-id-course",
+          "example_student.csv": "file-id-student"
+        },
+        "name": "example_batch_1",
+        "created_by": "uploader-uuid-here",
+        "deleted": false,
+        "completed": true,
+        "deletion_request_time": null,
+        "created_at": "2025-01-15T12:00:00",
+        "updated_at": "2025-01-15T12:00:00",
+        "updated_by": ""
+      }
+    ],
+    "files": [
+      {
+        "name": "example_course.csv",
+        "data_id": "file-id-course",
+        "batch_ids": ["batch-uuid-here"],
+        "inst_id": "inst-uuid-here",
+        "uploader": "uploader-uuid-here",
+        "source": "MANUAL_UPLOAD",
+        "schemas": ["COURSE"],
+        "deleted": false,
+        "deletion_request_time": null,
+        "retention_days": null,
+        "sst_generated": false,
+        "valid": true,
+        "uploaded_date": "2025-01-15T11:58:00"
+      },
+      {
+        "name": "example_student.csv",
+        "data_id": "file-id-student",
+        "batch_ids": ["batch-uuid-here"],
+        "inst_id": "inst-uuid-here",
+        "uploader": "uploader-uuid-here",
+        "source": "MANUAL_UPLOAD",
+        "schemas": ["STUDENT"],
+        "deleted": false,
+        "deletion_request_time": null,
+        "retention_days": null,
+        "sst_generated": false,
+        "valid": true,
+        "uploaded_date": "2025-01-15T11:57:00"
+      }
+    ]
+  }
+]

src/webapp/README.md

@@ -168,3 +168,25 @@ The process to upload a file involves three API calls:
 ## Local VSCode Debugging
 
 From the Run & Debug panel (⇧⌘D on 🍎) you can run the [debug launch config](../../.vscode/launch.json) for the webapp or worker modules. This will allow you to set breakpoints within the source code while the applications are running.
+
+## Local edvise development override
+
+Production uses a pinned Git reference for `edvise`. For local development, use an
+editable install after syncing the environment.
+
+1. Clone `edvise` alongside `edvise-api` (so `../edvise` exists).
+2. Run `uv sync`.
+3. Override locally: `uv pip install -e ../edvise`
+
+To revert back to the pinned Git dependency, run `uv sync --reinstall-package edvise`.
+
+## Local institutions (optional)
+
+You can seed the local database with institution, batch, and file metadata that matches dev or staging (names, UUIDs, batch membership) without checking secrets into Git.
+
+1. Copy `config/local_inst_data.example.json` to `config/local_inst_data.json`. The latter is gitignored.
+2. Edit `local_inst_data.json` to match your needs. Use the example file as the schema: one array element per institution, with `inst_id`, `name`, and optionally `state`, `pdp_id`, `batches`, and `files`.
+
+If the file is missing, startup skips this step and the default local seed in code still applies.
+
+**Limitation:** Endpoints that read uploaded CSV (for example EDA) load blobs from GCS under the bucket name `dev_<institution_uuid_hex>`, not from this JSON. To exercise those flows locally you still need GCP credentials and the corresponding objects in that bucket, or you rely on tests/mocks instead.

src/webapp/database.py

@@ -1,7 +1,9 @@
 """Database configuration."""
 
+import json
 import uuid
 import datetime
+from pathlib import Path
 from typing import Set, List, Any
 from contextvars import ContextVar
 import enum
@@ -61,6 +63,49 @@ class Base(DeclarativeBase):
 DATETIME_TESTING = datetime.datetime(2024, 12, 26, 19, 37, 59, 753357)
 
 
+def _setup_test_institutions(session: Session) -> None:
+    """Load optional local institution display data from config/local_inst_data.json (gitignored)."""
+    file = Path("config/local_inst_data.json")
+    if file.exists():
+        with open(file) as f:
+            for inst in json.load(f):
+                session.merge(
+                    InstTable(
+                        id=uuid.UUID(inst["inst_id"]),
+                        name=inst["name"],
+                        state=inst.get("state"),
+                        pdp_id=inst.get("pdp_id"),
+                        created_at=DATETIME_TESTING,
+                        updated_at=DATETIME_TESTING,
+                        created_by=LOCAL_USER_UUID,
+                    )
+                )
+                schemas_by_file_id = {
+                    f["data_id"]: f.get("schemas", []) for f in inst.get("files", [])
+                }
+                for batch in inst.get("batches", []):
+                    batch_table = BatchTable(
+                        id=uuid.UUID(batch["batch_id"]),
+                        inst_id=uuid.UUID(inst["inst_id"]),
+                        name=batch["name"],
+                        created_at=DATETIME_TESTING,
+                        updated_at=DATETIME_TESTING,
+                        created_by=LOCAL_USER_UUID,
+                    )
+                    for file_name, file_id in batch["file_names_to_ids"].items():
+                        batch_table.files.add(
+                            session.merge(
+                                FileTable(
+                                    id=uuid.UUID(file_id),
+                                    inst_id=uuid.UUID(inst["inst_id"]),
+                                    name=file_name,
+                                    schemas=schemas_by_file_id.get(file_id, []),
+                                )
+                            )  # type: ignore
+                        )
+                    session.merge(batch_table)
+
+
 @event.listens_for(Mapper, "before_insert")
 @event.listens_for(Mapper, "before_update")
 def validate_string_lengths(mapper, connection, target):
@@ -121,6 +166,7 @@ def init_db(env: str) -> None:
             )
             # Create test files and batches for LOCAL environment
             if env == "LOCAL":
+                _setup_test_institutions(session)
                 # Create test files
                 test_file_1 = FileTable(
                     id=uuid.UUID("f0bb3a20-6d92-4254-afed-6a72f43c562a"),