Merge pull request #214 from datakind/develop

Merge develop into staging

Author: chapmanhk

.devcontainer/devcontainer.json

@@ -15,11 +15,11 @@
         "vscode": {
             "extensions": [
                 "hashicorp.terraform",
-                "ms-python.black-formatter"
+                "charliermarsh.ruff"
             ],
             "settings": {
                 "[python]": {
-                    "editor.defaultFormatter": "ms-python.black-formatter",
+                    "editor.defaultFormatter": "charliermarsh.ruff",
                     "editor.formatOnSave": true
                   }
             }

.gitignore

@@ -114,3 +114,6 @@ dmypy.json
 # terraform
 **/.terraform/*
 **/terraform.tfvars
+
+# Cursor rule files
+.cursor/

.vscode/launch.json

@@ -29,6 +29,21 @@
       "env": {
         "ENV_FILE_PATH": "${workspaceFolder}/src/worker/.env"
       }
+    },
+    {
+      "name": "pytest (current file)",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "args": [
+        "${file}",
+        "-v",
+        "-s"
+      ],
+      "cwd": "${workspaceFolder}",
+      "env": {
+        "ENV_FILE_PATH": "${workspaceFolder}/src/webapp/.env"
+      }
     }
   ],
 }

CONTRIBUTING.md

@@ -10,8 +10,8 @@ To get an overview of the project, please read the [README](README.md) and our [
 ## Getting started
 ### Creating Issues
 
-If you spot a problem, [search if an issue already exists](https://github.com/datakind/sst-app-api/issues). If a related issue doesn't exist,
-you can open a new issue using a relevant [issue form](https://github.com/datakind/sst-app-api/issues/new).
+If you spot a problem, [search if an issue already exists](https://github.com/datakind/edvise-api/issues). If a related issue doesn't exist,
+you can open a new issue using a relevant [issue form](https://github.com/datakind/edvise-api/issues/new).
 
 As a general rule, we don’t assign issues to anyone. If you find an issue to work on, you are welcome to open a PR with a fix.
 
@@ -28,7 +28,7 @@ poetry install --no-interaction
 As many other open source projects, we use the famous [gitflow](https://nvie.com/posts/a-successful-git-branching-model/) to manage our branches.
 
 Summary of our git branching model:
-- Get all the latest work from the upstream `datakind/sst-app-api` repository
+- Get all the latest work from the upstream `datakind/edvise-api` repository
   (`git checkout main`)
 - Create a new branch off with a descriptive name (for example:
   `feature/new-test-macro`, `bugfix/bug-when-uploading-results`). You can
@@ -107,7 +107,7 @@ You can type `pytest` to run your tests, no matter which type of test it is.
 
 ## Continuous Integration
 
-We use [GitHub Actions](https://github.com/datakind/sst-app-api/actions)
+We use [GitHub Actions](https://github.com/datakind/edvise-api/actions)
 for continuous integration.
 See [here](https://docs.github.com/en/actions) for GitHub's documentation.
 

README.md

@@ -2,14 +2,25 @@
 
 This repo contains:
 
-* [src/webapp/](https://github.com/datakind/sst-app-api/tree/develop/src/webapp): The source code for the SST API (which is called by the SST frontend and by any direct API callers)
-* [src/worker/](https://github.com/datakind/sst-app-api/tree/develop/src/worker): The source code for the SFTP Worker (which calls the SST API)
+* [src/webapp/](https://github.com/datakind/edvise-api/tree/develop/src/webapp): The source code for the SST API (which is called by the SST frontend and by any direct API callers)
+* [src/worker/](https://github.com/datakind/edvise-api/tree/develop/src/worker): The source code for the SFTP Worker (which calls the SST API)
 * [terraform/]
-(https://github.com/datakind/sst-app-api/tree/develop/terraform): The Terraform configuration for the SST API/Frontend and other GCP resources including Cloud SQL setup, networking setup, secrets setup
+(https://github.com/datakind/edvise-api/tree/develop/terraform): The Terraform configuration for the SST API/Frontend and other GCP resources including Cloud SQL setup, networking setup, secrets setup
 * .devcontainer/ and .vscode/: which allow easy setup if you are using VSCode as your IDE.
-* [devtools/](https://github.com/datakind/sst-app-api/tree/develop/devtools): is a place to put utility scripts
-* .github/: contains mostly copied over files when this directory was forked from the student-success-tool repo, so likely much of it is outdated. The only Github action we've added is the [webapp-and-worker-precommit](https://github.com/datakind/sst-app-api/blob/develop/.github/workflows/webapp-and-worker-precommit.yml) which is run on every push to develop. This action contains a python linter (we use [black](https://black.readthedocs.io/en/stable/)), and automated runs of the unit tests in the src/webapp/ and src/worker/ directories.
-* Additionally, [pyproject.toml](https://github.com/datakind/sst-app-api/blob/develop/pyproject.toml) and [uv.lock](https://github.com/datakind/sst-app-api/blob/develop/uv.lock) are important for dependency management. At time of writing, the worker is just skeleton code so there's no separate dependency management. In the long-term consider separating out the dependency management for the two programs. 
+* [devtools/](https://github.com/datakind/edvise-api/tree/develop/devtools): is a place to put utility scripts
+* .github/: contains mostly copied over files when this directory was forked from the student-success-tool repo, so likely much of it is outdated. The only Github action we've added is the [webapp-and-worker-precommit](https://github.com/datakind/edvise-api/blob/develop/.github/workflows/webapp-and-worker-precommit.yml) which is run on every push to develop. This action contains a python linter (we use [black](https://black.readthedocs.io/en/stable/)), and automated runs of the unit tests in the src/webapp/ and src/worker/ directories.
+* Additionally, [pyproject.toml](https://github.com/datakind/edvise-api/blob/develop/pyproject.toml) and [uv.lock](https://github.com/datakind/edvise-api/blob/develop/uv.lock) are important for dependency management. At time of writing, the worker is just skeleton code so there's no separate dependency management. In the long-term consider separating out the dependency management for the two programs. 
 
 
 NOTE: this repo was forked from the https://github.com/datakind/student-success-tool repo, which means some of the static files (e.g. CONTRIBUTING.md) may be outdated or may include irrelevant information from that repo. Please update those as you see fit. For information about the specific items listed above, defer to the specific readmes in the relevant directory.
+
+## Local edvise development override
+
+Production uses a pinned Git reference for `edvise`. For local development, use an
+editable install after syncing the environment.
+
+1. Clone `edvise` alongside `edvise-api` (so `../edvise` exists).
+2. Run `uv sync`.
+3. Override locally: `uv pip install -e ../edvise`
+
+To revert back to the pinned Git dependency, run `uv sync --reinstall-package edvise`.

SECURITY.md

@@ -8,4 +8,4 @@ If we verify a reported security vulnerability, our policy is:
 
 ## Reporting a Security Issue
 
-To report any security issues, please [raise an issue](https://github.com/datakind/sst-app-api/issues/new/choose) and select **Security issue*
+To report any security issues, please [raise an issue](https://github.com/datakind/edvise-api/issues/new/choose) and select **Security issue*

cloudbuild-webapp.yaml

@@ -0,0 +1,55 @@
+# Cloud Build config for webapp (dev-webapp trigger).
+# _REGION and _ENVIRONMENT are set by the trigger (Terraform).
+steps:
+  - name: ghcr.io/astral-sh/uv:debian
+    entrypoint: bash
+    args:
+      - -c
+      - |
+        set -e
+        apt-get update && apt-get install -y --no-install-recommends git
+        uv lock --upgrade-package edvise
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - build
+      - '-f'
+      - src/webapp/Dockerfile
+      - '-t'
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/webapp:$COMMIT_SHA'
+      - '-t'
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/webapp:latest'
+      - .
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - push
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/webapp:$COMMIT_SHA'
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - push
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/webapp:latest'
+  - name: gcr.io/cloud-builders/gcloud
+    args:
+      - run
+      - deploy
+      - '${_ENVIRONMENT}-webapp'
+      - '--image'
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/webapp:$COMMIT_SHA'
+      - '--region'
+      - '${_REGION}'
+  - name: curlimages/curl
+    args:
+      - '-X'
+      - POST
+      - '-H'
+      - 'Content-Type: application/json'
+      - '-f'
+      - '-d'
+      - >-
+        {"text":"🚀 *$REPO_NAME* deployed · `$BRANCH_NAME` · $TRIGGER_NAME · $BUILD_ID"}
+      - >-
+        https://hooks.slack.com/triggers/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
+    id: notify-slack
+timeout: 600s
+options:
+  logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true

cloudbuild-worker.yaml

@@ -0,0 +1,34 @@
+# Cloud Build config for worker (dev-worker trigger).
+# _REGION and _ENVIRONMENT are set by the trigger (Terraform).
+steps:
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - build
+      - '-f'
+      - src/worker/Dockerfile
+      - '-t'
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/worker:$COMMIT_SHA'
+      - '-t'
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/worker:latest'
+      - .
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - push
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/worker:$COMMIT_SHA'
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - push
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/worker:latest'
+  - name: gcr.io/cloud-builders/gcloud
+    args:
+      - run
+      - deploy
+      - '${_ENVIRONMENT}-worker'
+      - '--image'
+      - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/edvise-api/worker:$COMMIT_SHA'
+      - '--region'
+      - '${_REGION}'
+timeout: 600s
+options:
+  logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true

pyproject.toml

@@ -8,7 +8,7 @@ dependencies = [
     "databricks-sdk~=0.38.0",
     "pydantic~=2.10",
     "fastapi[standard]~=0.115.4",
-    "google-cloud-storage~=2.18.2",
+    "google-cloud-storage==2.19.0",
     "paramiko~=3.5.0",
     "cloud-sql-python-connector[pymysql]~=1.14.0",
     "sqlalchemy~=2.0.36",
@@ -28,13 +28,14 @@ dependencies = [
     "thefuzz[speedup]~=0.22.1",
     "databricks-sql-connector~=3.5.0",
     "pandera~=0.13",
-    "mlflow~=2.15.0",
+    "mlflow~=2.22",
     "cachetools",
     "types-cachetools",
+    "edvise~=0.1.12",
 ]
 
 [project.urls]
-Repository = "https://github.com/datakind/sst-app-api"
+Repository = "https://github.com/datakind/edvise-api"
 
 [dependency-groups]
 dev = [
@@ -52,9 +53,15 @@ dev = [
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.uv]
 default-groups = ["dev"]
 
+[tool.uv.sources]
+edvise = { git = "https://github.com/datakind/edvise.git", rev = "develop" }
+
 [tool.ruff]
 line-length = 88
 indent-width = 4
@@ -86,8 +93,11 @@ lines-after-imports = 1
 [tool.pytest.ini_options]
 minversion = "8.0"
 addopts = ["--verbose", "--import-mode=importlib"]
-filterwarnings = ["ignore::DeprecationWarning"]
-testpaths = ["tests"]
+filterwarnings = [
+    "ignore::DeprecationWarning",
+    "ignore::FutureWarning:pandera",
+]
+testpaths = ["src"]
 
 [tool.mypy]
 files = ["src"]

src/webapp/Dockerfile

@@ -13,6 +13,11 @@ WORKDIR /app
 # Add project files
 ADD uv.lock pyproject.toml /app/
 
+# Install git and ca-certificates
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends git ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
 # Install dependencies
 RUN uv sync --frozen --no-install-project