refactor and add automated tests: reorganize nfgda_service directory structure by migrating algorithm components and cleaning up legacy files.

Author: Eoghan-Perkins

.github/workflows/integration-tests.yml

@@ -0,0 +1,48 @@
+name: Integration Tests
+
+on:
+  pull_request:
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install test dependencies
+        run: pip install pytest requests
+
+      - name: Start Docker Compose stack
+        run: docker compose up -d --build
+
+      - name: Wait for backend to be ready
+        run: |
+          echo "Waiting for backend..."
+          for i in $(seq 1 30); do
+            if curl -sf http://localhost:8001/apis/stations > /dev/null; then
+              echo "Backend is up"
+              exit 0
+            fi
+            echo "  attempt $i/30..."
+            sleep 5
+          done
+          echo "Backend did not become ready in time"
+          docker compose logs backend
+          exit 1
+
+      - name: Run integration tests (non-slow)
+        run: pytest test_endpoints.py -v -m "not slow"
+
+      - name: Dump logs on failure
+        if: failure()
+        run: docker compose logs
+
+      - name: Tear down stack
+        if: always()
+        run: docker compose down

backend/apis/retrieve_frames.py

@@ -1,7 +1,23 @@
 """
-Frame Data API: returns the list of rendered frames for a completed job.
+Frame Data API: returns a single rendered GeoTIFF frame for a completed job.
 """
 
-def get_frames(job_id: str):
-    
-    pass
+import os
+from flask import send_file, abort
+
+
+def get_frame(job_id: str, index: int):
+    """Return a single GeoTIFF frame file for the given job and frame index."""
+    job_dir = "/processed_data/" + job_id
+    if not os.path.exists(job_dir):
+        abort(404, description="Job not found")
+
+    frame_path = job_dir + f"/frame_{index}.tif"
+    if not os.path.exists(frame_path):
+        abort(404, description="Frame not found")
+
+    return send_file(
+        frame_path,
+        mimetype="image/tiff",
+        as_attachment=False
+    )

backend/apis/run_request.py

@@ -68,13 +68,14 @@ def send_job_to_redis_queue(redis_client, request_fields: dict):
 def validate_time_parameters(request_fields: dict):
     """Validate the time parameters recieved via the request."""
 
-    # Default timebox when not provided: look back 15 minutes from now
-    # so the algorithm captures 2-3 recent NEXRAD scans for detection + forecast
-    # (2 scan minimum needed for forcasting)
+    # Default timebox when not provided: look back over the last ~25 minutes, ending
+    # 10 minutes ago. The 10-minute buffer ensures the algorithm's end time is always
+    # fully in the past — if endUtc is too close to "now" the algorithm enters live
+    # polling mode and runs indefinitely.
     now = datetime.now(timezone.utc)
     if not request_fields.get("startUtc") and not request_fields.get("endUtc"):
-        request_fields["startUtc"] = (now - timedelta(minutes=15)).strftime("%Y-%m-%dT%H:%M:%SZ")
-        request_fields["endUtc"] = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+        request_fields["startUtc"] = (now - timedelta(minutes=35)).strftime("%Y-%m-%dT%H:%M:%SZ")
+        request_fields["endUtc"] = (now - timedelta(minutes=10)).strftime("%Y-%m-%dT%H:%M:%SZ")
     elif not request_fields.get("startUtc") or not request_fields.get("endUtc"):
         return jsonify({"error": "Must provide both startUtc and endUtc, or neither"})
 
@@ -102,8 +103,10 @@ def validate_time_parameters(request_fields: dict):
     if duration > max_duration:
         return jsonify({"error": f"Timebox duration must not exceed {max_hours:.0f} hours"})
 
-    # endUtc must not be in the future
-    if end_utc > now:
-        return jsonify({"error": "endUtc must not be later than the current time"})
+    # endUtc must be at least 5 minutes in the past — the algorithm enters a live
+    # polling loop if endUtc is too close to the current time, causing jobs to run
+    # indefinitely instead of processing a closed historical window.
+    if end_utc > now - timedelta(minutes=5):
+        return jsonify({"error": "endUtc must be at least 5 minutes in the past"})
 
     return None

backend/app.py

@@ -1,10 +1,9 @@
-import os
 import redis
-from flask import Flask, jsonify, request, send_file, abort
+from flask import Flask, jsonify, request
 from apis.stations import list_stations_api
 from apis.run_request import send_job_to_redis_queue
 from apis.status import get_job_status
-from apis.retrieve_frames import get_frames
+from apis.retrieve_frames import get_frame
 
 app = Flask(__name__)
 
@@ -38,22 +37,9 @@ def run_endpoint():
 
 # Frame Data API
 @app.route("/apis/jobs/<job_id>/frames/<int:index>", methods=["GET"])
-def get_frame(job_id, index):
+def get_frame_endpoint(job_id, index):
     """Takes job ID and frame index, returns a single GeoTIFF file."""
-    
-    job_dir = "/processed_data/" + job_id
-    if not os.path.exists(job_dir):
-        abort(404, description="Job not found")
-
-    frame_path = job_dir + f"/frame_{index}.tif"
-    if not os.path.exists(frame_path):
-        abort(404, description="Frame not found")
-
-    return send_file(
-        frame_path,
-        mimetype="image/tiff",
-        as_attachment=False
-    )
+    return get_frame(job_id, index)
 
 
 # Job Status API

nfgda_service/Dockerfile

@@ -23,4 +23,7 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . /app
 
+# algorithm modules (nfgda/, python/, scripts/) now live under /app/algorithm
+ENV PYTHONPATH="/app/algorithm"
+
 CMD ["python", "main.py"]