test: add unit tests for data pipeline and api

Author: whitebeard10

tests/test_api.py

@@ -0,0 +1,22 @@
+from fastapi.testclient import TestClient
+from app.main import app
+
+client = TestClient(app)
+
+def test_read_health():
+    # Since the model won't be loaded in a standard test environment, 
+    # we expect the model_loaded status to be false or for the startup to fail.
+    # We can mock the engine if needed, but a simple health check is a good start.
+    response = client.get("/health")
+    assert response.status_code == 200
+    assert "status" in response.json()
+    assert response.json()["status"] == "ok"
+
+def test_generate_fail_no_model():
+    # Test that /generate returns 503 if engine is not initialized
+    response = client.post(
+        "/generate",
+        json={"instruction": "How do I reset my password?"}
+    )
+    assert response.status_code == 503
+    assert response.json()["detail"] == "Model not loaded"

tests/test_data.py

@@ -0,0 +1,35 @@
+import pytest
+import os
+import json
+import shutil
+from src.data.preprocess import clean_data, format_prompt, load_raw_data
+
+def test_clean_data():
+    sample_data = [
+        {"instruction": "test 1", "response": "resp 1"},
+        {"instruction": "test 1", "response": "resp 1"}, # Duplicate
+        {"instruction": "test 2", "response": "resp 2"}
+    ]
+    cleaned = clean_data(sample_data)
+    assert len(cleaned) == 2
+    assert cleaned[0]["instruction"] == "test 1"
+    assert cleaned[1]["instruction"] == "test 2"
+
+def test_format_prompt():
+    sample = {"instruction": "Hello", "response": "Hi there"}
+    formatted = format_prompt(sample)
+    assert "text" in formatted
+    assert "<system>" in formatted["text"]
+    assert "<instruction> Hello </instruction>" in formatted["text"]
+    assert "<response> Hi there </response>" in formatted["text"]
+
+def test_load_raw_data(tmp_path):
+    d = tmp_path / "data"
+    d.mkdir()
+    p = d / "test.jsonl"
+    content = {"instruction": "q1", "response": "a1"}
+    p.write_text(json.dumps(content) + "\n")
+    
+    loaded = load_raw_data(str(p))
+    assert len(loaded) == 1
+    assert loaded[0]["instruction"] == "q1"