Merge branch 'dev'

Author: Francisco

.github/workflows/ci.yml

@@ -158,6 +158,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: "🧹 Free up disk space on runner"
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
       - name: "⚙️ Setup Docker Buildx"
         uses: docker/setup-buildx-action@v3
 
@@ -228,6 +237,15 @@ jobs:
       - name: "🕺 Checkout"
         uses: actions/checkout@v4
 
+      - name: "🧹 Free up disk space on runner"
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
       - name: "⚙️ Setup QEMU"
         uses: docker/setup-qemu-action@v3
 
@@ -357,6 +375,15 @@ jobs:
           persist-credentials: true
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: "🧹 Free up disk space on runner"
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
       - name: "⚙️ Setup QEMU"
         uses: docker/setup-qemu-action@v3
 

.gitignore

@@ -10,7 +10,6 @@
 logging_utility = UtilsInterface.LoggingUtility()
 
 # --- ALL ENGINE AND SESSION LOGIC IS NOW CENTRALIZED HERE ---
-
 DATABASE_URL = os.getenv("DATABASE_URL")
 SPECIAL_DB_URL = os.getenv("SPECIAL_DB_URL")
 
@@ -21,7 +20,24 @@ def running_in_docker() -> bool:
 
 
 def resolve_special_db_runtime_url(special_raw: str | None) -> str | None:
+    """
+    Resolve the effective SPECIAL_DB_URL for the current runtime.
+
+    SPECIAL_DB_URL is intended for host-shell use only (see .env.migrations).
+    Its value typically points at localhost:3307, which is meaningless inside
+    containers. If it leaks into container env via .env / env_file, we ignore
+    it and fall back to DATABASE_URL — and log a warning so the leak is
+    visible to the operator.
+    """
     if running_in_docker():
+        if special_raw:
+            logging_utility.warning(
+                "SPECIAL_DB_URL is set inside a container — ignoring and "
+                "falling back to DATABASE_URL. SPECIAL_DB_URL is host-shell-only; "
+                "it should live in .env.migrations, not .env. "
+                "See: https://github.com/project-david-ai/projectdavid-core "
+                "(container hygiene section)."
+            )
         return DATABASE_URL
     return special_raw or None
 
@@ -71,8 +87,6 @@ def get_db():
 
 # Optional: You can also move the wait logic here to keep all DB startup
 # code together, which makes app.py even cleaner.
-
-
 def _wait_for_engine(engine_to_check, db_name, logger, retries=30, delay=3):
     if not engine_to_check:
         return

src/api/entities_api/db/database.py

@@ -1,29 +1,20 @@
 # src/api/training/routers/datasets_router.py
 
-import os
 from typing import Optional
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Query
 from projectdavid_common import UtilsInterface, ValidationInterface
 from projectdavid_common.schemas.enums import StatusEnum
 from sqlalchemy.orm import Session
 
 from src.api.training.db.database import get_db
 from src.api.training.dependencies import get_current_user_id
-from src.api.training.services.dataset_service import (
-    create_dataset,
-    delete_dataset,
-    get_dataset,
-    list_datasets,
-    prepare_dataset,
-)
+from src.api.training.services.dataset_service import DatasetService
 
 logging_utility = UtilsInterface.LoggingUtility()
 
 router = APIRouter()
 
-# Note: API_BASE_URL and WORKER_API_KEY removed as they are no longer
-# needed by the router for internal file fetching.
 
 # ---------------------------------------------------------------------------
 # POST /v1/datasets
@@ -48,8 +39,7 @@ def create_dataset_endpoint(
         payload.format,
         payload.file_id,
     )
-    dataset = create_dataset(
-        db=db,
+    dataset = DatasetService(db).create(
         user_id=user_id,
         name=payload.name,
         fmt=payload.format,
@@ -86,8 +76,8 @@ def list_datasets_endpoint(
                 status_code=422, detail=f"Invalid status value '{status}'."
             )
 
-    datasets = list_datasets(
-        db=db, user_id=user_id, status=status_filter, limit=limit, offset=offset
+    datasets = DatasetService(db).list(
+        user_id=user_id, status=status_filter, limit=limit, offset=offset
     )
     return ValidationInterface.DatasetList(
         data=[ValidationInterface.DatasetRead.model_validate(d) for d in datasets],
@@ -110,7 +100,7 @@ def get_dataset_endpoint(
     user_id: str = Depends(get_current_user_id),
     db: Session = Depends(get_db),
 ):
-    dataset = get_dataset(db=db, dataset_id=dataset_id, user_id=user_id)
+    dataset = DatasetService(db).get(dataset_id=dataset_id, user_id=user_id)
     return ValidationInterface.DatasetRead.model_validate(dataset)
 
 
@@ -128,15 +118,7 @@ async def prepare_dataset_endpoint(
     user_id: str = Depends(get_current_user_id),
     db: Session = Depends(get_db),
 ):
-    """
-    Trigger background preparation.
-    Service logic now uses direct Samba/Shared DB access.
-    """
-    return prepare_dataset(
-        db=db,
-        dataset_id=dataset_id,
-        user_id=user_id,
-    )
+    return DatasetService(db).prepare(dataset_id=dataset_id, user_id=user_id)
 
 
 # ---------------------------------------------------------------------------
@@ -147,12 +129,23 @@ async def prepare_dataset_endpoint(
 @router.delete(
     "/{dataset_id}",
     response_model=ValidationInterface.DatasetDeleted,
-    summary="Soft delete a dataset",
+    summary="Delete a dataset (soft by default; ?hard=true cascades to storage)",
 )
 def delete_dataset_endpoint(
     dataset_id: str,
+    hard: bool = Query(
+        False,
+        description=(
+            "If true, permanently remove the dataset, its File/FileStorage DB rows, "
+            "and the physical .jsonl on Samba. Default is soft-delete (reversible)."
+        ),
+    ),
     user_id: str = Depends(get_current_user_id),
     db: Session = Depends(get_db),
 ):
-    result = delete_dataset(db=db, dataset_id=dataset_id, user_id=user_id)
+    result = DatasetService(db).delete(
+        dataset_id=dataset_id,
+        user_id=user_id,
+        hard=hard,
+    )
     return ValidationInterface.DatasetDeleted(**result)