fix: /cancel + dev_reset P1/P2 Review-Fixes

P1 AtomicBot-ai#1: /cancel false-positive bei nicht-existierender task_id
  - METRICS-Check jetzt für ALLE task_ids (nicht nur bei leerem Body)
  - task_id wird gegen laufende Slots validiert vor Cancel-Post
  - Nicht-existierende task_id → {cancelled: false, error: 'task not found'}
  - Leerer Body ohne laufende Tasks → {cancelled: false, message: 'no running tasks'}
  - Zusätzlich: ältester Task (nach start_time) statt niedrigster Slot-Index

P1 AtomicBot-ai#2: ggml_backend_cuda_device_reset thread-safety
  - device_mutex Lock hinzugefügt (wie ggml_backend_cuda_device_get_memory)
  - active_count > 0 → Reset verweigert (verhindert Context-Crash)
  - cudaGetLastError-Details in GGML_LOG_WARN

P1 AtomicBot-ai#3: Test-Skript — Cancel-Wirkung verifiziert
  - Test 3: Stream muss abgebrochen sein (aborted=True oder wenige chunks)
  - Test 3b neu: nicht-existierende task_id → cancelled=false + error
  - Test akzeptiert nicht mehr normal beendeten Stream als Erfolg

P1 AtomicBot-ai#4: dev_reset Rückgabewert nicht ignorieren
  - SRV_WRN bei fehlgeschlagenem Reset mit Device-Name

P2 AtomicBot-ai#7: proxy_post try/catch bei leerem/ungültigem Body
  - Statt 500-Exception → 400 'Invalid JSON body'
  - res_err() statt nicht-existenter .error() Methode

P2 AtomicBot-ai#8: Ältester Task statt niedrigster Slot-Index (in P1 AtomicBot-ai#1 fix enthalten)

Author: fukuro

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -5829,11 +5829,30 @@ static void ggml_backend_cuda_device_event_synchronize(ggml_backend_dev_t dev, g
 static bool ggml_backend_cuda_device_reset(ggml_backend_dev_t dev) {
     ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
 
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+    std::lock_guard<std::mutex> lock(dev_ctx->device_mutex);
+
+    // refuse to reset while backends/buffers are still active — cudaDeviceReset()
+    // would destroy the CUDA context and leave dangling handles
+    if (dev_ctx->active_count > 0) {
+        GGML_LOG_WARN("%s: refusing to reset device %d — %d active backend(s) still holding resources\n",
+                      __func__, dev_ctx->device, dev_ctx->active_count);
+        return false;
+    }
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+
     if (cudaSetDevice(dev_ctx->device) != cudaSuccess) {
+        GGML_LOG_WARN("%s: cudaSetDevice(%d) failed: %s\n", __func__, dev_ctx->device, cudaGetErrorString(cudaGetLastError()));
+        return false;
+    }
+
+    cudaError_t err = cudaDeviceReset();
+    if (err != cudaSuccess) {
+        GGML_LOG_WARN("%s: cudaDeviceReset() failed for device %d: %s\n", __func__, dev_ctx->device, cudaGetErrorString(err));
         return false;
     }
 
-    return cudaDeviceReset() == cudaSuccess;
+    return true;
 }
 
 static const ggml_backend_device_i ggml_backend_cuda_device_interface = {

scripts/test-cancel-endpoint.py

@@ -170,12 +170,31 @@ def stream_request():
     # Wait for streaming thread to finish (should get connection closed / abort)
     t.join(timeout=10)
     assert not t.is_alive(), "Streaming thread still alive after 10s — cancel did not terminate the stream"
-    # The streaming client should have been terminated (connection closed by server)
-    # On a fast CPU-only test setup, the stream may complete with [DONE] before cancel
-    # takes effect, but the thread must have ended (not still blocking on recv)
+    # The streaming client must have been terminated by the cancel, not completed normally.
+    # If done=True and not aborted, the stream finished before cancel took effect —
+    # that's a test failure (cancel was too slow or didn't work).
     print(f"[INFO] Stream done={stream_result['done']}, aborted={stream_result['aborted']}")
     print(f"[INFO] Stream collected {len(stream_content)} content chunks")
-    print("[PASS] Test 3: streaming request cancelled, thread terminated")
+    if stream_result.get("aborted"):
+        print("[PASS] Test 3: streaming request cancelled, stream was aborted by server")
+    elif stream_result.get("done") and len(stream_content) < 50:
+        # On very fast setups the stream may abort mid-chunk — few chunks + done is also acceptable
+        print("[PASS] Test 3: streaming request cancelled (stream ended with few chunks, likely aborted)")
+    else:
+        # Stream completed normally with lots of content — cancel didn't work
+        assert False, f"Stream completed normally ({len(stream_content)} chunks, done={stream_result['done']}) — cancel did not abort the stream"
+
+
+def test_cancel_nonexistent_task_id(base_url):
+    """Test 3b: /cancel with a task_id that doesn't exist should return cancelled=false."""
+    # Use a very high task_id that is unlikely to exist
+    r = requests.post(f"{base_url}/cancel", json={"task_id": 99999999}, timeout=10)
+    assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}"
+    data = r.json()
+    assert data["cancelled"] is False, f"Expected cancelled=false for non-existent task_id, got {data}"
+    assert "error" in data, f"Expected error field for non-existent task_id, got {data}"
+    assert "not found" in data["error"].lower(), f"Expected 'not found' in error, got {data['error']}"
+    print(f"[PASS] Test 3b: non-existent task_id correctly returns cancelled=false")
 
 
 def test_cancel_first_running_task(base_url):
@@ -278,6 +297,7 @@ def main():
         ("cancel invalid JSON", lambda: test_cancel_invalid_json(base_url)),
         ("cancel negative task_id", lambda: test_cancel_negative_task_id(base_url)),
         ("cancel running task", lambda: test_cancel_running_task(base_url)),
+        ("cancel nonexistent task_id", lambda: test_cancel_nonexistent_task_id(base_url)),
         ("cancel first running task", lambda: test_cancel_first_running_task(base_url)),
     ]
 

tools/server/server-context.cpp

@@ -862,7 +862,11 @@ struct server_context_impl {
         // in sleep mode, we need to reset the devices to free up memory
         if (reset_devices) {
             for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
-                ggml_backend_dev_reset(ggml_backend_dev_get(i));
+                ggml_backend_dev_t dev = ggml_backend_dev_get(i);
+                if (!ggml_backend_dev_reset(dev)) {
+                    SRV_WRN("failed to reset device %zu (%s) — VRAM may not be fully released\n",
+                            i, ggml_backend_dev_name(dev));
+                }
             }
         }
     }
@@ -4428,10 +4432,10 @@ void server_routes::init_routes() {
     };
 
     // POST /cancel — cancel a running request by task_id and release its slot
-    // Body: {"task_id": 12345} or {} (cancels first running task found)
-    // Router mode: {"model": "name", "task_id": 12345} — model is consumed by proxy_post
-    // Response: {"cancelled": true, "task_id": 12345} (fire-and-forget, idempotent)
+    // Body: {"task_id": 12345} or {} (cancels oldest running task found)
+    // Response: {"cancelled": true, "task_id": 12345}
     //           {"cancelled": false, "message": "no running tasks"} (empty body, nothing running)
+    //           {"cancelled": false, "error": "task not found"} (task_id not running or already finished)
     this->post_cancel = [this](const server_http_req & req) {
         auto res = create_response();
 
@@ -4450,44 +4454,62 @@ void server_routes::init_routes() {
                     }
                 }
             } catch (const std::exception &) {
-                res->error(format_error_response("Invalid JSON body", ERROR_TYPE_INVALID_REQUEST));
+                res->error(format_error_response("Invalid JSON body or wrong type for task_id", ERROR_TYPE_INVALID_REQUEST));
                 return res;
             }
         }
 
-        // if no task_id given, find the first running task via METRICS
-        if (target_id == NO_TASK_ID) {
-            server_task metrics_task(SERVER_TASK_TYPE_METRICS);
-            metrics_task.id = res->rd.get_new_id();
-            res->rd.post_task(std::move(metrics_task), true);
+        // query METRICS to find running tasks and validate task_id
+        server_task metrics_task(SERVER_TASK_TYPE_METRICS);
+        metrics_task.id = res->rd.get_new_id();
+        res->rd.post_task(std::move(metrics_task), true);
 
-            auto result = res->rd.next(req.should_stop);
-            if (!result) {
-                GGML_ASSERT(req.should_stop());
-                return res;
-            }
-            if (result->is_error()) {
-                res->error(result->to_json());
-                return res;
-            }
+        auto result = res->rd.next(req.should_stop);
+        if (!result) {
+            GGML_ASSERT(req.should_stop());
+            return res;
+        }
+        if (result->is_error()) {
+            res->error(result->to_json());
+            return res;
+        }
 
-            auto * res_metrics = dynamic_cast<server_task_result_metrics*>(result.get());
-            GGML_ASSERT(res_metrics != nullptr);
+        auto * res_metrics = dynamic_cast<server_task_result_metrics*>(result.get());
+        GGML_ASSERT(res_metrics != nullptr);
 
-            // find the first processing slot (lowest slot index, not necessarily oldest by start time)
-            for (const auto & slot_data : res_metrics->slots_data) {
-                if (slot_data.value("is_processing", false)) {
-                    target_id = slot_data.value("id_task", NO_TASK_ID);
+        // find running task: either the requested task_id or the oldest running task
+        int oldest_id    = NO_TASK_ID;
+        int64_t oldest_start = INT64_MAX;
+
+        for (const auto & slot_data : res_metrics->slots_data) {
+            if (slot_data.value("is_processing", false)) {
+                int slot_task_id = slot_data.value("id_task", NO_TASK_ID);
+                if (target_id == NO_TASK_ID) {
+                    // no task_id given — find oldest running task by start time
+                    int slot_start = slot_data.value("start_time", 0);
+                    if (slot_start < oldest_start) {
+                        oldest_start = slot_start;
+                        oldest_id    = slot_task_id;
+                    }
+                } else if (slot_task_id == target_id) {
+                    // requested task_id found and running
+                    oldest_id = target_id;
                     break;
                 }
             }
+        }
 
+        if (oldest_id == NO_TASK_ID) {
             if (target_id == NO_TASK_ID) {
                 res->ok(json {{"cancelled", false}, {"message", "no running tasks"}});
-                return res;
+            } else {
+                res->ok(json {{"cancelled", false}, {"error", "task not found"}});
             }
+            return res;
         }
 
+        target_id = oldest_id;
+
         // post cancel task with highest priority (fire-and-forget, idempotent)
         // the existing CANCEL handler in process_task_loop() calls slot.release()
         // if the task already finished, the cancel is a no-op

tools/server/server-models.cpp

@@ -1234,8 +1234,17 @@ void server_models_routes::init_routes() {
 
     this->proxy_post = [this](const server_http_req & req) {
         std::string method = "POST";
-        json body = json::parse(req.body);
-        std::string name = json_value(body, "model", std::string());
+        std::string name;
+        if (!req.body.empty()) {
+            try {
+                json body = json::parse(req.body);
+                name = json_value(body, "model", std::string());
+            } catch (const std::exception &) {
+                auto error_res = std::make_unique<server_http_res>();
+                res_err(error_res, format_error_response("Invalid JSON body", ERROR_TYPE_INVALID_REQUEST));
+                return error_res;
+            }
+        }
         bool autoload = is_autoload(params, req);
         auto error_res = std::make_unique<server_http_res>();
         if (!router_validate_model(name, models, autoload, error_res)) {