jobs/tests/docs: Add heartbeat command tests and lifecycle documentation

Add router-level tests covering JobCommand lifecycle:
- Kill command created when status → KILLED or DELETED
- Delivered exactly once via heartbeat
- No duplicate delivery on subsequent heartbeats
- Multiple jobs handled independently
- Non-terminal transitions do not create commands

Add developer documentation describing one-shot delivery semantics.
Ignore local virtualenv (.venv).

Author: mezzeddine

.gitignore

@@ -101,5 +101,8 @@ pixi.lock
 *.egg-info
 docs/templates/_builtin_markdown.jinja
 
+# virtualenv
+.venv
+
 # docs site
 site

diracx-routers/tests/jobs/test_heartbeat_commands.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from time import sleep
+
+import pytest
+from fastapi.testclient import TestClient
+
+from diracx.core.models.job import JobStatus
+
+pytestmark = pytest.mark.enabled_dependencies(
+    [
+        "AuthSettings",
+        "JobDB",
+        "JobLoggingDB",
+        "ConfigSource",
+        "TaskQueueDB",
+        "SandboxMetadataDB",
+        "WMSAccessPolicy",
+        "DevelopmentSettings",
+        "JobParametersDB",
+    ]
+)
+
+
+def test_multiple_jobs_receive_independent_kill_commands(
+    normal_user_client: TestClient,
+    valid_job_ids: list[int],
+):
+    """Verify that multiple jobs each receive exactly one Kill command."""
+    r = normal_user_client.patch(
+        "/api/jobs/status",
+        json={
+            job_id: {
+                datetime.now(timezone.utc).isoformat(): {
+                    "Status": JobStatus.KILLED,
+                    "MinorStatus": "Marked for termination",
+                }
+            }
+            for job_id in valid_job_ids
+        },
+    )
+    r.raise_for_status()
+
+    sleep(1)
+
+    r = normal_user_client.patch(
+        "/api/jobs/heartbeat",
+        json={job_id: {"Vsize": 2000} for job_id in valid_job_ids},
+    )
+    r.raise_for_status()
+
+    commands = r.json()
+    assert len(commands) == len(valid_job_ids)
+    assert {cmd["job_id"] for cmd in commands} == set(valid_job_ids)
+    assert {cmd["command"] for cmd in commands} == {"Kill"}
+
+    sleep(1)
+
+    r = normal_user_client.patch(
+        "/api/jobs/heartbeat",
+        json={job_id: {"Vsize": 2001} for job_id in valid_job_ids},
+    )
+    r.raise_for_status()
+
+    assert r.json() == []
+
+
+def test_non_killed_status_does_not_create_command(
+    normal_user_client: TestClient,
+    valid_job_id: int,
+):
+    """Verify statuses different from KILLED do not enqueue job commands."""
+    r = normal_user_client.patch(
+        "/api/jobs/status",
+        json={
+            valid_job_id: {
+                datetime.now(timezone.utc).isoformat(): {
+                    "Status": JobStatus.RUNNING,
+                    "MinorStatus": "Normal transition",
+                }
+            }
+        },
+    )
+    r.raise_for_status()
+
+    sleep(1)
+
+    r = normal_user_client.patch(
+        "/api/jobs/heartbeat",
+        json={valid_job_id: {"Vsize": 500}},
+    )
+    r.raise_for_status()
+
+    assert r.json() == []
+
+
+def test_deleted_creates_kill_command(
+    normal_user_client: TestClient,
+    valid_job_id: int,
+):
+    """Verify DELETED follows the same command path as KILLED."""
+    r = normal_user_client.patch(
+        "/api/jobs/status",
+        json={
+            valid_job_id: {
+                datetime.now(timezone.utc).isoformat(): {
+                    "Status": JobStatus.DELETED,
+                    "MinorStatus": "User removed job",
+                }
+            }
+        },
+    )
+    r.raise_for_status()
+
+    sleep(1)
+
+    r = normal_user_client.patch(
+        "/api/jobs/heartbeat",
+        json={valid_job_id: {"Vsize": 123}},
+    )
+    r.raise_for_status()
+
+    commands = r.json()
+    assert len(commands) == 1
+    assert commands[0]["job_id"] == valid_job_id
+    assert commands[0]["command"] == "Kill"
+
+    sleep(1)
+
+    r = normal_user_client.patch(
+        "/api/jobs/heartbeat",
+        json={valid_job_id: {"Vsize": 124}},
+    )
+    r.raise_for_status()
+
+    assert r.json() == []

docs/dev/explanations/job_commands.md

@@ -0,0 +1,82 @@
+## What Are Job Commands?
+
+Job commands implement a **per-job control queue** between the WMS and the pilot (JobAgent).
+
+They allow the WMS to send control instructions to running jobs via the heartbeat mechanism.
+
+Currently used for:
+
+- Remote job termination (`Kill` command)
+
+The mechanism is generic and could support additional commands in the future (e.g. debug actions, core dump requests, etc.).
+
+______________________________________________________________________
+
+## Behaviour Summary
+
+### Command Creation
+
+When a job transitions to a terminal state (`KILLED` or `DELETED`),
+`set_job_statuses()` enqueues a `Kill` command in the JobDB.
+
+### Command Delivery
+
+Commands are delivered during:
+
+```text
+PATCH /api/jobs/heartbeat
+```
+
+Flow:
+
+1. Heartbeat updates job state.
+2. `get_job_commands()` retrieves pending commands.
+3. Commands are marked as sent.
+4. Commands are returned to the pilot.
+
+This guarantees **one-shot delivery semantics**:
+
+- A command is delivered exactly once.
+- It is not re-delivered on subsequent heartbeats.
+
+______________________________________________________________________
+
+## Sequence Diagram (Kill Command Lifecycle)
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant User
+    participant Router
+    participant JobDB
+    participant WatchDog
+
+    User->>Router: PATCH /api/jobs/status (Killed)
+    Router->>JobDB: update status → KILLED
+    Router->>JobDB: enqueue "Kill"
+    Router-->>User: 200 OK
+
+    WatchDog->>Router: PATCH /api/jobs/heartbeat
+    Router->>JobDB: fetch + mark sent
+    Router-->>WatchDog: [Kill]
+
+    WatchDog->>Router: PATCH /api/jobs/heartbeat
+    Router-->>WatchDog: []
+```
+
+______________________________________________________________________
+
+## Activity View
+
+```mermaid
+flowchart TD
+    A[Status change → KILLED] --> B[Enqueue Kill command]
+    B --> C[Stored in JobDB]
+
+    D[Heartbeat] --> E[Fetch commands]
+    E --> F[Mark as sent]
+    F --> G[Return to pilot]
+    G --> D
+```
+
+______________________________________________________________________

mkdocs.yml

@@ -171,6 +171,7 @@ nav:
           - Components:
               - Databases: dev/explanations/components/db.md
               - Routes: dev/explanations/components/routes.md
+              - Job commands: dev/explanations/job_commands.md
           - Testing: dev/explanations/testing.md
           - Extensions: dev/explanations/extensions.md
           - Designing functionality: dev/explanations/designing-functionality.md