various: fix evaluator restart handling for inactive jobsets

Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I18c89d26f662d132783545f9ed13582a6a6a6964

Author: NotAShelf

crates/common/src/repo/evaluations.rs

@@ -366,17 +366,20 @@ pub async fn cancel(pool: &PgPool, id: Uuid) -> Result<Option<Evaluation>> {
 }
 
 /// Requeue a cancelled, failed, or timed-out evaluation after discarding its
-/// stale builds. One-shot jobsets are re-enabled for the new attempt.
+/// stale builds. A disabled jobset rejects the restart; a retained one-shot
+/// jobset is re-enabled for its new attempt.
 ///
 /// # Errors
 ///
 /// Returns an error if the database transaction fails.
 pub async fn restart(pool: &PgPool, id: Uuid) -> Result<Option<Evaluation>> {
   let mut tx = pool.begin().await?;
   let evaluation = sqlx::query_as::<_, Evaluation>(
-    "UPDATE evaluations SET status = 'pending', evaluation_time = NOW(), \
-     error_message = NULL, inputs_hash = NULL WHERE id = $1 AND status IN \
-     ('cancelled', 'failed', 'timed_out') RETURNING *",
+    "UPDATE evaluations e SET status = 'pending', evaluation_time = NOW(), \
+     error_message = NULL, inputs_hash = NULL FROM jobsets j WHERE e.id = $1 \
+     AND e.jobset_id = j.id AND e.status IN ('cancelled', 'failed', \
+     'timed_out') AND (j.state = 'one_shot' OR (j.enabled AND j.state IN \
+     ('enabled', 'one_at_a_time'))) RETURNING e.*",
   )
   .bind(id)
   .fetch_optional(&mut *tx)

crates/common/tests/repo_tests.rs

@@ -593,6 +593,44 @@ async fn test_restarting_one_shot_evaluation_resets_its_attempt() {
   assert_eq!(restarted_jobset.state, JobsetState::OneShot);
 }
 
+#[tokio::test]
+async fn test_restart_rejects_a_manually_disabled_jobset() {
+  let Some(pool) = get_pool().await else {
+    return;
+  };
+
+  let project = create_test_project(&pool, "restart-disabled").await;
+  let jobset = create_test_jobset(&pool, project.id).await;
+  let eval = create_test_eval(&pool, jobset.id).await;
+  repo::evaluations::update_status(
+    &pool,
+    eval.id,
+    EvaluationStatus::Failed,
+    Some("failed"),
+  )
+  .await
+  .expect("fail evaluation");
+  sqlx::query("UPDATE jobsets SET enabled = false WHERE id = $1")
+    .bind(jobset.id)
+    .execute(&pool)
+    .await
+    .expect("disable jobset");
+
+  assert!(
+    repo::evaluations::restart(&pool, eval.id)
+      .await
+      .expect("attempt restart")
+      .is_none()
+  );
+  assert_eq!(
+    repo::evaluations::get(&pool, eval.id)
+      .await
+      .expect("get failed evaluation")
+      .status,
+    EvaluationStatus::Failed
+  );
+}
+
 #[tokio::test]
 async fn test_cancellation_cannot_interleave_build_persistence() {
   let Some(pool) = get_pool().await else {

crates/server/src/routes/dashboard/admin.rs

@@ -770,7 +770,8 @@ pub(super) async fn evaluation_restart(
     .ok_or_else(|| {
       (
         StatusCode::CONFLICT,
-        "Only failed or cancelled evaluations can be restarted",
+        "Only failed, cancelled, or timed-out evaluations with active jobsets \
+         can be restarted",
       )
         .into_response()
     })?;

crates/server/templates/evaluation.html

@@ -39,14 +39,18 @@ <h1>Evaluation {{ eval.commit_short }}</h1>
                 {% endif %}
             </form>
             {% if eval.status_class == "running" || eval.status_class == "pending" %}
-                <form class="inline" method="POST" action="/evaluation/{{ eval.id }}/cancel"
+                <form class="inline"
+                      method="POST"
+                      action="/evaluation/{{ eval.id }}/cancel"
                       onsubmit="return confirm('Cancel this evaluation?')">
                     <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
                     <button class="btn btn-small btn-danger" type="submit">Cancel Evaluation</button>
                 </form>
             {% endif %}
-            {% if eval.status_class == "failed" || eval.status_class == "cancelled" %}
-                <form class="inline" method="POST" action="/evaluation/{{ eval.id }}/restart"
+            {% if eval.status_class == "failed" || eval.status_class == "cancelled" || eval.status_class == "timed-out" %}
+                <form class="inline"
+                      method="POST"
+                      action="/evaluation/{{ eval.id }}/restart"
                       onsubmit="return confirm('Restart this evaluation and discard its current builds?')">
                     <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
                     <button class="btn btn-small btn-secondary" type="submit">Restart Evaluation</button>
@@ -71,7 +75,8 @@ <h1>Evaluation {{ eval.commit_short }}</h1>
                 <h2>Evaluation Error</h2>
                 <span>Nix diagnostics</span>
             </div>
-            <pre class="error-detail">{{ eval.error_message }}</pre>
+            <pre class="error-detail">
+{{ eval.error_message }}</pre>
         </section>
     {% endif %}