Add AllStopped command and endpoint for downstairs status (#1718)

Closes #1660

This PR introduces the `dsc cmd all-stopped` command to reliably check
if all "downstairs" services are stopped. This replaces previous
`sleep`-based checks in tests, reducing potential flakiness.

### Changes

* Added `AllStopped` command to `dsc/src/client.rs` and corresponding
handling.
* Implemented the `/allstopped` GET endpoint in `dsc/src/control.rs`.
* Added the `all_stopped` logic to `DscInfo` in `dsc/src/main.rs`,
considering `Running`, `Starting`, or `Stopping` states as not stopped.
* Updated `openapi/dsc-control.json` with the new endpoint definition.
* Modified `tools/test_dsc.sh` to utilize the new `all-stopped` command.

All tests pass with these changes.

Author: nrempel

dsc/src/client.rs

@@ -10,6 +10,8 @@ use anyhow::Result;
 pub enum ClientCommand {
     /// Returns true if all downstairs are running
     AllRunning,
+    /// Returns true if all downstairs are stopped
+    AllStopped,
     /// Disable random stopping of downstairs
     DisableRandomStop,
     /// Disable auto restart on the given downstairs client ID
@@ -91,6 +93,10 @@ pub async fn client_main(server: String, cmd: ClientCommand) -> Result<()> {
             let res = dsc.dsc_all_running().await.unwrap();
             println!("{:?}", res);
         }
+        ClientCommand::AllStopped => {
+            let res = dsc.dsc_all_stopped().await.unwrap();
+            println!("{:?}", res);
+        }
         ClientCommand::DisableRandomStop => {
             let _ = dsc.dsc_disable_random_stop().await.unwrap();
         }

dsc/src/control.rs

@@ -19,6 +19,7 @@ use super::*;
 pub(crate) fn build_api() -> ApiDescription<Arc<DownstairsControl>> {
     let mut api = ApiDescription::new();
     api.register(dsc_all_running).unwrap();
+    api.register(dsc_all_stopped).unwrap();
     api.register(dsc_get_ds_state).unwrap();
     api.register(dsc_get_pid).unwrap();
     api.register(dsc_get_port).unwrap();
@@ -205,6 +206,22 @@ async fn dsc_all_running(
     Ok(HttpResponseOk(all_state))
 }
 
+/**
+ * Return true if all downstairs are stopped
+ */
+#[endpoint {
+    method = GET,
+    path = "/allstopped",
+}]
+async fn dsc_all_stopped(
+    rqctx: RequestContext<Arc<DownstairsControl>>,
+) -> Result<HttpResponseOk<bool>, HttpError> {
+    let api_context = rqctx.context();
+
+    let all_state = api_context.dsci.all_stopped().await;
+    Ok(HttpResponseOk(all_state))
+}
+
 /**
  * Fetch the current state for the requested client_id
  */

dsc/src/main.rs

@@ -715,6 +715,19 @@ impl DscInfo {
         true
     }
 
+    async fn all_stopped(&self) -> bool {
+        let rs = self.rs.lock().await;
+        for state in rs.ds_state.iter() {
+            if *state == DownstairsState::Running
+                || *state == DownstairsState::Starting
+                || *state == DownstairsState::Stopping
+            {
+                return false;
+            }
+        }
+        true
+    }
+
     async fn get_ds_state(&self, client_id: usize) -> Result<DownstairsState> {
         let rs = self.rs.lock().await;
         if rs.ds_state.len() <= client_id {

openapi/dsc-control.json

@@ -30,6 +30,31 @@
         }
       }
     },
+    "/allstopped": {
+      "get": {
+        "summary": "Return true if all downstairs are stopped",
+        "operationId": "dsc_all_stopped",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "title": "Boolean",
+                  "type": "boolean"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/disablerestart/all": {
       "post": {
         "summary": "Disable automatic restart on all downstairs",

tools/test_dsc.sh

@@ -266,26 +266,13 @@ fi
 # downstairs should automatically restart.
 sleep 4
 echo "Verify all ds are stopped"
-for cid in {0..2}; do
-    retry=1
-    # Loop until we find exit, or have exhausted our retry count
-	while :; do
-		state=$(curl -s "${dsc_url}"state/cid/0 | tr -d \")
-		if [[ "$state" == "exit" ]]; then
-			echo "cid $cid in $state after $retry attempt(s)"
-			break;
-		fi
-		echo "cid $cid Failed to stop: $state, try:$retry" | tee -a "$fail_log"
-		if [[ "$retry" -ge 10 ]]; then
-			echo "cid $cid Failed to stop: $state, abort" | tee -a "$fail_log"
-			(( res += 1 ))
-			exit 1
-			break;
-		fi
-		(( retry += 1 ))
-		sleep 3
-	done
-done
+all_stopped_check=$("${dsc}" cmd all-stopped)
+if [[ "$all_stopped_check" != "true" ]]; then
+    echo "Failed: all-stopped returned $all_stopped_check" | tee -a "$fail_log"
+    (( res += 1 ))
+else
+    echo "All downstairs are stopped"
+fi
 
 echo "Start up ds 1 manually"
 hc=$(curl ${curl_flags} -X POST -H "Content-Type: application/json" -w "%{http_code}\n" "${dsc_url}"start/cid/1)