drt: run roachtest-operations as a systemd unit

Previously, roachtest-operations ran directly in the foreground piped
through tee. This made it difficult to manage as a service — stopping
it required finding the process, and SIGINT handling via the tee pipe
was fragile.

Now the script uses systemd-run to create a transient systemd unit
(roachtest-operations-<cluster>), following the same self-reinvocation
pattern as roachprod's start.sh. The script detects re-invocation via
the ROACHTEST_OPS_SYSTEMD env var and uses exec to replace bash, so
roachtest-operations directly receives SIGTERM from systemctl stop for
graceful shutdown. TimeoutStopSec=150 ensures the full 2-minute cleanup
window is respected. Logs are available via journalctl.

Epic: none
Release note: None

Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>

Author: shailendra-patel

pkg/cmd/drt/scripts/roachtest_operations_run.sh

@@ -29,8 +29,11 @@ cd /home/ubuntu
 
 export ROACHPROD_GCE_DEFAULT_PROJECT=cockroach-drt
 export ROACHPROD_DNS="drt.crdb.io"
-./drtprod sync
-sleep 20
+
+if [[ "${ROACHTEST_OPS_SYSTEMD:-}" != "1" ]]; then
+  ./drtprod sync
+  sleep 20
+fi
 
 # Fetch secrets from cloud provider at runtime (not stored in any file)
 # AWS secrets are stored in us-east-1 region for consistency across all clusters
@@ -81,15 +84,30 @@ if [ -z "${DD_API_KEY}" ]; then
   exit 1
 fi
 
-# Ignore SIGINT in the shell so that tee (which inherits the signal
-# disposition) stays alive when Ctrl+C is pressed. Only
-# roachtest-operations receives SIGINT and handles cleanup gracefully.
-trap '' INT
+UNIT_NAME="roachtest-operations-${CLUSTER}"
+
+if [[ "${ROACHTEST_OPS_SYSTEMD:-}" == "1" ]]; then
+  exec ./roachtest-operations run-operation "${CLUSTER}" ".*" \
+    --datadog-api-key "${DD_API_KEY}" \
+    --datadog-app-key "unused" \
+    --datadog-tags "env:development,cluster:${CLUSTER},workload:${WORKLOAD_CLUSTER},team:drt,service:drt-cockroachdb" \
+    --certs-dir ./certs --prom-port 2115 \
+    --cloud "${CLOUD}" --workload-cluster "${WORKLOAD_CLUSTER}" --run-forever
+fi
+
+# Set up systemd unit and start it, which will recursively
+# invoke this script but hit the above conditional.
+
+if systemctl is-active -q "${UNIT_NAME}"; then
+  echo "${UNIT_NAME} service already active"
+  echo "To get more information: systemctl status ${UNIT_NAME}"
+  exit 1
+fi
 
-./roachtest-operations run-operation "${CLUSTER}" ".*" \
-  --datadog-api-key "${DD_API_KEY}" \
-  --datadog-app-key "unused" \
-  --datadog-tags "env:development,cluster:${CLUSTER},workload:${WORKLOAD_CLUSTER},team:drt,service:drt-cockroachdb" \
-  --certs-dir ./certs --prom-port 2115 \
-  --cloud "${CLOUD}" --workload-cluster "${WORKLOAD_CLUSTER}" --run-forever 2>&1 | tee -a roachtest_ops.log
+sudo systemctl reset-failed "${UNIT_NAME}" 2>/dev/null || true
 
+sudo systemd-run --unit "${UNIT_NAME}" \
+  --same-dir --uid "$(id -u)" --gid "$(id -g)" \
+  -p TimeoutStopSec=150 \
+  --setenv ROACHTEST_OPS_SYSTEMD=1 \
+  bash "${0}" "${CLUSTER}" "${WORKLOAD_CLUSTER}" "${CLOUD}"