Add diamondnode-ops: operational and functional commands for GTX 1650 environment (status, cleanup, reliable benchmarking with llama-bench, GPU watch)

Author: igor-holt

diamondnode-ops/README.md

@@ -0,0 +1,33 @@
+# diamondnode-ops
+
+Operational and functional commands for the diamondnode (GTX 1650) environment in the Genesis Conductor / Hermes-Openclaw project.
+
+## Scripts
+
+- `status.sh` — Comprehensive status (GPU, model, native binary, disk, podman, processes).
+- `cleanup.sh` — Safe kill of runaway llama processes + deletion of large benchmark logs.
+- `run_bench.sh` — Reliable statistical benchmarks using the proper `llama-bench` tool (JSON output, avoids log flooding).
+- `gpu_watch.sh` — Live GPU + disk monitor.
+
+## Usage on diamondnode
+
+```bash
+# Copy this directory to diamondnode (e.g. via rsync or git)
+rsync -avz diamondnode-ops/ diamondnode@192.168.1.228:~/diamondnode-ops/
+
+ssh diamondnode@192.168.1.228
+cd ~/diamondnode-ops
+./status.sh
+./cleanup.sh
+./run_bench.sh
+./gpu_watch.sh
+```
+
+These commands are designed for the real hardware environment (Ubuntu + GTX 1650 + native llama.cpp CUDA build).
+
+## Notes
+- Always run cleanup before new benchmark attempts.
+- Use `run_bench.sh` for clean statistical data (preferred over raw llama-cli loops).
+- All scripts are idempotent and safe where possible.
+
+Part of the ag-15 / Hermes-Openclaw operational tooling.

diamondnode-ops/cleanup.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# diamondnode-ops/cleanup.sh
+# Safe cleanup of runaway processes and large logs on diamondnode
+
+set -euo pipefail
+
+echo "=== DIAMONDNODE CLEANUP $(date -Iseconds) ==="
+
+echo "Killing llama* and benchmark processes..."
+pkill -9 -f "llama-bench\|llama-cli\|run_native" 2>/dev/null || true
+sleep 2
+
+echo "Current GPU:"
+nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader
+
+echo ""
+echo "Deleting large benchmark logs (>50MB)..."
+find /tmp -name "*diamondnode*bench*.log" -o -name "*native_bench*.log" | while read f; do
+    if [ -f "$f" ]; then
+        size=$(du -m "$f" | cut -f1)
+        if [ "$size" -gt 50 ]; then
+            echo "Deleting $f ($size MB)"
+            rm -f "$f"
+        fi
+    fi
+done
+
+echo ""
+echo "Disk after:"
+df -h /tmp
+
+echo "=== CLEANUP COMPLETE ==="

diamondnode-ops/gpu_watch.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# diamondnode-ops/gpu_watch.sh
+# Simple GPU monitor for diamondnode
+
+watch -n 2 'nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu --format=csv,noheader && echo "" && df -h /tmp'

diamondnode-ops/run_bench.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# diamondnode-ops/run_bench.sh
+# Reliable native benchmark using llama-bench (preferred for stats)
+
+set -euo pipefail
+
+MODEL="/home/diamondnode/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf"
+BENCH="/home/diamondnode/llama.cpp/build/bin/llama-bench"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUTDIR="/tmp/bench_results"
+mkdir -p "$OUTDIR"
+LOG="$OUTDIR/bench_${TIMESTAMP}.jsonl"
+
+if [ ! -x "$BENCH" ]; then
+    echo "llama-bench not found or not executable"
+    exit 1
+fi
+
+echo "=== RELIABLE NATIVE BENCHMARK $(date -Iseconds) ===" | tee -a "$LOG"
+echo "Model: $MODEL" | tee -a "$LOG"
+echo "Binary: $BENCH" | tee -a "$LOG"
+
+# Clean statistical runs with JSON output (avoids log flood)
+for LAYERS in 4 8 12; do   # Start conservative for 4GB VRAM
+  echo "Running layers=$LAYERS ..."
+  $BENCH \
+    -m "$MODEL" \
+    -ngl $LAYERS \
+    -p 512 \
+    -n 64 \
+    -r 5 \
+    -b 256 \
+    -ub 128 \
+    -t 4 \
+    --progress \
+    -o json \
+    --no-warmup 2>&1 | tee -a "$LOG"
+done
+
+echo "Results saved to $LOG"
+echo "=== BENCH COMPLETE ==="

diamondnode-ops/status.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# diamondnode-ops/status.sh
+# Operational status check for diamondnode (GTX 1650 setup)
+
+set -euo pipefail
+
+echo "=== DIAMONDNODE OPERATIONAL STATUS ==="
+echo "Timestamp: $(date -Iseconds)"
+echo "Host: $(hostname)"
+echo ""
+
+echo "=== GPU ==="
+nvidia-smi --query-gpu=name,memory.total,memory.free,utilization.gpu,driver_version --format=csv,noheader || echo "nvidia-smi failed"
+echo ""
+
+echo "=== MODEL ==="
+ls -lh /home/diamondnode/models/ 2>/dev/null || echo "No models directory"
+echo ""
+
+echo "=== NATIVE LLAMA.CPP BINARY ==="
+if [ -f /home/diamondnode/llama.cpp/build/bin/llama-cli ]; then
+    ls -l /home/diamondnode/llama.cpp/build/bin/llama-cli
+    sha256sum /home/diamondnode/llama.cpp/build/bin/llama-cli
+else
+    echo "llama-cli not found"
+fi
+if [ -f /home/diamondnode/llama.cpp/build/bin/llama-bench ]; then
+    echo "llama-bench present"
+fi
+echo ""
+
+echo "=== DISK ==="
+df -h /home/diamondnode /tmp 2>/dev/null | cat
+echo ""
+
+echo "=== PODMAN ==="
+podman --version 2>/dev/null || echo "podman not available"
+echo ""
+
+echo "=== KEY PROCESSES ==="
+ps aux | grep -E "llama|benchmark|python|node" | grep -v grep | head -10 | cat || echo "No relevant processes"
+echo ""
+
+echo "=== END STATUS ==="