feat: poll testnet status and open infra recovery issues (#742)

* ansible: codify dashmon status monitoring access

* feat: poll testnet status and open infra recovery issues

* fix: shorten dashmon authorized key ansible line

---------

Co-authored-by: dashinfraclaw <dashinfraclaw@users.noreply.github.com>

Author: vivekgsharma

.github/workflows/poll-testnet-status.yml

@@ -0,0 +1,46 @@
+name: Poll Testnet Status
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */2 * * *'
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  poll-testnet-status:
+    name: Poll testnet status and open recovery issues
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    concurrency:
+      group: poll-testnet-status
+      cancel-in-progress: false
+
+    env:
+      GH_TOKEN: ${{ secrets.INFRA_ISSUES_TOKEN }}
+      TESTNET_RECOVERY_ASSIGNEE: dashinfraclaw
+      TESTNET_RECOVERY_ISSUE_REPOSITORY: dashpay/infra
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Validate cross-repo issue token
+        run: |
+          if [[ -z "${GH_TOKEN}" ]]; then
+            echo "INFRA_ISSUES_TOKEN secret is required to create issues in dashpay/infra"
+            exit 1
+          fi
+
+      - name: Poll status API and open recovery issues
+        run: node bin/poll-testnet-status.js

ansible/roles/status_dashboard/defaults/main.yml

@@ -3,5 +3,7 @@
 status_dashboard_image: dashpay/status:latest
 status_dashboard_port: 3010
 status_dashboard_path: "{{ dashd_home }}/status_dashboard"
+status_dashboard_ssh_private_key_path: "{{ lookup('env', 'STATUS_DASHBOARD_SSH_KEY_PATH') | default('~/.ssh/dashmon-testnet', true) }}"
+status_dashboard_ssh_user: dashmon
 status_dashboard_poll_interval: 10000
 status_dashboard_poll_concurrency: 20

ansible/roles/status_dashboard/tasks/main.yml

@@ -12,9 +12,9 @@
     dest: "{{ status_dashboard_path }}/inventory"
     mode: "0644"
 
-- name: Copy SSH deploy key for status dashboard
+- name: Copy SSH monitoring key for status dashboard
   ansible.builtin.copy:
-    src: "{{ lookup('env', 'PRIVATE_KEY_PATH') | default('~/.ssh/evo-app-deploy.rsa', true) }}"
+    src: "{{ status_dashboard_ssh_private_key_path }}"
     dest: "{{ status_dashboard_path }}/ssh_key"
     mode: "0600"
     owner: root

ansible/roles/status_dashboard/templates/docker-compose.yml.j2

@@ -10,7 +10,7 @@ services:
     environment:
       - INVENTORY_PATH=/app/data/inventory
       - SSH_KEY_PATH=/app/data/ssh_key
-      - SSH_USER=ubuntu
+      - SSH_USER={{ status_dashboard_ssh_user }}
       - SSH_COMMAND=/usr/local/bin/dashmon-check
       - SSH_PORT=22
       - POLL_INTERVAL_MS={{ status_dashboard_poll_interval }}

ansible/roles/status_monitoring/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+
+status_monitoring_user: dashmon
+status_monitoring_home: "/home/{{ status_monitoring_user }}"
+status_monitoring_forced_command: /usr/local/bin/dashmon-check

ansible/roles/status_monitoring/files/dashmon-check.sh

@@ -7,7 +7,29 @@ set -euo pipefail
 
 if [[ -f /home/dashmate/.dashmate/config.json ]]; then
     # HP masternode: dashmate status as the dashmate user
-    sudo -u dashmate dashmate status 2>&1 || true
+    sudo -u dashmate dashmate status 2>&1
+    echo "===TENDERDASH==="
+    # Query Tenderdash RPC for proposer info (localhost only, no sudo needed)
+    python3 -c '
+import json, urllib.request
+try:
+    def fetch(path):
+        return json.loads(urllib.request.urlopen(
+            "http://127.0.0.1:36657" + path, timeout=5
+        ).read())
+    validators = fetch("/validators?per_page=100")
+    sorted_ptx = sorted(v["pro_tx_hash"] for v in validators["validators"])
+    block = fetch("/block")
+    header = block["block"]["header"]
+    cur_prop = header["proposer_pro_tx_hash"]
+    height = int(header["height"])
+    idx = sorted_ptx.index(cur_prop)
+    next_prop = sorted_ptx[(idx + 1) % len(sorted_ptx)]
+    print(json.dumps({"currentProposer": cur_prop,
+                       "nextProposer": next_prop, "platformHeight": height}))
+except Exception as e:
+    print(json.dumps({"error": str(e)}))
+' 2>/dev/null || echo '{"error":"tenderdash-unavailable"}'
     echo "===SYSMETRICS==="
 else
     # Regular masternode: dash-cli as the ubuntu user

ansible/roles/status_monitoring/files/dashmon-sudoers

@@ -0,0 +1,10 @@
+# /etc/sudoers.d/dashmon
+# Allow dashmon user to run read-only monitoring commands only.
+# Both rule sets present on all nodes; unused rules are harmless.
+
+# HP masternodes: dashmate status as dashmate user
+dashmon ALL=(dashmate) NOPASSWD: /usr/bin/dashmate status
+
+# Regular masternodes: dash-cli commands as ubuntu user
+dashmon ALL=(ubuntu) NOPASSWD: /usr/local/bin/dash-cli getblockchaininfo
+dashmon ALL=(ubuntu) NOPASSWD: /usr/local/bin/dash-cli masternode status

ansible/roles/status_monitoring/files/dashmon-testnet.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOZRnc5hqc+WjCLt9PHiVVfFPfkWSlWNscOwSZrUnRAu dashmon-readonly@testnet-dashboard

ansible/roles/status_monitoring/tasks/main.yml

@@ -1,9 +1,50 @@
 ---
 
+- name: Create dashmon monitoring user
+  ansible.builtin.user:
+    name: "{{ status_monitoring_user }}"
+    shell: /bin/bash
+    password: "!"
+    create_home: true
+
+- name: Create dashmon SSH directory
+  ansible.builtin.file:
+    path: "{{ status_monitoring_home }}/.ssh"
+    state: directory
+    owner: "{{ status_monitoring_user }}"
+    group: "{{ status_monitoring_user }}"
+    mode: "0700"
+
+- name: Install dashmon authorized key with forced command
+  ansible.builtin.copy:
+    dest: "{{ status_monitoring_home }}/.ssh/authorized_keys"
+    content: >-
+      {{ status_monitoring_authorized_key_options | join(',') }}
+      {{ lookup('file', role_path + '/files/dashmon-testnet.pub') | trim }}
+    owner: "{{ status_monitoring_user }}"
+    group: "{{ status_monitoring_user }}"
+    mode: "0600"
+  vars:
+    status_monitoring_authorized_key_options:
+      - 'command="{{ status_monitoring_forced_command }}"'
+      - no-port-forwarding
+      - no-X11-forwarding
+      - no-agent-forwarding
+      - no-pty
+
 - name: Copy dashmon-check monitoring script
   ansible.builtin.copy:
     src: dashmon-check.sh
-    dest: /usr/local/bin/dashmon-check
+    dest: "{{ status_monitoring_forced_command }}"
     mode: "0755"
     owner: root
     group: root
+
+- name: Install dashmon sudoers rules
+  ansible.builtin.copy:
+    src: dashmon-sudoers
+    dest: /etc/sudoers.d/dashmon
+    mode: "0440"
+    owner: root
+    group: root
+    validate: /usr/sbin/visudo -cf %s

bin/poll-testnet-status.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+
+const {
+  pollTestnetStatus,
+} = require('../lib/testnetStatus/pollTestnetStatus');
+
+async function main() {
+  const result = await pollTestnetStatus();
+
+  console.log(`Checked ${result.expectedNodeCount} expected testnet masternodes.`);
+  console.log(`Detected ${result.incidentCount} active incidents.`);
+
+  for (const incident of result.skippedIncidents) {
+    console.log(`Skipped existing issue for ${incident.nodeName} (${incident.observedState}).`);
+  }
+
+  for (const createdIssue of result.createdIssues) {
+    if (createdIssue.dryRun) {
+      console.log(`Would create issue for ${createdIssue.nodeName} (${createdIssue.observedState}).`);
+    } else {
+      console.log(
+        `Created recovery issue for ${createdIssue.nodeName} `
+        + `(${createdIssue.observedState}): ${createdIssue.issueUrl}`,
+      );
+    }
+  }
+}
+
+main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});