feat: poll testnet status and open infra recovery issues

.github/workflows/poll-testnet-status.yml

@@ -0,0 +1,46 @@
+name: Poll Testnet Status
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */2 * * *'
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  poll-testnet-status:
+    name: Poll testnet status and open recovery issues
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    concurrency:
+      group: poll-testnet-status
+      cancel-in-progress: false
+
+    env:
+      GH_TOKEN: ${{ secrets.INFRA_ISSUES_TOKEN }}
+      TESTNET_RECOVERY_ASSIGNEE: dashinfraclaw
+      TESTNET_RECOVERY_ISSUE_REPOSITORY: dashpay/infra
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Validate cross-repo issue token
+        run: |
+          if [[ -z "${GH_TOKEN}" ]]; then
+            echo "INFRA_ISSUES_TOKEN secret is required to create issues in dashpay/infra"
+            exit 1
+          fi
+
+      - name: Poll status API and open recovery issues
+        run: node bin/poll-testnet-status.js

ansible/roles/status_dashboard/defaults/main.yml

@@ -3,5 +3,7 @@
 status_dashboard_image: dashpay/status:latest
 status_dashboard_port: 3010
 status_dashboard_path: "{{ dashd_home }}/status_dashboard"
+status_dashboard_ssh_private_key_path: "{{ lookup('env', 'STATUS_DASHBOARD_SSH_KEY_PATH') | default('~/.ssh/dashmon-testnet', true) }}"
+status_dashboard_ssh_user: dashmon
 status_dashboard_poll_interval: 10000
 status_dashboard_poll_concurrency: 20

ansible/roles/status_dashboard/tasks/main.yml

@@ -12,9 +12,9 @@
     dest: "{{ status_dashboard_path }}/inventory"
     mode: "0644"
 
-- name: Copy SSH deploy key for status dashboard
+- name: Copy SSH monitoring key for status dashboard
   ansible.builtin.copy:
-    src: "{{ lookup('env', 'PRIVATE_KEY_PATH') | default('~/.ssh/evo-app-deploy.rsa', true) }}"
+    src: "{{ status_dashboard_ssh_private_key_path }}"
     dest: "{{ status_dashboard_path }}/ssh_key"
     mode: "0600"
     owner: root

ansible/roles/status_dashboard/templates/docker-compose.yml.j2

@@ -10,7 +10,7 @@ services:
     environment:
       - INVENTORY_PATH=/app/data/inventory
       - SSH_KEY_PATH=/app/data/ssh_key
-      - SSH_USER=ubuntu
+      - SSH_USER={{ status_dashboard_ssh_user }}
       - SSH_COMMAND=/usr/local/bin/dashmon-check
       - SSH_PORT=22
       - POLL_INTERVAL_MS={{ status_dashboard_poll_interval }}

ansible/roles/status_monitoring/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+
+status_monitoring_user: dashmon
+status_monitoring_home: "/home/{{ status_monitoring_user }}"
+status_monitoring_forced_command: /usr/local/bin/dashmon-check

ansible/roles/status_monitoring/files/dashmon-check.sh

@@ -7,7 +7,29 @@ set -euo pipefail
 
 if [[ -f /home/dashmate/.dashmate/config.json ]]; then
     # HP masternode: dashmate status as the dashmate user
-    sudo -u dashmate dashmate status 2>&1 || true
+    sudo -u dashmate dashmate status 2>&1
+    echo "===TENDERDASH==="
+    # Query Tenderdash RPC for proposer info (localhost only, no sudo needed)
+    python3 -c '
+import json, urllib.request
+try:
+    def fetch(path):
+        return json.loads(urllib.request.urlopen(
+            "http://127.0.0.1:36657" + path, timeout=5
+        ).read())
+    validators = fetch("/validators?per_page=100")
+    sorted_ptx = sorted(v["pro_tx_hash"] for v in validators["validators"])
+    block = fetch("/block")
+    header = block["block"]["header"]
+    cur_prop = header["proposer_pro_tx_hash"]
+    height = int(header["height"])
+    idx = sorted_ptx.index(cur_prop)
+    next_prop = sorted_ptx[(idx + 1) % len(sorted_ptx)]
+    print(json.dumps({"currentProposer": cur_prop,
+                       "nextProposer": next_prop, "platformHeight": height}))
+except Exception as e:
+    print(json.dumps({"error": str(e)}))
+' 2>/dev/null || echo '{"error":"tenderdash-unavailable"}'
     echo "===SYSMETRICS==="
 else
     # Regular masternode: dash-cli as the ubuntu user

ansible/roles/status_monitoring/files/dashmon-sudoers

@@ -0,0 +1,10 @@
+# /etc/sudoers.d/dashmon
+# Allow dashmon user to run read-only monitoring commands only.
+# Both rule sets present on all nodes; unused rules are harmless.
+
+# HP masternodes: dashmate status as dashmate user
+dashmon ALL=(dashmate) NOPASSWD: /usr/bin/dashmate status
+
+# Regular masternodes: dash-cli commands as ubuntu user
+dashmon ALL=(ubuntu) NOPASSWD: /usr/local/bin/dash-cli getblockchaininfo
+dashmon ALL=(ubuntu) NOPASSWD: /usr/local/bin/dash-cli masternode status

ansible/roles/status_monitoring/files/dashmon-testnet.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOZRnc5hqc+WjCLt9PHiVVfFPfkWSlWNscOwSZrUnRAu dashmon-readonly@testnet-dashboard

ansible/roles/status_monitoring/tasks/main.yml

@@ -1,9 +1,50 @@
 ---
 
+- name: Create dashmon monitoring user
+  ansible.builtin.user:
+    name: "{{ status_monitoring_user }}"
+    shell: /bin/bash
+    password: "!"
+    create_home: true
+
+- name: Create dashmon SSH directory
+  ansible.builtin.file:
+    path: "{{ status_monitoring_home }}/.ssh"
+    state: directory
+    owner: "{{ status_monitoring_user }}"
+    group: "{{ status_monitoring_user }}"
+    mode: "0700"
+
+- name: Install dashmon authorized key with forced command
+  ansible.builtin.copy:
+    dest: "{{ status_monitoring_home }}/.ssh/authorized_keys"
+    content: >-
+      {{ status_monitoring_authorized_key_options | join(',') }}
+      {{ lookup('file', role_path + '/files/dashmon-testnet.pub') | trim }}
+    owner: "{{ status_monitoring_user }}"
+    group: "{{ status_monitoring_user }}"
+    mode: "0600"
+  vars:
+    status_monitoring_authorized_key_options:
+      - 'command="{{ status_monitoring_forced_command }}"'
+      - no-port-forwarding
+      - no-X11-forwarding
+      - no-agent-forwarding
+      - no-pty
+
 - name: Copy dashmon-check monitoring script
   ansible.builtin.copy:
     src: dashmon-check.sh
-    dest: /usr/local/bin/dashmon-check
+    dest: "{{ status_monitoring_forced_command }}"
     mode: "0755"
     owner: root
     group: root
+
+- name: Install dashmon sudoers rules
+  ansible.builtin.copy:
+    src: dashmon-sudoers
+    dest: /etc/sudoers.d/dashmon
+    mode: "0440"
+    owner: root
+    group: root
+    validate: /usr/sbin/visudo -cf %s

bin/poll-testnet-status.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+
+const {
+  pollTestnetStatus,
+} = require('../lib/testnetStatus/pollTestnetStatus');
+
+async function main() {
+  const result = await pollTestnetStatus();
+
+  console.log(`Checked ${result.expectedNodeCount} expected testnet masternodes.`);
+  console.log(`Detected ${result.incidentCount} active incidents.`);
+
+  for (const incident of result.skippedIncidents) {
+    console.log(`Skipped existing issue for ${incident.nodeName} (${incident.observedState}).`);
+  }
+
+  for (const createdIssue of result.createdIssues) {
+    if (createdIssue.dryRun) {
+      console.log(`Would create issue for ${createdIssue.nodeName} (${createdIssue.observedState}).`);
+    } else {
+      console.log(
+        `Created recovery issue for ${createdIssue.nodeName} `
+        + `(${createdIssue.observedState}): ${createdIssue.issueUrl}`,
+      );
+    }
+  }
+}
+
+main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});

flake.nix

@@ -0,0 +1,102 @@
+{
+  description = "FHS development environment for dash-network-deploy";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs }:
+    let
+      system = "x86_64-linux";
+      pkgs = nixpkgs.legacyPackages.${system};
+
+      fhsEnv = pkgs.buildFHSEnv {
+        name = "dash-network-deploy-env";
+        targetPkgs = pkgs: with pkgs; [
+          # Node.js
+          nodejs_22
+          corepack_22
+
+          # Infrastructure
+          terraform
+          ansible
+          docker-client
+
+          # Python 3 (Ansible interpreter at /usr/bin/python3)
+          python3
+
+          # AWS
+          awscli2
+
+          # Network / VPN
+          openssh
+          openvpn
+          curl
+          wget
+
+          # Utilities
+          git
+          jq
+          bash
+          coreutils
+          gnugrep
+          gnused
+          gawk
+          findutils
+          gnutar
+          gzip
+          which
+
+          # Libraries for native node modules
+          stdenv.cc.cc.lib
+          openssl
+          zlib
+          cacert
+        ];
+        profile = ''
+          # Ensure Python 3 is at /usr/bin/python3 for Ansible
+          if [ ! -e /usr/bin/python3 ]; then
+            mkdir -p /usr/bin 2>/dev/null || true
+            ln -sf "$(command -v python3)" /usr/bin/python3 2>/dev/null || true
+          fi
+
+          # SSL certs
+          export SSL_CERT_FILE="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+          export NIX_SSL_CERT_FILE="$SSL_CERT_FILE"
+
+          # Pass through SSH agent
+          export SSH_AUTH_SOCK="''${SSH_AUTH_SOCK:-}"
+
+          # Docker socket
+          export DOCKER_HOST="unix:///var/run/docker.sock"
+        '';
+        runScript = pkgs.writeShellScript "dash-network-deploy-run" ''
+          cd "$HOME/code/dash-network-deploy" || exit 1
+          if [ $# -eq 0 ]; then
+            echo "Entered dash-network-deploy FHS environment"
+            echo "Working directory: $(pwd)"
+            exec bash
+          else
+            exec "$@"
+          fi
+        '';
+      };
+    in
+    {
+      packages.${system}.default = fhsEnv;
+
+      # `nix develop` drops you into the FHS env
+      devShells.${system}.default = pkgs.mkShell {
+        buildInputs = [ fhsEnv ];
+        shellHook = ''
+          echo "Run 'dash-network-deploy-env' to enter the FHS environment"
+        '';
+      };
+
+      # `nix run` launches the FHS env directly
+      apps.${system}.default = {
+        type = "app";
+        program = "${fhsEnv}/bin/dash-network-deploy-env";
+      };
+    };
+}