scripts: add validator automation tools

scripts/README.md

@@ -1,37 +1,77 @@
-## State Sync Bootstrapping
+# Republic AI - Validator Automation Scripts
 
-### [statesync.sh](https://github.com/RepublicAI/networks/blob/2460cda4a2a2258b1d0ae289a6316847acc3ec8e/scripts/statesync.sh)
+A collection of production-ready automation scripts for Republic AI validators.
 
-Configures a `republicd` node to use state sync instead of replaying the full chain.
+## Scripts
 
-**Prerequisites**
-- `republicd init <moniker>` has been run
-- `curl`, `jq`, `sed` installed
+### full-auto.sh
+Complete job automation script that handles the full compute workflow.
 
-### Run after copying or downloading the script
+**Features:**
+- Automatic job submission and result submission
+- GPU inference via Docker with 60s timeout
+- Thermal protection (75°C → 90s wait, 80°C → 3min, 85°C → 5min)
+- Network health check with auto-retry
+- Bech32 address bug fix (rai → raivaloper prefix)
+- Sequence mismatch protection (15s delay between TXs)
 
-```sh
-# Default home (~/.republic)
-./scripts/statesync.sh
+**Usage:**
+```bash
+# Edit configuration variables at the top of the script
+nano full-auto.sh
 
-# Custom home directory
-./scripts/statesync.sh --home /opt/republic
+# Run
+chmod +x full-auto.sh
+nohup ./full-auto.sh >> ~/full-auto.log 2>&1 &
+```
+
+---
+
+### watchdog.sh
+Monitors full-auto.sh and automatically restarts it if it crashes.
 
-# Optional: wipe existing state
-./scripts/statesync.sh --reset
-````
+**Usage:**
+```bash
+chmod +x watchdog.sh
+nohup ./watchdog.sh >> ~/watchdog.log 2>&1 &
+```
 
-### Run directly without cloning
+---
 
-```sh
-bash <(curl -sS https://raw.githubusercontent.com/RepublicAI/networks/main/scripts/statesync.sh)
+### unjail.sh
+Monitors validator jail status every 5 minutes and auto-unjails.
 
-# With flags
-bash <(curl -sS https://raw.githubusercontent.com/RepublicAI/networks/main/scripts/statesync.sh) --reset
+**Usage:**
+```bash
+chmod +x unjail.sh
+nohup ./unjail.sh >> ~/unjail.log 2>&1 &
 ```
 
-After running, start the node:
+---
+
+### monitor.sh
+Real-time dashboard showing node, validator, GPU, and job status.
 
-```sh
-republicd start
+**Usage:**
+```bash
+chmod +x monitor.sh
+./monitor.sh
 ```
+
+## Requirements
+
+- Ubuntu 22.04 / 24.04 (or WSL2)
+- NVIDIA GPU with CUDA support
+- Docker with NVIDIA Container Toolkit
+- `jq`, `curl`, `python3`
+- `bech32` Python library: `pip install bech32`
+
+## Tested On
+
+- Ubuntu 24.04 LTS + WSL2 (Windows 11)
+- NVIDIA RTX 4050 Laptop GPU (6GB)
+- Republic AI testnet v0.3.0
+
+## Author
+
+[@erhnysr](https://github.com/erhnysr)

scripts/full-auto.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+# Republic AI - Full Auto Compute Script
+# Automatically submits jobs, runs GPU inference, and submits results
+# GitHub: https://github.com/erhnysr/republic-ai-node
+
+# ─── CONFIGURATION ───────────────────────────────────────────────
+VALOPER="YOUR_VALOPER_ADDRESS"
+WALLET="YOUR_WALLET_ADDRESS"
+NODE="tcp://localhost:43657"
+CHAIN_ID="raitestnet_77701-1"
+SERVER_IP="YOUR_SERVER_IP_OR_CLOUDFLARE_TUNNEL"
+JOBS_DIR="/var/lib/republic/jobs"
+JOB_FEE="5000000000000000arai"
+# ─────────────────────────────────────────────────────────────────
+
+echo "Republic AI Full Auto started..."
+echo "Validator: $VALOPER"
+echo "Node: $NODE"
+
+while true; do
+  # Network check
+  BLOCK=$(curl -s http://localhost:43657/status 2>/dev/null | \
+    jq -r ".result.sync_info.latest_block_height" 2>/dev/null)
+  if [ -z "$BLOCK" ] || [ "$BLOCK" = "null" ]; then
+    echo "[$(date '+%H:%M:%S')] Network down, waiting 60s..."
+    sleep 60
+    continue
+  fi
+  echo "[$(date '+%H:%M:%S')] Block: $BLOCK"
+
+  # Thermal protection
+  TEMP=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
+  if [ -n "$TEMP" ]; then
+    echo "[$(date '+%H:%M:%S')] GPU Temp: ${TEMP}C"
+    if [ "$TEMP" -ge 85 ]; then
+      echo "CRITICAL: ${TEMP}C - cooling down 5 minutes..."
+      sleep 300; continue
+    elif [ "$TEMP" -ge 80 ]; then
+      echo "HOT: ${TEMP}C - cooling down 3 minutes..."
+      sleep 180; continue
+    elif [ "$TEMP" -ge 75 ]; then
+      echo "WARM: ${TEMP}C - slowing down..."
+      WAIT=90
+    else
+      WAIT=30
+    fi
+  else
+    WAIT=30
+  fi
+
+  # Submit job
+  echo "[$(date '+%H:%M:%S')] Submitting job..."
+  TX=$(republicd tx computevalidation submit-job \
+    $VALOPER \
+    republic-llm-inference:latest \
+    https://$SERVER_IP/upload \
+    https://$SERVER_IP/result \
+    example-verification:latest \
+    $JOB_FEE \
+    --from validator \
+    --home $HOME/.republicd \
+    --chain-id $CHAIN_ID \
+    --gas auto \
+    --gas-adjustment 1.5 \
+    --gas-prices 1000000000arai \
+    --node $NODE \
+    --keyring-backend test \
+    -y 2>/dev/null | grep txhash | awk '{print $2}')
+  echo "[$(date '+%H:%M:%S')] TX: $TX"
+
+  if [ -z "$TX" ]; then
+    echo "TX empty, network issue. Waiting 30s..."
+    sleep 30; continue
+  fi
+
+  sleep 15
+
+  # Get Job ID
+  JOB_ID=$(republicd query tx $TX --node $NODE -o json 2>/dev/null | \
+    jq -r '.events[] | select(.type=="job_submitted") | .attributes[] | select(.key=="job_id") | .value')
+  echo "[$(date '+%H:%M:%S')] Job ID: $JOB_ID"
+
+  if [ -z "$JOB_ID" ]; then
+    echo "Job ID not found, skipping..."
+    sleep 30; continue
+  fi
+
+  # Run GPU inference
+  RESULT_FILE="$JOBS_DIR/$JOB_ID/result.bin"
+  mkdir -p $JOBS_DIR/$JOB_ID
+
+  timeout 60 docker run --rm --gpus all \
+    -v $JOBS_DIR/$JOB_ID:/output \
+    republic-llm-inference:latest 2>/dev/null
+
+  if [ $? -ne 0 ]; then
+    echo "Docker error for job $JOB_ID, skipping..."
+    sleep 30; continue
+  fi
+
+  echo "[$(date '+%H:%M:%S')] Inference done for job $JOB_ID"
+
+  # Submit result (with bech32 fix)
+  if [ -f "$RESULT_FILE" ]; then
+    SHA256=$(sha256sum $RESULT_FILE | awk '{print $1}')
+
+    republicd tx computevalidation submit-job-result \
+      $JOB_ID \
+      https://$SERVER_IP/$JOB_ID/result.bin \
+      example-verification:latest \
+      $SHA256 \
+      --from validator \
+      --home $HOME/.republicd \
+      --chain-id $CHAIN_ID \
+      --gas 300000 \
+      --gas-prices 1000000000arai \
+      --node $NODE \
+      --keyring-backend test \
+      --generate-only 2>/dev/null > /tmp/tx_unsigned.json
+
+    # Fix bech32 address bug
+    python3 -c "
+import bech32, json
+tx = json.load(open('/tmp/tx_unsigned.json'))
+_, data = bech32.bech32_decode('$WALLET')
+valoper = bech32.bech32_encode('raivaloper', data)
+tx['body']['messages'][0]['validator'] = valoper
+json.dump(tx, open('/tmp/tx_unsigned.json', 'w'))
+print('Bech32 fix applied:', valoper)
+"
+    republicd tx sign /tmp/tx_unsigned.json \
+      --from validator \
+      --home $HOME/.republicd \
+      --chain-id $CHAIN_ID \
+      --node $NODE \
+      --keyring-backend test \
+      --output-document /tmp/tx_signed.json 2>/dev/null
+
+    republicd tx broadcast /tmp/tx_signed.json \
+      --node $NODE \
+      --chain-id $CHAIN_ID 2>/dev/null | grep txhash | \
+      awk "{print \"[$(date '+%H:%M:%S')] Job $JOB_ID submitted! TX: \"\$2}"
+
+    sleep 15
+  fi
+
+  echo "[$(date '+%H:%M:%S')] Waiting ${WAIT}s..."
+  sleep $WAIT
+done

scripts/monitor.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Republic AI - Validator Health Monitor
+# Displays real-time validator and node statistics
+
+VALOPER="YOUR_VALOPER_ADDRESS"
+NODE="tcp://localhost:43657"
+
+clear
+echo "╔═══════════════════════════════════════════╗"
+echo "║     Republic AI Validator Monitor          ║"
+echo "╚═══════════════════════════════════════════╝"
+echo ""
+
+# Node sync status
+echo "── NODE STATUS ─────────────────────────────"
+republicd status --node $NODE 2>/dev/null | jq '{
+  block: .sync_info.latest_block_height,
+  catching_up: .sync_info.catching_up,
+  peers: .node_info.other.rpc_address
+}' 2>/dev/null || echo "Node not responding"
+
+echo ""
+echo "── VALIDATOR STATUS ─────────────────────────"
+republicd query staking validator $VALOPER \
+  --node $NODE -o json 2>/dev/null | jq '{
+  moniker: .validator.description.moniker,
+  status: .validator.status,
+  jailed: .validator.jailed,
+  tokens: (.validator.tokens | tonumber / 1e18 | floor | tostring) + " RAI",
+  commission: .validator.commission.commission_rates.rate
+}' 2>/dev/null || echo "Validator not found"
+
+echo ""
+echo "── GPU STATUS ───────────────────────────────"
+nvidia-smi --query-gpu=name,temperature.gpu,utilization.gpu,memory.used,memory.total \
+  --format=csv,noheader 2>/dev/null || echo "No GPU detected"
+
+echo ""
+echo "── JOBS (last 10 minutes) ───────────────────"
+if [ -f "$HOME/full-auto.log" ]; then
+  grep "Job.*submitted! TX\|Inference done\|TX bos\|Docker error" \
+    $HOME/full-auto.log | tail -10
+else
+  echo "No job log found"
+fi
+
+echo ""
+echo "── PROCESS STATUS ───────────────────────────"
+pgrep -f "full-auto.sh" > /dev/null && echo "✅ full-auto.sh: RUNNING" || echo "❌ full-auto.sh: STOPPED"
+pgrep -f "watchdog.sh" > /dev/null && echo "✅ watchdog.sh: RUNNING" || echo "❌ watchdog.sh: STOPPED"
+pgrep -f "http.server" > /dev/null && echo "✅ HTTP server: RUNNING" || echo "❌ HTTP server: STOPPED"
+pgrep -f "republicd" > /dev/null && echo "✅ republicd: RUNNING" || echo "❌ republicd: STOPPED"

scripts/unjail.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Republic AI - Auto Unjail Script
+# Checks if validator is jailed and automatically unjails
+
+VALOPER="YOUR_VALOPER_ADDRESS"
+WALLET="YOUR_WALLET_ADDRESS"
+NODE="tcp://localhost:43657"
+CHAIN_ID="raitestnet_77701-1"
+CHECK_INTERVAL=300  # Check every 5 minutes
+
+echo "Auto-unjail monitor started..."
+echo "Validator: $VALOPER"
+
+while true; do
+  JAILED=$(republicd query staking validator $VALOPER \
+    --node $NODE -o json 2>/dev/null | jq -r '.validator.jailed')
+
+  if [ "$JAILED" = "true" ]; then
+    echo "[$(date '+%H:%M:%S')] Validator is JAILED! Attempting unjail..."
+    republicd tx slashing unjail \
+      --from validator \
+      --home $HOME/.republicd \
+      --chain-id $CHAIN_ID \
+      --gas auto \
+      --gas-adjustment 1.5 \
+      --gas-prices 1000000000arai \
+      --node $NODE \
+      --keyring-backend test \
+      -y 2>/dev/null | grep txhash | awk '{print "[$(date)] Unjail TX: "$2}'
+    echo "[$(date '+%H:%M:%S')] Unjail transaction sent!"
+  else
+    echo "[$(date '+%H:%M:%S')] Validator status: OK (not jailed)"
+  fi
+
+  sleep $CHECK_INTERVAL
+done

scripts/watchdog.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Republic AI - Watchdog Script
+# Monitors full-auto.sh and restarts it if it stops
+
+SCRIPT_PATH="$HOME/full-auto.sh"
+LOG_PATH="$HOME/full-auto.log"
+
+echo "Watchdog started. Monitoring: $SCRIPT_PATH"
+
+while true; do
+  if ! pgrep -f "full-auto.sh" > /dev/null; then
+    echo "[$(date '+%H:%M:%S')] full-auto.sh stopped! Restarting..."
+    nohup $SCRIPT_PATH >> $LOG_PATH 2>&1 &
+    echo "[$(date '+%H:%M:%S')] Restarted! PID: $!"
+  fi
+  sleep 30
+done