From 8a22d7787138a501401d515c3cecbb35d2540437 Mon Sep 17 00:00:00 2001 From: erhnysr Date: Mon, 2 Mar 2026 04:20:32 +0300 Subject: [PATCH 1/3] Add troubleshooting guide for common node setup issues --- testnet/TROUBLESHOOTING.md | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 testnet/TROUBLESHOOTING.md diff --git a/testnet/TROUBLESHOOTING.md b/testnet/TROUBLESHOOTING.md new file mode 100644 index 0000000..1f04661 --- /dev/null +++ b/testnet/TROUBLESHOOTING.md @@ -0,0 +1,44 @@ +# Republic AI Node - Troubleshooting Guide + +Common issues during Republic AI testnet node setup on WSL. + +## 1. Wrong RPC port +Default port is 26657 but Republic uses 43657. +Check your port: grep "laddr" ~/.republicd/config/config.toml +Use: republicd status --node tcp://localhost:43657 + +## 2. Block height stuck at 0 +State sync trust_height is outdated. Use snapshot instead: +curl -L https://snapshot.vinjan-inc.com/republic/latest.tar.lz4 | lz4 -dc - | tar -xf - -C $HOME/.republicd + +## 3. Wrong app version error +Download v0.3.0 binary: +curl -L https://github.com/RepublicAI/networks/releases/download/v0.3.0/republicd-linux-amd64 -o republicd +cp republicd ~/.republicd/cosmovisor/genesis/bin/republicd + +## 4. Wrong home directory +Use ~/.republicd not ~/.republic in all commands. + +## 5. Database locked error +pkill -9 -f republicd && pkill -9 -f cosmovisor +Or: sudo systemctl stop republicd + +## 6. Cosmovisor DAEMON_NAME not set +DAEMON_NAME=republicd DAEMON_HOME=$HOME/.republicd cosmovisor run start --home ~/.republicd --chain-id raitestnet_77701-1 + +## 7. Systemd service setup (WSL) +sudo tee /etc/systemd/system/republicd.service << EOF +[Unit] +Description=Republic Node +After=network-online.target +[Service] +User=$USER +Environment="DAEMON_NAME=republicd" +Environment="DAEMON_HOME=$HOME/.republicd" +ExecStart=/home/$USER/go/bin/cosmovisor run start --home /home/$USER/.republicd --chain-id raitestnet_77701-1 +Restart=always +RestartSec=3 +[Install] +WantedBy=multi-user.target +EOF +sudo systemctl daemon-reload && sudo systemctl enable republicd && sudo systemctl start republicd From d0b5ea3a95c3c1704e16f024843aa39d21bf0e9d Mon Sep 17 00:00:00 2001 From: erhnysr Date: Thu, 12 Mar 2026 12:48:54 +0300 Subject: [PATCH 2/3] scripts: add validator automation tools (full-auto, watchdog, unjail, monitor) --- scripts/full-auto.sh | 149 +++++++++++++++++++++++++++++++++++++++++++ scripts/monitor.sh | 52 +++++++++++++++ scripts/unjail.sh | 36 +++++++++++ scripts/watchdog.sh | 17 +++++ 4 files changed, 254 insertions(+) create mode 100755 scripts/full-auto.sh create mode 100755 scripts/monitor.sh create mode 100755 scripts/unjail.sh create mode 100755 scripts/watchdog.sh diff --git a/scripts/full-auto.sh b/scripts/full-auto.sh new file mode 100755 index 0000000..22cfd73 --- /dev/null +++ b/scripts/full-auto.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# Republic AI - Full Auto Compute Script +# Automatically submits jobs, runs GPU inference, and submits results +# GitHub: https://github.com/erhnysr/republic-ai-node + +# ─── CONFIGURATION ─────────────────────────────────────────────── +VALOPER="YOUR_VALOPER_ADDRESS" +WALLET="YOUR_WALLET_ADDRESS" +NODE="tcp://localhost:43657" +CHAIN_ID="raitestnet_77701-1" +SERVER_IP="YOUR_SERVER_IP_OR_CLOUDFLARE_TUNNEL" +JOBS_DIR="/var/lib/republic/jobs" +JOB_FEE="5000000000000000arai" +# ───────────────────────────────────────────────────────────────── + +echo "Republic AI Full Auto started..." +echo "Validator: $VALOPER" +echo "Node: $NODE" + +while true; do + # Network check + BLOCK=$(curl -s http://localhost:43657/status 2>/dev/null | \ + jq -r ".result.sync_info.latest_block_height" 2>/dev/null) + if [ -z "$BLOCK" ] || [ "$BLOCK" = "null" ]; then + echo "[$(date '+%H:%M:%S')] Network down, waiting 60s..." + sleep 60 + continue + fi + echo "[$(date '+%H:%M:%S')] Block: $BLOCK" + + # Thermal protection + TEMP=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null) + if [ -n "$TEMP" ]; then + echo "[$(date '+%H:%M:%S')] GPU Temp: ${TEMP}C" + if [ "$TEMP" -ge 85 ]; then + echo "CRITICAL: ${TEMP}C - cooling down 5 minutes..." + sleep 300; continue + elif [ "$TEMP" -ge 80 ]; then + echo "HOT: ${TEMP}C - cooling down 3 minutes..." + sleep 180; continue + elif [ "$TEMP" -ge 75 ]; then + echo "WARM: ${TEMP}C - slowing down..." + WAIT=90 + else + WAIT=30 + fi + else + WAIT=30 + fi + + # Submit job + echo "[$(date '+%H:%M:%S')] Submitting job..." + TX=$(republicd tx computevalidation submit-job \ + $VALOPER \ + republic-llm-inference:latest \ + https://$SERVER_IP/upload \ + https://$SERVER_IP/result \ + example-verification:latest \ + $JOB_FEE \ + --from validator \ + --home $HOME/.republicd \ + --chain-id $CHAIN_ID \ + --gas auto \ + --gas-adjustment 1.5 \ + --gas-prices 1000000000arai \ + --node $NODE \ + --keyring-backend test \ + -y 2>/dev/null | grep txhash | awk '{print $2}') + echo "[$(date '+%H:%M:%S')] TX: $TX" + + if [ -z "$TX" ]; then + echo "TX empty, network issue. Waiting 30s..." + sleep 30; continue + fi + + sleep 15 + + # Get Job ID + JOB_ID=$(republicd query tx $TX --node $NODE -o json 2>/dev/null | \ + jq -r '.events[] | select(.type=="job_submitted") | .attributes[] | select(.key=="job_id") | .value') + echo "[$(date '+%H:%M:%S')] Job ID: $JOB_ID" + + if [ -z "$JOB_ID" ]; then + echo "Job ID not found, skipping..." + sleep 30; continue + fi + + # Run GPU inference + RESULT_FILE="$JOBS_DIR/$JOB_ID/result.bin" + mkdir -p $JOBS_DIR/$JOB_ID + + timeout 60 docker run --rm --gpus all \ + -v $JOBS_DIR/$JOB_ID:/output \ + republic-llm-inference:latest 2>/dev/null + + if [ $? -ne 0 ]; then + echo "Docker error for job $JOB_ID, skipping..." + sleep 30; continue + fi + + echo "[$(date '+%H:%M:%S')] Inference done for job $JOB_ID" + + # Submit result (with bech32 fix) + if [ -f "$RESULT_FILE" ]; then + SHA256=$(sha256sum $RESULT_FILE | awk '{print $1}') + + republicd tx computevalidation submit-job-result \ + $JOB_ID \ + https://$SERVER_IP/$JOB_ID/result.bin \ + example-verification:latest \ + $SHA256 \ + --from validator \ + --home $HOME/.republicd \ + --chain-id $CHAIN_ID \ + --gas 300000 \ + --gas-prices 1000000000arai \ + --node $NODE \ + --keyring-backend test \ + --generate-only 2>/dev/null > /tmp/tx_unsigned.json + + # Fix bech32 address bug + python3 -c " +import bech32, json +tx = json.load(open('/tmp/tx_unsigned.json')) +_, data = bech32.bech32_decode('$WALLET') +valoper = bech32.bech32_encode('raivaloper', data) +tx['body']['messages'][0]['validator'] = valoper +json.dump(tx, open('/tmp/tx_unsigned.json', 'w')) +print('Bech32 fix applied:', valoper) +" + republicd tx sign /tmp/tx_unsigned.json \ + --from validator \ + --home $HOME/.republicd \ + --chain-id $CHAIN_ID \ + --node $NODE \ + --keyring-backend test \ + --output-document /tmp/tx_signed.json 2>/dev/null + + republicd tx broadcast /tmp/tx_signed.json \ + --node $NODE \ + --chain-id $CHAIN_ID 2>/dev/null | grep txhash | \ + awk "{print \"[$(date '+%H:%M:%S')] Job $JOB_ID submitted! TX: \"\$2}" + + sleep 15 + fi + + echo "[$(date '+%H:%M:%S')] Waiting ${WAIT}s..." + sleep $WAIT +done diff --git a/scripts/monitor.sh b/scripts/monitor.sh new file mode 100755 index 0000000..474fbf4 --- /dev/null +++ b/scripts/monitor.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Republic AI - Validator Health Monitor +# Displays real-time validator and node statistics + +VALOPER="YOUR_VALOPER_ADDRESS" +NODE="tcp://localhost:43657" + +clear +echo "╔═══════════════════════════════════════════╗" +echo "║ Republic AI Validator Monitor ║" +echo "╚═══════════════════════════════════════════╝" +echo "" + +# Node sync status +echo "── NODE STATUS ─────────────────────────────" +republicd status --node $NODE 2>/dev/null | jq '{ + block: .sync_info.latest_block_height, + catching_up: .sync_info.catching_up, + peers: .node_info.other.rpc_address +}' 2>/dev/null || echo "Node not responding" + +echo "" +echo "── VALIDATOR STATUS ─────────────────────────" +republicd query staking validator $VALOPER \ + --node $NODE -o json 2>/dev/null | jq '{ + moniker: .validator.description.moniker, + status: .validator.status, + jailed: .validator.jailed, + tokens: (.validator.tokens | tonumber / 1e18 | floor | tostring) + " RAI", + commission: .validator.commission.commission_rates.rate +}' 2>/dev/null || echo "Validator not found" + +echo "" +echo "── GPU STATUS ───────────────────────────────" +nvidia-smi --query-gpu=name,temperature.gpu,utilization.gpu,memory.used,memory.total \ + --format=csv,noheader 2>/dev/null || echo "No GPU detected" + +echo "" +echo "── JOBS (last 10 minutes) ───────────────────" +if [ -f "$HOME/full-auto.log" ]; then + grep "Job.*submitted! TX\|Inference done\|TX bos\|Docker error" \ + $HOME/full-auto.log | tail -10 +else + echo "No job log found" +fi + +echo "" +echo "── PROCESS STATUS ───────────────────────────" +pgrep -f "full-auto.sh" > /dev/null && echo "✅ full-auto.sh: RUNNING" || echo "❌ full-auto.sh: STOPPED" +pgrep -f "watchdog.sh" > /dev/null && echo "✅ watchdog.sh: RUNNING" || echo "❌ watchdog.sh: STOPPED" +pgrep -f "http.server" > /dev/null && echo "✅ HTTP server: RUNNING" || echo "❌ HTTP server: STOPPED" +pgrep -f "republicd" > /dev/null && echo "✅ republicd: RUNNING" || echo "❌ republicd: STOPPED" diff --git a/scripts/unjail.sh b/scripts/unjail.sh new file mode 100755 index 0000000..756e070 --- /dev/null +++ b/scripts/unjail.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Republic AI - Auto Unjail Script +# Checks if validator is jailed and automatically unjails + +VALOPER="YOUR_VALOPER_ADDRESS" +WALLET="YOUR_WALLET_ADDRESS" +NODE="tcp://localhost:43657" +CHAIN_ID="raitestnet_77701-1" +CHECK_INTERVAL=300 # Check every 5 minutes + +echo "Auto-unjail monitor started..." +echo "Validator: $VALOPER" + +while true; do + JAILED=$(republicd query staking validator $VALOPER \ + --node $NODE -o json 2>/dev/null | jq -r '.validator.jailed') + + if [ "$JAILED" = "true" ]; then + echo "[$(date '+%H:%M:%S')] Validator is JAILED! Attempting unjail..." + republicd tx slashing unjail \ + --from validator \ + --home $HOME/.republicd \ + --chain-id $CHAIN_ID \ + --gas auto \ + --gas-adjustment 1.5 \ + --gas-prices 1000000000arai \ + --node $NODE \ + --keyring-backend test \ + -y 2>/dev/null | grep txhash | awk '{print "[$(date)] Unjail TX: "$2}' + echo "[$(date '+%H:%M:%S')] Unjail transaction sent!" + else + echo "[$(date '+%H:%M:%S')] Validator status: OK (not jailed)" + fi + + sleep $CHECK_INTERVAL +done diff --git a/scripts/watchdog.sh b/scripts/watchdog.sh new file mode 100755 index 0000000..16f1ada --- /dev/null +++ b/scripts/watchdog.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Republic AI - Watchdog Script +# Monitors full-auto.sh and restarts it if it stops + +SCRIPT_PATH="$HOME/full-auto.sh" +LOG_PATH="$HOME/full-auto.log" + +echo "Watchdog started. Monitoring: $SCRIPT_PATH" + +while true; do + if ! pgrep -f "full-auto.sh" > /dev/null; then + echo "[$(date '+%H:%M:%S')] full-auto.sh stopped! Restarting..." + nohup $SCRIPT_PATH >> $LOG_PATH 2>&1 & + echo "[$(date '+%H:%M:%S')] Restarted! PID: $!" + fi + sleep 30 +done From 5d39365cf740e1e75af72b93e80901174499dccc Mon Sep 17 00:00:00 2001 From: erhnysr Date: Thu, 12 Mar 2026 12:56:43 +0300 Subject: [PATCH 3/3] scripts: add README with usage instructions --- scripts/README.md | 86 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 23 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 6852c03..a3c5d47 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,37 +1,77 @@ -## State Sync Bootstrapping +# Republic AI - Validator Automation Scripts -### [statesync.sh](https://github.com/RepublicAI/networks/blob/2460cda4a2a2258b1d0ae289a6316847acc3ec8e/scripts/statesync.sh) +A collection of production-ready automation scripts for Republic AI validators. -Configures a `republicd` node to use state sync instead of replaying the full chain. +## Scripts -**Prerequisites** -- `republicd init ` has been run -- `curl`, `jq`, `sed` installed +### full-auto.sh +Complete job automation script that handles the full compute workflow. -### Run after copying or downloading the script +**Features:** +- Automatic job submission and result submission +- GPU inference via Docker with 60s timeout +- Thermal protection (75°C → 90s wait, 80°C → 3min, 85°C → 5min) +- Network health check with auto-retry +- Bech32 address bug fix (rai → raivaloper prefix) +- Sequence mismatch protection (15s delay between TXs) -```sh -# Default home (~/.republic) -./scripts/statesync.sh +**Usage:** +```bash +# Edit configuration variables at the top of the script +nano full-auto.sh -# Custom home directory -./scripts/statesync.sh --home /opt/republic +# Run +chmod +x full-auto.sh +nohup ./full-auto.sh >> ~/full-auto.log 2>&1 & +``` + +--- + +### watchdog.sh +Monitors full-auto.sh and automatically restarts it if it crashes. -# Optional: wipe existing state -./scripts/statesync.sh --reset -```` +**Usage:** +```bash +chmod +x watchdog.sh +nohup ./watchdog.sh >> ~/watchdog.log 2>&1 & +``` -### Run directly without cloning +--- -```sh -bash <(curl -sS https://raw.githubusercontent.com/RepublicAI/networks/main/scripts/statesync.sh) +### unjail.sh +Monitors validator jail status every 5 minutes and auto-unjails. -# With flags -bash <(curl -sS https://raw.githubusercontent.com/RepublicAI/networks/main/scripts/statesync.sh) --reset +**Usage:** +```bash +chmod +x unjail.sh +nohup ./unjail.sh >> ~/unjail.log 2>&1 & ``` -After running, start the node: +--- + +### monitor.sh +Real-time dashboard showing node, validator, GPU, and job status. -```sh -republicd start +**Usage:** +```bash +chmod +x monitor.sh +./monitor.sh ``` + +## Requirements + +- Ubuntu 22.04 / 24.04 (or WSL2) +- NVIDIA GPU with CUDA support +- Docker with NVIDIA Container Toolkit +- `jq`, `curl`, `python3` +- `bech32` Python library: `pip install bech32` + +## Tested On + +- Ubuntu 24.04 LTS + WSL2 (Windows 11) +- NVIDIA RTX 4050 Laptop GPU (6GB) +- Republic AI testnet v0.3.0 + +## Author + +[@erhnysr](https://github.com/erhnysr)