From 51e3e86f09f4407ad0eedf150bf841385d8a63a2 Mon Sep 17 00:00:00 2001 From: Jeremy Cooper Date: Sun, 18 Jul 2021 17:10:59 +0000 Subject: [PATCH 1/4] Radical redesign around JSON/RPC. Remove all CLI-based scraping code and use JSON/RPC to interact with the validator programmatically, instead. Also remove all queries to the Helium API; they can be satisfied locally and likely more accurately. --- miner_exporter.py | 527 ++++++++++++++-------------------------------- miner_jsonrpc.py | 84 ++++++++ requirements.txt | 3 - 3 files changed, 239 insertions(+), 375 deletions(-) create mode 100755 miner_jsonrpc.py diff --git a/miner_exporter.py b/miner_exporter.py index cfeef73..ea7c124 100755 --- a/miner_exporter.py +++ b/miner_exporter.py @@ -2,19 +2,15 @@ # external packages import prometheus_client -import psutil -import docker import requests -import dateutil.parser # internal packages import datetime import time -import subprocess -import sys import os -import re import logging +import typing +from miner_jsonrpc import MinerJSONRPC # remember, levels: debug, info, warning, error, critical. there is no trace. logging.basicConfig(format="%(filename)s:%(funcName)s:%(lineno)d:%(levelname)s\t%(message)s", level=logging.WARNING) @@ -23,19 +19,11 @@ # time to sleep between scrapes UPDATE_PERIOD = int(os.environ.get('UPDATE_PERIOD', 30)) -VALIDATOR_CONTAINER_NAME = os.environ.get('VALIDATOR_CONTAINER_NAME', 'validator') # for testnet, https://testnet-api.helium.wtf/v1 API_BASE_URL = os.environ.get('API_BASE_URL', 'https://api.helium.io/v1') -# use the RPC calls where available. This means you have your RPC port open. -# Once all of the exec calls are replaced we can enable this by default. -ENABLE_RPC = os.environ.get('ENABLE_RPC', 0) - # prometheus exporter types Gauge,Counter,Summary,Histogram,Info and Enum SCRAPE_TIME = prometheus_client.Summary('validator_scrape_time', 'Time spent collecting miner data') -SYSTEM_USAGE = prometheus_client.Gauge('system_usage', - 'Hold current system resource usage', - ['resource_type','validator_name']) CHAIN_STATS = prometheus_client.Gauge('chain_stats', 'Stats about the global chain', ['resource_type']) VAL = prometheus_client.Gauge('validator_height', @@ -63,379 +51,174 @@ 'Version number of the miner container',['validator_name']) BALANCE = prometheus_client.Gauge('validator_api_balance', 'Balance of the validator owner account',['validator_name']) -UPTIME = prometheus_client.Gauge('validator_container_uptime', - 'Time container has been at a given state', - ['state_type','validator_name']) -miner_facts = {} - -def try_int(v): - if re.match(r"^\-?\d+$", v): - return int(v) - return v - -def try_float(v): - if re.match(r"^\-?[\d\.]+$", v): - return float(v) - return v - -def get_facts(docker_container_obj): - if miner_facts: - return miner_facts - #miner_facts = { - # 'name': None, - # 'address': None - #} - out = docker_container_obj.exec_run('miner print_keys') - # sample output: - # {pubkey,"1YBkf..."}. - # {onboarding_key,"1YBkf..."}. - # {animal_name,"one-two-three"}. - - log.debug(out.output) - printkeys = {} - for line in out.output.split(b"\n"): - strline = line.decode('utf-8') - - # := requires py3.8 - if m := re.match(r'{([^,]+),"([^"]+)"}.', strline): - log.debug(m) - k = m.group(1) - v = m.group(2) - log.debug(k,v) - printkeys[k] = v - - if v := printkeys.get('pubkey'): - miner_facts['address'] = v - if printkeys.get('animal_name'): - miner_facts['name'] = v - #$ docker exec validator miner print_keys - return miner_facts +# Last known HBBFT performance stats for this validator. +hval = {} # Decorate function with metric. @SCRAPE_TIME.time() -def stats(): - docker_container = None - - try: - dc = docker.DockerClient() - - # Try to find by specific name first - docker_container = dc.containers.get(VALIDATOR_CONTAINER_NAME) - except docker.errors.NotFound as ex: - # If find by specifc name fails, try to find by prefix - containers = dc.containers.list() - - for container in containers: - if container.name.startswith(VALIDATOR_CONTAINER_NAME): - docker_container = container - break - - # If container not found, then log error and return - if docker_container is None: - log.error(f"docker failed while bootstrapping. Not exporting anything. Error: {ex}") - return - - - miner_facts = get_facts(docker_container) - hotspot_name_str = get_miner_name(docker_container) - - # collect total cpu and memory usage. Might want to consider just the docker - # container with something like cadvisor instead - SYSTEM_USAGE.labels('CPU', hotspot_name_str).set(psutil.cpu_percent()) - SYSTEM_USAGE.labels('Memory', hotspot_name_str).set(psutil.virtual_memory()[2]) - SYSTEM_USAGE.labels('CPU-Steal', hotspot_name_str).set(psutil.cpu_times_percent().steal) - SYSTEM_USAGE.labels('Disk Used', hotspot_name_str).set(float(psutil.disk_usage('/').used) / float(psutil.disk_usage('/').total)) - SYSTEM_USAGE.labels('Disk Free', hotspot_name_str).set(float(psutil.disk_usage('/').free) / float(psutil.disk_usage('/').total)) - SYSTEM_USAGE.labels('Process-Count', hotspot_name_str).set(sum(1 for proc in psutil.process_iter())) - - collect_container_run_time(docker_container, hotspot_name_str) - collect_miner_version(docker_container, hotspot_name_str) - collect_block_age(docker_container, hotspot_name_str) - collect_miner_height(docker_container, hotspot_name_str) - collect_chain_stats() - collect_in_consensus(docker_container, hotspot_name_str) - # `miner ledger validators` fails on mainnet, 20210626 - #collect_ledger_validators(docker_container, hotspot_name_str) - collect_peer_book(docker_container, hotspot_name_str) - collect_hbbft_performance(docker_container, hotspot_name_str) - collect_balance(docker_container,miner_facts['address'],hotspot_name_str) - -def safe_get_json(url): - try: - ret = requests.get(url) - if not ret.status_code == requests.codes.ok: - log.error(f"bad status code ({ret.status_code}) from url: {url}") - return - retj = ret.json() - return retj - - - except (requests.exceptions.SSLError, requests.exceptions.ConnectionError) as ex: - log.error(f"error fetching {url}: {ex}") - return - -def collect_container_run_time(docker_container, miner_name): - attrs = docker_container.attrs - - # examples and other things we could track: - # "Created": "2021-05-18T22:11:48.962678927Z", - # "Id": "cd611b83a0f267a1000603db52aa2d21247a32cc195c9c2b8ebcade5d35cfe1a", - # "State": { - # "Status": "running", - # "Running": true, - # "Paused": false, - # "Restarting": false, - # "OOMKilled": false, - # "Dead": false, - # "Pid": 4159823, - # "ExitCode": 0, - # "Error": "", - # "StartedAt": "2021-05-18T22:11:49.50436001Z", - # "FinishedAt": "0001-01-01T00:00:00Z" - - now = datetime.datetime.now(datetime.timezone.utc) - if attrs: - if attrs.get("Created"): - create_time = attrs.get("Created") - create_dt = dateutil.parser.parse(create_time) - create_delta = (now-create_dt).total_seconds() - UPTIME.labels('create', miner_name).set(create_delta) - if attrs.get("State") and attrs["State"].get("StartedAt"): - start_time = attrs["State"]["StartedAt"] - start_dt = dateutil.parser.parse(start_time) - start_delta = (now-start_dt).total_seconds() - UPTIME.labels('start', miner_name).set(start_delta) - -def collect_chain_stats(): - api = safe_get_json(f'{API_BASE_URL}/blocks/height') - if not api: - log.error("chain height fetch returned empty JSON") - return - height_val = api['data']['height'] - CHAIN_STATS.labels('height').set(height_val) - - api = None - api = safe_get_json(f'{API_BASE_URL}/validators/stats') - if not api: - log.error("val stats stats fetch returned empty JSON") - return - count_val = api['data']['staked']['count'] - CHAIN_STATS.labels('staked_validators').set(count_val) - -def collect_balance(docker_container, addr, miner_name): - # should move pubkey to getfacts and then pass it in here - #out = docker_container.exec_run('miner print_keys') - #for line in out.output.decode('utf-8').split("\n"): - # if 'pubkey' in line: - # addr=line[9:60] - api_validators = safe_get_json(f'{API_BASE_URL}/validators/{addr}') - if not api_validators: - log.error("validator fetch returned empty JSON") - return - elif not api_validators.get('data') or not api_validators['data'].get('owner'): - log.error("could not find validator data owner in json") - return - owner = api_validators['data']['owner'] - - api_accounts = safe_get_json(f'{API_BASE_URL}/accounts/{owner}') - if not api_accounts: - return - if not api_accounts.get('data') or not api_accounts['data'].get('balance'): - return - balance = float(api_accounts['data']['balance'])/1E8 - #print(api_accounts) - #print('balance',balance) - BALANCE.labels(miner_name).set(balance) - - -def get_miner_name(docker_container): - # need to fix this. hotspot name really should only be queried once - out = docker_container.exec_run('miner info name') - log.debug(out.output) - hotspot_name = out.output.decode('utf-8').rstrip("\n") - return hotspot_name - -def collect_miner_height(docker_container, miner_name): - # grab the local blockchain height - out = docker_container.exec_run('miner info height') - log.debug(out.output) - txt = out.output.decode('utf-8').rstrip("\n") - VAL.labels('Height', miner_name).set(out.output.split()[1]) - -def collect_in_consensus(docker_container, miner_name): - # check if currently in consensus group - out = docker_container.exec_run('miner info in_consensus') - incon_txt = out.output.decode('utf-8').rstrip("\n") - incon = 0 - if incon_txt == 'true': - incon = 1 - log.info(f"in consensus? {incon} / {incon_txt}") - INCON.labels(miner_name).set(incon) - -def collect_block_age(docker_container, miner_name): - # collect current block age - out = docker_container.exec_run('miner info block_age') - ## transform into a number - age_val = try_int(out.output.decode('utf-8').rstrip("\n")) - - BLOCKAGE.labels('BlockAge', miner_name).set(age_val) - log.debug(f"age: {age_val}") - -# persist these between calls -hval = {} -def collect_hbbft_performance(docker_container, miner_name): - # parse the hbbft performance table for the penalty field - out = docker_container.exec_run('miner hbbft perf --format csv') - #print(out.output) - - for line in out.output.decode('utf-8').split("\n"): - c = [x.strip() for x in line.split(',')] - # samples: - - have_data = False - - if len(c) == 7 and miner_name == c[0]: - # name,bba_completions,seen_votes,last_bba,last_seen,tenure,penalty - # great-clear-chinchilla,5/5,237/237,0,0,2.91,2.91 - log.debug(f"resl7: {c}; {miner_name}/{c[0]}") - - (hval['bba_votes'],hval['bba_tot'])=c[1].split("/") - (hval['seen_votes'],hval['seen_tot'])=c[2].split("/") - hval['bba_last_val']=try_float(c[3]) - hval['seen_last_val']=try_float(c[4]) - hval['tenure'] = try_float(c[5]) - hval['pen_val'] = try_float(c[6]) - elif len(c) == 6 and miner_name == c[0]: - # name,bba_completions,seen_votes,last_bba,last_seen,penalty - # curly-peach-owl,11/11,368/368,0,0,1.86 - log.debug(f"resl6: {c}; {miner_name}/{c[0]}") - - (hval['bba_votes'],hval['bba_tot'])=c[1].split("/") - (hval['seen_votes'],hval['seen_tot'])=c[2].split("/") - hval['bba_last_val']=try_float(c[3]) - hval['seen_last_val']=try_float(c[4]) - hval['pen_val'] = try_float(c[5]) - - elif len(c) == 6: - # not our line - pass - elif len(line) == 0: - # empty line - pass - else: - log.debug(f"wrong len ({len(c)}) for hbbft: {c}") +def stats(miner: MinerJSONRPC): + try: + addr = miner.addr() + except: + # This is a non-recoverable error, so many things + # depend on knowing the address that it's silly + # to attempt to proceed without it. + log.error("can't get validator's address") + return + + try: + name = miner.name() + except: + # This is a non-recoverable error, so many things + # depend on knowing the address that it's silly + # to attempt to proceed without it. + log.error("can't get validator's name") + return + + # + # Safely try to obtain as many items as possible. + # + height_info = None + try: + height_info = miner.info_height() + except: + log.error("chain height fetch failure") + + in_consensus = None + try: + in_consensus = miner.in_consensus() + except: + log.error("in consensus fetch failure") + + validators = None + try: + validators = miner.ledger_validators() + except: + log.error("validator fetch failure") + + this_validator = None + if validators is not None: + for validator in validators: + if validator['address'] == addr: + this_validator = validator + break + + owner = None + if this_validator is not None: + owner = this_validator['owner_address'] + + balance = None + if owner is not None: + try: + balance_result = miner.ledger_balance({ "address" : owner }) + balance = balance_result['balance'] / 1.0e8 + except: + log.error("owner balance fetch failure") + + block_age = None + try: + block_age = miner.block_age() + except: + log.error("block age fetch failure") + + hbbft_perf = None + try: + hbbft_perf = miner.hbbft_perf() + except: + log.error("hbbft perf fetch failure") + + peer_book_info = None + try: + peer_book_info = miner.peer_book_self() + except: + log.error("peer book self fetch failure") + + # + # Parse results, update gauges. + # + + # Use the validator name as the label for all validator- + # related metrics + my_label = name + + if height_info is not None: + # If `sync_height` is present then the validator is + # syncing and behind, otherwise it is in sync. + chain_height = height_info['height'] + val_height = height_info.get('sync_height', chain_height) + + VAL.labels('Height', my_label).set(val_height) + # TODO, consider getting this from the API + CHAIN_STATS.labels('Height').set(chain_height) + + if in_consensus is not None: + INCON.labels(my_label).set(in_consensus) + + if validators is not None: + staked_validators = [ v for v in validators if v['status'] == 'staked' ] + CHAIN_STATS.labels('staked_validators').set(len(staked_validators)) + + if balance is not None: + BALANCE.labels(my_label).set(balance) + + if block_age is not None: + BLOCKAGE.labels('BlockAge', my_label).set(block_age) + + if this_validator is not None: + LEDGER_PENALTY.labels('ledger_penalties', 'tenure', my_label).set(this_validator['tenure_penalty']) + LEDGER_PENALTY.labels('ledger_penalties', 'dkg', my_label).set(this_validator['dkg_penalty']) + LEDGER_PENALTY.labels('ledger_penalties', 'performance', my_label).set(this_validator['performance_penalty']) + LEDGER_PENALTY.labels('ledger_penalties', 'total', my_label).set(this_validator['total_penalty']) + BLOCKAGE.labels('last_heartbeat', my_label).set(this_validator['last_heartbeat']) + + # Update HBBFT performance stats, if in CG + this_hbbft_perf = None + if hbbft_perf is not None: + for member in hbbft_perf['consensus_members']: + if member['address'] == addr: + this_hbbft_perf = member + break + + if this_hbbft_perf is not None: + # Values common to all members of the CG + hval['bba_tot'] = hbbft_perf['blocks_since_epoch'] + hval['seen_tot'] = hbbft_perf['max_seen'] + + # Values for this validator + hval['pen_val'] = this_hbbft_perf['penalty'] + hval['tenure'] = this_hbbft_perf['tenure'] + hval['seen_votes'] = this_hbbft_perf['seen_votes'] + hval['seen_last_val'] = this_hbbft_perf['last_seen'] + hval['bba_last_val'] = this_hbbft_perf['last_bba'] + hval['bba_completions'] = this_hbbft_perf['bba_completions'] # always set these, that way they get reset when out of CG - HBBFT_PERF.labels('hbbft_perf','Penalty', miner_name).set(hval.get('pen_val', 0)) - HBBFT_PERF.labels('hbbft_perf','BBA_Total', miner_name).set(hval.get('bba_tot', 0)) - HBBFT_PERF.labels('hbbft_perf','BBA_Votes', miner_name).set(hval.get('bba_votes', 0)) - HBBFT_PERF.labels('hbbft_perf','Seen_Total', miner_name).set(hval.get('seen_tot', 0)) - HBBFT_PERF.labels('hbbft_perf','Seen_Votes', miner_name).set(hval.get('seen_votes', 0)) - HBBFT_PERF.labels('hbbft_perf','BBA_Last', miner_name).set(hval.get('bba_last_val', 0)) - HBBFT_PERF.labels('hbbft_perf','Seen_Last', miner_name).set(hval.get('seen_last_val', 0)) - HBBFT_PERF.labels('hbbft_perf','Tenure', miner_name).set(hval.get('tenure', 0)) - -def collect_peer_book(docker_container, miner_name): - # peer book -s output - out = docker_container.exec_run('miner peer book -s --format csv') - # parse the peer book output - - # samples - # address,name,listen_addrs,connections,nat,last_updated - # /p2p/1YBkfTYH8iCvchuTevbCAbdni54geDjH95yopRRznZtAur3iPrM,bright-fuchsia-sidewinder,1,6,none,203.072s - # listen_addrs (prioritized) - # /ip4/174.140.164.130/tcp/2154 - # local,remote,p2p,name - # /ip4/192.168.0.4/tcp/2154,/ip4/72.224.176.69/tcp/2154,/p2p/1YU2cE9FNrwkTr8RjSBT7KLvxwPF9i6mAx8GoaHB9G3tou37jCM,clever-sepia-bull - - sessions = 0 - for line in out.output.decode('utf-8').split("\r\n"): - c = line.split(',') - if len(c) == 6: - log.debug(f"peerbook entry6: {c}") - (address,peer_name,listen_add,connections,nat,last_update) = c - conns_num = try_int(connections) - - if miner_name == peer_name and isinstance(conns_num, int): - CONNECTIONS.labels('connections', miner_name).set(conns_num) - - elif len(c) == 4: - # local,remote,p2p,name - log.debug(f"peerbook entry4: {c}") - if c[0] != 'local': - sessions += 1 - elif len(c) == 1: - log.debug(f"peerbook entry1: {c}") - # listen_addrs - pass - else: - log.warning(f"could not understand peer book line: {c}") - - log.debug(f"sess: {sessions}") - SESSIONS.labels('sessions', miner_name).set(sessions) - -def collect_ledger_validators(docker_container, miner_name): - # ledger validators output - out = docker_container.exec_run('miner ledger validators --format csv') - results = out.output.decode('utf-8').split("\n") - # parse the ledger validators output - for line in [x.rstrip("\r\n") for x in results]: - c = line.split(',') - #print(f"{len(c)} {c}") - if len(c) == 10: - if c[0] == 'name' and c[1] == 'owner_address': - # header line - continue - - (val_name,address,last_heartbeat,stake,status,version,tenure_penalty,dkg_penalty,performance_penalty,total_penalty) = c - if miner_name == val_name: - log.debug(f"have pen line: {c}") - tenure_penalty_val = try_float(tenure_penalty) - dkg_penalty_val = try_float(dkg_penalty) - performance_penalty_val = try_float(performance_penalty) - total_penalty_val = try_float(total_penalty) - least_heartbeat=try_float(last_heartbeat) - - log.info(f"L penalty: {total_penalty_val}") - LEDGER_PENALTY.labels('ledger_penalties', 'tenure', miner_name).set(tenure_penalty_val) - LEDGER_PENALTY.labels('ledger_penalties', 'dkg', miner_name).set(dkg_penalty_val) - LEDGER_PENALTY.labels('ledger_penalties', 'performance', miner_name).set(performance_penalty_val) - LEDGER_PENALTY.labels('ledger_penalties', 'total', miner_name).set(total_penalty_val) - BLOCKAGE.labels('last_heartbeat', miner_name).set(last_heartbeat) - - elif len(line) == 0: - # empty lines are fine - pass - else: - log.warning(f"failed to grok line: {c}; section count: {len(c)}") - - -def collect_miner_version(docker_container, miner_name): - out = docker_container.exec_run('miner versions') - results = out.output.decode('utf-8').split("\n") - # sample output - # $ docker exec validator miner versions - # Installed versions: - # * 0.1.48 permanent - for line in results: - if m := re.match('^\*\s+([\d\.]+)(.*)', line): - miner_version = m.group(1) - log.info(f"found miner version: {miner_version}") - VALIDATOR_VERSION.labels(miner_name).info({'version': miner_version}) + HBBFT_PERF.labels('hbbft_perf','Penalty', my_label).set(hval.get('pen_val', 0)) + HBBFT_PERF.labels('hbbft_perf','BBA_Total', my_label).set(hval.get('bba_tot', 0)) + HBBFT_PERF.labels('hbbft_perf','BBA_Votes', my_label).set(hval.get('bba_completions', 0)) + HBBFT_PERF.labels('hbbft_perf','Seen_Total', my_label).set(hval.get('seen_tot', 0)) + HBBFT_PERF.labels('hbbft_perf','Seen_Votes', my_label).set(hval.get('seen_votes', 0)) + HBBFT_PERF.labels('hbbft_perf','BBA_Last', my_label).set(hval.get('bba_last_val', 0)) + HBBFT_PERF.labels('hbbft_perf','Seen_Last', my_label).set(hval.get('seen_last_val', 0)) + HBBFT_PERF.labels('hbbft_perf','Tenure', my_label).set(hval.get('tenure', 0)) + + if peer_book_info is not None: + connections = peer_book_info[0]['connection_count'] + CONNECTIONS.labels('connections', my_label).set(connections) + sessions = len(peer_book_info[0]['sessions']) + SESSIONS.labels('sessions', my_label).set(sessions) if __name__ == '__main__': prometheus_client.start_http_server(9825) # 9-VAL on your phone + miner = MinerJSONRPC('http://localhost:4467/') while True: #log.warning("starting loop.") try: - stats() + stats(miner) except ValueError as ex: log.error(f"stats loop failed.", exc_info=ex) - except docker.errors.APIError as ex: - log.error(f"stats loop failed with a docker error.", exc_info=ex) - # sleep 30 seconds time.sleep(UPDATE_PERIOD) diff --git a/miner_jsonrpc.py b/miner_jsonrpc.py new file mode 100755 index 0000000..586b48e --- /dev/null +++ b/miner_jsonrpc.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +# external packages +import requests + +# internal packages +import logging +import typing +import functools + +# remember, levels: debug, info, warning, error, critical. there is no trace. +logging.basicConfig(format="%(filename)s:%(funcName)s:%(lineno)d:%(levelname)s\t%(message)s", level=logging.WARNING) +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + +class MinerJSONRPC: + """ + JSON/RPC interface to miner data. + """ + rpc_endpoint: str + rpc_call_id: int + headers: dict + + def __init__(self, endpoint: str): + self.rpc_endpoint = endpoint + self.rpc_call_id = 0 + self.headers = {'Content-Type': 'application/json' } + + def rpc_call(self, verb, params: dict=None) -> dict: + data = { "jsonrpc":"2.0", "id": self.rpc_call_id, "method": verb } + if params: + data['params'] = params + r = requests.post(self.rpc_endpoint, json=data) + self.rpc_call_id += 1 + body = r.json() + error = body.get('error') + if error is not None: + raise Exception('JSON/RPC error') + return body['result'] + + def addr(self) -> str: + result = self.rpc_call('peer_addr') + full_addr = result['peer_addr'] + return full_addr[5:] + + def name(self) -> str: + result = self.rpc_call('info_name') + return result['name'] + + def block_age(self) -> int: + return self.rpc_call('info_block_age')['block_age'] + + def info_height(self) -> dict: + return self.rpc_call('info_height') + + def in_consensus(self) -> bool: + return self.rpc_call('info_in_consensus')['in_consensus'] + + def peer_book_self(self) -> dict: + return self.rpc_call('peer_book', params={ 'addr' : 'self' }) + + def ledger_validators(self, params: typing.Optional[dict]=None) -> typing.List[dict]: + return self.rpc_call('ledger_validators', params) + + def ledger_balance(self, params: typing.Optional[dict]=None) -> typing.List[dict]: + return self.rpc_call('ledger_balance', params) + + def hbbft_perf(self) -> typing.List[dict]: + return self.rpc_call('hbbft_perf') + +def safe_get_json(url: str): + try: + ret = requests.get(url) + if not ret.status_code == requests.codes.ok: + log.error(f"bad status code ({ret.status_code}) from url: {url}") + return + retj = ret.json() + return retj + + + except (requests.exceptions.SSLError, requests.exceptions.ConnectionError) as ex: + log.error(f"error fetching {url}: {ex}") + return + diff --git a/requirements.txt b/requirements.txt index dd28cac..b1033a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,2 @@ prometheus_client -psutil -docker -python-dateutil requests From aa46c707a3c78559fae799d776ad3353b0fcbef0 Mon Sep 17 00:00:00 2001 From: Jeremy Cooper Date: Mon, 19 Jul 2021 04:42:35 +0000 Subject: [PATCH 2/4] Add systemd installation. --- .gitignore | 2 ++ Makefile | 36 +++++++++++++++++++++++++++++++++++ run.sh.in | 3 +++ validator_exporter.service.in | 11 +++++++++++ 4 files changed, 52 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 run.sh.in create mode 100644 validator_exporter.service.in diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3d43339 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +build/ +__pycache__/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b5cdb8a --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +PYTHON_SRCS= \ + miner_exporter.py \ + miner_jsonrpc.py + +PYTHON?= python3 + +DESTROOT?= /home/helium/validator_exporter + +BUILT_SERVICE= build/validator_exporter.service +STARTUP= build/run + +all: $(BUILT_SERVICE) $(STARTUP) + +install: $(DESTROOT)/pyenv $(STARTUP) + for pysrc in $(PYTHON_SRCS); do \ + install -D $$pysrc $(DESTROOT); \ + done + install $(STARTUP) $(DESTROOT) + +install-service: $(BUILT_SERVICE) + install $(BUILT_SERVICE) /etc/systemd/system + +$(DESTROOT)/pyenv: + $(PYTHON) -m venv $@ && . $(DESTROOT)/pyenv/bin/activate && pip install -r requirements.txt + +$(BUILT_SERVICE): validator_exporter.service.in + mkdir -p build + sed -e s,@@DESTROOT@@,$(DESTROOT),g < $< > $@ + +$(STARTUP): run.sh.in + mkdir -p build + sed -e s,@@DESTROOT@@,$(DESTROOT),g < $< > $@ + +clean: + rm -f $(BUILT_SERVICE) + diff --git a/run.sh.in b/run.sh.in new file mode 100644 index 0000000..39dcd96 --- /dev/null +++ b/run.sh.in @@ -0,0 +1,3 @@ +#!/bin/sh +. @@DESTROOT@@/pyenv/bin/activate +exec python miner_exporter.py diff --git a/validator_exporter.service.in b/validator_exporter.service.in new file mode 100644 index 0000000..6ebb149 --- /dev/null +++ b/validator_exporter.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Validator statistics for Prometheus + +[Service] +User=ubuntu +ExecStart=@@DESTROOT@@/run +WorkingDirectory=@@DESTROOT@@ +Restart=always + +[Install] +WantedBy=multi-user.target From 6269396cac06c9de85194acf1e81ae5163cea0ec Mon Sep 17 00:00:00 2001 From: Jeremy Cooper Date: Tue, 20 Jul 2021 18:12:27 +0000 Subject: [PATCH 3/4] Remove full validators scrape; pythonify API. Clean up the mechanism used for passing parameters to JSONRPC functions. Do it the Python way, using keyword arguments. Then unpack them into a JSON dictionary at the appropriate time. Also, for the time being, don't ask the validator to enumerate all validators recorded in the ledger. This is an expensive operation and doesn't help much in the long run. Instead, ask for this validator's stats, only. --- miner_exporter.py | 17 +++-------------- miner_jsonrpc.py | 4 ++-- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/miner_exporter.py b/miner_exporter.py index ea7c124..272b8d7 100755 --- a/miner_exporter.py +++ b/miner_exporter.py @@ -91,19 +91,12 @@ def stats(miner: MinerJSONRPC): except: log.error("in consensus fetch failure") - validators = None + this_validator = None try: - validators = miner.ledger_validators() + this_validator = miner.ledger_validators(address=addr) except: log.error("validator fetch failure") - this_validator = None - if validators is not None: - for validator in validators: - if validator['address'] == addr: - this_validator = validator - break - owner = None if this_validator is not None: owner = this_validator['owner_address'] @@ -111,7 +104,7 @@ def stats(miner: MinerJSONRPC): balance = None if owner is not None: try: - balance_result = miner.ledger_balance({ "address" : owner }) + balance_result = miner.ledger_balance(address=owner) balance = balance_result['balance'] / 1.0e8 except: log.error("owner balance fetch failure") @@ -155,10 +148,6 @@ def stats(miner: MinerJSONRPC): if in_consensus is not None: INCON.labels(my_label).set(in_consensus) - if validators is not None: - staked_validators = [ v for v in validators if v['status'] == 'staked' ] - CHAIN_STATS.labels('staked_validators').set(len(staked_validators)) - if balance is not None: BALANCE.labels(my_label).set(balance) diff --git a/miner_jsonrpc.py b/miner_jsonrpc.py index 586b48e..c3fa534 100755 --- a/miner_jsonrpc.py +++ b/miner_jsonrpc.py @@ -59,10 +59,10 @@ def in_consensus(self) -> bool: def peer_book_self(self) -> dict: return self.rpc_call('peer_book', params={ 'addr' : 'self' }) - def ledger_validators(self, params: typing.Optional[dict]=None) -> typing.List[dict]: + def ledger_validators(self, **params) -> typing.List[dict]: return self.rpc_call('ledger_validators', params) - def ledger_balance(self, params: typing.Optional[dict]=None) -> typing.List[dict]: + def ledger_balance(self, **params) -> typing.List[dict]: return self.rpc_call('ledger_balance', params) def hbbft_perf(self) -> typing.List[dict]: From 537f325c89d039bfbf88f66cf0222797d536dadb Mon Sep 17 00:00:00 2001 From: Jeremy Cooper Date: Thu, 22 Jul 2021 13:58:05 -0700 Subject: [PATCH 4/4] Update README with my local build/changes. --- README.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eea545c..d48ecea 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,32 @@ services: ## Running locally On the miner machine: -install python3 +install python3, python3-venv + ``` -pip install prometheus_client psutil docker +pip install prometheus_client ``` Details on the libraries: * [client\_python](https://github.com/prometheus/client_python) -* [psutil](https://github.com/giampaolo/psutil) -* [docker](https://pypi.org/project/docker/) +Then install the service in a home directory: + +``` +sudo make install +``` + +Then install the systemd unit file: + +``` +sudo make install-service +``` + +Then enable and start the service: + +``` +sudo systemctl enable validator_exporter +sudo systemctl start validator_exporter +``` ## Configuration