diff --git a/README.md b/README.md index 58e65a7..1b581d9 100644 --- a/README.md +++ b/README.md @@ -53,4 +53,4 @@ UPDATE_PERIOD # seconds between scrapes, int VALIDATOR_CONTAINER_NAME # eg 'validator', string API_BASE_URL # URL for api access, string. For testnet, set to "https://testnet-api.helium.wtf/v1" ENABLE_RPC # opt in to using the RPC API with a truthy value (defaults to falsey value until `exec` calls are fully replaced). -``` +``` \ No newline at end of file diff --git a/miner_exporter.py b/miner_exporter.py index 8654d6d..21716fd 100755 --- a/miner_exporter.py +++ b/miner_exporter.py @@ -7,6 +7,9 @@ import requests import dateutil.parser +# I just copied the python script from https://github.com/andrewboudreau/miner_httpclient into the folder for easy testing. +from miner_client import MinerClient + # internal packages import datetime import time @@ -74,120 +77,47 @@ UPTIME = prometheus_client.Gauge('validator_container_uptime', 'Time container has been at a given state', ['state_type','validator_name']) -miner_facts = {} -def try_int(v): - if re.match(r"^\-?\d+$", v): - return int(v) - return v +# create a client for the miner's json rpc interface +jsonRpcClient = MinerClient() def try_float(v): if re.match(r"^\-?[\d\.]+$", v): return float(v) return v -def get_facts(docker_container_obj): - if miner_facts: - return miner_facts - #miner_facts = { - # 'name': None, - # 'address': None - #} - out = docker_container_obj.exec_run('miner print_keys') - # sample output: - # {pubkey,"1YBkf..."}. - # {onboarding_key,"1YBkf..."}. - # {animal_name,"one-two-three"}. - - log.debug(out.output) - printkeys = {} - for line in out.output.split(b"\n"): - strline = line.decode('utf-8') - - # := requires py3.8 - if m := re.match(r'{([^,]+),"([^"]+)"}.', strline): - log.debug(m) - k = m.group(1) - v = m.group(2) - log.debug(k,v) - printkeys[k] = v - - if v := printkeys.get('pubkey'): - miner_facts['address'] = v - if printkeys.get('animal_name'): - miner_facts['name'] = v - #$ docker exec validator miner print_keys - - # this isn't as useful as it seems, because we'll run `du` inside the container. - miner_facts['data_mount'] = None - mts = docker_container_obj.attrs.get('Mounts',[]) - data_mount_list = [x['Source'] for x in mts if x['Destination'].startswith('/var/data')] - if len(data_mount_list): - miner_facts['data_mount'] = data_mount_list[0] - - return miner_facts - -def collect_volume_usage(docker_container_obj, data_mount_path, hotspot_name_str): - # note this won't come from the RPC, so it's either inside docker or on the root filesystem. - # busybox doesn't support -b (bytes), so we'll multiply it. - out = docker_container_obj.exec_run('du -ksx /var/data') - log.info(out.output) - disk_size = out.output.decode('utf-8').rstrip("\n").split("\t", 1)[0] - disk_size_i = try_int(disk_size) * 1024 - - log.info(f"DS: {disk_size_i} bytes ({disk_size} kb str)") - VALIDATOR_DISK_USAGE.labels(hotspot_name_str).set(disk_size_i) - - return disk_size_i - # Decorate function with metric. @SCRAPE_TIME.time() def stats(): - docker_container = None - + miner_name = jsonRpcClient.info_name() + try: dc = docker.DockerClient() # Try to find by specific name first docker_container = dc.containers.get(VALIDATOR_CONTAINER_NAME) except docker.errors.NotFound as ex: - # If find by specifc name fails, try to find by prefix - containers = dc.containers.list() - - for container in containers: - if container.name.startswith(VALIDATOR_CONTAINER_NAME): - docker_container = container - break - - # If container not found, then log error and return - if docker_container is None: - log.error(f"docker failed while bootstrapping. Not exporting anything. Error: {ex}") - return + log.error(f"docker failed while bootstrapping. Not exporting anything. Error: {ex}") + return + + # collect total cpu and memory usage. Might want to consider just the docker + # container with something like cadvisor instead + SYSTEM_USAGE.labels('CPU', miner_name).set(psutil.cpu_percent()) + SYSTEM_USAGE.labels('Memory', miner_name).set(psutil.virtual_memory()[2]) + SYSTEM_USAGE.labels('CPU-Steal', miner_name).set(psutil.cpu_times_percent().steal) + SYSTEM_USAGE.labels('Disk Used', miner_name).set(float(psutil.disk_usage('/').used) / float(psutil.disk_usage('/').total)) + SYSTEM_USAGE.labels('Disk Free', miner_name).set(float(psutil.disk_usage('/').free) / float(psutil.disk_usage('/').total)) - miner_facts = get_facts(docker_container) - hotspot_name_str = get_miner_name(docker_container) + collect_container_run_time(docker_container, miner_name) - # collect total cpu and memory usage. Might want to consider just the docker - # container with something like cadvisor instead - SYSTEM_USAGE.labels('CPU', hotspot_name_str).set(psutil.cpu_percent()) - SYSTEM_USAGE.labels('Memory', hotspot_name_str).set(psutil.virtual_memory()[2]) - SYSTEM_USAGE.labels('CPU-Steal', hotspot_name_str).set(psutil.cpu_times_percent().steal) - SYSTEM_USAGE.labels('Disk Used', hotspot_name_str).set(float(psutil.disk_usage('/').used) / float(psutil.disk_usage('/').total)) - SYSTEM_USAGE.labels('Disk Free', hotspot_name_str).set(float(psutil.disk_usage('/').free) / float(psutil.disk_usage('/').total)) - SYSTEM_USAGE.labels('Process-Count', hotspot_name_str).set(sum(1 for proc in psutil.process_iter())) - - collect_container_run_time(docker_container, hotspot_name_str) - collect_miner_version(docker_container, hotspot_name_str) - collect_block_age(docker_container, hotspot_name_str) - collect_miner_height(docker_container, hotspot_name_str) - collect_chain_stats() - collect_in_consensus(docker_container, hotspot_name_str) - collect_ledger_validators(docker_container, hotspot_name_str) - collect_peer_book(docker_container, hotspot_name_str) - collect_hbbft_performance(docker_container, hotspot_name_str) - collect_balance(docker_container,miner_facts['address'],hotspot_name_str) - collect_volume_usage(docker_container, miner_facts['data_mount'], hotspot_name_str) + collect_hbbft_performance(miner_name) + collect_block_age(miner_name) + collect_miner_height(miner_name) + collect_in_consensus(miner_name) + collect_ledger_validators(miner_name) + collect_peer_book(miner_name) + collect_balance(miner_name) def safe_get_json(url): try: @@ -197,12 +127,11 @@ def safe_get_json(url): return retj = ret.json() return retj - - + except (requests.exceptions.SSLError, requests.exceptions.ConnectionError) as ex: log.error(f"error fetching {url}: {ex}") return - + def collect_container_run_time(docker_container, miner_name): attrs = docker_container.attrs @@ -250,215 +179,106 @@ def collect_chain_stats(): return count_val = api['data']['staked']['count'] CHAIN_STATS.labels('staked_validators').set(count_val) +# persist these between calls -def collect_balance(docker_container, addr, miner_name): - # should move pubkey to getfacts and then pass it in here - #out = docker_container.exec_run('miner print_keys') - #for line in out.output.decode('utf-8').split("\n"): - # if 'pubkey' in line: - # addr=line[9:60] - api_validators = safe_get_json(f'{API_BASE_URL}/validators/{addr}') - if not api_validators: - log.error("validator fetch returned empty JSON") - return - elif not api_validators.get('data') or not api_validators['data'].get('owner'): - log.error("could not find validator data owner in json") - return - owner = api_validators['data']['owner'] - - api_accounts = safe_get_json(f'{API_BASE_URL}/accounts/{owner}') - if not api_accounts: - return - if not api_accounts.get('data') or not api_accounts['data'].get('balance'): - return - balance = float(api_accounts['data']['balance'])/1E8 - #print(api_accounts) - #print('balance',balance) +def collect_hbbft_performance(miner_name): + #{'address': '1126bQkTLZGXtt3Dx4kWUs4JufeBfsdnWTgNV3fptEmAYM2YPkzo', + # 'bba_completions': [10, 18], + # 'last_bba': 0, + # 'last_seen': 0, + # 'name': 'mythical-coconut-horse', + # 'penalty': [4.7695473020038905], + # 'seen_votes': [661, 661], + # 'tenure': [2.071502976190476]} + perf = jsonRpcClient.hbbft_perf(miner_name) + if perf is None: + HBBFT_PERF.labels('hbbft_perf','Penalty', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','Tenure', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','BBA_Total', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','BBA_Votes', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','BBA_Last', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','Seen_Total', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','Seen_Votes', miner_name).set(0) + HBBFT_PERF.labels('hbbft_perf','Seen_Last', miner_name).set(0) + else: + HBBFT_PERF.labels('hbbft_perf','Penalty', miner_name).set(float(perf.get('penalty')[0])) + HBBFT_PERF.labels('hbbft_perf','Tenure', miner_name).set(float(perf.get('tenure')[0])) + + HBBFT_PERF.labels('hbbft_perf','BBA_Votes', miner_name).set(perf.get('bba_completions')[0]) + HBBFT_PERF.labels('hbbft_perf','BBA_Total', miner_name).set(perf.get('bba_completions')[1]) + HBBFT_PERF.labels('hbbft_perf','BBA_Last', miner_name).set(perf.get('last_bba')) + + HBBFT_PERF.labels('hbbft_perf','Seen_Votes', miner_name).set(perf.get('seen_votes')[0]) + HBBFT_PERF.labels('hbbft_perf','Seen_Total', miner_name).set(perf.get('seen_votes')[1]) + HBBFT_PERF.labels('hbbft_perf','Seen_Last', miner_name).set(perf.get('last_seen')) + +def collect_balance(miner_name): + validator = get_validator_ledger(miner_name) + + if validator is None: + log.warning(f"failed to find validator {miner_name}") + return + + owner_address = validator['owner_address'] + bones = jsonRpcClient.ledger_balance(owner_address)["balance"] + balance = float(bones)/1E8 BALANCE.labels(miner_name).set(balance) - -def get_miner_name(docker_container): - # need to fix this. hotspot name really should only be queried once - out = docker_container.exec_run('miner info name') - log.debug(out.output) - hotspot_name = out.output.decode('utf-8').rstrip("\n") - return hotspot_name - -def collect_miner_height(docker_container, miner_name): +def collect_miner_height(miner_name): # grab the local blockchain height - out = docker_container.exec_run('miner info height') - log.debug(out.output) - txt = out.output.decode('utf-8').rstrip("\n") - VAL.labels('Height', miner_name).set(out.output.split()[1]) + height = jsonRpcClient.info_height()["height"] + VAL.labels('Height', miner_name).set(height) -def collect_in_consensus(docker_container, miner_name): +def collect_in_consensus(miner_name): # check if currently in consensus group - out = docker_container.exec_run('miner info in_consensus') - incon_txt = out.output.decode('utf-8').rstrip("\n") - incon = 0 - if incon_txt == 'true': - incon = 1 - log.info(f"in consensus? {incon} / {incon_txt}") - INCON.labels(miner_name).set(incon) - -def collect_block_age(docker_container, miner_name): - # collect current block age - out = docker_container.exec_run('miner info block_age') - ## transform into a number - age_val = try_int(out.output.decode('utf-8').rstrip("\n")) - log.debug(f"age: {age_val}") - - BLOCKAGE.labels('BlockAge', miner_name).set(age_val) + in_consensus = (1,0)[jsonRpcClient.info_in_consensus()] + INCON.labels(miner_name).set(in_consensus) -# persist these between calls -hval = {} -def collect_hbbft_performance(docker_container, miner_name): - # parse the hbbft performance table for the penalty field - out = docker_container.exec_run('miner hbbft perf --format csv') - #print(out.output) - - for line in out.output.decode('utf-8').split("\n"): - c = [x.strip() for x in line.split(',')] - # samples: - - have_data = False - - if len(c) == 7 and miner_name == c[0]: - # name,bba_completions,seen_votes,last_bba,last_seen,tenure,penalty - # great-clear-chinchilla,5/5,237/237,0,0,2.91,2.91 - log.debug(f"resl7: {c}; {miner_name}/{c[0]}") - - (hval['bba_votes'],hval['bba_tot'])=c[1].split("/") - (hval['seen_votes'],hval['seen_tot'])=c[2].split("/") - hval['bba_last_val']=try_float(c[3]) - hval['seen_last_val']=try_float(c[4]) - hval['tenure'] = try_float(c[5]) - hval['pen_val'] = try_float(c[6]) - elif len(c) == 6 and miner_name == c[0]: - # name,bba_completions,seen_votes,last_bba,last_seen,penalty - # curly-peach-owl,11/11,368/368,0,0,1.86 - log.debug(f"resl6: {c}; {miner_name}/{c[0]}") - - (hval['bba_votes'],hval['bba_tot'])=c[1].split("/") - (hval['seen_votes'],hval['seen_tot'])=c[2].split("/") - hval['bba_last_val']=try_float(c[3]) - hval['seen_last_val']=try_float(c[4]) - hval['pen_val'] = try_float(c[5]) - - elif len(c) == 6: - # not our line - pass - elif len(line) == 0: - # empty line - pass - else: - log.debug(f"wrong len ({len(c)}) for hbbft: {c}") - - # always set these, that way they get reset when out of CG - HBBFT_PERF.labels('hbbft_perf','Penalty', miner_name).set(hval.get('pen_val', 0)) - HBBFT_PERF.labels('hbbft_perf','BBA_Total', miner_name).set(hval.get('bba_tot', 0)) - HBBFT_PERF.labels('hbbft_perf','BBA_Votes', miner_name).set(hval.get('bba_votes', 0)) - HBBFT_PERF.labels('hbbft_perf','Seen_Total', miner_name).set(hval.get('seen_tot', 0)) - HBBFT_PERF.labels('hbbft_perf','Seen_Votes', miner_name).set(hval.get('seen_votes', 0)) - HBBFT_PERF.labels('hbbft_perf','BBA_Last', miner_name).set(hval.get('bba_last_val', 0)) - HBBFT_PERF.labels('hbbft_perf','Seen_Last', miner_name).set(hval.get('seen_last_val', 0)) - HBBFT_PERF.labels('hbbft_perf','Tenure', miner_name).set(hval.get('tenure', 0)) - -def collect_peer_book(docker_container, miner_name): - # peer book -s output - out = docker_container.exec_run('miner peer book -s --format csv') - # parse the peer book output - - # samples - # address,name,listen_addrs,connections,nat,last_updated - # /p2p/1YBkfTYH8iCvchuTevbCAbdni54geDjH95yopRRznZtAur3iPrM,bright-fuchsia-sidewinder,1,6,none,203.072s - # listen_addrs (prioritized) - # /ip4/174.140.164.130/tcp/2154 - # local,remote,p2p,name - # /ip4/192.168.0.4/tcp/2154,/ip4/72.224.176.69/tcp/2154,/p2p/1YU2cE9FNrwkTr8RjSBT7KLvxwPF9i6mAx8GoaHB9G3tou37jCM,clever-sepia-bull - - sessions = 0 - for line in out.output.decode('utf-8').split("\r\n"): - c = line.split(',') - if len(c) == 6: - log.debug(f"peerbook entry6: {c}") - (address,peer_name,listen_add,connections,nat,last_update) = c - conns_num = try_int(connections) - - if miner_name == peer_name and isinstance(conns_num, int): - CONNECTIONS.labels('connections', miner_name).set(conns_num) - - elif len(c) == 4: - # local,remote,p2p,name - log.debug(f"peerbook entry4: {c}") - if c[0] != 'local': - sessions += 1 - elif len(c) == 1: - log.debug(f"peerbook entry1: {c}") - # listen_addrs - pass - else: - log.warning(f"could not understand peer book line: {c}") - - log.debug(f"sess: {sessions}") - SESSIONS.labels('sessions', miner_name).set(sessions) +def collect_block_age(miner_name): + block_age = jsonRpcClient.info_block_age() + BLOCKAGE.labels('BlockAge', miner_name).set(block_age) + log.debug(f"block age: {block_age}") + +def collect_peer_book(miner_name): + book = jsonRpcClient.peer_book("self")[0] + connections = book["connection_count"] + CONNECTIONS.labels('connections', miner_name).set(connections) -def collect_ledger_validators(docker_container, miner_name): - # ledger validators output - out = docker_container.exec_run('miner ledger validators --format csv') - results = out.output.decode('utf-8').split("\n") - # parse the ledger validators output - for line in [x.rstrip("\r\n") for x in results]: - c = line.split(',') - #print(f"{len(c)} {c}") - if len(c) == 10: - if c[0] == 'name' and c[1] == 'owner_address': - # header line - continue - - (val_name,address,last_heartbeat,stake,status,version,tenure_penalty,dkg_penalty,performance_penalty,total_penalty) = c - if ALL_PENALTIES or miner_name == val_name: - log.debug(f"have pen line: {c}") - tenure_penalty_val = try_float(tenure_penalty) - dkg_penalty_val = try_float(dkg_penalty) - performance_penalty_val = try_float(performance_penalty) - total_penalty_val = try_float(total_penalty) - last_heartbeat = try_float(last_heartbeat) - - log.debug(f"L penalty: {total_penalty_val}") - if not ALL_PENALTIES or total_penalty_val > 0.0: - LEDGER_PENALTY.labels('ledger_penalties', 'tenure', val_name).set(tenure_penalty_val) - LEDGER_PENALTY.labels('ledger_penalties', 'dkg', val_name).set(dkg_penalty_val) - LEDGER_PENALTY.labels('ledger_penalties', 'performance', val_name).set(performance_penalty_val) - LEDGER_PENALTY.labels('ledger_penalties', 'total', val_name).set(total_penalty_val) - - # In an effort to reduce the number of metrics to track, only gather - # last_heartbear for this miner_name. Will this surprise users? - if miner_name == val_name: - BLOCKAGE.labels('last_heartbeat', val_name).set(last_heartbeat) - - elif len(line) == 0: - # empty lines are fine - pass - else: - log.warning(f"failed to grok line: {c}; section count: {len(c)}") - - -def collect_miner_version(docker_container, miner_name): - out = docker_container.exec_run('miner versions') - results = out.output.decode('utf-8').split("\n") - # sample output - # $ docker exec validator miner versions - # Installed versions: - # * 0.1.48 permanent - for line in results: - if m := re.match('^\*\s+([\d\.]+)(.*)', line): - miner_version = m.group(1) - log.info(f"found miner version: {miner_version}") - VALIDATOR_VERSION.labels(miner_name).info({'version': miner_version}) + sessions = len(book["sessions"]) + SESSIONS.labels('sessions', miner_name).set(sessions) +def collect_ledger_validators(miner_name): + validator = get_validator_ledger(miner_name) + + if validator is None: + log.warning(f"failed to find validator {miner_name}") + return + + tenure_penalty = float(validator['tenure_penalty']) + dkg_penalty = float(validator['dkg_penalty']) + performance_penalty = float(validator['performance_penalty']) + total_penalty = float(validator['total_penalty']) + + log.info(f"L penalty: {total_penalty}") + LEDGER_PENALTY.labels('ledger_penalties', 'tenure', miner_name).set(tenure_penalty) + LEDGER_PENALTY.labels('ledger_penalties', 'dkg', miner_name).set(dkg_penalty) + LEDGER_PENALTY.labels('ledger_penalties', 'performance', miner_name).set(performance_penalty) + LEDGER_PENALTY.labels('ledger_penalties', 'total', miner_name).set(total_penalty) + + last_heartbeat = validator['last_heartbeat'] + BLOCKAGE.labels('last_heartbeat', miner_name).set(last_heartbeat) + + miner_version = validator["version"] + VALIDATOR_VERSION.labels(miner_name).info({'version': miner_version}) + +def get_validator_ledger(miner_name): + validators = jsonRpcClient.ledger_validators() + validator = next((v for v in validators if v["name"] == miner_name), None) + return validator if __name__ == '__main__': + print(f"collecting metrics for: {jsonRpcClient.info_name()}") + prometheus_client.start_http_server(9825) # 9-VAL on your phone while True: #log.warning("starting loop.") @@ -472,4 +292,3 @@ def collect_miner_version(docker_container, miner_name): # sleep 30 seconds time.sleep(UPDATE_PERIOD) -