From f37fd376ecf8c3b86187574a515d1b5d0cdf5d9f Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:30:33 -0500 Subject: [PATCH 1/8] logger: optional compact SHARE lines for accepted and rejected work What this does -------------- When logger.log_shares is true, every stratum mining.submit result that passes the optional missingzeros gate is written at INFO as one compact line: SHARE 1x accepted bc1q... @192.168.4.115 ?ok diff=16384/16384/1141943014 missingzeros=6 SHARE rejected ? @192.168.4.115 ?high-hash diff=16384 Fields: Nx vardiff / job_diff when it divides evenly; omitted otherwise accepted|rejected username last-auth or submit username; "?" if unknown @host rem_host; "?" if empty ?reason ok | block | high-hash | stale-work | stale-prevblk | time-too-old | time-too-new | H-not-zero | duplicate | unknown-work | ... diff=job/vardiff/block block is nbits-derived uint64, 0 if unavailable missingzeros=N bits the hash is short of the *block* target (0 = block candidate). Omitted on early rejects that never hashed. A found block also emits a second line that matches the node-check format so log parsers can treat "we submitted a block" the same way: SHARE <64 hex> mode=block d= => submitted missingzeros is counted against the full 256-bit block target, not leading hex zeros. That matters on BLAKE2b where the interesting bits are not always in the high bytes the way people read SHA256d explorers. Configuration (all optional) ---------------------------- logger.log_shares bool default false mining.share_node_check_missingzeros int default -1 -1 means "do not gate on closeness": if log_shares is on, log every result that hits the helpers (early rejects have missingzeros=-1 and still log). >= 0 means only emit SHARE lines when missingzeros was computed and is <= this value. Use 2 for "near-block only" (~4 per block in expectation at random hashing), 4 for ~16, 0 for block-candidates only. This is the same knob later used by node-check. Benefits -------- - Solo / lab operators can see *why* a share died without turning on DEBUG and drowning the console. - Compact form fits a 120-column TTY with log_calling_function on. - missingzeros is a cheap closeness metric for "is this miner actually approaching the block target or just sending job-diff dust." - Same reason strings the miner already got in the stratum error, so the log and the ASIC agree. Risks ----- 1. Log volume. At INFO, an unfiltered farm can write tens of thousands of lines per minute. Default is off. missingzeros >= 0 is the intended production filter. 2. Username / IP in the log. This is the same data already on the dashboard and in stratum. Do not ship logs off-box if that is a problem. 3. Wrong closeness math would silently mis-filter. The helper first compares hash vs block_target; if the share already meets target it returns 0. Otherwise it estimates the bit gap from leading-zero difference and checks by shifting. Off-by-one is possible on the gap; it will not invert accept/reject. 4. Early rejects (unknown-work before username is parsed) use last_auth_username or "?". That can attribute a malformed submit to the previous authorized name on that connection. Better than inventing a user, still not a proof of identity. Latency / main thread --------------------- All work is on the stratum worker that already handled mining.submit: - a few 32-byte copies and a 256-bit shift for missingzeros - one snprintf + DLOG_INFO DLOG_INFO is the existing lock-free-ish queue into the logger thread (rwlock + double buffer). This patch does not fopen, does not malloc on the accept path, and does not wait on the logger. Cost is microseconds, dominated by the hash the submit already computed. It does *not* talk to bitcoind. Possible improvements --------------------- - Pass job_diff into the reject helpers so early rejects show a real diff instead of 0 / a single number. - Sample by 1-of-N for logs independently of missingzeros, so a busy farm can keep a heartbeat without the closeness filter. - Rate-limit per connection if someone turns log_shares on at a site with 10k miners. - Move missingzeros into a shared util and unit-test the shift against known hash/target pairs. What to validate before merge ----------------------------- - Default config: zero new lines, accept/reject behavior unchanged. - log_shares=true, missingzeros=-1: one SHARE line per submit, compact form, no user=/host= keys. - Reject reasons still match the JSON error sent to the miner. - Found-block path still submits and still prints BLOCK FOUND; the extra SHARE mode=block line is additional, not a replacement. - missingzeros=0 logs only was_block shares. - Logger queue does not stall stratum under a share flood (watch LOGGER OVERRUN). --- src/datum_conf.c | 4 ++ src/datum_conf.h | 2 + src/datum_stratum.c | 131 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/src/datum_conf.c b/src/datum_conf.c index 6801cb1c..fb44fa02 100644 --- a/src/datum_conf.c +++ b/src/datum_conf.c @@ -127,6 +127,8 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { .required = false, .ptr = &datum_config.coinbase_unique_id, .default_int = 4242 }, { .var_type = DATUM_CONF_STRING, .category = "mining", .name = "save_submitblocks_dir", .description = "Directory to save all submitted blocks to as submitblock JSON files", .required = false, .ptr = datum_config.mining_save_submitblocks_dir, .default_string[0] = "", .max_string_len = sizeof(datum_config.mining_save_submitblocks_dir) }, + { .var_type = DATUM_CONF_INT, .category = "mining", .name = "share_node_check_missingzeros", .description = "If >= 0, only log SHARE lines and node-check shares with missingzeros <= this value (overrides share_node_check_every). -1 = infinity (log all, use every-N).", + .required = false, .ptr = &datum_config.mining_share_node_check_missingzeros, .default_int = -1 }, { .var_type = DATUM_CONF_BOOL, .category = "mining", .name = "allow_hasher_time_rolling", .description = "Allow hasher time rolling for BLAKE2b jobs", .required = false, .ptr = &datum_config.mining_allow_hasher_time_rolling, .default_bool = false }, { .var_type = DATUM_CONF_BOOL, .category = "mining", .name = "abw_verify_all_shares_on_disclosure", .description = "Retain ABW proofs until key disclosure and alarm if the pool ignored a block", @@ -175,6 +177,8 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { { .var_type = DATUM_CONF_INT, .category = "logger", .name = "log_level_file", .description = "Minimum log level for log file messages (0=All, 1=Debug, 2=Info, 3=Warn, 4=Error, 5=Fatal)", .example_default = true, .required = false, .ptr = &datum_config.clog_level_file, .default_int = 1 }, + { .var_type = DATUM_CONF_BOOL, .category = "logger", .name = "log_shares", .description = "Log the result of every incoming share at INFO (level 2): accepted or rejected with reason", + .required = false, .ptr = &datum_config.logger_log_shares, .default_bool = false }, // datum options { .var_type = DATUM_CONF_STRING, .category = "datum", .name = "pool_host", .description = "Remote DATUM server host/ip to use for decentralized pooled mining (set to \"\" to disable pooled mining)", diff --git a/src/datum_conf.h b/src/datum_conf.h index 85f014e2..f75d31f9 100644 --- a/src/datum_conf.h +++ b/src/datum_conf.h @@ -131,6 +131,7 @@ typedef struct { char mining_coinbase_tag_primary[64]; char mining_coinbase_tag_secondary[64]; char mining_save_submitblocks_dir[256]; + int mining_share_node_check_missingzeros; bool mining_allow_hasher_time_rolling; bool mining_abw_verify_all_shares_on_disclosure; int coinbase_unique_id; @@ -155,6 +156,7 @@ typedef struct { bool clog_to_stderr; bool clog_rotate_daily; char clog_file[1024]; + bool logger_log_shares; char datum_pool_host[1024]; int datum_pool_port; diff --git a/src/datum_stratum.c b/src/datum_stratum.c index a1818faf..01e72c20 100644 --- a/src/datum_stratum.c +++ b/src/datum_stratum.c @@ -46,6 +46,8 @@ #include #include #include +#include +#include #include "datum_gateway.h" #include "datum_stratum.h" @@ -740,35 +742,152 @@ void send_error_to_client(T_DATUM_CLIENT_DATA *c, uint64_t id, char *e) { stratum_rpc_id_clear(c); } + +static unsigned datum_leading_zero_bits_le(const unsigned char *hash_le) +{ + unsigned n = 0; + int i, b; + if (!hash_le) return 0; + for (i = 31; i >= 0; i--) { + if (hash_le[i] == 0) { + n += 8; + continue; + } + for (b = 7; b >= 0; b--) { + if (hash_le[i] & (unsigned char)(1u << b)) return n; + n++; + } + } + return n; +} + +static void datum_shr_le256(unsigned char *h, unsigned k) +{ + unsigned bytes, bits, i, v, carry, mask; + if (!h || k == 0) return; + if (k >= 256) { + memset(h, 0, 32); + return; + } + bytes = k / 8; + bits = k % 8; + if (bytes) { + memmove(h, h + bytes, 32 - bytes); + memset(h + (32 - bytes), 0, bytes); + } + if (!bits) return; + mask = (1u << bits) - 1u; + carry = 0; + for (i = 31; i != (unsigned)-1; i--) { + v = h[i]; + h[i] = (unsigned char)((v >> bits) | carry); + carry = (v & mask) << (8 - bits); + } +} + +static unsigned datum_missing_block_zero_bits(const unsigned char *share_hash_le, const unsigned char *block_target_le, bool meets_block_target) +{ + unsigned char h[32]; + unsigned lz_h, lz_t, k; + if (meets_block_target) return 0; + if (!share_hash_le || !block_target_le) return 0; + if (compare_hashes(share_hash_le, block_target_le) <= 0) return 0; + lz_h = datum_leading_zero_bits_le(share_hash_le); + lz_t = datum_leading_zero_bits_le(block_target_le); + k = (lz_t > lz_h) ? (lz_t - lz_h) : 0; + memcpy(h, share_hash_le, 32); + datum_shr_le256(h, k); + if (compare_hashes(h, block_target_le) <= 0) return k; + return k + 1; +} + +static void datum_share_hash_to_hex(char *out, const unsigned char *hash_le) { + int i; + if (!hash_le) { out[0] = 0; return; } + for (i = 0; i < 32; i++) { + uchar_to_hex(&out[(31-i)*2], hash_le[i]); + } + out[64] = 0; +} + +static void stratum_log_share_result(const T_DATUM_CLIENT_DATA *c, const char *username, bool accepted, const char *reason, uint64_t job_diff, uint64_t vardiff, uint64_t blockdiff, int missing_zeros) +{ + const char *host; + const char *user; + char mult[32]; + char diffs[96]; + if (!datum_config.logger_log_shares) return; + if (datum_config.mining_share_node_check_missingzeros >= 0) { + if (missing_zeros < 0 || missing_zeros > datum_config.mining_share_node_check_missingzeros) return; + } + host = (c && c->rem_host[0]) ? c->rem_host : "?"; + user = (username && username[0]) ? username : "?"; + mult[0] = 0; + if (job_diff > 0 && vardiff >= job_diff && (vardiff % job_diff) == 0) { + snprintf(mult, sizeof(mult), " %" PRIu64 "x", vardiff / job_diff); + } + if (vardiff || blockdiff) { + snprintf(diffs, sizeof(diffs), "%" PRIu64 "/%" PRIu64 "/%" PRIu64, job_diff, vardiff, blockdiff); + } else { + snprintf(diffs, sizeof(diffs), "%" PRIu64, job_diff); + } + if (missing_zeros >= 0) { + DLOG_INFO("SHARE%s %s %s @%s ?%s diff=%s missingzeros=%d", + mult, accepted ? "accepted" : "rejected", user, host, + reason ? reason : (accepted ? "ok" : "unknown"), diffs, missing_zeros); + } else { + DLOG_INFO("SHARE%s %s %s @%s ?%s diff=%s", + mult, accepted ? "accepted" : "rejected", user, host, + reason ? reason : (accepted ? "ok" : "unknown"), diffs); + } +} + +static const char *stratum_share_user_guess(T_DATUM_CLIENT_DATA *c) +{ + T_DATUM_MINER_DATA *m; + if (!c || !c->app_client_data) return NULL; + m = c->app_client_data; + if (m->last_auth_username[0]) return m->last_auth_username; + return NULL; +} + static inline void send_unknown_work_error(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "unknown-work", 0, 0, 0, -1); send_error_to_client(c, id, "[20,\"unknown-work\",null]"); } static inline void send_rejected_high_hash_error(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "high-hash", 0, 0, 0, -1); send_error_to_client(c, id, "[23,\"high-hash\",null]"); } static inline void send_rejected_stale(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "stale-work", 0, 0, 0, -1); send_error_to_client(c, id, "[21,\"stale-work\",null]"); } static inline void send_rejected_time_too_old(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "time-too-old", 0, 0, 0, -1); send_error_to_client(c, id, "[21,\"time-too-old\",null]"); } static inline void send_rejected_time_too_new(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "time-too-new", 0, 0, 0, -1); send_error_to_client(c, id, "[21,\"time-too-new\",null]"); } static inline void send_rejected_stale_block(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "stale-prevblk", 0, 0, 0, -1); send_error_to_client(c, id, "[21,\"stale-prevblk\",null]"); } static inline void send_rejected_hnotzero_error(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "H-not-zero", 0, 0, 0, -1); send_error_to_client(c, id, "[23,\"H-not-zero\",null]"); } static inline void send_rejected_duplicate(T_DATUM_CLIENT_DATA *c, uint64_t id) { + stratum_log_share_result(c, stratum_share_user_guess(c), false, "duplicate", 0, 0, 0, -1); send_error_to_client(c, id, "[22,\"duplicate\",null]"); } @@ -1357,6 +1476,18 @@ int client_mining_submit(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_obj // update connection and gateway-local totals stratum_note_share(m, true, job_diff); + { + const int missing_zeros = (int)datum_missing_block_zero_bits(share_hash, job->block_target, was_block); + const uint64_t vardiff = m->current_diff ? m->current_diff : job_diff; + long double bld = job->nbits[0] ? calc_network_difficulty(job->nbits) : 0.0L; + const uint64_t blockdiff = (bld > 0.0L && bld < (long double)UINT64_MAX) ? (uint64_t)(bld + 0.5L) : 0; + stratum_log_share_result(c, username_s, true, was_block ? "block" : "ok", job_diff, vardiff, blockdiff, missing_zeros); + if (was_block) { + char hash_hex[65]; + datum_share_hash_to_hex(hash_hex, share_hash); + DLOG_INFO("SHARE %s mode=block d=%" PRIu64 " => submitted", hash_hex, job_diff); + } + } // update since-snap totals m->share_count_since_snap++; From 843c2f41e153d15f42aa01751c3177581d29afee Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:30:33 -0500 Subject: [PATCH 2/8] logger: optional TTY collapse of repeating job and notification lines What this does -------------- When logger.console_collapse_job_updates is true and stdout/stderr is a TTY, repeating INFO lines of these four kinds are rewritten in place instead of scrolling: 1 "Updating standard stratum job for block ..." 2 "Updating priority stratum job for block ..." 3 "NEW NETWORK BLOCK NOTIFICATION RECEIVED" 4 "NEW NETWORK BLOCK: ..." Same kind on the next line becomes a counter after the timestamp: ... INFO: x12 Updating standard stratum job for block ... A notification burst followed by the actual NEW NETWORK BLOCK line is folded into: ... INFO: xN NOTIFICATION + NEW NETWORK BLOCK: () All four kinds use the same in-place path: first line of a run is printed with no newline; later lines are only CR + rewrite + CSI K. CSI 1A (cursor-up) is not used. File logs are untouched and always get every original full line. Non-TTY stdout/stderr (systemd, pipes, script) never collapse. Configuration ------------- logger.console_collapse_job_updates bool default false Why job updates and notifications used to diverge (tmux) -------------------------------------------------------- Notification first-lines are short and usually fit one tmux row, so in-place rewrite looked reliable. Job-update lines are ~180-200 characters (timestamp + 44-char function field + BTC + txn/byte/client counts). In an 80-140 column tmux pane that line wraps. A cursor-up then lands on the wrapped half, so previous full job lines stay on screen. A wide raw tty does not wrap, so the same code looked fine there. An extra eligibility gate (only collapse if linelen+16 < cols) made it worse: long job lines printed in full in tmux; short notification lines still collapsed. That was the "works on a real console, only sometimes in tmux" report. This commit uses one mechanism for every kind, clips the *on-screen* rewrite to cols-1 so tmux cannot wrap it, and drops the length gate. ioctl-failed columns fall back to 80. The file log is never clipped. Benefits -------- - Interactive consoles stop losing the last useful ERROR/WARN under a storm of job updates at low work_update_seconds. - Operators watching a gateway during a block race can still see that work *is* updating (the counter moves). - tmux panes behave like a wide raw tty for these four line kinds. Risks ----- 1. Broken terminals. A client that is a TTY but does not honor CSI will show leftover fragments or a missing newline. Mitigation: default off; collapse only if isatty(). Any non-matching log line flushes a pending collapse with a real newline. 2. Lost history on screen. That is the point. File log remains the record. Do not enable this if the console *is* the record. 3. On-screen truncation in a narrow pane. The live row may lose the tail of a job line (client count, etc.). File log still has it. 4. Kind matching is prefix/strcmp on the message body. If CONVOY changes those four strings, collapse silently stops matching and falls back to normal prints. No functional mining impact. 5. Logger thread only. A crash in ioctl/fileno would be on the logger thread, not stratum. Latency / main thread --------------------- Zero on stratum. Collapse runs when the logger thread drains its queue, which already writes every console line. Extra work per line: isatty, optional TIOCGWINSZ, a few strstr/snprintf, one fwrite. isatty/ioctl every line is the sloppy part (see improvements). Possible improvements --------------------- - Cache isatty + columns and refresh on SIGWINCH instead of ioctl per line. - Collapse from the file-log side too if someone wants a daily log that is not 90% job updates (separate knob). - Match kinds by a logger-side tag (DLOG_INFO_JOB) rather than string prefixes so a wording change cannot break it. - Also fold "Empty work send completed" into the priority-job row; it currently ends the collapse because it is a different message. What to validate before merge ----------------------------- - Default: console identical to CONVOY, including under script(1) / systemd (not a TTY). - Enabled on a raw TTY and in an 80-column tmux pane: repeated "Updating standard stratum job" stays on one row with x2, x3, ... - Notification then NEW NETWORK BLOCK folds; a lone NEW NETWORK BLOCK still prints. - "Server stats" / "Empty work send" end the collapse and start on the next row. - File log still has one full untruncated line per event. - log_calling_function on/off both still match (kind is on msg->msg). --- src/datum_conf.c | 2 + src/datum_conf.h | 1 + src/datum_logger.c | 125 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/datum_conf.c b/src/datum_conf.c index fb44fa02..d4e3585b 100644 --- a/src/datum_conf.c +++ b/src/datum_conf.c @@ -179,6 +179,8 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { .required = false, .ptr = &datum_config.clog_level_file, .default_int = 1 }, { .var_type = DATUM_CONF_BOOL, .category = "logger", .name = "log_shares", .description = "Log the result of every incoming share at INFO (level 2): accepted or rejected with reason", .required = false, .ptr = &datum_config.logger_log_shares, .default_bool = false }, + { .var_type = DATUM_CONF_BOOL, .category = "logger", .name = "console_collapse_job_updates", .description = "Collapse repeating stratum job update logs into a single line counter to prevent console flooding.", + .required = false, .ptr = &datum_config.clog_console_collapse_jobs, .default_bool = false }, // datum options { .var_type = DATUM_CONF_STRING, .category = "datum", .name = "pool_host", .description = "Remote DATUM server host/ip to use for decentralized pooled mining (set to \"\" to disable pooled mining)", diff --git a/src/datum_conf.h b/src/datum_conf.h index f75d31f9..52362809 100644 --- a/src/datum_conf.h +++ b/src/datum_conf.h @@ -157,6 +157,7 @@ typedef struct { bool clog_rotate_daily; char clog_file[1024]; bool logger_log_shares; + bool clog_console_collapse_jobs; char datum_pool_host[1024]; int datum_pool_port; diff --git a/src/datum_logger.c b/src/datum_logger.c index 8f3af90e..8306dd69 100644 --- a/src/datum_logger.c +++ b/src/datum_logger.c @@ -51,9 +51,12 @@ #include #include #include +#include +#include #include "datum_logger.h" #include "datum_utils.h" +#include "datum_conf.h" const char *level_text[] = { " ALL", "DEBUG", " INFO", " WARN", "ERROR", "FATAL" }; @@ -70,6 +73,126 @@ bool log_to_stderr = false; bool log_rotate_daily = true; char log_file[1024] = { 0 }; +static int console_collapse_kind = 0; +static int console_collapse_count = 0; +static int console_collapse_open = 0; + +static int datum_console_job_kind(const char *msg) +{ + if (!msg) return 0; + if (!strncmp(msg, "Updating standard stratum job for block", 39)) return 1; + if (!strncmp(msg, "Updating priority stratum job for block", 39)) return 2; + if (!strcmp(msg, "NEW NETWORK BLOCK NOTIFICATION RECEIVED")) return 3; + if (!strncmp(msg, "NEW NETWORK BLOCK:", 18)) return 4; + return 0; +} + +static int datum_console_is_tty(FILE *out) +{ + int fd; + if (!out) return 0; + fd = fileno(out); + if (fd < 0) return 0; + return isatty(fd); +} + +static int datum_console_cols(FILE *out) +{ + struct winsize ws; + if (ioctl(fileno(out), TIOCGWINSZ, &ws) != 0) return 0; + return (int)ws.ws_col; +} + +static void datum_console_put_inplace(FILE *out, const char *line, int cols) +{ + size_t n; + char buf[1200]; + if (!line) return; + snprintf(buf, sizeof(buf), "%s", line); + n = strlen(buf); + if (n && buf[n - 1] == '\n') buf[--n] = 0; + /* Stay on one visual row so \\r overwrites in tmux. CUU (1A) is wrong + * once the previous line wrapped. */ + if (cols > 8 && (int)n >= cols) { + buf[cols - 1] = 0; + n = (size_t)(cols - 1); + } + if (console_collapse_open) { + fputc('\r', out); + } + fputs(buf, out); + fputs("\033[K", out); + fflush(out); + console_collapse_open = 1; +} + +static void datum_console_write(FILE *out, const char *line_with_nl, const char *msg) +{ + int kind = datum_console_job_kind(msg); + int tty = datum_console_is_tty(out); + int cols = tty ? datum_console_cols(out) : 0; + char rebuilt[1200]; + int can_collapse = datum_config.clog_console_collapse_jobs && tty && kind; + + if (can_collapse && cols <= 0) cols = 80; + + if (!can_collapse) { + if (console_collapse_open) { + fputc('\n', out); + console_collapse_open = 0; + } + console_collapse_kind = 0; + console_collapse_count = 0; + fputs(line_with_nl, out); + return; + } + + if (kind == 4 && console_collapse_kind == 3 && console_collapse_count > 0) { + char base[1200]; + size_t n; + snprintf(base, sizeof(base), "%s", line_with_nl); + n = strlen(base); + if (n && base[n-1] == '\n') base[--n] = 0; + { + const char *colon = strstr(base, ": "); + if (colon) { + size_t pre = (size_t)(colon + 2 - base); + snprintf(rebuilt, sizeof(rebuilt), "%.*sx%d NOTIFICATION + %s", (int)pre, base, console_collapse_count, colon + 2); + } else { + snprintf(rebuilt, sizeof(rebuilt), "x%d NOTIFICATION + %s", console_collapse_count, base); + } + } + datum_console_put_inplace(out, rebuilt, cols); + console_collapse_kind = 4; + console_collapse_count = 1; + return; + } + + if (console_collapse_kind == kind && console_collapse_count > 0 && kind != 4) { + const char *colon; + console_collapse_count++; + colon = strstr(line_with_nl, ": "); + if (colon) { + size_t pre = (size_t)(colon + 2 - line_with_nl); + snprintf(rebuilt, sizeof(rebuilt), "%.*sx%d %s", (int)pre, line_with_nl, console_collapse_count, colon + 2); + } else { + snprintf(rebuilt, sizeof(rebuilt), "%s", line_with_nl); + } + datum_console_put_inplace(out, rebuilt, cols); + return; + } + + if (console_collapse_open) { + fputc('\n', out); + console_collapse_open = 0; + } + console_collapse_kind = kind; + console_collapse_count = 1; + /* First line of a kind is also in-place (no trailing newline) so a + * wrapped job-update cannot sit under a later \\r. */ + datum_console_put_inplace(out, line_with_nl, cols); +} + int dlog_queue_max_entries = 0; int msg_buf_maxsz = DLOG_MSG_BUF_SIZE; @@ -362,7 +485,7 @@ void * datum_logger_thread(void *ptr) { log_line[1199] = 0; if ((log_to_console) && (msg->level >= log_level_console)) { - fprintf(log_to_stderr?stderr:stdout, "%s", log_line); + datum_console_write(log_to_stderr?stderr:stdout, log_line, msg->msg); } if ((log_to_file) && (msg->level >= log_level_file)) { From b4d8ab8a3db2070ec594a0b760a06cfb8542718a Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:30:33 -0500 Subject: [PATCH 3/8] mining: optional sampled node-side share validation (proposal / submitblock) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What this does -------------- When mining.validate_shares_on_node is true, a *sample* of accepted non-block shares is assembled into the same BLAKE2b v2 block hex the gateway would submit, then sent to the local node on a detached thread: share_node_check=proposal getblocktemplate mode=proposal TestBlockValidity(..., check_pow=false) share_node_check=submitblock submitblock CheckProofOfWork first; sub-target shares come back as high-hash Results are logged as: SHARE <64 hex> mode=proposal d= => null (seems valid) SHARE <64 hex> mode=proposal d= => bad-txnmrklroot SHARE <64 hex> mode=proposal d= => transport/HTTP error (no JSON) SHARE <64 hex> mode=submitblock d= => high-hash Real blocks are never sent through this path. They already go through assembleBlockAndSubmit + datum_submitblock_trigger. was_block returns immediately. At most one node-check RPC is in flight. Further candidates are skipped with a DEBUG line. That is the load shed. Sampling: mining.share_node_check_missingzeros < 0 take 1 of every mining.share_node_check_every accepted shares (default every=16) mining.share_node_check_missingzeros >= 0 ignore the 1-of-N sampler; only check shares with missingzeros <= that value (same gate as log_shares) Assembly uses CONVOY's own v2 header serialize (datum_blake2b_serialize_block_header), witness-aware coinbase hex, and the job's txn list. If proposal says the block is invalid, the template/header/coinbase packing is wrong — not "the miner is weak." Configuration ------------- mining.validate_shares_on_node bool default false mining.share_node_check string default "proposal" mining.share_node_check_every int default 16 mining.share_node_check_missingzeros int default -1 (declared in 01-log-shares) Benefits -------- - Catches silent packing bugs (wrong witness, wrong v2 fields, stale prevhdr) on shares that will never meet nBits, which submitblock would hide behind high-hash. - Gives lab operators a node-backed "this share is a real block except PoW" signal without submitting junk to the network. - Cheap enough to leave on at missingzeros=2 during a fork or gateway upgrade. Risks ----- 1. Extra bitcoind RPC. proposal is not free; a large block hex on every share would hurt the node. Mitigations: default off, 1-of-N or missingzeros gate, one-in-flight CAS, detached thread, never on was_block. 2. submitblock mode on a *valid* near-block share that somehow meets nBits would submit a real block through a second path. The was_block early-out plus "we only get here after the share was already classified" is the guard. Do not set share_node_check=submitblock unless you understand high-hash will dominate. 3. proposal on an old tip can return stale-prevblk after a race. That is useful, not a false "gateway is broken." 4. malloc of the block hex can fail; we drop the check and clear the in-flight flag. No retry. 5. Detached thread leak if pthread_create succeeds but the worker never runs — we only clear in-flight in the worker or on create failure. Same pattern as other gateway detached work. 6. Hex assembly duplication vs assembleBlockAndSubmit. If one path is updated and the other is not, node-check can lie. Reviewers should diff datum_write_assembled_block_hex against the submit builder when touching either. Latency / main thread --------------------- The stratum worker: filter arithmetic, one CAS, size estimate, two mallocs, a full block hex sprintf, one more malloc for the JSON wrapper, pthread_create. The hex walk is the costly part (copies every txn hex). That is why sampling and one-in-flight exist. The RPC itself (curl + bitcoind) runs on the detached worker. It does not block mining.submit, vardiff, or job broadcast. Worst case for the main worker is "we built a hex and spawned a thread." Do not point this at a remote bitcoind over a high-latency link with every=1. Possible improvements --------------------- - Build hex once and share it with assembleBlockAndSubmit on the was_block path (one serializer). - Reuse a thread-local curl handle instead of curl_easy_init per check. - Cap hex_guess / reject absurd templates so a malformed txn_total_size cannot malloc hundreds of MB on the stratum thread. - Bounded worker queue instead of skip-if-busy, if someone wants "check the closest share that arrived during the last RPC." - Unit test: known job + coinbase => proposal hex bit-identical to submitblock hex. What to validate before merge ----------------------------- - Default: no extra RPC, no extra threads, submit path unchanged. - proposal + missingzeros=2 against a real Knots node: near shares log "null (seems valid)"; a deliberately broken coinbase logs a reject reason, not a crash. - every=1 with a CPU miner does not stall stratum accepts; in-flight skips show at DEBUG. - was_block still only uses assembleBlockAndSubmit; no SHARE mode=proposal line for a found block. - Pooled mode: node-check does not talk to Prime and does not change DATUM pow_submit. - Memory: watch RSS while a 4 MB template is being checked; should return to baseline after the worker frees req+job. Includes Insulince's share-check improvements from the innerhat / test/insulince tree. Co-authored-by: Justin --- src/datum_conf.c | 6 ++ src/datum_conf.h | 3 + src/datum_stratum.c | 194 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+) diff --git a/src/datum_conf.c b/src/datum_conf.c index d4e3585b..56694fc8 100644 --- a/src/datum_conf.c +++ b/src/datum_conf.c @@ -129,6 +129,12 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { .required = false, .ptr = datum_config.mining_save_submitblocks_dir, .default_string[0] = "", .max_string_len = sizeof(datum_config.mining_save_submitblocks_dir) }, { .var_type = DATUM_CONF_INT, .category = "mining", .name = "share_node_check_missingzeros", .description = "If >= 0, only log SHARE lines and node-check shares with missingzeros <= this value (overrides share_node_check_every). -1 = infinity (log all, use every-N).", .required = false, .ptr = &datum_config.mining_share_node_check_missingzeros, .default_int = -1 }, + { .var_type = DATUM_CONF_BOOL, .category = "mining", .name = "validate_shares_on_node", .description = "Ask the local node about sampled accepted shares (proposal or submitblock). Real blocks are always submitted separately.", + .required = false, .ptr = &datum_config.mining_validate_shares_on_node, .default_bool = false }, + { .var_type = DATUM_CONF_STRING, .category = "mining", .name = "share_node_check", .description = "How to ask the node about a share: proposal (GBT mode=proposal, skips PoW) or submitblock (PoW checked first, usually high-hash).", + .required = false, .ptr = datum_config.mining_share_node_check, .default_string[0] = "proposal", .max_string_len = sizeof(datum_config.mining_share_node_check) }, + { .var_type = DATUM_CONF_INT, .category = "mining", .name = "share_node_check_every", .description = "Validate 1 of every N accepted shares on the node (1 = every share). Real blocks are always submitted separately.", + .required = false, .ptr = &datum_config.mining_share_node_check_every, .default_int = 16 }, { .var_type = DATUM_CONF_BOOL, .category = "mining", .name = "allow_hasher_time_rolling", .description = "Allow hasher time rolling for BLAKE2b jobs", .required = false, .ptr = &datum_config.mining_allow_hasher_time_rolling, .default_bool = false }, { .var_type = DATUM_CONF_BOOL, .category = "mining", .name = "abw_verify_all_shares_on_disclosure", .description = "Retain ABW proofs until key disclosure and alarm if the pool ignored a block", diff --git a/src/datum_conf.h b/src/datum_conf.h index 52362809..2d442b9a 100644 --- a/src/datum_conf.h +++ b/src/datum_conf.h @@ -132,6 +132,9 @@ typedef struct { char mining_coinbase_tag_secondary[64]; char mining_save_submitblocks_dir[256]; int mining_share_node_check_missingzeros; + bool mining_validate_shares_on_node; + char mining_share_node_check[32]; + int mining_share_node_check_every; bool mining_allow_hasher_time_rolling; bool mining_abw_verify_all_shares_on_disclosure; int coinbase_unique_id; diff --git a/src/datum_stratum.c b/src/datum_stratum.c index 01e72c20..c8547083 100644 --- a/src/datum_stratum.c +++ b/src/datum_stratum.c @@ -810,6 +810,7 @@ static void datum_share_hash_to_hex(char *out, const unsigned char *hash_le) { out[64] = 0; } +static void datum_maybe_validate_share_on_node(uint8_t *block_header, uint8_t *coinbase_txn, size_t coinbase_txn_size, T_DATUM_STRATUM_JOB *job, bool empty_work, const unsigned char *extranonce, const unsigned char *share_hash_le, uint64_t diff, bool was_block, int missing_zeros); static void stratum_log_share_result(const T_DATUM_CLIENT_DATA *c, const char *username, bool accepted, const char *reason, uint64_t job_diff, uint64_t vardiff, uint64_t blockdiff, int missing_zeros) { const char *host; @@ -1487,6 +1488,7 @@ int client_mining_submit(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_obj datum_share_hash_to_hex(hash_hex, share_hash); DLOG_INFO("SHARE %s mode=block d=%" PRIu64 " => submitted", hash_hex, job_diff); } + datum_maybe_validate_share_on_node(block_header, full_cb_txn, full_cb_txn_size, job, empty_work, extranonce_bin, share_hash, job_diff, was_block, missing_zeros); } // update since-snap totals @@ -2458,6 +2460,198 @@ bool datum_stratum_abw_finalize_block_request(char *request, size_t request_size return true; } +struct datum_share_node_check_job { + char *req; + char mode[16]; + char hash[65]; + uint64_t diff; +}; + +static volatile int datum_share_node_check_in_flight = 0; + +static void datum_share_node_check_clear_in_flight(void) { + __atomic_store_n(&datum_share_node_check_in_flight, 0, __ATOMIC_RELEASE); +} + +static void *datum_share_node_check_thread(void *arg) { + struct datum_share_node_check_job *job = arg; + CURL *curl; + json_t *r = NULL, *result = NULL, *err = NULL; + char *dump; + + curl = curl_easy_init(); + if (!curl) { + DLOG_WARN("SHARE node-check could not init curl"); + goto out; + } + + r = bitcoind_json_rpc_call(curl, &datum_config, job->req); + curl_easy_cleanup(curl); + + if (r) { + result = json_object_get(r, "result"); + err = json_object_get(r, "error"); + } + + if (r && result && json_is_null(result) && (!err || json_is_null(err))) { + DLOG_INFO("SHARE %s mode=%s d=%" PRIu64 " => null (seems valid)", job->hash, job->mode, job->diff); + } else if (!r) { + DLOG_INFO("SHARE %s mode=%s d=%" PRIu64 " => transport/HTTP error (no JSON)", job->hash, job->mode, job->diff); + } else if (err && !json_is_null(err)) { + dump = json_dumps(err, JSON_ENCODE_ANY); + DLOG_INFO("SHARE %s mode=%s d=%" PRIu64 " => RPC error %s", job->hash, job->mode, job->diff, dump ? dump : "(unknown)"); + free(dump); + } else if (result && json_is_string(result)) { + DLOG_INFO("SHARE %s mode=%s d=%" PRIu64 " => %s", job->hash, job->mode, job->diff, json_string_value(result)); + } else { + dump = json_dumps(result, JSON_ENCODE_ANY); + DLOG_INFO("SHARE %s mode=%s d=%" PRIu64 " => %s", job->hash, job->mode, job->diff, dump ? dump : "(unknown)"); + free(dump); + } + + if (r) json_decref(r); +out: + free(job->req); + free(job); + datum_share_node_check_clear_in_flight(); + return NULL; +} + +static char *datum_write_assembled_block_hex(char *ptr, uint8_t *block_header, uint8_t *coinbase_txn, size_t coinbase_txn_size, T_DATUM_STRATUM_JOB *job, bool empty_work, const unsigned char *extranonce) { + size_t i; + unsigned char v2hdr[DATUM_BLAKE2B_BLOCK_HEADER_SIZE]; + unsigned char merkle[32]; + unsigned char en[12]; + unsigned char cbh[32]; + uint16_t txcount; + bool add_witness; + char cbhex[(MAX_COINBASE_TXN_SIZE_BYTES + 36) * 2 + 1]; + size_t cbhex_len; + const T_DATUM_TEMPLATE_DATA *td; + + td = job->block_template; + txcount = (uint16_t)((empty_work ? 0 : td->txn_count) + 1); + if ((job->merklebranch_count) && (!empty_work)) { + double_sha256(cbh, coinbase_txn, coinbase_txn_size); + stratum_job_merkle_root_calc(job, cbh, merkle); + } else { + double_sha256(merkle, coinbase_txn, coinbase_txn_size); + } + memset(en, 0, sizeof(en)); + if (extranonce) memcpy(en, extranonce, 12); + datum_blake2b_serialize_block_header(v2hdr, job->version_uint, job->prevhash_bin, merkle, job->blake2b_time_on_wire, job->nbits_uint, block_header + 32, block_header + 40, en, txcount, job->blake2b_flags, 0, (const unsigned char[16]){0}, (uint32_t)job->height, (const unsigned char[32]){0}); + for(i=0;iblock_template->txn_count + 1, ptr); + } else { + ptr += append_bitcoin_varint_hex(1, ptr); + } + add_witness = datum_stratum_block_needs_witness(job, empty_work); + cbhex_len = datum_stratum_coinbase_for_block_hex(cbhex, sizeof(cbhex), coinbase_txn, coinbase_txn_size, add_witness); + if (!cbhex_len) { + for(i=0;iblock_template->txn_count;i++) { + memcpy(ptr, job->block_template->txns[i].txn_data_hex, job->block_template->txns[i].size*2); + ptr += job->block_template->txns[i].size*2; + } + } + *ptr = 0; + return ptr; +} + +static void datum_maybe_validate_share_on_node(uint8_t *block_header, uint8_t *coinbase_txn, size_t coinbase_txn_size, T_DATUM_STRATUM_JOB *job, bool empty_work, const unsigned char *extranonce, const unsigned char *share_hash_le, uint64_t diff, bool was_block, int missing_zeros) { + static uint64_t accepted_seen = 0; + uint64_t n; + int every; + size_t hex_guess; + char *buf, *hex_end, *req; + struct datum_share_node_check_job *cj; + pthread_t thr; + pthread_attr_t attr; + const char *mode; + size_t i; + + if (was_block) return; + if (!datum_config.mining_validate_shares_on_node) return; + if (!job || !job->block_template || !block_header || !coinbase_txn) return; + + if (datum_config.mining_share_node_check_missingzeros >= 0) { + if (missing_zeros < 0 || missing_zeros > datum_config.mining_share_node_check_missingzeros) return; + } else { + every = datum_config.mining_share_node_check_every; + if (every < 1) every = 1; + n = __atomic_add_fetch(&accepted_seen, 1, __ATOMIC_RELAXED); + if ((n % (uint64_t)every) != 0) return; + } + if (!__sync_bool_compare_and_swap(&datum_share_node_check_in_flight, 0, 1)) { + DLOG_DEBUG("SHARE node-check skipped (already in flight)"); + return; + } + + mode = datum_config.mining_share_node_check[0] ? datum_config.mining_share_node_check : "proposal"; + hex_guess = 1024 + coinbase_txn_size * 2; + if (!empty_work) { + for (i = 0; i < job->block_template->txn_count; i++) { + hex_guess += job->block_template->txns[i].size * 2; + } + } + buf = malloc(hex_guess + 8); + if (!buf) { + datum_share_node_check_clear_in_flight(); + return; + } + hex_end = datum_write_assembled_block_hex(buf, block_header, coinbase_txn, coinbase_txn_size, job, empty_work, extranonce); + if (!hex_end) { + free(buf); + datum_share_node_check_clear_in_flight(); + return; + } + + req = malloc(strlen(buf) + 192); + if (!req) { + free(buf); + datum_share_node_check_clear_in_flight(); + return; + } + if (strcasecmp(mode, "submitblock") == 0) { + sprintf(req, "{\"jsonrpc\":\"1.0\",\"id\":\"sharecheck\",\"method\":\"submitblock\",\"params\":[\"%s\"]}", buf); + } else { + sprintf(req, "{\"jsonrpc\":\"1.0\",\"id\":\"sharecheck\",\"method\":\"getblocktemplate\",\"params\":[{\"mode\":\"proposal\",\"data\":\"%s\"}]}", buf); + } + free(buf); + + cj = calloc(1, sizeof(*cj)); + if (!cj) { + free(req); + datum_share_node_check_clear_in_flight(); + return; + } + cj->req = req; + strncpy(cj->mode, mode, sizeof(cj->mode) - 1); + datum_share_hash_to_hex(cj->hash, share_hash_le); + cj->diff = diff; + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + if (pthread_create(&thr, &attr, datum_share_node_check_thread, cj) != 0) { + free(cj->req); + free(cj); + pthread_attr_destroy(&attr); + datum_share_node_check_clear_in_flight(); + return; + } + pthread_attr_destroy(&attr); +} + int assembleBlockAndSubmit(uint8_t *block_header, uint8_t *coinbase_txn, size_t coinbase_txn_size, T_DATUM_STRATUM_JOB *job, T_DATUM_STRATUM_THREADPOOL_DATA *sdata, const char *block_hash_hex, bool empty_work, const unsigned char *extranonce) { // TODO: Also submit directly to bitcoin P2P char *submitblock_req = NULL; From 862a413049c9a25dea6101fe8b2887362d8a65ab Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:30:33 -0500 Subject: [PATCH 4/8] mining: name saved submitblocks by height and hash, fsync, last.json What this does -------------- CONVOY already writes the assembled submitblock JSON-RPC request to disk when mining.save_submitblocks_dir is set. The payload is the full block: v2 header hex + varint tx count + coinbase + every template transaction, wrapped as {"method":"submitblock","params":["..."]}. That is exactly what bitcoin-cli submitblock / a watchdog would replay. Stock naming was only the hash: /datum_submitblock_.json This commit keeps the stock knob and the stock timing (write on the submit thread after datum_submitblock_trigger, before the inline bitcoind RPC) and changes the files to: /datum_submitblock__.json /datum_submitblock_last.json The unique file is never overwritten. The last file is the stable path for "what just happened." Both writes are fwrite of the exact assembled length, fflush, fsync, then fclose. Success is an INFO line with byte count and path. Failure is ERROR and does not change the submitblock return value. Startup logs the directory when the knob is set: submitblock save dir: /tmp/datum_blocks (datum_submitblock__.json + datum_submitblock_last.json) There is no second config key. mining.dump_submitblock_path is not added. A detached 500 ms dump thread is not added. datum_submitblock_trigger remains the async/redundant submit path; delaying the backup copy would only risk losing it. Configuration ------------- mining.save_submitblocks_dir string default "" (disabled) The directory must already exist. fopen does not mkdir. Benefits -------- - Height in the filename matches how operators look at logs ("block 123") without opening the JSON. - last.json gives a single path for a lab watcher or scp. - fsync makes the file durable if the process dies in the RPC that follows. - Same payload stock already wrote. No second serializer. Risks ----- 1. Sync write of a multi-MB JSON on the submit thread. CONVOY already accepted that cost; this adds fsync. Block-found is rare. The dedicated submit thread is already running. 2. last.json is overwritten. Two blocks in the same second leave only the latest last.json; the unique height_hash files remain. 3. No directory creation. A bad path is ERROR, submit still runs. 4. The file is a full block. Do not put the directory on a shared or untrusted volume. 5. Height 0 if job is NULL. Should not happen on the real path. Latency / main thread --------------------- On assembleBlockAndSubmit only (already a block-found rarity): two fopen/fwrite/fsync/fclose. Does not delay datum_submitblock_trigger. The inline curl submitblock still follows the writes, which is the stock order. Possible improvements --------------------- - mkdir -p for the configured directory. - Write to .tmp and rename so a watcher never reads a truncated JSON. - Also write a raw .blk (header+txs without the JSON wrapper). What to validate before merge ----------------------------- - Default "": no files, no extra INFO, submit unchanged. - Dir="/tmp/datum_blocks": after a found block, both files appear immediately, json_load_file succeeds, INFO lines show the paths. - Height in the unique filename matches the job height in the log. - fopen failure does not change submitblock return value. - Startup prints the save-dir line only when the knob is non-empty. --- src/datum_conf.c | 2 +- src/datum_gateway.c | 7 +++++++ src/datum_stratum.c | 50 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/datum_conf.c b/src/datum_conf.c index 56694fc8..18ca6a6c 100644 --- a/src/datum_conf.c +++ b/src/datum_conf.c @@ -125,7 +125,7 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { .required = false, .ptr = datum_config.mining_coinbase_tag_secondary, .default_string[0] = "DATUM User", .max_string_len = sizeof(datum_config.mining_coinbase_tag_secondary) }, { .var_type = DATUM_CONF_INT, .category = "mining", .name = "coinbase_unique_id", .description = "A unique ID between 1 and 65535. This is appended to the coinbase. Make unique per instance of datum with the same coinbase tags.", .required = false, .ptr = &datum_config.coinbase_unique_id, .default_int = 4242 }, - { .var_type = DATUM_CONF_STRING, .category = "mining", .name = "save_submitblocks_dir", .description = "Directory to save all submitted blocks to as submitblock JSON files", + { .var_type = DATUM_CONF_STRING, .category = "mining", .name = "save_submitblocks_dir", .description = "Directory to save submitted blocks as submitblock JSON (datum_submitblock__.json and datum_submitblock_last.json). Empty disables.", .required = false, .ptr = datum_config.mining_save_submitblocks_dir, .default_string[0] = "", .max_string_len = sizeof(datum_config.mining_save_submitblocks_dir) }, { .var_type = DATUM_CONF_INT, .category = "mining", .name = "share_node_check_missingzeros", .description = "If >= 0, only log SHARE lines and node-check shares with missingzeros <= this value (overrides share_node_check_every). -1 = infinity (log all, use every-N).", .required = false, .ptr = &datum_config.mining_share_node_check_missingzeros, .default_int = -1 }, diff --git a/src/datum_gateway.c b/src/datum_gateway.c index 88e786c5..baf52096 100644 --- a/src/datum_gateway.c +++ b/src/datum_gateway.c @@ -197,6 +197,13 @@ int main(const int argc, const char * const * const argv) { // Initialize logger thread datum_logger_init(); + if (datum_config.mining_save_submitblocks_dir[0]) { + char gitshort[8]; + memset(gitshort, 0, sizeof(gitshort)); + memcpy(gitshort, GIT_COMMIT_HASH, 7); + DLOG_INFO("submitblock save dir: %s (datum_submitblock__%s.json + datum_submitblock_last.json)", + datum_config.mining_save_submitblocks_dir, gitshort[0] ? gitshort : "unknown"); + } if (datum_protocol_init()) { DLOG_FATAL("Error initializing the DATUM protocol!"); diff --git a/src/datum_stratum.c b/src/datum_stratum.c index c8547083..8d496c76 100644 --- a/src/datum_stratum.c +++ b/src/datum_stratum.c @@ -2752,22 +2752,50 @@ int assembleBlockAndSubmit(uint8_t *block_header, uint8_t *coinbase_txn, size_t // this way we can have a (future) external watchdog monitoring the folder as a backup to submit the blocks if need be // for added security. The thread above should already be submitting this block anyway. if (datum_config.mining_save_submitblocks_dir[0] != 0) { - // save the block submission to a file named by the block's hash - char submitblockpath[384]; - int n = snprintf(submitblockpath, sizeof(submitblockpath), "%s/datum_submitblock_%s.json", datum_config.mining_save_submitblocks_dir, block_hash_hex); - - if (n >= sizeof(submitblockpath)) { + char unique_path[384]; + char last_path[384]; + const char *dir = datum_config.mining_save_submitblocks_dir; + size_t req_len = (size_t)(ptr - submitblock_req); + uint64_t height = job ? job->height : 0; + char gitshort[8]; + int n; + memset(gitshort, 0, sizeof(gitshort)); + memcpy(gitshort, GIT_COMMIT_HASH, 7); + n = snprintf(unique_path, sizeof(unique_path), "%s/datum_submitblock_%lu_%s.json", + dir, (unsigned long)height, gitshort[0] ? gitshort : "unknown"); + if (n < 0 || n >= (int)sizeof(unique_path)) { DLOG_ERROR("Overflow in construction of submitblock path!"); } else { - FILE *f; - f = fopen(submitblockpath, "w"); + FILE *f = fopen(unique_path, "w"); + size_t wr; + if (!f) { + DLOG_ERROR("Could not open %s for writing submitblock record: %s", unique_path, strerror(errno)); + } else { + wr = fwrite(submitblock_req, 1, req_len, f); + fflush(f); + if (fileno(f) >= 0) fsync(fileno(f)); + if (fclose(f) != 0 || wr != req_len) { + DLOG_ERROR("Could not write %s submitblock record: %s", unique_path, strerror(errno)); + } else { + DLOG_INFO("Wrote submitblock dump (%zu bytes) to %s", wr, unique_path); + } + } + } + n = snprintf(last_path, sizeof(last_path), "%s/datum_submitblock_last.json", dir); + if (n > 0 && n < (int)sizeof(last_path)) { + FILE *f = fopen(last_path, "w"); + size_t wr; if (!f) { - DLOG_ERROR("Could not open %s for writing submitblock record to disk: %s!", submitblockpath, strerror(errno)); + DLOG_ERROR("Could not open %s for writing submitblock record: %s", last_path, strerror(errno)); } else { - if (!fwrite(submitblock_req, ptr-submitblock_req, 1, f)) { - DLOG_ERROR("Could not write to %s when writing submitblock record to disk: %s!", submitblockpath, strerror(errno)); + wr = fwrite(submitblock_req, 1, req_len, f); + fflush(f); + if (fileno(f) >= 0) fsync(fileno(f)); + if (fclose(f) != 0 || wr != req_len) { + DLOG_ERROR("Could not write %s submitblock record: %s", last_path, strerror(errno)); + } else { + DLOG_INFO("Wrote submitblock dump (%zu bytes) to %s", wr, last_path); } - fclose(f); } } } From b88e146d456faa13a77ac7ed2b9c9e88ce36813f Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:30:34 -0500 Subject: [PATCH 5/8] stratum: optional second listen port for small-coinbase (legacy) miners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What this does -------------- CONVOY already builds six coinbase sizes per job (MAX_COINBASE_TYPES). Type 2 is the ~755 byte Antminer-safe cap and the default after subscribe. Type 4 is the 16 KB window. Fingerprint-by-UA already picks among them. This adds a second bind so the *port* can force that choice instead of a second gateway process or a second template thread. stratum.listen_port default 23334 stratum.legacy_listen_port default 0 (disabled) When legacy_listen_port is set and not equal to listen_port: clients accepted on listen_port coinbase type 4 (16 KB) clients accepted on legacy_listen_port coinbase type 2 (~755 B) Same template thread, same job cache, same worker pool. The listener accepts on both fds, tags T_DATUM_CLIENT_DATA.accepted_listen_port, and subscribe overrides coinbase_selection after fingerprint runs. A subscribe INFO line reports ua, port, and selected type. legacy_listen_port 0 (default) leaves stock behavior: type 2 plus UA fingerprint, single listen socket. There is no second job #1, no second GBT, and no 17-output aggregator. Type 2 is already the small coinbase old boards tolerate. Configuration ------------- stratum.listen_addr string unchanged stratum.listen_port int default 23334 stratum.legacy_listen_port int default 0 Keys live under "stratum". Flat names like stratum_port / legacy_stratum_port are not parsed. Benefits -------- - One process can feed modern rigs a full coinbase window and old Antminer-class boards a size they will not reject, without two gateways and two template polls. - Port policy is explicit. Fingerprint still runs; the port override wins when the legacy port is enabled. - Default remains a single port and type-2 default, so stock configs do not change behavior. Risks ----- 1. Two listen fds. Bind failure on the legacy port aborts the listener thread the same way a primary bind failure does. 2. IPv4+IPv6 on each port. listen_socks grew from 2 to 8. Four sockets (two ports × two families) is the expected case. 3. accepted_listen_port is 0 until assign_to_thread. Subscribe before that cannot happen on this codebase. 4. Forcing type 4 on the primary port can still break a miner that connected to the "modern" port but cannot take 16 KB. That is the point of the split; put that miner on the legacy port. 5. Fingerprint-set difficulty (NiceHash high min diff) is kept; only coinbase_selection is overridden by port. Latency / main thread --------------------- Accept path: one extra int copy onto the client slot. Subscribe: two integer compares and one INFO log when the legacy port is enabled. No extra template work. Jobs already carry all six coinbases. Possible improvements --------------------- - Per-port vardiff / min-diff so NiceHash-class clients can sit on the small port without fingerprint. - Config to choose which type each port forces (not hard-coded 4/2). - Log listen addresses at startup for both ports. What to validate before merge ----------------------------- - Default legacy_listen_port=0: single listen, type 2 + fingerprint unchanged, no new INFO on subscribe. - listen_port=23334, legacy_listen_port=23335: both accept, subscribe logs type 4 vs type 2, miners on each port get the matching coinbase size in mining.notify. - Same job id on both ports (one template). - Fingerprint still adjusts NiceHash min diff on either port. - Binding legacy to the same number as listen_port is a no-op (second bind skipped). --- doc/example_datum_gateway_config.json | 3 +- src/datum_conf.c | 4 +- src/datum_conf.h | 1 + src/datum_sockets.c | 53 ++++++++++++++++++++++----- src/datum_sockets.h | 3 +- src/datum_stratum.c | 10 +++++ 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/doc/example_datum_gateway_config.json b/doc/example_datum_gateway_config.json index 667db6d7..7d2de0a0 100644 --- a/doc/example_datum_gateway_config.json +++ b/doc/example_datum_gateway_config.json @@ -6,7 +6,8 @@ "notify_fallback": true }, "stratum": { - "listen_port": 23334 + "listen_port": 23334, + "legacy_listen_port": 0 }, "mining": { "pool_address": "put your own Bitcoin invoice address here", diff --git a/src/datum_conf.c b/src/datum_conf.c index 18ca6a6c..0ee8ed52 100644 --- a/src/datum_conf.c +++ b/src/datum_conf.c @@ -82,9 +82,11 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { // stratum v1 server configs { .var_type = DATUM_CONF_STRING, .category = "stratum", .name = "listen_addr", .description = "IP address to listen for Stratum Gateway connections", .required = false, .ptr = datum_config.stratum_v1_listen_addr, .default_string[0] = "", .max_string_len = sizeof(datum_config.stratum_v1_listen_addr) }, - { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "listen_port", .description = "Listening port for Stratum Gateway", + { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "listen_port", .description = "Listening port for Stratum Gateway (full 16KB coinbase / type 4 when legacy_listen_port is also set)", .example_default = true, .required = false, .ptr = &datum_config.stratum_v1_listen_port, .default_int = 23334 }, + { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "legacy_listen_port", .description = "Optional second Stratum port for old ASICs. Uses the small Antminer-safe coinbase (type 2, ~755 bytes). 0 disables.", + .required = false, .ptr = &datum_config.stratum_v1_legacy_listen_port, .default_int = 0 }, { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "max_clients_per_thread", .description = "Maximum clients per Stratum server thread", .required = false, .ptr = &datum_config.stratum_v1_max_clients_per_thread, .default_int = 128 }, { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "max_threads", .description = "Maximum Stratum server threads", diff --git a/src/datum_conf.h b/src/datum_conf.h index 2d442b9a..a53399b9 100644 --- a/src/datum_conf.h +++ b/src/datum_conf.h @@ -110,6 +110,7 @@ typedef struct { char stratum_v1_listen_addr[128]; int stratum_v1_listen_port; + int stratum_v1_legacy_listen_port; int stratum_v1_max_clients; int stratum_v1_max_threads; int stratum_v1_max_clients_per_thread; diff --git a/src/datum_sockets.c b/src/datum_sockets.c index 80d68ea4..189a24db 100644 --- a/src/datum_sockets.c +++ b/src/datum_sockets.c @@ -397,7 +397,7 @@ void clean_thread_data(T_DATUM_THREAD_DATA *d, T_DATUM_SOCKET_APP *app) { d->app = app; } -int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd) { +int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd, int accepted_listen_port) { // Only one thread will be calling this function for a particular "app" // under the current design. Safe to assume that multiple clients will // not cause overlap here. @@ -542,6 +542,7 @@ int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd) { // clear up and prep slot's client data without clobbering app_client_data app->datum_threads[tid].client_data[cid].fd = fd; app->datum_threads[tid].client_data[cid].cid = cid; + app->datum_threads[tid].client_data[cid].accepted_listen_port = accepted_listen_port; app->datum_threads[tid].client_data[cid].new_connection = true; app->datum_threads[tid].client_data[cid].datum_thread = (void *)&app->datum_threads[tid]; app->datum_threads[tid].client_data[cid].in_buf = 0; @@ -678,7 +679,10 @@ void *datum_gateway_listener_thread(void *arg) { T_DATUM_SOCKET_APP *app = (T_DATUM_SOCKET_APP *)arg; struct epoll_event ev, events[MAX_EVENTS]; - int listen_socks[2], conn_sock, nfds, epollfd; + int listen_socks[8]; + int listen_sock_ports[8]; + int conn_sock, nfds, epollfd; + int nlisten = 0; if (!app) { DLOG_FATAL("Called without application data structure. :("); @@ -707,11 +711,33 @@ void *datum_gateway_listener_thread(void *arg) { app->datum_active_threads = 0; - size_t listen_socks_len = 2; - if (!datum_sockets_setup_listening_sockets("stratum", datum_config.stratum_v1_listen_addr, app->listen_port, listen_socks, &listen_socks_len)) { - return NULL; + { + int tmp[2]; + size_t tmpn; + unsigned p; + tmpn = 2; + if (!datum_sockets_setup_listening_sockets("stratum", datum_config.stratum_v1_listen_addr, app->listen_port, tmp, &tmpn)) { + return NULL; + } + for (p = 0; p < tmpn && nlisten < 8; p++) { + listen_socks[nlisten] = tmp[p]; + listen_sock_ports[nlisten] = app->listen_port; + nlisten++; + } + if (datum_config.stratum_v1_legacy_listen_port > 0 && + datum_config.stratum_v1_legacy_listen_port != app->listen_port) { + tmpn = 2; + if (!datum_sockets_setup_listening_sockets("stratum-legacy", datum_config.stratum_v1_listen_addr, (uint16_t)datum_config.stratum_v1_legacy_listen_port, tmp, &tmpn)) { + return NULL; + } + for (p = 0; p < tmpn && nlisten < 8; p++) { + listen_socks[nlisten] = tmp[p]; + listen_sock_ports[nlisten] = datum_config.stratum_v1_legacy_listen_port; + nlisten++; + } + DLOG_INFO("Stratum also listening on legacy port %d (small coinbase)", datum_config.stratum_v1_legacy_listen_port); + } } - if (listen_socks_len < 2) listen_socks[1] = -1; epollfd = epoll_create1(0); if (epollfd < 0) { @@ -720,7 +746,7 @@ void *datum_gateway_listener_thread(void *arg) { return NULL; } - for (i = 0; i < 2; ++i) { + for (i = 0; i < nlisten; ++i) { if (listen_socks[i] == -1) continue; ev.events = EPOLLIN; ev.data.fd = listen_socks[i]; @@ -747,7 +773,16 @@ void *datum_gateway_listener_thread(void *arg) { } } for (int n = 0; n < nfds; ++n) { - if (events[n].data.fd == listen_socks[0] || events[n].data.fd == listen_socks[1]) { + int accepted_port = app->listen_port; + int is_listen = 0; + for (i = 0; i < nlisten; i++) { + if (events[n].data.fd == listen_socks[i]) { + is_listen = 1; + accepted_port = listen_sock_ports[i]; + break; + } + } + if (is_listen) { conn_sock = accept(events[n].data.fd, NULL, NULL); if (conn_sock < 0) { DLOG_ERROR("accept failed: %s", strerror(errno)); @@ -773,7 +808,7 @@ void *datum_gateway_listener_thread(void *arg) { } // assign socket to a thread - i = assign_to_thread(app, conn_sock); + i = assign_to_thread(app, conn_sock, accepted_port); if (!i) { // error finding a thread (too many connections?) DLOG_DEBUG("Closing socket we couldn't assign %d", conn_sock); diff --git a/src/datum_sockets.h b/src/datum_sockets.h index f3c32a3d..768bd720 100644 --- a/src/datum_sockets.h +++ b/src/datum_sockets.h @@ -75,6 +75,7 @@ typedef struct T_DATUM_CLIENT_DATA { int out_buf; char rem_host[DATUM_MAX_IP_LEN+1]; + int accepted_listen_port; bool kill_request; @@ -172,7 +173,7 @@ void *datum_gateway_listener_thread(void *arg); int datum_socket_send_string_to_client(T_DATUM_CLIENT_DATA *c, char *s); int datum_socket_send_chars_to_client(T_DATUM_CLIENT_DATA *c, char *s, int len); -int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd); +int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd, int accepted_listen_port); void *datum_threadpool_thread(void *arg); static inline void datum_socket_thread_client_count_decrement(T_DATUM_THREAD_DATA *my, int cid_who_left, bool not_already_locked) { diff --git a/src/datum_stratum.c b/src/datum_stratum.c index 8d496c76..f17d1505 100644 --- a/src/datum_stratum.c +++ b/src/datum_stratum.c @@ -1858,6 +1858,16 @@ int client_mining_subscribe(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_ m->current_diff = datum_config.stratum_v1_vardiff_min; } } + if (datum_config.stratum_v1_legacy_listen_port > 0) { + if (c->accepted_listen_port == datum_config.stratum_v1_legacy_listen_port) { + m->coinbase_selection = 2; /* Antminer-safe ~755 B */ + } else { + m->coinbase_selection = 4; /* 16 KB full window */ + } + DLOG_INFO("miner %s port %d coinbase type %d", + m->useragent[0] ? m->useragent : "(no-ua)", + c->accepted_listen_port, m->coinbase_selection); + } // get a new unique session ID for this connection (extranonce1) sid = get_new_session_id(c); From b8c075a95a993647c343429354f91f33be0ea3f5 Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:38:08 -0500 Subject: [PATCH 6/8] fix config example --- doc/example_datum_gateway_config.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/example_datum_gateway_config.json b/doc/example_datum_gateway_config.json index 7d2de0a0..667db6d7 100644 --- a/doc/example_datum_gateway_config.json +++ b/doc/example_datum_gateway_config.json @@ -6,8 +6,7 @@ "notify_fallback": true }, "stratum": { - "listen_port": 23334, - "legacy_listen_port": 0 + "listen_port": 23334 }, "mining": { "pool_address": "put your own Bitcoin invoice address here", From 7e9a527978c441c3e4936016f1218cc90baec7f8 Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:08:27 -0500 Subject: [PATCH 7/8] Drop entire fix for legacy port addition as it does not apply to blake2b, it does and it was the fix, but only worked if the legacy_port was configured --- src/datum_conf.c | 4 +--- src/datum_conf.h | 1 - src/datum_sockets.c | 53 ++++++++------------------------------------- src/datum_sockets.h | 3 +-- src/datum_stratum.c | 10 --------- 5 files changed, 11 insertions(+), 60 deletions(-) diff --git a/src/datum_conf.c b/src/datum_conf.c index 0ee8ed52..18ca6a6c 100644 --- a/src/datum_conf.c +++ b/src/datum_conf.c @@ -82,11 +82,9 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = { // stratum v1 server configs { .var_type = DATUM_CONF_STRING, .category = "stratum", .name = "listen_addr", .description = "IP address to listen for Stratum Gateway connections", .required = false, .ptr = datum_config.stratum_v1_listen_addr, .default_string[0] = "", .max_string_len = sizeof(datum_config.stratum_v1_listen_addr) }, - { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "listen_port", .description = "Listening port for Stratum Gateway (full 16KB coinbase / type 4 when legacy_listen_port is also set)", + { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "listen_port", .description = "Listening port for Stratum Gateway", .example_default = true, .required = false, .ptr = &datum_config.stratum_v1_listen_port, .default_int = 23334 }, - { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "legacy_listen_port", .description = "Optional second Stratum port for old ASICs. Uses the small Antminer-safe coinbase (type 2, ~755 bytes). 0 disables.", - .required = false, .ptr = &datum_config.stratum_v1_legacy_listen_port, .default_int = 0 }, { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "max_clients_per_thread", .description = "Maximum clients per Stratum server thread", .required = false, .ptr = &datum_config.stratum_v1_max_clients_per_thread, .default_int = 128 }, { .var_type = DATUM_CONF_INT, .category = "stratum", .name = "max_threads", .description = "Maximum Stratum server threads", diff --git a/src/datum_conf.h b/src/datum_conf.h index a53399b9..2d442b9a 100644 --- a/src/datum_conf.h +++ b/src/datum_conf.h @@ -110,7 +110,6 @@ typedef struct { char stratum_v1_listen_addr[128]; int stratum_v1_listen_port; - int stratum_v1_legacy_listen_port; int stratum_v1_max_clients; int stratum_v1_max_threads; int stratum_v1_max_clients_per_thread; diff --git a/src/datum_sockets.c b/src/datum_sockets.c index 189a24db..80d68ea4 100644 --- a/src/datum_sockets.c +++ b/src/datum_sockets.c @@ -397,7 +397,7 @@ void clean_thread_data(T_DATUM_THREAD_DATA *d, T_DATUM_SOCKET_APP *app) { d->app = app; } -int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd, int accepted_listen_port) { +int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd) { // Only one thread will be calling this function for a particular "app" // under the current design. Safe to assume that multiple clients will // not cause overlap here. @@ -542,7 +542,6 @@ int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd, int accepted_listen_port) // clear up and prep slot's client data without clobbering app_client_data app->datum_threads[tid].client_data[cid].fd = fd; app->datum_threads[tid].client_data[cid].cid = cid; - app->datum_threads[tid].client_data[cid].accepted_listen_port = accepted_listen_port; app->datum_threads[tid].client_data[cid].new_connection = true; app->datum_threads[tid].client_data[cid].datum_thread = (void *)&app->datum_threads[tid]; app->datum_threads[tid].client_data[cid].in_buf = 0; @@ -679,10 +678,7 @@ void *datum_gateway_listener_thread(void *arg) { T_DATUM_SOCKET_APP *app = (T_DATUM_SOCKET_APP *)arg; struct epoll_event ev, events[MAX_EVENTS]; - int listen_socks[8]; - int listen_sock_ports[8]; - int conn_sock, nfds, epollfd; - int nlisten = 0; + int listen_socks[2], conn_sock, nfds, epollfd; if (!app) { DLOG_FATAL("Called without application data structure. :("); @@ -711,33 +707,11 @@ void *datum_gateway_listener_thread(void *arg) { app->datum_active_threads = 0; - { - int tmp[2]; - size_t tmpn; - unsigned p; - tmpn = 2; - if (!datum_sockets_setup_listening_sockets("stratum", datum_config.stratum_v1_listen_addr, app->listen_port, tmp, &tmpn)) { - return NULL; - } - for (p = 0; p < tmpn && nlisten < 8; p++) { - listen_socks[nlisten] = tmp[p]; - listen_sock_ports[nlisten] = app->listen_port; - nlisten++; - } - if (datum_config.stratum_v1_legacy_listen_port > 0 && - datum_config.stratum_v1_legacy_listen_port != app->listen_port) { - tmpn = 2; - if (!datum_sockets_setup_listening_sockets("stratum-legacy", datum_config.stratum_v1_listen_addr, (uint16_t)datum_config.stratum_v1_legacy_listen_port, tmp, &tmpn)) { - return NULL; - } - for (p = 0; p < tmpn && nlisten < 8; p++) { - listen_socks[nlisten] = tmp[p]; - listen_sock_ports[nlisten] = datum_config.stratum_v1_legacy_listen_port; - nlisten++; - } - DLOG_INFO("Stratum also listening on legacy port %d (small coinbase)", datum_config.stratum_v1_legacy_listen_port); - } + size_t listen_socks_len = 2; + if (!datum_sockets_setup_listening_sockets("stratum", datum_config.stratum_v1_listen_addr, app->listen_port, listen_socks, &listen_socks_len)) { + return NULL; } + if (listen_socks_len < 2) listen_socks[1] = -1; epollfd = epoll_create1(0); if (epollfd < 0) { @@ -746,7 +720,7 @@ void *datum_gateway_listener_thread(void *arg) { return NULL; } - for (i = 0; i < nlisten; ++i) { + for (i = 0; i < 2; ++i) { if (listen_socks[i] == -1) continue; ev.events = EPOLLIN; ev.data.fd = listen_socks[i]; @@ -773,16 +747,7 @@ void *datum_gateway_listener_thread(void *arg) { } } for (int n = 0; n < nfds; ++n) { - int accepted_port = app->listen_port; - int is_listen = 0; - for (i = 0; i < nlisten; i++) { - if (events[n].data.fd == listen_socks[i]) { - is_listen = 1; - accepted_port = listen_sock_ports[i]; - break; - } - } - if (is_listen) { + if (events[n].data.fd == listen_socks[0] || events[n].data.fd == listen_socks[1]) { conn_sock = accept(events[n].data.fd, NULL, NULL); if (conn_sock < 0) { DLOG_ERROR("accept failed: %s", strerror(errno)); @@ -808,7 +773,7 @@ void *datum_gateway_listener_thread(void *arg) { } // assign socket to a thread - i = assign_to_thread(app, conn_sock, accepted_port); + i = assign_to_thread(app, conn_sock); if (!i) { // error finding a thread (too many connections?) DLOG_DEBUG("Closing socket we couldn't assign %d", conn_sock); diff --git a/src/datum_sockets.h b/src/datum_sockets.h index 768bd720..f3c32a3d 100644 --- a/src/datum_sockets.h +++ b/src/datum_sockets.h @@ -75,7 +75,6 @@ typedef struct T_DATUM_CLIENT_DATA { int out_buf; char rem_host[DATUM_MAX_IP_LEN+1]; - int accepted_listen_port; bool kill_request; @@ -173,7 +172,7 @@ void *datum_gateway_listener_thread(void *arg); int datum_socket_send_string_to_client(T_DATUM_CLIENT_DATA *c, char *s); int datum_socket_send_chars_to_client(T_DATUM_CLIENT_DATA *c, char *s, int len); -int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd, int accepted_listen_port); +int assign_to_thread(T_DATUM_SOCKET_APP *app, int fd); void *datum_threadpool_thread(void *arg); static inline void datum_socket_thread_client_count_decrement(T_DATUM_THREAD_DATA *my, int cid_who_left, bool not_already_locked) { diff --git a/src/datum_stratum.c b/src/datum_stratum.c index f17d1505..8d496c76 100644 --- a/src/datum_stratum.c +++ b/src/datum_stratum.c @@ -1858,16 +1858,6 @@ int client_mining_subscribe(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_ m->current_diff = datum_config.stratum_v1_vardiff_min; } } - if (datum_config.stratum_v1_legacy_listen_port > 0) { - if (c->accepted_listen_port == datum_config.stratum_v1_legacy_listen_port) { - m->coinbase_selection = 2; /* Antminer-safe ~755 B */ - } else { - m->coinbase_selection = 4; /* 16 KB full window */ - } - DLOG_INFO("miner %s port %d coinbase type %d", - m->useragent[0] ? m->useragent : "(no-ua)", - c->accepted_listen_port, m->coinbase_selection); - } // get a new unique session ID for this connection (extranonce1) sid = get_new_session_id(c); From f827701934e9111ecb022a64ae87131e18d06585 Mon Sep 17 00:00:00 2001 From: FlyTheElephant1 <259444470+FlyTheElephant1@users.noreply.github.com> Date: Tue, 15 Sep 2026 05:35:22 -0500 Subject: [PATCH 8/8] stratum: always serve YUGE coinbase, fit it to Blake weight and sigops What this does -------------- On Blake the miner never receives the generation transaction. mining.notify is a 39-byte stub (000000 || H2 || 00000000) and an empty coinb2. Coinbase class only decides how many of the pool's payout outputs the gateway commits into the block it will submit. Classes were sized for SHA256d firmware that hashes coinb1/coinb2: 0 tiny / empty 1 NiceHash ~500 B 2 Antminer-safe ~755 B (old subscribe default) 3 Whatsminer ~6500 B 4 YUGE ~16 KB 5 S21 / ANTMAIN2 ~2250 B Default class 2 plus UA fingerprint meant unknown UAs, empty UAs, and most stock Antminer strings stayed on type 2. A TIDES split with more outputs than that window fit (~17 P2WPKH with default tags) truncated the list. Leftover value landed on pool_address and ops had to top up LISTED_ONLY tails by hand. datum_stratum_coinbase_index no longer returns miner->coinbase_selection. Once the job is at or above JOB_STATE_FULL_PRIORITY_WAIT_COINBASER and full_coinbase_ready is set, it returns COINBASE_TYPE_YUGE for every miner. New-block work and not-ready jobs still use empty / type 0. miner stays on the function signature so existing call sites are unchanged. Subscribe defaults coinbase_selection to YUGE so the API and logs match the class that will actually be mined. UA fingerprinting still runs and still raises NiceHash min-diff; class is forced back to YUGE afterward. Always serving class 4 is only safe if that coinbase cannot make an invalid block against the template the miner is already hashing. Two leftover SHA256d constants said "fits" when it did not: - fit_to_template weight used +340 for an 80-byte header. Blake header is 164 bytes. Count is now ((DATUM_BLAKE2B_BLOCK_HEADER_SIZE+5)<<2)+36. Size limit already used the 164-byte header; weight did not. - Static overhead was 119 bytes with the witness commitment counted as 46. Real serialized witness commitment is 47 bytes (8 value, 1 length, 38 script). Static is 124. Payout packing skips an output that would exceed the template sigop budget (sigoplimit - txn_total_sigops - pool output cost) in both the counting pass and the write pass so the varint matches the outputs written. P2PKH (script[0]==0x76) costs 4; other scripts 0. Remainder value stays on pool_address. That is a short list, not a rejected block. This replaces the dual-port approach in b88e146 (reverted separately). That patch only forced type 4 on listen_port when legacy_listen_port was set and non-zero. The example config and the key default were 0, so stock configs never left type 2. What this does not do --------------------- Does not add a second listen port. Does not change vardiff except the existing NiceHash min-diff. Does not change the DATUM POW layout. Prime sees coinbase_id=4 and the body the gateway uploads. Configuration ------------- No new keys. No JSON edit required. Behavior changes for every miner on the existing listen_port. Benefits -------- - Finds from this gateway commit the full Prime payout list up to what the template can actually hold. - Unknown-UA and "legacy" Blake boxes are fine: they never needed the big coinbase on the wire. - A packed Knots template cannot be given a coinbase that overflows weight or sigops. - Matches Luke 4d0a045 / CONVOY#8 policy and iohzrd CONVOY#10 accounting without dropping fingerprint or changing the coinbase_index prototype. Risks ----- 1. Type-4 bytes differ from the old 119-byte packer. A pool that rebuilds class 4 itself with the old recipe will disagree with this body. It must use the uploaded blob. 2. API "coinbase class" column is YUGE after subscribe, so it no longer reports firmware fingerprint class. Job id still encodes the class that was mined. 3. SHA256d firmware that hashes coinb1/coinb2 is not a supported path on this tree. Latency / main thread --------------------- Index path: one constant return. Subscribe: one assignment after fingerprint. Packing: one extra integer compare per candidate output. What to validate before merge ----------------------------- - Ready job: datum_stratum_coinbase_index returns 4 for miners whose coinbase_selection is 2, 3, or MAX. - New-block / !full_coinbase_ready: still 0 / empty. - Subscribe + fingerprint: NiceHash min-diff still applied, class is 4. - fit_to_template(1000,0) on a weight-bound template returns the Blake leftover, not the old 1000. - Sigop tests: output-count hex 05 / 04 / 03 / 03. - A packed Knots template does not produce a coinbase that pushes block weight or sigops over the GBT limits. - Share submit / notify job id uses class 4 once the coinbaser is ready. - No stratum.legacy_listen_port key, no second bind. --- src/datum_coinbaser.c | 47 ++++++++++++++++------ src/datum_coinbaser_tests.c | 78 +++++++++++++++++++++++++++++++++++-- src/datum_stratum.c | 24 ++++++++---- src/datum_stratum_tests.c | 6 ++- 4 files changed, 132 insertions(+), 23 deletions(-) diff --git a/src/datum_coinbaser.c b/src/datum_coinbaser.c index 9ff4f019..a867fb37 100644 --- a/src/datum_coinbaser.c +++ b/src/datum_coinbaser.c @@ -206,9 +206,21 @@ void generate_coinbase_txns_for_stratum_job_subtypebysize(T_DATUM_STRATUM_JOB *s // technically an output script could be > 0x4B, meaning an extra byte would be eaten here... but that's not currently the standard // this needs to match the loop lower in this function, as the count will get thrown off if it does not. - // TODO: Enforce max sigops! Note: This is not currently enforced in eloipool, either, so punting for now and will monitor network stats to determine priority. + // The sigop cost available to these outputs: the template's sigoplimit + // (from GBT, in sigop cost units, where one legacy CHECKSIG counts 4) minus + // the cost of the template's transactions and minus the cost of the pool's + // own output. available_coinbase_outputs[].sigops is set by the coinbaser + // parser: 4 for a script whose first byte is OP_DUP (0x76, P2PKH) and 0 + // for every other script. The pool output is charged the same way. An + // output whose cost exceeds the remaining budget is skipped, the same as an + // output that exceeds the remaining size, in both this counting pass and + // the writing pass below. + int64_t sigops_budget = (int64_t)s->block_template->sigoplimit - (int64_t)s->block_template->txn_total_sigops; + if ((s->pool_addr_script_len > 0) && (s->pool_addr_script[0] == 0x76)) sigops_budget -= 4; + if (sigops_budget < 0) sigops_budget = 0; + int64_t sigops_left = sigops_budget; for(k=0;kavailable_coinbase_outputs_count;k++) { - if (((s->available_coinbase_outputs[k].output_script_len+9) <= i) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value)) { + if (((s->available_coinbase_outputs[k].output_script_len+9) <= i) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value) && (s->available_coinbase_outputs[k].sigops <= sigops_left)) { if ((special_coinb1) && (!c1full) && ((s->available_coinbase_outputs[k].output_script_len+9) <= i2)) { i2 -= (s->available_coinbase_outputs[k].output_script_len+9); c1cnt++; @@ -217,6 +229,7 @@ void generate_coinbase_txns_for_stratum_job_subtypebysize(T_DATUM_STRATUM_JOB *s } i -= (s->available_coinbase_outputs[k].output_script_len+9); + sigops_left -= s->available_coinbase_outputs[k].sigops; m++; mval += s->available_coinbase_outputs[k].value_sats; if (i < 30) break; @@ -244,9 +257,11 @@ void generate_coinbase_txns_for_stratum_job_subtypebysize(T_DATUM_STRATUM_JOB *s // append "m" payouts. find them the same way we did before mval = 0; + sigops_left = sigops_budget; for(k=0;kavailable_coinbase_outputs_count;k++) { - if (((s->available_coinbase_outputs[k].output_script_len+9) <= j) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value)) { + if (((s->available_coinbase_outputs[k].output_script_len+9) <= j) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value) && (s->available_coinbase_outputs[k].sigops <= sigops_left)) { j -= (s->available_coinbase_outputs[k].output_script_len+9); + sigops_left -= s->available_coinbase_outputs[k].sigops; m--; mval += s->available_coinbase_outputs[k].value_sats; @@ -326,8 +341,13 @@ int datum_stratum_coinbase_fit_to_template(int max_sz, int fixed_bytes, T_DATUM_ msz1 = j; } - if (((i<<2)+s->block_template->txn_total_weight+340+36) > s->block_template->weightlimit) { - j = ((s->block_template->weightlimit - (s->block_template->txn_total_weight+340+36))>>2) - fixed_bytes; + // Block weight: four units a byte for the header and the transaction count + // (at most five bytes), four a byte for the coinbase (no witness data of + // its own) plus the 36 bytes of witness the node adds to it (marker, flag, + // one 32-byte item), and the template's transactions at their weight. The + // original 340 covered the 80-byte SHA256d header. + if (((i<<2)+s->block_template->txn_total_weight+((DATUM_BLAKE2B_BLOCK_HEADER_SIZE+5)<<2)+36) > s->block_template->weightlimit) { + j = ((s->block_template->weightlimit - (s->block_template->txn_total_weight+((DATUM_BLAKE2B_BLOCK_HEADER_SIZE+5)<<2)+36))>>2) - fixed_bytes; if (j < 0) return 0; msz1 = j; } @@ -687,7 +707,7 @@ void generate_coinbase_txns_for_stratum_job(T_DATUM_STRATUM_JOB *s, bool empty_o // ok, let's figure out how much space, if any, we have for miner payout outputs // we first need to figure out how much space we are using for each type after required data, so let's do that - // witness output = 46 bytes + // witness commitment output = 47 bytes (8 value, 1 length, 38 script) // pool output = pool_addr_script_len + 9 // coinbase itself = cb_input_sz // coinbase len = 1 @@ -695,17 +715,22 @@ void generate_coinbase_txns_for_stratum_job(T_DATUM_STRATUM_JOB *s, bool empty_o // lock time = 4 bytes // "sequence" = 4 bytes // extranonce size = 15 bytes (w/len push needed for either coinbase or OP_RETURN formats) - // output count... could technically be up to three bytes for types 3 + 4, most likely 1 byte for 0,1,2. - // --- lets give ourselves the wiggle room and say 3 bytes + // output count: one byte up to 252 outputs, three bytes past that (the + // largest class holds several hundred); counted at three so the coinbase + // never exceeds what datum_stratum_coinbase_fit_to_template allowed. // - // total static bytes = 46+9+1+41+4+3+4+15 = 123 bytes + // total static bytes = 47+9+1+41+4+3+4+15 = 124 bytes // not-static bytes = pool_addr_script_len + cb_input_sz + (space_for_en_in_coinbase?0:10) // --- it costs 10 extra bytes to do the OP_RETURN based extranonce + // This was 119, three bytes under the transaction with a one-byte output + // count and five under it with a three-byte count; a coinbase built to a + // template's room then exceeded the block's weight limit by up to 20 + // weight units. if (!space_for_en_in_coinbase) { - cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 119 + s->pool_addr_script_len + cb_input_sz + 10; + cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 124 + s->pool_addr_script_len + cb_input_sz + 10; } else { - cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 119 + s->pool_addr_script_len + cb_input_sz; + cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 124 + s->pool_addr_script_len + cb_input_sz; cb_req_sz[2] += 10; // always OP_RETURN extranonce for type 2 } diff --git a/src/datum_coinbaser_tests.c b/src/datum_coinbaser_tests.c index d6b1dbcb..efed78a3 100644 --- a/src/datum_coinbaser_tests.c +++ b/src/datum_coinbaser_tests.c @@ -33,11 +33,13 @@ * */ +#include #include #include "datum_conf.h" #include "datum_stratum.h" #include "datum_coinbaser.h" +#include "datum_pow.h" #include "datum_utils.h" int datum_stratum_coinbase_fit_to_template( @@ -48,7 +50,7 @@ static void datum_prime_id_64bit_tests(void) { const uint16_t saved_unique_id = datum_config.coinbase_unique_id; char coinbase_input[1024] = {0}; int target_pot_index = -1; - + datum_config.prime_id = UINT64_C(0x887766555d965e4e); datum_config.coinbase_unique_id = 0x1234; const int coinbase_input_size = generate_coinbase_input( @@ -64,18 +66,88 @@ static void datum_prime_id_64bit_tests(void) { static void datum_blake2b_coinbase_limit_tests(void) { T_DATUM_TEMPLATE_DATA tdata; T_DATUM_STRATUM_JOB job; - + memset(&tdata, 0, sizeof(tdata)); memset(&job, 0, sizeof(job)); job.block_template = &tdata; tdata.sizelimit = 85 + 36 + 950; tdata.weightlimit = 4000000; - + /* The 164-byte header shrinks the coinbase leftover by 84 bytes. */ datum_test(datum_stratum_coinbase_fit_to_template(1000, 0, &job) == 866); + + /* The weight limit: the header and a five-byte count at four units a byte, + * the coinbase's 36 witness bytes, then 950 bytes of coinbase at four + * each. With the 80-byte header's 340 the leftover was 1000 (unbound). */ + tdata.sizelimit = 4000000; + tdata.weightlimit = ((DATUM_BLAKE2B_BLOCK_HEADER_SIZE + 5) * 4) + 36 + (4 * 950); + datum_test(datum_stratum_coinbase_fit_to_template(1000, 0, &job) == 950); + tdata.txn_total_weight = 4000; + tdata.weightlimit += 4000; + datum_test(datum_stratum_coinbase_fit_to_template(1000, 0, &job) == 950); + datum_test(datum_stratum_coinbase_fit_to_template(1000, 100, &job) == 850); +} + +/* P2PKH: OP_DUP OP_HASH160 <20 bytes> OP_EQUALVERIFY OP_CHECKSIG, 25 bytes. */ +static const unsigned char datum_test_p2pkh_script[25] = { 0x76, 0xa9, 0x14, [23] = 0x88, 0xac }; +/* P2WPKH: OP_0 <20 bytes>, 22 bytes. */ +static const unsigned char datum_test_p2wpkh_script[22] = { 0x00, 0x14 }; + +static const char *datum_coinbase_output_count_hex(T_DATUM_STRATUM_JOB *job, uint32_t sigops_used, bool pool_p2pkh) { + int cb1idx[MAX_COINBASE_TYPES] = { 0 }; + int cb2idx[MAX_COINBASE_TYPES] = { 0 }; + + job->block_template->txn_total_sigops = sigops_used; + if (pool_p2pkh) { + memcpy(job->pool_addr_script, datum_test_p2pkh_script, sizeof(datum_test_p2pkh_script)); + job->pool_addr_script_len = sizeof(datum_test_p2pkh_script); + } else { + memcpy(job->pool_addr_script, datum_test_p2wpkh_script, sizeof(datum_test_p2wpkh_script)); + job->pool_addr_script_len = sizeof(datum_test_p2wpkh_script); + } + memset(job->coinbase[1].coinb2, 0, sizeof(job->coinbase[1].coinb2)); + generate_coinbase_txns_for_stratum_job_subtypebysize(job, 1, 1000, true, cb1idx, cb2idx, false); + return job->coinbase[1].coinb2 + 8; +} + +static void datum_blake2b_coinbase_sigops_tests(void) { + T_DATUM_TEMPLATE_DATA tdata; + T_DATUM_STRATUM_JOB *job = calloc(1, sizeof(*job)); + int k; + + datum_test(job != NULL); + if (!job) return; + memset(&tdata, 0, sizeof(tdata)); + tdata.sigoplimit = 80000; + job->block_template = &tdata; + job->coinbase_value = 5000000000ULL; + for (k = 0; k < 3; k++) { + job->available_coinbase_outputs[k].value_sats = 100000000; + if (k < 2) { + memcpy(job->available_coinbase_outputs[k].output_script, datum_test_p2pkh_script, sizeof(datum_test_p2pkh_script)); + job->available_coinbase_outputs[k].output_script_len = sizeof(datum_test_p2pkh_script); + job->available_coinbase_outputs[k].sigops = 4; + } else { + memcpy(job->available_coinbase_outputs[k].output_script, datum_test_p2wpkh_script, sizeof(datum_test_p2wpkh_script)); + job->available_coinbase_outputs[k].output_script_len = sizeof(datum_test_p2wpkh_script); + job->available_coinbase_outputs[k].sigops = 0; + } + } + job->available_coinbase_outputs_count = 3; + + /* Space for every output: all three, the pool output and the witness commitment. */ + datum_test(!strncmp(datum_coinbase_output_count_hex(job, 0, false), "05", 2)); + /* Budget for one P2PKH output: first P2PKH in, second skipped, P2WPKH in. */ + datum_test(!strncmp(datum_coinbase_output_count_hex(job, 80000 - 4, false), "04", 2)); + /* Budget of 0: only the P2WPKH output is included. */ + datum_test(!strncmp(datum_coinbase_output_count_hex(job, 80000, false), "03", 2)); + /* P2PKH pool output takes the remaining 4 units, so neither P2PKH candidate is included. */ + datum_test(!strncmp(datum_coinbase_output_count_hex(job, 80000 - 4, true), "03", 2)); + free(job); } void datum_coinbaser_tests(void) { datum_prime_id_64bit_tests(); datum_blake2b_coinbase_limit_tests(); + datum_blake2b_coinbase_sigops_tests(); } diff --git a/src/datum_stratum.c b/src/datum_stratum.c index 8d496c76..c1efbce5 100644 --- a/src/datum_stratum.c +++ b/src/datum_stratum.c @@ -1583,15 +1583,20 @@ int client_mining_authorize(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_ return 0; } +// On Blake the miner never receives the generation tx (notify is a 39-byte +// stub). Coinbase class only decides how many of the pool's payout outputs +// land in the block. Per-miner type 2 was the TIDES truncate: leftover +// value dumped on pool_address. Once the coinbaser is ready, everyone +// gets YUGE. miner is unused; kept so call sites stay stable. unsigned int datum_stratum_coinbase_index( const T_DATUM_STRATUM_THREADPOOL_DATA *sdata, const T_DATUM_MINER_DATA *miner, bool new_block) { + (void)miner; if (new_block) return DATUM_COINBASE_ID_EMPTY; - if (!sdata || !miner || !sdata->cur_stratum_job || + if (!sdata || !sdata->cur_stratum_job || sdata->cur_stratum_job->job_state < JOB_STATE_FULL_PRIORITY_WAIT_COINBASER || - !sdata->full_coinbase_ready || - miner->coinbase_selection >= MAX_COINBASE_TYPES) return 0; - return miner->coinbase_selection; + !sdata->full_coinbase_ready) return 0; + return COINBASE_TYPE_YUGE; } int send_mining_notify(T_DATUM_CLIENT_DATA *c, bool clean, bool quickdiff, bool new_block) { @@ -1837,9 +1842,11 @@ int client_mining_subscribe(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_ // set default diff m->current_diff = datum_config.stratum_v1_vardiff_min; - // default to the antminer workaround, which appears to be universally compatible - // except for NiceHash. - m->coinbase_selection = 2; + // Blake miners are blind to generation-tx size. Default to YUGE so the + // API / logs match what datum_stratum_coinbase_index will actually + // commit once the coinbaser is ready. Fingerprinting may still raise + // NiceHash min-diff; class is forced back to YUGE after that. + m->coinbase_selection = COINBASE_TYPE_YUGE; m->useragent[0] = 0; if (params_obj) { @@ -1858,6 +1865,9 @@ int client_mining_subscribe(T_DATUM_CLIENT_DATA *c, uint64_t id, json_t *params_ m->current_diff = datum_config.stratum_v1_vardiff_min; } } + // Fingerprint may have overwritten class for SHA256d-era size limits. + // Those limits do not apply on Blake; keep NiceHash min-diff only. + m->coinbase_selection = COINBASE_TYPE_YUGE; // get a new unique session ID for this connection (extranonce1) sid = get_new_session_id(c); diff --git a/src/datum_stratum_tests.c b/src/datum_stratum_tests.c index 17199bba..3f71064a 100644 --- a/src/datum_stratum_tests.c +++ b/src/datum_stratum_tests.c @@ -212,9 +212,11 @@ static void datum_blake2b_coinbase_selection_tests(void) { sdata->full_coinbase_ready = true; datum_test(datum_stratum_coinbase_index(sdata, &miner, false) == 0); job.job_state = JOB_STATE_FULL_PRIORITY_WAIT_COINBASER; - datum_test(datum_stratum_coinbase_index(sdata, &miner, false) == 3); + datum_test(datum_stratum_coinbase_index(sdata, &miner, false) == COINBASE_TYPE_YUGE); miner.coinbase_selection = MAX_COINBASE_TYPES; - datum_test(datum_stratum_coinbase_index(sdata, &miner, false) == 0); + datum_test(datum_stratum_coinbase_index(sdata, &miner, false) == COINBASE_TYPE_YUGE); + miner.coinbase_selection = 2; + datum_test(datum_stratum_coinbase_index(sdata, &miner, false) == COINBASE_TYPE_YUGE); free(sdata); }