logger: optional TTY collapse of repeating job and notification lines

src/datum_coinbaser.c

@@ -206,9 +206,21 @@ void generate_coinbase_txns_for_stratum_job_subtypebysize(T_DATUM_STRATUM_JOB *s
 	// technically an output script could be > 0x4B, meaning an extra byte would be eaten here... but that's not currently the standard
 	// this needs to match the loop lower in this function, as the count will get thrown off if it does not.
 
-	// TODO: Enforce max sigops! Note: This is not currently enforced in eloipool, either, so punting for now and will monitor network stats to determine priority.
+	// The sigop cost available to these outputs: the template's sigoplimit
+	// (from GBT, in sigop cost units, where one legacy CHECKSIG counts 4) minus
+	// the cost of the template's transactions and minus the cost of the pool's
+	// own output. available_coinbase_outputs[].sigops is set by the coinbaser
+	// parser: 4 for a script whose first byte is OP_DUP (0x76, P2PKH) and 0
+	// for every other script. The pool output is charged the same way. An
+	// output whose cost exceeds the remaining budget is skipped, the same as an
+	// output that exceeds the remaining size, in both this counting pass and
+	// the writing pass below.
+	int64_t sigops_budget = (int64_t)s->block_template->sigoplimit - (int64_t)s->block_template->txn_total_sigops;
+	if ((s->pool_addr_script_len > 0) && (s->pool_addr_script[0] == 0x76)) sigops_budget -= 4;
+	if (sigops_budget < 0) sigops_budget = 0;
+	int64_t sigops_left = sigops_budget;
 	for(k=0;k<s->available_coinbase_outputs_count;k++) {
-		if (((s->available_coinbase_outputs[k].output_script_len+9) <= i) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value))  {
+		if (((s->available_coinbase_outputs[k].output_script_len+9) <= i) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value) && (s->available_coinbase_outputs[k].sigops <= sigops_left))  {
 			if ((special_coinb1) && (!c1full) && ((s->available_coinbase_outputs[k].output_script_len+9) <= i2)) {
 				i2 -= (s->available_coinbase_outputs[k].output_script_len+9);
 				c1cnt++;
@@ -217,6 +229,7 @@ void generate_coinbase_txns_for_stratum_job_subtypebysize(T_DATUM_STRATUM_JOB *s
 			}
 
 			i -= (s->available_coinbase_outputs[k].output_script_len+9);
+			sigops_left -= s->available_coinbase_outputs[k].sigops;
 			m++;
 			mval += s->available_coinbase_outputs[k].value_sats;
 			if (i < 30) break;
@@ -244,9 +257,11 @@ void generate_coinbase_txns_for_stratum_job_subtypebysize(T_DATUM_STRATUM_JOB *s
 
 	// append "m" payouts. find them the same way we did before
 	mval = 0;
+	sigops_left = sigops_budget;
 	for(k=0;k<s->available_coinbase_outputs_count;k++) {
-		if (((s->available_coinbase_outputs[k].output_script_len+9) <= j) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value)) {
+		if (((s->available_coinbase_outputs[k].output_script_len+9) <= j) && ((mval + s->available_coinbase_outputs[k].value_sats) <= s->coinbase_value) && (s->available_coinbase_outputs[k].sigops <= sigops_left)) {
 			j -= (s->available_coinbase_outputs[k].output_script_len+9);
+			sigops_left -= s->available_coinbase_outputs[k].sigops;
 			m--;
 
 			mval += s->available_coinbase_outputs[k].value_sats;
@@ -326,8 +341,13 @@ int datum_stratum_coinbase_fit_to_template(int max_sz, int fixed_bytes, T_DATUM_
 		msz1 = j;
 	}
 
-	if (((i<<2)+s->block_template->txn_total_weight+340+36) > s->block_template->weightlimit) {
-		j = ((s->block_template->weightlimit - (s->block_template->txn_total_weight+340+36))>>2) - fixed_bytes;
+	// Block weight: four units a byte for the header and the transaction count
+	// (at most five bytes), four a byte for the coinbase (no witness data of
+	// its own) plus the 36 bytes of witness the node adds to it (marker, flag,
+	// one 32-byte item), and the template's transactions at their weight. The
+	// original 340 covered the 80-byte SHA256d header.
+	if (((i<<2)+s->block_template->txn_total_weight+((DATUM_BLAKE2B_BLOCK_HEADER_SIZE+5)<<2)+36) > s->block_template->weightlimit) {
+		j = ((s->block_template->weightlimit - (s->block_template->txn_total_weight+((DATUM_BLAKE2B_BLOCK_HEADER_SIZE+5)<<2)+36))>>2) - fixed_bytes;
 		if (j < 0) return 0;
 		msz1 = j;
 	}
@@ -687,25 +707,30 @@ void generate_coinbase_txns_for_stratum_job(T_DATUM_STRATUM_JOB *s, bool empty_o
 		// ok, let's figure out how much space, if any, we have for miner payout outputs
 		// we first need to figure out how much space we are using for each type after required data, so let's do that
 
-		// witness output = 46 bytes
+		// witness commitment output = 47 bytes (8 value, 1 length, 38 script)
 		// pool output = pool_addr_script_len + 9
 		// coinbase itself = cb_input_sz
 		// coinbase len = 1
 		// cbstart = 41 bytes
 		// lock time = 4 bytes
 		// "sequence" = 4 bytes
 		// extranonce size = 15 bytes (w/len push needed for either coinbase or OP_RETURN formats)
-		// output count... could technically be up to three bytes for types 3 + 4, most likely 1 byte for 0,1,2.
-		//     --- lets give ourselves the wiggle room and say 3 bytes
+		// output count: one byte up to 252 outputs, three bytes past that (the
+		// largest class holds several hundred); counted at three so the coinbase
+		// never exceeds what datum_stratum_coinbase_fit_to_template allowed.
 		//
-		// total static bytes = 46+9+1+41+4+3+4+15 = 123 bytes
+		// total static bytes = 47+9+1+41+4+3+4+15 = 124 bytes
 		// not-static bytes = pool_addr_script_len + cb_input_sz + (space_for_en_in_coinbase?0:10)
 		//     --- it costs 10 extra bytes to do the OP_RETURN based extranonce
+		// This was 119, three bytes under the transaction with a one-byte output
+		// count and five under it with a three-byte count; a coinbase built to a
+		// template's room then exceeded the block's weight limit by up to 20
+		// weight units.
 
 		if (!space_for_en_in_coinbase) {
-			cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 119 + s->pool_addr_script_len + cb_input_sz + 10;
+			cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 124 + s->pool_addr_script_len + cb_input_sz + 10;
 		} else {
-			cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 119 + s->pool_addr_script_len + cb_input_sz;
+			cb_req_sz[1] = cb_req_sz[2] = cb_req_sz[3] = cb_req_sz[4] = cb_req_sz[5] = 124 + s->pool_addr_script_len + cb_input_sz;
 			cb_req_sz[2] += 10; // always OP_RETURN extranonce for type 2
 		}
 

src/datum_coinbaser_tests.c

@@ -33,11 +33,13 @@
  *
  */
 
+#include <stdlib.h>
 #include <string.h>
 
 #include "datum_conf.h"
 #include "datum_stratum.h"
 #include "datum_coinbaser.h"
+#include "datum_pow.h"
 #include "datum_utils.h"
 
 int datum_stratum_coinbase_fit_to_template(
@@ -48,7 +50,7 @@ static void datum_prime_id_64bit_tests(void) {
 	const uint16_t saved_unique_id = datum_config.coinbase_unique_id;
 	char coinbase_input[1024] = {0};
 	int target_pot_index = -1;
-	
+
 	datum_config.prime_id = UINT64_C(0x887766555d965e4e);
 	datum_config.coinbase_unique_id = 0x1234;
 	const int coinbase_input_size = generate_coinbase_input(
@@ -64,18 +66,88 @@ static void datum_prime_id_64bit_tests(void) {
 static void datum_blake2b_coinbase_limit_tests(void) {
 	T_DATUM_TEMPLATE_DATA tdata;
 	T_DATUM_STRATUM_JOB job;
-	
+
 	memset(&tdata, 0, sizeof(tdata));
 	memset(&job, 0, sizeof(job));
 	job.block_template = &tdata;
 	tdata.sizelimit = 85 + 36 + 950;
 	tdata.weightlimit = 4000000;
-	
+
 	/* The 164-byte header shrinks the coinbase leftover by 84 bytes. */
 	datum_test(datum_stratum_coinbase_fit_to_template(1000, 0, &job) == 866);
+
+	/* The weight limit: the header and a five-byte count at four units a byte,
+	 * the coinbase's 36 witness bytes, then 950 bytes of coinbase at four
+	 * each. With the 80-byte header's 340 the leftover was 1000 (unbound). */
+	tdata.sizelimit = 4000000;
+	tdata.weightlimit = ((DATUM_BLAKE2B_BLOCK_HEADER_SIZE + 5) * 4) + 36 + (4 * 950);
+	datum_test(datum_stratum_coinbase_fit_to_template(1000, 0, &job) == 950);
+	tdata.txn_total_weight = 4000;
+	tdata.weightlimit += 4000;
+	datum_test(datum_stratum_coinbase_fit_to_template(1000, 0, &job) == 950);
+	datum_test(datum_stratum_coinbase_fit_to_template(1000, 100, &job) == 850);
+}
+
+/* P2PKH: OP_DUP OP_HASH160 <20 bytes> OP_EQUALVERIFY OP_CHECKSIG, 25 bytes. */
+static const unsigned char datum_test_p2pkh_script[25] = { 0x76, 0xa9, 0x14, [23] = 0x88, 0xac };
+/* P2WPKH: OP_0 <20 bytes>, 22 bytes. */
+static const unsigned char datum_test_p2wpkh_script[22] = { 0x00, 0x14 };
+
+static const char *datum_coinbase_output_count_hex(T_DATUM_STRATUM_JOB *job, uint32_t sigops_used, bool pool_p2pkh) {
+	int cb1idx[MAX_COINBASE_TYPES] = { 0 };
+	int cb2idx[MAX_COINBASE_TYPES] = { 0 };
+
+	job->block_template->txn_total_sigops = sigops_used;
+	if (pool_p2pkh) {
+		memcpy(job->pool_addr_script, datum_test_p2pkh_script, sizeof(datum_test_p2pkh_script));
+		job->pool_addr_script_len = sizeof(datum_test_p2pkh_script);
+	} else {
+		memcpy(job->pool_addr_script, datum_test_p2wpkh_script, sizeof(datum_test_p2wpkh_script));
+		job->pool_addr_script_len = sizeof(datum_test_p2wpkh_script);
+	}
+	memset(job->coinbase[1].coinb2, 0, sizeof(job->coinbase[1].coinb2));
+	generate_coinbase_txns_for_stratum_job_subtypebysize(job, 1, 1000, true, cb1idx, cb2idx, false);
+	return job->coinbase[1].coinb2 + 8;
+}
+
+static void datum_blake2b_coinbase_sigops_tests(void) {
+	T_DATUM_TEMPLATE_DATA tdata;
+	T_DATUM_STRATUM_JOB *job = calloc(1, sizeof(*job));
+	int k;
+
+	datum_test(job != NULL);
+	if (!job) return;
+	memset(&tdata, 0, sizeof(tdata));
+	tdata.sigoplimit = 80000;
+	job->block_template = &tdata;
+	job->coinbase_value = 5000000000ULL;
+	for (k = 0; k < 3; k++) {
+		job->available_coinbase_outputs[k].value_sats = 100000000;
+		if (k < 2) {
+			memcpy(job->available_coinbase_outputs[k].output_script, datum_test_p2pkh_script, sizeof(datum_test_p2pkh_script));
+			job->available_coinbase_outputs[k].output_script_len = sizeof(datum_test_p2pkh_script);
+			job->available_coinbase_outputs[k].sigops = 4;
+		} else {
+			memcpy(job->available_coinbase_outputs[k].output_script, datum_test_p2wpkh_script, sizeof(datum_test_p2wpkh_script));
+			job->available_coinbase_outputs[k].output_script_len = sizeof(datum_test_p2wpkh_script);
+			job->available_coinbase_outputs[k].sigops = 0;
+		}
+	}
+	job->available_coinbase_outputs_count = 3;
+
+	/* Space for every output: all three, the pool output and the witness commitment. */
+	datum_test(!strncmp(datum_coinbase_output_count_hex(job, 0, false), "05", 2));
+	/* Budget for one P2PKH output: first P2PKH in, second skipped, P2WPKH in. */
+	datum_test(!strncmp(datum_coinbase_output_count_hex(job, 80000 - 4, false), "04", 2));
+	/* Budget of 0: only the P2WPKH output is included. */
+	datum_test(!strncmp(datum_coinbase_output_count_hex(job, 80000, false), "03", 2));
+	/* P2PKH pool output takes the remaining 4 units, so neither P2PKH candidate is included. */
+	datum_test(!strncmp(datum_coinbase_output_count_hex(job, 80000 - 4, true), "03", 2));
+	free(job);
 }
 
 void datum_coinbaser_tests(void) {
 	datum_prime_id_64bit_tests();
 	datum_blake2b_coinbase_limit_tests();
+	datum_blake2b_coinbase_sigops_tests();
 }

src/datum_conf.c

@@ -125,8 +125,16 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = {
 		.required = false, .ptr = datum_config.mining_coinbase_tag_secondary,			.default_string[0] = "DATUM User", .max_string_len = sizeof(datum_config.mining_coinbase_tag_secondary) },
 	{ .var_type = DATUM_CONF_INT, 		.category = "mining", 		.name = "coinbase_unique_id",		.description = "A unique ID between 1 and 65535. This is appended to the coinbase. Make unique per instance of datum with the same coinbase tags.",
 		.required = false, .ptr = &datum_config.coinbase_unique_id, 		.default_int = 4242 },
-	{ .var_type = DATUM_CONF_STRING, 	.category = "mining", 		.name = "save_submitblocks_dir",	.description = "Directory to save all submitted blocks to as submitblock JSON files",
+	{ .var_type = DATUM_CONF_STRING, 	.category = "mining", 		.name = "save_submitblocks_dir",	.description = "Directory to save submitted blocks as submitblock JSON (datum_submitblock_<height>_<first7 GIT_COMMIT_HASH>.json and datum_submitblock_last.json). Empty disables.",
 		.required = false, .ptr = datum_config.mining_save_submitblocks_dir,			.default_string[0] = "", .max_string_len = sizeof(datum_config.mining_save_submitblocks_dir) },
+	{ .var_type = DATUM_CONF_INT, 		.category = "mining", 		.name = "share_node_check_missingzeros",	.description = "If >= 0, only log SHARE lines and node-check shares with missingzeros <= this value (overrides share_node_check_every). -1 = infinity (log all, use every-N).",
+		.required = false, .ptr = &datum_config.mining_share_node_check_missingzeros, 	.default_int = -1 },
+	{ .var_type = DATUM_CONF_BOOL, 		.category = "mining", 		.name = "validate_shares_on_node",	.description = "Ask the local node about sampled accepted shares (proposal or submitblock). Real blocks are always submitted separately.",
+		.required = false, .ptr = &datum_config.mining_validate_shares_on_node, 		.default_bool = false },
+	{ .var_type = DATUM_CONF_STRING, 	.category = "mining", 		.name = "share_node_check",		.description = "How to ask the node about a share: proposal (GBT mode=proposal, skips PoW) or submitblock (PoW checked first, usually high-hash).",
+		.required = false, .ptr = datum_config.mining_share_node_check,				.default_string[0] = "proposal", .max_string_len = sizeof(datum_config.mining_share_node_check) },
+	{ .var_type = DATUM_CONF_INT, 		.category = "mining", 		.name = "share_node_check_every",	.description = "Validate 1 of every N accepted shares on the node (1 = every share). Real blocks are always submitted separately.",
+		.required = false, .ptr = &datum_config.mining_share_node_check_every, 		.default_int = 16 },
 	{ .var_type = DATUM_CONF_BOOL, 		.category = "mining", 		.name = "allow_hasher_time_rolling",	.description = "Allow hasher time rolling for BLAKE2b jobs",
 		.required = false, .ptr = &datum_config.mining_allow_hasher_time_rolling, 		.default_bool = false },
 	{ .var_type = DATUM_CONF_BOOL, 		.category = "mining", 		.name = "abw_verify_all_shares_on_disclosure",	.description = "Retain ABW proofs until key disclosure and alarm if the pool ignored a block",
@@ -175,6 +183,10 @@ const T_DATUM_CONFIG_ITEM datum_config_options[] = {
 	{ .var_type = DATUM_CONF_INT, 		.category = "logger",	.name = "log_level_file",				.description = "Minimum log level for log file messages (0=All, 1=Debug, 2=Info, 3=Warn, 4=Error, 5=Fatal)",
 		.example_default = true,
 		.required = false, .ptr = &datum_config.clog_level_file, .default_int = 1 },
+	{ .var_type = DATUM_CONF_BOOL, 		.category = "logger", 		.name = "log_shares",				.description = "Log the result of every incoming share at INFO (level 2): accepted or rejected with reason",
+		.required = false, .ptr = &datum_config.logger_log_shares, 		.default_bool = false },
+	{ .var_type = DATUM_CONF_BOOL,          .category = "logger",           .name = "console_collapse_job_updates",         .description = "Collapse repeating stratum job update logs into a single line counter to prevent console flooding.",
+		.required = false, .ptr = &datum_config.clog_console_collapse_jobs,          .default_bool = false },
 
 	// datum options
 	{ .var_type = DATUM_CONF_STRING, 	.category = "datum", 		.name = "pool_host",					.description = "Remote DATUM server host/ip to use for decentralized pooled mining (set to \"\" to disable pooled mining)",

src/datum_conf.h

@@ -131,6 +131,10 @@ typedef struct {
 	char mining_coinbase_tag_primary[64];
 	char mining_coinbase_tag_secondary[64];
 	char mining_save_submitblocks_dir[256];
+	int mining_share_node_check_missingzeros;
+	bool mining_validate_shares_on_node;
+	char mining_share_node_check[32];
+	int mining_share_node_check_every;
 	bool mining_allow_hasher_time_rolling;
 	bool mining_abw_verify_all_shares_on_disclosure;
 	int coinbase_unique_id;
@@ -155,6 +159,8 @@ typedef struct {
 	bool clog_to_stderr;
 	bool clog_rotate_daily;
 	char clog_file[1024];
+	bool logger_log_shares;
+	bool clog_console_collapse_jobs;
 
 	char datum_pool_host[1024];
 	int datum_pool_port;

src/datum_gateway.c

@@ -197,6 +197,13 @@ int main(const int argc, const char * const * const argv) {
 
 	// Initialize logger thread
 	datum_logger_init();
+	if (datum_config.mining_save_submitblocks_dir[0]) {
+		char gitshort[8];
+		memset(gitshort, 0, sizeof(gitshort));
+		memcpy(gitshort, GIT_COMMIT_HASH, 7);
+		DLOG_INFO("submitblock save dir: %s (datum_submitblock_<height>_%s.json + datum_submitblock_last.json)",
+			datum_config.mining_save_submitblocks_dir, gitshort[0] ? gitshort : "unknown");
+	}
 
 	if (datum_protocol_init()) {
 		DLOG_FATAL("Error initializing the DATUM protocol!");