From e29ee76f03e264d7162431c8611b0e90209c34de Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Tue, 8 Sep 2026 02:01:03 +0300 Subject: [PATCH 1/7] Add batched strong-weak RS product code decoding --- CMakeLists.txt | 8 +- .../strong_weak_rs_product_code_benchmarks.cc | 199 +++++++ include/reed_solomon/error_correction.h | 35 ++ .../strong_weak_rs_product_code.h | 91 +++ src/reed_solomon/error_correction/batched.cc | 80 ++- src/reed_solomon/error_correction/internal.h | 31 +- src/reed_solomon/error_correction/scalar.cc | 48 +- src/reed_solomon/product_code_internal.h | 26 + .../strong_weak_rs_product_code.cc | 237 ++++++++ tests/product_code_tests.cc | 525 ++++++++++++++++++ 10 files changed, 1240 insertions(+), 40 deletions(-) create mode 100644 benchmarks/strong_weak_rs_product_code_benchmarks.cc create mode 100644 include/reed_solomon/error_correction.h create mode 100644 include/reed_solomon/strong_weak_rs_product_code.h create mode 100644 src/reed_solomon/product_code_internal.h create mode 100644 src/reed_solomon/strong_weak_rs_product_code.cc create mode 100644 tests/product_code_tests.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index e385c25..1802f3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ add_library(gf256_core STATIC src/reed_solomon/decoder/high_rate.cc src/reed_solomon/lch_encoder.cc src/reed_solomon/lch_decoder.cc + src/reed_solomon/strong_weak_rs_product_code.cc ) target_compile_features(gf256_core PUBLIC cxx_std_20) target_include_directories(gf256_core @@ -142,7 +143,8 @@ if(GF256_BUILD_TESTS) target_link_libraries(gf_unittests PRIVATE gf256_core gtest gtest_main) add_test(NAME Unittests COMMAND gf_unittests) - add_executable(lch_rs_unittests tests/lch_tests.cc tests/rs_tests.cc) + add_executable(lch_rs_unittests tests/lch_tests.cc tests/rs_tests.cc + tests/product_code_tests.cc) target_include_directories(lch_rs_unittests PRIVATE ${PROJECT_SOURCE_DIR}/src) target_link_libraries(lch_rs_unittests PRIVATE gf256_core gtest gtest_main) if(GF256_ENABLE_NATIVE_ISA AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") @@ -170,8 +172,10 @@ endif() if(GF256_BUILD_BENCHMARKS) FetchContent_MakeAvailable(googlebenchmark) - add_executable(benchmarks benchmarks/matrix_multiplication.cc) + add_executable(benchmarks benchmarks/matrix_multiplication.cc + benchmarks/strong_weak_rs_product_code_benchmarks.cc) target_link_libraries(benchmarks PRIVATE gf256_core benchmark benchmark_main) + target_include_directories(benchmarks PRIVATE ${PROJECT_SOURCE_DIR}/src) add_executable(rs_verbose_benchmarks benchmarks/lch_rs_benchmarks.cc diff --git a/benchmarks/strong_weak_rs_product_code_benchmarks.cc b/benchmarks/strong_weak_rs_product_code_benchmarks.cc new file mode 100644 index 0000000..85f0993 --- /dev/null +++ b/benchmarks/strong_weak_rs_product_code_benchmarks.cc @@ -0,0 +1,199 @@ +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" +#include "reed_solomon/strong_weak_rs_product_code.h" +#include "reed_solomon/product_code_internal.h" + +namespace { + +void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = -1) { + using gf2p8::Element; + using gf2p8::rs::ProductCorrectionResult; + using gf2p8::rs::ProductTermination; + constexpr size_t kN = 256; + constexpr size_t kStrongK = 224; + constexpr size_t kWeakK = 254; + constexpr size_t kInformationBytes = kStrongK * kWeakK; + constexpr size_t kBlockBytes = kN * kN; + constexpr size_t kCorpusCount = 64; + constexpr size_t kPassLimit = 16; + constexpr uint32_t kSeed = 0x5b5c0224; + gf2p8::rs::StrongWeakRSProductCode code(kN, kStrongK, kN, kWeakK); + if (!code.Valid() || code.BlockSize() != kBlockBytes) { + state.SkipWithError("invalid product-code dimensions"); + return; + } + + std::mt19937 messages(kSeed); + std::mt19937 channel(kSeed ^ 0x9e3779b9U); + std::vector> original( + kCorpusCount, std::vector(kBlockBytes)); + auto corrupted = original; + auto work = original; + std::vector results(kCorpusCount); + uint64_t channel_bits = 0, channel_symbols = 0; + for (size_t sample = 0; sample < kCorpusCount; ++sample) { + auto& block = original[sample]; + for (size_t row = 0; row < kStrongK; ++row) { + for (size_t col = 0; col < kWeakK; ++col) { + block[row * kN + col] = static_cast(messages()); + } + } + if (code.Encode(block) != gf2p8::lch::Status::ok) { + state.SkipWithError("product corpus encoding failed"); + return; + } + work[sample] = block; + const auto clean = code.Correct(work[sample], kPassLimit); + if (!clean.all_zero_syndromes || clean.changed_symbols != 0 || + work[sample] != block) { + state.SkipWithError("encoded product corpus is not valid"); + return; + } + corrupted[sample] = block; + for (auto& value : corrupted[sample]) { + const auto before = value; + for (unsigned bit = 0; bit < 8; ++bit) { + // Reject the incomplete residue range: exact Bernoulli(1/200), + // reproducible across standard libraries, with no fixed error count. + uint32_t draw; + do { + draw = static_cast(channel()); + } while (draw >= 4294967200U); + if (draw % 200 == 0) { + value ^= static_cast(1U << bit); + ++channel_bits; + } + } + channel_symbols += value != before; + } + } + + uint64_t data_residual_bits = 0, codeword_residual_bits = 0; + uint64_t message_failures = 0, block_failures = 0, validity_failures = 0; + uint64_t valid_wrong_blocks = 0, pass_limits = 0, passes = 0; + uint64_t strong_lines = 0, weak_lines = 0; + size_t max_passes = 0; + // Validate this exact channel corpus against the retained single path outside + // timing, including scheduling outcomes, not just final message recovery. + for (size_t sample = 0; sample < kCorpusCount; ++sample) { + auto reference = corrupted[sample]; + const auto expected = gf2p8::rs::detail::ProductCorrectionAccess::Correct( + code, reference, kPassLimit, 0, false); + work[sample] = corrupted[sample]; + const auto actual = batch_passes < 0 ? code.Correct(work[sample], kPassLimit) + : gf2p8::rs::detail::ProductCorrectionAccess::Correct( + code, work[sample], kPassLimit, batch_passes); + if (work[sample] != reference || actual.termination != expected.termination || + actual.all_zero_syndromes != expected.all_zero_syndromes || + actual.directional_passes != expected.directional_passes || + actual.strong_lines_visited != expected.strong_lines_visited || + actual.weak_lines_visited != expected.weak_lines_visited || + actual.changed_symbols != expected.changed_symbols) { + state.SkipWithError("batch/single corpus differential mismatch"); + return; + } + } + for (auto _ : state) { + state.PauseTiming(); + for (size_t sample = 0; sample < kCorpusCount; ++sample) { + std::copy(corrupted[sample].begin(), corrupted[sample].end(), + work[sample].begin()); + } + state.ResumeTiming(); + for (size_t sample = 0; sample < kCorpusCount; ++sample) { + results[sample] = batch_passes < 0 ? code.Correct(work[sample], kPassLimit) + : gf2p8::rs::detail::ProductCorrectionAccess::Correct( + code, work[sample], kPassLimit, batch_passes); + benchmark::DoNotOptimize(results[sample]); + benchmark::ClobberMemory(); + } + state.PauseTiming(); + for (size_t sample = 0; sample < kCorpusCount; ++sample) { + const auto& result = results[sample]; + if (result.termination == ProductTermination::invalid_argument) { + state.SkipWithError("product correction rejected corpus input"); + break; + } + uint64_t data_bits = 0, block_bits = 0; + for (size_t pos = 0; pos < kBlockBytes; ++pos) { + const auto bits = std::popcount(static_cast( + work[sample][pos] ^ original[sample][pos])); + block_bits += bits; + if (pos / kN < kStrongK && pos % kN < kWeakK) { + data_bits += bits; + } + } + data_residual_bits += data_bits; + codeword_residual_bits += block_bits; + message_failures += data_bits != 0; + block_failures += block_bits != 0; + validity_failures += !result.all_zero_syndromes; + valid_wrong_blocks += result.all_zero_syndromes && block_bits != 0; + pass_limits += result.termination == ProductTermination::pass_limit; + passes += result.directional_passes; + max_passes = std::max(max_passes, result.directional_passes); + strong_lines += result.strong_lines_visited; + weak_lines += result.weak_lines_visited; + } + state.ResumeTiming(); + if (state.skipped()) break; + } + + const double sweeps = static_cast(state.iterations()); + if (sweeps == 0 || state.skipped()) return; + const double blocks = sweeps * kCorpusCount; + state.counters["corpus_blocks"] = kCorpusCount; + state.counters["timed_blocks"] = blocks; + state.counters["seed"] = kSeed; + state.counters["information_bytes_per_block"] = kInformationBytes; + state.counters["target_bit_probability"] = 0.005; + state.counters["channel_codeword_BER"] = + static_cast(channel_bits) / (kCorpusCount * kBlockBytes * 8); + state.counters["channel_flipped_bits"] = static_cast(channel_bits); + state.counters["channel_corrupted_bytes"] = static_cast(channel_symbols); + state.counters["corpus_data_residual_bits"] = data_residual_bits / sweeps; + state.counters["corpus_codeword_residual_bits"] = codeword_residual_bits / sweeps; + state.counters["data_residual_BER"] = + data_residual_bits / (blocks * kInformationBytes * 8); + state.counters["codeword_residual_BER"] = + codeword_residual_bits / (blocks * kBlockBytes * 8); + // Counts per unique corpus, not inflated by timed replays of the same noise. + state.counters["corpus_message_failures"] = message_failures / sweeps; + state.counters["corpus_block_failures"] = block_failures / sweeps; + state.counters["corpus_validity_failures"] = validity_failures / sweeps; + state.counters["corpus_valid_wrong_blocks"] = valid_wrong_blocks / sweeps; + state.counters["message_recovery_fraction"] = 1 - message_failures / blocks; + state.counters["block_recovery_fraction"] = 1 - block_failures / blocks; + state.counters["corpus_pass_limits"] = pass_limits / sweeps; + state.counters["mean_directional_passes"] = passes / blocks; + state.counters["max_directional_passes"] = static_cast(max_passes); + state.counters["mean_strong_lines"] = strong_lines / blocks; + state.counters["mean_weak_lines"] = weak_lines / blocks; + state.counters["pass_limit"] = kPassLimit; + state.SetItemsProcessed(state.iterations() * kCorpusCount); + state.SetBytesProcessed(state.iterations() * kCorpusCount * kInformationBytes); + state.SetLabel("64 blocks/iteration; information bytes=224*254; " + "Correct includes exact final validity; tuned backend"); +} + +const auto* kProductCorrectionBSC = benchmark::RegisterBenchmark( + "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/" + "Nstrong:256/Kstrong:224/Nweak:256/Kweak:254", + [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state); }); + +const auto* kProductCorrectionSingle = benchmark::RegisterBenchmark( + "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/Single", + [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state, 0); }); +const auto* kProductCorrectionStrongBatch = benchmark::RegisterBenchmark( + "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/StrongBatch", + [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state, 1); }); +const auto* kProductCorrectionBothBatch = benchmark::RegisterBenchmark( + "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/BothBatch", + [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state, 2); }); + +} // namespace diff --git a/include/reed_solomon/error_correction.h b/include/reed_solomon/error_correction.h new file mode 100644 index 0000000..fb6fa55 --- /dev/null +++ b/include/reed_solomon/error_correction.h @@ -0,0 +1,35 @@ +#pragma once + +#include "reed_solomon/lch_decoder.h" + +namespace gf2p8::rs { + +/** @brief Bounded-distance correction outcome. */ +enum class CorrectionStatus { + ok, + invalid_argument, + unsupported_dimensions, + uncorrectable, + reconstruction_failed, +}; + +/** @brief Status and number of repaired symbols (zero on failure). */ +struct CorrectionResult { + CorrectionStatus status = CorrectionStatus::invalid_argument; + size_t error_count = 0; +}; + +/** + * @brief Transactionally repairs data AND parity of one Cantor RS codeword. + * @param decoder Decoder defining K data and R recovery symbols. + * @param codeword Exactly K+R mutable symbols in [data][recovery] order. + * @return Status and corrected symbol count; zero count on an already valid word. + * @details Requires unshortened power-of-two N=K+R <=256 and power-of-two + * R<=K. Corrects up to floor(R/2) unknown symbol errors, with no erasures. + * On any non-ok status every input byte is unchanged. Success means the result + * is a codeword within that radius, NOT necessarily the transmitted codeword. + */ +CorrectionResult CorrectCodeword(const LCHDecoder& decoder, + std::span codeword); + +} // namespace gf2p8::rs diff --git a/include/reed_solomon/strong_weak_rs_product_code.h b/include/reed_solomon/strong_weak_rs_product_code.h new file mode 100644 index 0000000..4a2bc84 --- /dev/null +++ b/include/reed_solomon/strong_weak_rs_product_code.h @@ -0,0 +1,91 @@ +#pragma once + +#include "reed_solomon/error_correction.h" +#include "reed_solomon/lch_encoder.h" + +namespace gf2p8::rs { + +namespace detail { struct ProductCorrectionAccess; } + +/** @brief Reason iterative product correction stopped, independent of validity. */ +enum class ProductTermination { invalid_argument, no_change, pass_limit }; + +/** @brief Product correction outcome; validity does not prove original content. */ +struct ProductCorrectionResult { + ProductTermination termination = ProductTermination::invalid_argument; + bool all_zero_syndromes = false; + size_t directional_passes = 0; + size_t strong_lines_visited = 0; + size_t weak_lines_visited = 0; + /** @brief Accepted symbol writes, counting repeated repairs separately. */ + size_t changed_symbols = 0; +}; + +/** + * @brief Systematic Cantor RS product with strong columns and weak rows. + * @details Row-major block has Nstrong rows and Nweak columns. The top-left + * Kstrong by Kweak rectangle holds data; every column and every row, including + * parity regions, is a component codeword. Errors only: no erasures, + * shortening, or backtracking. Both N must be powers of two <=256; strong R + * must be a power of two with 2<=R<=K, and weak R must equal 2. + */ +class StrongWeakRSProductCode { + public: + /** + * @brief Constructs a product code; unsupported dimensions make Valid false. + * @param strong_n Number of rows. + * @param strong_k Number of systematic rows. + * @param weak_n Number of columns. + * @param weak_k Number of systematic columns. + */ + StrongWeakRSProductCode(size_t strong_n = 256, size_t strong_k = 224, + size_t weak_n = 256, size_t weak_k = 254); + + /** @brief Reports supported dimensions. @return Whether operations are valid. */ + bool Valid() const; + /** @brief Returns required row-major block size. @return Bytes, or zero if invalid. */ + size_t BlockSize() const; + + /** + * @brief Fills all product parity, preserving the systematic rectangle. + * @param block Exactly BlockSize() symbols, with data already in place. + * @param backend Owned encoder backend, scalar available for reference checks. + * @return Encoding status; on failure block is unchanged. + */ + lch::Status Encode(std::span block, + lch::Backend backend = lch::Backend::tuned) const; + + /** + * @brief Applies alternating strong BDD and gated weak single-error passes. + * @param block Exactly BlockSize() mutable symbols. + * @param max_directional_passes Cap counting each direction separately; >=2. + * @return Termination, pass/work counts, and final all-component validity. + * @details Always starts with a full strong pass then a full weak pass, even + * if strong changes nothing. From then on visits only lines intersecting + * accepted byte changes in the preceding pass. Stops on a no-change pass + * (including the initial weak pass), or the cap; no-change takes precedence. + * Strong success, including zero syndromes, protects that column; failure + * leaves its bytes unchanged and unprotects it. Unvisited protection persists. + * Weak candidates commit only for one changed symbol with popcount(old XOR + * new)<=2 in an unprotected column; rejected rows are unchanged. Protection + * changes alone never activate lines. Accepted repairs are retained at exit; + * the entire iterative operation is not transactional. Invalid arguments + * leave the block untouched. Validity is checked separately at exit and does + * not count as a directional pass or guarantee the original message. + */ + ProductCorrectionResult Correct(std::span block, + size_t max_directional_passes = 16) const; + + private: + friend struct detail::ProductCorrectionAccess; + ProductCorrectionResult CorrectImpl(std::span block, + size_t max_directional_passes, + unsigned batch_passes, + bool tracked_validation = true) const; + size_t strong_n_, strong_k_, weak_n_, weak_k_; + bool valid_; + LCHEncoder strong_encoder_, weak_encoder_; + LCHDecoder strong_decoder_, weak_decoder_; +}; + +} // namespace gf2p8::rs diff --git a/src/reed_solomon/error_correction/batched.cc b/src/reed_solomon/error_correction/batched.cc index 29cb8a2..38d68ee 100644 --- a/src/reed_solomon/error_correction/batched.cc +++ b/src/reed_solomon/error_correction/batched.cc @@ -111,7 +111,8 @@ CorrectionStatus CorrectColumnsScalar(const LCHDecoder& decoder, size_t byte_count, size_t first_column, std::span results, - std::span error_masks) { + std::span error_masks, + std::span mutable_recovery) { const size_t data_count = data.size(); const size_t recovery_count = recovery.size(); const size_t codeword_size = data_count + recovery_count; @@ -126,15 +127,29 @@ CorrectionStatus CorrectColumnsScalar(const LCHDecoder& decoder, for (size_t i = 0; i < recovery_count; ++i) { recovery_values[i] = recovery[i][column]; } - const CorrectionResult result = CorrectOne( - decoder, std::span(data_values).first(data_count), - std::span(recovery_values).first(recovery_count), - std::span(mask).first(codeword_size)); + CorrectionResult result; + if (mutable_recovery.empty()) { + result = CorrectOne( + decoder, std::span(data_values).first(data_count), + std::span(recovery_values).first(recovery_count), + std::span(mask).first(codeword_size)); + } else { + std::copy_n(recovery_values.begin(), recovery_count, + data_values.begin() + data_count); + const auto before = data_values; + result = CorrectCodeword(decoder, std::span(data_values).first(codeword_size)); + for (size_t i = 0; i < codeword_size; ++i) { + mask[i] = before[i] != data_values[i]; + } + } results[column] = result; if (result.status == CorrectionStatus::ok) { for (size_t i = 0; i < data_count; ++i) { data[i][column] = data_values[i]; } + for (size_t i = 0; i < mutable_recovery.size(); ++i) { + mutable_recovery[i][column] = data_values[data_count + i]; + } } for (size_t position = 0; position < codeword_size; ++position) { error_masks[position * byte_count + column] = mask[position]; @@ -584,7 +599,8 @@ void CorrectChunk32(std::span data, const CodeParameters& parameters, const lch::detail::ResolvedKernels& kernels, std::span results, - std::span error_masks) { + std::span error_masks, + std::span mutable_recovery) { #if !defined(__GFNI__) static_assert(!UseGFNI); #endif @@ -786,8 +802,10 @@ void CorrectChunk32(std::span data, data_error_lanes |= root_masks[native_position]; } } - const uint32_t location_only_lanes = candidate_lanes & ~data_error_lanes; - candidate_lanes &= data_error_lanes; + // Whole-codeword mode must evaluate and verify parity-only candidates too. + const uint32_t location_only_lanes = mutable_recovery.empty() + ? candidate_lanes & ~data_error_lanes : 0; + if (mutable_recovery.empty()) candidate_lanes &= data_error_lanes; if (candidate_lanes == 0) { PublishChunkResults(results, error_masks, byte_count, column, parameters.family, data_count, recovery_count, @@ -936,18 +954,20 @@ void CorrectChunk32(std::span data, ++native_position) { const size_t public_position = PublicPosition( parameters.family, data_count, recovery_count, native_position); - if (public_position >= data_count) { + if (public_position >= data_count && mutable_recovery.empty()) { continue; } + Element* destination = public_position < data_count + ? data[public_position] : mutable_recovery[public_position - data_count]; const uint32_t active = root_masks[native_position] & candidate_lanes; const __m256i old_data = _mm256_loadu_si256( - reinterpret_cast(data[public_position] + column)); + reinterpret_cast(destination + column)); const __m256i correction = _mm256_and_si256( _mm256_load_si256( reinterpret_cast(Row(work, native_position))), LaneMask(active)); _mm256_storeu_si256( - reinterpret_cast<__m256i*>(data[public_position] + column), + reinterpret_cast<__m256i*>(destination + column), _mm256_xor_si256(old_data, correction)); } PublishChunkResults(results, error_masks, byte_count, column, @@ -960,12 +980,13 @@ void CorrectChunk32(std::span data, } // namespace -CorrectionStatus CorrectBatch(const LCHDecoder& decoder, +static CorrectionStatus CorrectBatchImpl(const LCHDecoder& decoder, std::span data, std::span recovery, size_t byte_count, std::span results, - std::span error_masks) { + std::span error_masks, + std::span mutable_recovery) { if (!decoder.Valid()) { return CorrectionStatus::invalid_argument; } @@ -1026,19 +1047,46 @@ CorrectionStatus CorrectBatch(const LCHDecoder& decoder, #if defined(__GFNI__) for (; column + kBatchLanes <= byte_count; column += kBatchLanes) { CorrectChunk32(data, recovery, byte_count, column, parameters, - *kernels, results, error_masks); + *kernels, results, error_masks, mutable_recovery); } #else for (; column + kBatchLanes <= byte_count; column += kBatchLanes) { CorrectChunk32(data, recovery, byte_count, column, parameters, - *kernels, results, error_masks); + *kernels, results, error_masks, mutable_recovery); } #endif } } #endif return CorrectColumnsScalar(decoder, data, recovery, byte_count, column, - results, error_masks); + results, error_masks, mutable_recovery); +} + +CorrectionStatus CorrectBatch(const LCHDecoder& decoder, + std::span data, + std::span recovery, + size_t byte_count, + std::span results, + std::span error_masks) { + return CorrectBatchImpl(decoder, data, recovery, byte_count, results, + error_masks, {}); +} + +CorrectionStatus CorrectCodewordBatch(const LCHDecoder& decoder, + std::span shards, + size_t byte_count, + std::span results, + std::span error_masks) { + if (!decoder.Valid() || shards.size() > kFieldSize || + shards.size() != decoder.DataCount() + decoder.RecoveryCount()) { + return CorrectionStatus::invalid_argument; + } + const auto recovery = shards.subspan(decoder.DataCount()); + std::array immutable_recovery{}; + std::copy(recovery.begin(), recovery.end(), immutable_recovery.begin()); + return CorrectBatchImpl(decoder, shards.first(decoder.DataCount()), + std::span(immutable_recovery).first(recovery.size()), + byte_count, results, error_masks, recovery); } } // namespace gf2p8::rs::detail::error_correction diff --git a/src/reed_solomon/error_correction/internal.h b/src/reed_solomon/error_correction/internal.h index 216a6d8..459ae66 100644 --- a/src/reed_solomon/error_correction/internal.h +++ b/src/reed_solomon/error_correction/internal.h @@ -4,22 +4,12 @@ #include #include -#include "reed_solomon/lch_decoder.h" +#include "reed_solomon/error_correction.h" namespace gf2p8::rs::detail::error_correction { -enum class CorrectionStatus { - ok, - invalid_argument, - unsupported_dimensions, - uncorrectable, - reconstruction_failed, -}; - -struct CorrectionResult { - CorrectionStatus status = CorrectionStatus::invalid_argument; - size_t error_count = 0; -}; +using ::gf2p8::rs::CorrectionStatus; +using ::gf2p8::rs::CorrectionResult; /** * @brief Corrects one scalar LCH Reed-Solomon codeword. @@ -62,4 +52,19 @@ CorrectionStatus CorrectBatch(const LCHDecoder& decoder, std::span results, std::span error_masks); +/** + * @brief Repairs whole independent codewords, including recovery positions. + * @param decoder Component code dimensions. + * @param shards N disjoint mutable shard ranges in public data/recovery order. + * @param byte_count Independent codewords per shard. + * @param results Per-codeword outcomes; failed codewords remain unchanged. + * @param error_masks Position-major masks, as in CorrectBatch. + * @return Call-level status; all shard and output ranges must be disjoint. + */ +CorrectionStatus CorrectCodewordBatch(const LCHDecoder& decoder, + std::span shards, + size_t byte_count, + std::span results, + std::span error_masks); + } // namespace gf2p8::rs::detail::error_correction diff --git a/src/reed_solomon/error_correction/scalar.cc b/src/reed_solomon/error_correction/scalar.cc index 20389d9..4955662 100644 --- a/src/reed_solomon/error_correction/scalar.cc +++ b/src/reed_solomon/error_correction/scalar.cc @@ -228,10 +228,11 @@ size_t PublicPosition(CodeFamily family, : native_position - recovery_count; } -CorrectionStatus RecoverDataWithEvaluator( +CorrectionStatus RecoverWithEvaluator( CodeFamily family, std::span data, std::span recovery, + std::span mutable_recovery, size_t recovery_count, std::span root_positions, const Values& locator_samples, @@ -369,6 +370,8 @@ CorrectionStatus RecoverDataWithEvaluator( PublicPosition(family, data_count, recovery_count, position); if (data_position < data_count) { data[data_position] ^= corrections[position]; + } else if (!mutable_recovery.empty()) { + mutable_recovery[data_position - data_count] ^= corrections[position]; } } return CorrectionStatus::ok; @@ -376,10 +379,11 @@ CorrectionStatus RecoverDataWithEvaluator( } // namespace -CorrectionResult CorrectOne(const LCHDecoder& decoder, - std::span data, - std::span recovery, - std::span error_mask) { +static CorrectionResult CorrectOneImpl(const LCHDecoder& decoder, + std::span data, + std::span recovery, + std::span error_mask, + std::span mutable_recovery) { if (RangesOverlap(error_mask, data) || RangesOverlap(error_mask, recovery)) { return Result(CorrectionStatus::invalid_argument); } @@ -636,7 +640,9 @@ CorrectionResult CorrectOne(const LCHDecoder& decoder, } CorrectionStatus recovery_status = CorrectionStatus::ok; - if (has_data_error && root_count == 1) { + // Whole-codeword mode verifies every candidate, including parity-only roots. + // Retain the existing data-only fast path for CorrectOne/CorrectBatch callers. + if (has_data_error && root_count == 1 && mutable_recovery.empty()) { // Every aligned R-point native Cantor IFFT has unit leading Lagrange // coefficient. Therefore the highest syndrome coefficient is the error // magnitude when exactly one error is present. @@ -644,9 +650,9 @@ CorrectionResult CorrectOne(const LCHDecoder& decoder, const size_t data_index = PublicPosition(parameters.family, data_count, recovery_count, root_positions[0]); data[data_index] ^= magnitude; - } else if (has_data_error) { - recovery_status = RecoverDataWithEvaluator( - parameters.family, data, recovery, recovery_count, + } else if (has_data_error || !mutable_recovery.empty()) { + recovery_status = RecoverWithEvaluator( + parameters.family, data, recovery, mutable_recovery, recovery_count, std::span(root_positions).first(root_count), locator_samples, locator_coefficients, locator_degree, syndrome_samples, tables); } @@ -661,4 +667,28 @@ CorrectionResult CorrectOne(const LCHDecoder& decoder, return Result(CorrectionStatus::ok, root_count); } +CorrectionResult CorrectOne(const LCHDecoder& decoder, + std::span data, + std::span recovery, + std::span error_mask) { + return CorrectOneImpl(decoder, data, recovery, error_mask, {}); +} + } // namespace gf2p8::rs::detail::error_correction + +namespace gf2p8::rs { + +CorrectionResult CorrectCodeword(const LCHDecoder& decoder, + std::span codeword) { + if (!decoder.Valid() || + codeword.size() != decoder.DataCount() + decoder.RecoveryCount()) { + return {.status = CorrectionStatus::invalid_argument}; + } + std::array mask{}; + auto recovery = codeword.subspan(decoder.DataCount()); + return detail::error_correction::CorrectOneImpl( + decoder, codeword.first(decoder.DataCount()), recovery, + std::span(mask).first(codeword.size()), recovery); +} + +} // namespace gf2p8::rs diff --git a/src/reed_solomon/product_code_internal.h b/src/reed_solomon/product_code_internal.h new file mode 100644 index 0000000..cf94a1c --- /dev/null +++ b/src/reed_solomon/product_code_internal.h @@ -0,0 +1,26 @@ +#pragma once + +#include "reed_solomon/strong_weak_rs_product_code.h" + +namespace gf2p8::rs::detail { + +/** @brief Private differential-test and benchmark access to initial-pass choices. */ +struct ProductCorrectionAccess { + /** + * @brief Runs the same scheduler with zero, one, or two initial batched passes. + * @param code Product dimensions and component decoders. + * @param block Mutable row-major block. + * @param cap Directional pass limit. + * @param batch_passes Initial passes to batch (0: reference single path). + * @param tracked_validation Skip known-clean lines; false retains the full scan. + * @return The normal product outcome and work counts. + */ + static ProductCorrectionResult Correct(const StrongWeakRSProductCode& code, + std::span block, size_t cap, + unsigned batch_passes, + bool tracked_validation = true) { + return code.CorrectImpl(block, cap, batch_passes, tracked_validation); + } +}; + +} // namespace gf2p8::rs::detail diff --git a/src/reed_solomon/strong_weak_rs_product_code.cc b/src/reed_solomon/strong_weak_rs_product_code.cc new file mode 100644 index 0000000..221f680 --- /dev/null +++ b/src/reed_solomon/strong_weak_rs_product_code.cc @@ -0,0 +1,237 @@ +#include "reed_solomon/strong_weak_rs_product_code.h" +#include "reed_solomon/error_correction/internal.h" + +#include +#include +#include +#include + +namespace gf2p8::rs { +namespace { + +bool Aligned(size_t n, size_t k) { + return n <= 256 && std::has_single_bit(n) && k < n && + n - k >= 2 && n - k <= k && std::has_single_bit(n - k); +} + +} // namespace + +StrongWeakRSProductCode::StrongWeakRSProductCode(size_t strong_n, size_t strong_k, + size_t weak_n, size_t weak_k) + : strong_n_(strong_n), strong_k_(strong_k), weak_n_(weak_n), weak_k_(weak_k), + valid_(Aligned(strong_n, strong_k) && Aligned(weak_n, weak_k) && + weak_n - weak_k == 2), + strong_encoder_(valid_ ? strong_k : 0, valid_ ? strong_n - strong_k : 0), + weak_encoder_(valid_ ? weak_k : 0, valid_ ? weak_n - weak_k : 0), + strong_decoder_(valid_ ? strong_k : 0, valid_ ? strong_n - strong_k : 0), + weak_decoder_(valid_ ? weak_k : 0, valid_ ? weak_n - weak_k : 0) {} + +bool StrongWeakRSProductCode::Valid() const { + return valid_ && strong_encoder_.Valid() && weak_encoder_.Valid() && + strong_decoder_.Valid() && weak_decoder_.Valid(); +} + +size_t StrongWeakRSProductCode::BlockSize() const { + return Valid() ? strong_n_ * weak_n_ : 0; +} + +lch::Status StrongWeakRSProductCode::Encode(std::span block, + lch::Backend backend) const { + if (!Valid() || block.size() != BlockSize()) { + return lch::Status::invalid_argument; + } + std::vector candidate(block.begin(), block.end()); + std::array data{}; + std::array recovery{}; + std::vector workspace(std::max(strong_encoder_.WorkspaceSize(weak_n_), + weak_encoder_.WorkspaceSize(1))); + for (size_t row = 0; row < strong_k_; ++row) { + for (size_t col = 0; col < weak_k_; ++col) { + data[col] = &candidate[row * weak_n_ + col]; + } + for (size_t col = weak_k_; col < weak_n_; ++col) { + recovery[col - weak_k_] = &candidate[row * weak_n_ + col]; + } + const auto status = weak_encoder_.Encode( + std::span(data).first(weak_k_), std::span(recovery).first(2), 1, + workspace, backend); + if (status != lch::Status::ok) { + return status; + } + } + for (size_t row = 0; row < strong_k_; ++row) { + data[row] = &candidate[row * weak_n_]; + } + for (size_t row = strong_k_; row < strong_n_; ++row) { + recovery[row - strong_k_] = &candidate[row * weak_n_]; + } + const auto status = strong_encoder_.Encode( + std::span(data).first(strong_k_), + std::span(recovery).first(strong_n_ - strong_k_), weak_n_, workspace, + backend); + if (status == lch::Status::ok) { + std::copy(candidate.begin(), candidate.end(), block.begin()); + } + return status; +} + +ProductCorrectionResult StrongWeakRSProductCode::Correct( + std::span block, size_t max_directional_passes) const { + // Avoid packing overhead when no complete SIMD batch can run. + const unsigned batches = lch::BackendAvailable(lch::Backend::avx2) && + std::max(strong_n_, weak_n_) >= 32 ? 2 : 0; + return CorrectImpl(block, max_directional_passes, batches); +} + +ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( + std::span block, size_t max_directional_passes, + unsigned batch_passes, bool tracked_validation) const { + ProductCorrectionResult result; + if (!Valid() || block.size() != BlockSize() || max_directional_passes < 2) { + return result; + } + std::array protected_columns{}; + // False means unknown, never proven nonzero: intersecting edits can cancel. + std::array clean_columns{}, clean_rows{}; + std::array active{}; + std::array candidate{}; + std::vector packed(batch_passes != 0 ? block.size() : 0); + std::vector masks(packed.size()); + std::array outcomes{}; + std::array shards{}; + for (size_t pass = 0; pass < max_directional_passes; ++pass) { + const bool strong = pass % 2 == 0; + const size_t lines = strong ? weak_n_ : strong_n_; + const size_t length = strong ? strong_n_ : weak_n_; + std::array next{}; + size_t changes = 0; + const bool batched = pass < std::min(batch_passes, 2u); + if (batched) { + // Columns already have position-major layout; rows need transposition. + // Keep tentative repairs private until the existing weak gates accept. + if (strong) std::copy(block.begin(), block.end(), packed.begin()); + for (size_t pos = 0; pos < length; ++pos) { + shards[pos] = packed.data() + pos * lines; + if (!strong) { + for (size_t line = 0; line < lines; ++line) { + shards[pos][line] = block[line * weak_n_ + pos]; + } + } + } + const auto status = detail::error_correction::CorrectCodewordBatch( + strong ? strong_decoder_ : weak_decoder_, + std::span(shards).first(length), lines, + std::span(outcomes).first(lines), masks); + if (status != CorrectionStatus::ok) return result; + } + if (batched && strong) { + result.strong_lines_visited += lines; + for (size_t line = 0; line < lines; ++line) { + clean_columns[line] = protected_columns[line] = + outcomes[line].status == CorrectionStatus::ok; + } + // Batch failure is transactional and masks describe only verified edits. + // Walk position-major output rather than gathering every column again. + for (size_t pos = 0; pos < length; ++pos) { + for (size_t line = 0; line < lines; ++line) { + const size_t index = pos * lines + line; + if (masks[index]) { + block[index] = packed[index]; + next[pos] = true; + clean_rows[pos] = false; + ++changes; + } + } + } + } else for (size_t line = 0; line < lines; ++line) { + if (pass >= 2 && !active[line]) { + continue; + } + if (strong) { + ++result.strong_lines_visited; + } else { + ++result.weak_lines_visited; + } + const auto index = [&](size_t pos) { + return strong ? pos * weak_n_ + line : line * weak_n_ + pos; + }; + if (!batched) { + for (size_t pos = 0; pos < length; ++pos) { + candidate[pos] = block[index(pos)]; + } + } + const auto correction = batched ? outcomes[line] : CorrectCodeword( + strong ? strong_decoder_ : weak_decoder_, + std::span(candidate).first(length)); + auto& clean = strong ? clean_columns[line] : clean_rows[line]; + clean = correction.status == CorrectionStatus::ok && correction.error_count == 0; + if (strong) { + protected_columns[line] = correction.status == CorrectionStatus::ok; + } + if (correction.status != CorrectionStatus::ok) { + continue; + } + if (correction.error_count == 0) continue; + if (batched) { + for (size_t pos = 0; pos < length; ++pos) candidate[pos] = shards[pos][line]; + } + if (!strong) { + if (correction.error_count != 1) { + continue; + } + bool accept = true; + for (size_t pos = 0; pos < length; ++pos) { + const unsigned delta = block[index(pos)] ^ candidate[pos]; + if (delta != 0 && (protected_columns[pos] || std::popcount(delta) > 2)) { + accept = false; + } + } + if (!accept) { + continue; + } + } + for (size_t pos = 0; pos < length; ++pos) { + if (block[index(pos)] != candidate[pos]) { + block[index(pos)] = candidate[pos]; + next[pos] = true; + (strong ? clean_rows[pos] : clean_columns[pos]) = false; + ++changes; + } + } + // Successful BDD verifies its candidate internally; only committed + // candidates establish validity. Rejected weak candidates do not. + clean = true; + } + ++result.directional_passes; + result.changed_symbols += changes; + active = next; + if (pass >= 1 && changes == 0) { + result.termination = ProductTermination::no_change; + break; + } + result.termination = ProductTermination::pass_limit; + } + // Check the actual final block, not stale protection or tentative candidates. + result.all_zero_syndromes = true; + for (bool strong : {true, false}) { + const size_t lines = strong ? weak_n_ : strong_n_; + const size_t length = strong ? strong_n_ : weak_n_; + for (size_t line = 0; line < lines; ++line) { + if (tracked_validation && (strong ? clean_columns[line] : clean_rows[line])) { + continue; + } + for (size_t pos = 0; pos < length; ++pos) { + candidate[pos] = block[strong ? pos * weak_n_ + line : line * weak_n_ + pos]; + } + const auto check = CorrectCodeword( + strong ? strong_decoder_ : weak_decoder_, + std::span(candidate).first(length)); + if (check.status != CorrectionStatus::ok || check.error_count != 0) { + result.all_zero_syndromes = false; + } + } + } + return result; +} + +} // namespace gf2p8::rs diff --git a/tests/product_code_tests.cc b/tests/product_code_tests.cc new file mode 100644 index 0000000..659b85c --- /dev/null +++ b/tests/product_code_tests.cc @@ -0,0 +1,525 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "reed_solomon/strong_weak_rs_product_code.h" +#include "reed_solomon/error_correction/internal.h" +#include "reed_solomon/product_code_internal.h" + +namespace { + +using gf2p8::Element; +using gf2p8::lch::Backend; +using gf2p8::lch::Status; +using namespace gf2p8::rs; + +std::vector Codeword(size_t n, size_t k, uint32_t seed) { + LCHEncoder encoder(k, n - k); + std::mt19937 random(seed); + std::vector word(n); + for (size_t i = 0; i < k; ++i) word[i] = static_cast(random()); + std::vector data(k); + std::vector recovery(n - k); + for (size_t i = 0; i < k; ++i) data[i] = &word[i]; + for (size_t i = k; i < n; ++i) recovery[i - k] = &word[i]; + std::vector workspace(encoder.WorkspaceSize(1)); + EXPECT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + return word; +} + +TEST(WholeCodeword, RepairsEveryPositionAndFullRadiusIncludingParity) { + for (const auto [n, k] : {std::pair{4u, 2u}, {8u, 4u}, {16u, 12u}, + {256u, 224u}, {256u, 128u}, {256u, 254u}}) { + const auto original = Codeword(n, k, n + k); + LCHDecoder decoder(k, n - k); + for (size_t pos = 0; pos < n; ++pos) { + auto word = original; + word[pos] ^= 0xff; + const auto result = CorrectCodeword(decoder, word); + ASSERT_EQ(result.status, CorrectionStatus::ok) << n << ':' << pos; + EXPECT_EQ(result.error_count, 1u); + ASSERT_EQ(word, original); + } + std::mt19937 random(n); + for (size_t trial = 0; trial < 20; ++trial) { + auto word = original; + std::vector positions(n); + for (size_t i = 0; i < n; ++i) positions[i] = i; + if (trial != 0) std::shuffle(positions.begin(), positions.end(), random); + else std::reverse(positions.begin(), positions.end()); + for (size_t i = 0; i < (n - k) / 2; ++i) { + word[positions[i]] ^= static_cast(1 + random() % 255); + } + const auto result = CorrectCodeword(decoder, word); + ASSERT_EQ(result.status, CorrectionStatus::ok); + EXPECT_EQ(result.error_count, (n - k) / 2); + EXPECT_EQ(word, original); + } + auto clean = original; + EXPECT_EQ(CorrectCodeword(decoder, clean).error_count, 0u); + EXPECT_EQ(clean, original); + } +} + +TEST(WholeCodeword, TransactionalFailureAndInvalidDimensions) { + LCHDecoder decoder(6, 2); + auto word = Codeword(8, 6, 42); + // Equal magnitudes cancel the leading syndrome: no one-error candidate. + word[0] ^= 7; + word[7] ^= 7; + const auto before = word; + const auto result = CorrectCodeword(decoder, word); + EXPECT_EQ(result.status, CorrectionStatus::uncorrectable); + EXPECT_EQ(result.error_count, 0u); + EXPECT_EQ(word, before); + EXPECT_EQ(CorrectCodeword(decoder, std::span(word).first(7)).status, + CorrectionStatus::invalid_argument); + EXPECT_EQ(CorrectCodeword(LCHDecoder(0, 0), word).status, + CorrectionStatus::invalid_argument); + EXPECT_EQ(CorrectCodeword(LCHDecoder(5, 3), word).status, + CorrectionStatus::unsupported_dimensions); + EXPECT_EQ(word, before); + std::vector shortened(7, 1); + const auto saved = shortened; + EXPECT_EQ(CorrectCodeword(LCHDecoder(5, 2), shortened).status, + CorrectionStatus::unsupported_dimensions); + EXPECT_EQ(shortened, saved); +} + +TEST(ProductCode, DimensionsAndInvalidCalls) { + EXPECT_TRUE(StrongWeakRSProductCode().Valid()); + EXPECT_EQ(StrongWeakRSProductCode().BlockSize(), 65536u); + for (const auto [n, k] : {std::pair{0u, 0u}, {8u, 8u}, {8u, 9u}, {7u, 5u}, + {8u, 5u}, {8u, 2u}, {512u, 480u}, {8u, 7u}}) { + StrongWeakRSProductCode code(n, k, 8, 6); + EXPECT_FALSE(code.Valid()); + EXPECT_EQ(code.BlockSize(), 0u); + std::vector block(32, 17); + const auto before = block; + EXPECT_EQ(code.Encode(block), Status::invalid_argument); + EXPECT_EQ(code.Correct(block).termination, ProductTermination::invalid_argument); + EXPECT_EQ(block, before); + } + EXPECT_FALSE(StrongWeakRSProductCode(8, 4, 8, 4).Valid()); + EXPECT_FALSE(StrongWeakRSProductCode(std::numeric_limits::max(), 1).Valid()); + StrongWeakRSProductCode code(4, 2, 8, 6); + std::vector block(32, 17); + const auto before = block; + for (size_t cap : {0u, 1u}) { + const auto result = code.Correct(block, cap); + EXPECT_EQ(result.termination, ProductTermination::invalid_argument); + EXPECT_EQ(result.directional_passes, 0u); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_EQ(block, before); + } + EXPECT_EQ(code.Correct(std::span(block).first(31)).termination, + ProductTermination::invalid_argument); + EXPECT_EQ(code.Encode(std::span(block).first(31)), Status::invalid_argument); + EXPECT_EQ(block, before); +} + +TEST(ProductCode, SystematicEncodingScalarAgreementAndAllComponentValidity) { + for (const auto [ns, ks, nw, kw] : + {std::array{4, 2, 8, 6}, {16, 12, 16, 14}, {256, 224, 256, 254}}) { + StrongWeakRSProductCode code(ns, ks, nw, kw); + std::mt19937 random(901); + std::vector block(code.BlockSize()); + for (auto& value : block) value = static_cast(random()); + const auto input = block; + auto scalar = block; + ASSERT_EQ(code.Encode(block), Status::ok); + ASSERT_EQ(code.Encode(scalar, Backend::scalar), Status::ok); + EXPECT_EQ(block, scalar); + for (size_t row = 0; row < ks; ++row) { + for (size_t col = 0; col < kw; ++col) { + ASSERT_EQ(block[row * nw + col], input[row * nw + col]); + } + } + const auto result = code.Correct(block); + EXPECT_TRUE(result.all_zero_syndromes); + EXPECT_EQ(result.directional_passes, 2u); + EXPECT_EQ(result.strong_lines_visited, nw); + EXPECT_EQ(result.weak_lines_visited, ns); + EXPECT_EQ(result.changed_symbols, 0u); + EXPECT_EQ(result.termination, ProductTermination::no_change); + EXPECT_EQ(block, scalar); + // Arbitrary symbol damage in each of the four product regions. + for (auto [row, col] : {std::pair{size_t{0}, size_t{0}}, {ks, size_t{0}}, + {size_t{0}, kw}, {ks, kw}}) { + block[row * nw + col] ^= 0xff; + const auto repaired = code.Correct(block); + EXPECT_TRUE(repaired.all_zero_syndromes); + EXPECT_EQ(repaired.changed_symbols, 1u); + EXPECT_EQ(block, scalar); + } + } +} + +TEST(ProductCode, InitialWeakPassSelectiveActivationAndCap) { + StrongWeakRSProductCode code(4, 2, 8, 6); + for (size_t col : {0u, 6u, 7u}) { + for (Element magnitude : {Element{1}, Element{3}}) { + std::vector damaged(32); + // Strong cannot repair two equal errors; weak can repair both rows, + // including strong-parity rows and the parity/parity corner. + damaged[2 * 8 + col] = magnitude; + damaged[3 * 8 + col] = magnitude; + auto capped = damaged; + const auto cap = code.Correct(capped, 2); + EXPECT_EQ(cap.termination, ProductTermination::pass_limit); + EXPECT_EQ(cap.directional_passes, 2u); + EXPECT_TRUE(cap.all_zero_syndromes); + EXPECT_EQ(capped, std::vector(32)); + const auto result = code.Correct(damaged); + EXPECT_EQ(result.directional_passes, 3u); + EXPECT_EQ(result.strong_lines_visited, 9u); + EXPECT_EQ(result.weak_lines_visited, 4u); + EXPECT_EQ(result.changed_symbols, 2u); + EXPECT_EQ(result.termination, ProductTermination::no_change); + EXPECT_TRUE(result.all_zero_syndromes); + EXPECT_EQ(damaged, capped); + } + } +} + +TEST(ProductCode, DefaultStrongRadiusRepairsAllColumnsIncludingParity) { + StrongWeakRSProductCode code; + std::vector block(code.BlockSize()); + std::mt19937 random(1983); + for (auto& value : block) value = static_cast(random()); + ASSERT_EQ(code.Encode(block, Backend::scalar), Status::ok); + const auto original = block; + for (size_t col = 0; col < 256; ++col) { + for (size_t error = 0; error < 16; ++error) { + const size_t row = (col + error * 17) % 256; + block[row * 256 + col] ^= static_cast(1 + random() % 255); + } + } + const auto result = code.Correct(block); + EXPECT_EQ(result.directional_passes, 2u); + EXPECT_EQ(result.changed_symbols, 4096u); + EXPECT_TRUE(result.all_zero_syndromes); + EXPECT_EQ(block, original); +} + +TEST(ProductCode, WeakBitGateRejectsTransactionallyWithoutActivation) { + StrongWeakRSProductCode code(4, 2, 8, 6); + for (Element magnitude : {Element{7}, Element{255}}) { + std::vector block(32); + block[6] = magnitude; + block[3 * 8 + 6] = magnitude; + const auto before = block; + const auto result = code.Correct(block); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_EQ(result.termination, ProductTermination::no_change); + EXPECT_EQ(result.directional_passes, 2u); + EXPECT_EQ(result.changed_symbols, 0u); + EXPECT_EQ(block, before); + } +} + +TEST(ProductCode, ChangesPropagateThroughFourDirectionalPasses) { + StrongWeakRSProductCode code(4, 2, 8, 6); + std::vector block(32); + block[0] = block[8] = block[9] = block[17] = 1; + // Both columns initially fail, and the middle row fails. Weak fixes the + // outer two rows; strong then fixes the middle row in both active columns. + auto capped = block; + const auto limit = code.Correct(capped, 2); + EXPECT_EQ(limit.termination, ProductTermination::pass_limit); + EXPECT_FALSE(limit.all_zero_syndromes); + EXPECT_EQ(limit.changed_symbols, 2u); + EXPECT_EQ(capped[8], 1); + EXPECT_EQ(capped[9], 1); + const auto result = code.Correct(block); + EXPECT_TRUE(result.all_zero_syndromes); + EXPECT_EQ(result.termination, ProductTermination::no_change); + EXPECT_EQ(result.directional_passes, 4u); + EXPECT_EQ(result.strong_lines_visited, 10u); + EXPECT_EQ(result.weak_lines_visited, 5u); + EXPECT_EQ(result.changed_symbols, 4u); + EXPECT_EQ(block, std::vector(32)); +} + +TEST(ProductCode, SuccessfulStrongRepairProtectsItsColumn) { + StrongWeakRSProductCode code(4, 2, 8, 6); + std::vector block(32); + for (size_t row = 0; row < 4; ++row) block[row * 8] = 1; + const auto protected_word = block; + block[0] ^= 2; + const auto result = code.Correct(block); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_EQ(result.changed_symbols, 1u); + EXPECT_EQ(result.directional_passes, 2u); + EXPECT_EQ(block, protected_word); +} + +TEST(ProductCode, UnvisitedColumnRetainsProtectionOnLaterWeakPass) { + StrongWeakRSProductCode code(8, 4, 8, 6); + // A valid strong column with systematic symbols [0,1,0,0]. Obtain its + // parity using the owned scalar encoder, independently of product Encode. + LCHEncoder encoder(4, 4); + std::array protected_column{0, 1, 0, 0}; + std::array data{}; + std::array recovery{}; + for (size_t i = 0; i < 4; ++i) { + data[i] = &protected_column[i]; + recovery[i] = &protected_column[4 + i]; + } + std::vector workspace(encoder.WorkspaceSize(1)); + ASSERT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + std::vector block(64); + for (size_t row = 0; row < 8; ++row) block[row * 8 + 1] = protected_column[row]; + const auto expected = block; + for (size_t row = 0; row < 4; ++row) block[row * 8] = 1; + const auto result = code.Correct(block); + EXPECT_EQ(result.directional_passes, 4u); + EXPECT_EQ(result.strong_lines_visited, 9u); + EXPECT_EQ(result.weak_lines_visited, 9u); + EXPECT_EQ(result.changed_symbols, 4u); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_EQ(block, expected); +} + +TEST(WholeCodeword, OverRadiusOutcomesAreTransactionalOrVerifiedCodewords) { + std::mt19937 random(8181); + for (const auto [n, k] : {std::pair{8u, 6u}, {16u, 12u}, {32u, 16u}}) { + LCHDecoder decoder(k, n - k); + LCHEncoder encoder(k, n - k); + for (size_t trial = 0; trial < 100; ++trial) { + auto word = Codeword(n, k, random()); + for (size_t pos = 0; pos <= (n - k) / 2; ++pos) { + word[n - 1 - pos] ^= static_cast(1 + random() % 255); + } + const auto before = word; + const auto result = CorrectCodeword(decoder, word); + if (result.status != CorrectionStatus::ok) { + EXPECT_EQ(word, before); + EXPECT_EQ(result.error_count, 0u); + continue; + } + size_t distance = 0; + for (size_t pos = 0; pos < n; ++pos) distance += word[pos] != before[pos]; + EXPECT_EQ(distance, result.error_count); + EXPECT_LE(distance, (n - k) / 2); + std::vector data(k); + std::vector parity(n - k); + std::vector recovery(n - k); + for (size_t i = 0; i < k; ++i) data[i] = &word[i]; + for (size_t i = 0; i < n - k; ++i) recovery[i] = &parity[i]; + std::vector workspace(encoder.WorkspaceSize(1)); + ASSERT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + EXPECT_TRUE(std::equal(parity.begin(), parity.end(), word.begin() + k)); + } + } +} + +TEST(ProductCode, ProtectedColumnRejectsEvenLowWeightWeakCandidate) { + StrongWeakRSProductCode code(4, 2, 8, 6); + // Constant columns are valid RS words. Every row proposes a one-bit repair + // at column 0, but its zero-syndrome strong protection must reject them. + std::vector block(32); + for (size_t row = 0; row < 4; ++row) block[row * 8] = 1; + const auto before = block; + const auto result = code.Correct(block); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_EQ(result.changed_symbols, 0u); + EXPECT_EQ(result.directional_passes, 2u); + EXPECT_EQ(block, before); +} + +TEST(WholeCodewordBatch, DifferentialDataParityFailuresAndTails) { + using detail::error_correction::CorrectCodewordBatch; + std::mt19937 random(0xba7c224); + for (const auto [n, k] : {std::pair{4u, 2u}, {8u, 4u}, {16u, 12u}, + {256u, 128u}, {256u, 224u}, {256u, 254u}}) { + LCHDecoder decoder(k, n - k); + for (size_t lanes : {1u, 31u, 32u, 33u, 65u, 256u}) { + std::vector packed(n * lanes); + auto expected = packed; + std::vector results(lanes), reference(lanes); + std::vector masks(packed.size(), 0xff); + std::vector shards(n); + for (size_t pos = 0; pos < n; ++pos) shards[pos] = &packed[pos * lanes]; + for (size_t lane = 0; lane < lanes; ++lane) { + auto word = Codeword(n, k, random()); + const size_t radius = (n - k) / 2; + const size_t errors = lane % 6 == 0 ? 0 : lane % 6 == 1 ? 1 + : lane % 6 == 2 ? radius : lane % 6 == 3 ? radius + 1 + : lane % 6 == 4 ? n : radius; + std::vector positions(n); + for (size_t i = 0; i < n; ++i) positions[i] = i; + std::shuffle(positions.begin(), positions.end(), random); + // Include parity-only full-radius candidates in every vector chunk. + if (lane % 6 == 5) std::sort(positions.rbegin(), positions.rend()); + for (size_t i = 0; i < errors; ++i) { + word[positions[i]] ^= static_cast(1 + random() % 255); + } + for (size_t pos = 0; pos < n; ++pos) packed[pos * lanes + lane] = word[pos]; + const auto before = word; + reference[lane] = CorrectCodeword(decoder, word); + if (reference[lane].status != CorrectionStatus::ok) EXPECT_EQ(word, before); + for (size_t pos = 0; pos < n; ++pos) expected[pos * lanes + lane] = word[pos]; + } + const auto before = packed; + ASSERT_EQ(CorrectCodewordBatch(decoder, shards, lanes, results, masks), + CorrectionStatus::ok); + ASSERT_EQ(packed, expected) << n << ':' << lanes; + for (size_t lane = 0; lane < lanes; ++lane) { + EXPECT_EQ(results[lane].status, reference[lane].status) << lane; + EXPECT_EQ(results[lane].error_count, reference[lane].error_count) << lane; + for (size_t pos = 0; pos < n; ++pos) { + const auto index = pos * lanes + lane; + EXPECT_EQ(masks[index], before[index] != packed[index]); + } + } + } + } +} + +TEST(WholeCodewordBatch, InvalidRangesAreUntouched) { + using detail::error_correction::CorrectCodewordBatch; + LCHDecoder decoder(6, 2); + std::vector packed(8 * 33, 7); + std::array shards{}; + for (size_t i = 0; i < 8; ++i) shards[i] = packed.data() + i * 33; + std::vector results(33); + std::vector masks(packed.size(), 42); + const auto before = packed; + shards[7] = shards[0]; + EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 33, results, masks), + CorrectionStatus::invalid_argument); + shards[7] = nullptr; + EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 33, results, masks), + CorrectionStatus::invalid_argument); + shards[7] = packed.data() + 7 * 33; + EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 33, results, packed), + CorrectionStatus::invalid_argument); + EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 32, results, masks), + CorrectionStatus::invalid_argument); + EXPECT_EQ(CorrectCodewordBatch(LCHDecoder(5, 3), shards, 33, results, masks), + CorrectionStatus::unsupported_dimensions); + EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 0, {}, {}), CorrectionStatus::ok); + EXPECT_EQ(packed, before); + EXPECT_EQ(masks, std::vector(packed.size(), 42)); + for (const auto& result : results) EXPECT_EQ(result.status, CorrectionStatus::invalid_argument); +} + +// Independent parity oracle: no decoder outcomes or scheduler bookkeeping. +bool AllComponentsValid(const std::vector& block, + size_t ns, size_t ks, size_t nw, size_t kw) { + bool valid = true; + for (bool strong : {true, false}) { + const size_t n = strong ? ns : nw, k = strong ? ks : kw; + LCHEncoder encoder(k, n - k); + std::vector parity(n - k); + std::vector data(k); + std::vector recovery(n - k); + std::vector workspace(encoder.WorkspaceSize(1)); + for (size_t i = 0; i < n - k; ++i) recovery[i] = &parity[i]; + for (size_t line = 0; line < (strong ? nw : ns); ++line) { + const auto index = [&](size_t pos) { + return strong ? pos * nw + line : line * nw + pos; + }; + for (size_t i = 0; i < k; ++i) data[i] = &block[index(i)]; + EXPECT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + for (size_t i = k; i < n; ++i) valid &= parity[i - k] == block[index(i)]; + } + } + return valid; +} + +TEST(ProductCode, InitialBatchChoicesMatchSingleOutputsAndAllCounters) { + std::mt19937 random(0x5b5c0224); + for (const auto dims : {std::array{4, 2, 8, 6}, + {32, 16, 64, 62}, {256, 224, 256, 254}}) { + const auto [ns, ks, nw, kw] = dims; + StrongWeakRSProductCode code(ns, ks, nw, kw); + for (size_t trial = 0; trial < 16; ++trial) { + std::vector input(code.BlockSize()); + for (auto& value : input) value = static_cast(random()); + ASSERT_EQ(code.Encode(input), Status::ok); + if (trial < 8) { + for (auto& value : input) { + for (unsigned bit = 0; bit < 8; ++bit) { + if (random() % 200 == 0) value ^= static_cast(1u << bit); + } + } + } else { + // Valid strong columns propose weak repairs into protected columns; + // overloaded columns exercise rejection, bit gates, and activation. + std::fill(input.begin(), input.end(), Element{0}); + for (size_t row = 0; row < ns; ++row) input[row * nw] = 1; + for (size_t row = 0; row <= (ns - ks) / 2; ++row) { + input[row * nw + 1] = trial % 2 ? 7 : 3; + if (row % 2) input[row * nw + 2] = 1; + } + if (trial % 3 == 0) input[0] ^= 2; + } + for (size_t cap : {2u, 3u, 4u, 5u, 6u, 16u}) { + auto reference = input; + const auto expected = detail::ProductCorrectionAccess::Correct(code, reference, cap, 0, false); + EXPECT_EQ(expected.all_zero_syndromes, AllComponentsValid(reference, ns, ks, nw, kw)); + for (unsigned batches : {0u, 1u, 2u}) { + auto actual = input; + const auto result = detail::ProductCorrectionAccess::Correct(code, actual, cap, batches); + ASSERT_EQ(actual, reference) << ns << ':' << trial << ':' << cap << ':' << batches; + EXPECT_EQ(result.termination, expected.termination); + EXPECT_EQ(result.all_zero_syndromes, expected.all_zero_syndromes); + EXPECT_EQ(result.directional_passes, expected.directional_passes); + EXPECT_EQ(result.strong_lines_visited, expected.strong_lines_visited); + EXPECT_EQ(result.weak_lines_visited, expected.weak_lines_visited); + EXPECT_EQ(result.changed_symbols, expected.changed_symbols); + } + } + } + } +} + +TEST(ProductCode, TrackedValidityMatchesIndependentParityAcrossCapsAndCancellations) { + std::mt19937 random(0xc1ea0224); + for (const auto dims : {std::array{4, 2, 8, 6}, + {8, 4, 4, 2}, {32, 28, 32, 30}}) { + const auto [ns, ks, nw, kw] = dims; + StrongWeakRSProductCode code(ns, ks, nw, kw); + for (size_t trial = 0; trial < 128; ++trial) { + std::vector input(code.BlockSize()); + // Include undetected valid words, equal-magnitude cancellations, + // parity damage, dense failures, and both weak rejection gates. + if (trial % 4 == 0) { + for (auto& value : input) value = static_cast(random()); + ASSERT_EQ(code.Encode(input, Backend::scalar), Status::ok); + } + const size_t errors = trial % (ns * 2); + for (size_t i = 0; i < errors; ++i) { + input[random() % input.size()] ^= trial % 3 == 0 ? Element{1} + : trial % 3 == 1 ? Element{7} : static_cast(1 + random() % 255); + } + for (size_t cap : {2u, 3u, 4u, 5u, 6u, 7u, 8u, 16u}) { + auto reference = input; + const auto expected = detail::ProductCorrectionAccess::Correct(code, reference, cap, 0, false); + const bool valid = AllComponentsValid(reference, ns, ks, nw, kw); + EXPECT_EQ(expected.all_zero_syndromes, valid); + for (unsigned batches : {0u, 1u, 2u, 3u}) { + auto actual = input; + const auto result = batches == 3 ? code.Correct(actual, cap) + : detail::ProductCorrectionAccess::Correct(code, actual, cap, batches); + ASSERT_EQ(actual, reference) << ns << ':' << trial << ':' << cap << ':' << batches; + EXPECT_EQ(result.all_zero_syndromes, valid); + EXPECT_EQ(result.termination, expected.termination); + EXPECT_EQ(result.directional_passes, expected.directional_passes); + EXPECT_EQ(result.strong_lines_visited, expected.strong_lines_visited); + EXPECT_EQ(result.weak_lines_visited, expected.weak_lines_visited); + EXPECT_EQ(result.changed_symbols, expected.changed_symbols); + } + } + } + } +} + +} // namespace From a20891299dfc694ec91543dd477ddf21a9c05a89 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Wed, 9 Sep 2026 09:34:16 +0300 Subject: [PATCH 2/7] Add native RS product Monte Carlo experiments --- .github/workflows/format.yml | 21 + CMakeLists.txt | 62 +- CMakePresets.json | 24 + benchmarks/reference/xdrs_benchmarks.cc | 9 +- .../strong_weak_rs_product_code_benchmarks.cc | 74 +- include/reed_solomon/error_correction.h | 5 +- .../strong_weak_rs_product_code.h | 80 +- scripts/format.sh | 39 + scripts/install-lch-rs.sh | 26 + scripts/install-monte-carlo.sh | 26 + scripts/plot_product_monte_carlo.py | 383 ++++++++ src/reed_solomon/error_correction/batched.cc | 51 +- src/reed_solomon/error_correction/internal.h | 2 +- src/reed_solomon/error_correction/scalar.cc | 28 +- src/reed_solomon/product_code_internal.h | 38 +- .../strong_weak_rs_product_code.cc | 200 ++-- tests/plot_product_monte_carlo_test.py | 257 +++++ tests/product_code_tests.cc | 504 ++++++++-- tests/product_monte_carlo_data_tests.cc | 42 + tests/product_monte_carlo_legacy_test.py | 662 +++++++++++++ tests/product_monte_carlo_test.py | 279 ++++++ tests/reference/product_monte_carlo.py | 675 +++++++++++++ tools/lch_rs.cc | 48 +- tools/product_monte_carlo.cc | 912 ++++++++++++++++++ tools/product_monte_carlo_data.h | 331 +++++++ tools/product_monte_carlo_native.cc | 344 +++++++ tools/product_monte_carlo_trials.h | 25 + 27 files changed, 4863 insertions(+), 284 deletions(-) create mode 100644 .github/workflows/format.yml create mode 100644 scripts/format.sh create mode 100644 scripts/install-lch-rs.sh create mode 100644 scripts/install-monte-carlo.sh create mode 100644 scripts/plot_product_monte_carlo.py create mode 100644 tests/plot_product_monte_carlo_test.py create mode 100644 tests/product_monte_carlo_data_tests.cc create mode 100644 tests/product_monte_carlo_legacy_test.py create mode 100644 tests/product_monte_carlo_test.py create mode 100644 tests/reference/product_monte_carlo.py create mode 100644 tools/product_monte_carlo.cc create mode 100644 tools/product_monte_carlo_data.h create mode 100644 tools/product_monte_carlo_native.cc create mode 100644 tools/product_monte_carlo_trials.h diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000..70bcb30 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,21 @@ +name: C/C++ Formatting + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + clang-format: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install pinned formatter + run: python -m pip install clang-format==18.1.3 + - name: Check tracked owned C/C++ formatting + env: + CLANG_FORMAT: clang-format + run: bash scripts/format.sh --check diff --git a/CMakeLists.txt b/CMakeLists.txt index 1802f3a..9c45841 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ endif() option(GF256_BUILD_TESTS "Build unit tests" ON) option(GF256_BUILD_TOOLS "Build CLI tools" ON) +option(GF256_BUILD_MONTE_CARLO "Build the product Monte Carlo CLI (also in test profiles)" ON) option(GF256_BUILD_BENCHMARKS "Build benchmarks" ON) option(GF256_BUILD_REFERENCE_BENCHMARKS "Build external RS baselines" OFF) option(GF256_ENABLE_NATIVE_ISA "Compile owned sources for the build host" OFF) @@ -65,6 +66,7 @@ add_library(gf256_core STATIC src/reed_solomon/strong_weak_rs_product_code.cc ) target_compile_features(gf256_core PUBLIC cxx_std_20) +set_target_properties(gf256_core PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(gf256_core PUBLIC ${PROJECT_SOURCE_DIR}/include PRIVATE ${PROJECT_SOURCE_DIR}/src) @@ -86,6 +88,29 @@ if(GF256_ENABLE_CODEWORD_CANTOR_AFFINE_EXPERIMENT) GF256_ENABLE_CODEWORD_CANTOR_AFFINE_EXPERIMENT=1) endif() +if(GF256_BUILD_MONTE_CARLO) + find_package(Threads REQUIRED) + find_package(OpenSSL 3 REQUIRED COMPONENTS Crypto) + FetchContent_Declare(jsoncons + GIT_REPOSITORY https://github.com/danielaparker/jsoncons.git + GIT_TAG v0.177.0 + GIT_SHALLOW TRUE) + set(JSONCONS_BUILD_TESTS OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(jsoncons) + add_library(product_monte_carlo_trials OBJECT tools/product_monte_carlo_native.cc) + target_link_libraries(product_monte_carlo_trials PRIVATE gf256_core) + set_target_properties(product_monte_carlo_trials PROPERTIES POSITION_INDEPENDENT_CODE ON) + if(GF256_ENABLE_NATIVE_ISA AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(product_monte_carlo_trials PRIVATE -march=native) + endif() + add_executable(rs-product-monte-carlo tools/product_monte_carlo.cc) + target_link_libraries(rs-product-monte-carlo PRIVATE product_monte_carlo_trials + gf256_core Threads::Threads OpenSSL::Crypto jsoncons) + include(GNUInstallDirs) + install(TARGETS rs-product-monte-carlo + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT product-monte-carlo) +endif() + if(GF256_BUILD_TOOLS) set(XXHASH_BUILD_XXHSUM OFF CACHE BOOL "" FORCE) set(INDICATORS_BUILD_TESTS OFF CACHE BOOL "" FORCE) @@ -114,7 +139,7 @@ if(GF256_BUILD_TOOLS) target_link_libraries(lch-rs PRIVATE gf256_core xxhash indicators::indicators Threads::Threads) include(GNUInstallDirs) - install(TARGETS lch-rs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(TARGETS lch-rs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT lch-rs) endif() if(GF256_ENABLE_GFNI512_RADIX8_EXPERIMENT) @@ -161,6 +186,41 @@ if(GF256_BUILD_TESTS) GF256_ENABLE_CODEWORD_CANTOR_AFFINE_EXPERIMENT=1) endif() add_test(NAME LCHRSUnittests COMMAND lch_rs_unittests) + if(GF256_BUILD_MONTE_CARLO) + find_package(Python3 3.9 REQUIRED COMPONENTS Interpreter) + add_library(product_monte_carlo_native SHARED $) + set_target_properties(product_monte_carlo_native PROPERTIES PREFIX "") + target_link_libraries(product_monte_carlo_native PRIVATE gf256_core Threads::Threads) + configure_file(tests/reference/product_monte_carlo.py product_monte_carlo_reference.py COPYONLY) + add_executable(product_monte_carlo_data_tests tests/product_monte_carlo_data_tests.cc) + target_include_directories(product_monte_carlo_data_tests PRIVATE tools) + target_link_libraries(product_monte_carlo_data_tests PRIVATE jsoncons gtest_main) + target_compile_features(product_monte_carlo_data_tests PRIVATE cxx_std_20) + add_test(NAME ProductMonteCarloData COMMAND product_monte_carlo_data_tests) + add_executable(product_monte_carlo_fault_cli tools/product_monte_carlo.cc) + target_compile_definitions(product_monte_carlo_fault_cli PRIVATE GF256_MC_TEST_HOOKS=1) + target_link_libraries(product_monte_carlo_fault_cli PRIVATE product_monte_carlo_trials + gf256_core Threads::Threads OpenSSL::Crypto jsoncons) + add_test(NAME ProductMonteCarloCli + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tests/product_monte_carlo_test.py + ${PROJECT_BINARY_DIR}/rs-product-monte-carlo) + set_tests_properties(ProductMonteCarloCli PROPERTIES TIMEOUT 180) + add_test(NAME ProductMonteCarloLegacy + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tests/product_monte_carlo_legacy_test.py + ${PROJECT_BINARY_DIR}/product_monte_carlo_reference.py) + set_tests_properties(ProductMonteCarloLegacy PROPERTIES TIMEOUT 180) + add_test(NAME ProductMonteCarloPlot + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tests/plot_product_monte_carlo_test.py) + if(CMAKE_CXX_FLAGS MATCHES "fsanitize=address") + # ctypes loads an instrumented DSO into an otherwise uninstrumented Python. + execute_process(COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libasan.so + OUTPUT_VARIABLE GF256_ASAN_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE) + set_tests_properties(ProductMonteCarloLegacy PROPERTIES + ENVIRONMENT "LD_PRELOAD=${GF256_ASAN_RUNTIME}") + set_tests_properties(ProductMonteCarloCli PROPERTIES + ENVIRONMENT "MC_REFERENCE_LD_PRELOAD=${GF256_ASAN_RUNTIME}") + endif() + endif() if(GF256_BUILD_TOOLS) add_test(NAME LCHRSCli diff --git a/CMakePresets.json b/CMakePresets.json index 485e146..58a84f3 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -6,6 +6,22 @@ "patch": 0 }, "configurePresets": [ + { + "name": "experimental", + "displayName": "Experimental Tools", + "description": "Native Release build of the RS product Monte Carlo tool", + "binaryDir": "${sourceDir}/build/experimental-preset", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "GF256_BUILD_MONTE_CARLO": "ON", + "GF256_BUILD_TOOLS": "OFF", + "GF256_BUILD_TESTS": "OFF", + "GF256_BUILD_BENCHMARKS": "OFF", + "GF256_BUILD_REFERENCE_BENCHMARKS": "OFF", + "GF256_ENABLE_NATIVE_ISA": "ON" + } + }, { "name": "release", "displayName": "Release", @@ -61,6 +77,14 @@ } ], "buildPresets": [ + { + "name": "experimental", + "displayName": "Build Experimental Tools", + "configurePreset": "experimental", + "targets": [ + "rs-product-monte-carlo" + ] + }, { "name": "release", "displayName": "Build Release", diff --git a/benchmarks/reference/xdrs_benchmarks.cc b/benchmarks/reference/xdrs_benchmarks.cc index ceff514..8f5c05e 100644 --- a/benchmarks/reference/xdrs_benchmarks.cc +++ b/benchmarks/reference/xdrs_benchmarks.cc @@ -71,10 +71,11 @@ void InitializeXDRS(XDRSTables& tables, unsigned bytes, unsigned k) { ::function::init_dec(); } -bool VerifyXDRSRecovery( - unsigned k, unsigned bytes, bool low_rate, - const std::vector>& data, - const std::vector>& recovery) { +bool VerifyXDRSRecovery(unsigned k, + unsigned bytes, + bool low_rate, + const std::vector>& data, + const std::vector>& recovery) { const unsigned recovery_count = Size - k; auto data_pointers = ConstPointers(data); auto recovery_pointers = ConstPointers(recovery); diff --git a/benchmarks/strong_weak_rs_product_code_benchmarks.cc b/benchmarks/strong_weak_rs_product_code_benchmarks.cc index 85f0993..4ce8fd7 100644 --- a/benchmarks/strong_weak_rs_product_code_benchmarks.cc +++ b/benchmarks/strong_weak_rs_product_code_benchmarks.cc @@ -5,12 +5,13 @@ #include #include "benchmark/benchmark.h" -#include "reed_solomon/strong_weak_rs_product_code.h" #include "reed_solomon/product_code_internal.h" +#include "reed_solomon/strong_weak_rs_product_code.h" namespace { -void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = -1) { +void BenchmarkProductCorrectionBSC(benchmark::State& state, + int batch_passes = -1) { using gf2p8::Element; using gf2p8::rs::ProductCorrectionResult; using gf2p8::rs::ProductTermination; @@ -30,8 +31,8 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - std::mt19937 messages(kSeed); std::mt19937 channel(kSeed ^ 0x9e3779b9U); - std::vector> original( - kCorpusCount, std::vector(kBlockBytes)); + std::vector> original(kCorpusCount, + std::vector(kBlockBytes)); auto corrupted = original; auto work = original; std::vector results(kCorpusCount); @@ -77,6 +78,8 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - uint64_t message_failures = 0, block_failures = 0, validity_failures = 0; uint64_t valid_wrong_blocks = 0, pass_limits = 0, passes = 0; uint64_t strong_lines = 0, weak_lines = 0; + uint64_t accepted_bits = 0, accepted_symbols = 0; + uint64_t strong_bits = 0, strong_symbols = 0, weak_bits = 0, weak_symbols = 0; size_t max_passes = 0; // Validate this exact channel corpus against the retained single path outside // timing, including scheduling outcomes, not just final message recovery. @@ -85,15 +88,22 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - const auto expected = gf2p8::rs::detail::ProductCorrectionAccess::Correct( code, reference, kPassLimit, 0, false); work[sample] = corrupted[sample]; - const auto actual = batch_passes < 0 ? code.Correct(work[sample], kPassLimit) - : gf2p8::rs::detail::ProductCorrectionAccess::Correct( - code, work[sample], kPassLimit, batch_passes); - if (work[sample] != reference || actual.termination != expected.termination || + const auto actual = + batch_passes < 0 ? code.Correct(work[sample], kPassLimit) + : gf2p8::rs::detail::ProductCorrectionAccess::Correct( + code, work[sample], kPassLimit, batch_passes); + if (work[sample] != reference || + actual.termination != expected.termination || actual.all_zero_syndromes != expected.all_zero_syndromes || actual.directional_passes != expected.directional_passes || actual.strong_lines_visited != expected.strong_lines_visited || actual.weak_lines_visited != expected.weak_lines_visited || - actual.changed_symbols != expected.changed_symbols) { + actual.changed_symbols != expected.changed_symbols || + actual.changed_bits != expected.changed_bits || + actual.strong_changed_bits != expected.strong_changed_bits || + actual.weak_changed_bits != expected.weak_changed_bits || + actual.strong_changed_symbols != expected.strong_changed_symbols || + actual.weak_changed_symbols != expected.weak_changed_symbols) { state.SkipWithError("batch/single corpus differential mismatch"); return; } @@ -106,9 +116,11 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - } state.ResumeTiming(); for (size_t sample = 0; sample < kCorpusCount; ++sample) { - results[sample] = batch_passes < 0 ? code.Correct(work[sample], kPassLimit) - : gf2p8::rs::detail::ProductCorrectionAccess::Correct( - code, work[sample], kPassLimit, batch_passes); + results[sample] = + batch_passes < 0 + ? code.Correct(work[sample], kPassLimit) + : gf2p8::rs::detail::ProductCorrectionAccess::Correct( + code, work[sample], kPassLimit, batch_passes); benchmark::DoNotOptimize(results[sample]); benchmark::ClobberMemory(); } @@ -121,8 +133,8 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - } uint64_t data_bits = 0, block_bits = 0; for (size_t pos = 0; pos < kBlockBytes; ++pos) { - const auto bits = std::popcount(static_cast( - work[sample][pos] ^ original[sample][pos])); + const auto bits = std::popcount( + static_cast(work[sample][pos] ^ original[sample][pos])); block_bits += bits; if (pos / kN < kStrongK && pos % kN < kWeakK) { data_bits += bits; @@ -139,13 +151,23 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - max_passes = std::max(max_passes, result.directional_passes); strong_lines += result.strong_lines_visited; weak_lines += result.weak_lines_visited; + accepted_bits += result.changed_bits; + accepted_symbols += result.changed_symbols; + strong_bits += result.strong_changed_bits; + strong_symbols += result.strong_changed_symbols; + weak_bits += result.weak_changed_bits; + weak_symbols += result.weak_changed_symbols; } state.ResumeTiming(); - if (state.skipped()) break; + if (state.skipped()) { + break; + } } const double sweeps = static_cast(state.iterations()); - if (sweeps == 0 || state.skipped()) return; + if (sweeps == 0 || state.skipped()) { + return; + } const double blocks = sweeps * kCorpusCount; state.counters["corpus_blocks"] = kCorpusCount; state.counters["timed_blocks"] = blocks; @@ -155,9 +177,11 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - state.counters["channel_codeword_BER"] = static_cast(channel_bits) / (kCorpusCount * kBlockBytes * 8); state.counters["channel_flipped_bits"] = static_cast(channel_bits); - state.counters["channel_corrupted_bytes"] = static_cast(channel_symbols); + state.counters["channel_corrupted_bytes"] = + static_cast(channel_symbols); state.counters["corpus_data_residual_bits"] = data_residual_bits / sweeps; - state.counters["corpus_codeword_residual_bits"] = codeword_residual_bits / sweeps; + state.counters["corpus_codeword_residual_bits"] = + codeword_residual_bits / sweeps; state.counters["data_residual_BER"] = data_residual_bits / (blocks * kInformationBytes * 8); state.counters["codeword_residual_BER"] = @@ -174,11 +198,19 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, int batch_passes = - state.counters["max_directional_passes"] = static_cast(max_passes); state.counters["mean_strong_lines"] = strong_lines / blocks; state.counters["mean_weak_lines"] = weak_lines / blocks; + state.counters["mean_accepted_bit_changes"] = accepted_bits / blocks; + state.counters["mean_accepted_byte_changes"] = accepted_symbols / blocks; + state.counters["mean_strong_accepted_bit_changes"] = strong_bits / blocks; + state.counters["mean_strong_accepted_byte_changes"] = strong_symbols / blocks; + state.counters["mean_weak_accepted_bit_changes"] = weak_bits / blocks; + state.counters["mean_weak_accepted_byte_changes"] = weak_symbols / blocks; state.counters["pass_limit"] = kPassLimit; state.SetItemsProcessed(state.iterations() * kCorpusCount); - state.SetBytesProcessed(state.iterations() * kCorpusCount * kInformationBytes); - state.SetLabel("64 blocks/iteration; information bytes=224*254; " - "Correct includes exact final validity; tuned backend"); + state.SetBytesProcessed(state.iterations() * kCorpusCount * + kInformationBytes); + state.SetLabel( + "64 blocks/iteration; information bytes=224*254; " + "Correct includes exact final validity; tuned backend"); } const auto* kProductCorrectionBSC = benchmark::RegisterBenchmark( diff --git a/include/reed_solomon/error_correction.h b/include/reed_solomon/error_correction.h index fb6fa55..d2d026c 100644 --- a/include/reed_solomon/error_correction.h +++ b/include/reed_solomon/error_correction.h @@ -23,13 +23,14 @@ struct CorrectionResult { * @brief Transactionally repairs data AND parity of one Cantor RS codeword. * @param decoder Decoder defining K data and R recovery symbols. * @param codeword Exactly K+R mutable symbols in [data][recovery] order. - * @return Status and corrected symbol count; zero count on an already valid word. + * @return Status and corrected symbol count; zero count on an already valid + * word. * @details Requires unshortened power-of-two N=K+R <=256 and power-of-two * R<=K. Corrects up to floor(R/2) unknown symbol errors, with no erasures. * On any non-ok status every input byte is unchanged. Success means the result * is a codeword within that radius, NOT necessarily the transmitted codeword. */ CorrectionResult CorrectCodeword(const LCHDecoder& decoder, - std::span codeword); + std::span codeword); } // namespace gf2p8::rs diff --git a/include/reed_solomon/strong_weak_rs_product_code.h b/include/reed_solomon/strong_weak_rs_product_code.h index 4a2bc84..f7dd046 100644 --- a/include/reed_solomon/strong_weak_rs_product_code.h +++ b/include/reed_solomon/strong_weak_rs_product_code.h @@ -5,12 +5,28 @@ namespace gf2p8::rs { -namespace detail { struct ProductCorrectionAccess; } +namespace detail { +struct ProductCorrectionAccess; +} -/** @brief Reason iterative product correction stopped, independent of validity. */ +/** @brief Reason iterative product correction stopped, independent of validity. + */ enum class ProductTermination { invalid_argument, no_change, pass_limit }; -/** @brief Product correction outcome; validity does not prove original content. */ +/** @brief Per-call product correction limits and independent weak acceptance + * gates. */ +struct ProductDecodeOptions { + /** @brief Cap counting each direction separately; must be at least two. */ + size_t max_directional_passes = 16; + /** @brief Reject weak repairs into columns protected by strong BDD success. + */ + bool use_anchors = true; + /** @brief Require weak repair deltas to have at most two set bits. */ + bool use_binary_image = true; +}; + +/** @brief Product correction outcome; validity does not prove original content. + */ struct ProductCorrectionResult { ProductTermination termination = ProductTermination::invalid_argument; bool all_zero_syndromes = false; @@ -19,6 +35,12 @@ struct ProductCorrectionResult { size_t weak_lines_visited = 0; /** @brief Accepted symbol writes, counting repeated repairs separately. */ size_t changed_symbols = 0; + /** @brief Accepted bit toggles, including repeated committed repairs. */ + size_t changed_bits = 0; + /** @brief Accepted strong-direction symbol writes and bit toggles. */ + size_t strong_changed_symbols = 0, strong_changed_bits = 0; + /** @brief Accepted weak-direction symbol writes and bit toggles. */ + size_t weak_changed_symbols = 0, weak_changed_bits = 0; }; /** @@ -38,18 +60,23 @@ class StrongWeakRSProductCode { * @param weak_n Number of columns. * @param weak_k Number of systematic columns. */ - StrongWeakRSProductCode(size_t strong_n = 256, size_t strong_k = 224, - size_t weak_n = 256, size_t weak_k = 254); + StrongWeakRSProductCode(size_t strong_n = 256, + size_t strong_k = 224, + size_t weak_n = 256, + size_t weak_k = 254); - /** @brief Reports supported dimensions. @return Whether operations are valid. */ + /** @brief Reports supported dimensions. @return Whether operations are valid. + */ bool Valid() const; - /** @brief Returns required row-major block size. @return Bytes, or zero if invalid. */ + /** @brief Returns required row-major block size. @return Bytes, or zero if + * invalid. */ size_t BlockSize() const; /** * @brief Fills all product parity, preserving the systematic rectangle. * @param block Exactly BlockSize() symbols, with data already in place. - * @param backend Owned encoder backend, scalar available for reference checks. + * @param backend Owned encoder backend, scalar available for reference + * checks. * @return Encoding status; on failure block is unchanged. */ lch::Status Encode(std::span block, @@ -65,23 +92,38 @@ class StrongWeakRSProductCode { * accepted byte changes in the preceding pass. Stops on a no-change pass * (including the initial weak pass), or the cap; no-change takes precedence. * Strong success, including zero syndromes, protects that column; failure - * leaves its bytes unchanged and unprotects it. Unvisited protection persists. - * Weak candidates commit only for one changed symbol with popcount(old XOR - * new)<=2 in an unprotected column; rejected rows are unchanged. Protection - * changes alone never activate lines. Accepted repairs are retained at exit; - * the entire iterative operation is not transactional. Invalid arguments - * leave the block untouched. Validity is checked separately at exit and does - * not count as a directional pass or guarantee the original message. + * leaves its bytes unchanged and unprotects it. Unvisited protection + * persists. Weak candidates commit only for one changed symbol with + * popcount(old XOR new)<=2 in an unprotected column; rejected rows are + * unchanged. Protection changes alone never activate lines. Accepted repairs + * are retained at exit; the entire iterative operation is not transactional. + * Invalid arguments leave the block untouched. Validity is checked separately + * at exit and does not count as a directional pass or guarantee the original + * message. + */ + ProductCorrectionResult Correct(std::span block, + size_t max_directional_passes = 16) const; + + /** + * @brief Corrects with independent per-call weak acceptance gates. + * @param block Exactly BlockSize() mutable symbols. + * @param options Pass cap and optional anchor and binary-image gates. + * @return Termination, pass/work counts, and final all-component validity. + * @details Uses the same scheduling and stopping rules as the cap overload. + * Weak BDD always requires exactly one changed symbol. Disabling anchors + * bypasses only target-column protection; disabling binary image bypasses + * only the two-bit delta limit. Every accepted write still invalidates + * intersecting cached validity and activates the next direction. */ ProductCorrectionResult Correct(std::span block, - size_t max_directional_passes = 16) const; + ProductDecodeOptions options) const; private: friend struct detail::ProductCorrectionAccess; ProductCorrectionResult CorrectImpl(std::span block, - size_t max_directional_passes, - unsigned batch_passes, - bool tracked_validation = true) const; + ProductDecodeOptions options, + unsigned batch_passes, + bool tracked_validation = true) const; size_t strong_n_, strong_k_, weak_n_, weak_k_; bool valid_; LCHEncoder strong_encoder_, weak_encoder_; diff --git a/scripts/format.sh b/scripts/format.sh new file mode 100644 index 0000000..feba54c --- /dev/null +++ b/scripts/format.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "${1:---check}" in + --check) flags=(--dry-run --Werror) ;; + --write) flags=(-i) ;; + *) printf 'Usage: bash scripts/format.sh [--check|--write]\n' >&2; exit 2 ;; +esac +if [[ $# -gt 1 ]]; then + printf 'Expected at most one argument\n' >&2 + exit 2 +fi + +root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" +formatter="${CLANG_FORMAT:-clang-format-18}" +version="$("$formatter" --version)" +if [[ ! "$version" =~ (^|[[:space:]])version[[:space:]]18\.1\.3([[:space:]]|$) ]]; then + printf 'Expected clang-format 18.1.3, got: %s\n' "$version" >&2 + exit 2 +fi + +# Git's index defines ownership: never traverse submodules or untracked assets. +mapfile -d '' -t tracked < <(git -C "$root" ls-files -z) +files=() +for file in "${tracked[@]}"; do + case "$file" in + third_party/*|agentic/*|assets/*|reference/*|lessons/*) continue ;; + esac + case "$file" in + *.c|*.cc|*.cpp|*.cxx|*.h|*.hh|*.hpp|*.hxx|*.inc|*.inl) + [[ ! -f "$root/$file" || -L "$root/$file" ]] || files+=("$root/$file") ;; + esac +done +if [[ ${#files[@]} -eq 0 ]]; then + printf 'No tracked owned C/C++ files found\n' >&2 + exit 2 +fi +printf '%s: %s (%d tracked owned C/C++ files)\n' "${1:---check}" "$version" "${#files[@]}" +"$formatter" --style="file:$root/.clang-format" "${flags[@]}" "${files[@]}" diff --git a/scripts/install-lch-rs.sh b/scripts/install-lch-rs.sh new file mode 100644 index 0000000..0cf4744 --- /dev/null +++ b/scripts/install-lch-rs.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${1:-} == --help || ${1:-} == -h ]]; then + printf 'Usage: %s [PREFIX]\nBuild native Release lch-rs and install it (default: ~/.local).\nRequires CMake and a C++20 compiler; dependency downloads may require network access. No sudo is invoked.\n' "$0" + exit 0 +fi +if (( $# > 1 )) || [[ ${1:-} == -* ]]; then + printf 'Usage: %s [PREFIX]\n' "$0" >&2 + exit 2 +fi + +prefix=${1:-"$HOME/.local"} +if [[ -z $prefix ]]; then + printf 'Installation prefix must not be empty.\n' >&2 + exit 2 +fi +# Resolve relative prefixes before changing to the repository root. +[[ $prefix == /* ]] || prefix="$PWD/$prefix" +root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +cd -- "$root" +cmake --preset cli -DGF256_BUILD_MONTE_CARLO=OFF +cmake --build --preset cli --parallel +cmake --install "$root/build/cli-preset" \ + --prefix "$prefix" --component lch-rs +printf 'Installed lch-rs under %s. Ensure its bin directory is on PATH.\n' "$prefix" diff --git a/scripts/install-monte-carlo.sh b/scripts/install-monte-carlo.sh new file mode 100644 index 0000000..5d21b28 --- /dev/null +++ b/scripts/install-monte-carlo.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${1:-} == --help || ${1:-} == -h ]]; then + printf 'Usage: %s [PREFIX]\nBuild native Release rs-product-monte-carlo and install it (default: ~/.local).\nRequires CMake, a C++20 compiler, OpenSSL 3 development files, and Git/network for pinned jsoncons headers. No Python runtime or sudo.\n' "$0" + exit 0 +fi +if (( $# > 1 )) || [[ ${1:-} == -* ]]; then + printf 'Usage: %s [PREFIX]\n' "$0" >&2 + exit 2 +fi + +prefix=${1:-"$HOME/.local"} +if [[ -z $prefix ]]; then + printf 'Installation prefix must not be empty.\n' >&2 + exit 2 +fi +# Resolve relative prefixes before changing to the repository root. +[[ $prefix == /* ]] || prefix="$PWD/$prefix" +root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +cd -- "$root" +cmake --preset experimental +cmake --build --preset experimental --parallel +cmake --install "$root/build/experimental-preset" \ + --prefix "$prefix" --component product-monte-carlo +printf 'Installed rs-product-monte-carlo under %s. Ensure its bin directory is on PATH.\n' "$prefix" diff --git a/scripts/plot_product_monte_carlo.py b/scripts/plot_product_monte_carlo.py new file mode 100644 index 0000000..163a1fa --- /dev/null +++ b/scripts/plot_product_monte_carlo.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +"""Plot sampled-stratum BER contributions, not extrapolated decoder BER.""" + +import argparse +import csv +import hashlib +import json +import math +from pathlib import Path +import sys + +N = 524288 +DENOMINATORS = {"information": 455168, "full": N} +METRICS = {"information": "residual information bits", "full": "residual full block bits"} +CODE = "RS256,224 x RS256,254 Cantor systematic row major" +RANDOM = "splitmix64 domain seeds; mt19937_64; rejection modulo; Floyd complement v1" +RANDOM_FY = "splitmix64 domain seeds; mt19937_64; rejection modulo; persistent Fisher-Yates complement v1; replay saved flips" +NEG_INF = -math.inf + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def natural(value, maximum=None): + return type(value) is int and value >= 0 and (maximum is None or value <= maximum) + + +def strict_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_number(value): + raise ValueError(f"noninteger JSON number: {value}") + + +def read_json(path): + return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=strict_object, + parse_float=reject_number, parse_constant=reject_number) + + +def identity(metadata): + text = json.dumps(metadata, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + return hashlib.sha256(text.encode("ascii")).hexdigest() + + +def minimal_rows(summary, settings): + """Validate additive schema-2 counters and project the two BER numerators.""" + rows = summary["by flipped bit count"] + require(type(rows) is list, "invalid per-k rows") + pooled, totals = {}, None + for row in rows + [summary["overall"]]: + overall = row is summary["overall"] + require(type(row) is dict and set(row) == ({"statistics"} if overall else + {"flipped bit count", "statistics"}), "invalid row fields") + stats = row["statistics"] + require(type(stats) is dict and set(stats) == {"completed blocks", "total iterations", + "information bits", "full-codeword bits"}, "invalid minimal statistics") + trials, iterations = stats["completed blocks"], stats["total iterations"] + require(natural(trials) and (overall or trials > 0), "invalid completed blocks") + require(natural(iterations) and trials * 2 <= iterations <= trials * settings[ + "maximum directional passes"], "invalid total iterations") + flat = {"completed blocks": trials, "total iterations": iterations} + residuals = {} + for metric, name in (("information", "information bits"), ("full", "full-codeword bits")): + bits = stats[name] + require(type(bits) is dict and set(bits) == {"total bits", "raw corrupted bits", + "post decoding corrupted bits"}, "invalid bit fields") + total = trials * DENOMINATORS[metric] + require(bits["total bits"] == total and natural(bits["total bits"]), "invalid total bits") + for field in ("raw corrupted bits", "post decoding corrupted bits"): + require(natural(bits[field], total), "invalid corrupted bits") + flat.update({name + ":" + field: value for field, value in bits.items()}) + residuals[metric] = bits["post decoding corrupted bits"] + for field in ("raw corrupted bits", "post decoding corrupted bits"): + difference = stats["full-codeword bits"][field] - stats["information bits"][field] + require(0 <= difference <= trials * (N - DENOMINATORS["information"]), + "inconsistent full/information bits") + if overall: + require(flat == totals if rows else all(v == 0 for v in flat.values()), + "overall/per-k reconciliation failed") + continue + k = row["flipped bit count"] + require(natural(k, N) and settings["minimum flipped bits"] <= k <= settings[ + "maximum flipped bits"], "invalid flipped bit count") + require(k not in pooled, f"duplicate k: {k}") + require(stats["full-codeword bits"]["raw corrupted bits"] == trials*k, + "initial count is not exact k") + if totals is None: + totals = dict.fromkeys(flat, 0) + for key, value in flat.items(): + totals[key] += value + pooled[k] = {"trials": trials, **residuals} + return pooled + + +def load_report(path): + """Validate a summary snapshot without loading the native library or journal.""" + path = Path(path) + if path.is_dir(): + path /= "summary.json" + try: + metadata = read_json(path.parent / "metadata.json") + summary = read_json(path) + require(type(metadata) is dict and set(metadata) - {"codeword"} == { + "schema revision", "created at", "settings", "code", "random algorithm"}, + "incompatible metadata fields") + codeword = metadata.get("codeword", "random") + require(codeword in ("zero", "random"), "incompatible codeword convention") + require(type(metadata["schema revision"]) is int and metadata["schema revision"] in (1, 2) + and metadata["code"] == CODE and metadata["random algorithm"] in (RANDOM, RANDOM_FY), + "incompatible schema/code/random algorithm") + require(isinstance(metadata["created at"], str), "invalid created at") + settings = metadata["settings"] + integer_settings = {"root seed", "batch size", "batches", "threads", + "minimum flipped bits", "maximum flipped bits", + "maximum directional passes", "checkpoint trials", + "report seconds", "fsync seconds"} + require(type(settings) is dict and set(settings) == integer_settings | { + "anchors", "binary image"}, "incompatible settings") + for name in integer_settings: + require(natural(settings[name], (1 << 64) - 1), f"invalid {name}") + for name in ("anchors", "binary image"): + require(type(settings[name]) is bool, f"invalid {name}") + require(0 <= settings["minimum flipped bits"] <= settings["maximum flipped bits"] <= N, + "invalid sampled k range") + for name, low, high in (("batch size", 1, (1 << 64) - 1), ("threads", 1, 1024), + ("maximum directional passes", 2, 1000000), + ("checkpoint trials", 1, 4096), ("report seconds", 1, 86400), + ("fsync seconds", 1, 86400)): + require(low <= settings[name] <= high, f"invalid {name}") + require(type(summary) is dict and set(summary) == { + "schema revision", "run identity", "overall", "by flipped bit count"}, + "incompatible summary fields") + require(type(summary["schema revision"]) is int and summary["schema revision"] == metadata["schema revision"] + and summary["run identity"] == identity(metadata), "identity/schema mismatch") + if summary["schema revision"] == 2: + pooled = minimal_rows(summary, settings) + config = (settings["maximum directional passes"], settings["anchors"], settings["binary image"]) + return summary["run identity"], settings["root seed"], config, pooled, codeword + rows = summary["by flipped bit count"] + require(type(rows) is list, "invalid per-k rows") + pooled = {} + totals = {} + count = 0 + for row in rows + [summary["overall"]]: + overall = row is summary["overall"] + require(type(row) is dict and set(row) == ({"trial count", "statistics"} if overall + else {"flipped bit count", "trial count", "statistics"}), "invalid row fields") + trials = row["trial count"] + require(natural(trials) and (overall or trials > 0), "invalid trial count") + stats = row["statistics"] + require(type(stats) is dict and all(m in stats for m in METRICS.values()), + "missing residual statistics") + for name, moment in stats.items(): + require(type(moment) is dict and set(moment) == {"sum", "squared sum"}, + f"invalid moment fields: {name}") + total, square = moment["sum"], moment["squared sum"] + require(natural(total) and natural(square) and total <= square + and total * total <= trials * square, f"invalid moments: {name}") + require(trials > 0 or total == square == 0, "nonzero empty statistics") + if name in METRICS.values(): + d = DENOMINATORS[next(m for m in METRICS if METRICS[m] == name)] + require(total <= trials * d and square <= d * total, + f"residual moments exceed bit count: {name}") + require(stats[METRICS["information"]]["sum"] <= stats[METRICS["full"]]["sum"], + "information residual exceeds full residual") + if overall: + require(trials == count and (stats == totals if rows else all( + item == {"sum": 0, "squared sum": 0} for item in stats.values())), + "overall/per-k reconciliation failed") + continue + k = row["flipped bit count"] + require(natural(k, N) and settings["minimum flipped bits"] <= k <= settings[ + "maximum flipped bits"], "invalid flipped bit count") + require(k not in pooled, f"duplicate k: {k}") + if "initial full block corrupted bits" in stats: + require(stats["initial full block corrupted bits"] == { + "sum": trials * k, "squared sum": trials * k * k}, "initial count is not exact k") + if not totals: + totals = {name: {"sum": 0, "squared sum": 0} for name in stats} + require(set(stats) == set(totals), "inconsistent metric sets") + for name in stats: + for field in ("sum", "squared sum"): + totals[name][field] += stats[name][field] + count += trials + pooled[k] = {"trials": trials, **{m: stats[name]["sum"] for m, name in METRICS.items()}} + config = (settings["maximum directional passes"], settings["anchors"], settings["binary image"]) + return summary["run identity"], settings["root seed"], config, pooled, codeword + except (KeyError, TypeError, ValueError) as error: + raise ValueError(f"{path}: {error}") from error + + +def pool_reports(paths, allow_mixed_codewords=False): + groups, identities, seeds, conventions = {}, set(), set(), {} + for path in paths: + digest, seed, config, rows, codeword = load_report(path) + require(digest not in identities, f"duplicate run identity: {path}") + require((config, seed) not in seeds, + f"repeated seed {seed} within configuration {config}: {path}; " + "sample overlap cannot be excluded, even with different k ranges") + identities.add(digest) + seeds.add((config, seed)) + previous = conventions.setdefault(config, codeword) + require(allow_mixed_codewords or previous == codeword, + "mixed zero/random codewords require --allow-mixed-codewords") + group = groups.setdefault(config, {}) + for k, row in rows.items(): + target = group.setdefault(k, dict.fromkeys(row, 0)) + for name, value in row.items(): + target[name] += value + return groups + + +def logsumexp(values): + values = list(values) + maximum = max(values, default=NEG_INF) + if maximum == NEG_INF: + return maximum + return maximum + math.log(math.fsum(math.exp(v - maximum) for v in values)) + + +def log_binomial(n, k, p): + require(natural(n) and natural(k, n) and 0 < p < 1, "invalid binomial arguments") + return (math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1) + + k * math.log(p) + (n - k) * math.log1p(-p)) + + +def log_interval(n, low, high, p): + """Sum a binomial interval from its mode, bounding discarded relative tails.""" + if low > high: + return NEG_INF + mode = min(high, max(low, int((n + 1) * p))) + terms = [1.0] + for direction, end in ((-1, low), (1, high)): + k, term = mode, 1.0 + while k != end: + ratio = (k / (n - k + 1) * (1 - p) / p if direction < 0 + else (n - k) / (k + 1) * p / (1 - p)) + # Ratios decrease away from the mode. The geometric bound covers + # the entire omitted tail, not merely the next term. + if ratio < 1 and term * ratio / (1 - ratio) < 1e-16: + break + term *= ratio + terms.append(term) + k += direction + return log_binomial(n, mode, p) + math.log(math.fsum(terms)) + + +def log1mexp(value): + if value == NEG_INF: + return 0.0 + if value == 0: + return NEG_INF + return (math.log1p(-math.exp(value)) if value < -math.log(2) + else math.log(-math.expm1(value))) + + +def evaluate(rows, p, n=N, denominators=None): + """Return natural-log contributions and masses; never renormalize weights.""" + denominators = DENOMINATORS if denominators is None else denominators + weights = {k: log_binomial(n, k, p) for k in rows} + covered = logsumexp(weights.values()) + gaps, start = [], 0 + for k in sorted(rows): + if start < k: + gaps.append((start, k - 1)) + start = k + 1 + if start <= n: + gaps.append((start, n)) + missing = logsumexp(log_interval(n, low, high, p) for low, high in gaps) + # Derive only the larger probability by subtraction. Tiny missing tails + # must not be inferred from a rounded covered mass close to one. + if covered <= missing: + missing = log1mexp(min(0.0, covered)) + else: + covered = log1mexp(min(0.0, missing)) + contributions = {metric: logsumexp( + weights[k] + math.log(row[metric]) - math.log(row["trials"]) - math.log(d) + for k, row in rows.items() if row[metric] > 0) + for metric, d in denominators.items()} + return contributions, covered, missing + + +def config_label(config): + passes, anchors, binary = config + return f"passes={passes}, anchors={'on' if anchors else 'off'}, binary-image={'on' if binary else 'off'}" + + +def plot_value(value): + result = math.exp(value) + return result if result > 0 else math.nan + + +def main(argv=None): + if hasattr(sys, "set_int_max_str_digits"): + sys.set_int_max_str_digits(0) + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + epilog="Weights are Binomial(N,p), without renormalization. Missing k are unknown; " + "zero observed errors are not certainty. Arbitrarily low plotted values are not reliability " + "evidence. No extrapolation or MSE fit: a fitting model has not been specified. " + "Matching decoder configurations pool per-k sums/counts; different flags/caps stay separate. " + "Repeated seeds within a configuration are rejected. Metadata has no source revision.") + parser.add_argument("inputs", nargs="+", type=Path, help="run directories or summary.json with sibling metadata.json") + parser.add_argument("--output", type=Path, default=Path("product_monte_carlo.svg"), help="SVG output") + parser.add_argument("--csv", type=Path, help="CSV output (default: output path with .csv suffix)") + parser.add_argument("--points", type=int, default=200) + parser.add_argument("--metric", choices=("information", "full", "both"), default="information") + parser.add_argument("--allow-mixed-codewords", action="store_true", + help="pool zero/random inputs using linear-code BDD translation equivariance; " + "absent legacy convention means random; repeated seeds remain forbidden") + args = parser.parse_args(argv) + require(args.points >= 2, "--points must be at least 2") + require(args.output.suffix.lower() == ".svg", "--output must be an .svg path") + csv_path = args.csv or args.output.with_suffix(".csv") + protected = {p.resolve() for path in args.inputs for p in ( + (path / "summary.json" if path.is_dir() else path), + (path if path.is_dir() else path.parent) / "metadata.json")} + require(args.output.resolve() != csv_path.resolve() and not protected.intersection( + {args.output.resolve(), csv_path.resolve()}), "output paths must be distinct from each other and inputs") + groups = pool_reports(args.inputs, args.allow_mixed_codewords) + print("Warning: metadata lacks source revision; decoder implementation compatibility cannot be verified. " + "Missing strata are unknown; zero observed errors do not establish zero BER. " + "No extrapolation or MSE fit is performed.", file=sys.stderr) + metrics = list(METRICS) if args.metric == "both" else [args.metric] + ps = [0.008 + (0.0045 - 0.008) * i / (args.points - 1) for i in range(args.points)] + results = [] + for config, rows in groups.items(): + values = [evaluate(rows, p) for p in ps] + label = config_label(config) + for metric in metrics: + zeros = sum(row[metric] == 0 for row in rows.values()) + print(f"{label}; {metric}: {len(rows)}/{N + 1} sampled strata, " + f"{zeros} zero-observed strata; log10 missing mass range " + f"[{min(v[2] for v in values) / math.log(10):.6g}, " + f"{max(v[2] for v in values) / math.log(10):.6g}]", file=sys.stderr) + results.append((label, metric, zeros, len(rows), values)) + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError as error: + raise ValueError("plotting requires matplotlib; numerical core uses only the standard library") from error + with csv_path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.writer(stream) + writer.writerow(["configuration", "metric", "p", "log10_contribution", "log10_covered_mass", + "log10_missing_mass", "sampled_strata", "zero_observed_strata"]) + for label, metric, zeros, sampled, values in results: + for p, (contributions, covered, missing) in zip(ps, values): + writer.writerow([label, metric, p, contributions[metric] / math.log(10), + covered / math.log(10), missing / math.log(10), sampled, zeros]) + plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11, + "svg.hashsalt": "gf256-product-monte-carlo"}) + figure, axis = plt.subplots(figsize=(11, 7)) + for label, metric, _, _, values in results: + axis.plot(ps, [plot_value(v[0][metric]) for v in values], linewidth=2, + label=f"{metric}: {label}") + axis.set(xlim=(0.008, 0.0045), ylim=(1e-30, 1e-1), yscale="log", + xlabel="Channel bit-flip probability p", + ylabel="Sampled-stratum BER contribution", + title="Product-code sampled-stratum BER contribution") + axis.grid(True, which="major", color="#d1d5db", alpha=0.75) + axis.spines[["top", "right"]].set_visible(False) + axis.legend(fontsize=8) + figure.text(0.5, 0.02, "Missing strata unknown; zero observed errors are not certainty. No extrapolation.", + ha="center", fontsize=9) + figure.tight_layout(rect=(0, 0.04, 1, 1)) + figure.savefig(args.output, facecolor="white", metadata={"Creator": __file__, "Date": None}) + plt.close(figure) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + sys.exit(1) diff --git a/src/reed_solomon/error_correction/batched.cc b/src/reed_solomon/error_correction/batched.cc index 38d68ee..86eacf1 100644 --- a/src/reed_solomon/error_correction/batched.cc +++ b/src/reed_solomon/error_correction/batched.cc @@ -105,14 +105,15 @@ const Element* NativeSource(std::span data, } #endif -CorrectionStatus CorrectColumnsScalar(const LCHDecoder& decoder, - std::span data, - std::span recovery, - size_t byte_count, - size_t first_column, - std::span results, - std::span error_masks, - std::span mutable_recovery) { +CorrectionStatus CorrectColumnsScalar( + const LCHDecoder& decoder, + std::span data, + std::span recovery, + size_t byte_count, + size_t first_column, + std::span results, + std::span error_masks, + std::span mutable_recovery) { const size_t data_count = data.size(); const size_t recovery_count = recovery.size(); const size_t codeword_size = data_count + recovery_count; @@ -137,7 +138,8 @@ CorrectionStatus CorrectColumnsScalar(const LCHDecoder& decoder, std::copy_n(recovery_values.begin(), recovery_count, data_values.begin() + data_count); const auto before = data_values; - result = CorrectCodeword(decoder, std::span(data_values).first(codeword_size)); + result = + CorrectCodeword(decoder, std::span(data_values).first(codeword_size)); for (size_t i = 0; i < codeword_size; ++i) { mask[i] = before[i] != data_values[i]; } @@ -803,9 +805,11 @@ void CorrectChunk32(std::span data, } } // Whole-codeword mode must evaluate and verify parity-only candidates too. - const uint32_t location_only_lanes = mutable_recovery.empty() - ? candidate_lanes & ~data_error_lanes : 0; - if (mutable_recovery.empty()) candidate_lanes &= data_error_lanes; + const uint32_t location_only_lanes = + mutable_recovery.empty() ? candidate_lanes & ~data_error_lanes : 0; + if (mutable_recovery.empty()) { + candidate_lanes &= data_error_lanes; + } if (candidate_lanes == 0) { PublishChunkResults(results, error_masks, byte_count, column, parameters.family, data_count, recovery_count, @@ -958,7 +962,8 @@ void CorrectChunk32(std::span data, continue; } Element* destination = public_position < data_count - ? data[public_position] : mutable_recovery[public_position - data_count]; + ? data[public_position] + : mutable_recovery[public_position - data_count]; const uint32_t active = root_masks[native_position] & candidate_lanes; const __m256i old_data = _mm256_loadu_si256( reinterpret_cast(destination + column)); @@ -966,9 +971,8 @@ void CorrectChunk32(std::span data, _mm256_load_si256( reinterpret_cast(Row(work, native_position))), LaneMask(active)); - _mm256_storeu_si256( - reinterpret_cast<__m256i*>(destination + column), - _mm256_xor_si256(old_data, correction)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(destination + column), + _mm256_xor_si256(old_data, correction)); } PublishChunkResults(results, error_masks, byte_count, column, parameters.family, data_count, recovery_count, @@ -980,13 +984,14 @@ void CorrectChunk32(std::span data, } // namespace -static CorrectionStatus CorrectBatchImpl(const LCHDecoder& decoder, - std::span data, - std::span recovery, - size_t byte_count, - std::span results, - std::span error_masks, - std::span mutable_recovery) { +static CorrectionStatus CorrectBatchImpl( + const LCHDecoder& decoder, + std::span data, + std::span recovery, + size_t byte_count, + std::span results, + std::span error_masks, + std::span mutable_recovery) { if (!decoder.Valid()) { return CorrectionStatus::invalid_argument; } diff --git a/src/reed_solomon/error_correction/internal.h b/src/reed_solomon/error_correction/internal.h index 459ae66..2021f06 100644 --- a/src/reed_solomon/error_correction/internal.h +++ b/src/reed_solomon/error_correction/internal.h @@ -8,8 +8,8 @@ namespace gf2p8::rs::detail::error_correction { -using ::gf2p8::rs::CorrectionStatus; using ::gf2p8::rs::CorrectionResult; +using ::gf2p8::rs::CorrectionStatus; /** * @brief Corrects one scalar LCH Reed-Solomon codeword. diff --git a/src/reed_solomon/error_correction/scalar.cc b/src/reed_solomon/error_correction/scalar.cc index 4955662..3d0538c 100644 --- a/src/reed_solomon/error_correction/scalar.cc +++ b/src/reed_solomon/error_correction/scalar.cc @@ -228,18 +228,17 @@ size_t PublicPosition(CodeFamily family, : native_position - recovery_count; } -CorrectionStatus RecoverWithEvaluator( - CodeFamily family, - std::span data, - std::span recovery, - std::span mutable_recovery, - size_t recovery_count, - std::span root_positions, - const Values& locator_samples, - const Values& locator_coefficients, - size_t locator_degree, - const Values& syndrome_samples, - const MultiplicationTables& tables) { +CorrectionStatus RecoverWithEvaluator(CodeFamily family, + std::span data, + std::span recovery, + std::span mutable_recovery, + size_t recovery_count, + std::span root_positions, + const Values& locator_samples, + const Values& locator_coefficients, + size_t locator_degree, + const Values& syndrome_samples, + const MultiplicationTables& tables) { const size_t data_count = data.size(); const size_t codeword_size = data_count + recovery_count; const size_t correction_radius = recovery_count / 2; @@ -641,7 +640,8 @@ static CorrectionResult CorrectOneImpl(const LCHDecoder& decoder, CorrectionStatus recovery_status = CorrectionStatus::ok; // Whole-codeword mode verifies every candidate, including parity-only roots. - // Retain the existing data-only fast path for CorrectOne/CorrectBatch callers. + // Retain the existing data-only fast path for CorrectOne/CorrectBatch + // callers. if (has_data_error && root_count == 1 && mutable_recovery.empty()) { // Every aligned R-point native Cantor IFFT has unit leading Lagrange // coefficient. Therefore the highest syndrome coefficient is the error @@ -679,7 +679,7 @@ CorrectionResult CorrectOne(const LCHDecoder& decoder, namespace gf2p8::rs { CorrectionResult CorrectCodeword(const LCHDecoder& decoder, - std::span codeword) { + std::span codeword) { if (!decoder.Valid() || codeword.size() != decoder.DataCount() + decoder.RecoveryCount()) { return {.status = CorrectionStatus::invalid_argument}; diff --git a/src/reed_solomon/product_code_internal.h b/src/reed_solomon/product_code_internal.h index cf94a1c..a4930ff 100644 --- a/src/reed_solomon/product_code_internal.h +++ b/src/reed_solomon/product_code_internal.h @@ -4,22 +4,44 @@ namespace gf2p8::rs::detail { -/** @brief Private differential-test and benchmark access to initial-pass choices. */ +/** @brief Private differential-test and benchmark access to initial-pass + * choices. */ struct ProductCorrectionAccess { /** - * @brief Runs the same scheduler with zero, one, or two initial batched passes. + * @brief Runs the same scheduler with zero, one, or two initial batched + * passes. * @param code Product dimensions and component decoders. * @param block Mutable row-major block. * @param cap Directional pass limit. - * @param batch_passes Initial passes to batch (0: reference single path). - * @param tracked_validation Skip known-clean lines; false retains the full scan. + * @param batch_passes Initial passes to batch (0: reference single path). + * @param tracked_validation Skip known-clean lines; false retains the full + * scan. * @return The normal product outcome and work counts. */ static ProductCorrectionResult Correct(const StrongWeakRSProductCode& code, - std::span block, size_t cap, - unsigned batch_passes, - bool tracked_validation = true) { - return code.CorrectImpl(block, cap, batch_passes, tracked_validation); + std::span block, + size_t cap, + unsigned batch_passes, + bool tracked_validation = true) { + return Correct(code, block, ProductDecodeOptions{cap}, batch_passes, + tracked_validation); + } + + /** + * @brief Runs a private initial-pass/validation variant with per-call gates. + * @param code Product dimensions and component decoders. + * @param block Mutable row-major block. + * @param options Directional pass cap and independent weak gates. + * @param batch_passes Initial passes to batch (0: reference single path). + * @param tracked_validation Whether to skip known-clean lines at exit. + * @return The normal product outcome and work counts. + */ + static ProductCorrectionResult Correct(const StrongWeakRSProductCode& code, + std::span block, + ProductDecodeOptions options, + unsigned batch_passes, + bool tracked_validation = true) { + return code.CorrectImpl(block, options, batch_passes, tracked_validation); } }; diff --git a/src/reed_solomon/strong_weak_rs_product_code.cc b/src/reed_solomon/strong_weak_rs_product_code.cc index 221f680..87c80b0 100644 --- a/src/reed_solomon/strong_weak_rs_product_code.cc +++ b/src/reed_solomon/strong_weak_rs_product_code.cc @@ -1,24 +1,30 @@ #include "reed_solomon/strong_weak_rs_product_code.h" -#include "reed_solomon/error_correction/internal.h" #include #include #include #include +#include "reed_solomon/error_correction/internal.h" + namespace gf2p8::rs { namespace { bool Aligned(size_t n, size_t k) { - return n <= 256 && std::has_single_bit(n) && k < n && - n - k >= 2 && n - k <= k && std::has_single_bit(n - k); + return n <= 256 && std::has_single_bit(n) && k < n && n - k >= 2 && + n - k <= k && std::has_single_bit(n - k); } } // namespace -StrongWeakRSProductCode::StrongWeakRSProductCode(size_t strong_n, size_t strong_k, - size_t weak_n, size_t weak_k) - : strong_n_(strong_n), strong_k_(strong_k), weak_n_(weak_n), weak_k_(weak_k), +StrongWeakRSProductCode::StrongWeakRSProductCode(size_t strong_n, + size_t strong_k, + size_t weak_n, + size_t weak_k) + : strong_n_(strong_n), + strong_k_(strong_k), + weak_n_(weak_n), + weak_k_(weak_k), valid_(Aligned(strong_n, strong_k) && Aligned(weak_n, weak_k) && weak_n - weak_k == 2), strong_encoder_(valid_ ? strong_k : 0, valid_ ? strong_n - strong_k : 0), @@ -36,15 +42,15 @@ size_t StrongWeakRSProductCode::BlockSize() const { } lch::Status StrongWeakRSProductCode::Encode(std::span block, - lch::Backend backend) const { + lch::Backend backend) const { if (!Valid() || block.size() != BlockSize()) { return lch::Status::invalid_argument; } std::vector candidate(block.begin(), block.end()); std::array data{}; std::array recovery{}; - std::vector workspace(std::max(strong_encoder_.WorkspaceSize(weak_n_), - weak_encoder_.WorkspaceSize(1))); + std::vector workspace(std::max( + strong_encoder_.WorkspaceSize(weak_n_), weak_encoder_.WorkspaceSize(1))); for (size_t row = 0; row < strong_k_; ++row) { for (size_t col = 0; col < weak_k_; ++col) { data[col] = &candidate[row * weak_n_ + col]; @@ -52,9 +58,9 @@ lch::Status StrongWeakRSProductCode::Encode(std::span block, for (size_t col = weak_k_; col < weak_n_; ++col) { recovery[col - weak_k_] = &candidate[row * weak_n_ + col]; } - const auto status = weak_encoder_.Encode( - std::span(data).first(weak_k_), std::span(recovery).first(2), 1, - workspace, backend); + const auto status = weak_encoder_.Encode(std::span(data).first(weak_k_), + std::span(recovery).first(2), 1, + workspace, backend); if (status != lch::Status::ok) { return status; } @@ -65,10 +71,10 @@ lch::Status StrongWeakRSProductCode::Encode(std::span block, for (size_t row = strong_k_; row < strong_n_; ++row) { recovery[row - strong_k_] = &candidate[row * weak_n_]; } - const auto status = strong_encoder_.Encode( - std::span(data).first(strong_k_), - std::span(recovery).first(strong_n_ - strong_k_), weak_n_, workspace, - backend); + const auto status = + strong_encoder_.Encode(std::span(data).first(strong_k_), + std::span(recovery).first(strong_n_ - strong_k_), + weak_n_, workspace, backend); if (status == lch::Status::ok) { std::copy(candidate.begin(), candidate.end(), block.begin()); } @@ -76,16 +82,28 @@ lch::Status StrongWeakRSProductCode::Encode(std::span block, } ProductCorrectionResult StrongWeakRSProductCode::Correct( - std::span block, size_t max_directional_passes) const { + std::span block, + size_t max_directional_passes) const { + return Correct(block, ProductDecodeOptions{max_directional_passes}); +} + +ProductCorrectionResult StrongWeakRSProductCode::Correct( + std::span block, + ProductDecodeOptions options) const { // Avoid packing overhead when no complete SIMD batch can run. const unsigned batches = lch::BackendAvailable(lch::Backend::avx2) && - std::max(strong_n_, weak_n_) >= 32 ? 2 : 0; - return CorrectImpl(block, max_directional_passes, batches); + std::max(strong_n_, weak_n_) >= 32 + ? 2 + : 0; + return CorrectImpl(block, options, batches); } ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( - std::span block, size_t max_directional_passes, - unsigned batch_passes, bool tracked_validation) const { + std::span block, + ProductDecodeOptions options, + unsigned batch_passes, + bool tracked_validation) const { + const size_t max_directional_passes = options.max_directional_passes; ProductCorrectionResult result; if (!Valid() || block.size() != BlockSize() || max_directional_passes < 2) { return result; @@ -105,11 +123,14 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( const size_t length = strong ? strong_n_ : weak_n_; std::array next{}; size_t changes = 0; + size_t bit_changes = 0; const bool batched = pass < std::min(batch_passes, 2u); if (batched) { // Columns already have position-major layout; rows need transposition. // Keep tentative repairs private until the existing weak gates accept. - if (strong) std::copy(block.begin(), block.end(), packed.begin()); + if (strong) { + std::copy(block.begin(), block.end(), packed.begin()); + } for (size_t pos = 0; pos < length; ++pos) { shards[pos] = packed.data() + pos * lines; if (!strong) { @@ -122,7 +143,9 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( strong ? strong_decoder_ : weak_decoder_, std::span(shards).first(length), lines, std::span(outcomes).first(lines), masks); - if (status != CorrectionStatus::ok) return result; + if (status != CorrectionStatus::ok) { + return result; + } } if (batched && strong) { result.strong_lines_visited += lines; @@ -136,6 +159,8 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( for (size_t line = 0; line < lines; ++line) { const size_t index = pos * lines + line; if (masks[index]) { + bit_changes += std::popcount( + static_cast(block[index] ^ packed[index])); block[index] = packed[index]; next[pos] = true; clean_rows[pos] = false; @@ -143,67 +168,84 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( } } } - } else for (size_t line = 0; line < lines; ++line) { - if (pass >= 2 && !active[line]) { - continue; - } - if (strong) { - ++result.strong_lines_visited; - } else { - ++result.weak_lines_visited; - } - const auto index = [&](size_t pos) { - return strong ? pos * weak_n_ + line : line * weak_n_ + pos; - }; - if (!batched) { - for (size_t pos = 0; pos < length; ++pos) { - candidate[pos] = block[index(pos)]; - } - } - const auto correction = batched ? outcomes[line] : CorrectCodeword( - strong ? strong_decoder_ : weak_decoder_, - std::span(candidate).first(length)); - auto& clean = strong ? clean_columns[line] : clean_rows[line]; - clean = correction.status == CorrectionStatus::ok && correction.error_count == 0; - if (strong) { - protected_columns[line] = correction.status == CorrectionStatus::ok; - } - if (correction.status != CorrectionStatus::ok) { - continue; - } - if (correction.error_count == 0) continue; - if (batched) { - for (size_t pos = 0; pos < length; ++pos) candidate[pos] = shards[pos][line]; - } - if (!strong) { - if (correction.error_count != 1) { + } else { + for (size_t line = 0; line < lines; ++line) { + if (pass >= 2 && !active[line]) { continue; } - bool accept = true; - for (size_t pos = 0; pos < length; ++pos) { - const unsigned delta = block[index(pos)] ^ candidate[pos]; - if (delta != 0 && (protected_columns[pos] || std::popcount(delta) > 2)) { - accept = false; + if (strong) { + ++result.strong_lines_visited; + } else { + ++result.weak_lines_visited; + } + const auto index = [&](size_t pos) { + return strong ? pos * weak_n_ + line : line * weak_n_ + pos; + }; + if (!batched) { + for (size_t pos = 0; pos < length; ++pos) { + candidate[pos] = block[index(pos)]; } } - if (!accept) { + const auto correction = + batched ? outcomes[line] + : CorrectCodeword(strong ? strong_decoder_ : weak_decoder_, + std::span(candidate).first(length)); + auto& clean = strong ? clean_columns[line] : clean_rows[line]; + clean = correction.status == CorrectionStatus::ok && + correction.error_count == 0; + if (strong) { + protected_columns[line] = correction.status == CorrectionStatus::ok; + } + if (correction.status != CorrectionStatus::ok) { continue; } - } - for (size_t pos = 0; pos < length; ++pos) { - if (block[index(pos)] != candidate[pos]) { - block[index(pos)] = candidate[pos]; - next[pos] = true; - (strong ? clean_rows[pos] : clean_columns[pos]) = false; - ++changes; + if (correction.error_count == 0) { + continue; } + if (batched) { + for (size_t pos = 0; pos < length; ++pos) { + candidate[pos] = shards[pos][line]; + } + } + if (!strong) { + if (correction.error_count != 1) { + continue; + } + bool accept = true; + for (size_t pos = 0; pos < length; ++pos) { + const unsigned delta = block[index(pos)] ^ candidate[pos]; + if (delta != 0 && + ((options.use_anchors && protected_columns[pos]) || + (options.use_binary_image && std::popcount(delta) > 2))) { + accept = false; + } + } + if (!accept) { + continue; + } + } + for (size_t pos = 0; pos < length; ++pos) { + if (block[index(pos)] != candidate[pos]) { + bit_changes += std::popcount( + static_cast(block[index(pos)] ^ candidate[pos])); + block[index(pos)] = candidate[pos]; + next[pos] = true; + (strong ? clean_rows[pos] : clean_columns[pos]) = false; + ++changes; + } + } + // Successful BDD verifies its candidate internally; only committed + // candidates establish validity. Rejected weak candidates do not. + clean = true; } - // Successful BDD verifies its candidate internally; only committed - // candidates establish validity. Rejected weak candidates do not. - clean = true; } ++result.directional_passes; result.changed_symbols += changes; + result.changed_bits += bit_changes; + (strong ? result.strong_changed_symbols : result.weak_changed_symbols) += + changes; + (strong ? result.strong_changed_bits : result.weak_changed_bits) += + bit_changes; active = next; if (pass >= 1 && changes == 0) { result.termination = ProductTermination::no_change; @@ -217,15 +259,17 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( const size_t lines = strong ? weak_n_ : strong_n_; const size_t length = strong ? strong_n_ : weak_n_; for (size_t line = 0; line < lines; ++line) { - if (tracked_validation && (strong ? clean_columns[line] : clean_rows[line])) { + if (tracked_validation && + (strong ? clean_columns[line] : clean_rows[line])) { continue; } for (size_t pos = 0; pos < length; ++pos) { - candidate[pos] = block[strong ? pos * weak_n_ + line : line * weak_n_ + pos]; + candidate[pos] = + block[strong ? pos * weak_n_ + line : line * weak_n_ + pos]; } - const auto check = CorrectCodeword( - strong ? strong_decoder_ : weak_decoder_, - std::span(candidate).first(length)); + const auto check = + CorrectCodeword(strong ? strong_decoder_ : weak_decoder_, + std::span(candidate).first(length)); if (check.status != CorrectionStatus::ok || check.error_count != 0) { result.all_zero_syndromes = false; } diff --git a/tests/plot_product_monte_carlo_test.py b/tests/plot_product_monte_carlo_test.py new file mode 100644 index 0000000..5fbc17e --- /dev/null +++ b/tests/plot_product_monte_carlo_test.py @@ -0,0 +1,257 @@ +import copy +import importlib.util +import json +import math +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "plot_product_monte_carlo.py" +SPEC = importlib.util.spec_from_file_location("plot_product", SCRIPT) +plot = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(plot) + + +class NumericalTest(unittest.TestCase): + def test_binomial_identity_and_weighted_channel(self): + n, p = 12, 0.17 + rows = {k: {"trials": 1, "full": k} for k in range(n + 1)} + values, covered, missing = plot.evaluate(rows, p, n, {"full": n}) + self.assertAlmostEqual(math.exp(values["full"]), p, places=14) + self.assertEqual(covered, 0) + self.assertEqual(missing, -math.inf) + for low in range(n + 1): + for high in range(low, n + 1): + expected = sum(math.comb(n, k) * p**k * (1 - p)**(n - k) + for k in range(low, high + 1)) + self.assertAlmostEqual(plot.log_interval(n, low, high, p), math.log(expected), places=12) + + def test_tiny_exact_endpoint_and_large_counters(self): + n, p = plot.N, 0.0045 + count = 10**400 + rows = {n: {"trials": count, "full": count * n}} + values, covered, missing = plot.evaluate(rows, p, n, {"full": n}) + expected = n * math.log(p) + self.assertAlmostEqual(values["full"], expected, places=8) + self.assertEqual(covered, expected) + self.assertEqual(missing, 0) + self.assertTrue(math.isfinite(values["full"] / math.log(10))) + self.assertTrue(math.isnan(plot.plot_value(values["full"]))) + + def test_nearly_complete_coverage_keeps_tiny_missing_tail(self): + n, p = 100, 0.0045 + rows = {k: {"trials": 1, "full": 0} for k in range(n)} + values, covered, missing = plot.evaluate(rows, p, n, {"full": n}) + self.assertAlmostEqual(missing, n * math.log(p), places=10) + self.assertLess(covered, 0) + self.assertEqual(values["full"], -math.inf) + self.assertTrue(math.isnan(plot.plot_value(values["full"]))) + + def test_empty_and_sparse_coverage_not_renormalized(self): + values, covered, missing = plot.evaluate({}, 0.1, 10, {"full": 10}) + self.assertEqual((values["full"], covered, missing), (-math.inf, -math.inf, 0)) + rows = {2: {"trials": 2, "full": 10}} + values, covered, missing = plot.evaluate(rows, 0.1, 10, {"full": 10}) + self.assertAlmostEqual(values["full"], covered + math.log(0.5)) + self.assertAlmostEqual(math.exp(covered) + math.exp(missing), 1) + + def test_disjoint_missing_intervals(self): + n = 30 + for p in (0.0045, 0.5, 0.9955): + for sampled in ({0, 2, 10, 29}, set(range(1, n + 1))): + rows = {k: {"trials": 1, "full": 1} for k in sampled} + _, covered, missing = plot.evaluate(rows, p, n, {"full": n}) + expected = sum(math.comb(n, k) * p**k * (1 - p)**(n - k) + for k in range(n + 1) if k not in sampled) + self.assertAlmostEqual(missing, math.log(expected), places=11) + self.assertAlmostEqual(math.exp(covered) + math.exp(missing), 1) + + +class ReportTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="plot-product-") + self.root = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def fixture(self, name="run", seed=1, trials=1, residual=1, passes=16): + path = self.root / name + path.mkdir() + metadata = {"schema revision": 1, "created at": name, "code": plot.CODE, + "random algorithm": plot.RANDOM, + "settings": {"root seed": seed, "batch size": 1000, "batches": 0, + "threads": 1, "minimum flipped bits": 0, + "maximum flipped bits": plot.N, + "maximum directional passes": passes, "anchors": True, + "binary image": True, "checkpoint trials": 64, + "report seconds": 2, "fsync seconds": 5}} + stats = {metric: {"sum": residual * trials, "squared sum": residual**2 * trials} + for metric in plot.METRICS.values()} + row = {"flipped bit count": 2600, "trial count": trials, "statistics": stats} + summary = {"schema revision": 1, "run identity": plot.identity(metadata), + "overall": {"trial count": trials, "statistics": copy.deepcopy(stats)}, + "by flipped bit count": [row]} + self.save(path, metadata, summary) + return path, metadata, summary + + def save(self, path, metadata, summary): + (path / "metadata.json").write_text(json.dumps(metadata)) + (path / "summary.json").write_text(json.dumps(summary)) + + def test_pool_counts_not_means_and_separate_configs(self): + a, _, _ = self.fixture("a", trials=1, residual=10) + b, _, _ = self.fixture("b", seed=2, trials=9, residual=0) + c, _, _ = self.fixture("c", passes=32) + groups = plot.pool_reports([a, b / "summary.json", c]) + self.assertEqual(len(groups), 2) + row = groups[(16, True, True)][2600] + self.assertEqual(row, {"trials": 10, "information": 10, "full": 10}) + + def test_duplicate_identity_and_seed(self): + a, _, _ = self.fixture("a") + b, _, _ = self.fixture("b") + with self.assertRaisesRegex(ValueError, "duplicate run identity"): + plot.pool_reports([a, a / "summary.json"]) + with self.assertRaisesRegex(ValueError, "repeated seed"): + plot.pool_reports([a, b]) + + def test_schema2_pooling_exact_integers_and_reconciliation(self): + old, _, _ = self.fixture("old", seed=1, trials=1, residual=10) + path, metadata, summary = self.fixture("new", seed=2) + n = 10**80 + metadata["schema revision"] = summary["schema revision"] = 2 + summary["run identity"] = plot.identity(metadata) + stats = {"completed blocks": n, "total iterations": n*2, + "information bits": {"total bits": n*455168, "raw corrupted bits": n*2000, + "post decoding corrupted bits": n*3}, + "full-codeword bits": {"total bits": n*524288, "raw corrupted bits": n*2600, + "post decoding corrupted bits": n*4}} + summary["overall"] = {"statistics": copy.deepcopy(stats)} + summary["by flipped bit count"] = [{"flipped bit count": 2600, "statistics": copy.deepcopy(stats)}] + self.save(path, metadata, summary) + rows = plot.pool_reports([old, path])[(16, True, True)] + self.assertEqual(rows[2600], {"trials": n+1, "information": n*3+10, "full": n*4+10}) + for mutation in ( + lambda s: s["overall"]["statistics"].update({"completed blocks": n+1}), + lambda s: s["by flipped bit count"][0]["statistics"]["information bits"].update({"total bits": 1}), + lambda s: s["by flipped bit count"][0]["statistics"].update({"total iterations": True}), + lambda s: s["by flipped bit count"][0]["statistics"]["full-codeword bits"].update({"raw corrupted bits": 0})): + bad = copy.deepcopy(summary) + mutation(bad) + self.save(path, metadata, bad) + with self.assertRaises(ValueError): + plot.load_report(path) + + def test_fisher_yates_metadata_preserves_summary_format(self): + path, metadata, summary = self.fixture() + expected = plot.load_report(path)[1:] + metadata["random algorithm"] = plot.RANDOM_FY + summary["run identity"] = plot.identity(metadata) + self.save(path, metadata, summary) + self.assertEqual(plot.load_report(path)[1:], expected) + + def test_codeword_conventions_and_explicit_pooling(self): + legacy, _, _ = self.fixture("legacy") + zero, metadata, summary = self.fixture("zero", seed=2) + metadata["codeword"] = "zero" + summary["run identity"] = plot.identity(metadata) + self.save(zero, metadata, summary) + self.assertEqual(plot.load_report(legacy)[4], "random") + self.assertEqual(plot.load_report(zero)[4], "zero") + self.assertEqual(plot.pool_reports([zero])[(16, True, True)][2600]["trials"], 1) + with self.assertRaisesRegex(ValueError, "mixed zero/random"): + plot.pool_reports([legacy, zero]) + self.assertEqual(plot.pool_reports([legacy, zero], True)[(16, True, True)][2600]["trials"], 2) + metadata["settings"]["root seed"] = 1 + summary["run identity"] = plot.identity(metadata) + self.save(zero, metadata, summary) + with self.assertRaisesRegex(ValueError, "repeated seed"): + plot.pool_reports([legacy, zero], True) + metadata["codeword"] = "unsupported" + summary["run identity"] = plot.identity(metadata) + self.save(zero, metadata, summary) + with self.assertRaisesRegex(ValueError, "codeword convention"): + plot.load_report(zero) + + def test_flags_separate_groups_and_range_does_not_bypass_seed_check(self): + a, _, _ = self.fixture("a") + paths = [a] + for flag in ("anchors", "binary image"): + path, metadata, summary = self.fixture(flag) + metadata["settings"][flag] = False + summary["run identity"] = plot.identity(metadata) + self.save(path, metadata, summary) + paths.append(path) + self.assertEqual(len(plot.pool_reports(paths)), 3) + path, metadata, summary = self.fixture("range") + metadata["settings"]["minimum flipped bits"] = 2500 + summary["run identity"] = plot.identity(metadata) + self.save(path, metadata, summary) + with self.assertRaisesRegex(ValueError, "repeated seed"): + plot.pool_reports([a, path]) + + def test_malformed_reports(self): + path, metadata, summary = self.fixture() + mutations = [ + lambda s: s.update({"schema revision": True}), + lambda s: s.update({"run identity": "wrong"}), + lambda s: s["by flipped bit count"].append(copy.deepcopy(s["by flipped bit count"][0])), + lambda s: s["by flipped bit count"][0].update({"flipped bit count": plot.N + 1}), + lambda s: s["by flipped bit count"][0].update({"trial count": True}), + lambda s: s["overall"].update({"trial count": 2}), + lambda s: s["overall"]["statistics"][plot.METRICS["full"]].update({"sum": 0}), + lambda s: s["by flipped bit count"][0]["statistics"][plot.METRICS["full"]].update( + {"sum": plot.N + 1, "squared sum": (plot.N + 1)**2}), + ] + for mutate in mutations: + with self.subTest(mutate=mutate): + broken = copy.deepcopy(summary) + mutate(broken) + self.save(path, metadata, broken) + with self.assertRaises(ValueError): + plot.load_report(path) + for key in ("code", "random algorithm", "schema revision"): + broken = dict(metadata, **{key: "unsupported"}) + self.save(path, broken, summary) + with self.assertRaisesRegex(ValueError, "incompatible"): + plot.load_report(path) + + def test_strict_json_numbers_and_duplicate_keys(self): + path = self.root / "bad.json" + for text in ('{"x":1,"x":2}', '{"x":1.0}', '{"x":NaN}'): + path.write_text(text) + with self.assertRaises(ValueError): + plot.read_json(path) + + def test_large_json_counters(self): + path, _, _ = self.fixture(trials=10**400) + self.assertEqual(plot.load_report(path)[3][2600]["trials"], 10**400) + + def test_help_and_headless_outputs(self): + result = subprocess.run([sys.executable, str(SCRIPT), "--help"], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("No extrapolation", result.stdout) + if importlib.util.find_spec("matplotlib") is None: + self.skipTest("matplotlib not installed") + path, _, _ = self.fixture(residual=0) + metadata = plot.read_json(path / "metadata.json") + summary = plot.read_json(path / "summary.json") + metadata["codeword"] = "zero" + summary["run identity"] = plot.identity(metadata) + self.save(path, metadata, summary) + output = self.root / "plot.svg" + result = subprocess.run([sys.executable, str(SCRIPT), str(path), "--output", str(output), + "--points", "3", "--metric", "both"], + capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(" #include +#include #include #include #include @@ -7,9 +8,9 @@ #include #include "gtest/gtest.h" -#include "reed_solomon/strong_weak_rs_product_code.h" #include "reed_solomon/error_correction/internal.h" #include "reed_solomon/product_code_internal.h" +#include "reed_solomon/strong_weak_rs_product_code.h" namespace { @@ -22,19 +23,30 @@ std::vector Codeword(size_t n, size_t k, uint32_t seed) { LCHEncoder encoder(k, n - k); std::mt19937 random(seed); std::vector word(n); - for (size_t i = 0; i < k; ++i) word[i] = static_cast(random()); + for (size_t i = 0; i < k; ++i) { + word[i] = static_cast(random()); + } std::vector data(k); std::vector recovery(n - k); - for (size_t i = 0; i < k; ++i) data[i] = &word[i]; - for (size_t i = k; i < n; ++i) recovery[i - k] = &word[i]; + for (size_t i = 0; i < k; ++i) { + data[i] = &word[i]; + } + for (size_t i = k; i < n; ++i) { + recovery[i - k] = &word[i]; + } std::vector workspace(encoder.WorkspaceSize(1)); - EXPECT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + EXPECT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), + Status::ok); return word; } TEST(WholeCodeword, RepairsEveryPositionAndFullRadiusIncludingParity) { - for (const auto [n, k] : {std::pair{4u, 2u}, {8u, 4u}, {16u, 12u}, - {256u, 224u}, {256u, 128u}, {256u, 254u}}) { + for (const auto [n, k] : {std::pair{4u, 2u}, + {8u, 4u}, + {16u, 12u}, + {256u, 224u}, + {256u, 128u}, + {256u, 254u}}) { const auto original = Codeword(n, k, n + k); LCHDecoder decoder(k, n - k); for (size_t pos = 0; pos < n; ++pos) { @@ -49,9 +61,14 @@ TEST(WholeCodeword, RepairsEveryPositionAndFullRadiusIncludingParity) { for (size_t trial = 0; trial < 20; ++trial) { auto word = original; std::vector positions(n); - for (size_t i = 0; i < n; ++i) positions[i] = i; - if (trial != 0) std::shuffle(positions.begin(), positions.end(), random); - else std::reverse(positions.begin(), positions.end()); + for (size_t i = 0; i < n; ++i) { + positions[i] = i; + } + if (trial != 0) { + std::shuffle(positions.begin(), positions.end(), random); + } else { + std::reverse(positions.begin(), positions.end()); + } for (size_t i = 0; i < (n - k) / 2; ++i) { word[positions[i]] ^= static_cast(1 + random() % 255); } @@ -94,19 +111,27 @@ TEST(WholeCodeword, TransactionalFailureAndInvalidDimensions) { TEST(ProductCode, DimensionsAndInvalidCalls) { EXPECT_TRUE(StrongWeakRSProductCode().Valid()); EXPECT_EQ(StrongWeakRSProductCode().BlockSize(), 65536u); - for (const auto [n, k] : {std::pair{0u, 0u}, {8u, 8u}, {8u, 9u}, {7u, 5u}, - {8u, 5u}, {8u, 2u}, {512u, 480u}, {8u, 7u}}) { + for (const auto [n, k] : {std::pair{0u, 0u}, + {8u, 8u}, + {8u, 9u}, + {7u, 5u}, + {8u, 5u}, + {8u, 2u}, + {512u, 480u}, + {8u, 7u}}) { StrongWeakRSProductCode code(n, k, 8, 6); EXPECT_FALSE(code.Valid()); EXPECT_EQ(code.BlockSize(), 0u); std::vector block(32, 17); const auto before = block; EXPECT_EQ(code.Encode(block), Status::invalid_argument); - EXPECT_EQ(code.Correct(block).termination, ProductTermination::invalid_argument); + EXPECT_EQ(code.Correct(block).termination, + ProductTermination::invalid_argument); EXPECT_EQ(block, before); } EXPECT_FALSE(StrongWeakRSProductCode(8, 4, 8, 4).Valid()); - EXPECT_FALSE(StrongWeakRSProductCode(std::numeric_limits::max(), 1).Valid()); + EXPECT_FALSE( + StrongWeakRSProductCode(std::numeric_limits::max(), 1).Valid()); StrongWeakRSProductCode code(4, 2, 8, 6); std::vector block(32, 17); const auto before = block; @@ -124,12 +149,15 @@ TEST(ProductCode, DimensionsAndInvalidCalls) { } TEST(ProductCode, SystematicEncodingScalarAgreementAndAllComponentValidity) { - for (const auto [ns, ks, nw, kw] : - {std::array{4, 2, 8, 6}, {16, 12, 16, 14}, {256, 224, 256, 254}}) { + for (const auto [ns, ks, nw, kw] : {std::array{4, 2, 8, 6}, + {16, 12, 16, 14}, + {256, 224, 256, 254}}) { StrongWeakRSProductCode code(ns, ks, nw, kw); std::mt19937 random(901); std::vector block(code.BlockSize()); - for (auto& value : block) value = static_cast(random()); + for (auto& value : block) { + value = static_cast(random()); + } const auto input = block; auto scalar = block; ASSERT_EQ(code.Encode(block), Status::ok); @@ -149,12 +177,17 @@ TEST(ProductCode, SystematicEncodingScalarAgreementAndAllComponentValidity) { EXPECT_EQ(result.termination, ProductTermination::no_change); EXPECT_EQ(block, scalar); // Arbitrary symbol damage in each of the four product regions. - for (auto [row, col] : {std::pair{size_t{0}, size_t{0}}, {ks, size_t{0}}, - {size_t{0}, kw}, {ks, kw}}) { + for (auto [row, col] : {std::pair{size_t{0}, size_t{0}}, + {ks, size_t{0}}, + {size_t{0}, kw}, + {ks, kw}}) { block[row * nw + col] ^= 0xff; const auto repaired = code.Correct(block); EXPECT_TRUE(repaired.all_zero_syndromes); EXPECT_EQ(repaired.changed_symbols, 1u); + EXPECT_EQ(repaired.changed_bits, 8u); + EXPECT_EQ(repaired.strong_changed_bits, 8u); + EXPECT_EQ(repaired.weak_changed_bits, 0u); EXPECT_EQ(block, scalar); } } @@ -191,7 +224,9 @@ TEST(ProductCode, DefaultStrongRadiusRepairsAllColumnsIncludingParity) { StrongWeakRSProductCode code; std::vector block(code.BlockSize()); std::mt19937 random(1983); - for (auto& value : block) value = static_cast(random()); + for (auto& value : block) { + value = static_cast(random()); + } ASSERT_EQ(code.Encode(block, Backend::scalar), Status::ok); const auto original = block; for (size_t col = 0; col < 256; ++col) { @@ -249,7 +284,9 @@ TEST(ProductCode, ChangesPropagateThroughFourDirectionalPasses) { TEST(ProductCode, SuccessfulStrongRepairProtectsItsColumn) { StrongWeakRSProductCode code(4, 2, 8, 6); std::vector block(32); - for (size_t row = 0; row < 4; ++row) block[row * 8] = 1; + for (size_t row = 0; row < 4; ++row) { + block[row * 8] = 1; + } const auto protected_word = block; block[0] ^= 2; const auto result = code.Correct(block); @@ -272,11 +309,16 @@ TEST(ProductCode, UnvisitedColumnRetainsProtectionOnLaterWeakPass) { recovery[i] = &protected_column[4 + i]; } std::vector workspace(encoder.WorkspaceSize(1)); - ASSERT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + ASSERT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), + Status::ok); std::vector block(64); - for (size_t row = 0; row < 8; ++row) block[row * 8 + 1] = protected_column[row]; + for (size_t row = 0; row < 8; ++row) { + block[row * 8 + 1] = protected_column[row]; + } const auto expected = block; - for (size_t row = 0; row < 4; ++row) block[row * 8] = 1; + for (size_t row = 0; row < 4; ++row) { + block[row * 8] = 1; + } const auto result = code.Correct(block); EXPECT_EQ(result.directional_passes, 4u); EXPECT_EQ(result.strong_lines_visited, 9u); @@ -304,16 +346,23 @@ TEST(WholeCodeword, OverRadiusOutcomesAreTransactionalOrVerifiedCodewords) { continue; } size_t distance = 0; - for (size_t pos = 0; pos < n; ++pos) distance += word[pos] != before[pos]; + for (size_t pos = 0; pos < n; ++pos) { + distance += word[pos] != before[pos]; + } EXPECT_EQ(distance, result.error_count); EXPECT_LE(distance, (n - k) / 2); std::vector data(k); std::vector parity(n - k); std::vector recovery(n - k); - for (size_t i = 0; i < k; ++i) data[i] = &word[i]; - for (size_t i = 0; i < n - k; ++i) recovery[i] = &parity[i]; + for (size_t i = 0; i < k; ++i) { + data[i] = &word[i]; + } + for (size_t i = 0; i < n - k; ++i) { + recovery[i] = &parity[i]; + } std::vector workspace(encoder.WorkspaceSize(1)); - ASSERT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); + ASSERT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), + Status::ok); EXPECT_TRUE(std::equal(parity.begin(), parity.end(), word.begin() + k)); } } @@ -324,7 +373,9 @@ TEST(ProductCode, ProtectedColumnRejectsEvenLowWeightWeakCandidate) { // Constant columns are valid RS words. Every row proposes a one-bit repair // at column 0, but its zero-syndrome strong protection must reject them. std::vector block(32); - for (size_t row = 0; row < 4; ++row) block[row * 8] = 1; + for (size_t row = 0; row < 4; ++row) { + block[row * 8] = 1; + } const auto before = block; const auto result = code.Correct(block); EXPECT_FALSE(result.all_zero_syndromes); @@ -336,8 +387,12 @@ TEST(ProductCode, ProtectedColumnRejectsEvenLowWeightWeakCandidate) { TEST(WholeCodewordBatch, DifferentialDataParityFailuresAndTails) { using detail::error_correction::CorrectCodewordBatch; std::mt19937 random(0xba7c224); - for (const auto [n, k] : {std::pair{4u, 2u}, {8u, 4u}, {16u, 12u}, - {256u, 128u}, {256u, 224u}, {256u, 254u}}) { + for (const auto [n, k] : {std::pair{4u, 2u}, + {8u, 4u}, + {16u, 12u}, + {256u, 128u}, + {256u, 224u}, + {256u, 254u}}) { LCHDecoder decoder(k, n - k); for (size_t lanes : {1u, 31u, 32u, 33u, 65u, 256u}) { std::vector packed(n * lanes); @@ -345,26 +400,41 @@ TEST(WholeCodewordBatch, DifferentialDataParityFailuresAndTails) { std::vector results(lanes), reference(lanes); std::vector masks(packed.size(), 0xff); std::vector shards(n); - for (size_t pos = 0; pos < n; ++pos) shards[pos] = &packed[pos * lanes]; + for (size_t pos = 0; pos < n; ++pos) { + shards[pos] = &packed[pos * lanes]; + } for (size_t lane = 0; lane < lanes; ++lane) { auto word = Codeword(n, k, random()); const size_t radius = (n - k) / 2; - const size_t errors = lane % 6 == 0 ? 0 : lane % 6 == 1 ? 1 - : lane % 6 == 2 ? radius : lane % 6 == 3 ? radius + 1 - : lane % 6 == 4 ? n : radius; + const size_t errors = lane % 6 == 0 ? 0 + : lane % 6 == 1 ? 1 + : lane % 6 == 2 ? radius + : lane % 6 == 3 ? radius + 1 + : lane % 6 == 4 ? n + : radius; std::vector positions(n); - for (size_t i = 0; i < n; ++i) positions[i] = i; + for (size_t i = 0; i < n; ++i) { + positions[i] = i; + } std::shuffle(positions.begin(), positions.end(), random); // Include parity-only full-radius candidates in every vector chunk. - if (lane % 6 == 5) std::sort(positions.rbegin(), positions.rend()); + if (lane % 6 == 5) { + std::sort(positions.rbegin(), positions.rend()); + } for (size_t i = 0; i < errors; ++i) { word[positions[i]] ^= static_cast(1 + random() % 255); } - for (size_t pos = 0; pos < n; ++pos) packed[pos * lanes + lane] = word[pos]; + for (size_t pos = 0; pos < n; ++pos) { + packed[pos * lanes + lane] = word[pos]; + } const auto before = word; reference[lane] = CorrectCodeword(decoder, word); - if (reference[lane].status != CorrectionStatus::ok) EXPECT_EQ(word, before); - for (size_t pos = 0; pos < n; ++pos) expected[pos * lanes + lane] = word[pos]; + if (reference[lane].status != CorrectionStatus::ok) { + EXPECT_EQ(word, before); + } + for (size_t pos = 0; pos < n; ++pos) { + expected[pos * lanes + lane] = word[pos]; + } } const auto before = packed; ASSERT_EQ(CorrectCodewordBatch(decoder, shards, lanes, results, masks), @@ -372,7 +442,8 @@ TEST(WholeCodewordBatch, DifferentialDataParityFailuresAndTails) { ASSERT_EQ(packed, expected) << n << ':' << lanes; for (size_t lane = 0; lane < lanes; ++lane) { EXPECT_EQ(results[lane].status, reference[lane].status) << lane; - EXPECT_EQ(results[lane].error_count, reference[lane].error_count) << lane; + EXPECT_EQ(results[lane].error_count, reference[lane].error_count) + << lane; for (size_t pos = 0; pos < n; ++pos) { const auto index = pos * lanes + lane; EXPECT_EQ(masks[index], before[index] != packed[index]); @@ -387,7 +458,9 @@ TEST(WholeCodewordBatch, InvalidRangesAreUntouched) { LCHDecoder decoder(6, 2); std::vector packed(8 * 33, 7); std::array shards{}; - for (size_t i = 0; i < 8; ++i) shards[i] = packed.data() + i * 33; + for (size_t i = 0; i < 8; ++i) { + shards[i] = packed.data() + i * 33; + } std::vector results(33); std::vector masks(packed.size(), 42); const auto before = packed; @@ -404,15 +477,21 @@ TEST(WholeCodewordBatch, InvalidRangesAreUntouched) { CorrectionStatus::invalid_argument); EXPECT_EQ(CorrectCodewordBatch(LCHDecoder(5, 3), shards, 33, results, masks), CorrectionStatus::unsupported_dimensions); - EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 0, {}, {}), CorrectionStatus::ok); + EXPECT_EQ(CorrectCodewordBatch(decoder, shards, 0, {}, {}), + CorrectionStatus::ok); EXPECT_EQ(packed, before); EXPECT_EQ(masks, std::vector(packed.size(), 42)); - for (const auto& result : results) EXPECT_EQ(result.status, CorrectionStatus::invalid_argument); + for (const auto& result : results) { + EXPECT_EQ(result.status, CorrectionStatus::invalid_argument); + } } // Independent parity oracle: no decoder outcomes or scheduler bookkeeping. bool AllComponentsValid(const std::vector& block, - size_t ns, size_t ks, size_t nw, size_t kw) { + size_t ns, + size_t ks, + size_t nw, + size_t kw) { bool valid = true; for (bool strong : {true, false}) { const size_t n = strong ? ns : nw, k = strong ? ks : kw; @@ -421,14 +500,21 @@ bool AllComponentsValid(const std::vector& block, std::vector data(k); std::vector recovery(n - k); std::vector workspace(encoder.WorkspaceSize(1)); - for (size_t i = 0; i < n - k; ++i) recovery[i] = &parity[i]; + for (size_t i = 0; i < n - k; ++i) { + recovery[i] = &parity[i]; + } for (size_t line = 0; line < (strong ? nw : ns); ++line) { const auto index = [&](size_t pos) { return strong ? pos * nw + line : line * nw + pos; }; - for (size_t i = 0; i < k; ++i) data[i] = &block[index(i)]; - EXPECT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), Status::ok); - for (size_t i = k; i < n; ++i) valid &= parity[i - k] == block[index(i)]; + for (size_t i = 0; i < k; ++i) { + data[i] = &block[index(i)]; + } + EXPECT_EQ(encoder.Encode(data, recovery, 1, workspace, Backend::scalar), + Status::ok); + for (size_t i = k; i < n; ++i) { + valid &= parity[i - k] == block[index(i)]; + } } } return valid; @@ -437,54 +523,84 @@ bool AllComponentsValid(const std::vector& block, TEST(ProductCode, InitialBatchChoicesMatchSingleOutputsAndAllCounters) { std::mt19937 random(0x5b5c0224); for (const auto dims : {std::array{4, 2, 8, 6}, - {32, 16, 64, 62}, {256, 224, 256, 254}}) { + {32, 16, 64, 62}, + {256, 224, 256, 254}}) { const auto [ns, ks, nw, kw] = dims; StrongWeakRSProductCode code(ns, ks, nw, kw); for (size_t trial = 0; trial < 16; ++trial) { std::vector input(code.BlockSize()); - for (auto& value : input) value = static_cast(random()); + for (auto& value : input) { + value = static_cast(random()); + } ASSERT_EQ(code.Encode(input), Status::ok); if (trial < 8) { for (auto& value : input) { for (unsigned bit = 0; bit < 8; ++bit) { - if (random() % 200 == 0) value ^= static_cast(1u << bit); + if (random() % 200 == 0) { + value ^= static_cast(1u << bit); + } } } } else { // Valid strong columns propose weak repairs into protected columns; // overloaded columns exercise rejection, bit gates, and activation. std::fill(input.begin(), input.end(), Element{0}); - for (size_t row = 0; row < ns; ++row) input[row * nw] = 1; + for (size_t row = 0; row < ns; ++row) { + input[row * nw] = 1; + } for (size_t row = 0; row <= (ns - ks) / 2; ++row) { input[row * nw + 1] = trial % 2 ? 7 : 3; - if (row % 2) input[row * nw + 2] = 1; + if (row % 2) { + input[row * nw + 2] = 1; + } + } + if (trial % 3 == 0) { + input[0] ^= 2; } - if (trial % 3 == 0) input[0] ^= 2; } for (size_t cap : {2u, 3u, 4u, 5u, 6u, 16u}) { - auto reference = input; - const auto expected = detail::ProductCorrectionAccess::Correct(code, reference, cap, 0, false); - EXPECT_EQ(expected.all_zero_syndromes, AllComponentsValid(reference, ns, ks, nw, kw)); - for (unsigned batches : {0u, 1u, 2u}) { - auto actual = input; - const auto result = detail::ProductCorrectionAccess::Correct(code, actual, cap, batches); - ASSERT_EQ(actual, reference) << ns << ':' << trial << ':' << cap << ':' << batches; - EXPECT_EQ(result.termination, expected.termination); - EXPECT_EQ(result.all_zero_syndromes, expected.all_zero_syndromes); - EXPECT_EQ(result.directional_passes, expected.directional_passes); - EXPECT_EQ(result.strong_lines_visited, expected.strong_lines_visited); - EXPECT_EQ(result.weak_lines_visited, expected.weak_lines_visited); - EXPECT_EQ(result.changed_symbols, expected.changed_symbols); + for (bool anchors : {false, true}) { + for (bool binary : {false, true}) { + const ProductDecodeOptions options{cap, anchors, binary}; + auto reference = input; + const auto expected = detail::ProductCorrectionAccess::Correct( + code, reference, options, 0, false); + EXPECT_EQ(expected.all_zero_syndromes, + AllComponentsValid(reference, ns, ks, nw, kw)); + for (unsigned batches : {0u, 1u, 2u}) { + auto actual = input; + const auto result = detail::ProductCorrectionAccess::Correct( + code, actual, options, batches); + ASSERT_EQ(actual, reference) + << ns << ':' << trial << ':' << cap << ':' << batches; + EXPECT_EQ(result.termination, expected.termination); + EXPECT_EQ(result.all_zero_syndromes, expected.all_zero_syndromes); + EXPECT_EQ(result.directional_passes, expected.directional_passes); + EXPECT_EQ(result.strong_lines_visited, + expected.strong_lines_visited); + EXPECT_EQ(result.weak_lines_visited, expected.weak_lines_visited); + EXPECT_EQ(result.changed_symbols, expected.changed_symbols); + EXPECT_EQ(result.changed_bits, expected.changed_bits); + EXPECT_EQ(result.strong_changed_bits, + expected.strong_changed_bits); + EXPECT_EQ(result.weak_changed_bits, expected.weak_changed_bits); + EXPECT_EQ(result.strong_changed_symbols, + expected.strong_changed_symbols); + EXPECT_EQ(result.weak_changed_symbols, + expected.weak_changed_symbols); + } + } } } } } } -TEST(ProductCode, TrackedValidityMatchesIndependentParityAcrossCapsAndCancellations) { +TEST(ProductCode, + TrackedValidityMatchesIndependentParityAcrossCapsAndCancellations) { std::mt19937 random(0xc1ea0224); - for (const auto dims : {std::array{4, 2, 8, 6}, - {8, 4, 4, 2}, {32, 28, 32, 30}}) { + for (const auto dims : + {std::array{4, 2, 8, 6}, {8, 4, 4, 2}, {32, 28, 32, 30}}) { const auto [ns, ks, nw, kw] = dims; StrongWeakRSProductCode code(ns, ks, nw, kw); for (size_t trial = 0; trial < 128; ++trial) { @@ -492,32 +608,244 @@ TEST(ProductCode, TrackedValidityMatchesIndependentParityAcrossCapsAndCancellati // Include undetected valid words, equal-magnitude cancellations, // parity damage, dense failures, and both weak rejection gates. if (trial % 4 == 0) { - for (auto& value : input) value = static_cast(random()); + for (auto& value : input) { + value = static_cast(random()); + } ASSERT_EQ(code.Encode(input, Backend::scalar), Status::ok); } const size_t errors = trial % (ns * 2); for (size_t i = 0; i < errors; ++i) { - input[random() % input.size()] ^= trial % 3 == 0 ? Element{1} - : trial % 3 == 1 ? Element{7} : static_cast(1 + random() % 255); + input[random() % input.size()] ^= + trial % 3 == 0 ? Element{1} + : trial % 3 == 1 ? Element{7} + : static_cast(1 + random() % 255); } for (size_t cap : {2u, 3u, 4u, 5u, 6u, 7u, 8u, 16u}) { - auto reference = input; - const auto expected = detail::ProductCorrectionAccess::Correct(code, reference, cap, 0, false); - const bool valid = AllComponentsValid(reference, ns, ks, nw, kw); - EXPECT_EQ(expected.all_zero_syndromes, valid); + for (bool anchors : {false, true}) { + for (bool binary : {false, true}) { + const ProductDecodeOptions options{cap, anchors, binary}; + auto reference = input; + const auto expected = detail::ProductCorrectionAccess::Correct( + code, reference, options, 0, false); + const bool valid = AllComponentsValid(reference, ns, ks, nw, kw); + EXPECT_EQ(expected.all_zero_syndromes, valid); + for (unsigned batches : {0u, 1u, 2u, 3u}) { + auto actual = input; + const auto result = + batches == 3 ? code.Correct(actual, options) + : detail::ProductCorrectionAccess::Correct( + code, actual, options, batches); + ASSERT_EQ(actual, reference) + << ns << ':' << trial << ':' << cap << ':' << batches; + EXPECT_EQ(result.all_zero_syndromes, valid); + EXPECT_EQ(result.termination, expected.termination); + EXPECT_EQ(result.directional_passes, expected.directional_passes); + EXPECT_EQ(result.strong_lines_visited, + expected.strong_lines_visited); + EXPECT_EQ(result.weak_lines_visited, expected.weak_lines_visited); + EXPECT_EQ(result.changed_symbols, expected.changed_symbols); + EXPECT_EQ(result.changed_bits, expected.changed_bits); + EXPECT_EQ(result.strong_changed_bits, + expected.strong_changed_bits); + EXPECT_EQ(result.weak_changed_bits, expected.weak_changed_bits); + EXPECT_EQ(result.strong_changed_symbols, + expected.strong_changed_symbols); + EXPECT_EQ(result.weak_changed_symbols, + expected.weak_changed_symbols); + } + } + } + } + } + } +} + +TEST(ProductCode, IndependentGatesProtectParityAndKeepSingleSymbolBDD) { + for (size_t n : {4u, 32u}) { + StrongWeakRSProductCode code(n, n - 2, n, n - 2); + for (bool anchors : {false, true}) { + for (bool binary : {false, true}) { for (unsigned batches : {0u, 1u, 2u, 3u}) { - auto actual = input; - const auto result = batches == 3 ? code.Correct(actual, cap) - : detail::ProductCorrectionAccess::Correct(code, actual, cap, batches); - ASSERT_EQ(actual, reference) << ns << ':' << trial << ':' << cap << ':' << batches; - EXPECT_EQ(result.all_zero_syndromes, valid); - EXPECT_EQ(result.termination, expected.termination); - EXPECT_EQ(result.directional_passes, expected.directional_passes); - EXPECT_EQ(result.strong_lines_visited, expected.strong_lines_visited); - EXPECT_EQ(result.weak_lines_visited, expected.weak_lines_visited); - EXPECT_EQ(result.changed_symbols, expected.changed_symbols); + for (size_t cap : {2u, 16u}) { + const ProductDecodeOptions options{cap, anchors, binary}; + const auto correct = [&](std::vector& block) { + return batches == 3 ? code.Correct(block, options) + : detail::ProductCorrectionAccess::Correct( + code, block, options, batches); + }; + for (Element magnitude : {Element{1}, Element{7}}) { + // A clean strong parity column: each weak row proposes one + // repair. + std::vector block(n * n); + for (size_t row = 0; row < n; ++row) { + block[row * n + n - 1] = magnitude; + } + const auto before = block; + const bool accept = !anchors && (!binary || magnitude == 1); + const auto result = correct(block); + EXPECT_EQ(block, accept ? std::vector(n * n) : before); + EXPECT_EQ(result.changed_symbols, accept ? n : 0u); + EXPECT_EQ(result.changed_bits, + accept ? n * std::popcount(magnitude) : 0u); + EXPECT_EQ(result.weak_changed_bits, result.changed_bits); + EXPECT_EQ(result.strong_changed_bits, 0u); + EXPECT_EQ(result.all_zero_syndromes, accept); + EXPECT_EQ(result.directional_passes, accept && cap > 2 ? 3u : 2u); + EXPECT_EQ(result.termination, + accept && cap == 2 ? ProductTermination::pass_limit + : ProductTermination::no_change); + + // Strong failure leaves this parity column unprotected. Only the + // binary-image option may reject these weak repairs. + block.assign(n * n, 0); + block[(n - 2) * n + n - 1] = magnitude; + block[(n - 1) * n + n - 1] = magnitude; + const auto damaged = block; + const bool repair = !binary || magnitude == 1; + const auto unprotected = correct(block); + EXPECT_EQ(block, repair ? std::vector(n * n) : damaged); + EXPECT_EQ(unprotected.changed_symbols, repair ? 2u : 0u); + EXPECT_EQ(unprotected.changed_bits, + repair ? 2u * std::popcount(magnitude) : 0u); + EXPECT_EQ(unprotected.all_zero_syndromes, repair); + EXPECT_EQ(unprotected.strong_lines_visited, + n + (repair && cap > 2 ? 1 : 0)); + } + + // Equal errors in a 2x2 corner defeat both one-symbol component + // decoders even when both acceptance gates are disabled. + std::vector block(n * n); + for (size_t row : {n - 2, n - 1}) { + for (size_t col : {n - 2, n - 1}) { + block[row * n + col] = 1; + } + } + const auto before = block; + const auto result = correct(block); + EXPECT_EQ(block, before); + EXPECT_EQ(result.changed_symbols, 0u); + EXPECT_FALSE(result.all_zero_syndromes); + } + } + } + } + } +} + +TEST(ProductCode, AnchorFreeWritesInvalidatePreviouslyCleanColumnsAtCap) { + for (size_t n : {4u, 32u}) { + StrongWeakRSProductCode code(n, n - 2, n, n - 2); + for (bool binary : {false, true}) { + for (unsigned batches : {0u, 1u, 2u, 3u}) { + std::vector block(n * n); + for (size_t row = 0; row < n; ++row) { + block[row * n] = 1; } + block[1] = block[n + 1] = 1; + // Column 0 starts clean. Weak repairs only rows 2..N-1, making + // column 0 invalid. Cached strong success must not hide those writes. + const ProductDecodeOptions options{2, false, binary}; + const auto result = batches == 3 + ? code.Correct(block, options) + : detail::ProductCorrectionAccess::Correct( + code, block, options, batches); + EXPECT_EQ(result.changed_symbols, n - 2); + EXPECT_EQ(result.termination, ProductTermination::pass_limit); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_FALSE(AllComponentsValid(block, n, n - 2, n, n - 2)); + } + } + } +} + +TEST(ProductCode, CountsRepeatedCommittedWritesByIndependentPassDifferences) { + StrongWeakRSProductCode code(4, 2, 8, 6); + LCHDecoder strong(2, 2); + std::mt19937 random(0xacc37); + bool repeated = false; + for (size_t trial = 0; trial < 512; ++trial) { + std::vector input(32); + for (size_t i = 0; i < 12; ++i) { + input[random() % input.size()] ^= static_cast(1 + random() % 7); + } + auto previous = input; + // Independently reproduce just the initial strong pass. Each following + // cap extends the same deterministic prefix by one directional pass. + for (size_t col = 0; col < 8; ++col) { + std::array column{}; + for (size_t row = 0; row < 4; ++row) { + column[row] = input[row * 8 + col]; + } + CorrectCodeword(strong, column); + for (size_t row = 0; row < 4; ++row) { + previous[row * 8 + col] = column[row]; + } + } + size_t bits = 0, symbols = 0, strong_bits = 0, strong_symbols = 0; + std::array writes{}; + for (size_t pos = 0; pos < 32; ++pos) { + bits += std::popcount(static_cast(input[pos] ^ previous[pos])); + symbols += input[pos] != previous[pos]; + writes[pos] += input[pos] != previous[pos]; + } + strong_bits = bits; + strong_symbols = symbols; + for (size_t cap = 2; cap <= 8; ++cap) { + auto actual = input; + const auto result = + code.Correct(actual, ProductDecodeOptions{cap, false, false}); + for (size_t pos = 0; pos < 32; ++pos) { + const size_t delta_bits = + std::popcount(static_cast(previous[pos] ^ actual[pos])); + const size_t delta_symbols = previous[pos] != actual[pos]; + bits += delta_bits; + symbols += delta_symbols; + if (cap % 2 == 1) { + strong_bits += delta_bits; + strong_symbols += delta_symbols; + } + writes[pos] += delta_symbols; + repeated |= writes[pos] > 1; + } + EXPECT_EQ(result.changed_bits, bits); + EXPECT_EQ(result.changed_symbols, symbols); + EXPECT_EQ(result.strong_changed_bits, strong_bits); + EXPECT_EQ(result.strong_changed_symbols, strong_symbols); + EXPECT_EQ(result.weak_changed_bits, bits - strong_bits); + EXPECT_EQ(result.weak_changed_symbols, symbols - strong_symbols); + previous = actual; + } + } + EXPECT_TRUE(repeated); +} + +TEST(ProductCode, OptionsDefaultsAndInvalidCapsArePerCall) { + StrongWeakRSProductCode code(4, 2, 8, 6); + std::vector input(32); + input[6] = input[30] = 7; + for (bool anchors : {false, true}) { + for (bool binary : {false, true}) { + for (size_t cap : {0u, 1u}) { + auto block = input; + const auto result = + code.Correct(block, ProductDecodeOptions{cap, anchors, binary}); + EXPECT_EQ(result.termination, ProductTermination::invalid_argument); + EXPECT_EQ(result.directional_passes, 0u); + EXPECT_FALSE(result.all_zero_syndromes); + EXPECT_EQ(block, input); } + auto block = input; + code.Correct(block, ProductDecodeOptions{16, anchors, binary}); + auto defaults = input; + auto legacy = input; + const auto result = code.Correct(defaults, ProductDecodeOptions{}); + const auto old = code.Correct(legacy); + EXPECT_EQ(defaults, input); + EXPECT_EQ(defaults, legacy); + EXPECT_EQ(result.termination, old.termination); + EXPECT_EQ(result.all_zero_syndromes, old.all_zero_syndromes); + EXPECT_EQ(result.directional_passes, old.directional_passes); + EXPECT_EQ(result.changed_symbols, old.changed_symbols); } } } diff --git a/tests/product_monte_carlo_data_tests.cc b/tests/product_monte_carlo_data_tests.cc new file mode 100644 index 0000000..84e4e15 --- /dev/null +++ b/tests/product_monte_carlo_data_tests.cc @@ -0,0 +1,42 @@ +#include "product_monte_carlo_data.h" + +#include + +TEST(ProductMonteCarloData, ExactUnboundedIntegers) { + const auto n = mc::Big::from_string( + "184467440737095516160000000000000000000000000000001"); + mc::Stats s; + s.blocks = n; + s.iterations = n * 2; + s.info_raw = n; + s.full_raw = n; + s.info_post = 0; + s.full_post = 0; + auto json = s.ToJson(); + auto text = mc::Dump(json); + EXPECT_NE(text.find(n.to_string()), std::string::npos); + auto parsed = mc::Parse(text); + auto recovered = mc::Stats::FromJson(parsed, n, 1, 16); + recovered.Add(s); + EXPECT_EQ(recovered.blocks, n * 2); + EXPECT_EQ( + mc::Natural(recovered.ToJson().at("information bits").at("total bits")), + n * 910336); + EXPECT_EQ(mc::Dump(mc::Parse(mc::Dump(recovered.ToJson(), true))), + mc::Dump(recovered.ToJson())); +} + +TEST(ProductMonteCarloData, StrictJsonAndNaturalNumbers) { + for (const auto* text : + {"{\"a\":1,\"a\":2}", "{\"a\":{\"b\":0,\"b\":1}}", "1.0", "1e3", "NaN", + "/*comment*/1", "[1,]", "1 2"}) { + EXPECT_THROW(mc::Parse(text), std::exception) << text; + } + for (const auto* text : {"true", "\"123\"", "-1", "-184467440737095516160"}) { + EXPECT_THROW(mc::Natural(mc::Parse(text)), std::exception) << text; + } + EXPECT_EQ(mc::U64(mc::Parse("18446744073709551615")), UINT64_MAX); + EXPECT_THROW(mc::U64(mc::Parse("18446744073709551616")), std::exception); + EXPECT_EQ(mc::Dump(mc::Parse("{\"z\":1,\"a\":\"x\"}")), + "{\"a\":\"x\",\"z\":1}"); +} diff --git a/tests/product_monte_carlo_legacy_test.py b/tests/product_monte_carlo_legacy_test.py new file mode 100644 index 0000000..c1f997f --- /dev/null +++ b/tests/product_monte_carlo_legacy_test.py @@ -0,0 +1,662 @@ +"""Legacy coordinator and C ABI regression tests (test-only reference).""" + +import hashlib +import importlib.machinery +import importlib.util +import io +import json +import math +import os +from pathlib import Path +import pty +import re +import select +import shutil +import signal +import subprocess +import struct +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +CLI = Path(sys.argv.pop(1)).resolve() +loader = importlib.machinery.SourceFileLoader("experiment", str(CLI)) +spec = importlib.util.spec_from_loader(loader.name, loader) +experiment = importlib.util.module_from_spec(spec) +loader.exec_module(experiment) + + +class ExperimentTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="product-monte-carlo-") + self.root = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def invoke(self, *args, success=True): + result = subprocess.run([sys.executable, str(CLI), *map(str, args)], + capture_output=True, text=True, timeout=90) + self.assertEqual(result.returncode == 0, success, result.stderr) + return result + + def run_case(self, name, *args): + path = self.root / name + self.invoke("--output", path, "--seed", "18446744073709551615", + "--batches", "2", "--batch-size", "3", "--checkpoint-trials", "3", *args) + return path + + def read(self, path, name="summary.json"): + return json.loads((path / name).read_text()) + + def scientific(self, path): + result = self.read(path) + del result["run identity"] + return result + + def test_reproducible_threads_and_recovery(self): + args = ("--minimum-flipped-bits", "2590", "--maximum-flipped-bits", "2610") + a = self.run_case("a", "--threads", "1", *args) + b = self.run_case("b", "--threads", "3", *args) + c = self.run_case("c", "--threads", "3", *args) + self.assertEqual(self.scientific(a), self.scientific(b)) + self.assertEqual(self.scientific(a), self.scientific(c)) + before = self.read(b) + self.invoke("--report", b) + self.assertEqual(before, self.read(b)) + with (b / "journal.jsonl").open("ab") as stream: + stream.write(b'{"incomplete crash tail') + self.invoke("--report", b) + self.assertEqual(before, self.read(b)) + self.assertEqual(before["overall"]["trial count"], 6) + chunked = [self.run_case(f"chunked-{threads}", "--threads", threads, + "--checkpoint-trials", 64, "--batch-size", 17, *args) + for threads in (1, 4, 16)] + self.assertEqual(self.scientific(chunked[0]), self.scientific(chunked[1])) + self.assertEqual(self.scientific(chunked[0]), self.scientific(chunked[2])) + + def test_exact_channel_oracles_and_flags(self): + for k in (0, 1, 524287, 524288): + path = self.run_case(str(k), "--minimum-flipped-bits", k, + "--maximum-flipped-bits", k, "--batches", "1", + "--batch-size", "1", "--no-anchors", "--no-binary-image") + self.invoke("--report", path) + summary = self.read(path)["overall"] + stats = summary["statistics"] + self.assertEqual(stats["initial full block corrupted bits"], {"sum": k, "squared sum": k*k}) + if k <= 1: + self.assertEqual(stats["accepted bit changes"]["sum"], k) + self.assertEqual(stats["accepted byte changes"]["sum"], k) + self.assertEqual(stats["residual full block bits"]["sum"], 0) + else: + self.assertEqual(stats["initial full block corrupted bytes"]["sum"], 65536) + # The all-ones difference is itself a product codeword. With + # one bit absent BDD repairs to that wrong valid codeword. + self.assertEqual(stats["residual full block bits"]["sum"], 524288) + self.assertEqual(stats["residual information bits"]["sum"], 455168) + self.assertEqual(stats["accepted bit changes"]["sum"], 524288 - k) + self.assertEqual(stats["zero syndrome wrong full blocks"]["sum"], 1) + settings = self.read(path, "metadata.json")["settings"] + self.assertFalse(settings["anchors"]) + self.assertFalse(settings["binary image"]) + for anchors in ("--anchors", "--no-anchors"): + for binary in ("--binary-image", "--no-binary-image"): + path = self.run_case(anchors + binary, anchors, binary, + "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0") + self.assertEqual(self.read(path)["overall"]["statistics"]["directional passes"], + {"sum": 12, "squared sum": 24}) + + def test_native_known_answers_and_worker_reuse(self): + lib = experiment.native() + weights = (0, 1, 2590, 2600, 2610, 262143, 262144, 262145, 524287, 524288) + + def trials(_): + # Reuse both the ctypes output and native thread-local scratch across + # sparse/dense noise and different decoder gates. + output = (experiment.ctypes.c_uint64 * len(experiment.METRICS))() + values = [] + for index, k in enumerate(weights): + for i in range(len(output)): + output[i] = experiment.U64 + status = lib.product_trial(42, 7, index, k, 16, + index % 2, (index // 2) % 2, output) + self.assertEqual(status, 0) + self.assertEqual(output[0], k) + values.extend(output) + return hashlib.sha256(struct.pack("<" + "Q" * len(values), *values)).hexdigest() + + expected = "4a4257dff8b039967b33aba2a92dc387a8ccc5b05b39a7d91680910c6175c61d" + self.assertEqual(trials(0), expected) + with experiment.concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + self.assertEqual(list(pool.map(trials, range(6))), [expected] * 6) + + def test_native_chunk_prefix_and_replay(self): + lib = experiment.native() + c = experiment.ctypes + lib.product_interrupt_install() + output = (c.c_uint64 * 88)() + single = (c.c_uint64 * 22)() + count = c.c_uint64() + for k in (0, 2600, 524288): + stride = min(k, 524288 - k) + for sampler in (0, 1): + positions = (c.c_uint32 * (4 * stride))() + self.assertEqual(lib.product_trials(42, 0, 7, k, 16, 1, 1, + output, 4, sampler, positions, c.byref(count)), 0) + self.assertEqual(count.value, 4) + for i in range(4): + saved = (c.c_uint32 * stride).from_buffer(positions, i * stride * 4) + self.assertEqual(lib.product_trial_flips(42, 0, 7 + i, k, 16, 1, 1, + single, 2, saved), 0) + self.assertEqual(list(single), list(output[i * 22:(i + 1) * 22])) + before = list(output) + for first, size, passes in ((0, 5, 16), (experiment.U64, 2, 16), (0, 4, 1)): + self.assertNotEqual(lib.product_trials(42, 0, first, 2600, passes, 1, 1, + output, size, 0, None, c.byref(count)), 0) + self.assertEqual(count.value, 0) + self.assertEqual(list(output), before) + + def test_chunk_failure_preserves_prefix_and_later_successes(self): + source = self.run_case("chunk-settings", "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0") + settings = self.read(source, "metadata.json")["settings"] + settings.update({"threads": 3, "checkpoint trials": 12, "batch size": 12, "batches": 1}) + lib = experiment.native() + barrier = threading.Barrier(3) + + class FailChunk: + def __getattr__(self, name): + return getattr(lib, name) + + def product_trials(self, *args): + barrier.wait(timeout=10) + if args[2] == 4: + args = list(args) + args[8] = 1 + self_status = lib.product_trials(*args) + assert self_status == 0 + return 9 + return lib.product_trials(*args) + + for sampler in ("floyd", "fisher-yates"): + path = self.root / sampler + path.mkdir() + with self.assertRaisesRegex(RuntimeError, "index=5 failed: 9"): + experiment.run(path, settings, FailChunk(), sampler) + records = [json.loads(line) for line in (path / "journal.jsonl").read_text().splitlines()] + indices = [i for r in records for i in range(r["first trial index"], r["past last trial index"])] + self.assertEqual(indices, list(range(5)) + list(range(8, 12))) + self.invoke("--replay" if sampler == "fisher-yates" else "--report", path) + + def test_slow_first_trial_bounds_refill_and_orders_journal(self): + source = self.run_case("window-settings", "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0") + settings = self.read(source, "metadata.json")["settings"] + settings.update({"threads": 3, "checkpoint trials": 6, "batch size": 13, "batches": 1}) + submitted = [] + released_first = False + + def submit(function, index, k): + if not released_first: + self.assertLess(index, 6) + submitted.append(index) + future = experiment.concurrent.futures.Future() + future.index = index + future.set_result(function(index, k)) + return future + + def wait(pending, **kwargs): + nonlocal released_first + latest = max(pending, key=lambda f: f.index) + if not released_first and latest.index < 2: + self.assertEqual(submitted, list(range(6))) + latest = min(pending, key=lambda f: f.index) + released_first = True + return {latest}, pending - {latest} + + pool = mock.MagicMock() + pool.__enter__.return_value.submit.side_effect = submit + path = self.root / "window" + path.mkdir() + with mock.patch.object(experiment.concurrent.futures, "ThreadPoolExecutor", return_value=pool), \ + mock.patch.object(experiment.concurrent.futures, "wait", side_effect=wait): + experiment.run(path, settings, experiment.native()) + self.assertTrue(released_first) + self.assertEqual(submitted, list(range(13))) + records = [json.loads(line) for line in (path / "journal.jsonl").read_text().splitlines()] + self.assertTrue(all(r["trial count"] <= 6 for r in records)) + self.assertEqual([i for r in records for i in range(r["first trial index"], r["past last trial index"])], list(range(13))) + self.invoke("--report", path) + + def test_invalid_inputs_and_no_overwrite(self): + for flags in (("--minimum-flipped-bits", "5", "--maximum-flipped-bits", "4"), + ("--maximum-flipped-bits", "524289"), ("--seed", "18446744073709551616"), + ("--seed", "-1"), ("--threads", "0"), ("--threads", "1025"), + ("--max-directional-passes", "1"), ("--batch-size", "0"), + ("--checkpoint-trials", "0"), ("--fsync-seconds", "0")): + self.invoke("--output", self.root / "invalid", *flags, success=False) + self.assertFalse((self.root / "invalid").exists()) + path = self.run_case("existing", "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0") + before = (path / "journal.jsonl").read_bytes() + self.invoke("--output", path, success=False) + self.assertEqual(before, (path / "journal.jsonl").read_bytes()) + + def test_codeword_translation_equivariance(self): + lib = experiment.native() + c = experiment.ctypes + output = (c.c_uint64 * 22)() + residual = (c.c_uint8 * 65536)() + # Fixed bit patterns: strong radius 16 and beyond; weak radius 1 and + # beyond in columns that strong BDD cannot repair; parity boundaries. + patterns = [[], [0]] + patterns += [[8 * (r * 256 + col) for r in range(n) for col in range(width)] + for n in (16, 17, 33) for width in (1, 2, 3)] + patterns += [[8 * (r * 256 + col) + bit for r, col, bit in + ((223, 253, 7), (223, 254, 0), (224, 253, 1), (255, 255, 7))]] + failures = wrong = capped = False + for anchors in (0, 1): + for binary in (0, 1): + for passes in (2, 3, 16): + cases = [(len(p), p) for p in patterns] + # Fixed Floyd realizations, including dense complement boundary. + for k in (2600, 4000, 262143, 262144, 262145, 524287, 524288): + positions = (c.c_uint32 * min(k, 524288-k))() + self.assertEqual(lib.product_trial_flips( + 42, 7, 9, k, passes, anchors, binary, output, 0, positions), 0) + cases.append((k, positions)) + for k, positions in cases: + positions = (c.c_uint32 * len(positions))(*positions) + expected = None + # Alternate specializations on the same native thread; do + # not let a stale random original contaminate zero trials. + for random in (0, 1, 0): + self.assertEqual(lib.product_trial_reference( + 42, 7, 9, k, passes, anchors, binary, output, + 2, positions, random, residual), 0) + actual = (list(output), bytes(residual)) + if expected is None: + expected = actual + self.assertEqual(actual, expected, + (anchors, binary, passes, k, random)) + failures |= bool(output[9] and not output[10]) + wrong |= bool(output[11]) + capped |= bool(output[21]) + if k == 524288: + self.assertEqual(bytes(residual), b"\xff" * 65536) + self.assertTrue(failures) + self.assertTrue(wrong) # Dense all-ones valid-codeword miscorrection witness. + self.assertTrue(capped) + + def test_reference_floyd_positions_and_transactionality(self): + lib = experiment.native() + c = experiment.ctypes + outputs = [(c.c_uint64 * 22)() for _ in range(2)] + positions = [(c.c_uint32 * 2600)() for _ in range(2)] + for random in (0, 1): + self.assertEqual(lib.product_trial_reference( + experiment.U64, 3, 8, 2600, 16, 1, 1, outputs[random], + 0, positions[random], random, None), 0) + self.assertEqual(list(outputs[0]), list(outputs[1])) + self.assertEqual(list(positions[0]), list(positions[1])) + output = (c.c_uint64 * 22)(*([experiment.U64] * 22)) + residual = (c.c_uint8 * 65536)(*([123] * 65536)) + for random in (0, 1, 2): + for k, passes, sampler, p in ((524289, 16, 0, None), (0, 1, 0, None), + (0, 16, 3, None), (2, 16, 2, (c.c_uint32 * 2)(4, 4)), + (1, 16, 2, (c.c_uint32 * 1)(524288))): + self.assertNotEqual(lib.product_trial_reference( + 42, 0, 0, k, passes, 1, 1, output, sampler, p, random, residual), 0) + self.assertEqual(list(output), [experiment.U64] * 22) + self.assertEqual(bytes(residual), bytes([123]) * 65536) + + def test_legacy_random_replay_without_metadata_mutation(self): + path = self.run_case("legacy", "--sampler", "fisher-yates", + "--minimum-flipped-bits", "2600", "--maximum-flipped-bits", "2600") + metadata = self.read(path, "metadata.json") + self.assertEqual(metadata.pop("codeword"), "zero") + # Translate the identity only, preserving saved positions and counters. + # Equivariance means these records are valid random-reference fixtures. + digest = experiment.identity(metadata) + experiment.atomic_json(path / "metadata.json", metadata) + journal = [json.loads(line) for line in (path / "journal.jsonl").read_text().splitlines()] + for record in journal: + record["run identity"] = digest + (path / "journal.jsonl").write_text("".join(experiment.canonical(r) + "\n" for r in journal)) + flips = (path / "flips.bin").read_bytes() + (path / "flips.bin").write_bytes(experiment.FLIP_MAGIC + bytes.fromhex(digest) + flips[40:]) + before = (path / "metadata.json").read_bytes() + lib = experiment.native() + with mock.patch.object(lib, "product_trial_reference", wraps=lib.product_trial_reference) as trial: + experiment.recover(path, lib, replay=True) + self.assertEqual(trial.call_count, 6) + self.assertTrue(all(call.args[-2] for call in trial.call_args_list)) + self.assertEqual(before, (path / "metadata.json").read_bytes()) + self.assertEqual(self.read(path)["run identity"], digest) + self.invoke("--report", path) + self.assertEqual(before, (path / "metadata.json").read_bytes()) + + def test_saved_flips_replay_and_corruption(self): + for k in (0, 1, 2600, 262144, 524287, 524288): + path = self.run_case(f"fy-{k}", "--sampler", "fisher-yates", "--threads", "3", + "--minimum-flipped-bits", k, "--maximum-flipped-bits", k) + before = self.read(path) + self.invoke("--replay", path) + self.assertEqual(before, self.read(path)) + expected_size = 40 + 6 * (experiment.FLIP_HEADER.size + 4 * min(k, 524288-k) + 32) + self.assertEqual((path / "flips.bin").stat().st_size, expected_size) + data = (path / "flips.bin").read_bytes() + for bad in (data[:-1], data[:45] + bytes([data[45] ^ 1]) + data[46:], b"bad"): + (path / "flips.bin").write_bytes(bad) + self.invoke("--report", path, success=False) + self.assertEqual(before, self.read(path)) + (path / "flips.bin").write_bytes(data + b"uncommitted partial record") + self.invoke("--replay", path) + (path / "flips.bin").unlink() + self.invoke("--report", path, success=False) + + def test_native_saved_positions_and_persistent_permutation(self): + lib = experiment.native() + output = (experiment.ctypes.c_uint64 * len(experiment.METRICS))() + replay = type(output)() + first = None + for k in (2600, 2600, 0, 1, 262143, 262144, 262145, 524287, 524288): + count = min(k, 524288-k) + positions = (experiment.ctypes.c_uint32 * count)() + self.assertEqual(lib.product_trial_flips(42, 7, 0, k, 16, 1, 1, output, 1, positions), 0) + self.assertEqual(len(set(positions)), count) + self.assertTrue(all(p < 524288 for p in positions)) + self.assertEqual(output[0], k) + if k == 2600: + if first is None: + first = list(positions) + else: + self.assertNotEqual(first, list(positions)) + self.assertEqual(lib.product_trial_flips(42, 7, 0, k, 16, 1, 1, replay, 2, positions), 0) + self.assertEqual(list(output), list(replay)) + positions = (experiment.ctypes.c_uint32 * 2)(3, 3) + self.assertEqual(lib.product_trial_flips(42, 0, 0, 2, 16, 1, 1, replay, 2, positions), 6) + positions[1] = 524288 + self.assertEqual(lib.product_trial_flips(42, 0, 0, 2, 16, 1, 1, replay, 2, positions), 6) + + def test_flip_sync_precedes_journal_publication(self): + source = self.run_case("settings", "--minimum-flipped-bits", "1", "--maximum-flipped-bits", "1") + settings = self.read(source, "metadata.json")["settings"] + settings.update({"threads": 3, "checkpoint trials": 3}) + path = self.root / "sync-order" + path.mkdir() + real_sync = os.fsync + synced_end = [40] + + def sync(fd): + target = Path(os.readlink(f"/proc/self/fd/{fd}")) + if target == path / "flips.bin": + # Every already-published reference must have been covered by + # the PREVIOUS flip sync, not this one. + for line in (path / "journal.jsonl").read_text().splitlines(): + self.assertLessEqual(json.loads(line)["flip end"], synced_end[0]) + synced_end[0] = target.stat().st_size + if target == path / "journal.jsonl": + for line in target.read_text().splitlines(): + self.assertLessEqual(json.loads(line)["flip end"], synced_end[0]) + real_sync(fd) + + with mock.patch.object(experiment.os, "fsync", side_effect=sync): + experiment.run(path, settings, experiment.native(), "fisher-yates") + self.invoke("--replay", path) + + def test_corrupt_recovery_rejected_without_summary_replacement(self): + source = self.run_case("source", "--minimum-flipped-bits", "1", "--maximum-flipped-bits", "1", + "--checkpoint-trials", "1") + lines = (source / "journal.jsonl").read_text().splitlines(keepends=True) + record = json.loads(lines[1]) + record["run identity"] = "wrong run" + overlap = json.loads(lines[1]) + overlap["first trial index"] = 0 + overlap["past last trial index"] = 1 + moment = json.loads(lines[1]) + moment["statistics"]["initial full block corrupted bits"]["sum"] = 0 + for index, replacement in enumerate((lines[0], "garbage\n", json.dumps(record) + "\n", + json.dumps(overlap) + "\n", json.dumps(moment) + "\n", + '{"schema revision":1,"schema revision":1}\n')): + path = self.root / f"bad-{index}" + shutil.copytree(source, path) + before = (path / "summary.json").read_bytes() + (path / "journal.jsonl").write_text(lines[0] + replacement + "".join(lines[2:])) + self.invoke("--report", path, success=False) + self.assertEqual(before, (path / "summary.json").read_bytes()) + + def test_big_integer_moments(self): + aggregate = experiment.Aggregate("test") + values = [10**30] * len(experiment.METRICS) + stats = experiment.stats_for([values, values]) + aggregate.add({"flipped bit count": 7, "trial count": 2, "statistics": stats}) + output = json.loads(experiment.canonical(aggregate.summary())) + self.assertEqual(output["overall"]["statistics"]["accepted bit changes"]["squared sum"], 2 * 10**60) + + def test_progress_success_counts_and_wall_throughput(self): + for tty, scenario, step, expected in ( + (True, "success", 1.25, 12), (False, "success", 1.25, 12), + (True, "success", 0.05, 12), + (True, "short", 0.05, 1), (True, "short", 0.0, 1), + (True, "failure", 1.25, 5), (False, "failure", 1.25, 5), + (True, "interrupt", 1.25, 3), (False, "interrupt", 1.25, 3), + (True, "submit failure", 1.25, 1)): + with self.subTest(tty=tty, scenario=scenario, step=step): + path = self.root / f"{tty}-{scenario}-{step}" + path.mkdir() + settings = {"root seed": 42, "batch size": 1 if scenario == "short" else 6, + "batches": 2 if scenario == "success" else 1, "threads": 3, + "minimum flipped bits": 0, "maximum flipped bits": 0, + "maximum directional passes": 16, "anchors": True, "binary image": True, + "checkpoint trials": 6, "report seconds": 1, "fsync seconds": 5} + clock = [0.0] + interrupted = [False] + lib = mock.Mock() + lib.product_interrupt_install.return_value = 0 + lib.product_interrupted.side_effect = lambda: interrupted[0] + lib.product_batch_k.return_value = 0 + + def trial(seed, batch, index, k, passes, anchors, binary, output): + if scenario == "failure" and index == 4: + return 1 + return 0 + + lib.product_trial.side_effect = trial + + def submit(function, index, k): + if scenario == "submit failure" and index == 1: + raise RuntimeError("injected submit failure") + future = experiment.concurrent.futures.Future() + try: + future.set_result(function(index, k)) + except Exception as error: + future.set_exception(error) + future.index = index + return future + + def wait(pending, **kwargs): + # Deterministic completions with wall time independent of native execution. + clock[0] += step + if scenario == "interrupt": + interrupted[0] = True + done = {min(pending, key=lambda future: future.index)} + return done, pending - done + + console = io.StringIO() + console.isatty = lambda: tty + pool = mock.MagicMock() + pool.__enter__.return_value.submit.side_effect = submit + with mock.patch.object(experiment.sys, "stderr", console), \ + mock.patch.object(experiment.time, "monotonic", lambda: clock[0]), \ + mock.patch.object(experiment.concurrent.futures, "ThreadPoolExecutor", return_value=pool), \ + mock.patch.object(experiment.concurrent.futures, "wait", side_effect=wait): + if "failure" in scenario: + with self.assertRaises(RuntimeError): + experiment.run(path, settings, lib) + else: + experiment.run(path, settings, lib) + + output = console.getvalue() + log = (path / "progress.log").read_text() + self.assertNotIn("\r", log) + self.assertNotIn("\033", log) + self.assertEqual(self.read(path)["overall"]["trial count"], expected) + final = log.splitlines()[-1] + self.assertIn(f"finalizing trials={expected} interrupted={scenario == 'interrupt'}", final) + self.assertIn(final + "\n", output) + batch_count = 6 if scenario == "success" else expected + self.assertIn(f"completed trials={batch_count}/{settings['batch size']}", log) + rate = expected / clock[0] if clock[0] else 0.0 + rates = re.search(r"wall blocks/s=([\d.]+) information MiB/s=([\d.]+)", final) + self.assertIsNotNone(rates) + for actual, wanted in zip(map(float, rates.groups()), (rate, rate * 56896 / 1048576)): + self.assertTrue(math.isfinite(actual)) + self.assertAlmostEqual(actual, wanted, delta=0.00051) + snapshots = [line for line in log.splitlines() if "persisted overall trials=" in line] + if step >= 1: + self.assertTrue(snapshots) + # In-flight successes appear before the wave is journaled, exactly once. + self.assertIn("trials=1/6 overall trials=1 persisted overall trials=0", snapshots[0]) + self.assertIn("wall blocks/s=0.800", snapshots[0]) + for line in snapshots: + self.assertEqual(line in output, not tty) + if scenario == "success": + self.assertEqual([int(re.search(r"overall trials=(\d+) persisted", line)[1]) + for line in snapshots], list(range(1, 13))) + if tty: + self.assertIn("\r[", output) + self.assertIn(f"trials={batch_count}/{settings['batch size']} " + f"{batch_count / settings['batch size']:.1%}", output) + self.assertTrue(output.endswith("\n")) + if scenario == "short": + self.assertEqual(output.count("\r["), 1) + if scenario == "success" and step == 0.05: + # Two forced batch-end bars plus at most one timed update per 0.2s. + self.assertLessEqual(output.count("\r["), 2 + int(clock[0] / 0.2)) + else: + self.assertEqual(output, log) + + def test_worker_exception_drains_noncontiguous_successes(self): + source = self.run_case("source", "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0") + settings = self.read(source, "metadata.json")["settings"] + settings.update({"threads": 3, "batch size": 3, "batches": 1}) + path = self.root / "failure" + path.mkdir() + lib = experiment.native() + + class FailOne: + def __getattr__(self, name): + return getattr(lib, name) + + def product_trial(self, seed, batch, trial, *args): + if trial == 1: + raise RuntimeError("injected worker failure") + return lib.product_trial(seed, batch, trial, *args) + + with self.assertRaisesRegex(RuntimeError, "injected worker failure"): + experiment.run(path, settings, FailOne()) + summary = self.read(path) + self.assertEqual(summary["overall"]["trial count"], 2) + records = [json.loads(line) for line in (path / "journal.jsonl").read_text().splitlines()] + self.assertEqual([(r["first trial index"], r["past last trial index"]) for r in records], [(0, 1), (2, 3)]) + self.invoke("--report", path) + self.assertEqual(summary, self.read(path)) + recorded = self.root / "failure-recorded" + recorded.mkdir() + + class FailSaved(FailOne): + def product_trial_flips(self, seed, batch, trial, *args): + if trial == 1: + raise RuntimeError("injected worker failure") + return lib.product_trial_flips(seed, batch, trial, *args) + + with self.assertRaisesRegex(RuntimeError, "injected worker failure"): + experiment.run(recorded, settings, FailSaved(), "fisher-yates") + self.invoke("--replay", recorded) + self.assertEqual(self.read(recorded)["overall"]["trial count"], 2) + + def test_weighted_aggregate_and_generated_seed(self): + path = self.root / "generated" + self.invoke("--output", path, "--batches", "2", "--batch-size", "2", + "--checkpoint-trials", "1", + "--minimum-flipped-bits", "1", "--maximum-flipped-bits", "1") + seed = self.read(path, "metadata.json")["settings"]["root seed"] + self.assertTrue(0 <= seed <= experiment.U64) + self.assertIn(f"root seed={seed}", (path / "progress.log").read_text()) + # Retain a complete first batch plus one trial of the next batch. + lines = (path / "journal.jsonl").read_text().splitlines(keepends=True) + (path / "journal.jsonl").write_text("".join(lines[:3])) + self.invoke("--report", path) + entry = self.read(path)["by flipped bit count"][0] + self.assertEqual(entry["trial count"], 3) + self.assertEqual(entry["statistics"]["accepted bit changes"], {"sum": 3, "squared sum": 3}) + + def test_interrupt_partial_batch(self): + path = self.root / "signal" + with (self.root / "stderr").open("w") as stderr: + process = subprocess.Popen([sys.executable, str(CLI), "--output", str(path), + "--sampler", "fisher-yates", + "--checkpoint-trials", "4096", + "--seed", "42", "--batch-size", "1000000", "--threads", "3", + "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0"], + stderr=stderr) + try: + deadline = time.monotonic() + 30 + journal = path / "journal.jsonl" + while not journal.exists() or journal.stat().st_size == 0: + self.assertIsNone(process.poll()) + self.assertLess(time.monotonic(), deadline) + time.sleep(0.01) + # A live run cannot be reaggregated concurrently. + self.invoke("--report", path, success=False) + process.send_signal(signal.SIGINT) + self.assertEqual(process.wait(timeout=30), 0) + finally: + if process.poll() is None: + process.kill() + process.wait() + summary = self.read(path) + self.assertGreater(summary["overall"]["trial count"], 0) + self.assertLess(summary["overall"]["trial count"], 1000000) + self.invoke("--report", path) + self.assertEqual(summary, self.read(path)) + self.invoke("--replay", path) + seen = set() + for line in (path / "journal.jsonl").read_text().splitlines(): + record = json.loads(line) + for index in range(record["first trial index"], record["past last trial index"]): + self.assertNotIn(index, seen) + seen.add(index) + self.assertEqual(len(seen), summary["overall"]["trial count"]) + self.assertNotIn(b"\r", (path / "progress.log").read_bytes()) + + def test_tty_progress_is_in_place_but_log_is_plain(self): + path = self.root / "tty" + master, slave = pty.openpty() + process = subprocess.Popen([sys.executable, str(CLI), "--output", str(path), + "--seed", "42", "--batch-size", "1000000", "--threads", "2", + "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0"], stderr=slave) + os.close(slave) + output = b"" + try: + deadline = time.monotonic() + 30 + while b"\r[" not in output: + self.assertIsNone(process.poll()) + self.assertLess(time.monotonic(), deadline) + ready, _, _ = select.select([master], [], [], 0.1) + if ready: + output += os.read(master, 16384) + process.send_signal(signal.SIGINT) + self.assertEqual(process.wait(timeout=30), 0) + finally: + if process.poll() is None: + process.kill() + process.wait() + os.close(master) + self.assertNotIn(b"\r", (path / "progress.log").read_bytes()) + self.assertIn("finalizing", (path / "progress.log").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/product_monte_carlo_test.py b/tests/product_monte_carlo_test.py new file mode 100644 index 0000000..df35d7f --- /dev/null +++ b/tests/product_monte_carlo_test.py @@ -0,0 +1,279 @@ +"""Black-box tests of the native executable; Python is not a runtime dependency.""" +import copy +import fcntl +import hashlib +import json +import os +from pathlib import Path +import pty +import re +import select +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +CLI = Path(sys.argv.pop(1)).resolve() +REFERENCE = CLI.parent / "product_monte_carlo_reference.py" +FAULT = CLI.parent / "product_monte_carlo_fault_cli" + + +def canonical(value): + return json.dumps(value, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + + +def projected(row): + s = row["statistics"] + count = row["trial count"] + return {"completed blocks": count, "total iterations": s["directional passes"]["sum"], + "information bits": {"total bits": count * 455168, + "raw corrupted bits": s["initial information corrupted bits"]["sum"], + "post decoding corrupted bits": s["residual information bits"]["sum"]}, + "full-codeword bits": {"total bits": count * 524288, + "raw corrupted bits": s["initial full block corrupted bits"]["sum"], + "post decoding corrupted bits": s["residual full block bits"]["sum"]}} + + +class NativeTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="native-mc-") + self.root = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def invoke(self, *args, success=True, reference=False, fault=False, env=None): + command = [sys.executable, str(REFERENCE)] if reference else [str(FAULT if fault else CLI)] + environment = dict(os.environ) if env is None else dict(env) + if reference and environment.get("MC_REFERENCE_LD_PRELOAD"): + environment["LD_PRELOAD"] = environment["MC_REFERENCE_LD_PRELOAD"] + p = subprocess.run([*command, *map(str, args)], capture_output=True, + text=True, timeout=90, env=environment) + self.assertEqual(p.returncode == 0, success, p.stderr) + return p + + def run_case(self, name, *args, **kwargs): + path = self.root / name + self.invoke("--output", path, "--seed", 42, "--batches", 2, + "--batch-size", 17, *args, **kwargs) + return path + + def read(self, path, name="summary.json"): + return json.loads((path / name).read_text()) + + def records(self, path): + return [json.loads(s) for s in (path / "journal.jsonl").read_text().splitlines()] + + def count(self, path): + return self.read(path)["overall"]["statistics"]["completed blocks"] + + def test_floyd_matches_legacy_projection_and_thread_determinism(self): + old = self.run_case("old", reference=True) + expected = self.read(old) + for threads in (1, 4, 16): + path = self.run_case(str(threads), "--threads", threads) + result = self.read(path) + self.assertEqual(result["schema revision"], 2) + self.assertEqual(result["overall"]["statistics"], projected(expected["overall"])) + for a, b in zip(result["by flipped bit count"], expected["by flipped bit count"]): + self.assertEqual(a["flipped bit count"], b["flipped bit count"]) + self.assertEqual(a["statistics"], projected(b)) + self.invoke("--report", path) + self.assertEqual(result, self.read(path)) + self.assertNotIn("squared sum", (path / "summary.json").read_text()) + repeated = self.run_case("repeat", "--threads", 16) + self.assertEqual(self.read(repeated)["overall"], result["overall"]) + + def test_extremes_gates_and_pass_caps(self): + for k in (0, 1, 524287, 524288): + for sampler in ("floyd", "fisher-yates"): + path = self.run_case(f"{k}-{sampler}", "--minimum-flipped-bits", k, + "--maximum-flipped-bits", k, "--sampler", sampler, + "--batches", 1, "--batch-size", 1, "--no-anchors", "--no-binary-image") + s = self.read(path)["overall"]["statistics"] + self.assertEqual(s["full-codeword bits"]["raw corrupted bits"], k) + self.assertEqual(s["full-codeword bits"]["post decoding corrupted bits"], + 0 if k <= 1 else 524288) + self.invoke("--replay" if sampler == "fisher-yates" else "--report", path) + for anchors in ("--anchors", "--no-anchors"): + for binary in ("--binary-image", "--no-binary-image"): + args = (anchors, binary, "--max-directional-passes", 2, + "--minimum-flipped-bits", 2600, "--maximum-flipped-bits", 2600) + a = self.run_case(anchors+binary, *args) + b = self.run_case("old"+anchors+binary, *args, reference=True) + self.assertEqual(self.read(a)["overall"]["statistics"], projected(self.read(b)["overall"])) + + def test_legacy_report_and_saved_replay_hash_defaults(self): + for sampler in ("floyd", "fisher-yates"): + path = self.run_case(sampler, "--sampler", sampler, reference=True) + expected = self.read(path) + self.invoke("--report", path) + self.assertEqual(expected, self.read(path)) + if sampler == "fisher-yates": + self.invoke("--replay", path) + metadata = self.read(path, "metadata.json") + del metadata["codeword"] + # Legacy random-codeword fixture, identity hashed without a default. + digest = hashlib.sha256(canonical(metadata).encode("ascii")).hexdigest() + (path / "metadata.json").write_text(canonical(metadata)) + records = self.records(path) + for r in records: + r["run identity"] = digest + (path / "journal.jsonl").write_text("".join(canonical(r)+"\n" for r in records)) + data = (path / "flips.bin").read_bytes() + (path / "flips.bin").write_bytes(data[:8]+bytes.fromhex(digest)+data[40:]) + before = (path / "metadata.json").read_bytes() + self.invoke("--replay", path) + self.assertEqual(before, (path / "metadata.json").read_bytes()) + self.assertEqual(self.read(path)["run identity"], digest) + + def test_saved_replay_and_corruption(self): + for k in (0, 2600, 262144, 524288): + path = self.run_case(f"k-{k}", "--sampler", "fisher-yates", "--threads", 4, + "--batch-size", 3, "--minimum-flipped-bits", k, "--maximum-flipped-bits", k) + before = self.read(path) + p = self.invoke("--replay", path) + self.assertIn("all 22 metrics match", p.stderr) + self.assertEqual(before, self.read(path)) + self.assertEqual((path / "flips.bin").stat().st_size, 40+6*(240+4*min(k,524288-k))) + data = (path / "flips.bin").read_bytes() + for bad in (data[:-1], data[:45]+bytes([data[45]^1])+data[46:], b"bad"): + (path / "flips.bin").write_bytes(bad) + self.invoke("--report", path, success=False) + self.assertEqual(before, self.read(path)) + (path / "flips.bin").write_bytes(data+b"uncommitted tail") + self.invoke("--replay", path) + + def test_recovery_strictness_and_partial_tail(self): + path = self.run_case("source", "--checkpoint-trials", 1) + before = (path / "summary.json").read_bytes() + lines = (path / "journal.jsonl").read_text().splitlines(keepends=True) + bad_identity = json.loads(lines[1]); bad_identity["run identity"] = "wrong" + overlap = json.loads(lines[1]); overlap["first trial index"] = 0 + bad_stats = json.loads(lines[1]); bad_stats["statistics"]["full-codeword bits"]["raw corrupted bits"] = 0 + for replacement in (lines[0], "garbage\n", "{}\n", canonical(bad_identity)+"\n", + canonical(overlap)+"\n", canonical(bad_stats)+"\n", + '{"schema revision":2,"schema revision":2}\n'): + (path / "journal.jsonl").write_text(lines[0]+replacement+"".join(lines[2:])) + self.invoke("--report", path, success=False) + self.assertEqual(before, (path / "summary.json").read_bytes()) + for tail in ('{"incomplete', 'garbage', '{"complete but no newline":1}'): + (path / "journal.jsonl").write_text("".join(lines)+tail) + p = self.invoke("--report", path) + self.assertIn("incomplete tail=True", p.stderr) + self.assertEqual(json.loads(before), self.read(path)) + (path / "journal.jsonl").write_text("".join(lines)+"garbage\n") + self.invoke("--report", path, success=False) + + def test_validation_seed_install_runtime_and_no_overwrite(self): + for args in (("--seed", -1), ("--seed", 2**64), ("--seed", "1.0"), + ("--threads", 0), ("--threads", 1025), ("--batch-size", 0), + ("--minimum-flipped-bits", 5, "--maximum-flipped-bits", 4), + ("--max-directional-passes", 1), ("--checkpoint-trials", 4097), + ("--report-seconds", 0), ("--fsync-seconds", 0), ("--sampler", "unknown")): + self.invoke("--output", self.root / "invalid", *args, success=False) + self.assertFalse((self.root / "invalid").exists()) + path = self.root / "generated" + env = dict(os.environ, PATH="/nonexistent") + self.invoke("--output", path, "--batches", 1, "--batch-size", 1, + "--minimum-flipped-bits", 0, "--maximum-flipped-bits", 0, env=env) + self.assertEqual(CLI.read_bytes()[:4], b"\x7fELF") + seed = self.read(path, "metadata.json")["settings"]["root seed"] + self.assertTrue(0 <= seed < 2**64) + self.assertIn(f"root seed={seed}", (path / "progress.log").read_text()) + before = (path / "journal.jsonl").read_bytes() + self.invoke("--output", path, success=False) + self.assertEqual(before, (path / "journal.jsonl").read_bytes()) + self.invoke("--report", path, "--threads", 1, success=False) + with (path / "run.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + self.invoke("--report", path, success=False) + + def test_worker_failure_drains_ordered_successes(self): + for sampler in ("floyd", "fisher-yates"): + path = self.root / sampler + env = dict(os.environ, MC_TEST_FAIL="5", MC_TEST_SLOW_FIRST="200") + p = self.invoke("--output", path, "--batches", 1, "--batch-size", 30, + "--seed", 42, "--threads", 4, "--checkpoint-trials", 12, + "--minimum-flipped-bits", 0, "--maximum-flipped-bits", 0, + "--sampler", sampler, fault=True, env=env, success=False) + self.assertIn("index=5 failed: injected worker failure", p.stderr) + indices = [i for r in self.records(path) for i in range(r["first trial index"],r["past last trial index"])] + self.assertEqual(indices, sorted(set(indices))) + self.assertEqual(indices[:5], list(range(5))) + self.assertNotIn(5, indices) + self.assertTrue(any(i > 5 for i in indices)) + self.assertEqual(self.count(path), len(indices)) + self.invoke("--replay" if sampler == "fisher-yates" else "--report", path) + + def test_native_flip_sync_before_journal_publication(self): + path = self.run_case("sync-order", "--sampler", "fisher-yates", "--threads", 4, + "--checkpoint-trials", 3, fault=True, env=dict(os.environ, MC_TEST_SYNC_ORDER="1")) + self.assertEqual(self.count(path), 34) + self.invoke("--replay", path) + + def test_signal_drains_bounded_window_and_plain_progress(self): + for sig in (signal.SIGINT, signal.SIGTERM): + path = self.root / str(sig) + err = self.root / f"stderr-{sig}" + # Slow first block holds the ordered window; other workers must not + # progress past 12 starts, and SIGTERM must still finish block zero. + env = dict(os.environ, MC_TEST_SLOW_FIRST="1200") + with err.open("w") as stream: + p = subprocess.Popen([str(FAULT), "--output", str(path), "--seed", "42", + "--threads", "4", "--batch-size", "1000000", "--checkpoint-trials", "12", + "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0", + "--sampler", "fisher-yates", "--report-seconds", "1"], stderr=stream, env=env) + try: + deadline = time.monotonic()+20 + while not (path / "progress.log").exists() or "batch=0" not in (path / "progress.log").read_text(): + self.assertIsNone(p.poll()); self.assertLess(time.monotonic(), deadline); time.sleep(.01) + time.sleep(.2) + self.invoke("--report", path, success=False) + p.send_signal(sig) + self.assertEqual(p.wait(timeout=30), 0) + finally: + if p.poll() is None: p.kill(); p.wait() + self.assertEqual(self.count(path), 12) + indices = [i for r in self.records(path) for i in range(r["first trial index"],r["past last trial index"])] + self.assertEqual(indices, list(range(12))) + self.invoke("--replay", path) + log = (path / "progress.log").read_text() + self.assertEqual(log, err.read_text()) + self.assertNotIn("\x1b", log) + self.assertIn("interrupted=True", log) + final = log.splitlines()[-1] + match = re.search(r"wall blocks/s=([\d.]+) information MiB/s=([\d.]+) elapsed seconds=([\d.]+)", final) + self.assertIsNotNone(match) + rate, mib, elapsed = map(float, match.groups()) + self.assertAlmostEqual(rate, 12/elapsed, delta=.02) + self.assertAlmostEqual(mib, rate*56896/1048576, delta=.001) + + def test_tty_throttle_and_plain_log(self): + path = self.root / "tty" + master, slave = pty.openpty() + p = subprocess.Popen([str(CLI), "--output", str(path), "--seed", "42", + "--threads", "4", "--batch-size", "1000000"], stderr=slave) + os.close(slave) + output = b"" + start = time.monotonic() + try: + while output.count(b"\r[") < 3: + self.assertIsNone(p.poll()); self.assertLess(time.monotonic()-start, 20) + if select.select([master], [], [], .1)[0]: output += os.read(master, 65536) + p.send_signal(signal.SIGINT) + self.assertEqual(p.wait(timeout=30), 0) + finally: + if p.poll() is None: p.kill(); p.wait() + os.close(master) + self.assertLessEqual(output.count(b"\r["), int((time.monotonic()-start)/.2)+1) + log = (path / "progress.log").read_bytes() + self.assertNotIn(b"\r", log); self.assertNotIn(b"\x1b", log) + self.assertIn(b"finalizing", log) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/reference/product_monte_carlo.py b/tests/reference/product_monte_carlo.py new file mode 100644 index 0000000..4532ab0 --- /dev/null +++ b/tests/reference/product_monte_carlo.py @@ -0,0 +1,675 @@ +#!/usr/bin/env python3 +"""Frozen schema-1 coordinator for differential tests, never installed.""" + +import argparse +import array +import concurrent.futures +from contextlib import ExitStack +import ctypes +import datetime +import fcntl +import hashlib +import json +import os +from pathlib import Path +import secrets +import sys +import struct +import time + +SCHEMA = 1 +FLOYD = "splitmix64 domain seeds; mt19937_64; rejection modulo; Floyd complement v1" +FISHER_YATES = "splitmix64 domain seeds; mt19937_64; rejection modulo; persistent Fisher-Yates complement v1; replay saved flips" +# Little endian, LSB-first bit = byte * 8 + bit_in_byte, row-major block. +# Per record: batch, trial, k, count, complement, 22 metrics, uint32 positions, +# SHA256(header + positions). File header binds format/version and run identity. +FLIP_MAGIC = b"RSFLIP01" +FLIP_HEADER = struct.Struct(" U64: + raise argparse.ArgumentTypeError("integer exceeds uint64") + return value + + +def native(): + here = Path(__file__).resolve().parent + candidates = [here / "product_monte_carlo_native.so", + here.parent / "lib" / "product_monte_carlo_native.so", + here.parent / "lib64" / "product_monte_carlo_native.so"] + path = next((p for p in candidates if p.is_file()), None) + if path is None: + raise ValueError("native trial library not found beside CLI or in ../lib[64]") + lib = ctypes.CDLL(str(path)) + lib.product_batch_k.argtypes = [ctypes.c_uint64] * 4 + lib.product_batch_k.restype = ctypes.c_uint64 + lib.product_trial.argtypes = ([ctypes.c_uint64] * 5 + [ctypes.c_int] * 2 + + [ctypes.POINTER(ctypes.c_uint64)]) + lib.product_trial.restype = ctypes.c_int + lib.product_trial_flips.argtypes = lib.product_trial.argtypes + [ + ctypes.c_int, ctypes.POINTER(ctypes.c_uint32)] + lib.product_trial_flips.restype = ctypes.c_int + lib.product_trial_reference.argtypes = lib.product_trial_flips.argtypes + [ + ctypes.c_int, ctypes.POINTER(ctypes.c_uint8)] + lib.product_trial_reference.restype = ctypes.c_int + lib.product_trials.argtypes = lib.product_trial.argtypes + [ + ctypes.c_uint64, ctypes.c_int, ctypes.POINTER(ctypes.c_uint32), + ctypes.POINTER(ctypes.c_uint64)] + lib.product_trials.restype = ctypes.c_int + lib.product_interrupt_install.argtypes = [] + lib.product_interrupt_install.restype = ctypes.c_int + lib.product_interrupted.argtypes = [] + lib.product_interrupted.restype = ctypes.c_int + return lib + + +def strict_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_number(value): + raise ValueError(f"noninteger JSON number: {value}") + + +def loads(text): + return json.loads(text, object_pairs_hook=strict_object, + parse_float=reject_number, parse_constant=reject_number) + + +def canonical(value): + return json.dumps(value, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + + +def identity(metadata): + return hashlib.sha256(canonical(metadata).encode("ascii")).hexdigest() + + +def sync_directory(directory): + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def atomic_json(path, value): + tmp = path.with_name(path.name + ".tmp") + with tmp.open("w", encoding="ascii") as stream: + json.dump(value, stream, indent=2, sort_keys=True, ensure_ascii=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + sync_directory(path.parent) + + +def empty_stats(): + return {name: {"sum": 0, "squared sum": 0} for name in METRICS} + + +def add_stats(destination, source): + for name in METRICS: + for field in ("sum", "squared sum"): + destination[name][field] += source[name][field] + + +def stats_for(results): + stats = empty_stats() + for values in results: + for name, value in zip(METRICS, values): + stats[name]["sum"] += value + stats[name]["squared sum"] += value * value + return stats + + +class Aggregate: + def __init__(self, run_identity): + self.run_identity = run_identity + self.by_k = {} + self.count = 0 + self.stats = empty_stats() + + def add(self, record): + k, count = record["flipped bit count"], record["trial count"] + entry = self.by_k.setdefault(k, {"flipped bit count": k, "trial count": 0, + "statistics": empty_stats()}) + entry["trial count"] += count + add_stats(entry["statistics"], record["statistics"]) + self.count += count + add_stats(self.stats, record["statistics"]) + + def summary(self): + return {"schema revision": SCHEMA, "run identity": self.run_identity, + "overall": {"trial count": self.count, "statistics": self.stats}, + "by flipped bit count": [self.by_k[k] for k in sorted(self.by_k)]} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def natural(value, maximum=None): + return type(value) is int and value >= 0 and (maximum is None or value <= maximum) + + +def flip_bytes(batch, trial, k, metrics, positions): + header = FLIP_HEADER.pack(batch, trial, k, min(k, 524288 - k), k > 262144, *metrics) + payload = bytes(positions) + if sys.byteorder != "little": + converted = array.array("I") + converted.frombytes(payload) + converted.byteswap() + payload = converted.tobytes() + body = header + payload + return body + hashlib.sha256(body).digest() + + +def read_flips(stream, record, replay, settings, lib, codeword): + require(stream.tell() == record["flip start"], "noncontiguous flip offsets") + values = [] + for trial in range(record["first trial index"], record["past last trial index"]): + header = stream.read(FLIP_HEADER.size) + require(len(header) == FLIP_HEADER.size, "truncated flip header") + batch, index, k, count, complement, *metrics = FLIP_HEADER.unpack(header) + require((batch, index, k) == (record["batch id"], trial, record["flipped bit count"]), + "flip trial identity mismatch") + require(count == min(k, 524288 - k) and complement == int(k > 262144), + "invalid flip count/complement") + payload = stream.read(count * 4) + checksum = stream.read(32) + require(len(payload) == count * 4 and hashlib.sha256(header + payload).digest() == checksum, + "truncated or corrupt flip record") + positions = array.array("I") + positions.frombytes(payload) + if sys.byteorder != "little": + positions.byteswap() + require(all(p < 524288 for p in positions) and len(set(positions)) == count, + "invalid or duplicate flip position") + if replay: + output = (ctypes.c_uint64 * len(METRICS))() + buffer = (ctypes.c_uint32 * count)(*positions) + status = lib.product_trial_reference(settings["root seed"], batch, trial, k, + settings["maximum directional passes"], settings["anchors"], settings["binary image"], + output, 2, buffer, codeword == "random", None) + require(status == 0 and list(output) == metrics, + f"replay mismatch batch={batch} trial={trial} status={status}") + values.append(metrics) + require(stream.tell() == record["flip end"], "flip end offset mismatch") + require(stats_for(values) == record["statistics"], "flip metrics disagree with journal") + + +def validate_settings(settings): + expected = {"root seed", "batch size", "batches", "threads", "minimum flipped bits", + "maximum flipped bits", "maximum directional passes", "anchors", + "binary image", "checkpoint trials", "report seconds", "fsync seconds"} + require(type(settings) is dict and set(settings) == expected, "incompatible settings") + for key in expected - {"anchors", "binary image"}: + require(natural(settings[key], U64), f"invalid {key}") + require(type(settings["anchors"]) is bool and type(settings["binary image"]) is bool, + "gates must be boolean") + require(0 <= settings["minimum flipped bits"] <= settings["maximum flipped bits"] <= 524288, + "require 0 <= minimum flipped bits <= maximum flipped bits <= 524288") + for key, maximum in (("threads", 1024), ("checkpoint trials", 4096), + ("report seconds", 86400), ("fsync seconds", 86400)): + require(1 <= settings[key] <= maximum, f"{key} must be in [1,{maximum}]") + require(settings["batch size"] > 0, "batch size must be positive") + require(2 <= settings["maximum directional passes"] <= 1000000, + "maximum directional passes must be in [2,1000000]") + + +def validate_record(record, metadata, run_identity, expected_id, previous, lib): + fields = {"schema revision", "run identity", "increment id", "batch id", + "first trial index", "past last trial index", "trial count", + "flipped bit count", "statistics"} + if metadata["random algorithm"] == FISHER_YATES: + fields |= {"flip start", "flip end"} + require(type(record) is dict and set(record) == fields, "incompatible journal record") + require(record["schema revision"] == SCHEMA and type(record["schema revision"]) is int and + record["run identity"] == run_identity, "incompatible journal identity/schema") + for key in fields - {"run identity", "statistics"}: + require(natural(record[key]), f"invalid record {key}") + require(record["increment id"] == expected_id, "duplicate or out-of-order increment id") + s = metadata["settings"] + batch, first, end = (record[key] for key in + ("batch id", "first trial index", "past last trial index")) + require(batch <= U64 and (s["batches"] == 0 or batch < s["batches"]), "invalid batch id") + require(0 <= first < end <= s["batch size"] and end - first == record["trial count"] and + end - first <= s["checkpoint trials"], "invalid trial range/count") + if previous is not None: + old_batch, old_end = previous + require(batch >= old_batch and (batch != old_batch or first >= old_end), + "overlapping or out-of-order trial ranges") + k = lib.product_batch_k(s["root seed"], batch, s["minimum flipped bits"], s["maximum flipped bits"]) + require(record["flipped bit count"] == k, "batch k disagrees with seed/settings") + stats, count = record["statistics"], record["trial count"] + require(type(stats) is dict and set(stats) == set(METRICS), "incompatible statistics") + bounds = [524288, 65536, 455168, 56896, 524288, 65536, 455168, 56896, + 1, 1, 1, 1, s["maximum directional passes"]] + [ + s["maximum directional passes"] * 524288] * 8 + [1] + for name, bound in zip(METRICS, bounds): + item = stats[name] + require(type(item) is dict and set(item) == {"sum", "squared sum"}, "invalid moment fields") + total, square = item["sum"], item["squared sum"] + require(natural(total, count * bound) and natural(square, count * bound * bound), + f"invalid moment: {name}") + require(total * total <= count * square and total <= square <= bound * total, + f"inconsistent moments: {name}") + require(stats[METRICS[0]] == {"sum": count * k, "squared sum": count * k * k}, + "initial channel is not exact k") + for total, strong, weak in ((13, 15, 17), (14, 16, 18)): + require(stats[METRICS[total]]["sum"] == stats[METRICS[strong]]["sum"] + + stats[METRICS[weak]]["sum"], "directional accepted totals disagree") + return batch, end + + +def recover(directory, lib, replay=False): + with (directory / "metadata.json").open(encoding="ascii") as stream: + metadata = loads(stream.read(131073)) + require(type(metadata) is dict and set(metadata) - {"codeword"} == { + "schema revision", "created at", "settings", "code", "random algorithm"}, "incompatible metadata") + # Do not insert defaults into metadata: legacy run identities hash it as-is. + codeword = metadata.get("codeword", "random") + require(codeword in ("zero", "random"), "incompatible codeword convention") + require(type(metadata["schema revision"]) is int and metadata["schema revision"] == SCHEMA and + metadata["code"] == "RS256,224 x RS256,254 Cantor systematic row major" and + metadata["random algorithm"] in (FLOYD, FISHER_YATES), + "incompatible metadata schema/code/random algorithm") + validate_settings(metadata["settings"]) + digest = identity(metadata) + aggregate = Aggregate(digest) + previous = None + ignored = False + recorded = metadata["random algorithm"] == FISHER_YATES + require(not replay or recorded, "this run has no saved flips") + with ExitStack() as stack: + stream = stack.enter_context((directory / "journal.jsonl").open("rb")) + flips = stack.enter_context((directory / "flips.bin").open("rb")) if recorded else None + if flips: + require(flips.read(40) == FLIP_MAGIC + bytes.fromhex(digest), "invalid flip file identity/version") + index = 0 + while True: + line = stream.readline(131073) + if not line: + break + require(len(line) <= 131072, "journal line exceeds schema size limit") + if not line.endswith(b"\n"): + ignored = True + break + try: + record = loads(line) + previous = validate_record(record, metadata, digest, index, previous, lib) + if flips: + read_flips(flips, record, replay, metadata["settings"], lib, codeword) + aggregate.add(record) + except (ValueError, TypeError, KeyError) as error: + raise ValueError(f"journal line {index + 1}: {error}") from error + index += 1 + if flips and flips.read(1): + print("unreferenced flip tail ignored (not committed trials)", file=sys.stderr) + atomic_json(directory / "summary.json", aggregate.summary()) + print(f"{timestamp()} regenerated {aggregate.count} trials; ignored incomplete tail={ignored}", file=sys.stderr) + if replay: + print(f"verified replay: {aggregate.count} trials, all 22 metrics match", file=sys.stderr) + + +def run(directory, settings, lib, sampler="floyd"): + metadata = {"schema revision": SCHEMA, "created at": timestamp(), "settings": settings, + "codeword": "zero", + "code": "RS256,224 x RS256,254 Cantor systematic row major", + "random algorithm": FISHER_YATES if sampler == "fisher-yates" else FLOYD} + atomic_json(directory / "metadata.json", metadata) + aggregate = Aggregate(identity(metadata)) + atomic_json(directory / "summary.json", aggregate.summary()) + if lib.product_interrupt_install() != 0: + raise OSError("could not install native signal handlers") + with ExitStack() as stack: + journal = stack.enter_context((directory / "journal.jsonl").open("x", encoding="ascii")) + log = stack.enter_context((directory / "progress.log").open("x", encoding="ascii")) + flips = stack.enter_context((directory / "flips.bin").open("xb")) if sampler == "fisher-yates" else None + if flips: + flips.write(FLIP_MAGIC + bytes.fromhex(aggregate.run_identity)) + flips.flush() + os.fsync(flips.fileno()) + tty = sys.stderr.isatty() + bar_visible = False + increment = batch = 0 + staged = None + last_checkpoint = time.monotonic() + + def checkpoint(): + nonlocal staged, increment, last_checkpoint + if staged is not None: + if flips: + flips.flush() + os.fsync(flips.fileno()) + staged["increment id"] = increment + journal.write(canonical(staged) + "\n") + journal.flush() + aggregate.add(staged) + increment += 1 + staged = None + last_checkpoint = time.monotonic() + + def progress(text, console=True): + nonlocal bar_visible + if console and bar_visible: + print("\r\033[K", end="", file=sys.stderr) + bar_visible = False + line = f"{timestamp()} {text}" + if console: + print(line, file=sys.stderr, flush=True) + print(line, file=log, flush=True) + + def durable(): + checkpoint() + journal.flush() + os.fsync(journal.fileno()) + log.flush() + os.fsync(log.fileno()) + atomic_json(directory / "summary.json", aggregate.summary()) + + progress(f"root seed={settings['root seed']} settings persisted before trials") + durable() + sync_directory(directory) + sync_directory(directory.parent) + last_sync = last_report = last_bar = time.monotonic() + start = last_sync + completed_total = 0 + error = None + + def throughput(now): + # Whole simulation wall time, including coordinator work, not decoder time. + elapsed = max(0.0, now - start) + rate = completed_total / elapsed if elapsed > 0 else 0.0 + return (f"wall blocks/s={rate:.3f} information MiB/s={rate * INFORMATION_BYTES / (1 << 20):.3f} " + f"elapsed seconds={elapsed:.3f}") + + def bar(now, force=False): + nonlocal bar_visible, last_bar + if tty and (force or now - last_bar >= 0.2): + fraction = batch_completed / settings["batch size"] + width = int(30 * fraction) + print(f"\r[{('#' * width).ljust(30)}] batch={batch} k={k} " + f"trials={batch_completed}/{settings['batch size']} {fraction:.1%} " + f"overall trials={completed_total} {throughput(now)}\033[K", + end="", file=sys.stderr, flush=True) + bar_visible = True + last_bar = now + + def trial(index, k): + output = (ctypes.c_uint64 * len(METRICS))() + arguments = (settings["root seed"], batch, index, k, + settings["maximum directional passes"], + settings["anchors"], settings["binary image"], output) + if flips: + positions = (ctypes.c_uint32 * min(k, 524288 - k))() + status = lib.product_trial_flips(*arguments, 1, positions) + else: + status = lib.product_trial(*arguments) + if status: + raise RuntimeError(f"native trial batch={batch} index={index} failed: {status}") + return (list(output), positions) if flips else list(output) + + # Amortize GIL/executor handoffs without starving workers at small caps. + chunk = 4 if settings["checkpoint trials"] >= 4 * settings["threads"] else 1 + + def trials(first, k): + if chunk == 1: + if lib.product_interrupted(): + return [], None + try: + return [trial(first, k)], None + except Exception as exc: + return [], exc + count = min(chunk, settings["batch size"] - first) + output = (ctypes.c_uint64 * (len(METRICS) * count))() + stride = min(k, 524288 - k) + positions = (ctypes.c_uint32 * (stride * count))() if flips else None + completed = ctypes.c_uint64() + status = lib.product_trials(settings["root seed"], batch, first, k, + settings["maximum directional passes"], settings["anchors"], + settings["binary image"], output, count, bool(flips), positions, + ctypes.byref(completed)) + results = [] + for i in range(completed.value): + metrics = list(output[i * len(METRICS):(i + 1) * len(METRICS)]) + if flips: + saved = (ctypes.c_uint32 * stride).from_buffer(positions, i * stride * 4) + results.append((metrics, saved)) + else: + results.append(metrics) + failure = None + if status: + failure = RuntimeError( + f"native trial batch={batch} index={first + completed.value} failed: {status}") + return results, failure + + # Refill workers without wave barriers. Bound running plus out-of-order + # results so a slow early trial cannot accumulate an unbounded flip log. + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=settings["threads"]) as pool: + while not lib.product_interrupted() and not error and ( + settings["batches"] == 0 or batch < settings["batches"]): + k = lib.product_batch_k(settings["root seed"], batch, + settings["minimum flipped bits"], settings["maximum flipped bits"]) + first = 0 + cursor = 0 + futures, completed = {}, {} + window = min(settings["checkpoint trials"], 2 * chunk * settings["threads"]) + batch_completed = 0 + progress(f"batch={batch} k={k} trials=0/{settings['batch size']} overall trials={completed_total}") + while futures or (first < settings["batch size"] and not lib.product_interrupted() and not error): + while (not error and not lib.product_interrupted() and + first < settings["batch size"] and + first - cursor < window and len(futures) < settings["threads"]): + end = min(first + chunk, settings["batch size"]) + if end - cursor > window: + break + try: + futures[pool.submit(trials, first, k)] = (first, end) + first = end + except Exception as exc: + error = exc + break + if futures: + done, _ = concurrent.futures.wait(set(futures), timeout=0.1, + return_when=concurrent.futures.FIRST_COMPLETED) + for future in done: + index, end = futures.pop(future) + results = [] + try: + results, failure = future.result() + if failure is not None: + error = failure + except Exception as exc: + error = exc + completed_total += len(results) + batch_completed += len(results) + for i in range(index, end): + completed[i] = results[i - index] if i - index < len(results) else None + now = time.monotonic() + if now - last_checkpoint >= 1: + checkpoint() + if now - last_sync >= settings["fsync seconds"]: + durable() + last_sync = now + if now - last_report >= settings["report seconds"]: + progress(f"batch={batch} k={k} trials={batch_completed}/{settings['batch size']} " + f"overall trials={completed_total} " + f"persisted overall trials={aggregate.count} " + f"message failures={aggregate.stats['message failures']['sum']} " + f"full block failures={aggregate.stats['full block failures']['sum']} " + f"{throughput(now)}", console=not tty) + last_report = now + bar(now) + ready = {} + while cursor in completed: + result = completed.pop(cursor) + if result is not None: + ready[cursor] = result + cursor += 1 + indices = list(ready) + offset = 0 + while offset < len(indices): + remaining = settings["checkpoint trials"] - (staged["trial count"] if staged else 0) + stop = offset + 1 + while (stop < len(indices) and stop - offset < remaining and + indices[stop] == indices[stop - 1] + 1): + stop += 1 + flip_start = flips.tell() if flips else None + if flips: + for i in indices[offset:stop]: + metrics, positions = ready[i] + flips.write(flip_bytes(batch, i, k, metrics, positions)) + record = {"schema revision": SCHEMA, "run identity": aggregate.run_identity, + "increment id": increment, "batch id": batch, + "first trial index": indices[offset], "past last trial index": indices[stop - 1] + 1, + "trial count": stop - offset, "flipped bit count": k, + "statistics": stats_for(ready[i][0] if flips else ready[i] + for i in indices[offset:stop])} + if flips: + record.update({"flip start": flip_start, "flip end": flips.tell()}) + if staged is not None and staged["past last trial index"] != record["first trial index"]: + checkpoint() + if staged is None: + staged = record + else: + staged["past last trial index"] = record["past last trial index"] + staged["trial count"] += record["trial count"] + if flips: + staged["flip end"] = record["flip end"] + add_stats(staged["statistics"], record["statistics"]) + offset = stop + if staged["trial count"] >= settings["checkpoint trials"]: + checkpoint() + now = time.monotonic() + if (staged is not None and staged["trial count"] >= settings["checkpoint trials"]) or now - last_checkpoint >= 1: + checkpoint() + if now - last_sync >= settings["fsync seconds"]: + durable() + last_sync = now + checkpoint() + now = time.monotonic() + bar(now, force=True) + progress(f"batch={batch} k={k} completed trials={batch_completed}/{settings['batch size']} " + f"overall trials={completed_total} message failures={aggregate.stats['message failures']['sum']} " + f"full block failures={aggregate.stats['full block failures']['sum']} {throughput(now)}") + if batch == U64: + raise OverflowError("batch identity exhausted; start a new run") + batch += 1 + finally: + checkpoint() + progress(f"finalizing trials={completed_total} interrupted={bool(lib.product_interrupted())} " + f"error={error} {throughput(time.monotonic())}") + durable() + if error: + raise error + + +def main(): + # Python 3.11's decimal conversion guard is not a statistical counter limit. + if hasattr(sys, "set_int_max_str_digits"): + sys.set_int_max_str_digits(0) + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, epilog=( + "One uniform inclusive k per batch; each block has exactly k distinct flipped bits. " + "Defaults: RS256,224 x RS256,254, indefinite batches until Ctrl+C. " + "New trials send the all-zero codeword, bypassing message generation and encoding. " + "Linear syndromes and BDD delta gates make residual errors codeword-translation invariant. " + "Replay honors metadata; legacy records without codeword use the private random reference path. " + "Journal increments aggregate up to checkpoint-trials blocks, flushed at least " + "once per second subject to ordered in-flight completion. " + "Crash loss: bounded in-flight window and staged increment plus writes since last fsync (default 5 seconds, " + "subject to I/O scheduling). Ctrl+C drains in-flight blocks and fsyncs. " + "Report mode ignores only a final non-newline-terminated journal fragment. " + "Resume is not supported. Counts and squared sums are exact JSON integers. " + "Fisher-Yates always records flips.bin (RSFLIP01): little-endian uint32 bit positions, " + "bit=8*row-major byte index+LSB-first bit index. Dense records store N-k unflips " + "with a complement flag. Records include batch/trial identity, 22 metrics, SHA256; " + "journal flip start/end offsets reference data fsynced before journal publication. " + "At k=2600 recording costs 10640 bytes/trial, plus a 40-byte file header. " + "--replay verifies all committed trials; unreferenced crash tails are not trials.")) + parser.add_argument("--output", type=Path, help="new run directory; must not already exist") + parser.add_argument("--report", type=Path, help="regenerate summary in an existing run, without trials") + parser.add_argument("--replay", type=Path, help="verify every committed saved trial and its 22 metrics") + parser.add_argument("--sampler", choices=("floyd", "fisher-yates"), default="floyd", + help="Fisher-Yates saves flips.bin; seeds alone cannot replay its worker history") + parser.add_argument("--seed", type=integer, help="uint64 root seed; otherwise generated once and persisted") + for flag, default, help_text in ( + ("batch-size", 1000, "blocks per random-k batch"), + ("batches", 0, "batch limit, 0 means indefinite"), + ("threads", 1, "parallel independent product blocks (1..1024)"), + ("minimum-flipped-bits", 2500, "inclusive lower bound for batch k"), + ("maximum-flipped-bits", 2700, "inclusive upper bound for batch k"), + ("max-directional-passes", 16, "directional pass cap (2..1000000)"), + ("checkpoint-trials", 64, "maximum trials per journal increment; also caps concurrency (1..4096)"), + ("report-seconds", 2, "log/non-TTY progress snapshot interval (1..86400 seconds); TTY bar updates every 0.2 seconds"), + ("fsync-seconds", 5, "durability and summary interval (1..86400 seconds)")): + parser.add_argument("--" + flag, type=integer, default=default, help=help_text) + parser.add_argument("--anchors", action=argparse.BooleanOptionalAction, default=True, + help="enable strong-success anchor protection") + parser.add_argument("--binary-image", action=argparse.BooleanOptionalAction, default=True, + help="enable weak repair delta popcount <= 2 gate") + args = parser.parse_args() + require(sum(map(bool, (args.output, args.report, args.replay))) == 1, + "specify exactly one of --output, --report or --replay") + lib = native() + if args.report or args.replay: + flag = "--replay" if args.replay else "--report" + directory = args.replay or args.report + require((len(sys.argv) == 3 and sys.argv[1] == flag) or + (len(sys.argv) == 2 and sys.argv[1].startswith(flag + "=")), + f"{flag} accepts only a directory") + with (directory / "run.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + recover(directory, lib, replay=bool(args.replay)) + return + settings = {"root seed": args.seed if args.seed is not None else secrets.randbits(64), + "batch size": args.batch_size, "batches": args.batches, "threads": args.threads, + "minimum flipped bits": args.minimum_flipped_bits, + "maximum flipped bits": args.maximum_flipped_bits, + "maximum directional passes": args.max_directional_passes, + "anchors": args.anchors, "binary image": args.binary_image, + "checkpoint trials": args.checkpoint_trials, + "report seconds": args.report_seconds, "fsync seconds": args.fsync_seconds} + validate_settings(settings) + args.output.mkdir() # No exist_ok: never overwrite a previous experiment. + with (args.output / "run.lock").open("x") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + run(args.output, settings, lib, args.sampler) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, RuntimeError, OverflowError) as error: + print(f"{timestamp()} error: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tools/lch_rs.cc b/tools/lch_rs.cc index f56ec72..dbff40f 100644 --- a/tools/lch_rs.cc +++ b/tools/lch_rs.cc @@ -212,13 +212,12 @@ class ProgressDisplay { indicators::option::ShowRemainingTime{true}, indicators::option::Stream{std::cerr}); } else { - spinner_ = - std::make_unique( - indicators::option::BarWidth{30}, - indicators::option::PrefixText{label_ + " "}, - indicators::option::Start{"["}, indicators::option::Fill{"."}, - indicators::option::Lead{"<=>"}, indicators::option::End{"]"}, - indicators::option::Stream{std::cerr}); + spinner_ = std::make_unique( + indicators::option::BarWidth{30}, + indicators::option::PrefixText{label_ + " "}, + indicators::option::Start{"["}, indicators::option::Fill{"."}, + indicators::option::Lead{"<=>"}, indicators::option::End{"]"}, + indicators::option::Stream{std::cerr}); } reporter_ = std::thread([this] { Report(); }); } @@ -248,11 +247,11 @@ class ProgressDisplay { private: std::string Postfix(bool failed) const { const uint64_t completed = completed_.load(std::memory_order_relaxed); - const double elapsed = std::chrono::duration( - std::chrono::steady_clock::now() - start_) - .count(); - const double rate = elapsed > 0.0 ? static_cast(completed) / elapsed - : 0.0; + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start_) + .count(); + const double rate = + elapsed > 0.0 ? static_cast(completed) / elapsed : 0.0; std::string text = FormatBytes(static_cast(completed)); if (bar_) { text += "/" + FormatBytes(static_cast(total_)); @@ -1018,7 +1017,7 @@ bool ProcessEncodeStripe(EncodeState* state, std::memset(work->data(), 0, encoded_bytes); std::memcpy(work->data(), data->data(), live); if (!EncodeStripe(*state->encoder, work->data(), encoded_chunk_size, - workspace)) { + workspace)) { return false; } uint64_t payload_index = 0; @@ -1211,15 +1210,14 @@ int RunEncode(const Options& options) { return kExitUsage; } Verbose(options.verbose, "encode k=", options.k, " r=", options.r, - " chunk=", options.chunk_size, " jobs=", options.jobs, - " backend=", + " chunk=", options.chunk_size, " jobs=", options.jobs, " backend=", BackendName(gf2p8::lch::SelectBackend(options.chunk_size)), " input=", read_stdin ? "stdin" : options.input.c_str()); ProgressDisplay progress("Encoding", options.progress_mode, !read_stdin, source_size); auto consume = [&](uint32_t stripe, size_t live, AlignedBuffer* data, - AlignedBuffer* work, std::span workspace) { + AlignedBuffer* work, std::span workspace) { if (!ProcessEncodeStripe(&state, stripe, live, data, work, workspace)) { SetError(&state.error, kExitUsage); return false; @@ -1249,7 +1247,7 @@ int RunEncode(const Options& options) { AlignedBuffer workspace_buffer; const bool serial = options.jobs == 1; if (serial && (!work.Allocate(static_cast(n) * options.chunk_size) || - !workspace_buffer.Allocate(workspace_bytes))) { + !workspace_buffer.Allocate(workspace_bytes))) { progress.Finish(false); Error("out of memory"); return kExitUsage; @@ -1549,14 +1547,14 @@ int RunVerify(const Options& options) { geometry.m == 0 ? 1 : std::min(options.jobs, geometry.m); Verbose(options.verbose, "verify k=", geometry.k, " r=", geometry.r, " chunk=", geometry.c, " stripes=", geometry.m, " jobs=", jobs, - " shares=", shares.size(), " backend=", - BackendName(gf2p8::lch::SelectBackend(geometry.c))); + " shares=", shares.size(), + " backend=", BackendName(gf2p8::lch::SelectBackend(geometry.c))); ProgressDisplay progress("Verifying", options.progress_mode, true, geometry.original_size); auto worker = [&] { AlignedBuffer storage; if (!storage.Allocate(static_cast(ShareCount(geometry)) * - geometry.c)) { + geometry.c)) { operation_error.store(true); return; } @@ -1666,9 +1664,9 @@ int RunDecode(const Options& options) { const uint32_t jobs = serial ? 1 : std::min(options.jobs, geometry.m); Verbose(options.verbose, "decode k=", geometry.k, " r=", geometry.r, " chunk=", geometry.c, " stripes=", geometry.m, " jobs=", jobs, - " shares=", shares.size(), " backend=", - BackendName(gf2p8::lch::SelectBackend(geometry.c)), " output=", - write_stdout ? "stdout" : dest_path.c_str()); + " shares=", shares.size(), + " backend=", BackendName(gf2p8::lch::SelectBackend(geometry.c)), + " output=", write_stdout ? "stdout" : dest_path.c_str()); ProgressDisplay progress("Decoding", options.progress_mode, true, geometry.original_size); std::atomic error{0}; @@ -1708,7 +1706,7 @@ int RunDecode(const Options& options) { return false; } } else if (!PWriteAll(dest.get(), storage, static_cast(live), - offset)) { + offset)) { SetError(&error, kExitUsage); return false; } @@ -1742,7 +1740,7 @@ int RunDecode(const Options& options) { AlignedBuffer storage; AlignedBuffer workspace_buffer; if (!storage.Allocate(static_cast(ShareCount(geometry)) * - geometry.c) || + geometry.c) || !workspace_buffer.Allocate(workspace_bytes)) { progress.Finish(false); Error("out of memory"); diff --git a/tools/product_monte_carlo.cc b/tools/product_monte_carlo.cc new file mode 100644 index 0000000..d86e9d0 --- /dev/null +++ b/tools/product_monte_carlo.cc @@ -0,0 +1,912 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "product_monte_carlo_data.h" +#include "product_monte_carlo_trials.h" + +namespace mc { +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; + +std::string Timestamp() { + const auto now = std::time(nullptr); + std::tm tm{}; + gmtime_r(&now, &tm); + std::ostringstream out; + out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S+00:00"); + return out.str(); +} + +class File { + public: + File(const fs::path& path, int flags) { + fd_ = ::open(path.c_str(), flags | O_CLOEXEC, 0600); + if (fd_ < 0) { + throw std::system_error(errno, std::generic_category(), path.string()); + } +#ifdef GF256_MC_TEST_HOOKS + path_ = path; +#endif + } + ~File() { ::close(fd_); } + File(const File&) = delete; + File& operator=(const File&) = delete; + void Write(std::string_view bytes) { +#ifdef GF256_MC_TEST_HOOKS + if (path_.filename() == "journal.jsonl" && + std::getenv("MC_TEST_SYNC_ORDER")) { + const auto record = Parse(std::string(bytes)); + if (record.contains("flip end")) { + Require(U64(record.at("flip end")) <= synced_flip_end_, + "journal referenced unsynced flip data"); + } + } +#endif + while (!bytes.empty()) { + auto n = ::write(fd_, bytes.data(), bytes.size()); + if (n < 0 && errno == EINTR) { + continue; + } + if (n <= 0) { + throw std::system_error(n < 0 ? errno : EIO, std::generic_category(), + "write"); + } + bytes.remove_prefix(static_cast(n)); + } + } + void Sync() { + int status; + do { + status = ::fsync(fd_); + } while (status < 0 && errno == EINTR); + if (status < 0) { + throw std::system_error(errno, std::generic_category(), "fsync"); + } +#ifdef GF256_MC_TEST_HOOKS + if (path_.filename() == "flips.bin") { + synced_flip_end_ = Offset(); + } +#endif + } + uint64_t Offset() const { + auto pos = ::lseek(fd_, 0, SEEK_CUR); + Require(pos >= 0, "file offset unavailable"); + return static_cast(pos); + } + void Lock() { + if (::flock(fd_, LOCK_EX | LOCK_NB) < 0) { + throw std::system_error(errno, std::generic_category(), "run lock"); + } + } + + private: + int fd_; +#ifdef GF256_MC_TEST_HOOKS + fs::path path_; + inline static uint64_t synced_flip_end_ = 0; +#endif +}; + +void SyncDirectory(const fs::path& path) { + File dir(path.empty() ? fs::path(".") : path, O_RDONLY | O_DIRECTORY); + dir.Sync(); +} +void AtomicJson(const fs::path& path, const Json& value) { + auto temp = path; + temp += ".tmp"; + { + File out(temp, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW); + out.Write(Dump(value, true) + "\n"); + out.Sync(); + } + fs::rename(temp, path); + SyncDirectory(path.parent_path()); +} +std::string Hash(std::string_view data) { + std::string result(32, '\0'); + unsigned size = 0; + Require(EVP_Digest(data.data(), data.size(), + reinterpret_cast(result.data()), &size, + EVP_sha256(), nullptr) == 1 && + size == 32, + "OpenSSL SHA256 failed"); + return result; +} +std::string Hex(std::string_view data) { + constexpr char digits[] = "0123456789abcdef"; + std::string out; + for (unsigned char c : data) { + out += digits[c >> 4]; + out += digits[c & 15]; + } + return out; +} +std::string ReadExact(std::istream& stream, size_t n) { + std::string text(n, '\0'); + stream.read(text.data(), n); + Require(static_cast(stream.gcount()) == n, "truncated input"); + return text; +} +// Bounded line reads distinguish an incomplete final fragment from a complete +// malformed record. A corrupt interior/full line is never silently skipped. +std::optional ReadLine(std::istream& stream, bool& incomplete) { + std::string text; + char c; + while (stream.get(c)) { + if (c == '\n') { + return text; + } + Require(text.size() < 131072, "journal line exceeds schema size limit"); + text += c; + } + Require(stream.eof() && !stream.bad(), "journal read failed"); + incomplete = !text.empty(); + return std::nullopt; +} +void PutLE(std::string& bytes, size_t offset, uint64_t value, size_t width) { + for (size_t i = 0; i < width; ++i) { + bytes[offset + i] = static_cast(value >> (8 * i)); + } +} +uint64_t GetLE(std::string_view bytes, size_t offset, size_t width) { + uint64_t value = 0; + for (size_t i = 0; i < width; ++i) { + value |= uint64_t(static_cast(bytes[offset + i])) << (8 * i); + } + return value; +} + +struct Result { + uint64_t index = 0; + std::array metrics{}; + std::vector positions; + std::exception_ptr error; + bool ready = false; +}; + +class Workers { + public: + Workers(const Settings& settings, bool recorded) + : s_(settings), + recorded_(recorded), + slots_(std::min(s_.checkpoint, 8 * s_.threads)) { + try { + for (uint64_t i = 0; i < std::min(s_.threads, uint64_t(slots_.size())); + ++i) { + threads_.emplace_back([this] { Work(); }); + } + } catch (...) { + Shutdown(); + throw; + } + } + ~Workers() { Shutdown(); } + void Begin(uint64_t batch, uint64_t k) { + std::lock_guard lock(mutex_); + batch_ = batch; + k_ = k; + next_ = cursor_ = completed_ = 0; + active_ = true; + halted_ = false; + changed_.notify_all(); + } + // The coordinator alone consumes slots, in trial order. Credits include + // running and finished-but-not-consumed blocks, bounding memory even when + // trial zero is slow. Workers refill independently, without a wave barrier. + bool Pop(Result& result, uint64_t& completed, bool& done) { + std::unique_lock lock(mutex_); + auto finished = [&] { + return cursor_ == next_ && + (next_ == s_.size || halted_ || product_interrupted()); + }; + changed_.wait_for(lock, std::chrono::milliseconds(50), [&] { + return finished() || + (cursor_ < next_ && slots_[cursor_ % slots_.size()].ready); + }); + completed = completed_; + done = finished(); + if (done) { + active_ = false; + return false; + } + if (cursor_ == next_ || !slots_[cursor_ % slots_.size()].ready) { + return false; + } + auto& slot = slots_[cursor_ % slots_.size()]; + result = std::move(slot); + slot = Result{}; + ++cursor_; + changed_.notify_all(); + return true; + } + + private: + void Shutdown() noexcept { + { + std::lock_guard lock(mutex_); + shutdown_ = true; + changed_.notify_all(); + } + for (auto& t : threads_) { + if (t.joinable()) { + t.join(); + } + } + } + void Work() { + for (;;) { + uint64_t index; + { + std::unique_lock lock(mutex_); + changed_.wait(lock, [&] { + return shutdown_ || + (active_ && !halted_ && !product_interrupted() && + next_ < s_.size && next_ - cursor_ < slots_.size()); + }); + if (shutdown_) { + return; + } + // Claiming a block is its start boundary. Once claimed, it runs the + // complete product decoder even if a signal arrives immediately after. + if (product_interrupted()) { + continue; + } + index = next_++; + } + auto& slot = slots_[index % slots_.size()]; + slot.index = index; + try { + if (recorded_) { + slot.positions.resize( + std::max(1, std::min(k_, 524288 - k_))); + } +#ifdef GF256_MC_TEST_HOOKS + // Dedicated, noninstalled test executable only. + if (const char* fail = std::getenv("MC_TEST_FAIL")) { + if (index == std::stoull(fail)) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + throw std::runtime_error("injected worker failure"); + } + } + if (const char* slow = std::getenv("MC_TEST_SLOW_FIRST")) { + if (index == 0) { + std::this_thread::sleep_for( + std::chrono::milliseconds(std::stoul(slow))); + } + } +#endif + int status = product_trial_reference( + s_.seed, batch_, index, k_, s_.passes, s_.anchors, s_.binary, + slot.metrics.data(), recorded_ ? 1 : 0, + recorded_ ? slot.positions.data() : nullptr, 0, nullptr); + Require(status == 0, "native status=" + std::to_string(status)); + } catch (...) { + slot.error = std::current_exception(); + } + { + std::lock_guard lock(mutex_); + if (slot.error) { + halted_ = true; + } else { + ++completed_; + } + slot.ready = true; + changed_.notify_all(); + } + } + } + Settings s_; + bool recorded_; + std::vector slots_; + std::vector threads_; + std::mutex mutex_; + std::condition_variable changed_; + uint64_t batch_ = 0, k_ = 0, next_ = 0, cursor_ = 0, completed_ = 0; + bool active_ = false, halted_ = false, shutdown_ = false; +}; + +struct Aggregate { + std::string identity; + unsigned schema; + Stats overall; + std::map by_k; + Json legacy = LegacyStats(); + std::map legacy_k; + Aggregate(std::string id, unsigned revision) + : identity(std::move(id)), schema(revision) {} + void Add(uint64_t k, const Stats& stats, const Json& old = Json()) { + overall.Add(stats); + by_k[k].Add(stats); + if (schema == 1) { + AddLegacy(legacy, old); + auto [it, inserted] = legacy_k.try_emplace(k, LegacyStats()); + AddLegacy(it->second, old); + } + } + Json Summary() const { + Json out; + out["schema revision"] = schema; + out["run identity"] = identity; + Json rows(jsoncons::json_array_arg); + for (const auto& [k, s] : by_k) { + Json row; + row["flipped bit count"] = k; + if (schema == 1) { + row["trial count"] = Number(s.blocks); + row["statistics"] = legacy_k.at(k); + } else { + row["statistics"] = s.ToJson(); + } + rows.push_back(std::move(row)); + } + out["by flipped bit count"] = std::move(rows); + if (schema == 1) { + out["overall"]["trial count"] = Number(overall.blocks); + out["overall"]["statistics"] = legacy; + } else { + out["overall"]["statistics"] = overall.ToJson(); + } + return out; + } +}; + +void WriteFlips(File& file, uint64_t batch, uint64_t k, const Result& result) { + const auto count = std::min(k, 524288 - k); + std::string bytes(208 + 4 * count, '\0'); + PutLE(bytes, 0, batch, 8); + PutLE(bytes, 8, result.index, 8); + PutLE(bytes, 16, k, 4); + PutLE(bytes, 20, count, 4); + bytes[24] = k > 262144; + for (size_t i = 0; i < 22; ++i) { + PutLE(bytes, 32 + 8 * i, result.metrics[i], 8); + } + for (size_t i = 0; i < count; ++i) { + PutLE(bytes, 208 + 4 * i, result.positions[i], 4); + } + file.Write(bytes); + file.Write(Hash(bytes)); +} + +void Run(const fs::path& directory, const Settings& s, bool recorded) { + Require(product_interrupt_install() == 0, + "could not install signal handlers"); + Json metadata; + metadata["schema revision"] = 2; + metadata["created at"] = Timestamp(); + metadata["settings"] = s.ToJson(); + metadata["codeword"] = "zero"; + metadata["code"] = kCode; + metadata["random algorithm"] = recorded ? kFisherYates : kFloyd; + AtomicJson(directory / "metadata.json", metadata); + const auto digest = Hash(Dump(metadata)); + Aggregate aggregate(Hex(digest), 2); + AtomicJson(directory / "summary.json", aggregate.Summary()); + File journal(directory / "journal.jsonl", O_WRONLY | O_CREAT | O_EXCL); + File log(directory / "progress.log", O_WRONLY | O_CREAT | O_EXCL); + std::optional flips; + if (recorded) { + flips.emplace(directory / "flips.bin", O_WRONLY | O_CREAT | O_EXCL); + flips->Write("RSFLIP01" + digest); + flips->Sync(); + } + const bool tty = ::isatty(STDERR_FILENO); + bool bar_visible = false; + auto progress = [&](const std::string& text, bool console = true) { + auto line = Timestamp() + " " + text + "\n"; + if (console) { + if (bar_visible) { + std::cerr << "\r\033[K"; + bar_visible = false; + } + std::cerr << line << std::flush; + } + log.Write(line); + }; + progress("root seed=" + std::to_string(s.seed) + + " settings persisted before trials"); + journal.Sync(); + log.Sync(); + SyncDirectory(directory); + SyncDirectory(directory.parent_path()); + + const auto start = Clock::now(); + auto last_sync = start, last_report = start, last_bar = start, + last_checkpoint = start; + Big completed_total = 0, increment = 0; + uint64_t batch_completed = 0, batch = 0, k = 0; + Stats staged; + uint64_t first = 0, end = 0, flip_start = 0; + std::string error; + auto throughput = [&](Clock::time_point now) { + const double elapsed = std::chrono::duration(now - start).count(); + const double rate = + elapsed > 0 ? static_cast(completed_total) / elapsed : 0; + std::ostringstream out; + out << std::fixed << std::setprecision(3) << "wall blocks/s=" << rate + << " information MiB/s=" << rate * 56896 / 1048576 + << " elapsed seconds=" << elapsed; + return out.str(); + }; + auto counts = [&] { + return "batch=" + std::to_string(batch) + " k=" + std::to_string(k) + + " trials=" + std::to_string(batch_completed) + "/" + + std::to_string(s.size) + + " overall trials=" + completed_total.to_string(); + }; + auto checkpoint = [&] { + if (staged.blocks != 0) { + Json r; + r["schema revision"] = 2; + r["run identity"] = aggregate.identity; + r["increment id"] = Number(increment); + r["batch id"] = batch; + r["first trial index"] = first; + r["past last trial index"] = end; + r["trial count"] = end - first; + r["flipped bit count"] = k; + r["statistics"] = staged.ToJson(); + if (flips) { + flips->Sync(); + r["flip start"] = flip_start; + r["flip end"] = flips->Offset(); + } + journal.Write(Dump(r) + "\n"); + aggregate.Add(k, staged); + ++increment; + staged = Stats{}; + } + last_checkpoint = Clock::now(); + }; + auto durable = [&] { + checkpoint(); + journal.Sync(); + log.Sync(); + AtomicJson(directory / "summary.json", aggregate.Summary()); + last_sync = Clock::now(); + }; + auto bar = [&](Clock::time_point now, bool force = false) { + if (tty && (force || now - last_bar >= std::chrono::milliseconds(200))) { + double fraction = double(batch_completed) / s.size; + size_t width = static_cast(30 * fraction); + std::cerr << "\r[" << std::string(width, '#') + << std::string(30 - width, ' ') << "] batch=" << batch + << " k=" << k << " trials=" << batch_completed << "/" << s.size + << " " << std::fixed << std::setprecision(1) << 100 * fraction + << "% overall trials=" << completed_total << " " + << throughput(now) << "\033[K" << std::flush; + bar_visible = true; + last_bar = now; + } + }; + try { + Workers workers(s, recorded); + while (!product_interrupted() && error.empty() && + (!s.batches || batch < s.batches)) { + k = product_batch_k(s.seed, batch, s.lo, s.hi); + batch_completed = 0; + progress(counts()); + workers.Begin(batch, k); + bool done = false; + while (!done) { + Result result; + uint64_t observed; + bool ready = workers.Pop(result, observed, done); + completed_total += Big(observed - batch_completed); + batch_completed = observed; + if (ready) { + if (result.error) { + checkpoint(); + try { + std::rethrow_exception(result.error); + } catch (const std::exception& e) { + if (error.empty()) { + error = "batch=" + std::to_string(batch) + + " index=" + std::to_string(result.index) + + " failed: " + e.what(); + } + } catch (...) { + if (error.empty()) { + error = "unknown worker exception"; + } + } + } else { + if (staged.blocks != 0 && end != result.index) { + checkpoint(); + } + if (staged.blocks == 0) { + first = result.index; + if (flips) { + flip_start = flips->Offset(); + } + } + if (flips) { + WriteFlips(*flips, batch, k, result); + } + end = result.index + 1; + staged.Add(result.metrics); + if (staged.blocks == Big(s.checkpoint)) { + checkpoint(); + } + } + } + auto now = Clock::now(); + if (now - last_checkpoint >= std::chrono::seconds(1)) { + checkpoint(); + } + if (now - last_sync >= std::chrono::seconds(s.sync)) { + durable(); + } + if (now - last_report >= std::chrono::seconds(s.report)) { + progress(counts() + " persisted overall trials=" + + aggregate.overall.blocks.to_string() + " " + + throughput(now), + !tty); + last_report = now; + } + bar(now); + } + checkpoint(); + bar(Clock::now(), true); + progress("batch=" + std::to_string(batch) + " k=" + std::to_string(k) + + " completed trials=" + std::to_string(batch_completed) + "/" + + std::to_string(s.size) + " overall trials=" + + completed_total.to_string() + " " + throughput(Clock::now())); + Require(batch != UINT64_MAX, "batch identity exhausted; start a new run"); + ++batch; + } + } catch (const std::exception& e) { + // Worker failures are drained above. Coordinator I/O/allocation failures + // stop and join workers. Never retry a possibly partially written record + // or replace the last good summary with a partially updated aggregate. + // Report mode can recover the complete journal prefix after storage repair. + try { + journal.Sync(); + log.Sync(); + } catch (...) { + } + throw; + } + durable(); + progress("finalizing trials=" + completed_total.to_string() + + " interrupted=" + (product_interrupted() ? "True" : "False") + + " error=" + (error.empty() ? "None" : error) + " " + + throughput(Clock::now())); + log.Sync(); + Require(error.empty(), error); +} + +void ReadFlips(std::istream& stream, + const Json& r, + const Settings& s, + bool replay, + bool random, + unsigned schema) { + auto start = stream.tellg(); + Require(start >= 0 && static_cast(start) == U64(r.at("flip start")), + "noncontiguous flip offsets"); + uint64_t batch = U64(r.at("batch id")), + first = U64(r.at("first trial index")), + end = U64(r.at("past last trial index")), + k = U64(r.at("flipped bit count")); + Stats stats; + Json old = LegacyStats(); + const size_t count = std::min(k, 524288 - k); + std::vector positions(std::max(1, count)); + std::vector selected(524288); + for (uint64_t trial = first; trial < end; ++trial) { + auto body = ReadExact(stream, 208); + Require(GetLE(body, 0, 8) == batch && GetLE(body, 8, 8) == trial && + GetLE(body, 16, 4) == k, + "flip trial identity mismatch"); + Require(GetLE(body, 20, 4) == count && + GetLE(body, 24, 1) == uint64_t(k > 262144), + "invalid flip count/complement"); + body += ReadExact(stream, count * 4); + Require(ReadExact(stream, 32) == Hash(body), "corrupt flip checksum"); + std::fill(selected.begin(), selected.end(), false); + for (size_t i = 0; i < count; ++i) { + auto p = GetLE(body, 208 + 4 * i, 4); + Require(p < selected.size() && !selected[p], + "invalid or duplicate flip position"); + selected[p] = true; + positions[i] = static_cast(p); + } + std::array metrics{}; + for (size_t i = 0; i < 22; ++i) { + metrics[i] = GetLE(body, 32 + 8 * i, 8); + } + if (replay) { + std::array actual{}; + int status = product_trial_reference( + s.seed, batch, trial, k, s.passes, s.anchors, s.binary, actual.data(), + 2, positions.data(), random, nullptr); + Require(status == 0 && actual == metrics, + "replay mismatch batch=" + std::to_string(batch) + " trial=" + + std::to_string(trial) + " status=" + std::to_string(status)); + } + stats.Add(metrics); + if (schema == 1) { + AddLegacyTrial(old, metrics); + } + } + auto end_offset = stream.tellg(); + Require(end_offset >= 0 && + static_cast(end_offset) == U64(r.at("flip end")), + "flip end offset mismatch"); + Require((schema == 1 ? old : stats.ToJson()) == r.at("statistics"), + "flip metrics disagree with journal"); +} + +void Recover(const fs::path& directory, bool replay) { + std::ifstream meta_file(directory / "metadata.json", std::ios::binary); + Require(meta_file.good(), "cannot read metadata"); + std::string text; + char c; + while (meta_file.get(c)) { + Require(text.size() < 131072, "metadata exceeds size limit"); + text += c; + } + Require(meta_file.eof() && !meta_file.bad(), "metadata read failed"); + Json metadata = Parse(text); + std::set fields{"schema revision", "created at", "settings", + "code", "random algorithm"}; + if (metadata.contains("codeword")) { + fields.insert("codeword"); + } + Fields(metadata, fields); + auto schema = U64(metadata.at("schema revision")); + Require((schema == 1 || schema == 2) && metadata.at("code") == Json(kCode) && + (metadata.at("random algorithm") == Json(kFloyd) || + metadata.at("random algorithm") == Json(kFisherYates)), + "incompatible metadata schema/code/random algorithm"); + Require(metadata.at("created at").is_string(), "invalid created at"); + const auto codeword = metadata.contains("codeword") + ? metadata.at("codeword").as() + : "random"; + Require(codeword == "zero" || codeword == "random", + "incompatible codeword convention"); + auto settings = Settings::FromJson(metadata.at("settings")); + // Never insert defaults before hashing legacy metadata. + const auto digest = Hash(Dump(metadata)); + Aggregate aggregate(Hex(digest), static_cast(schema)); + const bool recorded = metadata.at("random algorithm") == Json(kFisherYates); + Require(!replay || recorded, "this run has no saved flips"); + std::ifstream journal(directory / "journal.jsonl", std::ios::binary), flips; + Require(journal.good(), "cannot read journal"); + if (recorded) { + flips.open(directory / "flips.bin", std::ios::binary); + Require(flips.good() && ReadExact(flips, 40) == "RSFLIP01" + digest, + "invalid flip file identity/version"); + } + Big index = 0; + std::optional> previous; + bool incomplete = false; + while (auto line = ReadLine(journal, incomplete)) { + try { + Json r = Parse(*line); + fields = {"schema revision", "run identity", "increment id", + "batch id", "first trial index", "past last trial index", + "trial count", "flipped bit count", "statistics"}; + if (recorded) { + fields.insert("flip start"); + fields.insert("flip end"); + } + Fields(r, fields); + Require(U64(r.at("schema revision")) == schema && + r.at("run identity") == Json(aggregate.identity), + "incompatible journal identity/schema"); + Require(Natural(r.at("increment id")) == index, + "duplicate or out-of-order increment id"); + uint64_t batch = U64(r.at("batch id")), + first = U64(r.at("first trial index")), + end = U64(r.at("past last trial index")), + count = U64(r.at("trial count")); + Require((settings.batches == 0 || batch < settings.batches) && + first < end && end <= settings.size && end - first == count && + count <= settings.checkpoint, + "invalid trial range/count"); + if (previous) { + Require(batch >= previous->first && + (batch != previous->first || first >= previous->second), + "overlapping or out-of-order trial ranges"); + } + uint64_t k = + product_batch_k(settings.seed, batch, settings.lo, settings.hi); + Require(U64(r.at("flipped bit count")) == k, + "batch k disagrees with seed/settings"); + Stats stats; + if (schema == 1) { + const auto& old = r.at("statistics"); + ValidateLegacy(old, count, k, settings.passes); + stats.blocks = count; + stats.iterations = Natural(old.at(kMetrics[12]).at("sum")); + stats.info_raw = Natural(old.at(kMetrics[2]).at("sum")); + stats.info_post = Natural(old.at(kMetrics[6]).at("sum")); + stats.full_raw = Natural(old.at(kMetrics[0]).at("sum")); + stats.full_post = Natural(old.at(kMetrics[4]).at("sum")); + } else { + stats = + Stats::FromJson(r.at("statistics"), Big(count), k, settings.passes); + } + if (recorded) { + ReadFlips(flips, r, settings, replay, codeword == "random", schema); + } + aggregate.Add(k, stats, r.at("statistics")); + previous = {batch, end}; + } catch (const std::exception& e) { + throw std::runtime_error("journal line " + (index + 1).to_string() + + ": " + e.what()); + } + ++index; + } + if (recorded && flips.peek() != std::char_traits::eof()) { + std::cerr << "unreferenced flip tail ignored (not committed trials)\n"; + } + AtomicJson(directory / "summary.json", aggregate.Summary()); + std::cerr << Timestamp() << " regenerated " << aggregate.overall.blocks + << " trials; ignored incomplete tail=" + << (incomplete ? "True" : "False") << "\n"; + if (replay) { + std::cerr << "verified replay: " << aggregate.overall.blocks + << " trials, all 22 metrics match\n"; + } +} + +int Main(int argc, char** argv) { + Settings s; + std::string mode, sampler = "floyd"; + fs::path directory; + bool seeded = false; + int options = 0; + for (int i = 1; i < argc; ++i) { + std::string flag = argv[i], value; + auto equals = flag.find('='); + if (equals != std::string::npos) { + value = flag.substr(equals + 1); + flag.resize(equals); + } + if (flag == "--help" || flag == "-h") { + std::cout + << "Native fixed-weight RS product Monte Carlo (schema 2)\n" + "Usage: rs-product-monte-carlo --output NEW_DIRECTORY [options]\n" + " rs-product-monte-carlo --report DIRECTORY | --replay " + "DIRECTORY\n" + "--seed UINT64 (default: generated, printed and persisted before " + "work)\n" + "--batch-size 1000 --batches 0 (infinite) --threads 1 (1..1024)\n" + "--minimum-flipped-bits 2500 --maximum-flipped-bits 2700 " + "(inclusive, 0..524288)\n" + "--max-directional-passes 16 (2..1000000)\n" + "--[no-]anchors --[no-]binary-image (both enabled)\n" + "--sampler floyd|fisher-yates (default floyd; Fisher-Yates saves " + "RSFLIP01)\n" + "--checkpoint-trials 64 (1..4096, also bounds concurrency)\n" + "--report-seconds 2 --fsync-seconds 5 (1..86400)\n" + "One uniform k per batch; exactly k flips per all-zero block. No " + "resume.\n" + "Counters are exact arbitrary-precision JSON integers; iterations " + "are directional passes.\n" + "SIGINT/SIGTERM stop starts, drain whole in-flight blocks, " + "checkpoint and fsync.\n" + "Crash loss: bounded worker window/staged increment plus journal " + "writes since fsync.\n" + "Ordered journal flush at least once/second subject to in-flight " + "completion and I/O.\n" + "Saved flips are fsynced before journal references; replay checks " + "all 22 private metrics.\n" + "Legacy absent codeword means random; report ignores only an " + "incomplete final line.\n"; + return 0; + } + ++options; + if (flag == "--anchors" || flag == "--no-anchors" || + flag == "--binary-image" || flag == "--no-binary-image") { + Require(equals == std::string::npos, "boolean flags take no value"); + if (flag == "--anchors" || flag == "--no-anchors") { + s.anchors = flag == "--anchors"; + } else { + s.binary = flag == "--binary-image"; + } + continue; + } + if (equals == std::string::npos) { + Require(i + 1 < argc, "missing value for " + flag); + value = argv[++i]; + } + if (flag == "--output" || flag == "--report" || flag == "--replay") { + Require(mode.empty() && !value.empty(), + "specify exactly one output/report/replay directory"); + mode = flag; + directory = value; + continue; + } + if (flag == "--sampler") { + sampler = value; + continue; + } + Require(!value.empty() && + std::all_of(value.begin(), value.end(), + [](char c) { return c >= '0' && c <= '9'; }), + "expected unsigned decimal integer for " + flag); + auto n = Big::from_string(value); + Require(n <= Big(UINT64_MAX), "integer exceeds uint64"); + auto v = static_cast(n); + if (flag == "--seed") { + s.seed = v; + seeded = true; + } else if (flag == "--batch-size") { + s.size = v; + } else if (flag == "--batches") { + s.batches = v; + } else if (flag == "--threads") { + s.threads = v; + } else if (flag == "--minimum-flipped-bits") { + s.lo = v; + } else if (flag == "--maximum-flipped-bits") { + s.hi = v; + } else if (flag == "--max-directional-passes") { + s.passes = v; + } else if (flag == "--checkpoint-trials") { + s.checkpoint = v; + } else if (flag == "--report-seconds") { + s.report = v; + } else if (flag == "--fsync-seconds") { + s.sync = v; + } else { + throw std::runtime_error("unknown option: " + flag); + } + } + Require(!mode.empty(), + "specify exactly one of --output, --report or --replay"); + if (mode != "--output") { + Require(options == 1, mode + " accepts only a directory"); + File lock(directory / "run.lock", O_WRONLY | O_CREAT | O_NOFOLLOW); + lock.Lock(); + Recover(directory, mode == "--replay"); + return 0; + } + s.Validate(); + Require(sampler == "floyd" || sampler == "fisher-yates", "invalid sampler"); + if (!seeded) { + Require(RAND_bytes(reinterpret_cast(&s.seed), + sizeof(s.seed)) == 1, + "OpenSSL random seed generation failed"); + } + Require(fs::create_directory(directory), + "output directory must not already exist"); + File lock(directory / "run.lock", O_WRONLY | O_CREAT | O_EXCL); + lock.Lock(); + Run(directory, s, sampler == "fisher-yates"); + return 0; +} +} // namespace mc + +int main(int argc, char** argv) { + try { + return mc::Main(argc, argv); + } catch (const std::exception& e) { + std::cerr << mc::Timestamp() << " error: " << e.what() << "\n"; + return 1; + } +} diff --git a/tools/product_monte_carlo_data.h b/tools/product_monte_carlo_data.h new file mode 100644 index 0000000..147502b --- /dev/null +++ b/tools/product_monte_carlo_data.h @@ -0,0 +1,331 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mc { +using Json = jsoncons::json; +using Big = jsoncons::bigint; +inline constexpr std::string_view kCode = + "RS256,224 x RS256,254 Cantor systematic row major"; +inline constexpr std::string_view kFloyd = + "splitmix64 domain seeds; mt19937_64; rejection modulo; Floyd complement " + "v1"; +inline constexpr std::string_view kFisherYates = + "splitmix64 domain seeds; mt19937_64; rejection modulo; persistent " + "Fisher-Yates complement v1; replay saved flips"; +inline constexpr std::array kMetrics{ + "initial full block corrupted bits", + "initial full block corrupted bytes", + "initial information corrupted bits", + "initial information corrupted bytes", + "residual full block bits", + "residual full block bytes", + "residual information bits", + "residual information bytes", + "message failures", + "full block failures", + "zero syndrome outcomes", + "zero syndrome wrong full blocks", + "directional passes", + "accepted bit changes", + "accepted byte changes", + "strong accepted bit changes", + "strong accepted byte changes", + "weak accepted bit changes", + "weak accepted byte changes", + "strong lines visited", + "weak lines visited", + "pass limit outcomes"}; + +inline void Require(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +inline Json Parse(const std::string& text) { + const auto options = jsoncons::json_options() + .lossless_number(true) + .max_nesting_depth(32) + .err_handler(jsoncons::strict_json_parsing()); + // Validate the library's event stream before DOM construction can discard + // duplicate keys. Number tokens larger than uint64 retain their exact digits. + std::vector> objects; + jsoncons::json_string_cursor cursor(text, options); + for (; !cursor.done(); cursor.next()) { + const auto& event = cursor.current(); + using E = jsoncons::staj_event_type; + if (event.event_type() == E::begin_object) { + objects.emplace_back(); + } + if (event.event_type() == E::end_object) { + objects.pop_back(); + } + if (event.event_type() == E::key) { + Require(objects.back().insert(event.get()).second, + "duplicate JSON key"); + } + Require(event.event_type() != E::double_value && + event.tag() != jsoncons::semantic_tag::bigdec, + "noninteger JSON number"); + } + return Json::parse(text, options); +} + +inline std::string Dump(const Json& value, bool pretty = false) { + std::string text; + auto options = jsoncons::json_options() + .bigint_format(jsoncons::bigint_chars_format::number) + .escape_all_non_ascii(true) + .indent_size(2); + if (!pretty) { + options.spaces_around_colon(jsoncons::spaces_option::no_spaces) + .spaces_around_comma(jsoncons::spaces_option::no_spaces); + } + value.dump( + text, options, + pretty ? jsoncons::indenting::indent : jsoncons::indenting::no_indent); + return text; +} + +inline Big Natural(const Json& value) { + if (value.is_uint64()) { + return Big(value.as()); + } + Require(value.tag() == jsoncons::semantic_tag::bigint, + "expected unsigned integer"); + auto n = Big::from_string(value.as()); + Require(n >= 0, "expected unsigned integer"); + return n; +} +inline uint64_t U64(const Json& value) { + auto n = Natural(value); + Require(n <= Big(UINT64_MAX), "integer exceeds uint64"); + return static_cast(n); +} +inline Json Number(const Big& value) { + if (value <= Big(UINT64_MAX)) { + return Json(static_cast(value)); + } + return Json(value.to_string(), jsoncons::semantic_tag::bigint); +} +inline void Fields(const Json& value, std::set expected) { + Require(value.is_object() && value.size() == expected.size(), + "incompatible object fields"); + for (const auto& item : value.object_range()) { + Require(expected.erase(std::string(item.key())) == 1, + "incompatible object field"); + } +} + +struct Settings { + uint64_t seed = 0, size = 1000, batches = 0, threads = 1, lo = 2500, + hi = 2700, passes = 16, checkpoint = 64, report = 2, sync = 5; + bool anchors = true, binary = true; + + void Validate() const { + Require(lo <= hi && hi <= 524288, + "require 0 <= minimum <= maximum <= 524288"); + Require(size > 0, "batch size must be positive"); + Require(threads >= 1 && threads <= 1024, "threads must be in [1,1024]"); + Require(passes >= 2 && passes <= 1000000, + "pass cap must be in [2,1000000]"); + Require(checkpoint >= 1 && checkpoint <= 4096, + "checkpoint trials must be in [1,4096]"); + Require(report >= 1 && report <= 86400 && sync >= 1 && sync <= 86400, + "report/fsync seconds must be in [1,86400]"); + } + Json ToJson() const { + Json j; + j["root seed"] = seed; + j["batch size"] = size; + j["batches"] = batches; + j["threads"] = threads; + j["minimum flipped bits"] = lo; + j["maximum flipped bits"] = hi; + j["maximum directional passes"] = passes; + j["checkpoint trials"] = checkpoint; + j["report seconds"] = report; + j["fsync seconds"] = sync; + j["anchors"] = anchors; + j["binary image"] = binary; + return j; + } + static Settings FromJson(const Json& j) { + Fields(j, {"root seed", "batch size", "batches", "threads", + "minimum flipped bits", "maximum flipped bits", + "maximum directional passes", "checkpoint trials", + "report seconds", "fsync seconds", "anchors", "binary image"}); + Settings s; + s.seed = U64(j.at("root seed")); + s.size = U64(j.at("batch size")); + s.batches = U64(j.at("batches")); + s.threads = U64(j.at("threads")); + s.lo = U64(j.at("minimum flipped bits")); + s.hi = U64(j.at("maximum flipped bits")); + s.passes = U64(j.at("maximum directional passes")); + s.checkpoint = U64(j.at("checkpoint trials")); + s.report = U64(j.at("report seconds")); + s.sync = U64(j.at("fsync seconds")); + Require(j.at("anchors").is_bool() && j.at("binary image").is_bool(), + "gates must be boolean"); + s.anchors = j.at("anchors").as(); + s.binary = j.at("binary image").as(); + s.Validate(); + return s; + } +}; + +struct Stats { + // All public accumulators are arbitrary precision. Trial metrics remain + // bounded uint64 values in the private ABI and legacy flip records. + Big blocks = 0, iterations = 0, info_raw = 0, info_post = 0, full_raw = 0, + full_post = 0; + void Add(const std::array& m) { + ++blocks; + iterations += m[12]; + info_raw += m[2]; + info_post += m[6]; + full_raw += m[0]; + full_post += m[4]; + } + void Add(const Stats& s) { + blocks += s.blocks; + iterations += s.iterations; + info_raw += s.info_raw; + info_post += s.info_post; + full_raw += s.full_raw; + full_post += s.full_post; + } + Json ToJson() const { + Json j; + j["completed blocks"] = Number(blocks); + j["total iterations"] = Number(iterations); + for (bool info : {true, false}) { + Json bits; + bits["total bits"] = Number(blocks * (info ? 455168 : 524288)); + bits["raw corrupted bits"] = Number(info ? info_raw : full_raw); + bits["post decoding corrupted bits"] = + Number(info ? info_post : full_post); + j[info ? "information bits" : "full-codeword bits"] = std::move(bits); + } + return j; + } + static Stats FromJson(const Json& j, + const Big& count, + uint64_t k, + uint64_t passes) { + Fields(j, {"completed blocks", "total iterations", "information bits", + "full-codeword bits"}); + Stats s; + s.blocks = Natural(j.at("completed blocks")); + s.iterations = Natural(j.at("total iterations")); + Require(s.blocks == count && s.iterations >= count * 2 && + s.iterations <= count * Big(passes), + "inconsistent block/iteration counts"); + for (bool info : {true, false}) { + const auto& bits = j.at(info ? "information bits" : "full-codeword bits"); + Fields(bits, {"total bits", "raw corrupted bits", + "post decoding corrupted bits"}); + const auto total = count * (info ? 455168 : 524288); + auto raw = Natural(bits.at("raw corrupted bits")); + auto post = Natural(bits.at("post decoding corrupted bits")); + Require(Natural(bits.at("total bits")) == total && raw <= total && + post <= total, + "invalid bit counters"); + (info ? s.info_raw : s.full_raw) = raw; + (info ? s.info_post : s.full_post) = post; + } + Require(s.full_raw == count * Big(k), "initial channel is not exact k"); + Require(s.info_raw <= s.full_raw && s.info_post <= s.full_post && + s.full_raw - s.info_raw <= count * 69120 && + s.full_post - s.info_post <= count * 69120, + "inconsistent information/full bit counters"); + return s; + } +}; + +inline Json LegacyStats() { + Json j; + for (auto name : kMetrics) { + j[name]["sum"] = 0; + j[name]["squared sum"] = 0; + } + return j; +} +inline void AddLegacy(Json& to, const Json& from) { + for (auto name : kMetrics) { + for (auto field : {"sum", "squared sum"}) { + to[name][field] = Number(Natural(to.at(name).at(field)) + + Natural(from.at(name).at(field))); + } + } +} +inline void AddLegacyTrial(Json& to, const std::array& m) { + for (size_t i = 0; i < m.size(); ++i) { + auto& item = to.at(kMetrics[i]); + item["sum"] = Number(Natural(item.at("sum")) + Big(m[i])); + item["squared sum"] = + Number(Natural(item.at("squared sum")) + Big(m[i]) * Big(m[i])); + } +} +inline void ValidateLegacy(const Json& j, + uint64_t count, + uint64_t k, + uint64_t passes) { + Fields(j, std::set(kMetrics.begin(), kMetrics.end())); + std::array bounds{524288, + 65536, + 455168, + 56896, + 524288, + 65536, + 455168, + 56896, + 1, + 1, + 1, + 1, + passes, + passes * 524288, + passes * 524288, + passes * 524288, + passes * 524288, + passes * 524288, + passes * 524288, + passes * 524288, + passes * 524288, + 1}; + for (size_t i = 0; i < kMetrics.size(); ++i) { + const auto& item = j.at(kMetrics[i]); + Fields(item, {"sum", "squared sum"}); + auto sum = Natural(item.at("sum")), + square = Natural(item.at("squared sum")); + Require(sum <= Big(count) * Big(bounds[i]) && + square <= Big(count) * Big(bounds[i]) * Big(bounds[i]) && + sum * sum <= Big(count) * square && sum <= square && + square <= Big(bounds[i]) * sum, + "inconsistent legacy moments"); + } + Require(Natural(j.at(kMetrics[0]).at("sum")) == Big(count) * Big(k) && + Natural(j.at(kMetrics[0]).at("squared sum")) == + Big(count) * Big(k) * Big(k), + "initial channel is not exact k"); + for (size_t i : {13, 14}) { + Require(Natural(j.at(kMetrics[i]).at("sum")) == + Natural(j.at(kMetrics[i + 2]).at("sum")) + + Natural(j.at(kMetrics[i + 4]).at("sum")), + "directional accepted totals disagree"); + } +} +} // namespace mc diff --git a/tools/product_monte_carlo_native.cc b/tools/product_monte_carlo_native.cc new file mode 100644 index 0000000..d758e8a --- /dev/null +++ b/tools/product_monte_carlo_native.cc @@ -0,0 +1,344 @@ +// 2026-09-08, Ryzen 8845HS/WSL, /check cli Release -march=native. +// CLI: seed=42, k=2600, 2 batches x 4000 trials, default gates/checkpoints; +// all workers verified with affinity 0-15. Whole-simulation wall blocks/s: +// threads 1 2 4 8 +// before 634.5 1132.6 2089.3 821.9 +// after 1124.4 1907.5 3191.0 1025.0 +// Single-worker native phase wall us/block: encode 383 -> 114, +// both difference scans 375 -> 30. Timers removed; seeded metrics unchanged. +// Eight-worker CPU-time inflation remains unresolved on this host. +// +// 2026-09-08 persistent Fisher-Yates experiment, same host and /check cli +// build. seed=42, k=2600, 2x4000 trials, affinity 0-15, default +// gates/checkpoints. Median of 3 rotated-order runs; whole-process wall +// blocks/s incl. final fsync: threads 1 4 8 +// Floyd 1081.5 2782.4 839.5 +// FY diagnostic/no disk 945.9 2662.9 956.3 +// FY saved flips 806.8 1959.8 785.8 +// Each saved run: 85,120,040 bytes; eight-worker measurements were noisy. +// Retained opt-in only: --sampler fisher-yates always saves replayable flips; +// default Floyd remains unchanged. No-disk FY is not exposed by the CLI. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "reed_solomon/strong_weak_rs_product_code.h" + +namespace { +static_assert(std::atomic::is_always_lock_free); +std::atomic interrupted{false}; +void Interrupt(int) { + interrupted = 1; +} + +uint64_t Mix(uint64_t x) { + x += 0x9e3779b97f4a7c15ULL; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ULL; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebULL; + return x ^ (x >> 31); +} +uint64_t Uniform(std::mt19937_64& rng, uint64_t n) { + const uint64_t threshold = -n % n; + uint64_t x; + do { + x = rng(); + } while (x < threshold); + return x % n; +} + +gf2p8::lch::Status Encode(std::span block) { + using gf2p8::Element; + thread_local gf2p8::rs::LCHEncoder weak(254, 2), strong(224, 32); + thread_local std::vector columns(256 * 224); + thread_local std::vector workspace( + std::max(weak.WorkspaceSize(224), strong.WorkspaceSize(256))); + std::array data{}; + std::array recovery{}; + // Independent weak rows become SIMD byte lanes of one shard encode. + for (size_t col = 0; col < 254; ++col) { + data[col] = columns.data() + col * 224; + for (size_t row = 0; row < 224; ++row) { + columns[col * 224 + row] = block[row * 256 + col]; + } + } + recovery[0] = columns.data() + 254 * 224; + recovery[1] = columns.data() + 255 * 224; + auto status = weak.Encode(data, std::span(recovery).first(2), 224, workspace); + if (status != gf2p8::lch::Status::ok) { + return status; + } + for (size_t row = 0; row < 224; ++row) { + block[row * 256 + 254] = recovery[0][row]; + block[row * 256 + 255] = recovery[1][row]; + data[row] = block.data() + row * 256; + } + for (size_t row = 0; row < 32; ++row) { + recovery[row] = block.data() + (224 + row) * 256; + } + return strong.Encode(std::span(data).first(224), recovery, 256, workspace); +} + +template +void CountDifferences(const std::vector& block, + const gf2p8::Element* original, + uint64_t* out) { + // Keep the contiguous reduction free of per-byte information-region branches. + for (size_t row = 0; row < 256; ++row) { + uint64_t bits = 0, bytes = 0; + for (size_t col = 0; col < 254; ++col) { + const unsigned d = + block[row * 256 + col] ^ (Random ? original[row * 256 + col] : 0); + bits += std::popcount(d); + bytes += d != 0; + } + out[0] += bits; + out[1] += bytes; + if (row < 224) { + out[2] += bits; + out[3] += bytes; + } + for (size_t col = 254; col < 256; ++col) { + const unsigned d = + block[row * 256 + col] ^ (Random ? original[row * 256 + col] : 0); + out[0] += std::popcount(d); + out[1] += d != 0; + } + } +} +template +int Trial(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + int sampler, + uint32_t* positions, + uint8_t* residual) { + try { + // Local counters cannot alias the byte buffers and are published only once. + uint64_t out[22]{}; + if (!output || k > 524288 || passes < 2 || passes > 1000000 || + sampler < 0 || sampler > 2 || (sampler == 2 && !positions)) { + return 1; + } + thread_local gf2p8::rs::StrongWeakRSProductCode code; + thread_local std::vector block(65536); + thread_local std::vector selected; + thread_local std::vector permutation; + const auto key = Mix(seed ^ Mix(batch) ^ Mix(trial ^ 0x545249414cULL)); + std::mt19937_64 noise(Mix(key ^ 0x4e4f495345ULL)); + const gf2p8::Element* original = nullptr; + if constexpr (Random) { + // Legacy replay/reference only. Zero trials neither allocate this buffer + // nor construct a message PRNG or encoder, even on first worker use. + thread_local std::vector reference(65536); + std::fill(reference.begin(), reference.end(), 0); + std::mt19937_64 message(Mix(key ^ 0x4d455353414745ULL)); + for (size_t r = 0; r < 224; ++r) { + for (size_t c = 0; c < 254; ++c) { + reference[r * 256 + c] = static_cast(message()); + } + } + if (Encode(reference) != gf2p8::lch::Status::ok) { + return 2; + } + original = reference.data(); + block = reference; + } else { + // Linearity gives S(c XOR e) = S(e). BDD deltas, anchor decisions and + // binary-image delta gates therefore depend on e, not the sent codeword. + std::fill(block.begin(), block.end(), 0); + } + // Sample the smaller of the flip set and its complement. Floyd uses a + // cleared bitmap; opt-in Fisher-Yates retains a 2 MiB worker permutation. + const bool complement = k > 262144; + const size_t count = complement ? 524288 - k : k; + if (sampler != 1) { + selected.assign(524288, 0); + } else if (permutation.empty()) { + permutation.resize(524288); + std::iota(permutation.begin(), permutation.end(), 0u); + } + if (complement) { + for (auto& byte : block) { + byte ^= 255; + } + } + for (size_t i = 0; i < count; ++i) { + size_t pos; + if (sampler == 1) { + // Any prior permutation gives a uniform subset. Do not reset it; + // replay requires saved positions, not just schedule-dependent seeds. + const size_t draw = i + Uniform(noise, 524288 - i); + std::swap(permutation[i], permutation[draw]); + pos = permutation[i]; + } else if (sampler == 2) { + pos = positions[i]; + if (pos >= 524288 || selected[pos]) { + return 6; + } + selected[pos] = 1; + } else { + const size_t j = 524288 - count + i; + const size_t draw = Uniform(noise, j + 1); + pos = selected[draw] ? j : draw; + selected[pos] = 1; + } + if (positions && sampler != 2) { + positions[i] = static_cast(pos); + } + block[pos / 8] ^= static_cast(1u << (pos % 8)); + } + CountDifferences(block, original, out); + if (out[0] != k) { + return 4; + } + const auto result = code.Correct( + block, {static_cast(passes), anchors != 0, binary != 0}); + if (result.termination == gf2p8::rs::ProductTermination::invalid_argument) { + return 3; + } + CountDifferences(block, original, out + 4); + out[8] = out[6] != 0; + out[9] = out[4] != 0; + out[10] = result.all_zero_syndromes; + out[11] = result.all_zero_syndromes && out[9]; + out[12] = result.directional_passes; + out[13] = result.changed_bits; + out[14] = result.changed_symbols; + out[15] = result.strong_changed_bits; + out[16] = result.strong_changed_symbols; + out[17] = result.weak_changed_bits; + out[18] = result.weak_changed_symbols; + out[19] = result.strong_lines_visited; + out[20] = result.weak_lines_visited; + out[21] = result.termination == gf2p8::rs::ProductTermination::pass_limit; + if (residual) { + for (size_t i = 0; i < block.size(); ++i) { + residual[i] = block[i] ^ (Random ? original[i] : 0); + } + } + std::copy(out, out + 22, output); + return 0; + } catch (...) { + return 5; + } +} +} // namespace + +// Private trial ABI for the native CLI and test-only reference. No C++ +// exception crosses the boundary. Counters/residuals publish only after +// successful trials; sampler output positions are scratch and must be ignored +// on failure. +extern "C" { +int product_interrupt_install() { + interrupted = 0; + struct sigaction action {}; + action.sa_handler = Interrupt; + sigemptyset(&action.sa_mask); + return sigaction(SIGINT, &action, nullptr) || + sigaction(SIGTERM, &action, nullptr); +} +int product_interrupted() { + return interrupted; +} +uint64_t product_batch_k(uint64_t seed, + uint64_t batch, + uint64_t lo, + uint64_t hi) { + std::mt19937_64 rng(Mix(seed ^ Mix(batch) ^ 0x4241544348ULL)); + return lo + Uniform(rng, hi - lo + 1); +} +// Explicit random reference for old saved records and differential tests. +// Optional residual is the complete decoded block XOR the original codeword. +int product_trial_reference(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + int sampler, + uint32_t* positions, + int random, + uint8_t* residual) { + if (random != 0 && random != 1) { + return 1; + } + return random ? Trial(seed, batch, trial, k, passes, anchors, binary, + output, sampler, positions, residual) + : Trial(seed, batch, trial, k, passes, anchors, binary, + output, sampler, positions, residual); +} +int product_trial_flips(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + int sampler, + uint32_t* positions) { + return Trial(seed, batch, trial, k, passes, anchors, binary, output, + sampler, positions, nullptr); +} +int product_trial(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output) { + return product_trial_flips(seed, batch, trial, k, passes, anchors, binary, + output, 0, nullptr); +} +// Bounded coordinator task. Publish each successful prefix trial separately; +// check signals between blocks rather than delaying them for the whole task. +int product_trials(uint64_t seed, + uint64_t batch, + uint64_t first, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + uint64_t count, + int sampler, + uint32_t* positions, + uint64_t* completed) { + if (!completed) { + return 1; + } + *completed = 0; + if (!output || count == 0 || count > 4 || first > UINT64_MAX - (count - 1) || + k > 524288 || sampler < 0 || sampler > 1 || + (sampler == 1 && !positions)) { + return 1; + } + const uint64_t stride = std::min(k, 524288 - k); + for (uint64_t i = 0; i < count && !interrupted; ++i) { + const int status = product_trial_flips( + seed, batch, first + i, k, passes, anchors, binary, output + 22 * i, + sampler, positions ? positions + stride * i : nullptr); + if (status) { + return status; + } + ++*completed; + } + return 0; +} +} diff --git a/tools/product_monte_carlo_trials.h b/tools/product_monte_carlo_trials.h new file mode 100644 index 0000000..1a933c7 --- /dev/null +++ b/tools/product_monte_carlo_trials.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +// Private trial ABI retained for legacy differential tests and RSFLIP01 replay. +extern "C" { +int product_interrupt_install(); +int product_interrupted(); +uint64_t product_batch_k(uint64_t seed, + uint64_t batch, + uint64_t lo, + uint64_t hi); +int product_trial_reference(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + int sampler, + uint32_t* positions, + int random, + uint8_t* residual); +} From 37a6b3fc80681a4d52d9dbccb409d48adb2cf9ad Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Wed, 9 Sep 2026 18:18:21 +0300 Subject: [PATCH 3/7] Refine Monte Carlo aggregation and pooled BER plotting --- CMakeLists.txt | 17 +- scripts/install-monte-carlo.sh | 2 +- scripts/plot_product_monte_carlo.py | 248 +++++++++++++---- tests/plot_product_monte_carlo_test.py | 351 ++++++++++++++++++++++++ tests/product_monte_carlo_data_tests.cc | 178 +++++++++++- tests/product_monte_carlo_test.py | 54 ++++ tools/product_monte_carlo.cc | 109 +++----- tools/product_monte_carlo_data.h | 318 ++++++++++++++------- tools/product_monte_carlo_native.cc | 2 + 9 files changed, 1041 insertions(+), 238 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c45841..f4071c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,12 +91,13 @@ endif() if(GF256_BUILD_MONTE_CARLO) find_package(Threads REQUIRED) find_package(OpenSSL 3 REQUIRED COMPONENTS Crypto) - FetchContent_Declare(jsoncons - GIT_REPOSITORY https://github.com/danielaparker/jsoncons.git - GIT_TAG v0.177.0 + FetchContent_Declare(nlohmann_json + GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v3.12.0 GIT_SHALLOW TRUE) - set(JSONCONS_BUILD_TESTS OFF CACHE BOOL "" FORCE) - FetchContent_MakeAvailable(jsoncons) + set(JSON_BuildTests OFF CACHE BOOL "" FORCE) + set(JSON_Install OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(nlohmann_json) add_library(product_monte_carlo_trials OBJECT tools/product_monte_carlo_native.cc) target_link_libraries(product_monte_carlo_trials PRIVATE gf256_core) set_target_properties(product_monte_carlo_trials PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -105,7 +106,7 @@ if(GF256_BUILD_MONTE_CARLO) endif() add_executable(rs-product-monte-carlo tools/product_monte_carlo.cc) target_link_libraries(rs-product-monte-carlo PRIVATE product_monte_carlo_trials - gf256_core Threads::Threads OpenSSL::Crypto jsoncons) + gf256_core Threads::Threads OpenSSL::Crypto nlohmann_json::nlohmann_json) include(GNUInstallDirs) install(TARGETS rs-product-monte-carlo RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT product-monte-carlo) @@ -194,13 +195,13 @@ if(GF256_BUILD_TESTS) configure_file(tests/reference/product_monte_carlo.py product_monte_carlo_reference.py COPYONLY) add_executable(product_monte_carlo_data_tests tests/product_monte_carlo_data_tests.cc) target_include_directories(product_monte_carlo_data_tests PRIVATE tools) - target_link_libraries(product_monte_carlo_data_tests PRIVATE jsoncons gtest_main) + target_link_libraries(product_monte_carlo_data_tests PRIVATE nlohmann_json::nlohmann_json gtest_main) target_compile_features(product_monte_carlo_data_tests PRIVATE cxx_std_20) add_test(NAME ProductMonteCarloData COMMAND product_monte_carlo_data_tests) add_executable(product_monte_carlo_fault_cli tools/product_monte_carlo.cc) target_compile_definitions(product_monte_carlo_fault_cli PRIVATE GF256_MC_TEST_HOOKS=1) target_link_libraries(product_monte_carlo_fault_cli PRIVATE product_monte_carlo_trials - gf256_core Threads::Threads OpenSSL::Crypto jsoncons) + gf256_core Threads::Threads OpenSSL::Crypto nlohmann_json::nlohmann_json) add_test(NAME ProductMonteCarloCli COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tests/product_monte_carlo_test.py ${PROJECT_BINARY_DIR}/rs-product-monte-carlo) diff --git a/scripts/install-monte-carlo.sh b/scripts/install-monte-carlo.sh index 5d21b28..6c10962 100644 --- a/scripts/install-monte-carlo.sh +++ b/scripts/install-monte-carlo.sh @@ -2,7 +2,7 @@ set -euo pipefail if [[ ${1:-} == --help || ${1:-} == -h ]]; then - printf 'Usage: %s [PREFIX]\nBuild native Release rs-product-monte-carlo and install it (default: ~/.local).\nRequires CMake, a C++20 compiler, OpenSSL 3 development files, and Git/network for pinned jsoncons headers. No Python runtime or sudo.\n' "$0" + printf 'Usage: %s [PREFIX]\nBuild native Release rs-product-monte-carlo and install it (default: ~/.local).\nRequires CMake, a C++20 compiler, OpenSSL 3 development files, and Git/network for pinned nlohmann/json v3.12.0 headers. No Python runtime or sudo.\n' "$0" exit 0 fi if (( $# > 1 )) || [[ ${1:-} == -* ]]; then diff --git a/scripts/plot_product_monte_carlo.py b/scripts/plot_product_monte_carlo.py index 163a1fa..4c5d45b 100644 --- a/scripts/plot_product_monte_carlo.py +++ b/scripts/plot_product_monte_carlo.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Plot sampled-stratum BER contributions, not extrapolated decoder BER.""" +"""Pool reports and plot fixed-weight conditional BER and/or BSC contributions.""" import argparse import csv @@ -196,9 +196,44 @@ def load_report(path): raise ValueError(f"{path}: {error}") from error +def discover_reports(paths): + """Expand containers in sorted order, stopping at runs and rejecting overlap.""" + reports, sources = [], {} + + def visit(path, source): + if path.is_dir(): + metadata, summary = path / "metadata.json", path / "summary.json" + if metadata.exists() or summary.exists(): + require(metadata.is_file() and summary.is_file(), + f"{path}: incomplete run; expected both metadata.json and summary.json " + "as files; finish the report or move the partial run outside the input tree") + visit(summary, source) + else: + for child in sorted(path.iterdir()): + if not child.is_symlink() and child.is_dir(): + visit(child, source) + return + require(path.is_file(), f"input does not exist or is not a report file: {path}") + resolved = path.resolve() + require(resolved not in sources, + f"duplicate report source (duplicate run identity): {path}; " + f"overlapping inputs {sources.get(resolved)} and {source}; supply each run only once") + sources[resolved] = source + reports.append(path) + + for source in paths: + before = len(reports) + visit(Path(source), source) + require(len(reports) > before, + f"{source}: no runs found; expected metadata.json + summary.json pairs " + "in this directory or its descendants (subdirectory symlinks are not followed)") + require(reports, "no runs found: supply run directories, containers, or summary files") + return reports + + def pool_reports(paths, allow_mixed_codewords=False): groups, identities, seeds, conventions = {}, set(), set(), {} - for path in paths: + for path in discover_reports(paths): digest, seed, config, rows, codeword = load_report(path) require(digest not in identities, f"duplicate run identity: {path}") require((config, seed) not in seeds, @@ -297,82 +332,193 @@ def plot_value(value): return result if result > 0 else math.nan +def conditional_values(rows, metric): + """Yield provenance and normalized conditional BER in ascending k.""" + for k, row in sorted(rows.items()): + total, trials = row[metric], row["trials"] + mean = total / trials + log_mean = math.log(total) - math.log(trials) if total else NEG_INF + log_ber = log_mean - math.log(DENOMINATORS[metric]) + yield {"k": k, "residual_bits_sum": total, "completed_blocks": trials, + "mean": mean, "log10_mean": log_mean / math.log(10), "raw_ber": k / N, + "conditional_ber": math.exp(log_ber), + "log10_conditional_ber": log_ber / math.log(10)} + + +def export_conditional(groups, metrics, path): + """Export pooled conditional BER points, independent of plotted mode/limits.""" + records = [] + for config, rows in groups.items(): + for metric in metrics: + points = [] + for value in conditional_values(rows, metric): + log_ber = value["log10_conditional_ber"] + points.append({"raw_ber": value["raw_ber"], + "residual_ber": value["conditional_ber"], + "log10_residual_ber": log_ber if math.isfinite(log_ber) else None}) + records.append({"configuration": config_label(config), "metric": metric, + "points": points}) + with path.open("w", encoding="utf-8") as stream: + json.dump(records, stream, indent=2, allow_nan=False) + stream.write("\n") + + +def plot_results(groups, metrics, mode, ps, output, csv_path, plt, thin=0.5): + require(math.isfinite(thin) and thin > 0, "--thin must be finite and greater than zero") + figure, axis = plt.subplots(figsize=(11, 7)) + positive = False + total_underflows = 0 + sampled = set() + colors = plt.rcParams["axes.prop_cycle"].by_key()["color"] + with csv_path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=[ + "configuration", "metric", "p", "log10_contribution", "log10_covered_mass", + "log10_missing_mass", "sampled_strata", "zero_observed_strata", "record_type", + "k", "residual_bits_sum", "completed_blocks", "mean", "log10_mean", + "raw_ber", "conditional_ber", "log10_conditional_ber"]) + writer.writeheader() + for index, (config, rows) in enumerate(groups.items()): + label = config_label(config) + color = colors[index % len(colors)] + sampled.update(rows) + weighted = [evaluate(rows, p) for p in ps] if mode != "conditional" else [] + for metric in metrics: + zeros = sum(row[metric] == 0 for row in rows.values()) + base = {"configuration": label, "metric": metric, + "sampled_strata": len(rows), "zero_observed_strata": zeros} + if mode != "ber": + values = list(conditional_values(rows, metric)) + for value in values: + writer.writerow({**base, "record_type": "conditional", **value}) + ys = [plot_value(v["log10_conditional_ber"] * math.log(10)) for v in values] + visible = [(v["raw_ber"], y) for v, y in zip(values, ys) if math.isfinite(y)] + underflows = sum(v["residual_bits_sum"] > 0 and not math.isfinite(y) + for v, y in zip(values, ys)) + total_underflows += underflows + positive |= bool(visible) + clipped = sum(not (0.0045 <= x <= 0.008 and 1e-30 <= y <= 1e-1) + for x, y in visible) + note = f"{zeros}/{len(values)} zero-observed strata omitted" + print(f"{label}; {metric}: {note}; {underflows} positive BERs underflowed " + f"(see CSV logs); {clipped} points outside axis limits", file=sys.stderr) + axis.scatter([x for x, _ in visible], [y for _, y in visible], s=18 * thin**2, + linewidths=thin, + color=color, marker="o" if metric == "information" else "x", + label=f"Fixed-weight conditional BER; {metric}: {label}\n{note}") + if mode != "conditional": + print(f"{label}; {metric}: {len(rows)}/{N + 1} sampled strata, " + f"{zeros} zero-observed strata; log10 missing mass range " + f"[{min(v[2] for v in weighted) / math.log(10):.6g}, " + f"{max(v[2] for v in weighted) / math.log(10):.6g}]", file=sys.stderr) + for p, (contributions, covered, missing) in zip(ps, weighted): + writer.writerow({**base, "record_type": "ber", "p": p, + "log10_contribution": contributions[metric] / math.log(10), + "log10_covered_mass": covered / math.log(10), + "log10_missing_mass": missing / math.log(10)}) + axis.plot(ps, [plot_value(v[0][metric]) for v in weighted], linewidth=2 * thin, + color=color, linestyle="-" if metric == "information" else "--", + label=f"Sampled-stratum BSC contribution; {metric}: {label}") + axis.set(xscale="linear", yscale="log", xlim=(0.008, 0.0045), ylim=(1e-30, 1e-1), + xlabel="Raw BER (k / 524288 for conditional; p for BSC)", + ylabel="Residual BER", title="Product-code BER") + if mode == "conditional" and not positive: + message = ("All observed residual sums are zero; no positive BERs to plot." + if sampled else "No completed blocks in these report snapshots.") + if total_underflows: + message = "No representable positive BERs to plot; see CSV logs." + axis.text(0.5, 0.5, message + "\nNo artificial floor is used.", + transform=axis.transAxes, ha="center", va="center") + axis.grid(True, which="major", color="#d1d5db", alpha=0.75) + axis.spines[["top", "right"]].set_visible(False) + axis.legend(fontsize=8) + figure.text(0.5, 0.02, + "Fixed-weight conditional BER is not a full BSC expectation. Missing strata unknown.\n" + "Zeros omitted, not floored; zero observed errors are not certainty. " + "No weight renormalization or extrapolation.", ha="center", fontsize=9) + figure.tight_layout(rect=(0, 0.06, 1, 1)) + figure.savefig(output, facecolor="white", metadata={"Creator": __file__, "Date": None}) + plt.close(figure) + + def main(argv=None): if hasattr(sys, "set_int_max_str_digits"): sys.set_int_max_str_digits(0) parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter, - epilog="Weights are Binomial(N,p), without renormalization. Missing k are unknown; " + epilog="BER mode uses Binomial(N,p) weights, without renormalization. Conditional mode uses " + "raw BER k/524288 and residual BER sum(residual bits)/(sum(completed blocks)*D), " + "D=455168 for information or 524288 for full. Both mode overlays conditional scatter and " + "sampled-stratum BSC contribution lines; conditional BER is not a full BSC expectation. " + "All modes use descending linear x 0.008 to 0.0045 and log y 1e-30 to 1e-1; " + "out-of-range points are clipped, not discarded from CSV. " + "Zeros are omitted and counted, never floored. The unified CSV uses record_type conditional/ber, " + "retains BER columns and conditional exact sums/counts and mean/log10_mean, and adds " + "raw_ber, conditional_ber/log10_conditional_ber. Inapplicable fields are blank. " + "Missing k are unknown; " "zero observed errors are not certainty. Arbitrarily low plotted values are not reliability " "evidence. No extrapolation or MSE fit: a fitting model has not been specified. " "Matching decoder configurations pool per-k sums/counts; different flags/caps stay separate. " - "Repeated seeds within a configuration are rejected. Metadata has no source revision.") - parser.add_argument("inputs", nargs="+", type=Path, help="run directories or summary.json with sibling metadata.json") + "Repeated seeds within a configuration and overlapping inputs are rejected. " + f"Only the fixed code {CODE} is supported; other dimensions/coordinates are rejected. " + "Zero/random conventions must match unless --allow-mixed-codewords is explicit. " + "Metadata has no source revision. " + "Discovery stops at run directories, ignores unrelated files, and does not follow subdirectory " + "symlinks. Partial or malformed runs are errors, not silently skipped. " + "Example: python3 -B scripts/plot_product_monte_carlo.py experiments --mode conditional " + "--metric both --output merged.svg (also writes merged.csv). " + "Direct inputs remain supported: run-a run-b/summary.json --output merged.svg.") + parser.add_argument("inputs", nargs="+", type=Path, + help="run directories, summary files with sibling metadata.json, or containers " + "recursively searched in sorted order for metadata.json + summary.json pairs") parser.add_argument("--output", type=Path, default=Path("product_monte_carlo.svg"), help="SVG output") - parser.add_argument("--csv", type=Path, help="CSV output (default: output path with .csv suffix)") - parser.add_argument("--points", type=int, default=200) + parser.add_argument("--csv", type=Path, help="unified CSV output, both row types in both mode " + "(default: output path with .csv suffix; no sidecar)") + parser.add_argument("--export", type=Path, metavar="JSON", + help="also export only pooled conditional raw/residual BER values to indented JSON, " + "with null log10 for zero estimates, " + "including zeros and out-of-axis points, regardless of --mode") + parser.add_argument("--mode", choices=("conditional", "ber", "both"), default="ber", + help="conditional: fixed-weight BER scatter; ber: BSC contribution lines; both: overlay") + parser.add_argument("--points", type=int, default=200, help="BER p-grid size; ignored in conditional mode") parser.add_argument("--metric", choices=("information", "full", "both"), default="information") + parser.add_argument("--thin", type=float, default=0.5, + help="positive size multiplier for curve widths, marker diameters and marker strokes; " + "smaller is thinner, 1 uses the previous curve width and marker area") parser.add_argument("--allow-mixed-codewords", action="store_true", help="pool zero/random inputs using linear-code BDD translation equivariance; " "absent legacy convention means random; repeated seeds remain forbidden") args = parser.parse_args(argv) - require(args.points >= 2, "--points must be at least 2") + require(math.isfinite(args.thin) and args.thin > 0, + "--thin must be finite and greater than zero") + require(args.mode == "conditional" or args.points >= 2, "--points must be at least 2") require(args.output.suffix.lower() == ".svg", "--output must be an .svg path") csv_path = args.csv or args.output.with_suffix(".csv") - protected = {p.resolve() for path in args.inputs for p in ( - (path / "summary.json" if path.is_dir() else path), - (path if path.is_dir() else path.parent) / "metadata.json")} - require(args.output.resolve() != csv_path.resolve() and not protected.intersection( - {args.output.resolve(), csv_path.resolve()}), "output paths must be distinct from each other and inputs") - groups = pool_reports(args.inputs, args.allow_mixed_codewords) + reports = discover_reports(args.inputs) + protected = {p.resolve() for path in reports for p in (path, path.parent / "metadata.json")} + outputs = [args.output.resolve(), csv_path.resolve()] + if args.export is not None: + outputs.append(args.export.resolve()) + require(len(set(outputs)) == len(outputs) and not protected.intersection(outputs), + "output paths must be distinct from each other and inputs") + groups = pool_reports(reports, args.allow_mixed_codewords) print("Warning: metadata lacks source revision; decoder implementation compatibility cannot be verified. " "Missing strata are unknown; zero observed errors do not establish zero BER. " - "No extrapolation or MSE fit is performed.", file=sys.stderr) + "No extrapolation or MSE fit is performed. " + f"Validated {len(reports)} report(s); only the fixed code {CODE} is supported.", file=sys.stderr) metrics = list(METRICS) if args.metric == "both" else [args.metric] - ps = [0.008 + (0.0045 - 0.008) * i / (args.points - 1) for i in range(args.points)] - results = [] - for config, rows in groups.items(): - values = [evaluate(rows, p) for p in ps] - label = config_label(config) - for metric in metrics: - zeros = sum(row[metric] == 0 for row in rows.values()) - print(f"{label}; {metric}: {len(rows)}/{N + 1} sampled strata, " - f"{zeros} zero-observed strata; log10 missing mass range " - f"[{min(v[2] for v in values) / math.log(10):.6g}, " - f"{max(v[2] for v in values) / math.log(10):.6g}]", file=sys.stderr) - results.append((label, metric, zeros, len(rows), values)) try: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError as error: raise ValueError("plotting requires matplotlib; numerical core uses only the standard library") from error - with csv_path.open("w", newline="", encoding="utf-8") as stream: - writer = csv.writer(stream) - writer.writerow(["configuration", "metric", "p", "log10_contribution", "log10_covered_mass", - "log10_missing_mass", "sampled_strata", "zero_observed_strata"]) - for label, metric, zeros, sampled, values in results: - for p, (contributions, covered, missing) in zip(ps, values): - writer.writerow([label, metric, p, contributions[metric] / math.log(10), - covered / math.log(10), missing / math.log(10), sampled, zeros]) plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11, "svg.hashsalt": "gf256-product-monte-carlo"}) - figure, axis = plt.subplots(figsize=(11, 7)) - for label, metric, _, _, values in results: - axis.plot(ps, [plot_value(v[0][metric]) for v in values], linewidth=2, - label=f"{metric}: {label}") - axis.set(xlim=(0.008, 0.0045), ylim=(1e-30, 1e-1), yscale="log", - xlabel="Channel bit-flip probability p", - ylabel="Sampled-stratum BER contribution", - title="Product-code sampled-stratum BER contribution") - axis.grid(True, which="major", color="#d1d5db", alpha=0.75) - axis.spines[["top", "right"]].set_visible(False) - axis.legend(fontsize=8) - figure.text(0.5, 0.02, "Missing strata unknown; zero observed errors are not certainty. No extrapolation.", - ha="center", fontsize=9) - figure.tight_layout(rect=(0, 0.04, 1, 1)) - figure.savefig(args.output, facecolor="white", metadata={"Creator": __file__, "Date": None}) - plt.close(figure) + ps = ([0.008 + (0.0045 - 0.008) * i / (args.points - 1) for i in range(args.points)] + if args.mode != "conditional" else []) + plot_results(groups, metrics, args.mode, ps, args.output, csv_path, plt, thin=args.thin) + if args.export is not None: + export_conditional(groups, metrics, args.export) if __name__ == "__main__": diff --git a/tests/plot_product_monte_carlo_test.py b/tests/plot_product_monte_carlo_test.py index 5fbc17e..5bd0eea 100644 --- a/tests/plot_product_monte_carlo_test.py +++ b/tests/plot_product_monte_carlo_test.py @@ -1,5 +1,7 @@ import copy +import csv import importlib.util +import io import json import math from pathlib import Path @@ -7,6 +9,7 @@ import sys import tempfile import unittest +from unittest import mock SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "plot_product_monte_carlo.py" @@ -16,6 +19,78 @@ class NumericalTest(unittest.TestCase): + def test_pure_conditional_export(self): + groups = {(16, True, True): { + 1: {"trials": 10, "information": 0, "full": 0}, + 3500: {"trials": 20, "information": 4, "full": 8}}} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "pure.json" + plot.export_conditional(groups, ["information", "full"], path) + with path.open() as stream: + rows = json.load(stream, parse_constant=lambda value: self.fail(value)) + self.assertEqual(len(rows), 2) + for row, metric in zip(rows, ("information", "full")): + self.assertEqual(set(row), {"configuration", "metric", "points"}) + self.assertEqual(row["configuration"], plot.config_label((16, True, True))) + self.assertEqual(row["metric"], metric) + self.assertEqual(len(row["points"]), 2) + for point in row["points"]: + self.assertEqual(set(point), {"raw_ber", "residual_ber", "log10_residual_ber"}) + points = rows[0]["points"] + self.assertEqual(points[0]["residual_ber"], 0) + self.assertIsNone(points[0]["log10_residual_ber"]) + self.assertEqual(points[0]["raw_ber"], 1 / plot.N) + self.assertAlmostEqual(points[1]["residual_ber"], 4 / (20 * 455168)) + self.assertAlmostEqual(rows[1]["points"][1]["residual_ber"], 8 / (20 * 524288)) + + def test_thin_rejects_invalid_values_before_reading_inputs(self): + for value in ("0", "-1", "nan", "inf", "-inf"): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "--thin"): + plot.main(["nonexistent-input", "--thin=" + value]) + + def test_thin_scales_markers_and_lines(self): + for thin in (0.25, 0.5, 1.0): + with self.subTest(thin=thin), tempfile.TemporaryDirectory() as directory: + plt = mock.MagicMock() + figure, axis = mock.MagicMock(), mock.MagicMock() + plt.subplots.return_value = (figure, axis) + plt.rcParams.__getitem__.return_value.by_key.return_value = {"color": ["blue"]} + groups = {(16, True, True): {3500: {"trials": 10, "information": 20, "full": 30}}} + with mock.patch.object(plot, "evaluate", return_value=( + {"information": -10, "full": -9}, -1, -1)), mock.patch("sys.stderr", io.StringIO()): + plot.plot_results(groups, ["information"], "both", [0.006], + Path(directory) / "out.svg", Path(directory) / "out.csv", plt, + thin=thin) + self.assertEqual(axis.scatter.call_args.kwargs["s"], 18 * thin**2) + self.assertEqual(axis.scatter.call_args.kwargs["linewidths"], thin) + self.assertEqual(axis.plot.call_args.kwargs["linewidth"], 2 * thin) + + def test_conditional_large_counters_and_zero(self): + count = 10**400 + rows = {3: {"trials": count, "full": 0}, + 1: {"trials": count, "full": count * 3}, + 2: {"trials": count, "full": 1}} + values = list(plot.conditional_values(rows, "full")) + for value, k, total, mean, log_mean in zip( + values, (1, 2, 3), (count * 3, 1, 0), (3, 0, 0), (math.log10(3), -400, -math.inf)): + self.assertEqual((value["k"], value["residual_bits_sum"], value["completed_blocks"], + value["mean"]), (k, total, count, mean)) + self.assertAlmostEqual(value["log10_mean"], log_mean) + self.assertEqual(value["raw_ber"], k / plot.N) + self.assertAlmostEqual(value["log10_conditional_ber"], log_mean - math.log10(plot.N)) + self.assertEqual(values[1]["conditional_ber"], 0) + self.assertTrue(math.isfinite(values[1]["log10_conditional_ber"])) + self.assertEqual(values[2]["conditional_ber"], 0) + + def test_conditional_metric_denominators(self): + rows = {3500: {"trials": 10, "information": 20, "full": 30}} + for metric, total in (("information", 20), ("full", 30)): + value, = plot.conditional_values(rows, metric) + self.assertEqual(value["raw_ber"], 3500 / 524288) + expected = total / (10 * plot.DENOMINATORS[metric]) + self.assertAlmostEqual(value["conditional_ber"], expected, places=18) + self.assertAlmostEqual(value["log10_conditional_ber"], math.log10(expected)) + def test_binomial_identity_and_weighted_channel(self): n, p = 12, 0.17 rows = {k: {"trials": 1, "full": k} for k in range(n + 1)} @@ -102,6 +177,122 @@ def save(self, path, metadata, summary): (path / "metadata.json").write_text(json.dumps(metadata)) (path / "summary.json").write_text(json.dumps(summary)) + def test_recursive_discovery_sorted_and_stops_at_runs(self): + z, _, _ = self.fixture("z", seed=2) + (self.root / "nested").mkdir() + a, _, _ = self.fixture("nested/a") + self.fixture("nested/a/internal", seed=3) + (self.root / "outputs").mkdir() + (self.root / "outputs" / "plot.svg").write_text("not a report") + (self.root / "outputs" / "plot.csv").write_text("not a report") + (self.root / "unrelated.json").write_text("not JSON") + self.assertEqual(plot.discover_reports([self.root]), + [a / "summary.json", z / "summary.json"]) + self.assertEqual(plot.discover_reports([a]), [a / "summary.json"]) + self.assertEqual(plot.discover_reports([z / "summary.json"]), [z / "summary.json"]) + + def test_discovery_does_not_follow_subdirectory_symlinks(self): + path, _, _ = self.fixture() + try: + (self.root / "loop").symlink_to(self.root, target_is_directory=True) + (self.root / "alias").symlink_to(path, target_is_directory=True) + except OSError as error: + self.skipTest(f"directory symlinks unavailable: {error}") + self.assertEqual(plot.discover_reports([self.root]), [path / "summary.json"]) + with self.assertRaisesRegex(ValueError, "duplicate report source"): + plot.discover_reports([path, self.root / "alias"]) + + def test_empty_missing_and_partial_containers(self): + with self.assertRaisesRegex(ValueError, "no runs found"): + plot.discover_reports([self.root]) + with self.assertRaisesRegex(ValueError, "input does not exist"): + plot.discover_reports([self.root / "missing"]) + self.fixture() + partial = self.root / "partial" + partial.mkdir() + for name in ("metadata.json", "summary.json"): + with self.subTest(name=name): + file = partial / name + file.write_text("{}") + with self.assertRaisesRegex(ValueError, "partial: incomplete run"): + plot.discover_reports([self.root]) + file.unlink() + + def test_discovered_malformed_and_unsupported_reports_fail(self): + path, metadata, summary = self.fixture() + (path / "summary.json").write_text("not JSON") + with self.assertRaisesRegex(ValueError, "summary.json"): + plot.pool_reports([self.root]) + for key in ("code", "random algorithm"): + broken = dict(metadata, **{key: "unsupported"}) + self.save(path, broken, summary) + with self.assertRaisesRegex(ValueError, "incompatible schema/code/random algorithm"): + plot.pool_reports([self.root]) + + def test_container_overlap_and_distinct_duplicate_identities(self): + a, metadata, summary = self.fixture("a") + for inputs in ([self.root, a], [a, self.root], [self.root, self.root], + [self.root, a / "summary.json"]): + with self.subTest(inputs=inputs): + with self.assertRaisesRegex(ValueError, "overlapping inputs.*supply each run only once"): + plot.pool_reports(inputs) + b, _, _ = self.fixture("b", seed=2) + self.save(b, metadata, summary) + with self.assertRaisesRegex(ValueError, "duplicate run identity"): + plot.pool_reports([self.root]) + + def test_container_pooling_keeps_configuration_and_seed_checks(self): + self.fixture("a", trials=1, residual=10) + b, metadata, summary = self.fixture("b", seed=2, trials=9, residual=0) + metadata["settings"].update({"threads": 2, "minimum flipped bits": 2500}) + summary["run identity"] = plot.identity(metadata) + self.save(b, metadata, summary) + self.fixture("cap", passes=32) + for flag in ("anchors", "binary image"): + path, metadata, summary = self.fixture(flag) + metadata["settings"][flag] = False + summary["run identity"] = plot.identity(metadata) + self.save(path, metadata, summary) + groups = plot.pool_reports([self.root]) + self.assertEqual(len(groups), 4) + self.assertEqual(groups[(16, True, True)][2600], + {"trials": 10, "information": 10, "full": 10}) + self.fixture("duplicate-seed") + with self.assertRaisesRegex(ValueError, "repeated seed"): + plot.pool_reports([self.root]) + + def test_discovered_input_output_protection(self): + path, _, _ = self.fixture() + before = {p: p.read_bytes() for p in path.iterdir()} + for protected in before: + with self.subTest(protected=protected): + with self.assertRaisesRegex(ValueError, "output paths must be distinct"): + plot.main([str(self.root), "--output", str(self.root / "plot.svg"), + "--csv", str(protected)]) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + + @unittest.skipUnless(importlib.util.find_spec("matplotlib"), "matplotlib not installed") + def test_container_conditional_smoke_preserves_sources(self): + a, _, _ = self.fixture("a", residual=10) + b, _, _ = self.fixture("b", seed=2, trials=9, residual=0) + sources = [p for run in (a, b) for p in run.iterdir()] + before = {p: p.read_bytes() for p in sources} + output = self.root / "merged.svg" + for mode in ("conditional", "ber", "both"): + with self.subTest(mode=mode), mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: + plot.main([str(self.root), "--mode", mode, "--metric", "both", + "--points", "2", "--output", str(output)]) + self.assertIn("Validated 2 report(s)", stderr.getvalue()) + self.assertIn(" -TEST(ProductMonteCarloData, ExactUnboundedIntegers) { - const auto n = mc::Big::from_string( - "184467440737095516160000000000000000000000000000001"); +TEST(ProductMonteCarloData, ExactBoundedIntegers) { + const uint64_t n = UINT64_MAX / 524288; mc::Stats s; s.blocks = n; s.iterations = n * 2; @@ -14,29 +13,192 @@ TEST(ProductMonteCarloData, ExactUnboundedIntegers) { s.full_post = 0; auto json = s.ToJson(); auto text = mc::Dump(json); - EXPECT_NE(text.find(n.to_string()), std::string::npos); + EXPECT_NE(text.find(std::to_string(n)), std::string::npos); auto parsed = mc::Parse(text); auto recovered = mc::Stats::FromJson(parsed, n, 1, 16); - recovered.Add(s); - EXPECT_EQ(recovered.blocks, n * 2); + EXPECT_THROW(recovered.Add(s), std::runtime_error); + EXPECT_EQ(recovered.ToJson(), s.ToJson()); + EXPECT_EQ(recovered.blocks, n); EXPECT_EQ( mc::Natural(recovered.ToJson().at("information bits").at("total bits")), - n * 910336); + n * 455168); EXPECT_EQ(mc::Dump(mc::Parse(mc::Dump(recovered.ToJson(), true))), mc::Dump(recovered.ToJson())); + s.iterations = UINT64_MAX; + EXPECT_NO_THROW( + mc::Stats::FromJson(mc::Parse(mc::Dump(s.ToJson())), n, 1, 1000000)); + EXPECT_THROW(mc::Stats::FromJson(s.ToJson(), n, 1, 16), std::runtime_error); +} + +TEST(ProductMonteCarloData, CheckedArithmeticAndTransactionalUpdates) { + EXPECT_EQ(mc::CheckedAdd(UINT64_MAX - 1, 1), UINT64_MAX); + EXPECT_EQ(mc::CheckedMultiply(UINT64_MAX, 1), UINT64_MAX); + EXPECT_EQ(mc::CheckedMultiply(UINT64_MAX, 0), 0); + EXPECT_THROW(mc::CheckedAdd(UINT64_MAX, 1), std::runtime_error); + EXPECT_THROW(mc::CheckedMultiply(UINT64_MAX, 2), std::runtime_error); + EXPECT_EQ(mc::Decimal("18446744073709551615"), UINT64_MAX); + EXPECT_EQ(mc::Decimal("00001"), 1); + for (auto text : {"18446744073709551616", "-1", "", "1.0", "+1", "1 "}) { + EXPECT_THROW(mc::Decimal(text), std::runtime_error); + } + for (auto member : + {&mc::Stats::iterations, &mc::Stats::info_raw, &mc::Stats::info_post, + &mc::Stats::full_raw, &mc::Stats::full_post}) { + mc::Stats s, delta; + s.*member = UINT64_MAX; + delta.blocks = 1; + delta.*member = 1; + const auto before = s.ToJson(); + EXPECT_THROW(s.Add(delta), std::runtime_error); + EXPECT_EQ(s.ToJson(), before); + } + mc::Stats s; + s.blocks = UINT64_MAX; + EXPECT_THROW(s.Add(mc::Stats{1}), std::runtime_error); + EXPECT_EQ(s.blocks, UINT64_MAX); + EXPECT_THROW(s.ToJson(), std::runtime_error); + s.blocks = UINT64_MAX / 524288; + const auto before = s.ToJson(); + EXPECT_THROW(s.Add(std::array{}), std::runtime_error); + EXPECT_EQ(s.ToJson(), before); + auto j = before; + j["completed blocks"] = s.blocks + 1; + j["total iterations"] = (s.blocks + 1) * 2; + EXPECT_THROW(mc::Stats::FromJson(j, s.blocks + 1, 0, 16), std::runtime_error); +} + +TEST(ProductMonteCarloData, LegacyBoundedMomentsAndTransactionality) { + auto old = mc::LegacyStats(); + old[mc::kMetrics.back()]["squared sum"] = UINT64_MAX; + auto delta = mc::LegacyStats(); + delta[mc::kMetrics.back()]["squared sum"] = uint64_t{1}; + const auto before = old; + EXPECT_THROW(mc::AddLegacy(old, delta), std::runtime_error); + EXPECT_EQ(old, before); + std::array m{}; + m.back() = UINT64_MAX; + EXPECT_THROW(mc::AddLegacyTrial(old, m), std::runtime_error); + EXPECT_EQ(old, before); + // A theoretical bound may exceed u64 even though every stored moment fits. + old = mc::LegacyStats(); + m = {}; + m[12] = 2; + mc::AddLegacyTrial(old, m); + EXPECT_NO_THROW(mc::ValidateLegacy(old, 1, 0, 1000000)); + old = mc::LegacyStats(); + old[mc::kMetrics[13]]["sum"] = UINT64_MAX; + old[mc::kMetrics[13]]["squared sum"] = UINT64_MAX; + old[mc::kMetrics[15]] = old[mc::kMetrics[13]]; + EXPECT_NO_THROW(mc::ValidateLegacy(old, UINT64_MAX, 0, 1000000)); } TEST(ProductMonteCarloData, StrictJsonAndNaturalNumbers) { for (const auto* text : {"{\"a\":1,\"a\":2}", "{\"a\":{\"b\":0,\"b\":1}}", "1.0", "1e3", "NaN", - "/*comment*/1", "[1,]", "1 2"}) { + "/*comment*/1", "[1,]", "1 2", "Infinity", "1e9999", + "{\"a\":0,\"\\u0061\":1}", "{\"squared sum\":18446744073709551616}"}) { EXPECT_THROW(mc::Parse(text), std::exception) << text; } for (const auto* text : {"true", "\"123\"", "-1", "-184467440737095516160"}) { EXPECT_THROW(mc::Natural(mc::Parse(text)), std::exception) << text; } EXPECT_EQ(mc::U64(mc::Parse("18446744073709551615")), UINT64_MAX); + EXPECT_EQ(mc::Natural(mc::Json(1)), 1); + EXPECT_EQ(mc::Dump(mc::Number(UINT64_MAX)), "18446744073709551615"); + EXPECT_THROW(mc::Natural(mc::Json(1.0)), std::exception); EXPECT_THROW(mc::U64(mc::Parse("18446744073709551616")), std::exception); EXPECT_EQ(mc::Dump(mc::Parse("{\"z\":1,\"a\":\"x\"}")), "{\"a\":\"x\",\"z\":1}"); + EXPECT_EQ( + mc::Dump(mc::Parse( + R"({"z":true,"a":"\u00e9\u000f/\ud83d\ude00","n":18446744073709551615})")), + R"({"a":"\u00e9\u000f/\ud83d\ude00","n":18446744073709551615,"z":true})"); + EXPECT_NO_THROW(mc::Parse(std::string(32, '[') + "0" + std::string(32, ']'))); + EXPECT_THROW(mc::Parse(std::string(33, '[') + "0" + std::string(33, ']')), + std::exception); + EXPECT_NO_THROW(mc::Parse(R"([{"a":1},{"a":2,"b":[{"a":3}]}])")); +} + +TEST(ProductMonteCarloData, AggregatePreparesOnlyOneStratum) { + for (unsigned schema : {1, 2}) { + mc::Aggregate aggregate("test", schema); + const auto old = mc::LegacyStats(); + for (uint64_t k = 0; k < 1024; ++k) { + aggregate.Add(k, mc::Stats{1}, old); + } + const auto* untouched = &aggregate.by_k.at(0); + const auto before = aggregate.Summary(); + for (uint64_t k : {512, 1024}) { + { + auto abandoned = aggregate.PrepareAdd(k, mc::Stats{1}, old); + EXPECT_EQ(abandoned.row.key(), k); + EXPECT_EQ(aggregate.Summary(), before); + } + EXPECT_EQ(aggregate.Summary(), before); + } + aggregate.Commit(aggregate.PrepareAdd(512, mc::Stats{1}, old)); + aggregate.Add(1024, mc::Stats{1}, old); + EXPECT_EQ(aggregate.by_k.size(), 1025); + EXPECT_EQ(aggregate.overall.blocks, 1026); + EXPECT_EQ(aggregate.by_k.at(512).blocks, 2); + EXPECT_EQ(aggregate.by_k.at(1024).blocks, 1); + EXPECT_EQ(&aggregate.by_k.at(0), untouched); + if (schema == 1) { + EXPECT_EQ(aggregate.legacy_k.size(), 1025); + } + } +} + +TEST(ProductMonteCarloData, AggregateCounterOverflowIsTransactional) { + for (auto member : + {&mc::Stats::blocks, &mc::Stats::iterations, &mc::Stats::info_raw, + &mc::Stats::info_post, &mc::Stats::full_raw, &mc::Stats::full_post}) { + for (bool overall : {false, true}) { + mc::Aggregate aggregate("test", 2); + aggregate.Add(7, mc::Stats{}); + const uint64_t maximum = + member == &mc::Stats::blocks ? UINT64_MAX / 524288 : UINT64_MAX; + if (overall) { + aggregate.overall.*member = maximum; + ASSERT_EQ(aggregate.overall.*member, maximum); + } else { + aggregate.by_k.at(7).*member = maximum; + ASSERT_EQ(aggregate.by_k.at(7).*member, maximum); + } + const auto before = aggregate.Summary(); + mc::Stats delta; + delta.*member = 1; + EXPECT_THROW(aggregate.Add(7, delta), std::runtime_error); + EXPECT_EQ(aggregate.Summary(), before); + if (overall) { + EXPECT_THROW(aggregate.Add(8, delta), std::runtime_error); + EXPECT_EQ(aggregate.Summary(), before); + EXPECT_FALSE(aggregate.by_k.contains(8)); + } + } + } +} + +TEST(ProductMonteCarloData, AggregateLegacyOverflowIsTransactional) { + for (const auto* field : {"sum", "squared sum"}) { + for (bool overall : {false, true}) { + mc::Aggregate aggregate("test", 1); + auto delta = mc::LegacyStats(); + aggregate.Add(7, mc::Stats{1}, delta); + (overall ? aggregate.legacy + : aggregate.legacy_k.at(7))[mc::kMetrics.back()][field] = + UINT64_MAX; + delta[mc::kMetrics.back()][field] = uint64_t{1}; + const auto before = aggregate.Summary(); + EXPECT_THROW(aggregate.Add(7, mc::Stats{1}, delta), std::runtime_error); + EXPECT_EQ(aggregate.Summary(), before); + EXPECT_EQ(aggregate.overall.blocks, 1); + if (overall) { + EXPECT_THROW(aggregate.Add(8, mc::Stats{1}, delta), std::runtime_error); + EXPECT_EQ(aggregate.Summary(), before); + EXPECT_FALSE(aggregate.by_k.contains(8)); + EXPECT_FALSE(aggregate.legacy_k.contains(8)); + } + } + } } diff --git a/tests/product_monte_carlo_test.py b/tests/product_monte_carlo_test.py index df35d7f..1b71d5a 100644 --- a/tests/product_monte_carlo_test.py +++ b/tests/product_monte_carlo_test.py @@ -147,6 +147,60 @@ def test_saved_replay_and_corruption(self): (path / "flips.bin").write_bytes(data+b"uncommitted tail") self.invoke("--replay", path) + def test_uint64_seed_and_canonical_unicode_identity(self): + for reference in (False, True): + path = self.run_case(str(reference), "--seed", 2**64-1, + "--batches", 1, "--batch-size", 1, "--sampler", "fisher-yates", + "--minimum-flipped-bits", 0, "--maximum-flipped-bits", 0, + reference=reference) + metadata = self.read(path, "metadata.json") + self.assertEqual(metadata["settings"]["root seed"], 2**64-1) + digest = hashlib.sha256(canonical(metadata).encode("ascii")).hexdigest() + self.assertEqual(self.read(path)["run identity"], digest) + self.assertEqual((path / "flips.bin").read_bytes()[8:40], bytes.fromhex(digest)) + metadata["created at"] = "unicode \u00e9 \U0001f600 / \x0f\n" + digest = hashlib.sha256(canonical(metadata).encode("ascii")).hexdigest() + # Exercise raw UTF-8 parsing as well as canonical ASCII serialization. + (path / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False), encoding="utf-8") + records = self.records(path) + for record in records: + record["run identity"] = digest + (path / "journal.jsonl").write_text("".join(canonical(r)+"\n" for r in records)) + data = (path / "flips.bin").read_bytes() + (path / "flips.bin").write_bytes(data[:8]+bytes.fromhex(digest)+data[40:]) + self.invoke("--replay", path) + self.assertEqual(self.read(path)["run identity"], digest) + + def test_oversized_counters_and_legacy_moments_rejected(self): + for reference in (False, True): + path = self.run_case(str(reference), "--batches", 1, "--batch-size", 1, + reference=reference) + before = (path / "summary.json").read_bytes() + original = self.records(path)[0] + for value in (2**64, 10**100, 10**400, 1.0, -1): + record = copy.deepcopy(original) + if reference: + record["statistics"]["accepted bit changes"]["squared sum"] = value + else: + record["statistics"]["total iterations"] = value + (path / "journal.jsonl").write_text(canonical(record)+"\n") + p = self.invoke("--report", path, success=False) + self.assertRegex(p.stderr, "uint64|unsigned integer") + self.assertEqual(before, (path / "summary.json").read_bytes()) + + def test_counter_overflow_preserves_committed_prefix(self): + for sampler in ("floyd", "fisher-yates"): + path = self.root / sampler + p = self.invoke("--output", path, "--seed", 42, "--batches", 1, + "--batch-size", 3, "--checkpoint-trials", 1, "--sampler", sampler, + "--minimum-flipped-bits", 0, "--maximum-flipped-bits", 0, + fault=True, env=dict(os.environ, MC_TEST_COUNTER_OVERFLOW="1"), success=False) + self.assertIn("uint64 counter addition overflow", p.stderr) + self.assertEqual(len(self.records(path)), 1) + self.assertEqual(self.count(path), 0) # Last durable summary is unchanged. + self.invoke("--replay" if sampler == "fisher-yates" else "--report", path) + self.assertEqual(self.count(path), 1) + def test_recovery_strictness_and_partial_tail(self): path = self.run_case("source", "--checkpoint-trials", 1) before = (path / "summary.json").read_bytes() diff --git a/tools/product_monte_carlo.cc b/tools/product_monte_carlo.cc index d86e9d0..de23d38 100644 --- a/tools/product_monte_carlo.cc +++ b/tools/product_monte_carlo.cc @@ -325,51 +325,6 @@ class Workers { bool active_ = false, halted_ = false, shutdown_ = false; }; -struct Aggregate { - std::string identity; - unsigned schema; - Stats overall; - std::map by_k; - Json legacy = LegacyStats(); - std::map legacy_k; - Aggregate(std::string id, unsigned revision) - : identity(std::move(id)), schema(revision) {} - void Add(uint64_t k, const Stats& stats, const Json& old = Json()) { - overall.Add(stats); - by_k[k].Add(stats); - if (schema == 1) { - AddLegacy(legacy, old); - auto [it, inserted] = legacy_k.try_emplace(k, LegacyStats()); - AddLegacy(it->second, old); - } - } - Json Summary() const { - Json out; - out["schema revision"] = schema; - out["run identity"] = identity; - Json rows(jsoncons::json_array_arg); - for (const auto& [k, s] : by_k) { - Json row; - row["flipped bit count"] = k; - if (schema == 1) { - row["trial count"] = Number(s.blocks); - row["statistics"] = legacy_k.at(k); - } else { - row["statistics"] = s.ToJson(); - } - rows.push_back(std::move(row)); - } - out["by flipped bit count"] = std::move(rows); - if (schema == 1) { - out["overall"]["trial count"] = Number(overall.blocks); - out["overall"]["statistics"] = legacy; - } else { - out["overall"]["statistics"] = overall.ToJson(); - } - return out; - } -}; - void WriteFlips(File& file, uint64_t batch, uint64_t k, const Result& result) { const auto count = std::min(k, 524288 - k); std::string bytes(208 + 4 * count, '\0'); @@ -433,7 +388,7 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { const auto start = Clock::now(); auto last_sync = start, last_report = start, last_bar = start, last_checkpoint = start; - Big completed_total = 0, increment = 0; + uint64_t completed_total = 0, increment = 0; uint64_t batch_completed = 0, batch = 0, k = 0; Stats staged; uint64_t first = 0, end = 0, flip_start = 0; @@ -452,10 +407,20 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { return "batch=" + std::to_string(batch) + " k=" + std::to_string(k) + " trials=" + std::to_string(batch_completed) + "/" + std::to_string(s.size) + - " overall trials=" + completed_total.to_string(); + " overall trials=" + std::to_string(completed_total); }; auto checkpoint = [&] { if (staged.blocks != 0) { + // Preflight every aggregate and derived total before publishing a record. +#ifdef GF256_MC_TEST_HOOKS + if (increment == 1 && std::getenv("MC_TEST_COUNTER_OVERFLOW")) { + auto exhausted = aggregate.overall; + exhausted.iterations = UINT64_MAX; + exhausted.Add(staged); + } +#endif + auto next = aggregate.PrepareAdd(k, staged); + const auto next_increment = CheckedAdd(increment, 1); Json r; r["schema revision"] = 2; r["run identity"] = aggregate.identity; @@ -472,8 +437,8 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { r["flip end"] = flips->Offset(); } journal.Write(Dump(r) + "\n"); - aggregate.Add(k, staged); - ++increment; + aggregate.Commit(std::move(next)); + increment = next_increment; staged = Stats{}; } last_checkpoint = Clock::now(); @@ -512,7 +477,8 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { Result result; uint64_t observed; bool ready = workers.Pop(result, observed, done); - completed_total += Big(observed - batch_completed); + completed_total = + CheckedAdd(completed_total, observed - batch_completed); batch_completed = observed; if (ready) { if (result.error) { @@ -540,12 +506,15 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { flip_start = flips->Offset(); } } + auto next_staged = staged; + next_staged.Add(result.metrics); + const auto next_end = CheckedAdd(result.index, 1); if (flips) { WriteFlips(*flips, batch, k, result); } - end = result.index + 1; - staged.Add(result.metrics); - if (staged.blocks == Big(s.checkpoint)) { + end = next_end; + staged = next_staged; + if (staged.blocks == s.checkpoint) { checkpoint(); } } @@ -559,7 +528,7 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { } if (now - last_report >= std::chrono::seconds(s.report)) { progress(counts() + " persisted overall trials=" + - aggregate.overall.blocks.to_string() + " " + + std::to_string(aggregate.overall.blocks) + " " + throughput(now), !tty); last_report = now; @@ -570,8 +539,9 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { bar(Clock::now(), true); progress("batch=" + std::to_string(batch) + " k=" + std::to_string(k) + " completed trials=" + std::to_string(batch_completed) + "/" + - std::to_string(s.size) + " overall trials=" + - completed_total.to_string() + " " + throughput(Clock::now())); + std::to_string(s.size) + + " overall trials=" + std::to_string(completed_total) + " " + + throughput(Clock::now())); Require(batch != UINT64_MAX, "batch identity exhausted; start a new run"); ++batch; } @@ -588,7 +558,7 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { throw; } durable(); - progress("finalizing trials=" + completed_total.to_string() + + progress("finalizing trials=" + std::to_string(completed_total) + " interrupted=" + (product_interrupted() ? "True" : "False") + " error=" + (error.empty() ? "None" : error) + " " + throughput(Clock::now())); @@ -682,7 +652,7 @@ void Recover(const fs::path& directory, bool replay) { "incompatible metadata schema/code/random algorithm"); Require(metadata.at("created at").is_string(), "invalid created at"); const auto codeword = metadata.contains("codeword") - ? metadata.at("codeword").as() + ? metadata.at("codeword").get() : "random"; Require(codeword == "zero" || codeword == "random", "incompatible codeword convention"); @@ -699,10 +669,11 @@ void Recover(const fs::path& directory, bool replay) { Require(flips.good() && ReadExact(flips, 40) == "RSFLIP01" + digest, "invalid flip file identity/version"); } - Big index = 0; + uint64_t index = 0; std::optional> previous; bool incomplete = false; while (auto line = ReadLine(journal, incomplete)) { + const auto next_index = CheckedAdd(index, 1); try { Json r = Parse(*line); fields = {"schema revision", "run identity", "increment id", @@ -746,19 +717,18 @@ void Recover(const fs::path& directory, bool replay) { stats.full_raw = Natural(old.at(kMetrics[0]).at("sum")); stats.full_post = Natural(old.at(kMetrics[4]).at("sum")); } else { - stats = - Stats::FromJson(r.at("statistics"), Big(count), k, settings.passes); + stats = Stats::FromJson(r.at("statistics"), count, k, settings.passes); } if (recorded) { ReadFlips(flips, r, settings, replay, codeword == "random", schema); } aggregate.Add(k, stats, r.at("statistics")); previous = {batch, end}; + index = next_index; } catch (const std::exception& e) { - throw std::runtime_error("journal line " + (index + 1).to_string() + + throw std::runtime_error("journal line " + std::to_string(next_index) + ": " + e.what()); } - ++index; } if (recorded && flips.peek() != std::char_traits::eof()) { std::cerr << "unreferenced flip tail ignored (not committed trials)\n"; @@ -805,8 +775,13 @@ int Main(int argc, char** argv) { "--report-seconds 2 --fsync-seconds 5 (1..86400)\n" "One uniform k per batch; exactly k flips per all-zero block. No " "resume.\n" - "Counters are exact arbitrary-precision JSON integers; iterations " - "are directional passes.\n" + "Counters are exact uint64 JSON integers " + "(0..18446744073709551615); " + "overflow is an error.\n" + "Total bits must also fit uint64; iterations are directional " + "passes.\n" + "Legacy schema 1 sums and squared sums exceeding uint64 are " + "rejected.\n" "SIGINT/SIGTERM stop starts, drain whole in-flight blocks, " "checkpoint and fsync.\n" "Crash loss: bounded worker window/staged increment plus journal " @@ -849,9 +824,7 @@ int Main(int argc, char** argv) { std::all_of(value.begin(), value.end(), [](char c) { return c >= '0' && c <= '9'; }), "expected unsigned decimal integer for " + flag); - auto n = Big::from_string(value); - Require(n <= Big(UINT64_MAX), "integer exceeds uint64"); - auto v = static_cast(n); + auto v = Decimal(value); if (flag == "--seed") { s.seed = v; seeded = true; diff --git a/tools/product_monte_carlo_data.h b/tools/product_monte_carlo_data.h index 147502b..29c1803 100644 --- a/tools/product_monte_carlo_data.h +++ b/tools/product_monte_carlo_data.h @@ -1,20 +1,19 @@ #pragma once #include +#include #include -#include -#include -#include #include +#include #include #include #include #include +#include #include namespace mc { -using Json = jsoncons::json; -using Big = jsoncons::bigint; +using Json = nlohmann::json; inline constexpr std::string_view kCode = "RS256,224 x RS256,254 Cantor systematic row major"; inline constexpr std::string_view kFloyd = @@ -54,75 +53,78 @@ inline void Require(bool condition, const std::string& message) { } inline Json Parse(const std::string& text) { - const auto options = jsoncons::json_options() - .lossless_number(true) - .max_nesting_depth(32) - .err_handler(jsoncons::strict_json_parsing()); - // Validate the library's event stream before DOM construction can discard - // duplicate keys. Number tokens larger than uint64 retain their exact digits. std::vector> objects; - jsoncons::json_string_cursor cursor(text, options); - for (; !cursor.done(); cursor.next()) { - const auto& event = cursor.current(); - using E = jsoncons::staj_event_type; - if (event.event_type() == E::begin_object) { - objects.emplace_back(); - } - if (event.event_type() == E::end_object) { - objects.pop_back(); - } - if (event.event_type() == E::key) { - Require(objects.back().insert(event.get()).second, - "duplicate JSON key"); - } - Require(event.event_type() != E::double_value && - event.tag() != jsoncons::semantic_tag::bigdec, - "noninteger JSON number"); + try { + return Json::parse( + text, [&](int depth, Json::parse_event_t event, Json& value) { + using E = Json::parse_event_t; + if (event == E::object_start || event == E::array_start) { + Require(depth < 32, "JSON nesting exceeds 32"); + } + if (event == E::object_start) { + objects.emplace_back(); + } + if (event == E::object_end) { + objects.pop_back(); + } + if (event == E::key) { + Require(objects.back().insert(value.get()).second, + "duplicate JSON key"); + } + // Overflowing integer tokens also become floats in the DOM. Never + // accept their rounded values, including unused legacy squared sums. + Require(!value.is_number_float(), + "noninteger JSON number or integer exceeds uint64 (including " + "legacy moments)"); + return true; + }); + } catch (const Json::out_of_range&) { + throw std::runtime_error( + "JSON number exceeds uint64 (including legacy moments)"); } - return Json::parse(text, options); } inline std::string Dump(const Json& value, bool pretty = false) { - std::string text; - auto options = jsoncons::json_options() - .bigint_format(jsoncons::bigint_chars_format::number) - .escape_all_non_ascii(true) - .indent_size(2); - if (!pretty) { - options.spaces_around_colon(jsoncons::spaces_option::no_spaces) - .spaces_around_comma(jsoncons::spaces_option::no_spaces); - } - value.dump( - text, options, - pretty ? jsoncons::indenting::indent : jsoncons::indenting::no_indent); - return text; + // The default ordered object map and ASCII escaping match Python's + // sort_keys=True, ensure_ascii=True, separators=(",", ":") identities. + return value.dump(pretty ? 2 : -1, ' ', true); } -inline Big Natural(const Json& value) { - if (value.is_uint64()) { - return Big(value.as()); - } - Require(value.tag() == jsoncons::semantic_tag::bigint, +inline uint64_t Natural(const Json& value) { + Require(value.is_number_unsigned() || + (value.is_number_integer() && value.get() >= 0), "expected unsigned integer"); - auto n = Big::from_string(value.as()); - Require(n >= 0, "expected unsigned integer"); - return n; + return value.get(); } inline uint64_t U64(const Json& value) { - auto n = Natural(value); - Require(n <= Big(UINT64_MAX), "integer exceeds uint64"); - return static_cast(n); + return Natural(value); } -inline Json Number(const Big& value) { - if (value <= Big(UINT64_MAX)) { - return Json(static_cast(value)); - } - return Json(value.to_string(), jsoncons::semantic_tag::bigint); +inline uint64_t Decimal(std::string_view text) { + Require(!text.empty(), "expected unsigned decimal integer"); + uint64_t value = 0; + const auto [end, error] = + std::from_chars(text.data(), text.data() + text.size(), value); + Require(error != std::errc::result_out_of_range, "integer exceeds uint64"); + Require(error == std::errc{} && end == text.data() + text.size(), + "expected unsigned decimal integer"); + return value; +} +inline uint64_t CheckedAdd(uint64_t a, uint64_t b) { + Require(b <= UINT64_MAX - a, "uint64 counter addition overflow"); + return a + b; +} +inline uint64_t CheckedMultiply(uint64_t a, uint64_t b) { + Require(b == 0 || a <= UINT64_MAX / b, + "uint64 counter multiplication overflow"); + return a * b; +} +inline Json Number(uint64_t value) { + return Json(value); } inline void Fields(const Json& value, std::set expected) { Require(value.is_object() && value.size() == expected.size(), "incompatible object fields"); - for (const auto& item : value.object_range()) { + for (const auto& item : value.items()) { Require(expected.erase(std::string(item.key())) == 1, "incompatible object field"); } @@ -177,35 +179,28 @@ struct Settings { s.checkpoint = U64(j.at("checkpoint trials")); s.report = U64(j.at("report seconds")); s.sync = U64(j.at("fsync seconds")); - Require(j.at("anchors").is_bool() && j.at("binary image").is_bool(), + Require(j.at("anchors").is_boolean() && j.at("binary image").is_boolean(), "gates must be boolean"); - s.anchors = j.at("anchors").as(); - s.binary = j.at("binary image").as(); + s.anchors = j.at("anchors").get(); + s.binary = j.at("binary image").get(); s.Validate(); return s; } }; struct Stats { - // All public accumulators are arbitrary precision. Trial metrics remain - // bounded uint64 values in the private ABI and legacy flip records. - Big blocks = 0, iterations = 0, info_raw = 0, info_post = 0, full_raw = 0, - full_post = 0; + uint64_t blocks = 0, iterations = 0, info_raw = 0, info_post = 0, + full_raw = 0, full_post = 0; void Add(const std::array& m) { - ++blocks; - iterations += m[12]; - info_raw += m[2]; - info_post += m[6]; - full_raw += m[0]; - full_post += m[4]; + Add(Stats{1, m[12], m[2], m[6], m[0], m[4]}); } void Add(const Stats& s) { - blocks += s.blocks; - iterations += s.iterations; - info_raw += s.info_raw; - info_post += s.info_post; - full_raw += s.full_raw; - full_post += s.full_post; + Stats next{ + CheckedAdd(blocks, s.blocks), CheckedAdd(iterations, s.iterations), + CheckedAdd(info_raw, s.info_raw), CheckedAdd(info_post, s.info_post), + CheckedAdd(full_raw, s.full_raw), CheckedAdd(full_post, s.full_post)}; + CheckedMultiply(next.blocks, 524288); + *this = next; } Json ToJson() const { Json j; @@ -213,7 +208,8 @@ struct Stats { j["total iterations"] = Number(iterations); for (bool info : {true, false}) { Json bits; - bits["total bits"] = Number(blocks * (info ? 455168 : 524288)); + bits["total bits"] = + Number(CheckedMultiply(blocks, info ? 455168 : 524288)); bits["raw corrupted bits"] = Number(info ? info_raw : full_raw); bits["post decoding corrupted bits"] = Number(info ? info_post : full_post); @@ -222,7 +218,7 @@ struct Stats { return j; } static Stats FromJson(const Json& j, - const Big& count, + uint64_t count, uint64_t k, uint64_t passes) { Fields(j, {"completed blocks", "total iterations", "information bits", @@ -230,14 +226,17 @@ struct Stats { Stats s; s.blocks = Natural(j.at("completed blocks")); s.iterations = Natural(j.at("total iterations")); - Require(s.blocks == count && s.iterations >= count * 2 && - s.iterations <= count * Big(passes), + // Compare the upper bound by division: count*passes need not fit u64 + // when the actual iteration count does. + Require(s.blocks == count && s.iterations >= CheckedMultiply(count, 2) && + (passes != 0 && s.iterations / passes <= count && + (s.iterations / passes < count || s.iterations % passes == 0)), "inconsistent block/iteration counts"); for (bool info : {true, false}) { const auto& bits = j.at(info ? "information bits" : "full-codeword bits"); Fields(bits, {"total bits", "raw corrupted bits", "post decoding corrupted bits"}); - const auto total = count * (info ? 455168 : 524288); + const auto total = CheckedMultiply(count, info ? 455168 : 524288); auto raw = Natural(bits.at("raw corrupted bits")); auto post = Natural(bits.at("post decoding corrupted bits")); Require(Natural(bits.at("total bits")) == total && raw <= total && @@ -246,10 +245,11 @@ struct Stats { (info ? s.info_raw : s.full_raw) = raw; (info ? s.info_post : s.full_post) = post; } - Require(s.full_raw == count * Big(k), "initial channel is not exact k"); + Require(s.full_raw == CheckedMultiply(count, k), + "initial channel is not exact k"); Require(s.info_raw <= s.full_raw && s.info_post <= s.full_post && - s.full_raw - s.info_raw <= count * 69120 && - s.full_post - s.info_post <= count * 69120, + s.full_raw - s.info_raw <= CheckedMultiply(count, 69120) && + s.full_post - s.info_post <= CheckedMultiply(count, 69120), "inconsistent information/full bit counters"); return s; } @@ -258,32 +258,37 @@ struct Stats { inline Json LegacyStats() { Json j; for (auto name : kMetrics) { - j[name]["sum"] = 0; - j[name]["squared sum"] = 0; + j[name]["sum"] = uint64_t{0}; + j[name]["squared sum"] = uint64_t{0}; } return j; } inline void AddLegacy(Json& to, const Json& from) { + Json next = to; for (auto name : kMetrics) { for (auto field : {"sum", "squared sum"}) { - to[name][field] = Number(Natural(to.at(name).at(field)) + - Natural(from.at(name).at(field))); + next[name][field] = Number(CheckedAdd(Natural(to.at(name).at(field)), + Natural(from.at(name).at(field)))); } } + to.swap(next); } inline void AddLegacyTrial(Json& to, const std::array& m) { + Json next = to; for (size_t i = 0; i < m.size(); ++i) { - auto& item = to.at(kMetrics[i]); - item["sum"] = Number(Natural(item.at("sum")) + Big(m[i])); - item["squared sum"] = - Number(Natural(item.at("squared sum")) + Big(m[i]) * Big(m[i])); + auto& item = next.at(kMetrics[i]); + item["sum"] = Number(CheckedAdd(Natural(item.at("sum")), m[i])); + item["squared sum"] = Number(CheckedAdd(Natural(item.at("squared sum")), + CheckedMultiply(m[i], m[i]))); } + to.swap(next); } inline void ValidateLegacy(const Json& j, uint64_t count, uint64_t k, uint64_t passes) { Fields(j, std::set(kMetrics.begin(), kMetrics.end())); + Require(passes >= 2 && passes <= 1000000, "invalid legacy pass cap"); std::array bounds{524288, 65536, 455168, @@ -311,21 +316,130 @@ inline void ValidateLegacy(const Json& j, Fields(item, {"sum", "squared sum"}); auto sum = Natural(item.at("sum")), square = Natural(item.at("squared sum")); - Require(sum <= Big(count) * Big(bounds[i]) && - square <= Big(count) * Big(bounds[i]) * Big(bounds[i]) && - sum * sum <= Big(count) * square && sum <= square && - square <= Big(bounds[i]) * sum, + // Only comparison products are widened, never stored counters. Two u64 + // factors fit exactly; the redundant count*bound*bound bound is implied + // by sum <= count*bound and square <= bound*sum. + using Wide = __uint128_t; + Require(sum <= Wide(count) * bounds[i] && + Wide(sum) * sum <= Wide(count) * square && sum <= square && + square <= Wide(bounds[i]) * sum, "inconsistent legacy moments"); } - Require(Natural(j.at(kMetrics[0]).at("sum")) == Big(count) * Big(k) && + Require(Natural(j.at(kMetrics[0]).at("sum")) == CheckedMultiply(count, k) && Natural(j.at(kMetrics[0]).at("squared sum")) == - Big(count) * Big(k) * Big(k), + CheckedMultiply(CheckedMultiply(count, k), k), "initial channel is not exact k"); for (size_t i : {13, 14}) { Require(Natural(j.at(kMetrics[i]).at("sum")) == - Natural(j.at(kMetrics[i + 2]).at("sum")) + - Natural(j.at(kMetrics[i + 4]).at("sum")), + CheckedAdd(Natural(j.at(kMetrics[i + 2]).at("sum")), + Natural(j.at(kMetrics[i + 4]).at("sum"))), "directional accepted totals disagree"); } } + +struct Aggregate { + std::string identity; + unsigned schema; + Stats overall; + std::map by_k; + Json legacy = LegacyStats(); + std::map legacy_k; + /** @brief Initialize an empty aggregate for a run and schema revision. */ + Aggregate(std::string id, unsigned revision) + : identity(std::move(id)), schema(revision) {} + + struct PreparedAdd { + Stats overall; + decltype(by_k)::node_type row; + Json legacy; + decltype(legacy_k)::node_type legacy_row; + }; + + /** + * @brief Check totals and allocate staged nodes without changing this + * aggregate. + * @param k Flipped-bit count of the updated stratum. + * @param stats Counter increment. + * @param old Legacy moment increment, required for schema 1. + * @return Prepared update that can be discarded without side effects. + */ + PreparedAdd PrepareAdd(uint64_t k, + const Stats& stats, + const Json& old = Json()) const { + PreparedAdd next; + next.overall = overall; + next.overall.Add(stats); + const auto it = by_k.find(k); + Stats row = it == by_k.end() ? Stats{} : it->second; + row.Add(stats); + // Stage just one node, with the same allocator as the destination map. + decltype(by_k) rows; + rows.emplace(k, row); + next.row = rows.extract(rows.begin()); + if (schema == 1) { + next.legacy = legacy; + AddLegacy(next.legacy, old); + const auto old_it = legacy_k.find(k); + Json old_row = old_it == legacy_k.end() ? LegacyStats() : old_it->second; + AddLegacy(old_row, old); + decltype(legacy_k) old_rows; + old_rows.emplace(k, std::move(old_row)); + next.legacy_row = old_rows.extract(old_rows.begin()); + } + return next; + } + + /** + * @brief Publish prepared totals without allocation or exceptions. + * @param next Update prepared by this aggregate; commit once, with no + * intervening updates. Node insertion allocates nothing, and the integer + * comparator cannot throw. + */ + void Commit(PreparedAdd&& next) noexcept { + auto row = by_k.insert(std::move(next.row)); + if (!row.inserted) { + row.position->second = row.node.mapped(); + } + if (schema == 1) { + auto old_row = legacy_k.insert(std::move(next.legacy_row)); + if (!old_row.inserted) { + old_row.position->second.swap(old_row.node.mapped()); + } + legacy.swap(next.legacy); + } + overall = next.overall; + } + + /** @brief Prepare and commit one increment with strong exception safety. */ + void Add(uint64_t k, const Stats& stats, const Json& old = Json()) { + Commit(PrepareAdd(k, stats, old)); + } + + /** @brief Serialize the overall totals and all strata in the run's schema. */ + Json Summary() const { + Json out; + out["schema revision"] = schema; + out["run identity"] = identity; + Json rows = Json::array(); + for (const auto& [k, s] : by_k) { + Json row; + row["flipped bit count"] = k; + if (schema == 1) { + row["trial count"] = Number(s.blocks); + row["statistics"] = legacy_k.at(k); + } else { + row["statistics"] = s.ToJson(); + } + rows.push_back(std::move(row)); + } + out["by flipped bit count"] = std::move(rows); + if (schema == 1) { + out["overall"]["trial count"] = Number(overall.blocks); + out["overall"]["statistics"] = legacy; + } else { + out["overall"]["statistics"] = overall.ToJson(); + } + return out; + } +}; } // namespace mc diff --git a/tools/product_monte_carlo_native.cc b/tools/product_monte_carlo_native.cc index d758e8a..623658c 100644 --- a/tools/product_monte_carlo_native.cc +++ b/tools/product_monte_carlo_native.cc @@ -127,6 +127,8 @@ int Trial(uint64_t seed, uint8_t* residual) { try { // Local counters cannot alias the byte buffers and are published only once. + // The validated cap bounds every metric by 1000000 * 524288 (< 2^39); + // scans count at most 524288 bits, so these per-trial additions fit u64. uint64_t out[22]{}; if (!output || k > 524288 || passes < 2 || passes > 1000000 || sampler < 0 || sampler > 2 || (sampler == 2 && !positions)) { From 63f3aed0e72585726b5bcc346c5de496f726b8c1 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Wed, 9 Sep 2026 19:35:16 +0300 Subject: [PATCH 4/7] Support shortened RS products and atomic experiment summaries --- .../strong_weak_rs_product_code.h | 8 +- scripts/install-monte-carlo.sh | 2 +- scripts/plot_product_monte_carlo.py | 116 +++++-- .../strong_weak_rs_product_code.cc | 78 +++-- tests/plot_product_monte_carlo_test.py | 108 ++++++- tests/product_code_tests.cc | 115 ++++++- tests/product_monte_carlo_data_tests.cc | 40 +++ tests/product_monte_carlo_legacy_test.py | 44 +++ tests/product_monte_carlo_test.py | 211 ++++++++++-- tools/product_monte_carlo.cc | 304 +++++++++++++----- tools/product_monte_carlo_data.h | 145 ++++++--- tools/product_monte_carlo_native.cc | 112 +++++-- tools/product_monte_carlo_trials.h | 16 + 13 files changed, 1062 insertions(+), 237 deletions(-) diff --git a/include/reed_solomon/strong_weak_rs_product_code.h b/include/reed_solomon/strong_weak_rs_product_code.h index f7dd046..9a297dd 100644 --- a/include/reed_solomon/strong_weak_rs_product_code.h +++ b/include/reed_solomon/strong_weak_rs_product_code.h @@ -47,9 +47,11 @@ struct ProductCorrectionResult { * @brief Systematic Cantor RS product with strong columns and weak rows. * @details Row-major block has Nstrong rows and Nweak columns. The top-left * Kstrong by Kweak rectangle holds data; every column and every row, including - * parity regions, is a component codeword. Errors only: no erasures, - * shortening, or backtracking. Both N must be powers of two <=256; strong R - * must be a power of two with 2<=R<=K, and weak R must equal 2. + * parity regions, is a component codeword. Errors only: no erasures or + * backtracking. Strong N and R must be powers of two, N<=256 and 2<=R<=K. + * Weak R=2, K>=2, N<=256 may be shortened from nextPow2(N): omitted data + * [K,nextPow2(N)-2) are known zeros. Public rows remain compact [data][parity]. + * Mother-code candidates changing any omitted zero are rejected in full. */ class StrongWeakRSProductCode { public: diff --git a/scripts/install-monte-carlo.sh b/scripts/install-monte-carlo.sh index 6c10962..441a973 100644 --- a/scripts/install-monte-carlo.sh +++ b/scripts/install-monte-carlo.sh @@ -2,7 +2,7 @@ set -euo pipefail if [[ ${1:-} == --help || ${1:-} == -h ]]; then - printf 'Usage: %s [PREFIX]\nBuild native Release rs-product-monte-carlo and install it (default: ~/.local).\nRequires CMake, a C++20 compiler, OpenSSL 3 development files, and Git/network for pinned nlohmann/json v3.12.0 headers. No Python runtime or sudo.\n' "$0" + printf 'Usage: %s [PREFIX]\nBuild native Release rs-product-monte-carlo and install it (default: ~/.local).\nRuntime dimensions: --n1 256 --k1 224 --n2 256 --k2 254. Strong N,R remain aligned powers of two; only weak R=2 shortening is supported (e.g. --n2 175 --k2 173). Small codes need explicit flip bounds <=8*n1*n2.\nRequires CMake, a C++20 compiler, OpenSSL 3 development files, and Git/network for pinned nlohmann/json v3.12.0 headers. No Python runtime or sudo.\n' "$0" exit 0 fi if (( $# > 1 )) || [[ ${1:-} == -* ]]; then diff --git a/scripts/plot_product_monte_carlo.py b/scripts/plot_product_monte_carlo.py index 4c5d45b..b24cd2a 100644 --- a/scripts/plot_product_monte_carlo.py +++ b/scripts/plot_product_monte_carlo.py @@ -15,7 +15,38 @@ CODE = "RS256,224 x RS256,254 Cantor systematic row major" RANDOM = "splitmix64 domain seeds; mt19937_64; rejection modulo; Floyd complement v1" RANDOM_FY = "splitmix64 domain seeds; mt19937_64; rejection modulo; persistent Fisher-Yates complement v1; replay saved flips" +SNAPSHOTS = "atomic summary v1" NEG_INF = -math.inf +DEFAULT_DIMENSIONS = (256, 224, 256, 254) + + +def dimensions(settings): + require(type(settings) is dict, "incompatible settings") + names = ("n1", "k1", "n2", "k2") + require(not any(name in settings for name in names) or all(name in settings for name in names), + "incomplete dimensions") + dims = tuple(settings.get(name, default) for name, default in zip(names, DEFAULT_DIMENSIONS)) + require(all(natural(value, 256) for value in dims), "invalid dimensions") + n1, k1, n2, k2 = dims + power2 = lambda n: n > 0 and n & (n - 1) == 0 + require(power2(n1) and 2 <= n1 - k1 <= k1 and power2(n1 - k1) + and 2 <= k2 < n2 <= 256 and n2 - k2 == 2, "unsupported dimensions") + return dims + + +def denominators(dims): + n1, k1, n2, k2 = dims + return {"information": 8 * k1 * k2, "full": 8 * n1 * n2} + + +def configuration(settings): + flags = (settings["maximum directional passes"], settings["anchors"], settings["binary image"]) + dims = dimensions(settings) + return flags if dims == DEFAULT_DIMENSIONS else flags + dims + + +def config_dimensions(config): + return config[3:] if len(config) == 7 else DEFAULT_DIMENSIONS def require(condition, message): @@ -51,6 +82,8 @@ def identity(metadata): def minimal_rows(summary, settings): """Validate additive schema-2 counters and project the two BER numerators.""" + ds = denominators(dimensions(settings)) + n = ds["full"] rows = summary["by flipped bit count"] require(type(rows) is list, "invalid per-k rows") pooled, totals = {}, None @@ -71,7 +104,7 @@ def minimal_rows(summary, settings): bits = stats[name] require(type(bits) is dict and set(bits) == {"total bits", "raw corrupted bits", "post decoding corrupted bits"}, "invalid bit fields") - total = trials * DENOMINATORS[metric] + total = trials * ds[metric] require(bits["total bits"] == total and natural(bits["total bits"]), "invalid total bits") for field in ("raw corrupted bits", "post decoding corrupted bits"): require(natural(bits[field], total), "invalid corrupted bits") @@ -79,14 +112,14 @@ def minimal_rows(summary, settings): residuals[metric] = bits["post decoding corrupted bits"] for field in ("raw corrupted bits", "post decoding corrupted bits"): difference = stats["full-codeword bits"][field] - stats["information bits"][field] - require(0 <= difference <= trials * (N - DENOMINATORS["information"]), + require(0 <= difference <= trials * (n - ds["information"]), "inconsistent full/information bits") if overall: require(flat == totals if rows else all(v == 0 for v in flat.values()), "overall/per-k reconciliation failed") continue k = row["flipped bit count"] - require(natural(k, N) and settings["minimum flipped bits"] <= k <= settings[ + require(natural(k, n) and settings["minimum flipped bits"] <= k <= settings[ "maximum flipped bits"], "invalid flipped bit count") require(k not in pooled, f"duplicate k: {k}") require(stats["full-codeword bits"]["raw corrupted bits"] == trials*k, @@ -107,41 +140,63 @@ def load_report(path): try: metadata = read_json(path.parent / "metadata.json") summary = read_json(path) - require(type(metadata) is dict and set(metadata) - {"codeword"} == { + require(type(metadata) is dict and set(metadata) - {"codeword", "storage"} == { "schema revision", "created at", "settings", "code", "random algorithm"}, "incompatible metadata fields") + snapshot = "storage" in metadata + require(not snapshot or (metadata["storage"] == SNAPSHOTS and metadata["schema revision"] == 2), + "incompatible snapshot storage/schema") codeword = metadata.get("codeword", "random") require(codeword in ("zero", "random"), "incompatible codeword convention") require(type(metadata["schema revision"]) is int and metadata["schema revision"] in (1, 2) - and metadata["code"] == CODE and metadata["random algorithm"] in (RANDOM, RANDOM_FY), + and metadata["random algorithm"] in (RANDOM, RANDOM_FY), "incompatible schema/code/random algorithm") require(isinstance(metadata["created at"], str), "invalid created at") settings = metadata["settings"] + dims = dimensions(settings) + n1, k1, n2, k2 = dims + ds = denominators(dims) + n = ds["full"] + require(metadata["code"] == f"RS{n1},{k1} x RS{n2},{k2} Cantor systematic row major", + "incompatible schema/code/random algorithm: code/dimensions/coordinates mismatch") integer_settings = {"root seed", "batch size", "batches", "threads", "minimum flipped bits", "maximum flipped bits", "maximum directional passes", "checkpoint trials", - "report seconds", "fsync seconds"} + "report seconds", "fsync seconds"} + if "n1" in settings: + integer_settings |= {"n1", "k1", "n2", "k2"} require(type(settings) is dict and set(settings) == integer_settings | { "anchors", "binary image"}, "incompatible settings") for name in integer_settings: require(natural(settings[name], (1 << 64) - 1), f"invalid {name}") for name in ("anchors", "binary image"): require(type(settings[name]) is bool, f"invalid {name}") - require(0 <= settings["minimum flipped bits"] <= settings["maximum flipped bits"] <= N, + require(0 <= settings["minimum flipped bits"] <= settings["maximum flipped bits"] <= n, "invalid sampled k range") for name, low, high in (("batch size", 1, (1 << 64) - 1), ("threads", 1, 1024), ("maximum directional passes", 2, 1000000), ("checkpoint trials", 1, 4096), ("report seconds", 1, 86400), ("fsync seconds", 1, 86400)): require(low <= settings[name] <= high, f"invalid {name}") - require(type(summary) is dict and set(summary) == { + require(type(summary) is dict and set(summary) - {"code parameters", "checkpoint"} == { "schema revision", "run identity", "overall", "by flipped bit count"}, "incompatible summary fields") + recorded_snapshot = snapshot and metadata["random algorithm"] == RANDOM_FY + require(("checkpoint" in summary) == recorded_snapshot, "invalid snapshot checkpoint") + if recorded_snapshot: + checkpoint = summary["checkpoint"] + require(type(checkpoint) is dict and set(checkpoint) == {"flip end"} + and natural(checkpoint["flip end"], (1 << 64) - 1) + and checkpoint["flip end"] >= 40, "invalid committed flip boundary") + if "code parameters" in summary: + parameters = summary["code parameters"] + require(type(parameters) is dict and set(parameters) == {"n1", "k1", "n2", "k2"} + and dimensions(parameters) == dims, "summary code parameters mismatch") require(type(summary["schema revision"]) is int and summary["schema revision"] == metadata["schema revision"] and summary["run identity"] == identity(metadata), "identity/schema mismatch") if summary["schema revision"] == 2: pooled = minimal_rows(summary, settings) - config = (settings["maximum directional passes"], settings["anchors"], settings["binary image"]) + config = configuration(settings) return summary["run identity"], settings["root seed"], config, pooled, codeword rows = summary["by flipped bit count"] require(type(rows) is list, "invalid per-k rows") @@ -165,7 +220,7 @@ def load_report(path): and total * total <= trials * square, f"invalid moments: {name}") require(trials > 0 or total == square == 0, "nonzero empty statistics") if name in METRICS.values(): - d = DENOMINATORS[next(m for m in METRICS if METRICS[m] == name)] + d = ds[next(m for m in METRICS if METRICS[m] == name)] require(total <= trials * d and square <= d * total, f"residual moments exceed bit count: {name}") require(stats[METRICS["information"]]["sum"] <= stats[METRICS["full"]]["sum"], @@ -176,7 +231,7 @@ def load_report(path): "overall/per-k reconciliation failed") continue k = row["flipped bit count"] - require(natural(k, N) and settings["minimum flipped bits"] <= k <= settings[ + require(natural(k, n) and settings["minimum flipped bits"] <= k <= settings[ "maximum flipped bits"], "invalid flipped bit count") require(k not in pooled, f"duplicate k: {k}") if "initial full block corrupted bits" in stats: @@ -190,7 +245,7 @@ def load_report(path): totals[name][field] += stats[name][field] count += trials pooled[k] = {"trials": trials, **{m: stats[name]["sum"] for m, name in METRICS.items()}} - config = (settings["maximum directional passes"], settings["anchors"], settings["binary image"]) + config = configuration(settings) return summary["run identity"], settings["root seed"], config, pooled, codeword except (KeyError, TypeError, ValueError) as error: raise ValueError(f"{path}: {error}") from error @@ -323,8 +378,10 @@ def evaluate(rows, p, n=N, denominators=None): def config_label(config): - passes, anchors, binary = config - return f"passes={passes}, anchors={'on' if anchors else 'off'}, binary-image={'on' if binary else 'off'}" + passes, anchors, binary = config[:3] + n1, k1, n2, k2 = config_dimensions(config) + return (f"RS{n1},{k1} x RS{n2},{k2}; passes={passes}, " + f"anchors={'on' if anchors else 'off'}, binary-image={'on' if binary else 'off'}") def plot_value(value): @@ -332,15 +389,16 @@ def plot_value(value): return result if result > 0 else math.nan -def conditional_values(rows, metric): +def conditional_values(rows, metric, n=N, ds=None): """Yield provenance and normalized conditional BER in ascending k.""" + ds = DENOMINATORS if ds is None else ds for k, row in sorted(rows.items()): total, trials = row[metric], row["trials"] mean = total / trials log_mean = math.log(total) - math.log(trials) if total else NEG_INF - log_ber = log_mean - math.log(DENOMINATORS[metric]) + log_ber = log_mean - math.log(ds[metric]) yield {"k": k, "residual_bits_sum": total, "completed_blocks": trials, - "mean": mean, "log10_mean": log_mean / math.log(10), "raw_ber": k / N, + "mean": mean, "log10_mean": log_mean / math.log(10), "raw_ber": k / n, "conditional_ber": math.exp(log_ber), "log10_conditional_ber": log_ber / math.log(10)} @@ -349,9 +407,10 @@ def export_conditional(groups, metrics, path): """Export pooled conditional BER points, independent of plotted mode/limits.""" records = [] for config, rows in groups.items(): + ds = denominators(config_dimensions(config)) for metric in metrics: points = [] - for value in conditional_values(rows, metric): + for value in conditional_values(rows, metric, ds["full"], ds): log_ber = value["log10_conditional_ber"] points.append({"raw_ber": value["raw_ber"], "residual_ber": value["conditional_ber"], @@ -378,16 +437,18 @@ def plot_results(groups, metrics, mode, ps, output, csv_path, plt, thin=0.5): "raw_ber", "conditional_ber", "log10_conditional_ber"]) writer.writeheader() for index, (config, rows) in enumerate(groups.items()): + ds = denominators(config_dimensions(config)) + n = ds["full"] label = config_label(config) color = colors[index % len(colors)] sampled.update(rows) - weighted = [evaluate(rows, p) for p in ps] if mode != "conditional" else [] + weighted = [evaluate(rows, p, n, ds) for p in ps] if mode != "conditional" else [] for metric in metrics: zeros = sum(row[metric] == 0 for row in rows.values()) base = {"configuration": label, "metric": metric, "sampled_strata": len(rows), "zero_observed_strata": zeros} if mode != "ber": - values = list(conditional_values(rows, metric)) + values = list(conditional_values(rows, metric, n, ds)) for value in values: writer.writerow({**base, "record_type": "conditional", **value}) ys = [plot_value(v["log10_conditional_ber"] * math.log(10)) for v in values] @@ -406,7 +467,7 @@ def plot_results(groups, metrics, mode, ps, output, csv_path, plt, thin=0.5): color=color, marker="o" if metric == "information" else "x", label=f"Fixed-weight conditional BER; {metric}: {label}\n{note}") if mode != "conditional": - print(f"{label}; {metric}: {len(rows)}/{N + 1} sampled strata, " + print(f"{label}; {metric}: {len(rows)}/{n + 1} sampled strata, " f"{zeros} zero-observed strata; log10 missing mass range " f"[{min(v[2] for v in weighted) / math.log(10):.6g}, " f"{max(v[2] for v in weighted) / math.log(10):.6g}]", file=sys.stderr) @@ -419,7 +480,7 @@ def plot_results(groups, metrics, mode, ps, output, csv_path, plt, thin=0.5): color=color, linestyle="-" if metric == "information" else "--", label=f"Sampled-stratum BSC contribution; {metric}: {label}") axis.set(xscale="linear", yscale="log", xlim=(0.008, 0.0045), ylim=(1e-30, 1e-1), - xlabel="Raw BER (k / 524288 for conditional; p for BSC)", + xlabel="Raw BER (k / transmitted bits for conditional; p for BSC)", ylabel="Residual BER", title="Product-code BER") if mode == "conditional" and not positive: message = ("All observed residual sums are zero; no positive BERs to plot." @@ -446,8 +507,8 @@ def main(argv=None): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter, epilog="BER mode uses Binomial(N,p) weights, without renormalization. Conditional mode uses " - "raw BER k/524288 and residual BER sum(residual bits)/(sum(completed blocks)*D), " - "D=455168 for information or 524288 for full. Both mode overlays conditional scatter and " + "raw BER k/N and residual BER sum(residual bits)/(sum(completed blocks)*D), " + "N=8*n1*n2; D=8*k1*k2 for information or N for full, per code. Both mode overlays conditional scatter and " "sampled-stratum BSC contribution lines; conditional BER is not a full BSC expectation. " "All modes use descending linear x 0.008 to 0.0045 and log y 1e-30 to 1e-1; " "out-of-range points are clipped, not discarded from CSV. " @@ -457,9 +518,10 @@ def main(argv=None): "Missing k are unknown; " "zero observed errors are not certainty. Arbitrarily low plotted values are not reliability " "evidence. No extrapolation or MSE fit: a fitting model has not been specified. " - "Matching decoder configurations pool per-k sums/counts; different flags/caps stay separate. " + "Matching decoder configurations pool per-k sums/counts; different dimensions/flags/caps stay separate. " "Repeated seeds within a configuration and overlapping inputs are rejected. " - f"Only the fixed code {CODE} is supported; other dimensions/coordinates are rejected. " + "Cantor systematic row-major codes only: strong N,R powers of two, N<=256, 2<=R<=K; " + "weak N<=256, K>=2, R=2, including shortening. Absent dimensions mean 256,224,256,254. " "Zero/random conventions must match unless --allow-mixed-codewords is explicit. " "Metadata has no source revision. " "Discovery stops at run directories, ignores unrelated files, and does not follow subdirectory " @@ -504,7 +566,7 @@ def main(argv=None): print("Warning: metadata lacks source revision; decoder implementation compatibility cannot be verified. " "Missing strata are unknown; zero observed errors do not establish zero BER. " "No extrapolation or MSE fit is performed. " - f"Validated {len(reports)} report(s); only the fixed code {CODE} is supported.", file=sys.stderr) + f"Validated {len(reports)} report(s); dimensions and Cantor coordinates checked.", file=sys.stderr) metrics = list(METRICS) if args.metric == "both" else [args.metric] try: import matplotlib diff --git a/src/reed_solomon/strong_weak_rs_product_code.cc b/src/reed_solomon/strong_weak_rs_product_code.cc index 87c80b0..274a8a7 100644 --- a/src/reed_solomon/strong_weak_rs_product_code.cc +++ b/src/reed_solomon/strong_weak_rs_product_code.cc @@ -25,12 +25,12 @@ StrongWeakRSProductCode::StrongWeakRSProductCode(size_t strong_n, strong_k_(strong_k), weak_n_(weak_n), weak_k_(weak_k), - valid_(Aligned(strong_n, strong_k) && Aligned(weak_n, weak_k) && - weak_n - weak_k == 2), + valid_(Aligned(strong_n, strong_k) && weak_n <= 256 && weak_k >= 2 && + weak_k < weak_n && weak_n - weak_k == 2), strong_encoder_(valid_ ? strong_k : 0, valid_ ? strong_n - strong_k : 0), weak_encoder_(valid_ ? weak_k : 0, valid_ ? weak_n - weak_k : 0), strong_decoder_(valid_ ? strong_k : 0, valid_ ? strong_n - strong_k : 0), - weak_decoder_(valid_ ? weak_k : 0, valid_ ? weak_n - weak_k : 0) {} + weak_decoder_(valid_ ? std::bit_ceil(weak_n) - 2 : 0, valid_ ? 2 : 0) {} bool StrongWeakRSProductCode::Valid() const { return valid_ && strong_encoder_.Valid() && weak_encoder_.Valid() && @@ -113,7 +113,12 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( std::array clean_columns{}, clean_rows{}; std::array active{}; std::array candidate{}; - std::vector packed(batch_passes != 0 ? block.size() : 0); + const size_t mother_n = std::bit_ceil(weak_n_); + const size_t mother_k = mother_n - 2; + const auto weak_position = [&](size_t pos) { + return pos < weak_k_ ? pos : mother_k + pos - weak_k_; + }; + std::vector packed(batch_passes != 0 ? strong_n_ * mother_n : 0); std::vector masks(packed.size()); std::array outcomes{}; std::array shards{}; @@ -121,6 +126,7 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( const bool strong = pass % 2 == 0; const size_t lines = strong ? weak_n_ : strong_n_; const size_t length = strong ? strong_n_ : weak_n_; + const size_t decoder_length = strong ? length : mother_n; std::array next{}; size_t changes = 0; size_t bit_changes = 0; @@ -131,18 +137,23 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( if (strong) { std::copy(block.begin(), block.end(), packed.begin()); } - for (size_t pos = 0; pos < length; ++pos) { + for (size_t pos = 0; pos < decoder_length; ++pos) { shards[pos] = packed.data() + pos * lines; if (!strong) { for (size_t line = 0; line < lines; ++line) { - shards[pos][line] = block[line * weak_n_ + pos]; + shards[pos][line] = + pos >= weak_k_ && pos < mother_k + ? Element{0} + : block[line * weak_n_ + + (pos < weak_k_ ? pos : weak_k_ + pos - mother_k)]; } } } const auto status = detail::error_correction::CorrectCodewordBatch( strong ? strong_decoder_ : weak_decoder_, - std::span(shards).first(length), lines, - std::span(outcomes).first(lines), masks); + std::span(shards).first(decoder_length), lines, + std::span(outcomes).first(lines), + std::span(masks).first(decoder_length * lines)); if (status != CorrectionStatus::ok) { return result; } @@ -182,38 +193,47 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( return strong ? pos * weak_n_ + line : line * weak_n_ + pos; }; if (!batched) { + candidate.fill(0); for (size_t pos = 0; pos < length; ++pos) { - candidate[pos] = block[index(pos)]; + candidate[strong ? pos : weak_position(pos)] = block[index(pos)]; } } const auto correction = - batched ? outcomes[line] - : CorrectCodeword(strong ? strong_decoder_ : weak_decoder_, - std::span(candidate).first(length)); + batched + ? outcomes[line] + : CorrectCodeword(strong ? strong_decoder_ : weak_decoder_, + std::span(candidate).first(decoder_length)); + if (batched) { + for (size_t pos = 0; pos < decoder_length; ++pos) { + candidate[pos] = shards[pos][line]; + } + } + // A mother-code repair is not a shortened-code candidate if it + // changes any known-zero data. Reject before validity or gate updates. + const bool shortened_valid = + strong || std::all_of(candidate.begin() + weak_k_, + candidate.begin() + mother_k, + [](Element value) { return value == 0; }); auto& clean = strong ? clean_columns[line] : clean_rows[line]; - clean = correction.status == CorrectionStatus::ok && + clean = shortened_valid && correction.status == CorrectionStatus::ok && correction.error_count == 0; if (strong) { protected_columns[line] = correction.status == CorrectionStatus::ok; } - if (correction.status != CorrectionStatus::ok) { + if (!shortened_valid || correction.status != CorrectionStatus::ok) { continue; } if (correction.error_count == 0) { continue; } - if (batched) { - for (size_t pos = 0; pos < length; ++pos) { - candidate[pos] = shards[pos][line]; - } - } if (!strong) { if (correction.error_count != 1) { continue; } bool accept = true; for (size_t pos = 0; pos < length; ++pos) { - const unsigned delta = block[index(pos)] ^ candidate[pos]; + const unsigned delta = + block[index(pos)] ^ candidate[weak_position(pos)]; if (delta != 0 && ((options.use_anchors && protected_columns[pos]) || (options.use_binary_image && std::popcount(delta) > 2))) { @@ -225,10 +245,11 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( } } for (size_t pos = 0; pos < length; ++pos) { - if (block[index(pos)] != candidate[pos]) { - bit_changes += std::popcount( - static_cast(block[index(pos)] ^ candidate[pos])); - block[index(pos)] = candidate[pos]; + const auto value = candidate[strong ? pos : weak_position(pos)]; + if (block[index(pos)] != value) { + bit_changes += + std::popcount(static_cast(block[index(pos)] ^ value)); + block[index(pos)] = value; next[pos] = true; (strong ? clean_rows[pos] : clean_columns[pos]) = false; ++changes; @@ -263,13 +284,14 @@ ProductCorrectionResult StrongWeakRSProductCode::CorrectImpl( (strong ? clean_columns[line] : clean_rows[line])) { continue; } + candidate.fill(0); for (size_t pos = 0; pos < length; ++pos) { - candidate[pos] = + candidate[strong ? pos : weak_position(pos)] = block[strong ? pos * weak_n_ + line : line * weak_n_ + pos]; } - const auto check = - CorrectCodeword(strong ? strong_decoder_ : weak_decoder_, - std::span(candidate).first(length)); + const auto check = CorrectCodeword( + strong ? strong_decoder_ : weak_decoder_, + std::span(candidate).first(strong ? length : mother_n)); if (check.status != CorrectionStatus::ok || check.error_count != 0) { result.all_zero_syndromes = false; } diff --git a/tests/plot_product_monte_carlo_test.py b/tests/plot_product_monte_carlo_test.py index 5bd0eea..f179051 100644 --- a/tests/plot_product_monte_carlo_test.py +++ b/tests/plot_product_monte_carlo_test.py @@ -19,6 +19,36 @@ class NumericalTest(unittest.TestCase): + def test_dimension_specific_conditional_export_and_weighted_plot(self): + dims = (4, 2, 5, 3) + ds = plot.denominators(dims) + n = ds["full"] + config = (16, True, True) + dims + rows = {k: {"trials": 2, "information": min(k, ds["information"])*2, + "full": 2*k} for k in range(n+1)} + p = 0.17 + expected = sum(math.comb(n, k)*p**k*(1-p)**(n-k)*min(k, ds["information"]) + / ds["information"] for k in rows) + values, covered, missing = plot.evaluate(rows, p, n, ds) + self.assertAlmostEqual(math.exp(values["full"]), p, places=13) + self.assertAlmostEqual(math.exp(values["information"]), expected, places=13) + self.assertEqual((covered, missing), (0, -math.inf)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "export.json" + plot.export_conditional({config: rows}, ["information", "full"], path) + exports = json.loads(path.read_text()) + self.assertIn("RS4,2 x RS5,3", exports[0]["configuration"]) + self.assertEqual(exports[0]["points"][1]["raw_ber"], 1/n) + self.assertAlmostEqual(exports[0]["points"][1]["residual_ber"], 1/ds["information"]) + plt = mock.MagicMock() + plt.subplots.return_value = (mock.MagicMock(), mock.MagicMock()) + plt.rcParams.__getitem__.return_value.by_key.return_value = {"color": ["blue"]} + with mock.patch.object(plot, "evaluate", wraps=plot.evaluate) as evaluate, \ + mock.patch("sys.stderr", io.StringIO()): + plot.plot_results({config: rows}, ["full"], "both", [p], + Path(directory)/"out.svg", Path(directory)/"out.csv", plt) + evaluate.assert_called_once_with(rows, p, n, ds) + def test_pure_conditional_export(self): groups = {(16, True, True): { 1: {"trials": 10, "information": 0, "full": 0}, @@ -337,6 +367,60 @@ def test_schema2_pooling_exact_integers_and_reconciliation(self): with self.assertRaises(ValueError): plot.load_report(path) + def test_shortened_and_default_dimensions_never_pool(self): + default, _, _ = self.fixture("default") + short, metadata, summary = self.fixture("short") + metadata["settings"].update(n1=256, k1=224, n2=175, k2=173) + metadata["settings"]["maximum flipped bits"] = 8*256*175 + metadata["code"] = "RS256,224 x RS175,173 Cantor systematic row major" + summary["run identity"] = plot.identity(metadata) + self.save(short, metadata, summary) + groups = plot.pool_reports([default, short]) + self.assertEqual(len(groups), 2) + self.assertEqual({plot.config_dimensions(config) for config in groups}, + {(256, 224, 256, 254), (256, 224, 175, 173)}) + before = (short / "metadata.json").read_bytes() + self.assertEqual(plot.load_report(short)[0], summary["run identity"]) + self.assertEqual(before, (short / "metadata.json").read_bytes()) + metadata["schema revision"] = summary["schema revision"] = 2 + stats = {"completed blocks": 1, "total iterations": 2, + "information bits": {"total bits": 8*224*173, "raw corrupted bits": 2000, + "post decoding corrupted bits": 1}, + "full-codeword bits": {"total bits": 8*256*175, "raw corrupted bits": 2600, + "post decoding corrupted bits": 1}} + summary["overall"] = {"statistics": copy.deepcopy(stats)} + summary["by flipped bit count"] = [{"flipped bit count": 2600, "statistics": stats}] + summary["run identity"] = plot.identity(metadata) + self.save(short, metadata, summary) + self.assertEqual(plot.pool_reports([default, short]), groups) + summary["code parameters"] = {"n1": 256, "k1": 224, "n2": 175, "k2": 173} + self.save(short, metadata, summary) + self.assertEqual(plot.pool_reports([default, short]), groups) + summary["code parameters"]["k2"] = 172 + self.save(short, metadata, summary) + with self.assertRaises(ValueError): + plot.load_report(short) + summary["code parameters"]["k2"] = 173 + for mutation in (lambda m: m["settings"].pop("k2"), + lambda m: m["settings"].update(n2=174), + lambda m: m["settings"].update({"maximum flipped bits": 358401}), + lambda m: m.update(code=plot.CODE)): + broken = copy.deepcopy(metadata) + mutation(broken) + self.save(short, broken, summary) + with self.assertRaises(ValueError): + plot.load_report(short) + + def test_explicit_default_dimensions_preserve_legacy_group_and_hash(self): + path, metadata, summary = self.fixture() + implicit = plot.load_report(path) + metadata["settings"].update(zip(("n1", "k1", "n2", "k2"), plot.DEFAULT_DIMENSIONS)) + summary["run identity"] = plot.identity(metadata) + self.save(path, metadata, summary) + explicit = plot.load_report(path) + self.assertEqual(implicit[1:], explicit[1:]) + self.assertEqual(explicit[0], plot.identity(metadata)) + def test_fisher_yates_metadata_preserves_summary_format(self): path, metadata, summary = self.fixture() expected = plot.load_report(path)[1:] @@ -345,6 +429,28 @@ def test_fisher_yates_metadata_preserves_summary_format(self): self.save(path, metadata, summary) self.assertEqual(plot.load_report(path)[1:], expected) + def test_snapshot_storage_and_checkpoint_validation(self): + path, metadata, summary = self.fixture() + metadata["schema revision"] = summary["schema revision"] = 2 + metadata["storage"] = plot.SNAPSHOTS + stats = {"completed blocks": 0, "total iterations": 0, + "information bits": {"total bits": 0, "raw corrupted bits": 0, "post decoding corrupted bits": 0}, + "full-codeword bits": {"total bits": 0, "raw corrupted bits": 0, "post decoding corrupted bits": 0}} + summary["overall"] = {"statistics": stats} + summary["by flipped bit count"] = [] + for sampler in (plot.RANDOM, plot.RANDOM_FY): + metadata["random algorithm"] = sampler + summary["run identity"] = plot.identity(metadata) + if sampler == plot.RANDOM_FY: + summary["checkpoint"] = {"flip end": 40} + self.save(path, metadata, summary) + self.assertEqual(plot.load_report(path)[3], {}) + for checkpoint in ({"flip end": 39}, {"flip end": True}, {"flip end": 2**64}, {}): + summary["checkpoint"] = checkpoint + self.save(path, metadata, summary) + with self.assertRaises(ValueError): + plot.load_report(path) + def test_codeword_conventions_and_explicit_pooling(self): legacy, _, _ = self.fixture("legacy") zero, metadata, summary = self.fixture("zero", seed=2) @@ -585,7 +691,7 @@ def test_help_and_headless_outputs(self): self.assertIn("No extrapolation", result.stdout) self.assertIn("recursively", result.stdout) self.assertIn("merged.svg", result.stdout) - self.assertIn("Partial or malformed runs", result.stdout) + self.assertIn("Partial or malformed runs", " ".join(result.stdout.split())) if importlib.util.find_spec("matplotlib") is None: self.skipTest("matplotlib not installed") path, _, _ = self.fixture(residual=0) diff --git a/tests/product_code_tests.cc b/tests/product_code_tests.cc index aa7f69c..e37c3ec 100644 --- a/tests/product_code_tests.cc +++ b/tests/product_code_tests.cc @@ -130,6 +130,17 @@ TEST(ProductCode, DimensionsAndInvalidCalls) { EXPECT_EQ(block, before); } EXPECT_FALSE(StrongWeakRSProductCode(8, 4, 8, 4).Valid()); + for (auto [n, k] : {std::pair{3u, 1u}, + {257u, 255u}, + {175u, 174u}, + {175u, 175u}, + {175u, 176u}}) { + EXPECT_FALSE(StrongWeakRSProductCode(256, 224, n, k).Valid()); + } + for (auto [n, k] : + {std::pair{4u, 2u}, {5u, 3u}, {175u, 173u}, {255u, 253u}}) { + EXPECT_TRUE(StrongWeakRSProductCode(256, 224, n, k).Valid()); + } EXPECT_FALSE( StrongWeakRSProductCode(std::numeric_limits::max(), 1).Valid()); StrongWeakRSProductCode code(4, 2, 8, 6); @@ -151,6 +162,8 @@ TEST(ProductCode, DimensionsAndInvalidCalls) { TEST(ProductCode, SystematicEncodingScalarAgreementAndAllComponentValidity) { for (const auto [ns, ks, nw, kw] : {std::array{4, 2, 8, 6}, {16, 12, 16, 14}, + {4, 2, 5, 3}, + {256, 224, 175, 173}, {256, 224, 256, 254}}) { StrongWeakRSProductCode code(ns, ks, nw, kw); std::mt19937 random(901); @@ -394,7 +407,7 @@ TEST(WholeCodewordBatch, DifferentialDataParityFailuresAndTails) { {256u, 224u}, {256u, 254u}}) { LCHDecoder decoder(k, n - k); - for (size_t lanes : {1u, 31u, 32u, 33u, 65u, 256u}) { + for (size_t lanes : {1u, 31u, 32u, 33u, 65u, 175u, 256u}) { std::vector packed(n * lanes); auto expected = packed; std::vector results(lanes), reference(lanes); @@ -524,10 +537,11 @@ TEST(ProductCode, InitialBatchChoicesMatchSingleOutputsAndAllCounters) { std::mt19937 random(0x5b5c0224); for (const auto dims : {std::array{4, 2, 8, 6}, {32, 16, 64, 62}, + {256, 224, 175, 173}, {256, 224, 256, 254}}) { const auto [ns, ks, nw, kw] = dims; StrongWeakRSProductCode code(ns, ks, nw, kw); - for (size_t trial = 0; trial < 16; ++trial) { + for (size_t trial = 0; trial < (nw == 175 ? 4u : 16u); ++trial) { std::vector input(code.BlockSize()); for (auto& value : input) { value = static_cast(random()); @@ -599,8 +613,11 @@ TEST(ProductCode, InitialBatchChoicesMatchSingleOutputsAndAllCounters) { TEST(ProductCode, TrackedValidityMatchesIndependentParityAcrossCapsAndCancellations) { std::mt19937 random(0xc1ea0224); - for (const auto dims : - {std::array{4, 2, 8, 6}, {8, 4, 4, 2}, {32, 28, 32, 30}}) { + for (const auto dims : {std::array{4, 2, 8, 6}, + {8, 4, 4, 2}, + {32, 28, 32, 30}, + {4, 2, 5, 3}, + {32, 28, 31, 29}}) { const auto [ns, ks, nw, kw] = dims; StrongWeakRSProductCode code(ns, ks, nw, kw); for (size_t trial = 0; trial < 128; ++trial) { @@ -819,6 +836,96 @@ TEST(ProductCode, CountsRepeatedCommittedWritesByIndependentPassDifferences) { EXPECT_TRUE(repeated); } +TEST(ProductCode, ShortenedMotherParityAndVirtualRepairRejection) { + LCHEncoder encoder(254, 2); + LCHDecoder decoder(254, 2); + std::array mother{}; + std::array data{}; + std::array parity{&mother[254], &mother[255]}; + for (size_t i = 0; i < data.size(); ++i) { + data[i] = &mother[i]; + } + std::vector workspace(encoder.WorkspaceSize(1)); + StrongWeakRSProductCode code(32, 28, 175, 173); + // Every one of the 81 omitted positions can be a plausible mother repair. + // The actual rows contain only its two parity symbols. Constant strong + // columns are already valid, leaving these weak candidates untouched. + for (size_t missing = 173; missing < 254; ++missing) { + mother.fill(0); + mother[missing] = 1; + ASSERT_EQ(encoder.Encode(data, parity, 1, workspace, Backend::scalar), + Status::ok); + auto received = mother; + received[missing] = 0; + const auto witness = CorrectCodeword(decoder, received); + ASSERT_EQ(witness.status, CorrectionStatus::ok); + ASSERT_EQ(witness.error_count, 1u); + ASSERT_EQ(received, mother); + std::vector input(code.BlockSize()); + for (size_t row = 0; row < 32; ++row) { + input[row * 175 + 173] = mother[254]; + input[row * 175 + 174] = mother[255]; + } + for (bool anchors : {false, true}) { + for (bool binary : {false, true}) { + for (unsigned batches : {0u, 1u, 2u}) { + auto actual = input; + const auto result = detail::ProductCorrectionAccess::Correct( + code, actual, {16, anchors, binary}, batches); + EXPECT_EQ(actual, input) << missing; + EXPECT_EQ(result.changed_symbols, 0u); + EXPECT_EQ(result.changed_bits, 0u); + EXPECT_EQ(result.weak_changed_symbols, 0u); + EXPECT_EQ(result.strong_lines_visited, 175u); + EXPECT_EQ(result.weak_lines_visited, 32u); + EXPECT_EQ(result.directional_passes, 2u); + EXPECT_EQ(result.termination, ProductTermination::no_change); + EXPECT_FALSE(result.all_zero_syndromes); + } + } + } + } + StrongWeakRSProductCode target(256, 224, 175, 173); + std::vector block(target.BlockSize()); + std::mt19937 random(173); + for (auto& value : block) { + value = static_cast(random()); + } + ASSERT_EQ(target.Encode(block), Status::ok); + ASSERT_TRUE(AllComponentsValid(block, 256, 224, 175, 173)); + for (size_t row = 0; row < 256; ++row) { + mother.fill(0); + std::copy_n(block.begin() + row * 175, 173, mother.begin()); + ASSERT_EQ(encoder.Encode(data, parity, 1, workspace, Backend::scalar), + Status::ok); + EXPECT_EQ(mother[254], block[row * 175 + 173]); + EXPECT_EQ(mother[255], block[row * 175 + 174]); + } +} + +TEST(ProductCode, ShortenedWeakActualParityRepairsAndSelectiveActivation) { + for (size_t ns : {4u, 32u, 256u}) { + StrongWeakRSProductCode code(ns, ns - 2, 175, 173); + for (size_t col : {0u, 172u, 173u, 174u}) { + for (unsigned batches : {0u, 1u, 2u}) { + std::vector block(code.BlockSize()); + block[col] = block[175 + col] = 1; + const auto result = detail::ProductCorrectionAccess::Correct( + code, block, {16, true, true}, batches); + EXPECT_EQ(block, std::vector(code.BlockSize())); + EXPECT_EQ(result.changed_symbols, 2u); + EXPECT_EQ(result.weak_changed_symbols, 2u); + EXPECT_EQ(result.strong_changed_symbols, 0u); + EXPECT_EQ(result.changed_bits, 2u); + EXPECT_EQ(result.strong_lines_visited, 176u); + EXPECT_EQ(result.weak_lines_visited, ns); + EXPECT_EQ(result.directional_passes, 3u); + EXPECT_TRUE(result.all_zero_syndromes); + } + } + } +} + TEST(ProductCode, OptionsDefaultsAndInvalidCapsArePerCall) { StrongWeakRSProductCode code(4, 2, 8, 6); std::vector input(32); diff --git a/tests/product_monte_carlo_data_tests.cc b/tests/product_monte_carlo_data_tests.cc index c4bfa2b..7643057 100644 --- a/tests/product_monte_carlo_data_tests.cc +++ b/tests/product_monte_carlo_data_tests.cc @@ -2,6 +2,46 @@ #include +TEST(ProductMonteCarloData, DimensionsMetadataAndDynamicOverflowBounds) { + mc::Settings settings; + const auto legacy = settings.ToJson(); + EXPECT_FALSE(legacy.contains("n1")); + EXPECT_EQ(mc::Settings::FromJson(legacy).ToJson(), legacy); + settings.n2 = 175; + settings.k2 = 173; + settings.Validate(); + EXPECT_EQ(settings.FullBits(), 358400u); + EXPECT_EQ(settings.InfoBits(), 310016u); + EXPECT_EQ(mc::Settings::FromJson(settings.ToJson()).ToJson(), + settings.ToJson()); + auto partial = settings.ToJson(); + partial.erase("k2"); + EXPECT_THROW(mc::Settings::FromJson(partial), std::runtime_error); + const uint64_t count = UINT64_MAX / settings.FullBits(); + mc::Stats stats{count, 2 * count, count, 0, count, 0}; + const auto json = stats.ToJson(settings); + EXPECT_NO_THROW(mc::Stats::FromJson(json, count, 1, 16, settings)); + EXPECT_THROW(mc::Stats::FromJson(json, count, 1, 16), std::runtime_error); + mc::Aggregate aggregate("shortened", 2, settings); + aggregate.Add(1, stats); + const auto before = aggregate.Summary(); + EXPECT_EQ(before.at("code parameters"), + (mc::Json{{"n1", 256}, {"k1", 224}, {"n2", 175}, {"k2", 173}})); + EXPECT_THROW(aggregate.Add(1, mc::Stats{1}), std::runtime_error); + EXPECT_EQ(aggregate.Summary(), before); + EXPECT_THROW(stats.Add(mc::Stats{1}, settings.FullBits()), + std::runtime_error); + EXPECT_EQ(stats.ToJson(settings), json); + settings.n1 = 4; + settings.k1 = 2; + settings.n2 = 5; + settings.k2 = 3; + EXPECT_THROW(settings.Validate(), std::runtime_error); + settings.lo = 0; + settings.hi = settings.FullBits(); + EXPECT_NO_THROW(settings.Validate()); +} + TEST(ProductMonteCarloData, ExactBoundedIntegers) { const uint64_t n = UINT64_MAX / 524288; mc::Stats s; diff --git a/tests/product_monte_carlo_legacy_test.py b/tests/product_monte_carlo_legacy_test.py index c1f997f..4f1dfcf 100644 --- a/tests/product_monte_carlo_legacy_test.py +++ b/tests/product_monte_carlo_legacy_test.py @@ -159,6 +159,50 @@ def test_native_chunk_prefix_and_replay(self): self.assertEqual(count.value, 0) self.assertEqual(list(output), before) + def test_dimension_state_reuse_equivariance_and_saved_flips(self): + lib = experiment.native() + c = experiment.ctypes + trial = lib.product_trial_dimensions + trial.argtypes = [c.c_uint64] * 5 + [c.c_int, c.c_int, + c.POINTER(c.c_uint64), c.c_int, c.POINTER(c.c_uint32), c.c_int, + c.POINTER(c.c_uint8)] + [c.c_uint64] * 4 + trial.restype = c.c_int + seen = {} + for dims in ((256, 224, 175, 173), (4, 2, 5, 3), (256, 224, 256, 254), + (32, 28, 31, 29), (256, 224, 175, 173)): + n1, k1, n2, k2 = dims + n = 8 * n1 * n2 + for k in (0, 1, min(1800, n // 3), n // 2 + 1, n - 1, n): + for anchors, binary in ((0, 0), (1, 1), (0, 1), (1, 0)): + outputs, residuals = [], [] + positions = (c.c_uint32 * max(1, min(k, n-k)))() + for random, sampler in ((0, 0), (1, 0), (0, 2), (1, 2)): + output = (c.c_uint64 * 22)() + residual = (c.c_uint8 * (n1*n2))() + self.assertEqual(trial(42, 7, 0, k, 4, anchors, binary, + output, sampler, positions, random, residual, *dims), 0) + self.assertEqual(output[0], k) + self.assertLessEqual(output[2], 8*k1*k2) + self.assertLessEqual(output[19], 4*n2) + self.assertLessEqual(output[20], 4*n1) + outputs.append(list(output)); residuals.append(bytes(residual)) + self.assertTrue(all(v == outputs[0] for v in outputs)) + self.assertTrue(all(v == residuals[0] for v in residuals)) + key = dims, k, anchors, binary + self.assertEqual(seen.setdefault(key, outputs[0]), outputs[0]) + # A persistent FY permutation must be resized when dimensions change. + k = n - 3 + positions = (c.c_uint32 * 3)() + output, replay = (c.c_uint64 * 22)(), (c.c_uint64 * 22)() + self.assertEqual(trial(42, 0, 0, k, 2, 0, 0, output, 1, positions, 0, None, *dims), 0) + self.assertEqual(trial(42, 0, 0, k, 2, 0, 0, replay, 2, positions, 1, None, *dims), 0) + self.assertEqual(list(output), list(replay)) + before = list(output) + for dims, k in (((256, 224, 175, 173), 358401), ((256, 224, 175, 172), 0), + ((255, 223, 175, 173), 0), ((4, 2, 3, 1), 0)): + self.assertNotEqual(trial(42, 0, 0, k, 2, 0, 0, output, 0, None, 0, None, *dims), 0) + self.assertEqual(list(output), before) + def test_chunk_failure_preserves_prefix_and_later_successes(self): source = self.run_case("chunk-settings", "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0") settings = self.read(source, "metadata.json")["settings"] diff --git a/tests/product_monte_carlo_test.py b/tests/product_monte_carlo_test.py index 1b71d5a..6aba77a 100644 --- a/tests/product_monte_carlo_test.py +++ b/tests/product_monte_carlo_test.py @@ -10,6 +10,7 @@ import select import shutil import signal +import struct import subprocess import sys import tempfile @@ -59,6 +60,9 @@ def run_case(self, name, *args, **kwargs): path = self.root / name self.invoke("--output", path, "--seed", 42, "--batches", 2, "--batch-size", 17, *args, **kwargs) + if not kwargs.get("reference"): + self.assertFalse((path / "journal.jsonl").exists()) + self.assertEqual(self.read(path, "metadata.json")["storage"], "atomic summary v1") return path def read(self, path, name="summary.json"): @@ -67,6 +71,17 @@ def read(self, path, name="summary.json"): def records(self, path): return [json.loads(s) for s in (path / "journal.jsonl").read_text().splitlines()] + def saved_indices(self, path): + data = (path / "flips.bin").read_bytes() + end = self.read(path)["checkpoint"]["flip end"] + result, offset = [], 40 + while offset < end: + batch, trial, k, count = struct.unpack_from(" 5 for i in indices)) - self.assertEqual(self.count(path), len(indices)) + self.assertFalse((path / "journal.jsonl").exists()) + self.assertGreater(self.count(path), 5) + if sampler == "fisher-yates": + indices = self.saved_indices(path) + self.assertEqual(indices, sorted(set(indices))) + self.assertEqual(indices[:5], list(range(5))) + self.assertNotIn(5, indices) + self.assertTrue(any(i > 5 for i in indices)) + self.assertEqual(self.count(path), len(indices)) self.invoke("--replay" if sampler == "fisher-yates" else "--report", path) - def test_native_flip_sync_before_journal_publication(self): + def test_native_flip_sync_before_snapshot_publication(self): path = self.run_case("sync-order", "--sampler", "fisher-yates", "--threads", 4, "--checkpoint-trials", 3, fault=True, env=dict(os.environ, MC_TEST_SYNC_ORDER="1")) self.assertEqual(self.count(path), 34) @@ -271,6 +432,7 @@ def test_native_flip_sync_before_journal_publication(self): def test_signal_drains_bounded_window_and_plain_progress(self): for sig in (signal.SIGINT, signal.SIGTERM): + dimensions = ["--n2", "175", "--k2", "173"] if sig == signal.SIGTERM else [] path = self.root / str(sig) err = self.root / f"stderr-{sig}" # Slow first block holds the ordered window; other workers must not @@ -280,7 +442,7 @@ def test_signal_drains_bounded_window_and_plain_progress(self): p = subprocess.Popen([str(FAULT), "--output", str(path), "--seed", "42", "--threads", "4", "--batch-size", "1000000", "--checkpoint-trials", "12", "--minimum-flipped-bits", "0", "--maximum-flipped-bits", "0", - "--sampler", "fisher-yates", "--report-seconds", "1"], stderr=stream, env=env) + "--sampler", "fisher-yates", "--report-seconds", "1", *dimensions], stderr=stream, env=env) try: deadline = time.monotonic()+20 while not (path / "progress.log").exists() or "batch=0" not in (path / "progress.log").read_text(): @@ -292,7 +454,8 @@ def test_signal_drains_bounded_window_and_plain_progress(self): finally: if p.poll() is None: p.kill(); p.wait() self.assertEqual(self.count(path), 12) - indices = [i for r in self.records(path) for i in range(r["first trial index"],r["past last trial index"])] + self.assertFalse((path / "journal.jsonl").exists()) + indices = self.saved_indices(path) self.assertEqual(indices, list(range(12))) self.invoke("--replay", path) log = (path / "progress.log").read_text() @@ -304,7 +467,7 @@ def test_signal_drains_bounded_window_and_plain_progress(self): self.assertIsNotNone(match) rate, mib, elapsed = map(float, match.groups()) self.assertAlmostEqual(rate, 12/elapsed, delta=.02) - self.assertAlmostEqual(mib, rate*56896/1048576, delta=.001) + self.assertAlmostEqual(mib, rate*224*(173 if dimensions else 254)/1048576, delta=.001) def test_tty_throttle_and_plain_log(self): path = self.root / "tty" diff --git a/tools/product_monte_carlo.cc b/tools/product_monte_carlo.cc index de23d38..321c260 100644 --- a/tools/product_monte_carlo.cc +++ b/tools/product_monte_carlo.cc @@ -51,17 +51,10 @@ class File { ~File() { ::close(fd_); } File(const File&) = delete; File& operator=(const File&) = delete; - void Write(std::string_view bytes) { #ifdef GF256_MC_TEST_HOOKS - if (path_.filename() == "journal.jsonl" && - std::getenv("MC_TEST_SYNC_ORDER")) { - const auto record = Parse(std::string(bytes)); - if (record.contains("flip end")) { - Require(U64(record.at("flip end")) <= synced_flip_end_, - "journal referenced unsynced flip data"); - } - } + static uint64_t SyncedFlipEnd() { return synced_flip_end_; } #endif + void Write(std::string_view bytes) { while (!bytes.empty()) { auto n = ::write(fd_, bytes.data(), bytes.size()); if (n < 0 && errno == EINTR) { @@ -112,6 +105,13 @@ void SyncDirectory(const fs::path& path) { dir.Sync(); } void AtomicJson(const fs::path& path, const Json& value) { +#ifdef GF256_MC_TEST_HOOKS + if (path.filename() == "summary.json" && value.contains("checkpoint") && + std::getenv("MC_TEST_SYNC_ORDER")) { + Require(U64(value.at("checkpoint").at("flip end")) <= File::SyncedFlipEnd(), + "summary referenced unsynced flip data"); + } +#endif auto temp = path; temp += ".tmp"; { @@ -119,6 +119,14 @@ void AtomicJson(const fs::path& path, const Json& value) { out.Write(Dump(value, true) + "\n"); out.Sync(); } +#ifdef GF256_MC_TEST_HOOKS + if (path.filename() == "summary.json" && + std::getenv("MC_TEST_SNAPSHOT_FAIL") && + Natural(value.at("overall").at("statistics").at("completed blocks")) != + 0) { + throw std::runtime_error("injected failure before atomic summary rename"); + } +#endif fs::rename(temp, path); SyncDirectory(path.parent_path()); } @@ -278,7 +286,7 @@ class Workers { try { if (recorded_) { slot.positions.resize( - std::max(1, std::min(k_, 524288 - k_))); + std::max(1, std::min(k_, s_.FullBits() - k_))); } #ifdef GF256_MC_TEST_HOOKS // Dedicated, noninstalled test executable only. @@ -295,10 +303,11 @@ class Workers { } } #endif - int status = product_trial_reference( + int status = product_trial_dimensions( s_.seed, batch_, index, k_, s_.passes, s_.anchors, s_.binary, slot.metrics.data(), recorded_ ? 1 : 0, - recorded_ ? slot.positions.data() : nullptr, 0, nullptr); + recorded_ ? slot.positions.data() : nullptr, 0, nullptr, s_.n1, + s_.k1, s_.n2, s_.k2); Require(status == 0, "native status=" + std::to_string(status)); } catch (...) { slot.error = std::current_exception(); @@ -325,14 +334,18 @@ class Workers { bool active_ = false, halted_ = false, shutdown_ = false; }; -void WriteFlips(File& file, uint64_t batch, uint64_t k, const Result& result) { - const auto count = std::min(k, 524288 - k); +void WriteFlips(File& file, + uint64_t batch, + uint64_t k, + const Result& result, + uint64_t bits) { + const auto count = std::min(k, bits - k); std::string bytes(208 + 4 * count, '\0'); PutLE(bytes, 0, batch, 8); PutLE(bytes, 8, result.index, 8); PutLE(bytes, 16, k, 4); PutLE(bytes, 20, count, 4); - bytes[24] = k > 262144; + bytes[24] = k > bits / 2; for (size_t i = 0; i < 22; ++i) { PutLE(bytes, 32 + 8 * i, result.metrics[i], 8); } @@ -351,13 +364,12 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { metadata["created at"] = Timestamp(); metadata["settings"] = s.ToJson(); metadata["codeword"] = "zero"; - metadata["code"] = kCode; + metadata["code"] = s.Code(); metadata["random algorithm"] = recorded ? kFisherYates : kFloyd; + metadata["storage"] = kSnapshots; AtomicJson(directory / "metadata.json", metadata); const auto digest = Hash(Dump(metadata)); - Aggregate aggregate(Hex(digest), 2); - AtomicJson(directory / "summary.json", aggregate.Summary()); - File journal(directory / "journal.jsonl", O_WRONLY | O_CREAT | O_EXCL); + Aggregate aggregate(Hex(digest), 2, s); File log(directory / "progress.log", O_WRONLY | O_CREAT | O_EXCL); std::optional flips; if (recorded) { @@ -365,6 +377,15 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { flips->Write("RSFLIP01" + digest); flips->Sync(); } + auto snapshot = [&] { + auto summary = aggregate.Summary(); + if (flips) { + flips->Sync(); + summary["checkpoint"]["flip end"] = flips->Offset(); + } + AtomicJson(directory / "summary.json", summary); + }; + snapshot(); const bool tty = ::isatty(STDERR_FILENO); bool bar_visible = false; auto progress = [&](const std::string& text, bool console = true) { @@ -380,7 +401,6 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { }; progress("root seed=" + std::to_string(s.seed) + " settings persisted before trials"); - journal.Sync(); log.Sync(); SyncDirectory(directory); SyncDirectory(directory.parent_path()); @@ -388,10 +408,9 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { const auto start = Clock::now(); auto last_sync = start, last_report = start, last_bar = start, last_checkpoint = start; - uint64_t completed_total = 0, increment = 0; + uint64_t completed_total = 0, increment = 0, persisted_total = 0; uint64_t batch_completed = 0, batch = 0, k = 0; Stats staged; - uint64_t first = 0, end = 0, flip_start = 0; std::string error; auto throughput = [&](Clock::time_point now) { const double elapsed = std::chrono::duration(now - start).count(); @@ -399,7 +418,7 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { elapsed > 0 ? static_cast(completed_total) / elapsed : 0; std::ostringstream out; out << std::fixed << std::setprecision(3) << "wall blocks/s=" << rate - << " information MiB/s=" << rate * 56896 / 1048576 + << " information MiB/s=" << rate * (s.InfoBits() / 8) / 1048576 << " elapsed seconds=" << elapsed; return out.str(); }; @@ -411,32 +430,16 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { }; auto checkpoint = [&] { if (staged.blocks != 0) { - // Preflight every aggregate and derived total before publishing a record. + // Stage bounded increments in memory; only atomic summaries are durable. #ifdef GF256_MC_TEST_HOOKS if (increment == 1 && std::getenv("MC_TEST_COUNTER_OVERFLOW")) { auto exhausted = aggregate.overall; exhausted.iterations = UINT64_MAX; - exhausted.Add(staged); + exhausted.Add(staged, s.FullBits()); } #endif auto next = aggregate.PrepareAdd(k, staged); const auto next_increment = CheckedAdd(increment, 1); - Json r; - r["schema revision"] = 2; - r["run identity"] = aggregate.identity; - r["increment id"] = Number(increment); - r["batch id"] = batch; - r["first trial index"] = first; - r["past last trial index"] = end; - r["trial count"] = end - first; - r["flipped bit count"] = k; - r["statistics"] = staged.ToJson(); - if (flips) { - flips->Sync(); - r["flip start"] = flip_start; - r["flip end"] = flips->Offset(); - } - journal.Write(Dump(r) + "\n"); aggregate.Commit(std::move(next)); increment = next_increment; staged = Stats{}; @@ -445,9 +448,9 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { }; auto durable = [&] { checkpoint(); - journal.Sync(); log.Sync(); - AtomicJson(directory / "summary.json", aggregate.Summary()); + snapshot(); + persisted_total = aggregate.overall.blocks; last_sync = Clock::now(); }; auto bar = [&](Clock::time_point now, bool force = false) { @@ -497,22 +500,11 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { } } } else { - if (staged.blocks != 0 && end != result.index) { - checkpoint(); - } - if (staged.blocks == 0) { - first = result.index; - if (flips) { - flip_start = flips->Offset(); - } - } auto next_staged = staged; - next_staged.Add(result.metrics); - const auto next_end = CheckedAdd(result.index, 1); + next_staged.Add(result.metrics, s.FullBits()); if (flips) { - WriteFlips(*flips, batch, k, result); + WriteFlips(*flips, batch, k, result, s.FullBits()); } - end = next_end; staged = next_staged; if (staged.blocks == s.checkpoint) { checkpoint(); @@ -528,8 +520,7 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { } if (now - last_report >= std::chrono::seconds(s.report)) { progress(counts() + " persisted overall trials=" + - std::to_string(aggregate.overall.blocks) + " " + - throughput(now), + std::to_string(persisted_total) + " " + throughput(now), !tty); last_report = now; } @@ -547,11 +538,9 @@ void Run(const fs::path& directory, const Settings& s, bool recorded) { } } catch (const std::exception& e) { // Worker failures are drained above. Coordinator I/O/allocation failures - // stop and join workers. Never retry a possibly partially written record - // or replace the last good summary with a partially updated aggregate. - // Report mode can recover the complete journal prefix after storage repair. + // stop and join workers. Leave the last atomic summary authoritative; + // uncommitted flip tails and temporary snapshots are ignored by report. try { - journal.Sync(); log.Sync(); } catch (...) { } @@ -581,16 +570,16 @@ void ReadFlips(std::istream& stream, k = U64(r.at("flipped bit count")); Stats stats; Json old = LegacyStats(); - const size_t count = std::min(k, 524288 - k); + const size_t count = std::min(k, s.FullBits() - k); std::vector positions(std::max(1, count)); - std::vector selected(524288); + std::vector selected(s.FullBits()); for (uint64_t trial = first; trial < end; ++trial) { auto body = ReadExact(stream, 208); Require(GetLE(body, 0, 8) == batch && GetLE(body, 8, 8) == trial && GetLE(body, 16, 4) == k, "flip trial identity mismatch"); Require(GetLE(body, 20, 4) == count && - GetLE(body, 24, 1) == uint64_t(k > 262144), + GetLE(body, 24, 1) == uint64_t(k > s.FullBits() / 2), "invalid flip count/complement"); body += ReadExact(stream, count * 4); Require(ReadExact(stream, 32) == Hash(body), "corrupt flip checksum"); @@ -608,14 +597,14 @@ void ReadFlips(std::istream& stream, } if (replay) { std::array actual{}; - int status = product_trial_reference( + int status = product_trial_dimensions( s.seed, batch, trial, k, s.passes, s.anchors, s.binary, actual.data(), - 2, positions.data(), random, nullptr); + 2, positions.data(), random, nullptr, s.n1, s.k1, s.n2, s.k2); Require(status == 0 && actual == metrics, "replay mismatch batch=" + std::to_string(batch) + " trial=" + std::to_string(trial) + " status=" + std::to_string(status)); } - stats.Add(metrics); + stats.Add(metrics, s.FullBits()); if (schema == 1) { AddLegacyTrial(old, metrics); } @@ -624,10 +613,130 @@ void ReadFlips(std::istream& stream, Require(end_offset >= 0 && static_cast(end_offset) == U64(r.at("flip end")), "flip end offset mismatch"); - Require((schema == 1 ? old : stats.ToJson()) == r.at("statistics"), + Require((schema == 1 ? old : stats.ToJson(s)) == r.at("statistics"), "flip metrics disagree with journal"); } +void ValidateSnapshot(const fs::path& directory, + const Settings& settings, + const std::string& digest, + bool recorded, + bool replay, + bool random) { + std::ifstream input(directory / "summary.json", std::ios::binary); + Require(input.good(), "cannot read authoritative summary snapshot"); + std::string text; + char c; + while (input.get(c)) { + Require(text.size() < 512 * 1024 * 1024, "summary exceeds size limit"); + text += c; + } + Require(input.eof() && !input.bad(), "summary read failed"); + auto summary = Parse(text); + std::set fields{"schema revision", "run identity", "overall", + "by flipped bit count"}; + if (!settings.DefaultDimensions()) { + fields.insert("code parameters"); + } + if (recorded) { + fields.insert("checkpoint"); + } + Fields(summary, fields); + uint64_t flip_end = 0; + if (recorded) { + Fields(summary.at("checkpoint"), {"flip end"}); + flip_end = U64(summary.at("checkpoint").at("flip end")); + Require(flip_end >= 40, "invalid committed flip boundary"); + summary.erase("checkpoint"); + } + Aggregate aggregate(Hex(digest), 2, settings); + const auto& overall = summary.at("overall").at("statistics"); + Natural(overall.at("completed blocks")); + Natural(overall.at("total iterations")); + for (const auto* region : {"information bits", "full-codeword bits"}) { + for (const auto* field : + {"total bits", "raw corrupted bits", "post decoding corrupted bits"}) { + Natural(overall.at(region).at(field)); + } + } + const auto& rows = summary.at("by flipped bit count"); + Require(rows.is_array(), "invalid summary strata"); + for (const auto& row : rows) { + Fields(row, {"flipped bit count", "statistics"}); + const auto k = U64(row.at("flipped bit count")); + Require(k >= settings.lo && k <= settings.hi && !aggregate.by_k.contains(k), + "invalid or duplicate summary stratum"); + const auto count = U64(row.at("statistics").at("completed blocks")); + Require(count > 0, "empty summary stratum"); + aggregate.Add(k, Stats::FromJson(row.at("statistics"), count, k, + settings.passes, settings)); + } + Require(summary == aggregate.Summary(), + "summary identity/parameters/totals reconciliation failed"); + Require(settings.batches == 0 || + __uint128_t(aggregate.overall.blocks) <= + __uint128_t(settings.batches) * settings.size, + "summary exceeds configured trials"); + if (recorded) { + std::ifstream flips(directory / "flips.bin", std::ios::binary); + Require(flips.good() && ReadExact(flips, 40) == "RSFLIP01" + digest, + "invalid flip file identity/version"); + Aggregate verified(Hex(digest), 2, settings); + uint64_t offset = 40; + std::optional> previous; + while (offset < flip_end) { + Require(flip_end - offset >= 240, "invalid committed flip boundary"); + const auto start = flips.tellg(); + const auto header = ReadExact(flips, 208); + const auto batch = GetLE(header, 0, 8), trial = GetLE(header, 8, 8), + k = GetLE(header, 16, 4); + const auto position = std::pair{batch, trial}; + Require((settings.batches == 0 || batch < settings.batches) && + trial < settings.size && (!previous || position > *previous), + "invalid or overlapping saved trial identity"); + Require( + k == product_batch_k(settings.seed, batch, settings.lo, settings.hi), + "saved batch k disagrees with seed/settings"); + const auto next = + CheckedAdd(offset, 240 + 4 * std::min(k, settings.FullBits() - k)); + Require(next <= flip_end, "flip record crosses committed boundary"); + std::array metrics{}; + for (size_t i = 0; i < metrics.size(); ++i) { + metrics[i] = GetLE(header, 32 + 8 * i, 8); + } + Stats stats; + stats.Add(metrics, settings.FullBits()); + const auto json = stats.ToJson(settings); + Stats::FromJson(json, 1, k, settings.passes, settings); + // Reuse the legacy record verifier without persisting a journal record. + Json record{{"batch id", batch}, + {"first trial index", trial}, + {"past last trial index", trial + 1}, + {"flipped bit count", k}, + {"flip start", offset}, + {"flip end", next}, + {"statistics", json}}; + flips.seekg(start); + ReadFlips(flips, record, settings, replay, random, 2); + verified.Add(k, stats); + previous = position; + offset = next; + } + Require(verified.Summary() == summary, + "saved flips disagree with summary snapshot"); + if (flips.peek() != std::char_traits::eof()) { + std::cerr << "unreferenced flip tail ignored (not committed trials)\n"; + } + } + std::cerr << Timestamp() + << " validated snapshot: " << aggregate.overall.blocks + << " trials; summary unchanged\n"; + if (replay) { + std::cerr << "verified replay: " << aggregate.overall.blocks + << " trials, all 22 metrics match\n"; + } +} + void Recover(const fs::path& directory, bool replay) { std::ifstream meta_file(directory / "metadata.json", std::ios::binary); Require(meta_file.good(), "cannot read metadata"); @@ -644,9 +753,16 @@ void Recover(const fs::path& directory, bool replay) { if (metadata.contains("codeword")) { fields.insert("codeword"); } + const bool snapshot = metadata.contains("storage"); + if (snapshot) { + fields.insert("storage"); + Require(metadata.at("storage") == kSnapshots && + U64(metadata.at("schema revision")) == 2, + "incompatible snapshot storage/schema"); + } Fields(metadata, fields); auto schema = U64(metadata.at("schema revision")); - Require((schema == 1 || schema == 2) && metadata.at("code") == Json(kCode) && + Require((schema == 1 || schema == 2) && (metadata.at("random algorithm") == Json(kFloyd) || metadata.at("random algorithm") == Json(kFisherYates)), "incompatible metadata schema/code/random algorithm"); @@ -657,11 +773,18 @@ void Recover(const fs::path& directory, bool replay) { Require(codeword == "zero" || codeword == "random", "incompatible codeword convention"); auto settings = Settings::FromJson(metadata.at("settings")); + Require(metadata.at("code") == settings.Code(), + "incompatible code/dimensions/coordinates"); // Never insert defaults before hashing legacy metadata. const auto digest = Hash(Dump(metadata)); - Aggregate aggregate(Hex(digest), static_cast(schema)); + Aggregate aggregate(Hex(digest), static_cast(schema), settings); const bool recorded = metadata.at("random algorithm") == Json(kFisherYates); Require(!replay || recorded, "this run has no saved flips"); + if (snapshot) { + ValidateSnapshot(directory, settings, digest, recorded, replay, + codeword == "random"); + return; + } std::ifstream journal(directory / "journal.jsonl", std::ios::binary), flips; Require(journal.good(), "cannot read journal"); if (recorded) { @@ -709,7 +832,7 @@ void Recover(const fs::path& directory, bool replay) { Stats stats; if (schema == 1) { const auto& old = r.at("statistics"); - ValidateLegacy(old, count, k, settings.passes); + ValidateLegacy(old, count, k, settings.passes, settings); stats.blocks = count; stats.iterations = Natural(old.at(kMetrics[12]).at("sum")); stats.info_raw = Natural(old.at(kMetrics[2]).at("sum")); @@ -717,7 +840,8 @@ void Recover(const fs::path& directory, bool replay) { stats.full_raw = Natural(old.at(kMetrics[0]).at("sum")); stats.full_post = Natural(old.at(kMetrics[4]).at("sum")); } else { - stats = Stats::FromJson(r.at("statistics"), count, k, settings.passes); + stats = Stats::FromJson(r.at("statistics"), count, k, settings.passes, + settings); } if (recorded) { ReadFlips(flips, r, settings, replay, codeword == "random", schema); @@ -765,8 +889,12 @@ int Main(int argc, char** argv) { "--seed UINT64 (default: generated, printed and persisted before " "work)\n" "--batch-size 1000 --batches 0 (infinite) --threads 1 (1..1024)\n" + "--n1 256 --k1 224 --n2 256 --k2 254 (Cantor [data][parity])\n" + "Strong n1,R1 powers of two, n1<=256, 2<=R1<=k1; weak R2=2,\n" + "n2<=256, k2>=2; only weak shortening is supported.\n" "--minimum-flipped-bits 2500 --maximum-flipped-bits 2700 " - "(inclusive, 0..524288)\n" + "(inclusive, 0..8*n1*n2; small codes need explicit smaller " + "bounds, no cap)\n" "--max-directional-passes 16 (2..1000000)\n" "--[no-]anchors --[no-]binary-image (both enabled)\n" "--sampler floyd|fisher-yates (default floyd; Fisher-Yates saves " @@ -784,13 +912,21 @@ int Main(int argc, char** argv) { "rejected.\n" "SIGINT/SIGTERM stop starts, drain whole in-flight blocks, " "checkpoint and fsync.\n" - "Crash loss: bounded worker window/staged increment plus journal " - "writes since fsync.\n" - "Ordered journal flush at least once/second subject to in-flight " - "completion and I/O.\n" - "Saved flips are fsynced before journal references; replay checks " + "New runs create no journal: atomic summary snapshots are " + "authoritative.\n" + "Snapshots are fsynced every --fsync-seconds and on graceful " + "exit;\n" + "crash loss is work since the last snapshot, plus in-flight " + "blocks.\n" + "Report validates snapshot-only runs without rewriting the " + "summary;\n" + "legacy journal runs still support report regeneration.\n" + "Saved flips are fsynced before the summary's committed boundary; " + "replay checks " "all 22 private metrics.\n" - "Legacy absent codeword means random; report ignores only an " + "Uncommitted flip tails are ignored. Legacy absent codeword means " + "random;\n" + "legacy journal report ignores only an " "incomplete final line.\n"; return 0; } @@ -834,6 +970,14 @@ int Main(int argc, char** argv) { s.batches = v; } else if (flag == "--threads") { s.threads = v; + } else if (flag == "--n1") { + s.n1 = v; + } else if (flag == "--k1") { + s.k1 = v; + } else if (flag == "--n2") { + s.n2 = v; + } else if (flag == "--k2") { + s.k2 = v; } else if (flag == "--minimum-flipped-bits") { s.lo = v; } else if (flag == "--maximum-flipped-bits") { diff --git a/tools/product_monte_carlo_data.h b/tools/product_monte_carlo_data.h index 29c1803..99e40d9 100644 --- a/tools/product_monte_carlo_data.h +++ b/tools/product_monte_carlo_data.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -16,6 +17,7 @@ namespace mc { using Json = nlohmann::json; inline constexpr std::string_view kCode = "RS256,224 x RS256,254 Cantor systematic row major"; +inline constexpr std::string_view kSnapshots = "atomic summary v1"; inline constexpr std::string_view kFloyd = "splitmix64 domain seeds; mt19937_64; rejection modulo; Floyd complement " "v1"; @@ -131,13 +133,35 @@ inline void Fields(const Json& value, std::set expected) { } struct Settings { + uint64_t n1 = 256, k1 = 224, n2 = 256, k2 = 254; uint64_t seed = 0, size = 1000, batches = 0, threads = 1, lo = 2500, hi = 2700, passes = 16, checkpoint = 64, report = 2, sync = 5; bool anchors = true, binary = true; + /** @brief Transmitted bits per block after dimension validation. */ + uint64_t FullBits() const { return 8 * n1 * n2; } + /** @brief Information bits per block after dimension validation. */ + uint64_t InfoBits() const { return 8 * k1 * k2; } + /** @brief Whether legacy implicit dimensions describe this code. */ + bool DefaultDimensions() const { + return n1 == 256 && k1 == 224 && n2 == 256 && k2 == 254; + } + /** @brief Code and coordinate convention, preserving the legacy spelling. */ + std::string Code() const { + return "RS" + std::to_string(n1) + "," + std::to_string(k1) + " x RS" + + std::to_string(n2) + "," + std::to_string(k2) + + " Cantor systematic row major"; + } + void Validate() const { - Require(lo <= hi && hi <= 524288, - "require 0 <= minimum <= maximum <= 524288"); + Require(n1 <= 256 && std::has_single_bit(n1) && k1 < n1 && n1 - k1 >= 2 && + n1 - k1 <= k1 && std::has_single_bit(n1 - k1) && n2 <= 256 && + k2 >= 2 && k2 < n2 && n2 - k2 == 2, + "invalid dimensions: strong N,R powers of two, N<=256, 2<=R<=K; " + "weak N<=256, K>=2, R=2 (shortening supported)"); + Require(lo <= hi && hi <= FullBits(), + "require 0 <= minimum <= maximum <= 8*n1*n2; set smaller explicit " + "flip bounds for small codes (defaults 2500..2700 are not capped)"); Require(size > 0, "batch size must be positive"); Require(threads >= 1 && threads <= 1024, "threads must be in [1,1024]"); Require(passes >= 2 && passes <= 1000000, @@ -161,14 +185,40 @@ struct Settings { j["fsync seconds"] = sync; j["anchors"] = anchors; j["binary image"] = binary; + if (!DefaultDimensions()) { + j["n1"] = n1; + j["k1"] = k1; + j["n2"] = n2; + j["k2"] = k2; + } return j; } static Settings FromJson(const Json& j) { - Fields(j, {"root seed", "batch size", "batches", "threads", - "minimum flipped bits", "maximum flipped bits", - "maximum directional passes", "checkpoint trials", - "report seconds", "fsync seconds", "anchors", "binary image"}); + std::set fields{"root seed", + "batch size", + "batches", + "threads", + "minimum flipped bits", + "maximum flipped bits", + "maximum directional passes", + "checkpoint trials", + "report seconds", + "fsync seconds", + "anchors", + "binary image"}; + const bool dimensions = j.contains("n1") || j.contains("k1") || + j.contains("n2") || j.contains("k2"); + if (dimensions) { + fields.insert({"n1", "k1", "n2", "k2"}); + } + Fields(j, fields); Settings s; + if (dimensions) { + s.n1 = U64(j.at("n1")); + s.k1 = U64(j.at("k1")); + s.n2 = U64(j.at("n2")); + s.k2 = U64(j.at("k2")); + } s.seed = U64(j.at("root seed")); s.size = U64(j.at("batch size")); s.batches = U64(j.at("batches")); @@ -191,25 +241,25 @@ struct Settings { struct Stats { uint64_t blocks = 0, iterations = 0, info_raw = 0, info_post = 0, full_raw = 0, full_post = 0; - void Add(const std::array& m) { - Add(Stats{1, m[12], m[2], m[6], m[0], m[4]}); + void Add(const std::array& m, uint64_t full_bits = 524288) { + Add(Stats{1, m[12], m[2], m[6], m[0], m[4]}, full_bits); } - void Add(const Stats& s) { + void Add(const Stats& s, uint64_t full_bits = 524288) { Stats next{ CheckedAdd(blocks, s.blocks), CheckedAdd(iterations, s.iterations), CheckedAdd(info_raw, s.info_raw), CheckedAdd(info_post, s.info_post), CheckedAdd(full_raw, s.full_raw), CheckedAdd(full_post, s.full_post)}; - CheckedMultiply(next.blocks, 524288); + CheckedMultiply(next.blocks, full_bits); *this = next; } - Json ToJson() const { + Json ToJson(const Settings& settings = {}) const { Json j; j["completed blocks"] = Number(blocks); j["total iterations"] = Number(iterations); for (bool info : {true, false}) { Json bits; - bits["total bits"] = - Number(CheckedMultiply(blocks, info ? 455168 : 524288)); + bits["total bits"] = Number(CheckedMultiply( + blocks, info ? settings.InfoBits() : settings.FullBits())); bits["raw corrupted bits"] = Number(info ? info_raw : full_raw); bits["post decoding corrupted bits"] = Number(info ? info_post : full_post); @@ -220,7 +270,8 @@ struct Stats { static Stats FromJson(const Json& j, uint64_t count, uint64_t k, - uint64_t passes) { + uint64_t passes, + const Settings& settings = {}) { Fields(j, {"completed blocks", "total iterations", "information bits", "full-codeword bits"}); Stats s; @@ -236,7 +287,8 @@ struct Stats { const auto& bits = j.at(info ? "information bits" : "full-codeword bits"); Fields(bits, {"total bits", "raw corrupted bits", "post decoding corrupted bits"}); - const auto total = CheckedMultiply(count, info ? 455168 : 524288); + const auto total = CheckedMultiply( + count, info ? settings.InfoBits() : settings.FullBits()); auto raw = Natural(bits.at("raw corrupted bits")); auto post = Natural(bits.at("post decoding corrupted bits")); Require(Natural(bits.at("total bits")) == total && raw <= total && @@ -248,8 +300,12 @@ struct Stats { Require(s.full_raw == CheckedMultiply(count, k), "initial channel is not exact k"); Require(s.info_raw <= s.full_raw && s.info_post <= s.full_post && - s.full_raw - s.info_raw <= CheckedMultiply(count, 69120) && - s.full_post - s.info_post <= CheckedMultiply(count, 69120), + s.full_raw - s.info_raw <= + CheckedMultiply( + count, settings.FullBits() - settings.InfoBits()) && + s.full_post - s.info_post <= + CheckedMultiply(count, + settings.FullBits() - settings.InfoBits()), "inconsistent information/full bit counters"); return s; } @@ -286,30 +342,32 @@ inline void AddLegacyTrial(Json& to, const std::array& m) { inline void ValidateLegacy(const Json& j, uint64_t count, uint64_t k, - uint64_t passes) { + uint64_t passes, + const Settings& settings = {}) { Fields(j, std::set(kMetrics.begin(), kMetrics.end())); Require(passes >= 2 && passes <= 1000000, "invalid legacy pass cap"); - std::array bounds{524288, - 65536, - 455168, - 56896, - 524288, - 65536, - 455168, - 56896, + const auto full = settings.FullBits(), info = settings.InfoBits(); + std::array bounds{full, + full / 8, + info, + info / 8, + full, + full / 8, + info, + info / 8, 1, 1, 1, 1, passes, - passes * 524288, - passes * 524288, - passes * 524288, - passes * 524288, - passes * 524288, - passes * 524288, - passes * 524288, - passes * 524288, + passes * full, + passes * (full / 8), + passes * full, + passes * (full / 8), + passes * full, + passes * (full / 8), + passes * settings.n2, + passes * settings.n1, 1}; for (size_t i = 0; i < kMetrics.size(); ++i) { const auto& item = j.at(kMetrics[i]); @@ -340,13 +398,14 @@ inline void ValidateLegacy(const Json& j, struct Aggregate { std::string identity; unsigned schema; + Settings settings; Stats overall; std::map by_k; Json legacy = LegacyStats(); std::map legacy_k; /** @brief Initialize an empty aggregate for a run and schema revision. */ - Aggregate(std::string id, unsigned revision) - : identity(std::move(id)), schema(revision) {} + Aggregate(std::string id, unsigned revision, Settings dimensions = {}) + : identity(std::move(id)), schema(revision), settings(dimensions) {} struct PreparedAdd { Stats overall; @@ -368,10 +427,10 @@ struct Aggregate { const Json& old = Json()) const { PreparedAdd next; next.overall = overall; - next.overall.Add(stats); + next.overall.Add(stats, settings.FullBits()); const auto it = by_k.find(k); Stats row = it == by_k.end() ? Stats{} : it->second; - row.Add(stats); + row.Add(stats, settings.FullBits()); // Stage just one node, with the same allocator as the destination map. decltype(by_k) rows; rows.emplace(k, row); @@ -420,6 +479,12 @@ struct Aggregate { Json out; out["schema revision"] = schema; out["run identity"] = identity; + if (!settings.DefaultDimensions()) { + out["code parameters"] = {{"n1", settings.n1}, + {"k1", settings.k1}, + {"n2", settings.n2}, + {"k2", settings.k2}}; + } Json rows = Json::array(); for (const auto& [k, s] : by_k) { Json row; @@ -428,7 +493,7 @@ struct Aggregate { row["trial count"] = Number(s.blocks); row["statistics"] = legacy_k.at(k); } else { - row["statistics"] = s.ToJson(); + row["statistics"] = s.ToJson(settings); } rows.push_back(std::move(row)); } @@ -437,7 +502,7 @@ struct Aggregate { out["overall"]["trial count"] = Number(overall.blocks); out["overall"]["statistics"] = legacy; } else { - out["overall"]["statistics"] = overall.ToJson(); + out["overall"]["statistics"] = overall.ToJson(settings); } return out; } diff --git a/tools/product_monte_carlo_native.cc b/tools/product_monte_carlo_native.cc index 623658c..dca91ae 100644 --- a/tools/product_monte_carlo_native.cc +++ b/tools/product_monte_carlo_native.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -89,25 +90,29 @@ gf2p8::lch::Status Encode(std::span block) { template void CountDifferences(const std::vector& block, const gf2p8::Element* original, - uint64_t* out) { + uint64_t* out, + size_t n1, + size_t k1, + size_t n2, + size_t k2) { // Keep the contiguous reduction free of per-byte information-region branches. - for (size_t row = 0; row < 256; ++row) { + for (size_t row = 0; row < n1; ++row) { uint64_t bits = 0, bytes = 0; - for (size_t col = 0; col < 254; ++col) { + for (size_t col = 0; col < k2; ++col) { const unsigned d = - block[row * 256 + col] ^ (Random ? original[row * 256 + col] : 0); + block[row * n2 + col] ^ (Random ? original[row * n2 + col] : 0); bits += std::popcount(d); bytes += d != 0; } out[0] += bits; out[1] += bytes; - if (row < 224) { + if (row < k1) { out[2] += bits; out[3] += bytes; } - for (size_t col = 254; col < 256; ++col) { + for (size_t col = k2; col < n2; ++col) { const unsigned d = - block[row * 256 + col] ^ (Random ? original[row * 256 + col] : 0); + block[row * n2 + col] ^ (Random ? original[row * n2 + col] : 0); out[0] += std::popcount(d); out[1] += d != 0; } @@ -124,18 +129,36 @@ int Trial(uint64_t seed, uint64_t* output, int sampler, uint32_t* positions, - uint8_t* residual) { + uint8_t* residual, + size_t n1 = 256, + size_t k1 = 224, + size_t n2 = 256, + size_t k2 = 254) { try { // Local counters cannot alias the byte buffers and are published only once. // The validated cap bounds every metric by 1000000 * 524288 (< 2^39); // scans count at most 524288 bits, so these per-trial additions fit u64. uint64_t out[22]{}; - if (!output || k > 524288 || passes < 2 || passes > 1000000 || - sampler < 0 || sampler > 2 || (sampler == 2 && !positions)) { + if (!output || n1 > 256 || n2 > 256 || k > 8 * n1 * n2 || passes < 2 || + passes > 1000000 || sampler < 0 || sampler > 2 || + (sampler == 2 && !positions)) { return 1; } - thread_local gf2p8::rs::StrongWeakRSProductCode code; - thread_local std::vector block(65536); + const std::array dimensions{n1, k1, n2, k2}; + thread_local std::array previous{}; + thread_local std::unique_ptr code; + if (!code || previous != dimensions) { + auto next = + std::make_unique(n1, k1, n2, k2); + if (!next->Valid()) { + return 1; + } + code = std::move(next); + previous = dimensions; + } + const size_t bits = 8 * code->BlockSize(); + thread_local std::vector block; + block.resize(code->BlockSize()); thread_local std::vector selected; thread_local std::vector permutation; const auto key = Mix(seed ^ Mix(batch) ^ Mix(trial ^ 0x545249414cULL)); @@ -144,15 +167,19 @@ int Trial(uint64_t seed, if constexpr (Random) { // Legacy replay/reference only. Zero trials neither allocate this buffer // nor construct a message PRNG or encoder, even on first worker use. - thread_local std::vector reference(65536); - std::fill(reference.begin(), reference.end(), 0); + thread_local std::vector reference; + reference.assign(block.size(), 0); std::mt19937_64 message(Mix(key ^ 0x4d455353414745ULL)); - for (size_t r = 0; r < 224; ++r) { - for (size_t c = 0; c < 254; ++c) { - reference[r * 256 + c] = static_cast(message()); + for (size_t r = 0; r < k1; ++r) { + for (size_t c = 0; c < k2; ++c) { + reference[r * n2 + c] = static_cast(message()); } } - if (Encode(reference) != gf2p8::lch::Status::ok) { + const auto status = + dimensions == std::array{256, 224, 256, 254} + ? Encode(reference) + : code->Encode(reference); + if (status != gf2p8::lch::Status::ok) { return 2; } original = reference.data(); @@ -164,12 +191,12 @@ int Trial(uint64_t seed, } // Sample the smaller of the flip set and its complement. Floyd uses a // cleared bitmap; opt-in Fisher-Yates retains a 2 MiB worker permutation. - const bool complement = k > 262144; - const size_t count = complement ? 524288 - k : k; + const bool complement = k > bits / 2; + const size_t count = complement ? bits - k : k; if (sampler != 1) { - selected.assign(524288, 0); - } else if (permutation.empty()) { - permutation.resize(524288); + selected.assign(bits, 0); + } else if (permutation.size() != bits) { + permutation.resize(bits); std::iota(permutation.begin(), permutation.end(), 0u); } if (complement) { @@ -182,17 +209,17 @@ int Trial(uint64_t seed, if (sampler == 1) { // Any prior permutation gives a uniform subset. Do not reset it; // replay requires saved positions, not just schedule-dependent seeds. - const size_t draw = i + Uniform(noise, 524288 - i); + const size_t draw = i + Uniform(noise, bits - i); std::swap(permutation[i], permutation[draw]); pos = permutation[i]; } else if (sampler == 2) { pos = positions[i]; - if (pos >= 524288 || selected[pos]) { + if (pos >= bits || selected[pos]) { return 6; } selected[pos] = 1; } else { - const size_t j = 524288 - count + i; + const size_t j = bits - count + i; const size_t draw = Uniform(noise, j + 1); pos = selected[draw] ? j : draw; selected[pos] = 1; @@ -202,16 +229,16 @@ int Trial(uint64_t seed, } block[pos / 8] ^= static_cast(1u << (pos % 8)); } - CountDifferences(block, original, out); + CountDifferences(block, original, out, n1, k1, n2, k2); if (out[0] != k) { return 4; } - const auto result = code.Correct( + const auto result = code->Correct( block, {static_cast(passes), anchors != 0, binary != 0}); if (result.termination == gf2p8::rs::ProductTermination::invalid_argument) { return 3; } - CountDifferences(block, original, out + 4); + CountDifferences(block, original, out + 4, n1, k1, n2, k2); out[8] = out[6] != 0; out[9] = out[4] != 0; out[10] = result.all_zero_syndromes; @@ -244,6 +271,33 @@ int Trial(uint64_t seed, // successful trials; sampler output positions are scratch and must be ignored // on failure. extern "C" { +int product_trial_dimensions(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + int sampler, + uint32_t* positions, + int random, + uint8_t* residual, + uint64_t n1, + uint64_t k1, + uint64_t n2, + uint64_t k2) { + if ((random != 0 && random != 1) || n1 > 256 || k1 > 256 || n2 > 256 || + k2 > 256) { + return 1; + } + return random + ? Trial(seed, batch, trial, k, passes, anchors, binary, + output, sampler, positions, residual, n1, k1, n2, k2) + : Trial(seed, batch, trial, k, passes, anchors, binary, + output, sampler, positions, residual, n1, k1, n2, + k2); +} int product_interrupt_install() { interrupted = 0; struct sigaction action {}; diff --git a/tools/product_monte_carlo_trials.h b/tools/product_monte_carlo_trials.h index 1a933c7..d06949c 100644 --- a/tools/product_monte_carlo_trials.h +++ b/tools/product_monte_carlo_trials.h @@ -4,6 +4,22 @@ // Private trial ABI retained for legacy differential tests and RSFLIP01 replay. extern "C" { +int product_trial_dimensions(uint64_t seed, + uint64_t batch, + uint64_t trial, + uint64_t k, + uint64_t passes, + int anchors, + int binary, + uint64_t* output, + int sampler, + uint32_t* positions, + int random, + uint8_t* residual, + uint64_t n1, + uint64_t k1, + uint64_t n2, + uint64_t k2); int product_interrupt_install(); int product_interrupted(); uint64_t product_batch_k(uint64_t seed, From bcc3df674e4da6a808a0901be490cd2a91cf0bcd Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Thu, 10 Sep 2026 03:22:50 +0300 Subject: [PATCH 5/7] Accelerate weak RS correction and product batch processing --- .../strong_weak_rs_product_code_benchmarks.cc | 95 ++- docs/strong_weak_rs_acceleration_report.md | 763 ++++++++++++++++++ .../strong_weak_rs_product_code.h | 22 +- src/reed_solomon/product_code_internal.h | 30 + .../strong_weak_rs_product_code.cc | 228 +++++- tests/product_code_tests.cc | 137 +++- 6 files changed, 1223 insertions(+), 52 deletions(-) create mode 100644 docs/strong_weak_rs_acceleration_report.md diff --git a/benchmarks/strong_weak_rs_product_code_benchmarks.cc b/benchmarks/strong_weak_rs_product_code_benchmarks.cc index 4ce8fd7..df45ed8 100644 --- a/benchmarks/strong_weak_rs_product_code_benchmarks.cc +++ b/benchmarks/strong_weak_rs_product_code_benchmarks.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "benchmark/benchmark.h" @@ -11,19 +12,32 @@ namespace { void BenchmarkProductCorrectionBSC(benchmark::State& state, - int batch_passes = -1) { + int batch_passes = -1, + size_t weak_n = 256, + int optimizations = -1) { using gf2p8::Element; using gf2p8::rs::ProductCorrectionResult; using gf2p8::rs::ProductTermination; constexpr size_t kN = 256; constexpr size_t kStrongK = 224; - constexpr size_t kWeakK = 254; - constexpr size_t kInformationBytes = kStrongK * kWeakK; - constexpr size_t kBlockBytes = kN * kN; + const size_t kWeakK = weak_n - 2; + const size_t kInformationBytes = kStrongK * kWeakK; + const size_t kBlockBytes = kN * weak_n; constexpr size_t kCorpusCount = 64; constexpr size_t kPassLimit = 16; constexpr uint32_t kSeed = 0x5b5c0224; - gf2p8::rs::StrongWeakRSProductCode code(kN, kStrongK, kN, kWeakK); + gf2p8::rs::StrongWeakRSProductCode code(kN, kStrongK, weak_n, kWeakK); + const auto correct = [&](std::vector& block) { + if (optimizations >= 0) { + return gf2p8::rs::detail::ProductCorrectionAccess::Experiment( + code, block, gf2p8::rs::ProductDecodeOptions{kPassLimit}, + optimizations); + } + return batch_passes < 0 + ? code.Correct(block, kPassLimit) + : gf2p8::rs::detail::ProductCorrectionAccess::Correct( + code, block, kPassLimit, batch_passes); + }; if (!code.Valid() || code.BlockSize() != kBlockBytes) { state.SkipWithError("invalid product-code dimensions"); return; @@ -41,7 +55,7 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, auto& block = original[sample]; for (size_t row = 0; row < kStrongK; ++row) { for (size_t col = 0; col < kWeakK; ++col) { - block[row * kN + col] = static_cast(messages()); + block[row * weak_n + col] = static_cast(messages()); } } if (code.Encode(block) != gf2p8::lch::Status::ok) { @@ -88,10 +102,7 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, const auto expected = gf2p8::rs::detail::ProductCorrectionAccess::Correct( code, reference, kPassLimit, 0, false); work[sample] = corrupted[sample]; - const auto actual = - batch_passes < 0 ? code.Correct(work[sample], kPassLimit) - : gf2p8::rs::detail::ProductCorrectionAccess::Correct( - code, work[sample], kPassLimit, batch_passes); + const auto actual = correct(work[sample]); if (work[sample] != reference || actual.termination != expected.termination || actual.all_zero_syndromes != expected.all_zero_syndromes || @@ -116,11 +127,7 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, } state.ResumeTiming(); for (size_t sample = 0; sample < kCorpusCount; ++sample) { - results[sample] = - batch_passes < 0 - ? code.Correct(work[sample], kPassLimit) - : gf2p8::rs::detail::ProductCorrectionAccess::Correct( - code, work[sample], kPassLimit, batch_passes); + results[sample] = correct(work[sample]); benchmark::DoNotOptimize(results[sample]); benchmark::ClobberMemory(); } @@ -136,7 +143,7 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, const auto bits = std::popcount( static_cast(work[sample][pos] ^ original[sample][pos])); block_bits += bits; - if (pos / kN < kStrongK && pos % kN < kWeakK) { + if (pos / weak_n < kStrongK && pos % weak_n < kWeakK) { data_bits += bits; } } @@ -209,7 +216,7 @@ void BenchmarkProductCorrectionBSC(benchmark::State& state, state.SetBytesProcessed(state.iterations() * kCorpusCount * kInformationBytes); state.SetLabel( - "64 blocks/iteration; information bytes=224*254; " + "64 blocks/iteration; information bytes=Kstrong*Kweak; " "Correct includes exact final validity; tuned backend"); } @@ -218,9 +225,28 @@ const auto* kProductCorrectionBSC = benchmark::RegisterBenchmark( "Nstrong:256/Kstrong:224/Nweak:256/Kweak:254", [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state); }); +const auto kProductExperiments = [] { + for (int variant : {0, 1, 2, 3, 4, 6, 7}) { + for (size_t n : {256u, 175u}) { + const auto name = "LCH/Owned/StrongWeakRSProductCode/Experiment/" + + std::to_string(variant) + "/" + std::to_string(n); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + BenchmarkProductCorrectionBSC(state, -1, n, variant); + }); + } + } + return true; +}(); + const auto* kProductCorrectionSingle = benchmark::RegisterBenchmark( "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/Single", [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state, 0); }); +const auto* kProductCorrectionShortened = benchmark::RegisterBenchmark( + "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/" + "Nstrong:256/Kstrong:224/Nweak:175/Kweak:173", + [](benchmark::State& state) { + BenchmarkProductCorrectionBSC(state, -1, 175); + }); const auto* kProductCorrectionStrongBatch = benchmark::RegisterBenchmark( "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/StrongBatch", [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state, 1); }); @@ -228,4 +254,39 @@ const auto* kProductCorrectionBothBatch = benchmark::RegisterBenchmark( "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/BothBatch", [](benchmark::State& state) { BenchmarkProductCorrectionBSC(state, 2); }); +const auto* kProductCorrectionGenericShortened = benchmark::RegisterBenchmark( + "LCH/Owned/StrongWeakRSProductCode/Correct/BSC005/Generic175", + [](benchmark::State& state) { + BenchmarkProductCorrectionBSC(state, 2, 175); + }); + +void BenchmarkProductEncode(benchmark::State& state) { + const size_t n = state.range(0); + gf2p8::rs::StrongWeakRSProductCode code(256, 224, n, n - 2); + std::vector block(code.BlockSize()); + std::mt19937 random(0x5b5c0224); + for (auto& value : block) { + value = static_cast(random()); + } + if (code.Encode(block) != gf2p8::lch::Status::ok) { + state.SkipWithError("encoding failed"); + return; + } + const auto original = block; + for (auto _ : state) { + benchmark::DoNotOptimize(code.Encode(block)); + benchmark::ClobberMemory(); + } + if (block != original || !code.Correct(block).all_zero_syndromes) { + state.SkipWithError("encoding mismatch"); + } + state.SetBytesProcessed(state.iterations() * 224 * (n - 2)); +} + +const auto* kProductEncode = + benchmark::RegisterBenchmark("LCH/Owned/StrongWeakRSProductCode/Encode", + BenchmarkProductEncode) + ->Arg(256) + ->Arg(175); + } // namespace diff --git a/docs/strong_weak_rs_acceleration_report.md b/docs/strong_weak_rs_acceleration_report.md new file mode 100644 index 0000000..65d2d8f --- /dev/null +++ b/docs/strong_weak_rs_acceleration_report.md @@ -0,0 +1,763 @@ +# Strong-Weak Reed-Solomon Product Code Acceleration Report + +## Post-Profile Experiments (2026-09-10) + +All three requested experiments were implemented, differentially tested and +measured. Retained: sparse strong-mask traversal, coordinate-bit weak syndrome +reduction, and padding independent strong batch lanes. Public APIs, gates, +transactionality, stopping rules, counters, mother codes and R=2 policy are +unchanged. The earlier uncommitted direct-R2 implementation and the report below +are preserved. No commits were made. + +### Retained Techniques and Limits + +1. **Strong masks:** AVX2 compares 32 mask bytes against zero and extracts a + bitset. Set-bit traversal visits only verified repairs; old/new XOR popcount, + byte write counts, next-direction activation and clean-state invalidation + still occur for every committed byte. Real partial tails use the exact scalar + scan. Runtime availability is checked once per product correction, not once + per position row. Staging remains transactional per lane; failed lanes are + not committed. This is a modest, workload-dependent improvement, not removal + of the entire old 19-23% profiled region. +2. **Weak reduction:** for each complete 32-byte group, XOR symbols into eight + accumulators selected by the coordinate bits of weight `(j+2) XOR 1`. + Linearity gives `S0 = XOR_b MultiplyCantor(1<