diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..df32be2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +build +results +docs/figures +.git +*.pyc +__pycache__ diff --git a/.gitmessage.txt b/.gitmessage.txt index 1248072..adaf523 100644 --- a/.gitmessage.txt +++ b/.gitmessage.txt @@ -17,7 +17,6 @@ # - shipovnik # - hypericum # - kryzhovnik -# - iaik_merkle_tree # - Ensures reproducible builds with pinned submodule commits # # chore(repo): add .gitignore and basic CI workflow diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..434fa39 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + gcc \ + git \ + libcppunit-dev \ + libntl-dev \ + libssl-dev \ + make \ + pkg-config \ + python3 \ + python3-pip \ + python3-matplotlib \ + \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/batch-pqc + +# Copy repository into the image +COPY . /opt/batch-pqc + +# Build the project inside the image. +# Note: submodules are not initialized in the image build context. Ensure +# submodules are checked out on the host before building the image (or add +# them to the build context). Attempting to run `git submodule` here will +# usually fail because `.git` is excluded from the Docker build context. +RUN mkdir -p build \ + && cmake -S . -B build \ + && cmake --build build --parallel + +CMD ["/bin/bash"] diff --git a/README.md b/README.md index a65a40e..3fb5317 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The goal is to measure the speedup of signing a batch of messages with a single - [Shipovnik](https://github.com/QAPP-tech/shipovnik_tc26) - [Hypericum](https://github.com/QAPP-tech/hypericum_tc26) - [Kryzhovnik](https://github.com/ElenaKirshanova/pqc_LWR_signature) -- Sequential benchmark (`bench_seq`) with configurable paramsets, verification pass, and warmup iterations +- Sequential and batch benchmarks (`bench_seq`, `bench_batch`) with configurable paramsets, verification pass, and warmup iterations - Test suite with unified `test_*` naming in CTest ## Original Repos and Forks @@ -102,8 +102,8 @@ git submodule update --init --recursive # Run tests ./scripts/third_party.sh tests -# Run benchmark (sequential signing) -./build/bench/bench_seq --algo hypericum --batch-size 16 --iters 100 --verify 1 +# Run benchmark comparison (sequential + batch + plots) +./scripts/benchmark.sh --algo hypericum --batch-sizes 1,4,16 --iters 100 --verify 1 ``` ### Paramset Selection @@ -140,8 +140,11 @@ Note: Kryzhovnik constants are aligned with `security.sage` for `small/medium/la After running benchmarks, the `results/` directory will contain: -- `raw_data.csv` – timings for each (algorithm, batch_size) -- `plots/speedup_vs_batch.png` – graph for all three algorithms +- `bench-/*_seq.csv` and `bench-/*_batch.csv` – raw benchmark slices +- `bench-/raw_data.csv` – aggregated timings for each `(algorithm, batch_size)` +- `bench-/plots/speedup_vs_batch.png` – graph for all three algorithms + +- Note: automated PDF generation has been removed from this repository; the final report is maintained as Markdown at `docs/report.md`. To produce a PDF locally, install `pandoc` and a LaTeX distribution and convert the Markdown manually. ## Current Status (MVP) @@ -151,8 +154,8 @@ After running benchmarks, the `results/` directory will contain: - [x] Sequential benchmark `bench_seq` with warmup and corrected signature-size metric - [x] Merkle-based batch signer/verifier implementation - [x] Test coverage: `test_adapters_smoke`, `test_adapters_batch`, `test_kryzhovnik*`, `test_merkle`, `test_batch_signing`, `test_streebog` -- [ ] Sequential vs real batch signing benchmark comparison -- [ ] Final report (PDF) +- [x] Sequential vs real batch signing benchmark comparison +- [x] Final report (`docs/report.md`) **MVP is being developed in `dev` branch. After completion, a Pull Request to `main` will be opened for review.** diff --git a/README_ru.md b/README_ru.md index 93ea471..82413e4 100644 --- a/README_ru.md +++ b/README_ru.md @@ -33,7 +33,7 @@ - [Шиповник](https://github.com/QAPP-tech/shipovnik_tc26) - [Гиперикум](https://github.com/QAPP-tech/hypericum_tc26) - [Крыжовник](https://github.com/ElenaKirshanova/pqc_LWR_signature) -- Последовательный бенчмарк (`bench_seq`) с параметрами paramset, verify-проходом и прогревочными итерациями +- Последовательный и пакетный бенчмарки (`bench_seq`, `bench_batch`) с параметрами paramset, verify-проходом и прогревочными итерациями - Набор тестов с унифицированными именами `test_*` в CTest ## Оригинальные репозитории и форки @@ -107,8 +107,8 @@ git submodule update --init --recursive # Запуск тестов ./scripts/third_party.sh tests -# Запуск бенчмарка (последовательная подпись) -./build/bench/bench_seq --algo hypericum --batch-size 16 --iters 100 --verify 1 +# Запуск бенчмарков (последовательная и пакетная подпись + графики) +./scripts/benchmark.sh --algo hypericum --batch-sizes 1,4,16 --iters 100 --verify 1 ``` ### Выбор paramset @@ -145,8 +145,9 @@ cmake --build build --parallel После запуска бенчмарков в директории `results/` появятся: -- `raw_data.csv` – замеры времени для каждого (алгоритм, размер пакета) -- `plots/speedup_vs_batch.png` – график ускорения для всех трёх алгоритмов +- `bench-/*_seq.csv` и `bench-/*_batch.csv` – исходные замеры +- `bench-/raw_data.csv` – агрегированные замеры времени для каждого `(алгоритм, размер пакета)` +- `bench-/plots/speedup_vs_batch.png` – график ускорения для всех трёх алгоритмов ## Текущий статус (MVP) @@ -156,8 +157,8 @@ cmake --build build --parallel - [x] Последовательный бенчмарк `bench_seq` с warmup и исправленной метрикой размера подписей - [x] Реализация пакетной подписи/верификации на основе дерева Меркла - [x] Покрытие тестами: `test_adapters_smoke`, `test_adapters_batch`, `test_kryzhovnik*`, `test_merkle`, `test_batch_signing`, `test_streebog` -- [ ] Сравнение бенчмарков: последовательная vs реальная пакетная подпись -- [ ] Финальный отчёт (PDF) +- [x] Сравнение бенчмарков: последовательная vs реальная пакетная подпись +- [x] Финальный отчёт (`docs/report.md`) **MVP разрабатывается в ветке `dev`. После завершения будет открыт Pull Request в `main` для ревью.** diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 51d7000..02b56e9 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -3,3 +3,9 @@ add_executable(bench_seq bench_seq.c) target_link_libraries(bench_seq PRIVATE batch_bench_adapters) target_compile_features(bench_seq PRIVATE c_std_99) + +add_executable(bench_batch bench_batch.c) + +target_link_libraries(bench_batch PRIVATE batch_adapters batch_bench_adapters streebog) + +target_compile_features(bench_batch PRIVATE c_std_99) diff --git a/bench/bench_batch.c b/bench/bench_batch.c new file mode 100644 index 0000000..9ace35b --- /dev/null +++ b/bench/bench_batch.c @@ -0,0 +1,568 @@ +#include "batch_adapters.h" +#include "batch_bench.h" +#include "batch_signing.h" +#include "utils/message_gen.h" +#include "utils/timer.h" +#include "gost3411-2012-core.h" + +#include +#include +#include +#include +#include +#include + +#define BATCH_HASH_BITS 256u +#define BATCH_HASH_SIZE (BATCH_HASH_BITS / 8u) + +typedef struct bench_config { + const char *algo; + const char *params; + size_t batch_size; + size_t iters; + size_t warmup; + size_t msg_size; + uint64_t seed; + int verify; + const char *out_csv; + const char *out_json; +} bench_config; + +typedef struct bench_result { + const char *algorithm; + const char *params; + size_t batch_size; + double keygen_ms; + double time_batch_ms; + double verify_ms; + double serialize_time_ms; + double peak_memory_mb; + double total_sig_size_mb; +} bench_result; + +static void print_usage(const char *prog) { + printf("Usage: %s [options]\n", prog); + printf(" --algo Algorithm name or all (default: all)\n"); + printf(" --params Paramset tag for result output (default: default)\n"); + printf(" --batch-size Messages per iteration (default: 1)\n"); + printf(" --iters Iterations (default: 100)\n"); + printf(" --warmup Warmup iterations (default: 0)\n"); + printf(" --msg-size Message size in bytes (default: 1024)\n"); + printf(" --seed Deterministic message seed (default: 1)\n"); + printf(" --verify <0|1> Run verify pass (default: 1)\n"); + printf(" --out-csv Output CSV path\n"); + printf(" --out-json Output JSON path\n"); + printf(" --help Show this message\n"); +} + +static int parse_size(const char *s, size_t *out) { + char *end = NULL; + unsigned long long value; + + if (s == NULL || out == NULL) { + return -1; + } + + errno = 0; + value = strtoull(s, &end, 10); + if (errno != 0 || end == s || *end != '\0') { + return -1; + } + + *out = (size_t)value; + return 0; +} + +static int parse_u64(const char *s, uint64_t *out) { + char *end = NULL; + unsigned long long value; + + if (s == NULL || out == NULL) { + return -1; + } + + errno = 0; + value = strtoull(s, &end, 10); + if (errno != 0 || end == s || *end != '\0') { + return -1; + } + + *out = (uint64_t)value; + return 0; +} + +static int parse_int(const char *s, int *out) { + char *end = NULL; + long value; + + if (s == NULL || out == NULL) { + return -1; + } + + errno = 0; + value = strtol(s, &end, 10); + if (errno != 0 || end == s || *end != '\0') { + return -1; + } + + *out = (int)value; + return 0; +} + +static int parse_args(int argc, char **argv, bench_config *cfg) { + int index; + + cfg->algo = "all"; + cfg->params = "default"; + cfg->batch_size = 1; + cfg->iters = 100; + cfg->warmup = 0; + cfg->msg_size = BB_DEFAULT_MESSAGE_SIZE; + cfg->seed = 1; + cfg->verify = 1; + cfg->out_csv = NULL; + cfg->out_json = NULL; + + for (index = 1; index < argc; ++index) { + const char *arg = argv[index]; + + if (strcmp(arg, "--help") == 0) { + print_usage(argv[0]); + return 1; + } + if (strcmp(arg, "--algo") == 0 && index + 1 < argc) { + cfg->algo = argv[++index]; + continue; + } + if (strcmp(arg, "--params") == 0 && index + 1 < argc) { + cfg->params = argv[++index]; + continue; + } + if (strcmp(arg, "--batch-size") == 0 && index + 1 < argc) { + if (parse_size(argv[++index], &cfg->batch_size) != 0 || cfg->batch_size == 0) { + return -1; + } + continue; + } + if (strcmp(arg, "--iters") == 0 && index + 1 < argc) { + if (parse_size(argv[++index], &cfg->iters) != 0 || cfg->iters == 0) { + return -1; + } + continue; + } + if (strcmp(arg, "--warmup") == 0 && index + 1 < argc) { + if (parse_size(argv[++index], &cfg->warmup) != 0) { + return -1; + } + continue; + } + if (strcmp(arg, "--msg-size") == 0 && index + 1 < argc) { + if (parse_size(argv[++index], &cfg->msg_size) != 0 || cfg->msg_size == 0) { + return -1; + } + continue; + } + if (strcmp(arg, "--seed") == 0 && index + 1 < argc) { + if (parse_u64(argv[++index], &cfg->seed) != 0) { + return -1; + } + continue; + } + if (strcmp(arg, "--verify") == 0 && index + 1 < argc) { + if (parse_int(argv[++index], &cfg->verify) != 0 || + (cfg->verify != 0 && cfg->verify != 1)) { + return -1; + } + continue; + } + if (strcmp(arg, "--out-csv") == 0 && index + 1 < argc) { + cfg->out_csv = argv[++index]; + continue; + } + if (strcmp(arg, "--out-json") == 0 && index + 1 < argc) { + cfg->out_json = argv[++index]; + continue; + } + return -1; + } + + return 0; +} + +static double get_peak_memory_mb(void) { + struct rusage usage; + + if (getrusage(RUSAGE_SELF, &usage) != 0) { + return 0.0; + } + + return (double)usage.ru_maxrss / 1024.0; +} + +static void streebog256_hash(const uint8_t *in, size_t in_len, uint8_t *out) { + GOST34112012Context ctx; + + GOST34112012Init(&ctx, 256u); + GOST34112012Update(&ctx, in, in_len); + GOST34112012Final(&ctx, out); + GOST34112012Cleanup(&ctx); +} + +static int run_one(const bb_algorithm *algo, const bench_config *cfg, bench_result *out) { + batch_bb_adapter_ctx adapter_ctx; + batch_signer_t *signer = NULL; + batch_signature_t *signature = NULL; + uint8_t *sk = NULL; + uint8_t *pk = NULL; + uint8_t *messages_buf = NULL; + const uint8_t **messages = NULL; + size_t *msg_lens = NULL; + uint8_t *serialized = NULL; + size_t sk_len = 0; + size_t pk_len = 0; + size_t total_sig_bytes = 0; + size_t total_iters; + size_t iteration; + bb_timer timer; + + sk = (uint8_t *)malloc(algo->secret_key_bytes); + pk = (uint8_t *)malloc(algo->public_key_bytes); + + /* check for overflow when computing messages buffer size */ + if (cfg->batch_size > 0 && cfg->msg_size > SIZE_MAX / cfg->batch_size) { + goto fail; + } + messages_buf = (uint8_t *)malloc(cfg->batch_size * cfg->msg_size); + + /* check for overflow when allocating pointer/length arrays */ + if (cfg->batch_size > 0 && sizeof(*messages) > SIZE_MAX / cfg->batch_size) { + goto fail; + } + messages = (const uint8_t **)calloc(cfg->batch_size, sizeof(*messages)); + if (cfg->batch_size > 0 && sizeof(*msg_lens) > SIZE_MAX / cfg->batch_size) { + goto fail; + } + msg_lens = (size_t *)calloc(cfg->batch_size, sizeof(*msg_lens)); + if (sk == NULL || pk == NULL || messages_buf == NULL || messages == NULL || msg_lens == NULL) { + goto fail; + } + + bb_timer_start(&timer); + if (algo->keygen(sk, algo->secret_key_bytes, &sk_len, + pk, algo->public_key_bytes, &pk_len) != BB_OK) { + goto fail; + } + bb_timer_stop(&timer); + out->keygen_ms = bb_timer_elapsed_ms(&timer); + + if (batch_bb_adapter_init(&adapter_ctx, algo, sk, sk_len, pk, pk_len) != 0) { + goto fail; + } + + signer = batch_signer_create(BATCH_HASH_SIZE, + batch_bb_signature_capacity(&adapter_ctx), + streebog256_hash, + batch_bb_sign_callback, + &adapter_ctx); + if (signer == NULL) { + goto fail; + } + + out->time_batch_ms = 0.0; + out->verify_ms = 0.0; + out->serialize_time_ms = 0.0; + + total_iters = cfg->warmup + cfg->iters; + for (iteration = 0; iteration < total_iters; ++iteration) { + const int measured = iteration >= cfg->warmup; + size_t message_index; + size_t serialized_len = 0; + + batch_signer_reset(signer); + if (signature != NULL) { + batch_signature_free(signature); + signature = NULL; + } + + for (message_index = 0; message_index < cfg->batch_size; ++message_index) { + uint8_t *message = messages_buf + (message_index * cfg->msg_size); + + bb_fill_message(message, + cfg->msg_size, + cfg->seed, + iteration * cfg->batch_size + message_index); + messages[message_index] = message; + msg_lens[message_index] = cfg->msg_size; + + if (batch_signer_add_message(signer, message, cfg->msg_size) != 0) { + goto fail; + } + } + + if (measured) { + bb_timer_start(&timer); + signature = batch_signer_sign(signer); + bb_timer_stop(&timer); + out->time_batch_ms += bb_timer_elapsed_ms(&timer); + } else { + signature = batch_signer_sign(signer); + } + + if (signature == NULL) { + goto fail; + } + + if (batch_signature_serialize(signature, NULL, &serialized_len) != 0) { + goto fail; + } + + if (measured) { + uint8_t *grown = (uint8_t *)realloc(serialized, serialized_len == 0 ? 1 : serialized_len); + + if (grown == NULL) { + goto fail; + } + serialized = grown; + + bb_timer_start(&timer); + if (batch_signature_serialize(signature, serialized, &serialized_len) != 0) { + bb_timer_stop(&timer); + goto fail; + } + bb_timer_stop(&timer); + out->serialize_time_ms += bb_timer_elapsed_ms(&timer); + total_sig_bytes += serialized_len; + } + + if (cfg->verify) { + int ok; + + if (measured) { + bb_timer_start(&timer); + ok = batch_verify(messages, + msg_lens, + cfg->batch_size, + signature, + BATCH_HASH_SIZE, + streebog256_hash, + batch_bb_verify_callback, + &adapter_ctx); + bb_timer_stop(&timer); + out->verify_ms += bb_timer_elapsed_ms(&timer); + } else { + ok = batch_verify(messages, + msg_lens, + cfg->batch_size, + signature, + BATCH_HASH_SIZE, + streebog256_hash, + batch_bb_verify_callback, + &adapter_ctx); + } + + if (!ok) { + goto fail; + } + } + } + + out->algorithm = algo->name; + out->params = cfg->params; + out->batch_size = cfg->batch_size; + out->peak_memory_mb = get_peak_memory_mb(); + out->total_sig_size_mb = ((double)total_sig_bytes / (double)cfg->iters) / (1024.0 * 1024.0); + + batch_signature_free(signature); + batch_signer_free(signer); + free(sk); + free(pk); + free(messages_buf); + free(messages); + free(msg_lens); + free(serialized); + return 0; + +fail: + batch_signature_free(signature); + batch_signer_free(signer); + free(sk); + free(pk); + free(messages_buf); + free(messages); + free(msg_lens); + free(serialized); + return -1; +} + +static void print_result(const bench_result *result) { + printf("algo=%s params=%s batch=%zu keygen_ms=%.3f batch_ms=%.3f verify_ms=%.3f serialize_ms=%.3f peak_mem_mb=%.2f total_sig_mb=%.3f\n", + result->algorithm, + result->params, + result->batch_size, + result->keygen_ms, + result->time_batch_ms, + result->verify_ms, + result->serialize_time_ms, + result->peak_memory_mb, + result->total_sig_size_mb); +} + +static int write_csv(FILE *file, const bench_result *result) { + return fprintf(file, + "%s,%s,%zu,%.6f,%.6f,%.6f,%.6f,%.6f,%.6f\n", + result->algorithm, + result->params, + result->batch_size, + 0.0, + result->time_batch_ms, + 0.0, + result->peak_memory_mb, + result->total_sig_size_mb, + result->serialize_time_ms) < 0 + ? -1 + : 0; +} + +static int write_json(FILE *file, + const bench_result *results, + size_t count, + const bench_config *cfg) { + size_t index; + + if (fprintf(file, + "{\n" + " \"config\": {\n" + " \"algo\": \"%s\",\n" + " \"params\": \"%s\",\n" + " \"batch_size\": %zu,\n" + " \"iters\": %zu,\n" + " \"warmup\": %zu,\n" + " \"msg_size\": %zu,\n" + " \"seed\": %llu,\n" + " \"verify\": %d\n" + " },\n" + " \"results\": [\n", + cfg->algo, + cfg->params, + cfg->batch_size, + cfg->iters, + cfg->warmup, + cfg->msg_size, + (unsigned long long)cfg->seed, + cfg->verify) < 0) { + return -1; + } + + for (index = 0; index < count; ++index) { + const bench_result *result = &results[index]; + + if (fprintf(file, + " {\"algorithm\":\"%s\",\"params\":\"%s\",\"batch_size\":%zu," + "\"time_seq_ms\":%.6f,\"time_batch_ms\":%.6f," + "\"speedup\":%.6f,\"peak_memory_MB\":%.6f," + "\"total_sig_size_MB\":%.6f,\"serialize_time_ms\":%.6f}%s\n", + result->algorithm, + result->params, + result->batch_size, + 0.0, + result->time_batch_ms, + 0.0, + result->peak_memory_mb, + result->total_sig_size_mb, + result->serialize_time_ms, + index + 1 == count ? "" : ",") < 0) { + return -1; + } + } + + return fprintf(file, " ]\n}\n") < 0 ? -1 : 0; +} + +int main(int argc, char **argv) { + bench_config cfg; + bench_result *results = NULL; + size_t results_count = 0; + FILE *csv = NULL; + FILE *json = NULL; + size_t index; + int parse_result; + + parse_result = parse_args(argc, argv, &cfg); + if (parse_result == 1) { + return 0; + } + if (parse_result != 0) { + print_usage(argv[0]); + return 1; + } + + results = (bench_result *)calloc(bb_algorithm_count(), sizeof(bench_result)); + if (results == NULL) { + return 1; + } + + if (cfg.out_csv != NULL) { + csv = fopen(cfg.out_csv, "w"); + if (csv == NULL) { + free(results); + return 1; + } + fprintf(csv, + "algorithm,params,batch_size,time_seq_ms,time_batch_ms,speedup," + "peak_memory_MB,total_sig_size_MB,serialize_time_ms\n"); + } + + for (index = 0; index < bb_algorithm_count(); ++index) { + const bb_algorithm *algo = bb_algorithm_at(index); + + if (strcmp(cfg.algo, "all") != 0 && strcmp(cfg.algo, algo->name) != 0) { + continue; + } + + if (run_one(algo, &cfg, &results[results_count]) != 0) { + if (csv != NULL) { + fclose(csv); + } + free(results); + return 1; + } + + print_result(&results[results_count]); + if (csv != NULL && write_csv(csv, &results[results_count]) != 0) { + fclose(csv); + free(results); + return 1; + } + + ++results_count; + } + + if (results_count == 0) { + if (csv != NULL) { + fclose(csv); + } + free(results); + return 1; + } + + if (csv != NULL) { + fclose(csv); + } + + if (cfg.out_json != NULL) { + json = fopen(cfg.out_json, "w"); + if (json == NULL) { + free(results); + return 1; + } + if (write_json(json, results, results_count, &cfg) != 0) { + fclose(json); + free(results); + return 1; + } + fclose(json); + } + + free(results); + return 0; +} \ No newline at end of file diff --git a/docs/report.md b/docs/report.md new file mode 100644 index 0000000..7d71137 --- /dev/null +++ b/docs/report.md @@ -0,0 +1,131 @@ +# Final Report: Batch PQC Benchmarking + +> Note: This report is maintained as Markdown (`docs/report.md`). The repository no longer provides an automated PDF generation path. If you need a PDF version, convert the Markdown locally using `pandoc` and a LaTeX distribution. + +## 1. Goal + +This project evaluates whether Merkle-tree-based batch signing can reduce total signing cost for Russian post-quantum signature schemes when signing many independent messages. The comparison baseline is straightforward sequential signing of the same batch. + +The implementation targets three schemes already integrated in the repository: + +- Shipovnik +- Hypericum +- Kryzhovnik + +## 2. Architecture + +The benchmark stack is split into four layers: + +1. `src/signature/*` exposes a unified `bb_algorithm` interface for key generation, signing, and verification. +2. `src/batch_adapters.*` bridges the benchmark-facing `bb_algorithm` interface to the callback-oriented batch signing core. +3. `src/batch_signing.*` implements Merkle-tree-based batch signing and verification. +4. `bench/bench_seq.c` and `bench/bench_batch.c` measure the sequential and real batch workflows with a compatible CSV/JSON format. + +In the batch path, each message becomes a Merkle leaf after hashing with Streebog-256. The root hash is signed once, and each message receives its authentication proof. Verification reconstructs the root from the message hash and proof, then checks the single root signature. + +## 3. Measurement Methodology + +Both benchmark executables support the same core parameters: + +- `--algo` +- `--batch-size` +- `--iters` +- `--warmup` +- `--msg-size` +- `--seed` +- `--verify` + +Measured metrics: + +- `keygen_ms`: one-time key generation cost per benchmark run +- `time_seq_ms`: total time for sequential signing across measured iterations +- `time_batch_ms`: total time for real batch signing across measured iterations +- `verify_ms`: total verification time when `--verify 1` +- `serialize_time_ms`: time spent serializing produced signatures +- `peak_memory_MB`: process peak RSS during the run +- `total_sig_size_MB`: average serialized signature payload per measured iteration + +Sequential runs and batch runs are emitted separately as `*_seq.csv` and `*_batch.csv`. The plotting step joins both datasets by `(algorithm, params, batch_size)` and computes: + +$$ +speedup = \frac{time\_seq\_ms}{time\_batch\_ms} +$$ + +## 4. Running the Full Comparison + +Example workflow: + +```bash +./scripts/benchmark.sh --algo all --batch-sizes 1,2,4,8,16,32,64 --iters 100 --verify 1 +``` + +Generated artifacts: + +- `results/bench-/*_seq.csv` +- `results/bench-/*_batch.csv` +- `results/bench-/raw_data.csv` +- `results/bench-/plots/speedup_vs_batch.png` +- `results/bench-/plots/_times.png` + +The helper script invokes `scripts/plot_results.py` automatically when `python3` is available. Plot generation additionally requires `matplotlib`. + +## 5. Interpreting the Results + +The main chart is `speedup_vs_batch.png`. + +- `speedup > 1` means the Merkle-based batch path outperforms sequential signing for that batch size. +- `speedup = 1` means both strategies are equivalent. +- `speedup < 1` means the Merkle tree overhead is still larger than the saved signing work. + +The per-algorithm timing plots are useful to separate two effects: + +- whether root-signature amortization dominates total cost +- whether proof construction and serialization become the bottleneck for larger batches + +### Measured Results (run: `results/bench-20260423-004124`) + +Observed speedup ranges on this machine: + +![Speedup](figures/speedup_vs_batch.png) + +**Per-algorithm timings:** + + +| Shipovnik | Hypericum | Kryzhovnik | +| :-: | :-: | :-: | +| ![Shipovnik](figures/shipovnik_times.png) | ![Hypericum](figures/hypericum_times.png) | ![Kryzhovnik](figures/kryzhovnik_times.png) | + + +- Shipovnik: from `1.01x` at `batch_size=1` up to `62.34x` at `batch_size=64`. +- Hypericum (`debug`): from `0.99x` at `batch_size=1` up to `63.23x` at `batch_size=64`. +- Kryzhovnik (`debug`): from `0.15x` to `0.44x` (batch mode slower than sequential for all tested sizes). + +Interpretation: + +- For Shipovnik and Hypericum, batch signing shows near-linear amortization as batch size grows. +- For Kryzhovnik in the current implementation and parameter set, Merkle overhead dominates and no crossover point was observed up to `batch_size=64`. +- Signature payload behavior matches the design expectation: batch mode keeps approximately one root signature plus proofs, while sequential mode grows linearly with the number of signed messages. + +## 6. Practical Notes + +- Hypericum and Kryzhovnik paramsets are build-time selectable through CMake. +- The benchmark uses deterministic message generation for reproducibility. +- `debug` paramsets remain suitable only for diagnostics and quick iteration, not for security claims. +- This repository is still a PoC and is not intended for production deployment. + +## 7. Conclusion + +The repository now contains the full comparison pipeline needed for the MVP: + +- sequential benchmark +- real batch benchmark +- common automation script +- aggregated CSV output +- plot generation +- reproducible reportable workflow + +For this benchmark run, crossover behavior is already clear: + +- Shipovnik: batch is beneficial from `batch_size >= 1` and scales strongly. +- Hypericum (`debug`): batch becomes beneficial from `batch_size >= 2`. +- Kryzhovnik (`debug`): batch is not beneficial in the tested range and requires further optimization or different parameter choices. \ No newline at end of file diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 9487162..990b1b8 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -14,6 +14,20 @@ SEED=1 VERIFY=1 DRY_RUN=0 SKIP_EXISTING=1 +MODE="all" + +find_plot_python() { + local candidate + for candidate in python3 /usr/bin/python3; do + if command -v "$candidate" >/dev/null 2>&1; then + if "$candidate" -c 'import matplotlib' >/dev/null 2>&1; then + printf '%s\n' "$candidate" + return 0 + fi + fi + done + return 1 +} usage() { cat < (default: 1024) --seed (default: 1) --verify <0|1> (default: 1) + --mode (default: all) --build-dir (default: ./build) --output-dir (default: ./results/bench-) --dry-run @@ -79,6 +94,10 @@ while [[ $# -gt 0 ]]; do VERIFY="$2" shift 2 ;; + --mode) + MODE="$2" + shift 2 + ;; --build-dir) BUILD_DIR="$2" shift 2 @@ -115,6 +134,13 @@ echo "[benchmark] output: $OUTPUT_DIR" echo "[benchmark] hypericum: $HYPERICUM_PARAMS" echo "[benchmark] kryzhovnik:$KRYZHOVNIK_PARAMS" echo "[benchmark] skip: $SKIP_EXISTING" +echo "[benchmark] mode: $MODE" + +if [[ "$MODE" != "seq" && "$MODE" != "batch" && "$MODE" != "all" ]]; then + echo "Unsupported mode: $MODE" + usage + exit 1 +fi auto_configure_build() { cmake -S "$ROOT_DIR" -B "$BUILD_DIR" \ @@ -126,12 +152,18 @@ auto_configure_build() { run_one() { local algo="$1" local batch_size="$2" - local csv_out="$OUTPUT_DIR/${algo}_b${batch_size}.csv" - local json_out="$OUTPUT_DIR/${algo}_b${batch_size}.json" + local kind="$3" + local bench_bin="bench_seq" + local csv_out="$OUTPUT_DIR/${algo}_b${batch_size}_${kind}.csv" + local json_out="$OUTPUT_DIR/${algo}_b${batch_size}_${kind}.json" local csv_tmp="$csv_out.tmp" local json_tmp="$json_out.tmp" local params_label="default" + if [[ "$kind" == "batch" ]]; then + bench_bin="bench_batch" + fi + case "$algo" in hypericum) params_label="$HYPERICUM_PARAMS" @@ -143,7 +175,7 @@ run_one() { if [[ "$SKIP_EXISTING" -eq 1 ]]; then if [[ -s "$csv_out" && -s "$json_out" ]]; then - echo "[skip] algo=$algo batch_size=$batch_size (artifacts exist)" + echo "[skip] kind=$kind algo=$algo batch_size=$batch_size (artifacts exist)" return fi fi @@ -157,7 +189,7 @@ run_one() { fi local cmd=( - "$BUILD_DIR/bench/bench_seq" + "$BUILD_DIR/bench/$bench_bin" --algo "$algo" --batch-size "$batch_size" --iters "$ITERS" @@ -172,13 +204,26 @@ run_one() { if [[ "$DRY_RUN" -eq 1 ]]; then echo "[dry-run] ${cmd[*]}" else - echo "[run] algo=$algo batch_size=$batch_size" + echo "[run] kind=$kind algo=$algo batch_size=$batch_size" "${cmd[@]}" mv "$csv_tmp" "$csv_out" mv "$json_tmp" "$json_out" fi } +run_modes() { + local algo="$1" + local batch_size="$2" + + if [[ "$MODE" == "seq" || "$MODE" == "all" ]]; then + run_one "$algo" "$batch_size" "seq" + fi + + if [[ "$MODE" == "batch" || "$MODE" == "all" ]]; then + run_one "$algo" "$batch_size" "batch" + fi +} + if [[ "$DRY_RUN" -eq 1 ]]; then echo "[dry-run] cmake -S $ROOT_DIR -B $BUILD_DIR -DHYPERICUM_PARAMSET=$HYPERICUM_PARAMS -DKRYZHOVNIK_PARAMSET=$KRYZHOVNIK_PARAMS" echo "[dry-run] cmake --build $BUILD_DIR --parallel" @@ -191,13 +236,24 @@ IFS=',' read -r -a batches <<< "$BATCH_SIZES" if [[ "$ALGOS" == "all" ]]; then for algo in shipovnik hypericum kryzhovnik; do for b in "${batches[@]}"; do - run_one "$algo" "$b" + run_modes "$algo" "$b" done done else for b in "${batches[@]}"; do - run_one "$ALGOS" "$b" + run_modes "$ALGOS" "$b" done fi +if PLOT_PYTHON="$(find_plot_python)"; then + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "[dry-run] $PLOT_PYTHON $ROOT_DIR/scripts/plot_results.py --results-dir $OUTPUT_DIR" + else + "$PLOT_PYTHON" "$ROOT_DIR/scripts/plot_results.py" --results-dir "$OUTPUT_DIR" || \ + echo "[benchmark] warning: plot generation failed" + fi +else + echo "[benchmark] warning: no python interpreter with matplotlib found, skipping plot generation" +fi + echo "[benchmark] done" diff --git a/scripts/docker_run.sh b/scripts/docker_run.sh new file mode 100755 index 0000000..2c2dfe0 --- /dev/null +++ b/scripts/docker_run.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE_NAME="batch-pqc-bench:latest" +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +RESULTS_DIR="$ROOT_DIR/results" +DOCS_DIR="$ROOT_DIR/docs/figures" +BUILD_VOLUME="batch-pqc_build" + +echo "Building Docker image: $IMAGE_NAME" +docker build -t "$IMAGE_NAME" "$ROOT_DIR" + +mkdir -p "$RESULTS_DIR" "$DOCS_DIR" + +CMD="scripts/third_party.sh build && ./scripts/benchmark.sh ${*:---algo all --batch-sizes 1,2,4,8,16 --iters 100 --verify 1}" + +echo "Running benchmark in container (results -> $RESULTS_DIR, docs -> $DOCS_DIR)" +docker run --rm -it \ + -v "$ROOT_DIR":/opt/batch-pqc \ + -v "$BUILD_VOLUME":/opt/batch-pqc/build \ + -v "$RESULTS_DIR":/opt/batch-pqc/results \ + -v "$DOCS_DIR":/opt/batch-pqc/docs/figures \ + -w /opt/batch-pqc \ + "$IMAGE_NAME" /bin/bash -lc "$CMD" diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh new file mode 100755 index 0000000..b5fa81d --- /dev/null +++ b/scripts/install_deps.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + if command -v sudo >/dev/null 2>&1; then + exec sudo "$0" "$@" + fi + echo "This script requires root privileges or sudo." >&2 + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive + +APT_PACKAGES=( + build-essential + cmake + gcc + git + libcppunit-dev + libntl-dev + libssl-dev + make + pkg-config + python3 + python3-matplotlib + +) + +echo "[deps] apt-get update" +apt-get update + +echo "[deps] installing: ${APT_PACKAGES[*]}" +apt-get install -y "${APT_PACKAGES[@]}" + +echo "[deps] done" diff --git a/scripts/plot_results.py b/scripts/plot_results.py new file mode 100644 index 0000000..dfe927c --- /dev/null +++ b/scripts/plot_results.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import math +from collections import defaultdict +from pathlib import Path +import shutil + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Aggregate seq/batch benchmark results and render plots.") + parser.add_argument("--results-dir", required=True, help="Directory containing *_seq.csv and *_batch.csv files") + parser.add_argument("--output-dir", help="Directory for raw_data.csv and plots/ (defaults to results dir)") + parser.add_argument("--copy-to-docs", action="store_true", help="Copy generated raw_data.csv and plots to the docs figures directory") + parser.add_argument("--docs-dir", default="docs/figures", help="Destination docs figures directory when --copy-to-docs is used") + parser.add_argument("--algo", default="all", help="Algorithm to filter, or 'all'") + return parser.parse_args() + + +def load_rows(results_dir: Path, suffix: str) -> dict[tuple[str, str, int], dict[str, str]]: + rows: dict[tuple[str, str, int], dict[str, str]] = {} + pattern = f"*_{{suffix}}.csv".format(suffix=suffix) + for csv_path in sorted(results_dir.glob(pattern)): + with csv_path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + for row in reader: + key = (row["algorithm"], row["params"], int(row["batch_size"])) + rows[key] = row + return rows + + +def build_combined_rows(seq_rows: dict, batch_rows: dict, algo_filter: str) -> list[dict[str, float | int | str]]: + keys = sorted(set(seq_rows) | set(batch_rows)) + combined: list[dict[str, float | int | str]] = [] + + for key in keys: + algorithm, params, batch_size = key + if algo_filter != "all" and algorithm != algo_filter: + continue + + seq = seq_rows.get(key) + batch = batch_rows.get(key) + time_seq = float(seq["time_seq_ms"]) if seq else 0.0 + time_batch = float(batch["time_batch_ms"]) if batch else 0.0 + serialize_seq = float(seq["serialize_time_ms"]) if seq else 0.0 + serialize_batch = float(batch["serialize_time_ms"]) if batch else 0.0 + peak_memory = float(batch["peak_memory_MB"]) if batch else float(seq["peak_memory_MB"]) if seq else 0.0 + total_sig_size = float(batch["total_sig_size_MB"]) if batch else float(seq["total_sig_size_MB"]) if seq else 0.0 + speedup = time_seq / time_batch if time_seq > 0.0 and time_batch > 0.0 else 0.0 + + combined.append( + { + "algorithm": algorithm, + "params": params, + "batch_size": batch_size, + "time_seq_ms": time_seq, + "time_batch_ms": time_batch, + "speedup": speedup, + "peak_memory_MB": peak_memory, + "total_sig_size_MB": total_sig_size, + "serialize_time_seq_ms": serialize_seq, + "serialize_time_batch_ms": serialize_batch, + } + ) + + return combined + + +def write_raw_csv(rows: list[dict[str, float | int | str]], output_path: Path) -> None: + fieldnames = [ + "algorithm", + "params", + "batch_size", + "time_seq_ms", + "time_batch_ms", + "speedup", + "peak_memory_MB", + "total_sig_size_MB", + "serialize_time_seq_ms", + "serialize_time_batch_ms", + ] + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def render_plots(rows: list[dict[str, float | int | str]], plots_dir: Path) -> None: + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError("matplotlib is required for plot generation") from exc + + grouped: dict[str, list[dict[str, float | int | str]]] = defaultdict(list) + for row in rows: + grouped[str(row["algorithm"])].append(row) + + if not grouped: + return + + plt.style.use("ggplot") + + fig, ax = plt.subplots(figsize=(9, 5)) + for algorithm, points in sorted(grouped.items()): + ordered = sorted(points, key=lambda item: int(item["batch_size"])) + ax.plot( + [int(item["batch_size"]) for item in ordered], + [float(item["speedup"]) for item in ordered], + marker="o", + linewidth=2, + label=algorithm, + ) + + ax.set_title("Sequential vs Batch Signing Speedup") + ax.set_xlabel("Batch size") + ax.set_ylabel("Speedup = time_seq / time_batch") + ax.set_xscale("log", base=2) + ax.set_xticks(sorted({int(row["batch_size"]) for row in rows})) + ax.get_xaxis().set_major_formatter(plt.ScalarFormatter()) + ax.legend() + ax.grid(True, which="both", alpha=0.3) + fig.tight_layout() + fig.savefig(plots_dir / "speedup_vs_batch.png", dpi=150) + plt.close(fig) + + for algorithm, points in sorted(grouped.items()): + ordered = sorted(points, key=lambda item: int(item["batch_size"])) + fig, ax = plt.subplots(figsize=(9, 5)) + batches = [int(item["batch_size"]) for item in ordered] + seq_times = [float(item["time_seq_ms"]) for item in ordered] + batch_times = [float(item["time_batch_ms"]) for item in ordered] + + ax.plot(batches, seq_times, marker="o", linewidth=2, label="sequential") + ax.plot(batches, batch_times, marker="s", linewidth=2, label="batch") + ax.set_title(f"{algorithm}: sequential vs batch timings") + ax.set_xlabel("Batch size") + ax.set_ylabel("Time, ms") + ax.set_xscale("log", base=2) + ax.set_xticks(batches) + ax.get_xaxis().set_major_formatter(plt.ScalarFormatter()) + ax.grid(True, which="both", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(plots_dir / f"{algorithm}_times.png", dpi=150) + plt.close(fig) + + +def main() -> int: + args = parse_args() + results_dir = Path(args.results_dir).resolve() + output_dir = Path(args.output_dir).resolve() if args.output_dir else results_dir + plots_dir = output_dir / "plots" + + if not results_dir.is_dir(): + raise SystemExit(f"results directory not found: {results_dir}") + + seq_rows = load_rows(results_dir, "seq") + batch_rows = load_rows(results_dir, "batch") + combined_rows = build_combined_rows(seq_rows, batch_rows, args.algo) + if not combined_rows: + raise SystemExit("no matching benchmark rows found") + + output_dir.mkdir(parents=True, exist_ok=True) + plots_dir.mkdir(parents=True, exist_ok=True) + write_raw_csv(combined_rows, output_dir / "raw_data.csv") + + try: + render_plots(combined_rows, plots_dir) + except RuntimeError as exc: + print(f"[plot] warning: {exc}") + + print(f"[plot] wrote {(output_dir / 'raw_data.csv')} and plots in {plots_dir}") + if args.copy_to_docs: + docs_dir = Path(args.docs_dir).resolve() + docs_dir.mkdir(parents=True, exist_ok=True) + # copy raw_data.csv + src_raw = output_dir / "raw_data.csv" + if src_raw.exists(): + shutil.copy2(src_raw, docs_dir / "raw_data.csv") + # copy all png plots + for p in plots_dir.glob("*.png"): + try: + shutil.copy2(p, docs_dir / p.name) + except Exception as e: + print(f"[plot] warning: failed to copy {p}: {e}") + + print(f"[plot] copied raw_data.csv and plots to {docs_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/scripts/third_party.sh b/scripts/third_party.sh index e7915f1..cfa3624 100755 --- a/scripts/third_party.sh +++ b/scripts/third_party.sh @@ -28,8 +28,7 @@ do_rebuild() { do_install() { echo "[1/3] Installing dependencies..." - sudo apt-get update - sudo apt-get install -y cmake build-essential pkg-config libssl-dev libcppunit-dev libntl-dev + "$SCRIPT_DIR/install_deps.sh" echo "[2/3] Initializing/updating git submodules..." git submodule sync --recursive git submodule update --init --recursive diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index cba9b69..adcd5ed 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -1,6 +1,6 @@ # third_party/CMakeLists.txt -# Collects all submodules: shipovnik, hypericum, kryzhovnik, iaik_merkle_tree +# Collects all submodules: shipovnik, hypericum, kryzhovnik cmake_minimum_required(VERSION 3.14) project(ThirdPartyModules LANGUAGES C CXX) diff --git a/third_party/README.md b/third_party/README.md index 45dc943..f2ca65a 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -1,6 +1,6 @@ # third_party -This directory contains git submodules for cryptographic primitives and Merkle tree logic used in batch-pqc. +This directory contains git submodules for cryptographic primitives used in batch-pqc. Original algorithm repositories: @@ -13,7 +13,6 @@ Current submodule remotes used by this project: - shipovnik/ → https://github.com/cherninkiy/shipovnik-wrapper-tc26 - hypericum/ → https://github.com/cherninkiy/hypericum-wrapper-tc26 - kryzhovnik/ → https://github.com/cherninkiy/kryzhovnik-wrapper-tc26 -- iaik_merkle_tree/ → https://github.com/IAIK/merkle-tree Fork rationale: