From 7a825a8c9b85996b91b84d356adbd47e65560226 Mon Sep 17 00:00:00 2001 From: flamboh Date: Wed, 26 Aug 2026 02:42:21 -0700 Subject: [PATCH 1/2] feat(netflow-db): add coordinated active-source subsets --- .gitignore | 5 + Cargo.lock | 1 + docs/code/pipeline-contract.md | 49 + docs/user/datasets.md | 38 + docs/user/setup-pipeline.md | 646 + shell.nix | 2 + tools/netflow-db/Cargo.toml | 1 + tools/netflow-db/src/domain.rs | 401 +- tools/netflow-db/src/ingest.rs | 822 +- tools/netflow-db/src/maad.rs | 38 +- tools/netflow-db/src/main.rs | 48 +- tools/netflow-db/src/nfdump.rs | 488 +- tools/netflow-db/src/pipeline.rs | 14533 +++++++++++++++--- tools/netflow-db/src/provenance.rs | 42 + tools/netflow-db/src/publish.rs | 172 +- tools/netflow-db/src/registry.rs | 71 +- tools/netflow-db/src/storage.rs | 1815 ++- tools/netflow-db/tests/pipeline_cli_help.rs | 317 + 18 files changed, 16801 insertions(+), 2688 deletions(-) create mode 100644 tools/netflow-db/tests/pipeline_cli_help.rs diff --git a/.gitignore b/.gitignore index ecdb0ba..6868f9b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,8 @@ scripts/local/ugr16-csv.pipeline-v2.json **/playwright-mcp data/** .codex + +# Local native helper artifacts +/tools/netflow-db/maad_fast +/tools/netflow-db/nfdump_reducer +/tools/netflow-db/nfdump_reducer.build-id diff --git a/Cargo.lock b/Cargo.lock index dbea0c4..56fbd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,6 +206,7 @@ dependencies = [ "ipnet", "jiff", "libc", + "nix", "parquet", "rayon", "regex", diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 0ae77d2..72f4da3 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -26,6 +26,45 @@ Coverage is observed before selection. Thus, selected-out buckets remain as dens Native nfcapd input pushes the IP prefix condition into the nfdump filter. Visibility conditions apply before statistics accumulate. +`daily_active_sources` is a separate, fixed selection policy for the UOregon `/16` candidate +products. It accepts exactly one IPv4 `/16` and exactly one `nfcapd_tree` input. For each complete +local calendar day, it makes two bounded passes over every unique physical member: + +1. Select anonymized IPv4 source traffic in the `/16` that uses TCP or UDP and source port 1024 or + greater. Sum flows, packets, and bytes by exact source address across physical members. +2. Mark sources active at the inclusive thresholds of 3 flows, 20 packets, and 2,000 bytes. Publish + only the same qualifying-flow population from active sources into the existing five-minute and + rollup contracts. + +There is no destination-port filter and no TCP-flag or SYN filter. Overlapping logical sources do +not double-count activity because the first pass deduplicates their physical members. A day with +any missing physical capture is not published. Activity resets at local midnight, including DST +days. + +The normalized product identity records the entire fixed policy. It is not compatible with an old +prefix-only database. A late or changed input can alter the active set for every five-minute bucket +in a day, so repair requires a whole-day `--force` rebuild inside the existing day transaction. + +## Coordinated subset runs + +Repeat `--dataset` for two or more registry entries to build coordinated subset products. Each entry +supplies its own root and source configuration. Entries do not point to a parent dataset or a +`source_dataset`. + +Multi mode supports only `daily_active_sources`. The selected entries must resolve to the same +nfcapd root and logical source layout, and one whole local-day window. The command rejects `--config`, +`--database-path`, partial time bounds, and ambiguous selection overrides. It takes each output path +from the corresponding registry entry. + +The run coordinates one daily eligibility scan and one publication scan across the subsets. These +remain two physical phases because qualification needs the complete local day before any bucket can +publish. Local-day completeness is shared, so an incomplete required day blocks publication for every +subset. + +Each subset keeps its own immutable product database, product identity, transactions, resume state, +and MAAD configuration. Active sets are still resolved independently. Overlapping subsets may both +receive the same qualifying flow. + ## Input identity Each input records an exact revision. The revision contains a SHA-256 content identity and a canonical decoder fingerprint. @@ -81,6 +120,16 @@ The fork's stdout is the private `atlantis-flow-stream-v1` binary contract. The The pipeline stops when the fork executable is absent or incompatible. +Request resolution stores the executable's canonical path, one SHA-256 content identity, and a +cheap device/inode/size/timestamp snapshot. The binary identity is part of the product config and +the native input decoder fingerprint, so replacing the executable requires a fresh product and +cannot mix revisions in a resumed database. The snapshot is rechecked around activity and decode +scans and immediately before native publication commits; a change rolls that transaction back. + +Before creating an output directory, lock, or database, the pipeline runs one bounded probe against +an isolated empty `-R` directory. The probe requires the exact empty Atlantis stream (header, +terminator, and EOF), including for incomplete-day requests. + To update the fork, rebase its `atlantis-binary-v1` branch onto a reviewed upstream nfdump tag and run the fork's serial test suite. Then advance this repository's submodule pointer and run `./vendor/scripts/compile-nfdump.sh`. Treat a protocol or normalization change as a versioned wire-contract change. Update the Rust decoder and the provenance revision in the same change. ## Analysis-window exports diff --git a/docs/user/datasets.md b/docs/user/datasets.md index 2761ed2..200c9e2 100644 --- a/docs/user/datasets.md +++ b/docs/user/datasets.md @@ -61,6 +61,41 @@ A logical source combines the captures from more than one collector directory. E } ``` +## Define coordinated subsets + +Give each subset its own registry entry. Repeat those dataset IDs in one pipeline command. Each entry +defines its own logical sources and `daily_active_sources` selection. + +```json +[ + { + "dataset_id": "campus-a", + "root_path": "/data/netflow/campus", + "source_ids": ["router-a"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16" + }, + "db_path": "data/campus-a/netflow.sqlite" + }, + { + "dataset_id": "campus-b", + "root_path": "/data/netflow/campus", + "source_ids": ["router-a"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "0.221.0.0/16" + }, + "db_path": "data/campus-b/netflow.sqlite" + } +] +``` + +The entries share a capture root and the same logical source layout. Their active sets remain independent, +so one flow may publish to both products when the selections overlap. Do not add a parent or +`source_dataset` relation. The multi-dataset command infers each subset from the selected entry and +uses each entry's `db_path`. + ## Required fields | Field | Purpose | @@ -80,8 +115,11 @@ A logical source combines the captures from more than one collector directory. E | `source_ids` | None | Simple source names for datasets without member directories. | | `discovery_mode` | `static` | `live` marks a dataset that continues to receive new captures. `static` marks a complete dataset. | | `sort_order` | `0` | The dataset order in the dashboard. Lower values sort first. | +| `selection` | All flows | A normalized flow-selection object applied automatically by dataset-mode pipeline runs. | Set `db_path` only for a database that must stay separate, such as a [flow selection](setup-pipeline.md#select-flows) product. +Persist `selection` with a dedicated `db_path` when the dataset is itself a selected product. Command-line +selection flags may only override it when `--database-path` names a different output product. Each pipeline run calculates `default_start_date` again. A run that adds earlier days moves the date back. Set the field to hold the dashboard at one date. diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 56151c3..9298eb9 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -29,6 +29,11 @@ On macOS, bind mounts over large capture trees are slower than native filesystem `scripts/netflow-db.sh` runs the crate with `cargo run --locked --release`, which compiles it when necessary. Set `NETFLOW_DB_BIN` to run a prebuilt binary instead. +The native gate below also needs `jq`, Python 3, GNU `time` with verbose (`-v`) support, and a +running per-user `systemd` manager on a cgroup-v2 host. `jq`, Python, and GNU `time` are supplied +on `PATH` by `shell.nix`; `systemd-run` and `systemctl` normally come from the host. Run the gate +from `nix-shell shell.nix` (or an equivalent environment). + nfcapd input also needs the pinned ATLANTIS nfdump fork. A system nfdump installation does not work: the pipeline uses an output mode that only the fork has. CSV input does not need nfdump. 1. Initialize the Git submodules. @@ -64,6 +69,599 @@ Dataset mode calculates MAAD statistics by default. MAAD statistics describe the If a command fails, read [Troubleshooting](troubleshooting.md). +## Process coordinated subsets + +Repeat `--dataset` for two or more registry entries that describe subsets of one nfcapd tree: + +```bash +./scripts/netflow-db.sh pipeline \ + --dataset campus-a \ + --dataset campus-b \ + --start-date \ + --end-date +``` + +Multi mode supports `daily_active_sources` only. Put that selection in every registry entry. The +command infers each root and source configuration from its entry. It does not use a parent or +`source_dataset` relation. + +Use the same root and logical source layout for all entries. Use whole local-day dates. Do not pass +`--config`, `--database-path`, partial time bounds, or selection flags in multi mode. The command +uses each registry entry's database path. + +The pipeline performs one shared daily eligibility scan and one shared publication scan. It still +needs two physical phases because it must qualify sources over the whole local day first. A missing +required capture makes that local day incomplete for every subset. + +Each subset keeps its own product database, identity, transactions, resume state, and MAAD settings. +The active set is independent for each subset, so an overlapping subset may receive the same flow. + +### Gate a coordinated run + +Before a full MAAD run, gate one local day with temporary output paths. Keep the gate directory +under `data/`: the Docker wrapper mounts that directory at `/workspace/data`. Copy the registry and +rewrite only its `db_path` values; keep the roots, sources, and selections unchanged: + +```bash +mkdir -p data +gate_dir=$(mktemp -d data/netflow-gate.XXXXXX) +gate_root=$(realpath "$gate_dir") +gate_date=2025-06-01 # replace with one complete local day +jq --arg dir "$gate_dir" \ + '(if type == "array" then {datasets: .} + elif type == "object" then . + else error("registry must be an array or an object with a datasets array") + end) + | (.datasets // .) as $entries + | if ($entries | type) != "array" then + error("registry must be an array or an object with a datasets array") + elif ($entries | length) == 0 then + error("registry cannot be empty") + elif any($entries[]; type != "object") then + error("registry entries must be objects") + else + $entries + | to_entries + | map(.value.db_path = ($dir + "/" + ((.key + 1) | tostring) + ".sqlite") | .value) + end' \ + datasets.json >"$gate_dir/datasets.json" + +registry_db_paths() { + local registry=$1 + jq -er ' + (if type == "array" then {datasets: .} + elif type == "object" then . + else error("registry must be an array or an object with a datasets array") + end) + | (.datasets // .) as $entries + | if ($entries | type) != "array" then + error("registry must be an array or an object with a datasets array") + elif any($entries[]; type != "object") then + error("registry entries must be objects") + else + ["campus-a", "campus-b"][] as $dataset_id + | ($entries | map(select(.dataset_id == $dataset_id))) as $matches + | if ($matches | length) != 1 then + error("registry must contain exactly one entry for " + $dataset_id) + elif (($matches[0].db_path | type) != "string") then + error("db_path for " + $dataset_id + " must be a string") + elif ($matches[0].db_path | length) == 0 then + error("db_path for " + $dataset_id + " cannot be empty") + else + $matches[0].db_path + end + end + ' "$registry" +} + +mapfile -t gate_db_paths < <(registry_db_paths "$gate_dir/datasets.json") +if (( ${#gate_db_paths[@]} != 2 )); then + echo "the gate requires campus-a and campus-b database paths" >&2 + exit 1 +fi +for path in "${gate_db_paths[@]}"; do + resolved=$(realpath -m -- "$path") + case "$resolved" in + "$gate_root"/*.sqlite) ;; + *) echo "temporary database escaped gate directory: $path" >&2; exit 1 ;; + esac +done +``` + +Run the same MAAD-enabled native command twice for one complete local day, saving separate cold and +no-op resource/profile logs. The cold invocation checks positive publication cardinality; the +second invocation checks the resume path and zero new publications. Record the cgroup aggregate +peak, elapsed wall time, every `netflow_db::profile` phase, and the byte sizes of each temporary +SQLite output and its `-wal` file. GNU `time -v` still writes its per-process +`Maximum resident set size` for diagnostics, but that value is not a gate: it is not the sum of +the pipeline's concurrent nfdump children. + +The native gate puts the whole pipeline process tree in a transient per-user systemd scope. Its +16 GiB `MemoryMax` (`16777216` KiB) is an aggregate cgroup-v2 limit, and the gate reads the scope's +`memory.peak` and fails closed if peak accounting or the cgroup's OOM counters cannot be read. +This leaves a safe margin below roughly 20 GiB available on Barbera without requiring root. Set +`NETFLOW_GATE_MAX_MEMORY_KIB`, `NETFLOW_GATE_COLD_MAX_ELAPSED_SECONDS`, +`NETFLOW_GATE_FULL_COLD_MAX_ELAPSED_SECONDS`, `NETFLOW_GATE_FULL_NOOP_MAX_ELAPSED_SECONDS`, or +`NETFLOW_GATE_SPACE_HEADROOM_PERCENT` before running the block to change the ceilings. The +full-cold budget defaults to 30 days (`2592000` seconds); the gate conservatively projects two +times the measured one-day cold elapsed time across 394 days before launching it. The space +headroom defaults to 100% (a 2x projection). These positive-integer defaults are fail-closed: +raise them explicitly only when the host and run window justify it. +The gate refuses hosts without cgroup v2 or a usable user systemd manager; it does not silently +fall back to GNU `time` or an incomplete process-tree RSS sample. + +Run this Bash block from that Nix shell; it keeps the one-day and full-history phases in the same +gated session: + +```bash +set -euo pipefail +pipeline=(./scripts/netflow-db.sh) +time_bin="$(type -P time || true)" +if [[ -z "$time_bin" ]] || ! "$time_bin" -v true >/dev/null 2>&1; then + echo "GNU time with -v is required on PATH; enter nix-shell shell.nix" >&2 + exit 1 +fi +systemd_run_bin="$(type -P systemd-run || true)" +systemctl_bin="$(type -P systemctl || true)" +true_bin="$(type -P true || true)" +sleep_bin="$(type -P sleep || true)" +if [[ -z "$systemd_run_bin" || -z "$systemctl_bin" || -z "$true_bin" ]] || + ! [[ "$(stat -fc %T -- /sys/fs/cgroup 2>/dev/null || true)" == cgroup2fs ]] || + ! "$systemd_run_bin" --user --scope --quiet -p MemoryMax=64M "$true_bin" >/dev/null 2>&1; then + echo "a running user systemd manager with cgroup v2 is required for the aggregate memory gate" >&2 + exit 1 +fi +runtime_probe_unit="netflow-gate-runtime-probe-$$.scope" +if [[ -z "$sleep_bin" ]] || + "$systemd_run_bin" --user --scope --quiet --collect --unit="$runtime_probe_unit" \ + -p MemoryMax=64M -p RuntimeMaxSec=1s -- "$sleep_bin" 5 >/dev/null 2>&1; then + echo "a user-systemd scope with RuntimeMaxSec is required for the elapsed-time gate" >&2 + exit 1 +fi +max_memory_kib_limit="${NETFLOW_GATE_MAX_MEMORY_KIB:-16777216}" +cold_elapsed_limit_seconds="${NETFLOW_GATE_COLD_MAX_ELAPSED_SECONDS:-1800}" +full_cold_elapsed_limit_seconds="${NETFLOW_GATE_FULL_COLD_MAX_ELAPSED_SECONDS:-2592000}" +full_noop_elapsed_limit_seconds="${NETFLOW_GATE_FULL_NOOP_MAX_ELAPSED_SECONDS:-1800}" +space_headroom_percent="${NETFLOW_GATE_SPACE_HEADROOM_PERCENT:-100}" +if ! [[ "$max_memory_kib_limit" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$cold_elapsed_limit_seconds" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$full_cold_elapsed_limit_seconds" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$full_noop_elapsed_limit_seconds" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$space_headroom_percent" =~ ^[1-9][0-9]*$ ]]; then + echo "native gate ceilings must be positive integers" >&2 + exit 1 +fi + +parse_elapsed_seconds() { + awk ' + /^[[:space:]]*Elapsed \(wall clock\) time \(h:mm:ss or m:ss\):[[:space:]]*/ { + value = $0 + sub(/^.*\):[[:space:]]*/, "", value) + gsub(/[[:space:]]/, "", value) + matches++ + if (value !~ /^[0-9]+:[0-9][0-9](:[0-9][0-9])?([.][0-9]+)?$/) { + invalid = 1 + next + } + fields = split(value, part, ":") + if (fields == 2) { + if (part[2] + 0 >= 60) invalid = 1 + else seconds = (part[1] * 60) + part[2] + } else if (fields == 3) { + if (part[2] + 0 >= 60 || part[3] + 0 >= 60) invalid = 1 + else seconds = (part[1] * 3600) + (part[2] * 60) + part[3] + } else { + invalid = 1 + } + } + END { + if (matches != 1 || invalid) exit 1 + printf "%.6f\n", seconds + } + ' "$1" +} + +assert_resources() { + local log_name=$1 + local elapsed_limit_seconds=$2 + local log_path="$gate_dir/$log_name" + local peak_bytes peak_kib elapsed_seconds + if ! peak_bytes="$(cat "$gate_dir/$log_name.cgroup-peak-bytes" 2>/dev/null)" || + ! [[ "$peak_bytes" =~ ^[1-9][0-9]*$ ]]; then + echo "missing or malformed aggregate cgroup peak in $log_path" >&2 + return 1 + fi + if ! elapsed_seconds="$(parse_elapsed_seconds "$log_path")"; then + echo "missing or malformed elapsed wall time in $log_path" >&2 + return 1 + fi + peak_kib=$(( (peak_bytes + 1023) / 1024 )) + if (( peak_bytes > max_memory_kib_limit * 1024 )); then + echo "aggregate memory limit exceeded in $log_path: ${peak_kib} KiB > ${max_memory_kib_limit} KiB" >&2 + return 1 + fi + if ! awk -v actual="$elapsed_seconds" -v limit="$elapsed_limit_seconds" ' + BEGIN { + if (actual !~ /^[0-9]+([.][0-9]+)?$/ || actual > limit) exit 1 + } + '; then + echo "elapsed limit exceeded in $log_path: ${elapsed_seconds}s > ${elapsed_limit_seconds}s" >&2 + return 1 + fi + printf 'Resource gate %s: %s KiB aggregate cgroup peak, %ss elapsed\n' \ + "$log_name" "$peak_kib" "$elapsed_seconds" +} + +stop_gate_unit() { + local unit=$1 + "$systemctl_bin" --user stop "$unit" >/dev/null 2>&1 || true +} + +gate_pid_is_running() { + local pid=$1 + local state + if ! [[ "$pid" =~ ^[1-9][0-9]*$ ]] || + ! state="$(awk '$1 == "State:" { print $2; exit }' "/proc/$pid/status" 2>/dev/null)"; then + return 1 + fi + [[ "$state" =~ ^[A-Z]$ && "$state" != Z ]] +} + +monitor_cgroup_peak() { + local cgroup_dir=$1 + local peak_path=$2 + local launch_pid=$3 + local unit=$4 + local peak_bytes events_values oom oom_kill oom_group_kill saw_sample=0 + while true; do + if [[ ! -r "$cgroup_dir/memory.peak" || ! -r "$cgroup_dir/memory.events" ]]; then + if gate_pid_is_running "$launch_pid"; then + stop_gate_unit "$unit" + return 1 + fi + break + fi + if ! peak_bytes="$(cat "$cgroup_dir/memory.peak")" || + ! [[ "$peak_bytes" =~ ^[0-9]+$ ]]; then + if gate_pid_is_running "$launch_pid"; then + stop_gate_unit "$unit" + return 1 + fi + break + fi + if ! events_values="$(awk ' + $1 == "oom" { + oom_count++ + if (NF != 2 || $2 !~ /^[0-9]+$/) invalid = 1 + else oom_value = $2 + next + } + $1 == "oom_kill" { + oom_kill_count++ + if (NF != 2 || $2 !~ /^[0-9]+$/) invalid = 1 + else oom_kill_value = $2 + next + } + $1 == "oom_group_kill" { + oom_group_kill_count++ + if (NF != 2 || $2 !~ /^[0-9]+$/) invalid = 1 + else oom_group_kill_value = $2 + next + } + END { + if (invalid || oom_count != 1 || oom_kill_count != 1 || oom_group_kill_count != 1) exit 1 + printf "%s %s %s\n", oom_value, oom_kill_value, oom_group_kill_value + } + ' "$cgroup_dir/memory.events")"; then + if gate_pid_is_running "$launch_pid"; then + stop_gate_unit "$unit" + return 1 + fi + break + fi + read -r oom oom_kill oom_group_kill <<<"$events_values" + if [[ "$oom" != 0 || "$oom_kill" != 0 || "$oom_group_kill" != 0 ]]; then + if gate_pid_is_running "$launch_pid"; then + stop_gate_unit "$unit" + fi + return 1 + fi + printf '%s\n' "$peak_bytes" >"$peak_path" + saw_sample=1 + gate_pid_is_running "$launch_pid" || break + sleep 0.1 + done + (( saw_sample == 1 )) +} + +gate_active_unit= +gate_active_launch_pid= +gate_active_monitor_pid= +cleanup_gate_processes() { + local unit=$gate_active_unit + local launch_pid=$gate_active_launch_pid + local monitor_pid=$gate_active_monitor_pid + gate_active_unit= + gate_active_launch_pid= + gate_active_monitor_pid= + + if [[ -n "$unit" ]]; then + stop_gate_unit "$unit" + fi + if [[ -n "$monitor_pid" ]] && gate_pid_is_running "$monitor_pid"; then + kill "$monitor_pid" >/dev/null 2>&1 || true + fi + if [[ -n "$monitor_pid" ]]; then + wait "$monitor_pid" >/dev/null 2>&1 || true + fi + if [[ -n "$launch_pid" ]]; then + wait "$launch_pid" >/dev/null 2>&1 || true + fi +} + +gate_error() { + local status=$? + cleanup_gate_processes + trap - ERR + exit "$status" +} + +gate_signal() { + local status=$1 + cleanup_gate_processes + trap - ERR + exit "$status" +} + +trap gate_error ERR +trap 'gate_signal 129' HUP +trap 'gate_signal 130' INT +trap 'gate_signal 143' TERM +trap cleanup_gate_processes EXIT + +run_gate() { + local log_name=$1 + local published_pattern=$2 + local start_date="${3:-$gate_date}" + local end_date="${4:-$gate_date}" + local registry="${5:-$gate_dir/datasets.json}" + local elapsed_limit_seconds="${6:-$cold_elapsed_limit_seconds}" + local unit="netflow-gate-${log_name%.log}-$$.scope" + local launch_pid pipeline_status monitor_pid monitor_status control_group cgroup_dir= + gate_active_unit=$unit + RUST_LOG=netflow_db::profile=info "$time_bin" -v \ + "$systemd_run_bin" --user --scope --quiet --collect --unit="$unit" \ + -p "MemoryMax=${max_memory_kib_limit}K" \ + -p "RuntimeMaxSec=${elapsed_limit_seconds}s" -- \ + "${pipeline[@]}" pipeline \ + --datasets "$registry" \ + --dataset campus-a --dataset campus-b \ + --start-date "$start_date" --end-date "$end_date" \ + --require-complete \ + >"$gate_dir/$log_name" 2>&1 & + launch_pid=$! + gate_active_launch_pid=$launch_pid + for _ in {1..300}; do + control_group="$("$systemctl_bin" --user show "$unit" --property=ControlGroup --value 2>/dev/null || true)" + if [[ "$control_group" == /* && "$control_group" != *..* && + "$control_group" != *$'\n'* && "$control_group" != *$'\t'* ]]; then + cgroup_dir="/sys/fs/cgroup$control_group" + if [[ -r "$cgroup_dir/memory.peak" && -r "$cgroup_dir/memory.events" ]]; then + break + fi + cgroup_dir= + fi + gate_pid_is_running "$launch_pid" || break + sleep 0.1 + done + if [[ -z "$cgroup_dir" ]]; then + echo "could not locate aggregate cgroup for $unit" >&2 + cleanup_gate_processes + return 1 + fi + monitor_cgroup_peak "$cgroup_dir" "$gate_dir/$log_name.cgroup-peak-bytes" "$launch_pid" "$unit" & + monitor_pid=$! + gate_active_monitor_pid=$monitor_pid + if wait "$monitor_pid"; then monitor_status=0; else monitor_status=$?; fi + if (( monitor_status != 0 )); then + stop_gate_unit "$unit" + fi + if wait "$launch_pid"; then pipeline_status=0; else pipeline_status=$?; fi + gate_active_unit= + gate_active_launch_pid= + gate_active_monitor_pid= + if (( pipeline_status != 0 || monitor_status != 0 )); then + echo "aggregate memory gate failed for $log_name (pipeline=$pipeline_status monitor=$monitor_status)" >&2 + return 1 + fi + cat "$gate_dir/$log_name" + grep -Eq '^Five-minute coverage: [1-9][0-9]* complete' "$gate_dir/$log_name" + grep -Eq '^Five-minute coverage: [1-9][0-9]* complete, 0 partial, 0 unknown$' \ + "$gate_dir/$log_name" + grep -Eq "$published_pattern" "$gate_dir/$log_name" + assert_resources "$log_name" "$elapsed_limit_seconds" +} +run_gate one-day-cold.log '^Published five-minute buckets: [1-9][0-9]*$' \ + "$gate_date" "$gate_date" "$gate_dir/datasets.json" "$cold_elapsed_limit_seconds" + +mapfile -t destination_db_paths < <(registry_db_paths datasets.json) +if (( ${#destination_db_paths[@]} != 2 )); then + echo "the gate requires campus-a and campus-b database paths" >&2 + exit 1 +fi + +destination_related_paths() { + local database=$1 + local resolved path parent name suffix + if ! resolved="$(realpath -m -- "$database")"; then + echo "could not resolve final database path: $database" >&2 + return 1 + fi + for path in "$database" "$resolved"; do + parent=${path%/*} + [[ "$parent" == "$path" ]] && parent=. + name=${path##*/} + printf '%s\n' "$path" + for suffix in -journal -wal -shm; do + printf '%s%s\n' "$path" "$suffix" + done + printf '%s/.%s.operation.lock\n' "$parent" "$name" + done +} + +assert_fresh_destination_paths() { + local database=$1 + local related + while IFS= read -r related; do + if [[ -e "$related" || -L "$related" ]]; then + echo "full-cold requires a fresh final database path; found existing database, SQLite sidecar, or operation lock: $related" >&2 + echo "Remove that path and retry; the final destination must be fresh before full-cold: $database" >&2 + return 1 + fi + done < <(destination_related_paths "$database") +} + +for db_path in "${destination_db_paths[@]}"; do + assert_fresh_destination_paths "$db_path" +done + +for db_path in "${gate_db_paths[@]}"; do + python3 - "$db_path" <<'PY' +import sqlite3 +import sys + +database = sys.argv[1] +with sqlite3.connect(database, timeout=30) as connection: + result = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() +if result is None or result[0] != 0: + raise SystemExit(f"WAL checkpoint was busy for {database!r}: {result!r}") +PY + if [[ -e "$db_path-wal" && "$(stat -c %s -- "$db_path-wal")" != 0 ]]; then + echo "WAL was not truncated for $db_path" >&2 + exit 1 + fi +done + +python3 - "$space_headroom_percent" \ + "${gate_db_paths[0]}" "${gate_db_paths[1]}" \ + "${destination_db_paths[0]}" "${destination_db_paths[1]}" <<'PY' | tee "$gate_dir/space.log" +import os +import shutil +import sys +from collections import defaultdict + +headroom_percent = int(sys.argv[1]) +source_paths = sys.argv[2:4] +destination_paths = sys.argv[4:6] +if headroom_percent < 1 or len(source_paths) != 2 or len(destination_paths) != 2: + raise SystemExit("invalid space-gate arguments") + +history_days = 394 +factor = history_days * (100 + headroom_percent) +projections = [] +for source, destination in zip(source_paths, destination_paths): + source_size = os.stat(source).st_size + if source_size < 1: + raise SystemExit(f"one-day database is empty: {source!r}") + destination_realpath = os.path.realpath(destination) + destination_parent = os.path.dirname(destination_realpath) or "." + if not os.path.isdir(destination_parent): + raise SystemExit(f"destination directory does not exist: {destination_parent!r}") + projected = (source_size * factor + 99) // 100 + projections.append((destination_parent, destination, source_size, projected)) + +by_filesystem = defaultdict(list) +for projection in projections: + by_filesystem[os.stat(projection[0]).st_dev].append(projection) + +for device, files in by_filesystem.items(): + free = shutil.disk_usage(files[0][0]).free + required = sum(item[3] for item in files) + print(f"filesystem {device}: {required} projected bytes required, {free} bytes free") + for _, destination, source_size, projected in files: + print(f" {destination}: {source_size} one-day bytes -> {projected} projected bytes") + if required > free: + raise SystemExit(f"insufficient free space on filesystem {device}") +PY +run_gate one-day-noop.log '^Published five-minute buckets: 0$' \ + "$gate_date" "$gate_date" "$gate_dir/datasets.json" "$cold_elapsed_limit_seconds" + +for path in "$gate_dir"/*.sqlite "$gate_dir"/*.sqlite-wal; do + if [[ -e "$path" ]]; then + stat --printf='%n %s bytes\n' "$path" + fi +done | tee "$gate_dir/sizes.log" + +one_day_cold_elapsed_seconds="$(parse_elapsed_seconds "$gate_dir/one-day-cold.log")" +full_cold_projection_seconds="$(awk -v one_day="$one_day_cold_elapsed_seconds" ' + BEGIN { + if (one_day !~ /^[0-9]+([.][0-9]+)?$/ || one_day <= 0) exit 1 + projected = one_day * 394 * 2 + rounded = int(projected) + if (projected > rounded) rounded++ + printf "%d\n", rounded + } +')" +if ! [[ "$full_cold_projection_seconds" =~ ^[1-9][0-9]*$ ]] || + ! awk -v projected="$full_cold_projection_seconds" -v limit="$full_cold_elapsed_limit_seconds" ' + BEGIN { + if (projected !~ /^[0-9]+$/ || limit !~ /^[0-9]+$/ || projected > limit) exit 1 + } + '; then + echo "projected full-cold elapsed time exceeds its configured budget: ${full_cold_projection_seconds}s > ${full_cold_elapsed_limit_seconds}s" >&2 + exit 1 +fi +printf 'Full cold elapsed projection: %ss (2x %s-day one-day cold elapsed of %ss); budget: %ss\n' \ + "$full_cold_projection_seconds" 394 "$one_day_cold_elapsed_seconds" \ + "$full_cold_elapsed_limit_seconds" + +run_gate full-cold.log '^Published five-minute buckets: [1-9][0-9]*$' \ + 2025-06-01 2026-06-29 datasets.json "$full_cold_elapsed_limit_seconds" +run_gate full-noop.log '^Published five-minute buckets: 0$' \ + 2025-06-01 2026-06-29 datasets.json "$full_noop_elapsed_limit_seconds" +``` + +The block runs four distinct assertions: `one-day-cold.log` requires positive publication from +the temporary databases, `one-day-noop.log` requires zero new publication while retaining complete +coverage, `full-cold.log` requires positive publication from the real registry, and +`full-noop.log` requires zero new publication over the complete history. Every run is covered by +the same aggregate cgroup memory monitor and process-tree-safe `RuntimeMaxSec` limit. The full +cold run is launched only after the one-day elapsed time has been parsed and its conservative +2x/394-day projection fits the explicit full-cold budget; the full no-op remains after that build. + +The full-history no-op asserts zero published buckets, positive complete coverage with no partial +or unknown buckets, the same 16 GiB-by-default aggregate cgroup memory ceiling, and its separate +elapsed ceiling. The one-day space projection is deliberately conservative: it scales each +checkpointed output by 394 days and adds the configured headroom for SQLite growth and WAL space, +then compares the combined requirement with free bytes on the actual destination filesystems +before the full build is launched. + +The Docker wrapper is suitable for a functional smoke check, not this RSS gate. It does not expose +a container memory limit or cgroup peak sampler, and timing the wrapper measures the local Docker +client rather than the process inside the container. If you use Docker for the smoke check, keep +the cold and no-op logs separate and keep the capture root absolute: + +```bash +set -euo pipefail +docker_gate() { + local log_name=$1 + local published_pattern=$2 + ./scripts/netflow-db-docker.sh --capture-root /absolute/path/to/captures pipeline \ + --datasets "$gate_dir/datasets.json" \ + --dataset campus-a --dataset campus-b \ + --start-date "$gate_date" --end-date "$gate_date" \ + --require-complete \ + 2>&1 | tee "$gate_dir/$log_name" + grep -Eq '^Five-minute coverage: [1-9][0-9]* complete' "$gate_dir/$log_name" + grep -Eq "$published_pattern" "$gate_dir/$log_name" +} +docker_gate docker-cold.log '^Published five-minute buckets: [1-9][0-9]*$' +docker_gate docker-noop.log '^Published five-minute buckets: 0$' +``` + +Both the temporary registry and SQLite outputs work in this Docker smoke check because their paths +are relative to the mounted repository `data/` directory. + ## Select flows Selection conditions use AND logic. The IP prefix can match the source endpoint or the destination endpoint. @@ -79,13 +677,40 @@ Selection conditions use AND logic. The IP prefix can match the source endpoint ``` A selected population is a different database product. Thus, selection options require an explicit `--database-path`. +Dataset registry entries may instead persist a `selection` beside their dedicated `db_path`; dataset +mode applies that selection automatically. Available selection options are: - `--ip-prefix` +- `--daily-active-sources` - `--src-visibility literal|anonymized` - `--dst-visibility literal|anonymized` +`--daily-active-sources` applies the fixed active-user definition used to choose the UOregon +candidate subnets. It requires an IPv4 `/16` and cannot be combined with the visibility flags: + +```bash +./scripts/netflow-db.sh pipeline \ + --dataset example \ + --start-date \ + --end-date \ + --database-path data/example-active/netflow.sqlite \ + --ip-prefix 0.220.0.0/16 \ + --daily-active-sources +``` + +For each complete local day, the pipeline sums qualifying traffic by exact source address across +each unique physical capture member. A source is active when it has at least 3 flows, 20 packets, +and 2,000 bytes that day. Qualifying traffic is IPv4 TCP or UDP from an anonymized source in the +target `/16`, with source port at least 1024. Destination ports and TCP flags are unrestricted. +Only that qualifying traffic from active sources is published. + +This mode supports exactly one `nfcapd_tree` input and whole local days. A day missing any expected +physical capture is skipped rather than published as zero. If input evidence changes after a day +was published, rebuild the whole day with `--force`; a single five-minute repair is not safe because +it can change the active-source set for every bucket in that day. + ## Use a pipeline configuration Configuration mode supports CSV input, nfcapd input, and mixed input. Explicit `csv` and `nfcapd` inputs and `csv_tree` and `nfcapd_tree` discovery inputs go in the top-level `inputs` list. @@ -109,6 +734,27 @@ Put flow selection in the top-level `selection` object: } ``` +The equivalent active-source selection is deliberately a named policy rather than configurable +thresholds: + +```json +{ + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16" + }, + "inputs": [ + { + "input_kind": "nfcapd_tree", + "root_path": "/path/to/captures", + "source_ids": ["gateway-a", "gateway-b"], + "start_date": "2025-06-01", + "end_date": "2026-06-29" + } + ] +} +``` + On the native path, nfcapd input needs the fork path: set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`, or pass `--nfdump` when the configuration does not set it. ## Common options diff --git a/shell.nix b/shell.nix index 4e27246..ce49730 100644 --- a/shell.nix +++ b/shell.nix @@ -15,11 +15,13 @@ mkShell { pkgs.git pkgs.gnumake pkgs.gnutar + pkgs.jq pkgs.libtool pkgs.nodejs pkgs.pkg-config pkgs.python3 pkgs.rustup + pkgs.time pkgs.playwright-driver.browsers ]; diff --git a/tools/netflow-db/Cargo.toml b/tools/netflow-db/Cargo.toml index b01215f..54cd946 100644 --- a/tools/netflow-db/Cargo.toml +++ b/tools/netflow-db/Cargo.toml @@ -26,6 +26,7 @@ fs2 = "0.4" ipnet = { version = "2", features = ["serde"] } jiff = "0.2" libc = "0.2" +nix = { version = "0.27", default-features = false, features = ["fs"] } parquet = { version = "59.2", default-features = false, features = ["arrow", "zstd"] } rayon = "1" regex = "1" diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index 7b64ab8..6fb0e11 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -36,10 +36,18 @@ pub enum DomainError { UnknownSelectionKeys(String), #[error("selection version must be 1")] InvalidSelectionVersion, - #[error("selection kind must be 'all' or 'flows'")] + #[error("selection kind must be 'all', 'flows', or 'daily_active_sources'")] InvalidSelectionKind, #[error("selection kind 'all' cannot define flow criteria")] AllSelectionHasCriteria, + #[error("daily_active_sources selection requires one IPv4 /16 ip_prefix")] + DailyActiveSourcesRequireIpv4Prefix, + #[error( + "daily_active_sources selection fixes src_visibility to 'anonymized' and leaves dst_visibility unrestricted" + )] + InvalidDailyActiveSourceVisibility, + #[error("daily_active_sources criteria do not match the finalized active-source definition")] + InvalidDailyActiveSourceCriteria, #[error("Invalid selection ip_prefix: {0}")] InvalidIpPrefix(String), #[error("selection {0} must be 'literal' or 'anonymized'")] @@ -265,9 +273,21 @@ impl FlowObservation { } } -/// A validated predicate shared by all input adapters. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum FlowSelectionKind { + #[default] + Flows, + DailyActiveSources, +} + +pub(crate) const DAILY_ACTIVE_MIN_FLOWS: i64 = 3; +pub(crate) const DAILY_ACTIVE_MIN_PACKETS: i64 = 20; +pub(crate) const DAILY_ACTIVE_MIN_BYTES: i64 = 2_000; + +/// A validated selection shared by input adapters and pipeline product identity. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct FlowSelection { + kind: FlowSelectionKind, ip_prefix: Option, src_visibility: Option, dst_visibility: Option, @@ -286,7 +306,12 @@ impl FlowSelection { .filter(|key| { !matches!( key.as_str(), - "version" | "kind" | "ip_prefix" | "src_visibility" | "dst_visibility" + "version" + | "kind" + | "ip_prefix" + | "src_visibility" + | "dst_visibility" + | "criteria" ) }) .cloned() @@ -302,11 +327,13 @@ impl FlowSelection { Some(Value::String(kind)) => Some(kind.as_str()), Some(_) => return Err(DomainError::InvalidSelectionKind), }; - if !matches!(kind, None | Some("all" | "flows")) { - return Err(DomainError::InvalidSelectionKind); - } + let selection_kind = match kind { + None | Some("all" | "flows") => FlowSelectionKind::Flows, + Some("daily_active_sources") => FlowSelectionKind::DailyActiveSources, + Some(_) => return Err(DomainError::InvalidSelectionKind), + }; if kind == Some("all") - && ["ip_prefix", "src_visibility", "dst_visibility"] + && ["ip_prefix", "src_visibility", "dst_visibility", "criteria"] .iter() .any(|key| !is_empty_value(object.get(*key))) { @@ -322,16 +349,47 @@ impl FlowSelection { .map(|network| network.trunc()) }) .transpose()?; - Ok(Self { + let mut selection = Self { + kind: selection_kind, ip_prefix, src_visibility: parse_visibility(object, "src_visibility")?, dst_visibility: parse_visibility(object, "dst_visibility")?, - }) + }; + if selection.kind == FlowSelectionKind::DailyActiveSources { + if !matches!(selection.ip_prefix, Some(IpNet::V4(prefix)) if prefix.prefix_len() == 16) + { + return Err(DomainError::DailyActiveSourcesRequireIpv4Prefix); + } + if !matches!( + selection.src_visibility, + None | Some(ExactVisibility::Anonymized) + ) || selection.dst_visibility.is_some() + { + return Err(DomainError::InvalidDailyActiveSourceVisibility); + } + if object.get("criteria").is_some_and(|criteria| { + !criteria.is_null() && criteria != &daily_active_source_criteria() + }) { + return Err(DomainError::InvalidDailyActiveSourceCriteria); + } + selection.src_visibility = Some(ExactVisibility::Anonymized); + } else if !is_empty_value(object.get("criteria")) { + return Err(DomainError::InvalidDailyActiveSourceCriteria); + } + Ok(selection) } #[must_use] pub const fn is_unrestricted(&self) -> bool { - self.ip_prefix.is_none() && self.src_visibility.is_none() && self.dst_visibility.is_none() + matches!(self.kind, FlowSelectionKind::Flows) + && self.ip_prefix.is_none() + && self.src_visibility.is_none() + && self.dst_visibility.is_none() + } + + #[must_use] + pub const fn selects_daily_active_sources(&self) -> bool { + matches!(self.kind, FlowSelectionKind::DailyActiveSources) } #[must_use] @@ -350,11 +408,16 @@ impl FlowSelection { } #[must_use] - pub fn matches(&self, observation: &FlowObservation) -> bool { - let prefix_matches = self.ip_prefix.as_ref().is_none_or(|prefix| { - prefix.contains(&observation.src_ip) || prefix.contains(&observation.dst_ip) - }); - prefix_matches && self.allows_src_tos(observation.src_tos) + pub fn matches_qualifying_flow(&self, observation: &FlowObservation) -> bool { + let (source, destination) = exact_visibility_pair_from_tos(observation.src_tos); + self.matches_flow_fields( + observation.src_ip, + observation.dst_ip, + observation.protocol, + observation.src_port, + source, + destination, + ) } #[must_use] @@ -368,10 +431,56 @@ impl FlowSelection { } #[must_use] - pub fn nfdump_prefix_filter(&self) -> Option { - self.ip_prefix - .as_ref() - .map(|prefix| format!("net {prefix}")) + pub fn nfdump_filter(&self) -> Option { + self.ip_prefix.as_ref().map(|prefix| { + if self.selects_daily_active_sources() { + // nfdump filters the original record before -o atlantis expands its + // tunnel extension into a synthetic row; retain both branches and + // let the Rust decoder authoritatively match emitted rows. + format!( + "(src net {prefix} and ipv4 and (proto tcp or proto udp) and src port > 1023) or (src tun net {prefix} and (tun proto tcp or tun proto udp) and src port > 1023)" + ) + } else { + format!("net {prefix}") + } + }) + } + + #[must_use] + pub(crate) fn matches_flow_fields( + &self, + src_ip: IpAddr, + dst_ip: IpAddr, + protocol: u8, + src_port: Option, + source_visibility: ExactVisibility, + destination_visibility: ExactVisibility, + ) -> bool { + let prefix_matches = self.ip_prefix.as_ref().is_none_or(|prefix| { + if self.selects_daily_active_sources() { + prefix.contains(&src_ip) + } else { + prefix.contains(&src_ip) || prefix.contains(&dst_ip) + } + }); + let visibility_matches = self + .src_visibility + .is_none_or(|required| required == source_visibility) + && self + .dst_visibility + .is_none_or(|required| required == destination_visibility); + let activity_candidate_matches = !self.selects_daily_active_sources() + || (matches!(src_ip, IpAddr::V4(_)) + && matches!(protocol, 6 | 17) + && src_port.is_some_and(|port| port >= 1_024)); + prefix_matches && visibility_matches && activity_candidate_matches + } + + #[must_use] + pub const fn daily_activity_threshold_met(flows: i64, packets: i64, bytes: i64) -> bool { + flows >= DAILY_ACTIVE_MIN_FLOWS + && packets >= DAILY_ACTIVE_MIN_PACKETS + && bytes >= DAILY_ACTIVE_MIN_BYTES } #[must_use] @@ -379,6 +488,16 @@ impl FlowSelection { if self.is_unrestricted() { return json!({"version": 1, "kind": "all"}); } + if self.selects_daily_active_sources() { + return json!({ + "version": 1, + "kind": "daily_active_sources", + "ip_prefix": self.ip_prefix.map(|prefix| prefix.to_string()), + "src_visibility": self.src_visibility.map(ExactVisibility::as_str), + "dst_visibility": self.dst_visibility.map(ExactVisibility::as_str), + "criteria": daily_active_source_criteria(), + }); + } json!({ "version": 1, "kind": "flows", @@ -389,6 +508,23 @@ impl FlowSelection { } } +fn daily_active_source_criteria() -> Value { + json!({ + "address_side": "source", + "ip_version": 4, + "protocols": [6, 17], + "minimum_source_port": 1024, + "destination_ports": "all", + "tcp_flags": "all", + "activity_window": "local_day", + "minimum_flows": DAILY_ACTIVE_MIN_FLOWS, + "minimum_packets": DAILY_ACTIVE_MIN_PACKETS, + "minimum_bytes": DAILY_ACTIVE_MIN_BYTES, + "union_across_physical_sources": true, + "requires_complete_physical_day": true, + }) +} + fn optional_non_empty_string<'a>( object: &'a Map, key: &'static str, @@ -516,6 +652,11 @@ impl AddressSet { self.0.iter() } + #[must_use] + pub fn contains(&self, address: &IpAddr) -> bool { + self.0.contains(address) + } + #[must_use] pub fn len(&self) -> usize { self.0.len() @@ -1165,6 +1306,54 @@ impl StatisticalBucket { } } + /// Consume this builder into its canonical representation without cloning + /// the large aggregate collections. + #[must_use] + pub fn finish_owned(self) -> CanonicalBucket { + let Self { + key, + coverage, + traffic, + protocols, + addresses, + ports, + five_minute_starts, + } = self; + + CanonicalBucket { + key, + coverage, + traffic: traffic + .into_iter() + .map(|(scope, metrics)| ScopedTraffic { scope, metrics }) + .collect(), + protocols: protocols + .into_iter() + .map(|(scope, protocols)| ScopedProtocols { + scope, + protocols: protocols.into_iter().collect(), + }) + .collect(), + addresses: addresses + .into_iter() + .map(|((scope, address_side), addresses)| ScopedAddresses { + scope, + address_side, + addresses, + }) + .collect(), + ports: ports + .into_iter() + .map(|((scope, port_side), ports)| ScopedPorts { + scope, + port_side, + ports, + }) + .collect(), + five_minute_starts, + } + } + fn add_observation(&mut self, observation: FlowObservation) -> Result<(), DomainError> { if IpVersion::of(observation.src_ip) != IpVersion::of(observation.dst_ip) { return Err(DomainError::MixedIpVersions); @@ -1341,11 +1530,11 @@ mod tests { let wrong_visibility = FlowObservation::new(matching.src_ip, matching.dst_ip, 6, 1, 100, 0).unwrap(); - assert!(selection.matches(&matching)); - assert!(!selection.matches(&wrong_visibility)); + assert!(selection.matches_qualifying_flow(&matching)); + assert!(!selection.matches_qualifying_flow(&wrong_visibility)); assert_eq!(selection.src_visibility(), Some(ExactVisibility::Literal)); assert_eq!( - selection.nfdump_prefix_filter().as_deref(), + selection.nfdump_filter().as_deref(), Some("net 192.0.2.0/24") ); assert_eq!( @@ -1360,6 +1549,140 @@ mod tests { ); } + #[test] + fn daily_active_source_selection_preserves_the_finalized_definition() { + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.99.1/16" + }))) + .unwrap(); + let matching = FlowObservation::new( + address([0, 220, 1, 2]), + address([198, 51, 100, 2]), + 6, + 20, + 2_000, + 2, + ) + .unwrap() + .with_ports(Some(55_000), Some(443)); + let low_source_port = matching.clone().with_ports(Some(443), Some(55_000)); + let boundary_source_port = matching.clone().with_ports(Some(1_024), Some(65_535)); + let icmp = FlowObservation::new( + address([0, 220, 1, 2]), + address([198, 51, 100, 2]), + 1, + 20, + 2_000, + 2, + ) + .unwrap() + .with_ports(None, None); + let literal_source = FlowObservation::new( + address([0, 220, 1, 2]), + address([198, 51, 100, 2]), + 17, + 20, + 2_000, + 0, + ) + .unwrap() + .with_ports(Some(55_000), Some(53)); + let destination_only = FlowObservation::new( + address([198, 51, 100, 2]), + address([0, 220, 1, 2]), + 6, + 20, + 2_000, + 1, + ) + .unwrap() + .with_ports(Some(55_000), Some(443)); + + assert!(selection.matches_qualifying_flow(&matching)); + assert!(selection.matches_qualifying_flow(&boundary_source_port)); + assert!(!selection.matches_qualifying_flow(&low_source_port)); + assert!(!selection.matches_qualifying_flow(&icmp)); + assert!(!selection.matches_qualifying_flow(&literal_source)); + assert!(!selection.matches_qualifying_flow(&destination_only)); + assert!(selection.selects_daily_active_sources()); + assert_eq!( + selection.src_visibility(), + Some(ExactVisibility::Anonymized) + ); + assert_eq!( + selection.nfdump_filter().as_deref(), + Some( + "(src net 0.220.0.0/16 and ipv4 and (proto tcp or proto udp) and src port > 1023) or (src tun net 0.220.0.0/16 and (tun proto tcp or tun proto udp) and src port > 1023)" + ) + ); + assert!(FlowSelection::daily_activity_threshold_met(3, 20, 2_000)); + assert!(!FlowSelection::daily_activity_threshold_met(2, 20, 2_000)); + assert!(!FlowSelection::daily_activity_threshold_met(3, 19, 2_000)); + assert!(!FlowSelection::daily_activity_threshold_met(3, 20, 1_999)); + + let normalized = json!({ + "version": 1, + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16", + "src_visibility": "anonymized", + "dst_visibility": null, + "criteria": { + "address_side": "source", + "ip_version": 4, + "protocols": [6, 17], + "minimum_source_port": 1024, + "destination_ports": "all", + "tcp_flags": "all", + "activity_window": "local_day", + "minimum_flows": 3, + "minimum_packets": 20, + "minimum_bytes": 2000, + "union_across_physical_sources": true, + "requires_complete_physical_day": true + } + }); + assert_eq!(selection.normalized_payload(), normalized); + assert_eq!( + FlowSelection::from_payload(Some(&normalized)).unwrap(), + selection + ); + } + + #[test] + fn daily_active_source_selection_rejects_semantic_drift() { + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "2001:db8::/32" + }))), + Err(DomainError::DailyActiveSourcesRequireIpv4Prefix) + ); + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/24" + }))), + Err(DomainError::DailyActiveSourcesRequireIpv4Prefix) + ); + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16", + "dst_visibility": "literal" + }))), + Err(DomainError::InvalidDailyActiveSourceVisibility) + ); + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16", + "criteria": {"minimum_flows": 4} + }))), + Err(DomainError::InvalidDailyActiveSourceCriteria) + ); + } + #[test] fn selection_rejects_unknown_keys_and_invalid_all_criteria() { assert_eq!( @@ -1422,6 +1745,40 @@ mod tests { ); } + #[test] + fn consuming_finalizer_matches_borrowing_finalizer_for_dense_and_sparse_buckets() { + let dense = StatisticalBucket::dense(key(Granularity::FiveMinutes, 0, 300)); + let dense_expected = dense.finish(); + assert_eq!(dense_expected, dense.finish_owned()); + + let mut sparse = StatisticalBucket::new(key(Granularity::ThirtyMinutes, 0, 1_800)); + sparse + .add( + observation([192, 0, 2, 1], [198, 51, 100, 1], 6, 2) + .with_ports(Some(53), Some(1_024)), + ) + .unwrap(); + sparse + .add(GroupedTrafficFact { + ip_version: IpVersion::V4, + protocol: 17, + src_tos: 0, + flows: 2, + packets: 4, + bytes: 40, + }) + .unwrap(); + sparse + .add(ScopedAddressesFact::new( + Scope::new(IpVersion::V4, Visibility::Literal, Visibility::Anonymized), + AddressSide::Destination, + [address([203, 0, 113, 1]), address([203, 0, 113, 2])], + )) + .unwrap(); + let sparse_expected = sparse.finish(); + assert_eq!(sparse_expected, sparse.finish_owned()); + } + #[test] fn bucket_coverage_is_explicit_and_additive_across_rollups() { let partial = StatisticalBucket::new(key(Granularity::FiveMinutes, 0, 300)) diff --git a/tools/netflow-db/src/ingest.rs b/tools/netflow-db/src/ingest.rs index 306963a..2a9da68 100644 --- a/tools/netflow-db/src/ingest.rs +++ b/tools/netflow-db/src/ingest.rs @@ -1,13 +1,14 @@ //! Streaming adapters that turn external CSV inputs into canonical five-minute buckets. use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashMap}, ffi::OsString, fs, io::{BufReader, Read, Seek, SeekFrom}, + net::IpAddr, path::{Path, PathBuf}, process::{Command, Stdio}, - sync::mpsc, + sync::{Arc, mpsc}, thread, time::{Duration, Instant}, }; @@ -24,8 +25,8 @@ use crate::{ config::{CsvSourceConfig, InputOrder}, coverage::BucketCoverage, domain::{ - BucketKey, CanonicalBucket, DomainError, FlowObservation, FlowSelection, Granularity, - StatisticalBucket, + AddressSet, BucketKey, CanonicalBucket, DomainError, FlowObservation, FlowSelection, + Granularity, StatisticalBucket, }, nfdump, normalize::{NormalizeError, field_indexes, normalize_csv_values}, @@ -33,6 +34,8 @@ use crate::{ const BUCKET_SECONDS: i64 = 300; const NFDUMP_TIMEOUT: Duration = Duration::from_secs(300); +const NFDUMP_DAY_TIMEOUT: Duration = Duration::from_secs(3_600); +const NFDUMP_PROBE_TIMEOUT: Duration = Duration::from_secs(10); const MAX_DIAGNOSTIC_BYTES: usize = 64 * 1024; const TIMESTAMP_KEYS: [&str; 3] = ["time_received", "time_end", "time_start"]; @@ -113,6 +116,9 @@ pub struct NfcapdInputSpec { pub bucket_start: i64, } +/// One coordinated daily selection and its resolved source set. +pub type NfcapdSelectionAndActiveSources = (FlowSelection, Arc); + /// Discover configured CSV inputs under one flat directory. pub fn discover_csv_inputs( root: impl AsRef, @@ -388,7 +394,7 @@ fn scan_csv_reader( match normalize_csv_values(&values, config, &indexes) { Ok(row) => { state.mark_valid(&row.source_id, row.bucket_start)?; - if selection.matches(&row.observation) { + if selection.matches_qualifying_flow(&row.observation) { state.accept(row)?; } } @@ -1131,12 +1137,243 @@ pub fn build_nfdump_command( "-o".into(), nfdump::OUTPUT_MODE.into(), ]; - if let Some(filter) = selection.nfdump_prefix_filter() { + if let Some(filter) = selection.nfdump_filter() { command.push(filter.into()); } command } +/// Build one nfcapd command for several daily active-source selections. +pub fn build_nfdump_command_for_selections( + path: impl AsRef, + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result, IngestError> { + let mut command = vec![ + executable.as_ref().to_owned(), + "-r".into(), + path.as_ref().as_os_str().to_owned(), + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + command.push(daily_active_union_filter(selections)?.into()); + Ok(command) +} + +/// Build one range command for a complete physical member day. +pub fn build_nfdump_range_command( + paths: &[PathBuf], + selection: &FlowSelection, + executable: impl AsRef, +) -> Result, IngestError> { + build_nfdump_range_command_for_selections(paths, std::slice::from_ref(selection), executable) +} + +/// Build one range command for a complete physical member day and several selections. +pub fn build_nfdump_range_command_for_selections( + paths: &[PathBuf], + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result, IngestError> { + let first = paths + .first() + .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; + let last = paths.last().expect("nonempty range has a last path"); + if paths.iter().any(|path| path.parent() != first.parent()) { + return Err(IngestError::InvalidInput(format!( + "nfcapd day range spans directories, including {} and {}", + first.display(), + last.display() + ))); + } + if paths.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(IngestError::InvalidInput( + "nfcapd day range paths must be strictly chronological".into(), + )); + } + first.file_name().ok_or_else(|| { + IngestError::InvalidInput(format!("invalid nfcapd path: {}", first.display())) + })?; + let last_name = last.file_name().ok_or_else(|| { + IngestError::InvalidInput(format!("invalid nfcapd path: {}", last.display())) + })?; + let mut range = first.to_path_buf().into_os_string(); + range.push(":"); + range.push(last_name); + let mut command = vec![ + executable.as_ref().to_owned(), + "-R".into(), + range, + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + command.push(daily_active_union_filter(selections)?.into()); + Ok(command) +} + +fn daily_active_union_filter(selections: &[FlowSelection]) -> Result { + if selections.is_empty() { + return Err(IngestError::InvalidInput( + "daily active-source selections are empty".into(), + )); + } + let mut prefixes = selections + .iter() + .map(|selection| { + if !selection.selects_daily_active_sources() { + return Err(IngestError::InvalidInput( + "nfdump subset decoding requires daily_active_sources selections".into(), + )); + } + selection + .ip_prefix() + .map(ToString::to_string) + .ok_or_else(|| { + IngestError::InvalidInput( + "daily_active_sources selection is missing an ip_prefix".into(), + ) + }) + }) + .collect::, _>>()?; + prefixes.sort_unstable(); + prefixes.dedup(); + let source_filter = |qualifier: &str| { + if prefixes.len() == 1 { + format!("{qualifier} net {}", prefixes[0]) + } else { + format!( + "({})", + prefixes + .iter() + .map(|prefix| format!("{qualifier} net {prefix}")) + .collect::>() + .join(" or ") + ) + } + }; + let outer_source_filter = source_filter("src"); + let tunnel_source_filter = source_filter("src tun"); + Ok(format!( + "({outer_source_filter} and ipv4 and (proto tcp or proto udp) and src port > 1023) or ({tunnel_source_filter} and (tun proto tcp or tun proto udp) and src port > 1023)" + )) +} + +/// Create a private nfdump input directory containing exactly the requested captures. +/// +/// `nfdump -R first:last` selects every alphabetically intervening file. A manifest +/// directory keeps the pinned nfdump range reader while making the input membership +/// explicit and bounded to the paths discovered by the pipeline. +fn prepare_nfcapd_manifest(paths: &[PathBuf]) -> Result<(tempfile::TempDir, PathBuf), IngestError> { + let context = paths + .first() + .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; + if paths.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(IngestError::InvalidInput( + "nfcapd day range paths must be strictly chronological".into(), + )); + } + let manifest = tempfile::Builder::new() + .prefix("atlantis-nfcapd-") + .tempdir() + .map_err(|source| IngestError::Io { + path: context.clone(), + source, + })?; + for (index, path) in paths.iter().enumerate() { + let target = if path.is_absolute() { + path.clone() + } else { + std::env::current_dir() + .map_err(|source| IngestError::Io { + path: path.clone(), + source, + })? + .join(path) + }; + let link = manifest.path().join(format!("nfcapd.{index:020}")); + link_nfcapd_manifest_entry(&target, &link).map_err(|source| IngestError::Io { + path: path.clone(), + source, + })?; + } + let manifest_path = manifest.path().to_owned(); + Ok((manifest, manifest_path)) +} + +#[cfg(unix)] +fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(target, link) +} + +#[cfg(windows)] +fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { + std::os::windows::fs::symlink_file(target, link) +} + +#[cfg(not(any(unix, windows)))] +fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { + fs::hard_link(target, link) +} + +fn build_nfdump_manifest_command_for_selections( + manifest: &Path, + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result, IngestError> { + let mut command = vec![ + executable.as_ref().to_owned(), + "-R".into(), + manifest.as_os_str().to_owned(), + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + command.push(daily_active_union_filter(selections)?.into()); + Ok(command) +} + +/// Prove that an executable implements the private Atlantis output contract before any pipeline +/// output is created. An empty `-R` directory makes the probe independent of capture contents, +/// while the normal streaming decoder still enforces the header, terminator, and EOF contract. +pub(crate) fn probe_nfdump_compatibility( + executable: impl AsRef, +) -> Result<(), IngestError> { + let manifest = tempfile::tempdir().map_err(|source| IngestError::Io { + path: PathBuf::from(""), + source, + })?; + let command = vec![ + executable.as_ref().to_owned(), + "-R".into(), + manifest.path().as_os_str().to_owned(), + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + let key = BucketKey::new("", Granularity::FiveMinutes, 0, BUCKET_SECONDS); + let bucket = run_nfdump( + command, + manifest.path(), + NFDUMP_PROBE_TIMEOUT, + move |stdout| nfdump::reduce_to_bucket(stdout, key, &FlowSelection::default()), + ) + .map_err(|error| { + IngestError::InvalidInput(format!( + "nfdump compatibility probe for {:?} failed: {error}", + executable.as_ref() + )) + })?; + if bucket.traffic.iter().any(|scope| scope.metrics.flows != 0) { + return Err(IngestError::InvalidInput(format!( + "nfdump compatibility probe for {:?} emitted a non-empty Atlantis stream", + executable.as_ref() + ))); + } + Ok(()) +} + /// Decode one canonical nfcapd file into its dense five-minute bucket. pub fn read_nfcapd_bucket( path: impl AsRef, @@ -1172,6 +1409,123 @@ fn read_nfcapd_bucket_with_timeout( bucket_start + BUCKET_SECONDS, ); let command = build_nfdump_command(path, selection, executable.as_ref()); + let selection = selection.clone(); + run_nfdump(command, path, timeout, move |stdout| { + nfdump::reduce_to_bucket(stdout, key, &selection) + }) +} + +pub fn read_nfcapd_bucket_with_active_sources( + path: impl AsRef, + source_id: &str, + selection: &FlowSelection, + active_sources: Arc, + executable: impl AsRef, + timezone: &str, +) -> Result { + let path = path.as_ref(); + let bucket_start = parse_nfcapd_bucket_start(path, timezone)?; + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + BUCKET_SECONDS, + ); + let command = build_nfdump_command(path, selection, executable.as_ref()); + let selection = selection.clone(); + run_nfdump(command, path, NFDUMP_TIMEOUT, move |stdout| { + nfdump::reduce_to_bucket_with_active_sources( + stdout, + key, + &selection, + active_sources.as_ref(), + ) + }) +} + +pub fn read_nfcapd_buckets_with_active_sources( + path: impl AsRef, + source_id: &str, + selections_and_active_sources: &[NfcapdSelectionAndActiveSources], + executable: impl AsRef, + timezone: &str, +) -> Result, IngestError> { + if selections_and_active_sources.is_empty() { + return Err(IngestError::InvalidInput( + "daily active-source selection pairs are empty".into(), + )); + } + let path = path.as_ref(); + let bucket_start = parse_nfcapd_bucket_start(path, timezone)?; + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + BUCKET_SECONDS, + ); + let command = build_nfdump_command_for_selections( + path, + &selections_and_active_sources + .iter() + .map(|(selection, _)| selection.clone()) + .collect::>(), + executable.as_ref(), + )?; + let selections_and_active_sources = selections_and_active_sources.to_vec(); + run_nfdump(command, path, NFDUMP_TIMEOUT, move |stdout| { + nfdump::reduce_to_buckets_with_active_sources(stdout, key, &selections_and_active_sources) + }) +} + +pub(crate) fn read_nfcapd_daily_source_activities( + paths: &[PathBuf], + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result>, IngestError> { + let context = paths + .first() + .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; + let (_manifest, manifest_path) = prepare_nfcapd_manifest(paths)?; + let command = + build_nfdump_manifest_command_for_selections(&manifest_path, selections, executable)?; + let selections = selections.to_vec(); + run_nfdump(command, context, NFDUMP_DAY_TIMEOUT, move |stdout| { + nfdump::reduce_to_daily_source_activities(stdout, &selections) + }) +} + +pub(crate) fn read_nfcapd_daily_source_activity( + paths: &[PathBuf], + selection: &FlowSelection, + executable: impl AsRef, +) -> Result, IngestError> { + let context = paths + .first() + .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; + let (_manifest, manifest_path) = prepare_nfcapd_manifest(paths)?; + let command = build_nfdump_manifest_command_for_selections( + &manifest_path, + std::slice::from_ref(selection), + executable, + )?; + let selection = selection.clone(); + run_nfdump(command, context, NFDUMP_DAY_TIMEOUT, move |stdout| { + nfdump::reduce_to_daily_source_activity(stdout, &selection) + }) +} + +fn run_nfdump( + command: Vec, + path: &Path, + timeout: Duration, + decode: F, +) -> Result +where + T: Send + 'static, + F: FnOnce(BufReader) -> Result + + Send + + 'static, +{ let executable_name = command[0].clone(); let stderr_file = tempfile::tempfile().map_err(|source| IngestError::Io { path: path.to_owned(), @@ -1195,10 +1549,9 @@ fn read_nfcapd_bucket_with_timeout( .stdout .take() .ok_or_else(|| IngestError::InvalidInput("nfdump stdout was not captured".into()))?; - let selection = selection.clone(); let (sender, receiver) = mpsc::sync_channel(1); let reader = thread::spawn(move || { - let result = nfdump::reduce_to_bucket(BufReader::new(stdout), key, &selection); + let result = decode(BufReader::new(stdout)); let _ = sender.send(result); }); let deadline = Instant::now() + timeout; @@ -1283,7 +1636,7 @@ fn read_tail(mut file: fs::File, limit: usize) -> std::io::Result { #[cfg(test)] mod tests { - use std::{collections::BTreeMap, fs, io::Write}; + use std::{collections::BTreeMap, fs, io::Write, sync::Arc}; use flate2::{Compression, write::GzEncoder}; use serde_json::json; @@ -1613,6 +1966,457 @@ mod tests { ); } + #[test] + fn daily_activity_command_uses_one_source_only_member_range() { + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16" + }))) + .unwrap(); + let paths = [ + PathBuf::from("/captures/edge/nfcapd.202506010000"), + PathBuf::from("/captures/edge/nfcapd.202506012355"), + ]; + + let command = build_nfdump_range_command(&paths, &selection, "nfdump").unwrap(); + + assert_eq!( + command, + [ + "nfdump", + "-R", + "/captures/edge/nfcapd.202506010000:nfcapd.202506012355", + "-q", + "-o", + "atlantis", + "(src net 0.220.0.0/16 and ipv4 and (proto tcp or proto udp) and src port > 1023) or (src tun net 0.220.0.0/16 and (tun proto tcp or tun proto udp) and src port > 1023)", + ] + .map(OsString::from) + ); + let filter = command.last().unwrap().to_string_lossy(); + assert!(!filter.contains("dst port")); + assert!(!filter.contains("flags")); + } + + #[test] + fn multi_daily_activity_command_unions_prefixes_and_keeps_fixed_filter() { + let selections = [ + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16", + }))) + .unwrap(), + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(), + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(), + ]; + let paths = [ + PathBuf::from("/captures/edge/nfcapd.202506010000"), + PathBuf::from("/captures/edge/nfcapd.202506012355"), + ]; + + let command = + build_nfdump_range_command_for_selections(&paths, &selections, "nfdump").unwrap(); + + assert_eq!( + command.last().unwrap().to_string_lossy(), + "((src net 192.0.0.0/16 or src net 198.51.0.0/16) and ipv4 and (proto tcp or proto udp) and src port > 1023) or ((src tun net 192.0.0.0/16 or src tun net 198.51.0.0/16) and (tun proto tcp or tun proto udp) and src port > 1023)" + ); + } + + #[test] + fn multi_nfdump_commands_reject_empty_and_non_daily_selection_sets() { + let paths = [PathBuf::from("/captures/edge/nfcapd.202506010000")]; + assert!(matches!( + build_nfdump_range_command_for_selections(&paths, &[], "nfdump"), + Err(IngestError::InvalidInput(message)) if message.contains("empty") + )); + assert!(matches!( + build_nfdump_command_for_selections("capture", &[FlowSelection::default()], "nfdump"), + Err(IngestError::InvalidInput(message)) if message.contains("daily_active_sources") + )); + } + + #[cfg(unix)] + #[test] + fn multi_bucket_reader_decodes_one_process_into_one_bucket_per_pair() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let invocation_log = directory.path().join("invocations.log"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 32..16 + 40].copy_from_slice(&20_u64.to_le_bytes()); + binary[16 + 40..16 + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + binary[16 + 48..16 + 56].copy_from_slice(&3_u64.to_le_bytes()); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + invocation_log.display(), + stream.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(); + let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); + let pairs = [ + ( + selection.clone(), + Arc::new([source].into_iter().collect::()), + ), + ( + selection, + Arc::new([source].into_iter().collect::()), + ), + ]; + + let buckets = read_nfcapd_buckets_with_active_sources( + &capture, + "edge-a", + &pairs, + &executable, + "America/Los_Angeles", + ) + .unwrap(); + + assert_eq!(buckets.len(), 2); + for bucket in buckets { + assert_eq!( + bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics + .flows, + 3 + ); + } + assert_eq!( + fs::read_to_string(invocation_log).unwrap().lines().count(), + 1 + ); + } + + #[cfg(unix)] + #[test] + fn single_bucket_reader_accepts_shared_active_source_set() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!("#!/bin/sh\ncat '{}'\n", stream.display()), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(); + let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); + let active_sources = Arc::new([source].into_iter().collect::()); + + let bucket = read_nfcapd_bucket_with_active_sources( + &capture, + "edge-a", + &selection, + active_sources, + &executable, + "America/Los_Angeles", + ) + .unwrap(); + + assert_eq!( + bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics + .flows, + 3 + ); + } + + #[cfg(unix)] + #[test] + fn multi_daily_activity_reader_decodes_one_range_into_distinct_maps() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let invocation_log = directory.path().join("invocations.log"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 32..16 + 40].copy_from_slice(&20_u64.to_le_bytes()); + binary[16 + 40..16 + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + binary[16 + 48..16 + 56].copy_from_slice(&3_u64.to_le_bytes()); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + invocation_log.display(), + stream.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selections = [ + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(), + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16", + }))) + .unwrap(), + ]; + + let activities = read_nfcapd_daily_source_activities( + std::slice::from_ref(&capture), + &selections, + &executable, + ) + .unwrap(); + + assert_eq!(activities.len(), 2); + assert_eq!(activities[0].len(), 1); + assert!(activities[1].is_empty()); + assert_eq!( + fs::read_to_string(invocation_log).unwrap().lines().count(), + 1 + ); + } + + #[cfg(unix)] + #[test] + fn daily_activity_reader_uses_only_the_discovered_capture_paths() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let manifest_members = directory.path().join("manifest-members.log"); + let manifest_path = directory.path().join("manifest-path.log"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + let first = directory.path().join("nfcapd.202506010000"); + let second = directory.path().join("nfcapd.202506010010"); + fs::write(&first, "selected first").unwrap(); + fs::write(&second, "selected second").unwrap(); + // These files are alphabetically between the selected captures. A physical + // -R first:last range would read them even though they were not snapshotted. + fs::write( + directory.path().join("nfcapd.202506010005"), + "untracked valid capture", + ) + .unwrap(); + fs::write( + directory.path().join("nfcapd.202506010007.backup"), + "untracked backup", + ) + .unwrap(); + fs::write( + directory.path().join("nfcapd.202506010008.aria2"), + "untracked sidecar", + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\n\ + set -eu\n\ + manifest=\"\"\n\ + while [ \"$#\" -gt 0 ]; do\n\ + if [ \"$1\" = \"-R\" ]; then manifest=\"$2\"; shift 2; else shift; fi\n\ + done\n\ + test -n \"$manifest\"\n\ + test -d \"$manifest\"\n\ + printf '%s\\n' \"$manifest\" > '{}'\n\ + for member in \"$manifest\"/*; do\n\ + test -L \"$member\"\n\ + readlink \"$member\" >> '{}'\n\ + done\n\ + cat '{}'\n", + manifest_path.display(), + manifest_members.display(), + stream.display(), + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(); + let activity = read_nfcapd_daily_source_activity( + &[first.clone(), second.clone()], + &selection, + &executable, + ) + .unwrap(); + + let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); + assert_eq!(activity[&source].flows, 3); + assert_eq!( + fs::read_to_string(manifest_members).unwrap(), + format!("{}\n{}\n", first.display(), second.display()) + ); + let manifest_directory = fs::read_to_string(manifest_path).unwrap(); + assert!(!Path::new(manifest_directory.trim()).exists()); + } + + #[cfg(unix)] + #[test] + fn daily_activity_reader_keeps_a_qualifying_synthetic_tunnel_flow_once() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let invocation_log = directory.path().join("invocation.log"); + + let mut synthetic: [u8; 72] = ONE_V4_BINARY_STREAM[16..88].try_into().unwrap(); + synthetic[..16].fill(0); + synthetic[..4].copy_from_slice(&[10, 0, 0, 1]); + synthetic[16..32].fill(0); + synthetic[16..20].copy_from_slice(&[10, 0, 0, 2]); + synthetic[32..40].copy_from_slice(&210_u64.to_le_bytes()); + synthetic[40..48].copy_from_slice(&4_467_904_u64.to_le_bytes()); + synthetic[48..56].copy_from_slice(&1_u64.to_le_bytes()); + synthetic[64..66].copy_from_slice(&22_222_u16.to_le_bytes()); + synthetic[66..68].copy_from_slice(&80_u16.to_le_bytes()); + synthetic[68] = 6; + synthetic[69] = 0b010; + synthetic[70..72].fill(0); + + let mut outer = synthetic; + outer[..16].fill(0); + outer[..4].copy_from_slice(&[72, 138, 170, 101]); + outer[16..32].fill(0); + outer[16..20].copy_from_slice(&[42, 16, 32, 6]); + outer[48..56].copy_from_slice(&7_u64.to_le_bytes()); + outer[70..72].copy_from_slice(&[40, 255]); + + let mut binary = Vec::new(); + binary.extend_from_slice(&ONE_V4_BINARY_STREAM[..12]); + binary.extend_from_slice(&2_u32.to_le_bytes()); + binary.extend_from_slice(&synthetic); + binary.extend_from_slice(&outer); + binary.extend_from_slice(&[0, 0, 0, 0]); + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nset -eu\nprintf '%s\\n' \"$*\" > '{}'\ncat '{}'\n", + invocation_log.display(), + stream.display(), + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "10.0.0.0/16", + }))) + .unwrap(); + + let activity = read_nfcapd_daily_source_activity( + std::slice::from_ref(&capture), + &selection, + &executable, + ) + .unwrap(); + + let source = IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 1)); + assert_eq!(activity.len(), 1); + assert_eq!( + activity[&source], + nfdump::SourceActivity { + flows: 1, + packets: 20, + bytes: 2_000, + } + ); + + let active_sources = Arc::new([source].into_iter().collect::()); + let bucket = read_nfcapd_bucket_with_active_sources( + &capture, + "edge-a", + &selection, + active_sources, + &executable, + "America/Los_Angeles", + ) + .unwrap(); + let metrics = &bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics; + assert_eq!(metrics.flows, 1); + assert_eq!(metrics.packets, 210); + assert_eq!(metrics.bytes, 4_467_904); + + let invocation = fs::read_to_string(invocation_log).unwrap(); + assert!(invocation.contains("src net 10.0.0.0/16")); + assert!(invocation.contains("src tun net 10.0.0.0/16")); + assert!(invocation.contains("tun proto tcp or tun proto udp")); + } + #[cfg(unix)] #[test] fn nfdump_decoder_builds_the_canonical_bucket() { diff --git a/tools/netflow-db/src/maad.rs b/tools/netflow-db/src/maad.rs index 0331bf9..5c07458 100644 --- a/tools/netflow-db/src/maad.rs +++ b/tools/netflow-db/src/maad.rs @@ -136,7 +136,7 @@ pub fn compute_with_config( if addresses.len() < MIN_MAAD_ADDRESSES { return Ok(empty_result(addresses.len())); } - let counts = build_prefix_counts(&addresses); + let counts = build_prefix_counts(&addresses, config.max_prefix_length); let prepared = prepare_valid_moments(&counts, &config); if prepared.is_empty() { return Ok(empty_result(addresses.len())); @@ -182,9 +182,13 @@ fn empty_result(total_addrs: usize) -> MaadResult { } } -fn build_prefix_counts(addresses: &[u32]) -> Vec> { - let mut counts = Vec::with_capacity(33); - for prefix_length in 0..=32_u8 { +fn build_prefix_counts(addresses: &[u32], max_prefix_length: u8) -> Vec> { + // Moment preparation needs each configured parent level and its children; + // dimensions only read the configured parent levels. Do not retain the + // unused /32 level range for the default /24 analysis. + let count_levels = usize::from(max_prefix_length) + 2; + let mut counts = Vec::with_capacity(count_levels); + for prefix_length in 0..=max_prefix_length + 1 { let mut prefixes = Vec::new(); for &address in addresses { let prefix = prefix_of(address, prefix_length); @@ -930,6 +934,32 @@ mod tests { assert_eq!(result.structure.len(), 33); } + #[test] + fn prefix_counts_stop_after_the_configured_parent_and_child_levels() { + let addresses = [ + u32::from(Ipv4Addr::new(192, 0, 2, 1)), + u32::from(Ipv4Addr::new(192, 0, 2, 2)), + ]; + + let default_counts = + build_prefix_counts(&addresses, MaadConfig::default().max_prefix_length); + assert_eq!(default_counts.len(), 26); + assert_eq!( + default_counts.last().unwrap(), + &vec![(prefix_of(addresses[0], 25), 2)] + ); + + let deepest_counts = build_prefix_counts(&addresses, 31); + assert_eq!(deepest_counts.len(), 33); + assert_eq!( + deepest_counts.last().unwrap(), + &vec![ + (u32::from(Ipv4Addr::new(192, 0, 2, 1)), 1), + (u32::from(Ipv4Addr::new(192, 0, 2, 2)), 1), + ] + ); + } + #[test] fn configurable_q_grid_is_uniform_and_includes_dimension_qs() { let config = MaadConfig { diff --git a/tools/netflow-db/src/main.rs b/tools/netflow-db/src/main.rs index e047d5b..9525fe5 100644 --- a/tools/netflow-db/src/main.rs +++ b/tools/netflow-db/src/main.rs @@ -61,8 +61,9 @@ enum Command { struct PipelineArgs { #[arg(long, conflicts_with = "dataset")] config: Option, + /// Dataset ID. Repeat --dataset for two or more values to start one coordinated fixed daily-active subset run; one value keeps the normal single-dataset path. #[arg(long, requires = "start_date", conflicts_with = "config")] - dataset: Option, + dataset: Vec, #[arg(long)] start_date: Option, #[arg(long)] @@ -77,6 +78,13 @@ struct PipelineArgs { datasets: Option, #[arg(long)] ip_prefix: Option, + /// Select qualifying flows from sources active over each complete local day. + #[arg( + long, + requires = "ip_prefix", + conflicts_with_all = ["src_visibility", "dst_visibility"] + )] + daily_active_sources: bool, #[arg(long, value_enum)] src_visibility: Option, #[arg(long, value_enum)] @@ -328,14 +336,24 @@ fn main() -> Result<()> { } fn run_pipeline(args: PipelineArgs) -> Result<()> { - let selection = serde_json::json!({ - "ip_prefix": args.ip_prefix, - "src_visibility": args.src_visibility.map(VisibilityArg::as_str), - "dst_visibility": args.dst_visibility.map(VisibilityArg::as_str), - }); - let report = netflow_db::pipeline::run(netflow_db::pipeline::PipelineRequest { + let mut selection = serde_json::Map::new(); + if args.daily_active_sources { + selection.insert("kind".into(), serde_json::json!("daily_active_sources")); + } + selection.insert("ip_prefix".into(), serde_json::json!(args.ip_prefix)); + selection.insert( + "src_visibility".into(), + serde_json::json!(args.src_visibility.map(VisibilityArg::as_str)), + ); + selection.insert( + "dst_visibility".into(), + serde_json::json!(args.dst_visibility.map(VisibilityArg::as_str)), + ); + let selection = serde_json::Value::Object(selection); + let dataset_ids = args.dataset; + let request = netflow_db::pipeline::PipelineRequest { config_path: args.config, - dataset_id: args.dataset, + dataset_id: (dataset_ids.len() == 1).then(|| dataset_ids[0].clone()), datasets_path: args.datasets, start_date: args.start_date, end_date: args.end_date, @@ -347,13 +365,25 @@ fn run_pipeline(args: PipelineArgs) -> Result<()> { force: args.force, run_maad: !args.no_maad, require_complete: args.require_complete, - })?; + }; + let report = if dataset_ids.len() > 1 { + netflow_db::pipeline::run_many(request, dataset_ids)? + } else { + netflow_db::pipeline::run(request)? + }; println!( "Five-minute coverage: {} complete, {} partial, {} unknown", report.complete_five_minute_buckets, report.partial_five_minute_buckets, report.unknown_five_minute_buckets ); + println!( + "Published five-minute buckets: {}", + report.five_minute_buckets + ); + if report.skipped_inputs != 0 { + println!("Skipped inputs: {}", report.skipped_inputs); + } Ok(()) } diff --git a/tools/netflow-db/src/nfdump.rs b/tools/netflow-db/src/nfdump.rs index 2fcbcaa..403cbe8 100644 --- a/tools/netflow-db/src/nfdump.rs +++ b/tools/netflow-db/src/nfdump.rs @@ -1,16 +1,18 @@ //! Private decoder for the Atlantis Flow Stream emitted by the nfdump fork. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::fmt; use std::io::{self, Read}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::sync::Arc; use fixedbitset::FixedBitSet; use crate::{ coverage::BucketCoverage, domain::{ - AddressSet, AddressSide, BucketKey, CanonicalBucket, ExactVisibility, FlowSelection, + AddressSet, AddressSide, BucketKey, CanonicalBucket, DAILY_ACTIVE_MIN_BYTES, + DAILY_ACTIVE_MIN_FLOWS, DAILY_ACTIVE_MIN_PACKETS, ExactVisibility, FlowSelection, Granularity, IpVersion, Scope, ScopedAddresses, ScopedPorts, ScopedProtocols, ScopedTraffic, TrafficMetrics, Visibility, }, @@ -130,6 +132,9 @@ enum ErrorReason { NonzeroIcmpDestinationPort(u16), InvalidTtlOrder { minimum: u8, maximum: u8 }, AggregateOverflow, + DailyActivityRequiresSelection, + MissingDailyActiveSources, + DailyActivityRequiresDailyActiveSourceSelection, } impl fmt::Display for ErrorReason { @@ -178,10 +183,42 @@ impl fmt::Display for ErrorReason { ) } Self::AggregateOverflow => formatter.write_str("exceeds signed 64-bit aggregate range"), + Self::DailyActivityRequiresSelection => { + formatter.write_str("daily source activity requires at least one selection") + } + Self::MissingDailyActiveSources => formatter + .write_str("daily active-source selection was not resolved for this local day"), + Self::DailyActivityRequiresDailyActiveSourceSelection => formatter + .write_str("daily source activity requires a daily_active_sources selection"), } } } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct SourceActivity { + pub flows: i64, + pub packets: i64, + pub bytes: i64, +} + +impl SourceActivity { + /// Qualification only needs threshold-capped counters, keeping addition overflow-free. + pub fn include(&mut self, other: Self) { + self.flows = self + .flows + .saturating_add(other.flows) + .min(DAILY_ACTIVE_MIN_FLOWS); + self.packets = self + .packets + .saturating_add(other.packets) + .min(DAILY_ACTIVE_MIN_PACKETS); + self.bytes = self + .bytes + .saturating_add(other.bytes) + .min(DAILY_ACTIVE_MIN_BYTES); + } +} + #[derive(Debug)] pub(crate) struct NfdumpError { phase: Phase, @@ -356,7 +393,7 @@ pub(crate) fn reduce_to_bucket( key: BucketKey, selection: &FlowSelection, ) -> Result { - match reduce_stream(&mut input, selection) { + match reduce_stream(&mut input, selection, None) { Ok(scopes) => Ok(finish_bucket(scopes, key)), Err(error) => { drain_to_eof(&mut input); @@ -365,10 +402,208 @@ pub(crate) fn reduce_to_bucket( } } +pub(crate) fn reduce_to_bucket_with_active_sources( + mut input: R, + key: BucketKey, + selection: &FlowSelection, + active_sources: &AddressSet, +) -> Result { + match reduce_stream(&mut input, selection, Some(active_sources)) { + Ok(scopes) => Ok(finish_bucket(scopes, key)), + Err(error) => { + drain_to_eof(&mut input); + Err(error) + } + } +} + +pub(crate) fn reduce_to_buckets_with_active_sources( + mut input: R, + key: BucketKey, + selections_and_active_sources: &[(FlowSelection, Arc)], +) -> Result, NfdumpError> { + match reduce_stream_for_active_sources(&mut input, selections_and_active_sources) { + Ok(scopes) => Ok(scopes + .into_iter() + .map(|scopes| finish_bucket(scopes, key.clone())) + .collect()), + Err(error) => { + drain_to_eof(&mut input); + Err(error) + } + } +} + +pub(crate) fn reduce_to_daily_source_activity( + mut input: R, + selection: &FlowSelection, +) -> Result, NfdumpError> { + if !selection.selects_daily_active_sources() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection, + )); + } + let mut activities = + reduce_to_daily_source_activities(&mut input, std::slice::from_ref(selection))?; + Ok(activities + .pop() + .expect("one daily selection produces one activity map")) +} + +pub(crate) fn reduce_to_daily_source_activities( + mut input: R, + selections: &[FlowSelection], +) -> Result>, NfdumpError> { + validate_daily_active_selections(selections)?; + let mut activities = (0..selections.len()) + .map(|_| HashMap::::new()) + .collect::>(); + let result = visit_stream(&mut input, |flow, _, _| { + for (selection, activity) in selections.iter().zip(&mut activities) { + if !matches_selection(flow, selection) { + continue; + } + let entry = activity.entry(flow.source_address).or_default(); + entry.include(SourceActivity { + flows: flow.flow_count, + packets: flow.packets, + bytes: flow.bytes, + }); + } + Ok(()) + }); + match result { + Ok(()) => Ok(activities), + Err(error) => { + drain_to_eof(&mut input); + Err(error) + } + } +} + +fn reduce_stream_for_active_sources( + input: &mut impl Read, + selections_and_active_sources: &[(FlowSelection, Arc)], +) -> Result, NfdumpError> { + validate_daily_active_selection_pairs(selections_and_active_sources)?; + let mut scopes = (0..selections_and_active_sources.len()) + .map(|_| std::array::from_fn(|_| ScopeAccumulator::default())) + .collect::>(); + visit_stream(input, |flow, block_index, record_ordinal| { + for ((selection, active_sources), scopes) in + selections_and_active_sources.iter().zip(&mut scopes) + { + if !matches_selection(flow, selection) || !active_sources.contains(&flow.source_address) + { + continue; + } + add_flow_to_scopes(scopes, flow, block_index, record_ordinal)?; + } + Ok(()) + })?; + Ok(scopes) +} + +fn validate_daily_active_selections(selections: &[FlowSelection]) -> Result<(), NfdumpError> { + if selections.is_empty() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresSelection, + )); + } + if selections + .iter() + .any(|selection| !selection.selects_daily_active_sources()) + { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection, + )); + } + Ok(()) +} + +fn validate_daily_active_selection_pairs( + selections_and_active_sources: &[(FlowSelection, Arc)], +) -> Result<(), NfdumpError> { + if selections_and_active_sources.is_empty() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresSelection, + )); + } + if selections_and_active_sources + .iter() + .any(|(selection, _)| !selection.selects_daily_active_sources()) + { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection, + )); + } + Ok(()) +} + fn reduce_stream( input: &mut R, selection: &FlowSelection, + active_sources: Option<&AddressSet>, ) -> Result<[ScopeAccumulator; 10], NfdumpError> { + if selection.selects_daily_active_sources() && active_sources.is_none() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::MissingDailyActiveSources, + )); + } + let mut scopes = std::array::from_fn(|_| ScopeAccumulator::default()); + visit_stream(input, |flow, block_index, record_ordinal| { + if !matches_selection(flow, selection) + || active_sources.is_some_and(|sources| !sources.contains(&flow.source_address)) + { + return Ok(()); + } + add_flow_to_scopes(&mut scopes, flow, block_index, record_ordinal) + })?; + Ok(scopes) +} + +fn add_flow_to_scopes( + scopes: &mut [ScopeAccumulator; 10], + flow: &Flow, + block_index: u64, + record_ordinal: u64, +) -> Result<(), NfdumpError> { + let family_base = if flow.ip_version == IpVersion::V4 { + 0 + } else { + 5 + }; + let exact_index = + family_base + exact_scope_index(flow.source_anonymized, flow.destination_anonymized); + for index in [family_base, exact_index] { + scopes[index].validate_add(flow).map_err(|field| { + NfdumpError::new(Phase::Aggregate, field, ErrorReason::AggregateOverflow) + .at_block(block_index) + .at_record(record_ordinal) + })?; + } + for index in [family_base, exact_index] { + scopes[index].add(flow); + } + Ok(()) +} + +fn visit_stream(input: &mut R, mut visit: F) -> Result<(), NfdumpError> +where + F: FnMut(&Flow, u64, u64) -> Result<(), NfdumpError>, +{ let mut header = [0_u8; 12]; let read = read_fully(input, &mut header).map_err(|failure| { NfdumpError::new( @@ -411,7 +646,6 @@ fn reduce_stream( )); } - let mut scopes = std::array::from_fn(|_| ScopeAccumulator::default()); let mut payload = [0_u8; MAX_BLOCK_BYTES]; let mut block_index = 1_u64; let mut record_ordinal = 0_u64; @@ -447,7 +681,7 @@ fn reduce_stream( let count = u32::from_le_bytes(count_bytes); if count == 0 { ensure_eof(input, block_index)?; - return Ok(scopes); + return Ok(()); } if count > MAX_BLOCK_RECORDS as u32 { return Err(NfdumpError::new( @@ -484,33 +718,8 @@ fn reduce_stream( for record in payload[..payload_len].chunks_exact(RECORD_LEN) { record_ordinal += 1; let validated = validate_record(record, block_index, record_ordinal)?; - if !matches_visibility(&validated, selection) { - continue; - } let flow = validated.into_flow(); - if selection.ip_prefix().is_some_and(|prefix| { - !prefix.contains(&flow.source_address) - && !prefix.contains(&flow.destination_address) - }) { - continue; - } - let family_base = if flow.ip_version == IpVersion::V4 { - 0 - } else { - 5 - }; - let exact_index = family_base - + exact_scope_index(flow.source_anonymized, flow.destination_anonymized); - for index in [family_base, exact_index] { - scopes[index].validate_add(&flow).map_err(|field| { - NfdumpError::new(Phase::Aggregate, field, ErrorReason::AggregateOverflow) - .at_block(block_index) - .at_record(record_ordinal) - })?; - } - for index in [family_base, exact_index] { - scopes[index].add(&flow); - } + visit(&flow, block_index, record_ordinal)?; } block_index += 1; } @@ -710,12 +919,23 @@ where i64::try_from(value).map_err(|_| error(field, ErrorReason::NumericOverflow(value))) } -fn matches_visibility(record: &ValidatedRecord<'_>, selection: &FlowSelection) -> bool { - selection.src_visibility().is_none_or(|required| { - matches!(required, ExactVisibility::Anonymized) == record.source_anonymized - }) && selection.dst_visibility().is_none_or(|required| { - matches!(required, ExactVisibility::Anonymized) == record.destination_anonymized - }) +fn matches_selection(flow: &Flow, selection: &FlowSelection) -> bool { + selection.matches_flow_fields( + flow.source_address, + flow.destination_address, + flow.protocol, + Some(flow.source_port), + if flow.source_anonymized { + ExactVisibility::Anonymized + } else { + ExactVisibility::Literal + }, + if flow.destination_anonymized { + ExactVisibility::Anonymized + } else { + ExactVisibility::Literal + }, + ) } const fn exact_scope_index(source_anonymized: bool, destination_anonymized: bool) -> usize { @@ -924,6 +1144,7 @@ fn traffic_metrics(metrics: [i64; METRIC_COUNT]) -> TrafficMetrics { mod tests { use std::io::{self, Cursor, Read}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + use std::sync::Arc; use super::*; use crate::domain::{Granularity, Scope}; @@ -943,6 +1164,31 @@ mod tests { .expect("the exported fixture contains one fixed record") } + fn daily_selection(prefix: &str) -> FlowSelection { + FlowSelection::from_payload(Some(&serde_json::json!({ + "kind": "daily_active_sources", + "ip_prefix": prefix, + }))) + .unwrap() + } + + fn daily_record( + source: [u8; 4], + tag: u8, + flows: u64, + packets: u64, + bytes: u64, + ) -> [u8; RECORD_LEN] { + let mut record = base_record(); + record[..4].copy_from_slice(&source); + record[32..40].copy_from_slice(&packets.to_le_bytes()); + record[40..48].copy_from_slice(&bytes.to_le_bytes()); + record[48..56].copy_from_slice(&flows.to_le_bytes()); + record[64..66].copy_from_slice(&55_000_u16.to_le_bytes()); + record[69] = tag; + record + } + fn stream(records: &[[u8; RECORD_LEN]]) -> Vec { let mut bytes = vec![65, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0]; bytes.extend_from_slice(&(records.len() as u32).to_le_bytes()); @@ -1109,6 +1355,174 @@ mod tests { assert_eq!(bucket.protocols[0].protocols, ["47", "6"]); } + #[test] + fn daily_activity_resolves_exact_sources_before_bucket_reduction() { + let selection = FlowSelection::from_payload(Some(&serde_json::json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + }))) + .unwrap(); + let mut first = base_record(); + first[32..40].copy_from_slice(&5_u64.to_le_bytes()); + first[40..48].copy_from_slice(&500_u64.to_le_bytes()); + first[48..56].copy_from_slice(&1_u64.to_le_bytes()); + first[64..66].copy_from_slice(&55_000_u16.to_le_bytes()); + first[66..68].copy_from_slice(&443_u16.to_le_bytes()); + first[69] = 0b010; + let mut second = first; + second[32..40].copy_from_slice(&15_u64.to_le_bytes()); + second[40..48].copy_from_slice(&1_500_u64.to_le_bytes()); + second[48..56].copy_from_slice(&2_u64.to_le_bytes()); + let mut inactive = first; + inactive[0..4].copy_from_slice(&[192, 0, 3, 2]); + inactive[32..40].copy_from_slice(&19_u64.to_le_bytes()); + inactive[40..48].copy_from_slice(&1_999_u64.to_le_bytes()); + inactive[48..56].copy_from_slice(&2_u64.to_le_bytes()); + + let activity = reduce_to_daily_source_activity( + Cursor::new(stream(&[first, second, inactive])), + &selection, + ) + .unwrap(); + assert_eq!( + activity[&IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))], + SourceActivity { + flows: 3, + packets: 20, + bytes: 2_000, + } + ); + let mut active = AddressSet::default(); + active.extend(activity.into_iter().filter_map(|(address, metrics)| { + FlowSelection::daily_activity_threshold_met( + metrics.flows, + metrics.packets, + metrics.bytes, + ) + .then_some(address) + })); + let mut low_port = first; + low_port[64..66].copy_from_slice(&1_023_u16.to_le_bytes()); + + let bucket = reduce_to_bucket_with_active_sources( + Cursor::new(stream(&[first, inactive, low_port])), + key(), + &selection, + &active, + ) + .unwrap(); + + assert_eq!(active.len(), 1); + assert_eq!(bucket.traffic[0].metrics.flows, 1); + assert_eq!(bucket.traffic[0].metrics.packets, 5); + assert_eq!(bucket.traffic[0].metrics.bytes, 500); + assert!(reduce_to_bucket(Cursor::new(stream(&[first])), key(), &selection).is_err()); + } + + #[test] + fn daily_activity_maps_are_reduced_independently_with_overlapping_prefixes() { + let source_a = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + let source_b = IpAddr::V4(Ipv4Addr::new(198, 51, 2, 1)); + let records = [ + daily_record([192, 0, 2, 1], 0b010, 3, 20, 2_000), + daily_record([198, 51, 2, 1], 0b010, 4, 21, 2_100), + daily_record([192, 0, 2, 1], 0, 1, 1, 1), + ]; + let selections = [ + daily_selection("192.0.0.0/16"), + daily_selection("198.51.0.0/16"), + daily_selection("192.0.0.0/16"), + ]; + + let activities = + reduce_to_daily_source_activities(Cursor::new(stream(&records)), &selections).unwrap(); + + assert_eq!(activities.len(), 3); + assert_eq!(activities[0][&source_a].flows, 3); + assert_eq!(activities[0][&source_a].packets, 20); + assert!(!activities[0].contains_key(&source_b)); + assert_eq!(activities[1][&source_b].flows, 3); + assert!(!activities[1].contains_key(&source_a)); + assert_eq!(activities[2], activities[0]); + } + + #[test] + fn active_source_buckets_fan_out_per_pair_and_allow_overlap() { + let source_a = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + let source_b = IpAddr::V4(Ipv4Addr::new(198, 51, 2, 1)); + let selection_a = daily_selection("192.0.0.0/16"); + let selection_b = daily_selection("198.51.0.0/16"); + let pairs = [ + ( + selection_a.clone(), + Arc::new([source_a].into_iter().collect::()), + ), + ( + selection_a.clone(), + Arc::new([source_b].into_iter().collect::()), + ), + ( + selection_a, + Arc::new([source_a, source_b].into_iter().collect::()), + ), + ( + selection_b, + Arc::new([source_b].into_iter().collect::()), + ), + ]; + let records = [ + daily_record([192, 0, 2, 1], 0b010, 3, 20, 2_000), + daily_record([198, 51, 2, 1], 0b010, 4, 21, 2_100), + ]; + + let buckets = + reduce_to_buckets_with_active_sources(Cursor::new(stream(&records)), key(), &pairs) + .unwrap(); + + assert_eq!(buckets.len(), 4); + let all_v4 = |bucket: &CanonicalBucket| { + bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics + .flows + }; + assert_eq!(buckets.iter().map(all_v4).collect::>(), [3, 0, 3, 4]); + assert_eq!( + buckets[0].addresses[1].addresses, + [source_a].into_iter().collect::() + ); + assert_eq!( + buckets[3].addresses[1].addresses, + [source_b].into_iter().collect::() + ); + } + + #[test] + fn multi_subset_decoding_rejects_empty_and_non_daily_inputs() { + let empty = + reduce_to_daily_source_activities(Cursor::new(Vec::::new()), &[]).unwrap_err(); + assert!(matches!( + empty.reason, + ErrorReason::DailyActivityRequiresSelection + )); + + let non_daily = reduce_to_buckets_with_active_sources( + Cursor::new(Vec::::new()), + key(), + &[(FlowSelection::default(), Arc::new(AddressSet::default()))], + ) + .unwrap_err(); + assert!(matches!( + non_daily.reason, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection + )); + } + #[test] fn malformed_excluded_record_is_rejected_and_input_is_drained() { let mut record = base_record(); diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index 0f8b89f..fb3a2e9 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -2,9 +2,11 @@ use std::{ borrow::Cow, - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashMap}, fs, + net::IpAddr, path::{Path, PathBuf}, + sync::{Arc, OnceLock}, time::{Duration, Instant}, }; @@ -15,40 +17,373 @@ use serde::Deserialize; use serde_json::{Value, json}; use thiserror::Error; +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; + use crate::{ config::{ConfigError, CsvSourceConfig}, coverage::BucketCoverage, domain::{ - BucketKey, CanonicalBucket, DomainError, FlowSelection, Granularity, StatisticalBucket, - StatisticalBucketIncludeProfile, + AddressSet, BucketKey, CanonicalBucket, DomainError, FlowSelection, Granularity, + StatisticalBucket, StatisticalBucketIncludeProfile, }, ingest::{self, IngestError, ProducerError}, nfdump, provenance::{ - ExpectedAbsence, FileSnapshot, InputRevision, ProvenanceError, capture_file_revision, - csv_decoder_fingerprint, nfcapd_decoder_fingerprint, revision_for_locator, - verify_file_snapshot, + ExecutableRevision, ExpectedAbsence, FileSnapshot, InputRevision, ProvenanceError, + capture_file_revision, csv_decoder_fingerprint, file_sha256, nfcapd_decoder_fingerprint, + revision_for_locator, verify_file_snapshot, }, publish::{PublishError, WriteBucketsProfile, write_buckets, write_buckets_profiled}, registry::{Dataset, DatasetRegistry, DatasetSource, RegistryError, is_safe_path_component}, storage::{ - BucketCoverageRow, DatabaseOperationLock, DatasetMetadata, InputBucket, InputEvidenceRow, - InputEvidenceState, InputKind, InputStatus, ProductIdentity, SourceDefinition, - StatsBucketKey, StorageError, bind_nfcapd_source_layout, bind_product_identity, - cached_content_fingerprint, complete_input_scan, connect_pipeline_writer, - delete_stats_bucket_keys, earliest_traffic_bucket_start, init_schema, - input_scan_fully_processed, insert_bucket_coverage_rows, mark_input_bucket_status, - nfcapd_logical_bucket_processed, optimize_all_query_planner_statistics, - query_bucket_coverage, query_input_evidence, replace_input_evidence, - set_dataset_default_start_date, upsert_dataset_metadata, upsert_input_bucket, + BucketCoverageRow, DailyProductCompletionState, DatabaseOperationLock, DatasetMetadata, + InputBucket, InputEvidenceRow, InputEvidenceState, InputKind, InputStatus, ProductIdentity, + STATS_TABLE_NAMES, SourceDefinition, StatsBucketKey, StorageError, + bind_nfcapd_source_layout, bind_product_identity, cached_content_fingerprint, + canonical_path, complete_input_scan, connect_pipeline_writer, current_product_fingerprint, + daily_product_completion_state, database_operation_lock_path, delete_stats_bucket_keys, + delete_stats_time_range, earliest_traffic_bucket_start, + ensure_daily_product_completion_bucket_guard, init_schema, input_scan_fully_processed, + insert_bucket_coverage_rows, mark_input_bucket_status, nfcapd_logical_bucket_processed, + optimize_all_query_planner_statistics, provision_daily_product_completion_bucket_guards, + query_bucket_coverage, query_input_evidence, query_input_evidence_range, + query_processed_nfcapd_range, replace_input_evidence, set_dataset_default_start_date, + upsert_daily_product_completion, upsert_dataset_metadata, upsert_input_bucket, + validate_database_path_separation, }, }; const FIVE_MINUTES: i64 = 300; const NFCAPD_DECODE_BATCH_SIZE: usize = 12; const NFCAPD_REVISION_HASH_MAX_WORKERS: usize = NFCAPD_DECODE_BATCH_SIZE * 2; +const MAX_MISSING_DAY_WARNING_DETAILS: usize = 8; const DEFAULT_TIMEZONE: &str = "America/Los_Angeles"; +static NFCAPD_DENSE_TRAFFIC_SCOPE_COUNT: OnceLock = OnceLock::new(); + +fn nfcapd_dense_traffic_scope_count() -> i64 { + *NFCAPD_DENSE_TRAFFIC_SCOPE_COUNT.get_or_init(|| { + let key = BucketKey::new("", Granularity::FiveMinutes, 0, FIVE_MINUTES); + i64::try_from(StatisticalBucket::dense(key).finish_owned().traffic.len()) + .expect("dense traffic scope count fits SQLite INTEGER") + }) +} + +#[cfg(test)] +type MissingDayAbsenceHook = Box; + +#[cfg(test)] +type CoordinatedCommitGuardHook = Box; + +#[cfg(test)] +type CoordinatedPlanHook = Box; + +#[cfg(test)] +type SinglePlanHook = Box; + +#[cfg(test)] +type SingleCommitGuardHook = Box; + +#[cfg(test)] +thread_local! { + static PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static NFCAPD_CAPTURE_IDENTITY_CALLS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static NFCAPD_REVISION_POOL_BUILDS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static NFCAPD_DECODE_POOL_BUILDS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static NFCAPD_ACTIVITY_POOL_BUILDS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static DATASET_REGISTRY_LOAD_CALLS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; + static MISSING_DAY_ABSENCE_HOOK: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + static COORDINATED_COMMIT_GUARD_HOOK: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + static COORDINATED_PLAN_HOOK: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + static SINGLE_COMMIT_GUARD_HOOK: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + static SINGLE_PLAN_HOOK: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn reset_prepare_nfcapd_tree_timestamp_calls() { + PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +fn prepare_nfcapd_tree_timestamp_calls() -> usize { + PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn reset_nfcapd_logical_bucket_topology_calls() { + NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +fn nfcapd_logical_bucket_topology_calls() -> usize { + NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn reset_nfcapd_day_topology_audit_calls() { + NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +fn nfcapd_day_topology_audit_calls() -> usize { + NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn reset_nfcapd_capture_identity_calls() { + NFCAPD_CAPTURE_IDENTITY_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +fn nfcapd_capture_identity_calls() -> usize { + NFCAPD_CAPTURE_IDENTITY_CALLS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn reset_nfcapd_pool_builds() { + NFCAPD_REVISION_POOL_BUILDS.with(|builds| builds.set(0)); + NFCAPD_DECODE_POOL_BUILDS.with(|builds| builds.set(0)); + NFCAPD_ACTIVITY_POOL_BUILDS.with(|builds| builds.set(0)); +} + +#[cfg(test)] +fn nfcapd_pool_builds() -> (usize, usize, usize) { + ( + NFCAPD_REVISION_POOL_BUILDS.with(std::cell::Cell::get), + NFCAPD_DECODE_POOL_BUILDS.with(std::cell::Cell::get), + NFCAPD_ACTIVITY_POOL_BUILDS.with(std::cell::Cell::get), + ) +} + +#[cfg(test)] +fn reset_dataset_registry_load_calls() { + DATASET_REGISTRY_LOAD_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +fn dataset_registry_load_calls() -> usize { + DATASET_REGISTRY_LOAD_CALLS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn reset_coordinated_postflight_snapshot_verifications() { + COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS.with(|calls| calls.set(0)); +} + +#[cfg(test)] +fn coordinated_postflight_snapshot_verifications() -> usize { + COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS.with(std::cell::Cell::get) +} + +#[cfg(test)] +fn set_missing_day_absence_hook(hook: impl FnMut(&Path, &[(String, i64)], &str) + 'static) { + MISSING_DAY_ABSENCE_HOOK.with(|current| { + *current.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn clear_missing_day_absence_hook() { + MISSING_DAY_ABSENCE_HOOK.with(|current| { + *current.borrow_mut() = None; + }); +} + +#[cfg(test)] +fn set_coordinated_commit_guard_hook(hook: impl FnMut() + 'static) { + COORDINATED_COMMIT_GUARD_HOOK.with(|current| { + *current.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn clear_coordinated_commit_guard_hook() { + COORDINATED_COMMIT_GUARD_HOOK.with(|current| { + *current.borrow_mut() = None; + }); +} + +#[cfg(test)] +fn invoke_coordinated_commit_guard_hook() { + COORDINATED_COMMIT_GUARD_HOOK.with(|current| { + if let Some(hook) = current.borrow_mut().as_mut() { + hook(); + } + }); +} + +#[cfg(test)] +fn set_coordinated_plan_hook(hook: impl FnMut(&Path) + 'static) { + COORDINATED_PLAN_HOOK.with(|current| { + *current.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn clear_coordinated_plan_hook() { + COORDINATED_PLAN_HOOK.with(|current| { + *current.borrow_mut() = None; + }); +} + +#[cfg(test)] +fn invoke_coordinated_plan_hook(root: &Path) { + COORDINATED_PLAN_HOOK.with(|current| { + if let Some(hook) = current.borrow_mut().as_mut() { + hook(root); + } + }); +} + +#[cfg(test)] +fn set_single_commit_guard_hook(hook: impl FnMut() + 'static) { + SINGLE_COMMIT_GUARD_HOOK.with(|current| { + *current.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn clear_single_commit_guard_hook() { + SINGLE_COMMIT_GUARD_HOOK.with(|current| { + *current.borrow_mut() = None; + }); +} + +#[cfg(test)] +fn invoke_single_commit_guard_hook() { + SINGLE_COMMIT_GUARD_HOOK.with(|current| { + if let Some(hook) = current.borrow_mut().as_mut() { + hook(); + } + }); +} + +#[cfg(test)] +fn set_single_plan_hook(hook: impl FnMut(&Path) + 'static) { + SINGLE_PLAN_HOOK.with(|current| { + *current.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn clear_single_plan_hook() { + SINGLE_PLAN_HOOK.with(|current| { + *current.borrow_mut() = None; + }); +} + +#[cfg(test)] +fn invoke_single_plan_hook(root: &Path) { + SINGLE_PLAN_HOOK.with(|current| { + if let Some(hook) = current.borrow_mut().as_mut() { + hook(root); + } + }); +} + +#[cfg(not(test))] +fn invoke_coordinated_commit_guard_hook() {} + +#[cfg(not(test))] +fn invoke_coordinated_plan_hook(_root: &Path) {} + +#[cfg(not(test))] +fn invoke_single_commit_guard_hook() {} + +#[cfg(not(test))] +fn invoke_single_plan_hook(_root: &Path) {} + +#[cfg(test)] +fn invoke_missing_day_absence_hook(root: &Path, missing: &[(String, i64)], timezone: &str) { + MISSING_DAY_ABSENCE_HOOK.with(|current| { + if let Some(hook) = current.borrow_mut().as_mut() { + hook(root, missing, timezone); + } + }); +} + +#[cfg(not(test))] +fn invoke_missing_day_absence_hook(_root: &Path, _missing: &[(String, i64)], _timezone: &str) {} + +fn build_revision_hash_pool() -> Result { + #[cfg(test)] + NFCAPD_REVISION_POOL_BUILDS.with(|builds| builds.set(builds.get() + 1)); + let revision_hash_workers = std::thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .min(NFCAPD_REVISION_HASH_MAX_WORKERS); + rayon::ThreadPoolBuilder::new() + .num_threads(revision_hash_workers) + .thread_name(|index| format!("nfcapd-revision-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build revision hash pool: {error}")) + }) +} + +fn build_nfcapd_snapshot_pool() -> Result { + let snapshot_workers = std::thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .min(NFCAPD_REVISION_HASH_MAX_WORKERS); + rayon::ThreadPoolBuilder::new() + .num_threads(snapshot_workers) + .thread_name(|index| format!("nfcapd-snapshot-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build nfcapd snapshot pool: {error}")) + }) +} + +fn build_nfcapd_decode_pool() -> Result { + #[cfg(test)] + NFCAPD_DECODE_POOL_BUILDS.with(|builds| builds.set(builds.get() + 1)); + rayon::ThreadPoolBuilder::new() + .num_threads(NFCAPD_DECODE_BATCH_SIZE) + .thread_name(|index| format!("nfcapd-decode-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build nfcapd decode pool: {error}")) + }) +} + +fn build_nfcapd_activity_pool() -> Result { + #[cfg(test)] + NFCAPD_ACTIVITY_POOL_BUILDS.with(|builds| builds.set(builds.get() + 1)); + rayon::ThreadPoolBuilder::new() + .num_threads(NFCAPD_DECODE_BATCH_SIZE) + .thread_name(|index| format!("nfcapd-activity-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build nfcapd activity pool: {error}")) + }) +} + #[derive(Clone, Debug)] pub struct PipelineRequest { pub config_path: Option, @@ -188,15 +523,162 @@ enum InputSpec { #[derive(Clone, Debug)] struct ResolvedPipeline { database_path: PathBuf, + /// Files that configure or execute the pipeline and must remain read-only during output setup. + control_paths: Vec, timezone: String, run_maad: bool, - nfdump: String, + nfdump: PathBuf, + nfdump_revision: Option, selection: FlowSelection, inputs: Vec, datasets: Vec, require_complete: bool, } +fn nfdump_control_path(value: &str) -> Option { + let path = Path::new(value); + (path.is_absolute() + || path + .parent() + .is_some_and(|parent| !parent.as_os_str().is_empty()) + || value.contains(['/', '\\'])) + .then(|| path.to_owned()) +} + +#[cfg(unix)] +fn has_effective_execute_access(path: &Path) -> bool { + #[cfg(all(not(target_os = "android"), not(target_os = "redox")))] + { + nix::unistd::faccessat( + None, + path, + nix::unistd::AccessFlags::X_OK, + nix::fcntl::AtFlags::AT_EACCESS, + ) + .is_ok() + } + + #[cfg(any(target_os = "android", target_os = "redox"))] + { + // These targets do not expose AT_EACCESS. Their normal process lookup uses the same + // credentials as access(2) for this non-set-id pipeline. + nix::unistd::access(path, nix::unistd::AccessFlags::X_OK).is_ok() + } +} + +#[cfg(not(unix))] +fn has_effective_execute_access(_path: &Path) -> bool { + true +} + +/// Resolve the executable that a nfdump command will select. +/// +/// Resolve explicit paths before output setup as well as bare names. The canonical path is stored +/// in the resolved pipeline so later command invocations use the same executable that preflight +/// checked, and output alias checks see the actual control file. +fn resolved_nfdump_control_path(value: &str) -> Result { + if let Some(path) = nfdump_control_path(value) { + let resolved = canonical_path(&path)?; + let metadata = fs::metadata(&resolved).map_err(|error| { + PipelineError::InvalidConfig(format!( + "cannot resolve explicit nfdump executable {value:?} at {}: {error}", + path.display() + )) + })?; + if !metadata.is_file() { + return Err(PipelineError::InvalidConfig(format!( + "explicit nfdump executable {value:?} at {} is not a regular file", + path.display() + ))); + } + if !has_effective_execute_access(&resolved) { + return Err(PipelineError::InvalidConfig(format!( + "explicit nfdump executable {value:?} at {} is not executable by this process", + path.display() + ))); + } + return Ok(resolved); + } + if value.is_empty() { + return Err(PipelineError::InvalidConfig( + "nfdump executable name is empty".into(), + )); + } + + let path_variable = std::env::var_os("PATH").ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "cannot resolve bare nfdump executable {value:?}: PATH is not set" + )) + })?; + for directory in std::env::split_paths(&path_variable) { + // An empty PATH component means the current working directory for process lookup. + let directory = if directory.as_os_str().is_empty() { + std::env::current_dir()? + } else { + directory + }; + let candidate = directory.join(value); + let Ok(metadata) = fs::metadata(&candidate) else { + continue; + }; + if !metadata.is_file() { + continue; + } + if !has_effective_execute_access(&candidate) { + continue; + } + return Ok(canonical_path(candidate)?); + } + + Err(PipelineError::InvalidConfig(format!( + "cannot resolve bare nfdump executable {value:?} through PATH" + ))) +} + +/// Capture and validate the exact native decoder before output setup. The digest is paid once; +/// every later boundary uses only the stored file snapshot. +fn resolve_nfdump_revision(value: &str) -> Result<(PathBuf, ExecutableRevision), PipelineError> { + let path = resolved_nfdump_control_path(value)?; + let revision = ExecutableRevision::capture(&path)?; + Ok((path, revision)) +} + +fn verify_nfdump_revision(pipeline: &ResolvedPipeline) -> Result<(), PipelineError> { + if let Some(revision) = &pipeline.nfdump_revision { + verify_nfdump_revision_snapshot(revision)?; + } + Ok(()) +} + +fn verify_nfdump_revision_snapshot(revision: &ExecutableRevision) -> Result<(), PipelineError> { + verify_file_snapshot(Path::new(&revision.locator), &revision.snapshot).map_err(|error| { + PipelineError::InvalidConfig(format!( + "nfdump executable changed during pipeline execution at {}: {error}", + revision.locator + )) + }) +} + +fn nfdump_decoder_fingerprint_for_pipeline( + pipeline: &ResolvedPipeline, +) -> Result { + if let Some(revision) = &pipeline.nfdump_revision { + return Ok(revision.decoder_fingerprint.clone()); + } + // Manually assembled test pipelines can exercise native helpers without going through + // request resolution. Production native requests always carry a revision. + Ok(nfcapd_decoder_fingerprint()?) +} + +fn inputs_require_nfdump(inputs: &[InputSpec]) -> bool { + inputs.iter().any(|input| { + matches!( + input, + InputSpec::Nfcapd { .. } | InputSpec::NfcapdTree { .. } + ) + }) +} + fn default_timezone() -> String { DEFAULT_TIMEZONE.into() } @@ -208,2780 +690,11401 @@ pub fn run( execute(pipeline) } -fn resolve_request(request: &PipelineRequest) -> Result { - match (&request.config_path, &request.dataset_id) { - (Some(_), Some(_)) => return Err(PipelineError::ConflictingModes), - (None, None) => return Err(PipelineError::MissingMode), - _ => {} +/// Run several registry datasets as coordinated daily-active-source products. +/// +/// The datasets share discovery and nfdump work, while each output retains its own product +/// identity, provenance, transactions, and resume state. This deliberately stays separate from +/// [`run`] so the established single-dataset path remains unchanged. +pub fn run_many( + request: impl std::borrow::Borrow, + dataset_ids: Vec, +) -> Result { + let request = request.borrow(); + if dataset_ids.len() < 2 { + return Err(PipelineError::InvalidConfig( + "coordinated pipeline mode requires at least two --dataset values".into(), + )); } - if let Some(path) = &request.config_path { - let mut config: PipelineConfigFile = serde_json::from_slice(&fs::read(path)?)?; - if let Some(path) = &request.database_path { - config.database_path = path.clone(); - } - let configured_selection = selection_from_value(&config.selection)?; - let requested_selection = selection_from_value(&request.selection)?; - let selection = if requested_selection.is_unrestricted() { - configured_selection - } else { - requested_selection - }; - if request.force { - let tree_count = config - .inputs - .iter() - .filter(|input| matches!(input, InputSpec::NfcapdTree { .. })) - .count(); - if tree_count != 1 { - return Err(PipelineError::InvalidConfig( - "--force in config mode requires exactly one nfcapd_tree input".into(), - )); - } - for input in &mut config.inputs { - if let InputSpec::NfcapdTree { force, .. } = input { - *force = true; - } - } - } - return Ok(ResolvedPipeline { - database_path: config.database_path, - timezone: config.timezone, - run_maad: config.run_maad.unwrap_or(true) && request.run_maad, - nfdump: config.nfdump.unwrap_or_else(|| request.nfdump.clone()), - selection, - inputs: config.inputs, - datasets: config.datasets, - require_complete: request.require_complete, - }); + if request.config_path.is_some() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode cannot combine --config with repeated --dataset".into(), + )); } - - let repository_root = std::env::current_dir()?; - let registry = match &request.datasets_path { - Some(path) => DatasetRegistry::load(path, &repository_root)?, - None => DatasetRegistry::load_default(&repository_root)?, - }; - let dataset_id = request - .dataset_id - .as_deref() - .ok_or(PipelineError::MissingMode)?; - let dataset = registry.get(dataset_id)?.clone(); - let start_date = request.start_date.clone().ok_or_else(|| { - PipelineError::InvalidConfig("--start-date is required with --dataset".into()) - })?; - let selection = selection_from_value(&request.selection)?; - if !selection.is_unrestricted() && request.database_path.is_none() { + if request.database_path.is_some() { return Err(PipelineError::InvalidConfig( - "flow selection requires an explicit --database-path".into(), + "coordinated dataset mode cannot override --database-path".into(), )); } - Ok(ResolvedPipeline { - database_path: request - .database_path - .clone() - .unwrap_or_else(|| dataset.db_path.clone()), - timezone: DEFAULT_TIMEZONE.into(), - run_maad: request.run_maad, - nfdump: request.nfdump.clone(), - selection, - inputs: vec![InputSpec::NfcapdTree { - root_path: dataset.root_path.clone(), - source_ids: dataset.source_ids.clone(), - sources: dataset.sources.clone(), - start_date, - end_date: request.end_date.clone(), - start_time: request.start_time.clone(), - end_time: request.end_time.clone(), - force: request.force, - }], - datasets: vec![dataset], - require_complete: request.require_complete, - }) + if selection_override_requested(&request.selection) { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode cannot override registry selections from the CLI".into(), + )); + } + if request.start_time.is_some() || request.end_time.is_some() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode requires a whole-day date window; --start-time and --end-time are unsupported".into(), + )); + } + + let mut seen = BTreeSet::new(); + if let Some(duplicate) = dataset_ids.iter().find(|id| !seen.insert(id.as_str())) { + return Err(PipelineError::InvalidConfig(format!( + "coordinated dataset mode cannot repeat dataset {duplicate:?}" + ))); + } + + let repository_root = std::env::current_dir()?; + let registry_path = request + .datasets_path + .clone() + .unwrap_or_else(|| DatasetRegistry::default_path(&repository_root)); + let registry = load_dataset_registry(®istry_path, &repository_root)?; + let shared_nfdump = resolve_nfdump_revision(&request.nfdump)?; + let mut pipelines = Vec::with_capacity(dataset_ids.len()); + for dataset_id in &dataset_ids { + let mut single = request.clone(); + single.dataset_id = Some(dataset_id.clone()); + pipelines.push(resolve_dataset_request( + &single, + ®istry_path, + ®istry, + Some((&shared_nfdump.0, &shared_nfdump.1)), + )?); + } + validate_compatible_pipelines(&pipelines)?; + execute_many(pipelines) } -fn selection_from_value(value: &Value) -> Result { - FlowSelection::from_payload((!value.is_null()).then_some(value)) +fn selection_override_requested(value: &Value) -> bool { + value + .as_object() + .is_some_and(|object| object.values().any(|entry| !entry.is_null())) } -fn execute(pipeline: ResolvedPipeline) -> Result { - if let Some(parent) = pipeline.database_path.parent() { - fs::create_dir_all(parent)?; +fn validate_compatible_pipelines(pipelines: &[ResolvedPipeline]) -> Result<(), PipelineError> { + let Some(first) = pipelines.first() else { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode requires at least two datasets".into(), + )); + }; + if !first.selection.selects_daily_active_sources() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode requires every registry selection to be daily_active_sources" + .into(), + )); } - let _lock = DatabaseOperationLock::acquire(&pipeline.database_path, "pipeline build")?; - let connection = connect_pipeline_writer(&pipeline.database_path)?; - init_schema(&connection)?; - initialize_metadata(&connection, &pipeline)?; - - let mut report = PipelineReport::default(); - let mut csv_inputs = pipeline - .inputs - .iter() - .filter_map(|input| match input { - InputSpec::Csv { path, mapping_path } => Some(ingest::CsvInputSpec { - path: path.clone(), - mapping_path: mapping_path.clone(), - }), - _ => None, - }) - .collect::>(); - let explicit_nfcapd = pipeline - .inputs - .iter() - .filter(|input| matches!(input, InputSpec::Nfcapd { .. })) - .cloned() - .collect::>(); - for input in &pipeline.inputs { - match input { - InputSpec::Csv { .. } | InputSpec::Nfcapd { .. } => {} - InputSpec::CsvTree { - root_path, - mapping_path, - } => { - let mapping = CsvSourceConfig::load(mapping_path)?; - csv_inputs.extend(ingest::discover_csv_inputs( - root_path, - mapping_path, - &mapping, - )?); + let first_input = only_nfcapd_tree(first)?; + let first_config = nfcapd_tree_config(first_input)?; + let first_root = canonical_path(first_config.root_path)?; + for pipeline in pipelines.iter().skip(1) { + if !pipeline.selection.selects_daily_active_sources() { + return Err(PipelineError::InvalidConfig(format!( + "dataset {:?} does not use a daily_active_sources selection", + pipeline + .datasets + .first() + .map(|dataset| dataset.dataset_id.as_str()) + .unwrap_or("") + ))); + } + let input = only_nfcapd_tree(pipeline)?; + let config = nfcapd_tree_config(input)?; + if first_root != canonical_path(config.root_path)? { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfcapd root".into(), + )); + } + if first_config.start_date != config.start_date + || first_config.end_date != config.end_date + || first_config.start_time != config.start_time + || first_config.end_time != config.end_time + || first_config.force != config.force + { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same whole-day window and force settings".into(), + )); + } + if first.timezone != pipeline.timezone { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same timezone".into(), + )); + } + if first.run_maad != pipeline.run_maad { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same MAAD setting".into(), + )); + } + if first.nfdump != pipeline.nfdump { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfdump executable/configuration".into(), + )); + } + let same_executable_revision = match (&first.nfdump_revision, &pipeline.nfdump_revision) { + (Some(left), Some(right)) => { + left.locator == right.locator + && left.content_fingerprint == right.content_fingerprint + && left.decoder_fingerprint == right.decoder_fingerprint } - InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - start_date, - end_date, - start_time, - end_time, - force, - } => process_nfcapd_tree( - &connection, - root_path, - source_ids, - sources, - start_date, - end_date.as_deref(), - start_time.as_deref(), - end_time.as_deref(), - *force, - &pipeline, - &mut report, - )?, + (None, None) => true, + _ => false, + }; + if !same_executable_revision { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfdump executable revision".into(), + )); } - } - merge_report( - &mut report, - process_csv_inputs(&connection, &csv_inputs, &pipeline)?, - ); - merge_report( - &mut report, - process_explicit_nfcapd_inputs(&connection, &explicit_nfcapd, &pipeline)?, - ); - infer_default_start_dates(&connection, &pipeline)?; - populate_coverage_summary(&connection, &mut report)?; - // This is the publication seam for the in-place pipeline product. Keep it before the strict - // coverage error so an incomplete-but-inspectable database also gets useful planner stats. - if let Err(error) = optimize_all_query_planner_statistics(&connection) { - tracing::warn!(%error, "could not refresh SQLite planner statistics"); - } - if pipeline.require_complete { - let incomplete = count_incomplete_requested_coverage(&connection, &pipeline)?; - if incomplete != 0 { - return Err(PipelineError::IncompleteCoverage(incomplete)); + if first.require_complete != pipeline.require_complete { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same coverage settings".into(), + )); } } - Ok(report) -} - -/// Give every dataset without a configured `default_start_date` the earliest ingested local day. -/// -/// This runs after ingestion so that newly ingested earlier days move the stored date back. Until -/// the database holds traffic, the row keeps the fallback that [`upsert_dataset_metadata`] wrote. -fn infer_default_start_dates( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result<(), PipelineError> { - let inferred = pipeline - .datasets + let output_paths = pipelines .iter() - .filter(|dataset| dataset.default_start_date.trim().is_empty()) + .map(|pipeline| pipeline.database_path.as_path()) .collect::>(); - if inferred.is_empty() { - return Ok(()); + validate_database_path_separation(&output_paths)?; + Ok(()) +} + +fn only_nfcapd_tree(pipeline: &ResolvedPipeline) -> Result<&InputSpec, PipelineError> { + if pipeline.inputs.len() != 1 { + return Err(PipelineError::InvalidConfig( + "coordinated datasets require exactly one nfcapd_tree input".into(), + )); } - let Some(bucket_start) = earliest_traffic_bucket_start(connection)? else { - return Ok(()); + match pipeline.inputs.first() { + Some(input @ InputSpec::NfcapdTree { .. }) => Ok(input), + _ => Err(PipelineError::InvalidConfig( + "coordinated datasets require an nfcapd_tree input".into(), + )), + } +} + +struct NfcapdTreeConfig<'a> { + root_path: &'a Path, + start_date: &'a str, + end_date: Option<&'a str>, + start_time: Option<&'a str>, + end_time: Option<&'a str>, + force: bool, +} + +fn nfcapd_tree_config(input: &InputSpec) -> Result, PipelineError> { + let InputSpec::NfcapdTree { + root_path, + start_date, + end_date, + start_time, + end_time, + force, + .. + } = input + else { + return Err(PipelineError::InvalidConfig( + "coordinated datasets require an nfcapd_tree input".into(), + )); }; - let date = local_date(bucket_start, &pipeline.timezone)?; - with_transaction(connection, || { - for dataset in inferred { - set_dataset_default_start_date(connection, &dataset.dataset_id, &date)?; - } - Ok(()) + Ok(NfcapdTreeConfig { + root_path, + start_date, + end_date: end_date.as_deref(), + start_time: start_time.as_deref(), + end_time: end_time.as_deref(), + force: *force, }) } -/// Local calendar day that contains `timestamp`, formatted as `YYYY-MM-DD`. -fn local_date(timestamp: i64, timezone: &str) -> Result { - Ok(Timestamp::from_second(timestamp) - .map_err(|error| PipelineError::Time(error.to_string()))? - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .date() - .to_string()) +fn canonical_logical_sources(input: &InputSpec) -> Result, PipelineError> { + let InputSpec::NfcapdTree { + root_path, + source_ids, + sources, + .. + } = input + else { + return Err(PipelineError::InvalidConfig( + "coordinated datasets require an nfcapd_tree input".into(), + )); + }; + let mut sources = normalize_sources(root_path, source_ids, sources)?; + for source in &mut sources { + source.members.sort_unstable(); + } + Ok(sources) } -fn with_transaction( - connection: &Connection, - operation: impl FnOnce() -> Result, -) -> Result { - connection - .execute_batch("BEGIN IMMEDIATE") - .map_err(StorageError::from)?; - let result = operation(); - match result { - Ok(value) => { - connection - .execute_batch("COMMIT") - .map_err(StorageError::from)?; - Ok(value) - } - Err(error) => { - let _ = connection.execute_batch("ROLLBACK"); - Err(error) +fn normalized_path_key(path: &Path) -> PathBuf { + let mut result = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + result.pop(); + } + _ => result.push(component.as_os_str()), } } + result } -fn initialize_metadata( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result<(), PipelineError> { - let layouts = pipeline - .inputs - .iter() - .filter_map(|input| match input { - InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - .. - } => Some(normalize_sources(root_path, source_ids, sources)), - _ => None, - }) - .collect::, _>>()? - .into_iter() - .flatten() - .collect::>(); - let mut source_ids = BTreeSet::new(); - if let Some(duplicate) = layouts - .iter() - .find(|source| !source_ids.insert(source.source_id.clone())) - { - return Err(PipelineError::InvalidConfig(format!( - "nfcapd_tree inputs define duplicate logical source ID {:?}", - duplicate.source_id - ))); - } - with_transaction(connection, || { - bind_identity(connection, pipeline)?; - for dataset in &pipeline.datasets { - upsert_dataset(connection, dataset)?; - } - if !layouts.is_empty() { - let layout = layouts - .iter() - .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) - .collect::>(); - bind_nfcapd_source_layout(connection, &layout)?; - } - Ok(()) - }) +fn absolute_lexical_path(path: &Path) -> Result { + let absolute = if path.is_absolute() { + path.to_owned() + } else { + std::env::current_dir()?.join(path) + }; + Ok(normalized_path_key(&absolute)) } -fn process_atomic( - connection: &Connection, - pipeline: &ResolvedPipeline, - operation: impl FnOnce(&mut AggregateBuckets, &mut PipelineReport) -> Result<(), PipelineError>, -) -> Result { - let mut aggregates = AggregateBuckets::default(); - let mut report = PipelineReport::default(); - with_transaction(connection, || { - operation(&mut aggregates, &mut report)?; - publish_rollups(connection, aggregates, pipeline, &mut report) +fn sqlite_related_path(path: &Path, suffix: &str) -> Result { + let parent = path.parent().ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "database path has no parent directory: {}", + path.display() + )) })?; - Ok(report) + let name = path.file_name().ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "database path has no file name: {}", + path.display() + )) + })?; + Ok(parent.join(format!("{}{}", name.to_string_lossy(), suffix))) } -fn merge_report(total: &mut PipelineReport, addition: PipelineReport) { - total.input_scans += addition.input_scans; - total.skipped_inputs += addition.skipped_inputs; - total.five_minute_buckets += addition.five_minute_buckets; - total.rollup_buckets += addition.rollup_buckets; -} +/// Return every path SQLite or the pipeline lock can touch for an output path. +/// +/// Keep both the caller spelling and the resolved spelling here. SQLite receives the caller +/// spelling, while the operation lock resolves the database first; a symlink can therefore make +/// those two sets differ even when the database itself is absent. +fn output_related_paths(path: &Path) -> Result, PipelineError> { + let raw = absolute_lexical_path(path)?; + let resolved = canonical_path(path)?; + let mut candidates = BTreeSet::new(); + let mut add = |candidate: PathBuf| -> Result<(), PipelineError> { + let candidate = absolute_lexical_path(&candidate)?; + candidates.insert(candidate.clone()); + candidates.insert(canonical_path(candidate)?); + Ok(()) + }; -fn populate_coverage_summary( - connection: &Connection, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - let mut statement = connection - .prepare( - "SELECT coverage_state, COUNT(*) - FROM bucket_coverage - WHERE granularity = '5m' - GROUP BY coverage_state", - ) - .map_err(StorageError::from)?; - let rows = statement - .query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - for (state, count) in rows { - let count = usize::try_from(count) - .map_err(|_| PipelineError::InvalidConfig("coverage summary count overflow".into()))?; - match state.as_str() { - "complete" => report.complete_five_minute_buckets = count, - "partial" => report.partial_five_minute_buckets = count, - "unknown" => report.unknown_five_minute_buckets = count, - _ => { - return Err(PipelineError::InvalidConfig(format!( - "invalid five-minute coverage state in database: {state:?}" - ))); - } + for database in [&raw, &resolved] { + add(database.to_owned())?; + for suffix in ["-journal", "-wal", "-shm"] { + add(sqlite_related_path(database, suffix)?)?; } } - Ok(()) + add(database_operation_lock_path(&resolved)?)?; + add(raw.with_file_name(format!( + ".{}.operation.lock", + raw.file_name() + .ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "database path has no file name: {}", + raw.display() + )) + })? + .to_string_lossy() + )))?; + Ok(candidates.into_iter().collect()) } -#[derive(Clone, Debug, PartialEq, Eq)] -struct CoverageScope { - source_ids: Vec, - start: i64, - end: i64, +#[cfg(unix)] +fn existing_path_identity(path: &Path) -> Result, PipelineError> { + use std::os::unix::fs::MetadataExt; + + match fs::metadata(path) { + Ok(metadata) => Ok(Some((metadata.dev(), metadata.ino()))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } } -/// A finite native request can be checked independently of incomplete data -/// already stored outside that request. CSV and literal-input configurations -/// have no separately declared time window, so their configured product is -/// the strict scope. -fn requested_coverage_scopes( - pipeline: &ResolvedPipeline, -) -> Result>, PipelineError> { - let mut scopes = Vec::new(); - for input in &pipeline.inputs { - let InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - start_date, - end_date, - start_time, - end_time, - .. - } = input - else { - return Ok(None); - }; - let selected_start = parse_date_start(start_date, &pipeline.timezone)?; - let start = match start_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_start, - }; - let end = match (end_time, end_date) { - (Some(value), _) => parse_local_datetime(value, &pipeline.timezone)?, - (None, Some(value)) => next_date_start(value, &pipeline.timezone)?, - (None, None) => return Ok(None), - }; - let source_ids = normalize_sources(root_path, source_ids, sources)? - .into_iter() - .map(|source| source.source_id) - .collect(); - scopes.push(CoverageScope { - source_ids, - start, - end, - }); +#[cfg(unix)] +#[cfg(test)] +fn nfcapd_capture_identity(path: &Path) -> Result, PipelineError> { + #[cfg(test)] + NFCAPD_CAPTURE_IDENTITY_CALLS.with(|calls| calls.set(calls.get() + 1)); + existing_path_identity(path) +} + +fn capture_nfcapd_snapshot(path: &Path) -> Result { + FileSnapshot::capture(path).map_err(PipelineError::from) +} + +/// Capture the cheap identities for a discovered capture set with bounded parallelism. +/// +/// The caller keeps the resulting metadata alongside the already-discovered paths. Hashing and +/// decode work can then reuse the same observation instead of doing a serial alias pass followed +/// by another metadata walk. +fn capture_nfcapd_snapshots( + paths: &BTreeSet, +) -> Result, PipelineError> { + capture_nfcapd_snapshots_with(paths, capture_nfcapd_snapshot) +} + +fn capture_nfcapd_snapshots_with( + paths: &BTreeSet, + capture: F, +) -> Result, PipelineError> +where + F: Fn(&Path) -> Result + Sync, +{ + if paths.is_empty() { + return Ok(BTreeMap::new()); } - Ok(Some(scopes)) + let pool = build_nfcapd_snapshot_pool()?; + pool.install(|| { + paths + .par_iter() + .map(|path| capture(path).map(|snapshot| (path.clone(), snapshot))) + .collect::, _>>() + }) } -fn count_incomplete_requested_coverage( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result { - let Some(scopes) = requested_coverage_scopes(pipeline)? else { - return connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state <> 'complete'", - [], - |row| row.get(0), - ) - .map_err(StorageError::from) - .map_err(PipelineError::from); - }; +#[cfg(test)] +fn capture_nfcapd_snapshots_counted( + paths: &BTreeSet, + calls: &AtomicUsize, +) -> Result, PipelineError> { + capture_nfcapd_snapshots_with(paths, |path| { + calls.fetch_add(1, Ordering::Relaxed); + capture_nfcapd_snapshot(path) + }) +} - connection - .execute_batch( - "CREATE TEMP TABLE IF NOT EXISTS requested_coverage_scope ( - source_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - bucket_end INTEGER NOT NULL, - PRIMARY KEY (source_id, bucket_start, bucket_end) - ); - DELETE FROM requested_coverage_scope;", - ) - .map_err(StorageError::from)?; - for scope in scopes { - for source_id in scope.source_ids { - connection - .execute( - "INSERT OR IGNORE INTO requested_coverage_scope ( - source_id, bucket_start, bucket_end - ) VALUES (?1, ?2, ?3)", - params![source_id, scope.start, scope.end], - ) - .map_err(StorageError::from)?; +#[cfg(unix)] +fn output_has_existing_identity(output_paths: &[&Path]) -> Result { + for output in output_paths { + for related in output_related_paths(output)? { + if existing_path_identity(&related)?.is_some() { + return Ok(true); + } } } - connection - .query_row( - "SELECT COUNT(*) - FROM bucket_coverage AS coverage - WHERE coverage.granularity = '5m' - AND coverage.coverage_state <> 'complete' - AND EXISTS ( - SELECT 1 FROM requested_coverage_scope AS scope - WHERE scope.source_id = coverage.source_id - AND coverage.bucket_start >= scope.bucket_start - AND coverage.bucket_start < scope.bucket_end - )", - [], - |row| row.get(0), - ) - .map_err(StorageError::from) - .map_err(PipelineError::from) + Ok(false) } -fn bind_identity( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result<(), PipelineError> { - let maad_config = serde_json::to_value(crate::maad::MaadConfig::default())?; - let schema = json!({ - "version": 3, - "tables": [ - {"name":"traffic_stats","version":2}, - {"name":"protocol_stats","version":1}, - {"name":"address_count_stats","version":1}, - {"name":"port_count_stats","version":1}, - {"name":"address_structure_stats","version":1}, - {"name":"bucket_coverage","version":1} - ] - }); - let result_config = json!({ - "version": 3, - "timezone": pipeline.timezone, - "nfcapd_decoder": { - "protocol_version": nfdump::CONTRACT_VERSION, - "input_contract": nfdump::INPUT_CONTRACT, - "output_contract": nfdump::OUTPUT_CONTRACT - }, - "maad": { - "enabled": pipeline.run_maad, - "backend": "in-process", - "contract_version": 2, - "config": maad_config - } - }); - let identity = ProductIdentity::create( - &schema, - &pipeline.selection.normalized_payload(), - &result_config, - )?; - bind_product_identity(connection, &identity, &crate::storage::STATS_TABLE_NAMES)?; - Ok(()) +#[cfg(not(unix))] +fn output_has_existing_identity(_output_paths: &[&Path]) -> Result { + Ok(false) } -fn upsert_dataset(connection: &Connection, dataset: &Dataset) -> Result<(), PipelineError> { - let sources = dataset - .logical_sources()? - .into_iter() - .map(|source| SourceDefinition::new(source.source_id, source.members)) - .collect::>(); - let mut metadata = DatasetMetadata::new(&dataset.dataset_id); - metadata.label = dataset.label.clone(); - metadata.default_start_date = dataset.default_start_date.clone(); - metadata.source_mode = dataset.source_mode.clone(); - metadata.discovery_mode = dataset.discovery_mode.clone(); - metadata.sort_order = dataset.sort_order; - metadata.sources = sources; - upsert_dataset_metadata(connection, &metadata)?; +/// Reject an output database, its SQLite sidecars, or its operation lock when any aliases an +/// input file. This must run after input discovery and before output setup. +fn validate_output_input_separation<'a, I>( + output_paths: &[&Path], + input_paths: I, + input_label: &str, +) -> Result<(), PipelineError> +where + I: IntoIterator, +{ + // Outputs are few, while a discovered capture tree can contain hundreds of thousands of + // paths. Index the small output side and stream the input side so validation does not retain a + // second copy of the discovered corpus. + let mut outputs_by_path = BTreeMap::::new(); + #[cfg(unix)] + let mut outputs_by_identity = BTreeMap::<(u64, u64), (usize, PathBuf)>::new(); + for (output_index, output) in output_paths.iter().enumerate() { + for related in output_related_paths(output)? { + let resolved = canonical_path(&related)?; + outputs_by_path + .entry(resolved) + .or_insert_with(|| (output_index, related.clone())); + #[cfg(unix)] + if let Some(identity) = existing_path_identity(&related)? { + outputs_by_identity + .entry(identity) + .or_insert_with(|| (output_index, related)); + } + } + } + + for input in input_paths { + let resolved = canonical_path(input)?; + if let Some((output_index, related)) = outputs_by_path.get(&resolved) { + return Err(PipelineError::InvalidConfig(format!( + "output database {} aliases {input_label} {} through {}", + output_paths[*output_index].display(), + input.display(), + related.display() + ))); + } + #[cfg(unix)] + if let Some(identity) = existing_path_identity(input)? + && let Some((output_index, related)) = outputs_by_identity.get(&identity) + { + return Err(PipelineError::InvalidConfig(format!( + "output database {} aliases {input_label} {} through device/inode {:?} at {}", + output_paths[*output_index].display(), + input.display(), + identity, + related.display() + ))); + } + } Ok(()) } -struct PreparedCsvInput { - path: PathBuf, - mapping: CsvSourceConfig, - revision: InputRevision, - snapshot: FileSnapshot, +/// Reject an output database, its SQLite sidecars, or its operation lock when any aliases a +/// discovered nfcapd capture. This must run after capture discovery and before output setup. +fn validate_output_capture_separation<'a, I>( + output_paths: &[&Path], + capture_paths: I, +) -> Result<(), PipelineError> +where + I: IntoIterator, +{ + validate_output_input_separation(output_paths, capture_paths, "discovered nfcapd capture") } -fn prepare_file_revision( - connection: &Connection, - path: &Path, - input_kind: InputKind, - decoder_fingerprint: String, -) -> Result<(InputRevision, FileSnapshot), PipelineError> { - prepare_file_revision_with(connection, path, input_kind, decoder_fingerprint, || { - capture_file_revision(path) - }) +/// Protect a discovered nfcapd tree without resolving every capture path. +/// +/// Capture discovery already walks the configured member namespaces, so a capture's lexical +/// locator is known. Output aliases are checked against the configured and resolved namespace +/// spellings, including locators that do not exist yet. Only an existing output-side inode needs +/// a physical capture scan for hard-link aliases; the common new-output case performs no metadata +/// call per capture. +#[cfg(test)] +fn validate_output_nfcapd_capture_separation<'a, I>( + output_paths: &[&Path], + namespaces: &[PathBuf], + _timezone: &str, + capture_paths: I, +) -> Result<(), PipelineError> +where + I: IntoIterator, +{ + validate_output_nfcapd_locator_separation(output_paths, namespaces)?; + + #[cfg(unix)] + let mut outputs_by_identity = BTreeMap::<(u64, u64), (usize, PathBuf)>::new(); + #[cfg(unix)] + for (output_index, output) in output_paths.iter().enumerate() { + for related in output_related_paths(output)? { + if let Some(identity) = existing_path_identity(&related)? { + outputs_by_identity + .entry(identity) + .or_insert_with(|| (output_index, related)); + } + } + } + + #[cfg(unix)] + if outputs_by_identity.is_empty() { + return Ok(()); + } + + #[cfg(unix)] + for input in capture_paths { + if let Some(identity) = nfcapd_capture_identity(input)? + && let Some((output_index, related)) = outputs_by_identity.get(&identity) + { + return Err(PipelineError::InvalidConfig(format!( + "output database {} aliases discovered nfcapd capture {} through device/inode {:?} at {}", + output_paths[*output_index].display(), + input.display(), + identity, + related.display() + ))); + } + } + + #[cfg(not(unix))] + let _ = capture_paths; + Ok(()) } -fn prepare_file_revision_with( - connection: &Connection, - path: &Path, - input_kind: InputKind, - decoder_fingerprint: String, - hash_file: impl FnOnce() -> Result<(String, FileSnapshot), ProvenanceError>, -) -> Result<(InputRevision, FileSnapshot), PipelineError> { - let locator = path.to_string_lossy().into_owned(); - let observed = FileSnapshot::capture(path)?; - let (content_fingerprint, snapshot) = - match cached_content_fingerprint(connection, input_kind, &locator, &observed)? { - Some(content_fingerprint) => (content_fingerprint, observed), - None => hash_file()?, - }; - let revision = InputRevision::create( - input_kind.as_str(), - locator, - content_fingerprint, - decoder_fingerprint, - )?; - Ok((revision, snapshot)) -} +/// Validate discovered nfcapd captures using identities captured by the bounded snapshot pass. +/// +/// This is the existing-output path: the snapshot map is also consumed by revision preparation, +/// so hard-link protection does not require a second serial metadata walk. +fn validate_output_nfcapd_capture_separation_with_snapshots<'a, I>( + output_paths: &[&Path], + namespaces: &[PathBuf], + capture_snapshots: I, +) -> Result<(), PipelineError> +where + I: IntoIterator, +{ + validate_output_nfcapd_locator_separation(output_paths, namespaces)?; -fn process_csv_inputs( - connection: &Connection, - inputs: &[ingest::CsvInputSpec], - pipeline: &ResolvedPipeline, -) -> Result { - let mut prepared = Vec::new(); - let mut skipped_inputs = 0_usize; - let mut needs_rescan = false; - for input in inputs { - let mapping = CsvSourceConfig::load(&input.mapping_path)?; - let (revision, snapshot) = prepare_file_revision( - connection, - &input.path, - InputKind::Csv, - csv_decoder_fingerprint(&mapping)?, - )?; - if input_scan_fully_processed(connection, InputKind::Csv, &revision.locator, &revision)? { - skipped_inputs += 1; - } else { - needs_rescan = true; + #[cfg(unix)] + let mut outputs_by_identity = BTreeMap::<(u64, u64), (usize, PathBuf)>::new(); + #[cfg(unix)] + for (output_index, output) in output_paths.iter().enumerate() { + for related in output_related_paths(output)? { + if let Some(identity) = existing_path_identity(&related)? { + outputs_by_identity + .entry(identity) + .or_insert_with(|| (output_index, related)); + } } - prepared.push(PreparedCsvInput { - path: input.path.clone(), - mapping, - revision, - snapshot, - }); } - if !needs_rescan { - return Ok(PipelineReport { - skipped_inputs, - ..PipelineReport::default() - }); + + #[cfg(unix)] + if outputs_by_identity.is_empty() { + return Ok(()); } - prepared.sort_unstable_by(|left, right| left.path.cmp(&right.path)); - let mut aggregates = AggregateBuckets::default(); - let mut report = PipelineReport::default(); - with_transaction(connection, || { - connection - .execute_batch( - "CREATE TEMP TABLE csv_bucket_stage ( - source_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - input_locator TEXT NOT NULL, - revision_fingerprint TEXT, - payload BLOB NOT NULL - ); - CREATE INDEX csv_bucket_stage_order - ON csv_bucket_stage(source_id, bucket_start);", - ) - .map_err(StorageError::from)?; - for input in &prepared { - process_csv( - connection, - &input.path, - &input.mapping, - &input.revision, - &input.snapshot, - pipeline, - &mut report, - )?; + #[cfg(unix)] + for (input, snapshot) in capture_snapshots { + let identity = (snapshot.device, snapshot.inode); + if let Some((output_index, related)) = outputs_by_identity.get(&identity) { + return Err(PipelineError::InvalidConfig(format!( + "output database {} aliases discovered nfcapd capture {} through device/inode {:?} at {}", + output_paths[*output_index].display(), + input.display(), + identity, + related.display() + ))); } - publish_csv_stage(connection, pipeline, &mut aggregates, &mut report)?; - publish_rollups(connection, aggregates, pipeline, &mut report) - })?; - Ok(report) -} + } -#[allow(clippy::too_many_arguments)] -fn process_csv( - connection: &Connection, - path: &Path, - mapping: &CsvSourceConfig, - revision: &InputRevision, - snapshot: &FileSnapshot, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - connection - .execute( - "DELETE FROM processed_inputs - WHERE input_kind = 'csv' AND scan_locator = ?1", - params![revision.locator], - ) - .map_err(StorageError::from)?; - let completion = match ingest::scan_csv(path, mapping, &pipeline.selection, |event| { - let bucket_revision = revision_for_locator(revision, &event.input_locator)?; - let owner = InputBucket { - input_kind: InputKind::Csv, - input_locator: event.input_locator.clone(), - scan_locator: event.scan_locator, - source_id: event.bucket.key.source_id.clone(), - bucket_start: event.bucket.key.bucket_start, - bucket_end: event.bucket.key.bucket_end, - revision: bucket_revision.clone(), - file_snapshot: Some(snapshot.clone()), - }; - upsert_input_bucket(connection, &owner, false)?; - mark_input_bucket_status( - connection, - InputKind::Csv, - &event.input_locator, - &event.bucket.key.source_id, - event.bucket.key.bucket_start, - InputStatus::Processed, - &bucket_revision, - None, - )?; - let payload = serde_json::to_vec(&event.bucket)?; - connection - .execute( - "INSERT INTO csv_bucket_stage ( - source_id, bucket_start, input_locator, - revision_fingerprint, payload - ) VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - event.bucket.key.source_id, - event.bucket.key.bucket_start, - event.input_locator, - bucket_revision.fingerprint, - payload, - ], - ) - .map_err(StorageError::from)?; - Ok::<_, PipelineError>(()) - }) { - Ok(completion) => completion, - Err(ProducerError::Input(error)) => return Err(error.into()), - Err(ProducerError::Sink(error)) => return Err(error), - }; - verify_file_snapshot(path, snapshot)?; - complete_input_scan( - connection, - InputKind::Csv, - &completion.scan_locator, - i64::try_from(completion.rejected_rows) - .map_err(|_| PipelineError::InvalidConfig("rejected row count overflow".into()))?, - i64::try_from(completion.skipped_bad_column_count).map_err(|_| { - PipelineError::InvalidConfig("skipped bad-column count overflow".into()) - })?, - revision, - Some(snapshot), - )?; - verify_file_snapshot(path, snapshot)?; - report.input_scans += 1; + #[cfg(not(unix))] + let _ = capture_snapshots; Ok(()) } -struct CsvStageMember { - bucket: CanonicalBucket, - input_locator: String, - revision_fingerprint: Option, +/// Return the canonical and configured spellings of each member's locator namespace. +/// +/// The output preflight uses these prefixes instead of materializing every possible capture +/// path in the selected window. Missing future captures are still protected because the output +/// path itself is checked against the namespace shape. +fn nfcapd_locator_namespaces( + root: &Path, + physical_ids: &[String], +) -> Result, PipelineError> { + let mut namespaces = BTreeSet::new(); + for member in physical_ids { + let configured = absolute_lexical_path(&root.join(member))?; + namespaces.insert(configured.clone()); + namespaces.insert(canonical_path(configured)?); + } + Ok(namespaces.into_iter().collect()) } -/// Merge all staged CSV buckets in source/time order. The stage is indexed on -/// disk, so only one overlapping bucket group is held in memory at a time. -fn publish_csv_stage( - connection: &Connection, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, +/// Return whether two paths overlap as a namespace and a path. +/// +/// The equality and ancestor cases matter when the output or one of its sidecars is the +/// namespace itself, or when it would replace a root/ancestor needed to discover captures. +fn paths_overlap_namespace(path: &Path, namespace: &Path) -> bool { + path == namespace || path.starts_with(namespace) || namespace.starts_with(path) +} + +/// Reject output databases, sidecars, and operation locks anywhere in a configured member +/// namespace, even when the output is not itself a valid nfcapd capture locator yet. +fn validate_output_nfcapd_locator_separation( + output_paths: &[&Path], + namespaces: &[PathBuf], ) -> Result<(), PipelineError> { - let mut statement = connection - .prepare( - "SELECT source_id, bucket_start, input_locator, - revision_fingerprint, payload - FROM csv_bucket_stage - ORDER BY source_id, bucket_start, input_locator", - ) - .map_err(StorageError::from)?; - let mut rows = statement.query([]).map_err(StorageError::from)?; - let mut group: Option<(String, i64, Vec)> = None; - let mut current_source = None; - let mut next_expected = None; - loop { - let Some((source_id, bucket_start, input_locator, revision_fingerprint, payload)) = rows - .next() - .map_err(StorageError::from)? - .map(|row| { - Ok::<_, rusqlite::Error>(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, String>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Vec>(4)?, - )) - }) - .transpose() - .map_err(StorageError::from)? - else { - break; - }; - let member = CsvStageMember { - bucket: serde_json::from_slice(&payload)?, - input_locator, - revision_fingerprint, - }; - match group.as_mut() { - Some((group_source, group_start, members)) - if group_source == &source_id && *group_start == bucket_start => - { - members.push(member); + for output in output_paths { + for related in output_related_paths(output)? { + for namespace in namespaces { + if paths_overlap_namespace(&related, namespace) { + return Err(PipelineError::InvalidConfig(format!( + "output database {} aliases discovered nfcapd capture locator in configured nfcapd member namespace {} through {}", + output.display(), + namespace.display(), + related.display() + ))); + } } - _ => { - if let Some((group_source, group_start, members)) = group.take() { - publish_csv_stage_group( - connection, - pipeline, - aggregates, - report, - &group_source, - group_start, - &members, - &mut current_source, - &mut next_expected, - )?; + } + } + Ok(()) +} + +/// Reject output databases, sidecars, and operation locks anywhere below an auto-discovered +/// nfcapd root. The member directory may not exist during preflight, so checking only discovered +/// captures would leave a future member namespace writable by output setup. +fn validate_output_nfcapd_auto_namespace_separation( + output_paths: &[&Path], + roots: &[PathBuf], +) -> Result<(), PipelineError> { + for output in output_paths { + for related in output_related_paths(output)? { + for root in roots { + if paths_overlap_namespace(&related, root) { + return Err(PipelineError::InvalidConfig(format!( + "output database {} aliases discovered nfcapd capture locator in the auto-discovered member namespace under nfcapd root {} (including direct-child directory paths) through {}", + output.display(), + root.display(), + related.display() + ))); } - group = Some((source_id, bucket_start, vec![member])); } } } - drop(rows); - drop(statement); - if let Some((group_source, group_start, members)) = group { - publish_csv_stage_group( - connection, - pipeline, - aggregates, - report, - &group_source, - group_start, - &members, - &mut current_source, - &mut next_expected, - )?; - } Ok(()) } -#[allow(clippy::too_many_arguments)] -fn publish_csv_stage_group( - connection: &Connection, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, - source_id: &str, - bucket_start: i64, - members: &[CsvStageMember], - current_source: &mut Option, - next_expected: &mut Option, +/// Reject output paths that a CSV tree would discover after SQLite creates them. Discovery is +/// intentionally flat, so only a direct child of the configured tree root can change its input +/// set. +fn validate_output_csv_tree_separation( + output_paths: &[&Path], + trees: &[(PathBuf, CsvSourceConfig)], ) -> Result<(), PipelineError> { - if current_source.as_deref() != Some(source_id) { - *current_source = Some(source_id.to_owned()); - *next_expected = None; - } else { - let mut expected = next_expected.ok_or_else(|| { - PipelineError::InvalidConfig("CSV stage lost its source envelope".into()) - })?; - while expected < bucket_start { - let (bucket, evidence) = merged_csv_bucket(source_id, expected, &[])?; - publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; - expected = expected.checked_add(FIVE_MINUTES).ok_or_else(|| { - PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) - })?; + for output in output_paths { + for related in output_related_paths(output)? { + for (root, mapping) in trees { + if related.parent() != Some(root.as_path()) { + continue; + } + let Some(name) = related.file_name().and_then(|name| name.to_str()) else { + continue; + }; + let lowercase_name = name.to_ascii_lowercase(); + let excluded = mapping + .discovery_exclude_suffixes + .iter() + .any(|suffix| lowercase_name.ends_with(&suffix.to_ascii_lowercase())); + if !excluded && ingest::matches_csv_discovery(&lowercase_name, mapping) { + return Err(PipelineError::InvalidConfig(format!( + "output database {} would be discovered as a CSV tree input at {} under {}", + output.display(), + related.display(), + root.display() + ))); + } + } } } - let (bucket, evidence) = merged_csv_bucket(source_id, bucket_start, members)?; - publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; - *next_expected = Some(bucket_start.checked_add(FIVE_MINUTES).ok_or_else(|| { - PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) - })?); Ok(()) } -fn merged_csv_bucket( - source_id: &str, - bucket_start: i64, - members: &[CsvStageMember], -) -> Result<(CanonicalBucket, InputEvidenceRow), PipelineError> { - let key = BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - ); - let any_observed = members - .iter() - .any(|member| member.bucket.coverage.observed_units() != 0); - let any_rejected = members - .iter() - .any(|member| member.bucket.coverage.rejected_units() != 0); - let mut builder = if any_observed { - StatisticalBucket::dense(key) - } else { - StatisticalBucket::new(key) +fn validate_daily_active_source_layout( + sources: &[DatasetSource], + physical_ids: &[String], +) -> Result<(), PipelineError> { + if sources.is_empty() { + return Err(PipelineError::InvalidConfig( + "daily_active_sources requires at least one logical source".into(), + )); } - .with_coverage(BucketCoverage::empty()); - for member in members { - builder.include(&member.bucket)?; + if physical_ids.is_empty() { + return Err(PipelineError::InvalidConfig( + "daily_active_sources requires at least one physical source member".into(), + )); } - let coverage = BucketCoverage::new(1, u64::from(any_observed), u64::from(any_rejected)) - .map_err(DomainError::from)?; - let bucket = builder.with_coverage(coverage).finish(); - let evidence_state = if any_rejected { - InputEvidenceState::Rejected - } else if any_observed { - InputEvidenceState::Observed - } else { - InputEvidenceState::Missing - }; - let (input_locator, revision_fingerprint) = match members { - [member] => ( - member.input_locator.clone(), - member.revision_fingerprint.clone(), - ), - [] => (format!("csv://{source_id}"), None), - _ => (format!("csv://{source_id}"), None), - }; - let evidence = InputEvidenceRow::new( - source_id, - source_id, - bucket_start, - bucket_start + FIVE_MINUTES, - input_locator, - evidence_state, - revision_fingerprint, - ); - Ok((bucket, evidence)) + Ok(()) } -fn publish_csv_bucket( - connection: &Connection, - bucket: &CanonicalBucket, - evidence: &InputEvidenceRow, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - reject_cross_kind_overlap(connection, bucket, InputKind::Csv)?; - aggregates.reject_persisted_csv_siblings(connection, bucket, &pipeline.timezone)?; - write_buckets(connection, std::slice::from_ref(bucket), pipeline.run_maad)?; - replace_input_evidence( - connection, - &bucket.key.source_id, - bucket.key.bucket_start, - std::slice::from_ref(evidence), - )?; - aggregates.include(bucket, &pipeline.timezone)?; - report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; - report.five_minute_buckets += 1; - Ok(()) +/// Identity of a planned physical member directory. +/// +/// The canonical path catches a retargeted root/member symlink or a renamed directory. On Unix, +/// the device/inode pair also catches a replacement at the same configured path. Directory +/// timestamps are intentionally not part of this identity: normal capture creation changes them. +#[derive(Clone, Debug, PartialEq, Eq)] +struct MemberDirectoryIdentity { + canonical_path: PathBuf, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, } -fn reject_cross_kind_overlap( - connection: &Connection, - bucket: &CanonicalBucket, - input_kind: InputKind, -) -> Result<(), PipelineError> { - let conflict = connection - .query_row( - "SELECT input_kind, input_locator FROM processed_inputs - WHERE source_id = ?1 AND bucket_start = ?2 AND input_kind <> ?3 - ORDER BY input_kind, input_locator LIMIT 1", - params![ - bucket.key.source_id, - bucket.key.bucket_start, - input_kind.as_str(), - ], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional() - .map_err(StorageError::from)?; - if let Some((kind, locator)) = conflict { - return Err(PipelineError::InvalidConfig(format!( - "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", - bucket.key.source_id, bucket.key.bucket_start - ))); +fn capture_member_directory_identity( + root: &Path, + member: &str, +) -> Result { + let member_path = root.join(member); + let canonical_member_path = canonical_path(&member_path)?; + #[cfg(unix)] + { + let (device, inode) = existing_path_identity(&member_path)?.ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "source member directory {:?} disappeared while its identity was being captured", + member + )) + })?; + Ok(MemberDirectoryIdentity { + canonical_path: canonical_member_path, + device, + inode, + }) + } + #[cfg(not(unix))] + { + Ok(MemberDirectoryIdentity { + canonical_path: canonical_member_path, + }) } - Ok(()) } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_tree( - connection: &Connection, +fn capture_member_directory_identities( root: &Path, - source_ids: &[String], - configured_sources: &[DatasetSource], - start_date: &str, - end_date: Option<&str>, - start_time: Option<&str>, - end_time: Option<&str>, - force: bool, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - let sources = normalize_sources(root, source_ids, configured_sources)?; - let physical_ids = sources + physical_ids: &[String], +) -> Result, PipelineError> { + physical_ids .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>() - .into_iter() - .collect::>(); - let discovery_started = Instant::now(); - let discovered = ingest::discover_nfcapd_source_paths(root, &physical_ids, &pipeline.timezone)?; - tracing::info!( - target: "netflow_db::profile", - phase = "discovery", - elapsed_seconds = discovery_started.elapsed().as_secs_f64(), - physical_sources = physical_ids.len(), - discovered_inputs = discovered.len(), - ); - let mut by_member_and_start = BTreeMap::new(); - let mut member_bounds = BTreeMap::new(); - for input in discovered { - member_bounds - .entry(input.source_id.clone()) - .and_modify(|(first, last): &mut (i64, i64)| { - *first = (*first).min(input.bucket_start); - *last = (*last).max(input.bucket_start); - }) - .or_insert((input.bucket_start, input.bucket_start)); - by_member_and_start.insert((input.source_id, input.bucket_start), input.path); - } - let selected_start = parse_date_start(start_date, &pipeline.timezone)?; - let discovered_end = by_member_and_start - .keys() - .map(|(_, bucket_start)| *bucket_start) - .max() - .map(|start| aggregate_bounds(start, Granularity::OneDay, &pipeline.timezone)) - .transpose()? - .map(|(_, end)| end) - .unwrap_or(selected_start); - let selected_end = match end_date { - Some(date) => next_date_start(date, &pipeline.timezone)?, - None => discovered_end, - }; - let start = match start_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_start, - }; - let end = match end_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_end, - }; - validate_window(selected_start, selected_end, start, end, &pipeline.timezone)?; + .map(|member| { + capture_member_directory_identity(root, member) + .map(|identity| (member.clone(), identity)) + }) + .collect() +} - let mut day_start = start; - while day_start < end { - let day_end = aggregate_bounds(day_start, Granularity::OneDay, &pipeline.timezone)?.1; - let mut owned_keys = BTreeSet::new(); - let mut bucket_start = day_start; - while bucket_start < day_end { - for source in &sources { - if force - && source_has_candidate( - source, - bucket_start, - &by_member_and_start, - &member_bounds, - end_date.is_some(), - ) - { - owned_keys.insert((source.source_id.clone(), bucket_start)); - } - } - bucket_start = next_local_five_minute_start(bucket_start, &pipeline.timezone)?; +fn verify_member_directory_identities( + root: &Path, + expected: &BTreeMap, +) -> Result<(), PipelineError> { + for (member, expected_identity) in expected { + let current = capture_member_directory_identity(root, member)?; + if current != *expected_identity { + return Err(PipelineError::InvalidConfig(format!( + "nfcapd member directory {:?} changed during the pipeline: planned {} but found {}", + member, + expected_identity.canonical_path.display(), + current.canonical_path.display() + ))); } - let transaction_started = Instant::now(); - let (day_report, day_profile) = with_transaction(connection, || { - let mut aggregates = AggregateBuckets::with_owned_keys(owned_keys); - let mut day_report = PipelineReport::default(); - let mut day_profile = process_nfcapd_tree_day( - connection, - root, - &sources, - &by_member_and_start, - &member_bounds, - day_start, - day_end, - end_date.is_some(), - force, - pipeline, - &mut aggregates, - &mut day_report, - )?; - day_profile.final_rollups = - publish_rollups_profiled(connection, aggregates, pipeline, &mut day_report)?; - Ok((day_report, day_profile)) - })?; - day_profile.log(day_start, day_end, transaction_started.elapsed()); - merge_report(report, day_report); - day_start = day_end; } Ok(()) } -fn source_has_candidate( - source: &DatasetSource, - bucket_start: i64, - paths: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - extend_gaps_to_window: bool, -) -> bool { - let has_file = source - .members - .iter() - .any(|member| paths.contains_key(&(member.clone(), bucket_start))); - has_file - || extend_gaps_to_window - || source.members.iter().any(|member| { - member_bounds - .get(member) - .is_some_and(|(first, last)| *first <= bucket_start && bucket_start <= *last) - }) +#[derive(Clone, Debug)] +struct FrozenNfcapdTreeLayout { + root_path: PathBuf, + sources: Vec, + physical_ids: Vec, + member_identities: BTreeMap, + auto_discovered: bool, } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_tree_day( - connection: &Connection, - root: &Path, - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - start: i64, - end: i64, - extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result { - let day_started = Instant::now(); - let mut prepare_elapsed = Duration::ZERO; - let mut decode_elapsed = Duration::ZERO; - let mut publish_elapsed = Duration::ZERO; - let mut publish_profile = NfcapdDayPublishProfile::default(); - let revision_hash_workers = std::thread::available_parallelism() - .map_or(1, std::num::NonZeroUsize::get) - .min(NFCAPD_REVISION_HASH_MAX_WORKERS); - let revision_pool = rayon::ThreadPoolBuilder::new() - .num_threads(revision_hash_workers) - .thread_name(|index| format!("nfcapd-revision-{index}")) - .build() - .map_err(|error| { - PipelineError::InvalidConfig(format!("failed to build revision hash pool: {error}")) - })?; - let revision_context = NfcapdRevisionContext { - connection, - sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - force, - revision_pool: &revision_pool, - }; - let mut bucket_start = start; - while bucket_start < end { - let prepare_started = Instant::now(); - let mut batch_starts = Vec::with_capacity(NFCAPD_DECODE_BATCH_SIZE); - while bucket_start < end && batch_starts.len() < NFCAPD_DECODE_BATCH_SIZE { - batch_starts.push(bucket_start); - bucket_start = next_local_five_minute_start(bucket_start, &pipeline.timezone)?; - } - let revisions = resolve_nfcapd_batch_revisions(&revision_context, &batch_starts)?; - let mut batch = Vec::with_capacity(batch_starts.len()); - for bucket_start in batch_starts { - batch.push(prepare_nfcapd_tree_timestamp( - connection, - root, - sources, - by_member_and_start, - member_bounds, - bucket_start, - extend_gaps_to_window, - force, - pipeline, - report, - &revisions, - )?); - } - prepare_elapsed += prepare_started.elapsed(); +#[derive(Clone, Debug, Default)] +struct SingleOutputPlan { + trees: BTreeMap, + dataset_sources: BTreeMap>, + capture_snapshots: BTreeMap, +} - let decode_started = Instant::now(); - let needed = batch - .iter() - .flat_map(|timestamp| { - timestamp.jobs.iter().flat_map(|job| { - job.present.iter().map(|(member, path)| { - let snapshot = timestamp - .revision_cache - .get(member) - .and_then(|owner| owner.snapshot.clone()) - .expect("present member has a snapshot"); - ( - (member.clone(), timestamp.bucket_start), - (path.clone(), snapshot), - ) - }) - }) - }) - .collect::>(); - let mut decoded_cache = needed - .par_iter() - .map(|((member, bucket_start), (path, snapshot))| { - let bucket = ingest::read_nfcapd_bucket( - path, - member, - &pipeline.selection, - &pipeline.nfdump, - &pipeline.timezone, - )?; - verify_file_snapshot(path, snapshot)?; - Ok::<_, PipelineError>(((member.clone(), *bucket_start), bucket)) - }) - .collect::, _>>()?; - decode_elapsed += decode_started.elapsed(); +impl SingleOutputPlan { + fn first_root(&self) -> Option<&Path> { + self.trees + .values() + .next() + .map(|tree| tree.root_path.as_path()) + } +} - let publish_started = Instant::now(); - for timestamp in batch { - for job in timestamp.jobs { - let member_buckets = job - .present +/// Perform the read-only nfcapd discovery needed to protect a single output before opening it. +/// +/// The returned layout is the only source membership snapshot used by output initialization, +/// processing, and strict coverage checks. Auto-discovered layouts are revalidated by +/// [`verify_single_auto_source_layouts`] after all read-only planning and immediately before +/// output setup. +fn plan_single_output(pipeline: &ResolvedPipeline) -> Result { + verify_nfdump_revision(pipeline)?; + let mut locator_namespaces = BTreeSet::new(); + let mut auto_discovery_roots = BTreeSet::new(); + let mut csv_tree_configs = Vec::new(); + let mut nfcapd_windows = Vec::new(); + let mut nfcapd_capture_paths = BTreeSet::new(); + let mut trees = BTreeMap::new(); + let output_path = pipeline.database_path.as_path(); + let output_paths = std::slice::from_ref(&output_path); + validate_output_input_separation( + output_paths, + pipeline.control_paths.iter().map(PathBuf::as_path), + "pipeline control path", + )?; + for (input_index, input) in pipeline.inputs.iter().enumerate() { + match input { + InputSpec::NfcapdTree { + root_path, + source_ids, + sources, + start_date, + end_date, + start_time, + end_time, + .. + } => { + let root = canonical_path(root_path)?; + let auto_discovered = source_ids.is_empty() && sources.is_empty(); + if auto_discovered { + auto_discovery_roots.insert(root.clone()); + } + let sources = normalize_sources(&root, source_ids, sources)?; + let physical_ids = sources .iter() - .map(|(member, _)| { - decoded_cache - .get(&(member.clone(), timestamp.bucket_start)) - .expect("requested physical member was decoded") - }) + .flat_map(|source| source.members.iter().cloned()) + .collect::>() + .into_iter() .collect::>(); - let logical_started = Instant::now(); - let logical = logical_source_bucket( - &job.source_id, - timestamp.bucket_start, - job.expected_units, - &member_buckets, - )?; - publish_profile.logical_source_elapsed += logical_started.elapsed(); - let sibling_started = Instant::now(); - if !job.is_repair { - aggregates.reject_persisted_siblings( - connection, - &logical, - &pipeline.timezone, + let member_identities = capture_member_directory_identities(&root, &physical_ids)?; + trees.insert( + input_index, + FrozenNfcapdTreeLayout { + root_path: root.clone(), + sources: sources.clone(), + physical_ids: physical_ids.clone(), + member_identities, + auto_discovered, + }, + ); + locator_namespaces.extend(nfcapd_locator_namespaces(&root, &physical_ids)?); + if pipeline.selection.selects_daily_active_sources() { + validate_daily_active_source_layout(&sources, &physical_ids)?; + if start_time.is_some() || end_time.is_some() { + return Err(PipelineError::InvalidConfig( + "daily_active_sources selection requires whole local calendar days; start_time and end_time are unsupported".into(), + )); + } + } + let discovered = + ingest::discover_nfcapd_source_paths(&root, &physical_ids, &pipeline.timezone)?; + nfcapd_capture_paths.extend(discovered.iter().map(|input| input.path.clone())); + nfcapd_windows.push(( + start_date.clone(), + end_date.clone(), + start_time.clone(), + end_time.clone(), + discovered.iter().map(|input| input.bucket_start).max(), + )); + } + InputSpec::Nfcapd { + path, + gap, + expected_path, + .. + } => { + if *gap { + if let Some(expected_path) = expected_path { + validate_output_capture_separation( + output_paths, + std::iter::once(expected_path.as_path()), + )?; + } + } else { + validate_output_capture_separation( + output_paths, + std::iter::once(path.as_path()), )?; } - publish_profile.persisted_sibling_elapsed += sibling_started.elapsed(); - let bucket_profile = publish_nfcapd_bucket_profiled( - connection, - &logical, - &job.owners, - &job.absences, - &job.evidence, - true, - force, - pipeline.run_maad, + } + InputSpec::Csv { path, mapping_path } => { + validate_output_input_separation( + output_paths, + [path.as_path(), mapping_path.as_path()], + "discovered CSV input", )?; - publish_profile.bucket_publish.include(bucket_profile); - let flushed = if job.is_repair { - refresh_rollups_after_five_minute_repair( - connection, - &logical, - &pipeline.timezone, - )?; - 0 - } else { - let aggregate_profile = - aggregates.include_profiled(&logical, &pipeline.timezone)?; - publish_profile.aggregate_include.include(aggregate_profile); - let flush_started = Instant::now(); - let (flushed, rollup_write) = - aggregates.flush_complete_profiled(connection, pipeline.run_maad)?; - publish_profile.completed_rollup_flush_elapsed += flush_started.elapsed(); - publish_profile.completed_rollup_write.include(rollup_write); - publish_profile.completed_rollup_flushes += 1; - if flushed > 0 { - publish_profile.nonempty_rollup_flushes += 1; - } - flushed - }; - publish_profile.logical_buckets += 1; - report.rollup_buckets += flushed; - report.five_minute_buckets += 1; } - decoded_cache.retain(|(_, start), _| *start != timestamp.bucket_start); + InputSpec::CsvTree { + root_path, + mapping_path, + } => { + let mapping = CsvSourceConfig::load(mapping_path)?; + let discovered = ingest::discover_csv_inputs(root_path, mapping_path, &mapping)?; + validate_output_input_separation( + output_paths, + std::iter::once(mapping_path.as_path()) + .chain(discovered.iter().map(|input| input.path.as_path())), + "discovered CSV input", + )?; + csv_tree_configs.push((canonical_path(root_path)?, mapping)); + } } - publish_elapsed += publish_started.elapsed(); } - publish_profile.day_elapsed = day_started.elapsed(); - publish_profile.prepare_elapsed = prepare_elapsed; - publish_profile.decode_elapsed = decode_elapsed; - publish_profile.batch_publish_elapsed = publish_elapsed; - tracing::info!( - target: "netflow_db::profile", - phase = "nfcapd_tree_day", - day_start = start, - day_end = end, - elapsed_seconds = publish_profile.day_elapsed.as_secs_f64(), - prepare_seconds = prepare_elapsed.as_secs_f64(), - decode_seconds = decode_elapsed.as_secs_f64(), - publish_seconds = publish_elapsed.as_secs_f64(), - ); - Ok(publish_profile) + let locator_namespaces = locator_namespaces.into_iter().collect::>(); + validate_output_nfcapd_locator_separation(output_paths, &locator_namespaces)?; + validate_output_nfcapd_auto_namespace_separation( + output_paths, + &auto_discovery_roots.into_iter().collect::>(), + )?; + validate_output_csv_tree_separation(output_paths, &csv_tree_configs)?; + let capture_snapshots = if output_has_existing_identity(output_paths)? { + let capture_snapshots = capture_nfcapd_snapshots(&nfcapd_capture_paths)?; + validate_output_nfcapd_capture_separation_with_snapshots( + output_paths, + &locator_namespaces, + capture_snapshots + .iter() + .map(|(path, snapshot)| (path.as_path(), snapshot)), + )?; + capture_snapshots + } else { + BTreeMap::new() + }; + for (start_date, end_date, start_time, end_time, discovered_end) in nfcapd_windows { + resolve_nfcapd_tree_window( + &start_date, + end_date.as_deref(), + start_time.as_deref(), + end_time.as_deref(), + discovered_end, + &pipeline.timezone, + )?; + } + if let Some(revision) = &pipeline.nfdump_revision { + ingest::probe_nfdump_compatibility(&pipeline.nfdump)?; + verify_file_snapshot(&pipeline.nfdump, &revision.snapshot)?; + } + verify_nfdump_revision(pipeline)?; + let mut dataset_sources = BTreeMap::new(); + for dataset in &pipeline.datasets { + let dataset_root = canonical_path(&dataset.root_path)?; + let sources = trees + .values() + .find(|tree| tree.root_path == dataset_root) + .map(|tree| tree.sources.clone()) + .unwrap_or(dataset.logical_sources()?); + dataset_sources.insert(dataset.dataset_id.clone(), sources); + } + Ok(SingleOutputPlan { + trees, + dataset_sources, + capture_snapshots, + }) } -struct NfcapdRevisionProbe { - path: PathBuf, - observed: FileSnapshot, - cached_content_fingerprint: Option, +fn verify_single_auto_source_layouts( + pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, +) -> Result<(), PipelineError> { + for tree in plan.trees.values() { + verify_member_directory_identities(&tree.root_path, &tree.member_identities)?; + if tree.auto_discovered { + let current = normalize_sources(&tree.root_path, &[], &[])?; + if current != tree.sources { + return Err(PipelineError::InvalidConfig( + "auto-discovered source layout changed during single-output planning".into(), + )); + } + } + } + // Keep this parameter in the validation seam so a future pipeline with multiple roots can + // report the owning dataset without rediscovering its metadata. + let _ = pipeline; + Ok(()) } -struct NfcapdRevisionContext<'a> { - connection: &'a Connection, - sources: &'a [DatasetSource], - by_member_and_start: &'a BTreeMap<(String, i64), PathBuf>, - member_bounds: &'a BTreeMap, - extend_gaps_to_window: bool, - force: bool, - revision_pool: &'a rayon::ThreadPool, +#[cfg(test)] +fn preflight_single_output(pipeline: &ResolvedPipeline) -> Result<(), PipelineError> { + plan_single_output(pipeline).map(|_| ()) } -/// Resolve the physical files needed by a decode batch before making any job decisions. -/// SQLite access stays on the pipeline thread; only exact hashes run in parallel. -fn resolve_nfcapd_batch_revisions( - context: &NfcapdRevisionContext<'_>, - batch_starts: &[i64], -) -> Result, PipelineError> { - let mut paths = BTreeSet::new(); - for &bucket_start in batch_starts { - for source in context.sources { - if !source_has_candidate( - source, - bucket_start, - context.by_member_and_start, - context.member_bounds, - context.extend_gaps_to_window, - ) { - continue; - } - paths.extend(source.members.iter().filter_map(|member| { - context - .by_member_and_start - .get(&(member.clone(), bucket_start)) - .cloned() - })); - } +fn resolve_request(request: &PipelineRequest) -> Result { + match (&request.config_path, &request.dataset_id) { + (Some(_), Some(_)) => return Err(PipelineError::ConflictingModes), + (None, None) => return Err(PipelineError::MissingMode), + _ => {} } - - let decoder_fingerprint = nfcapd_decoder_fingerprint()?; - let probes = paths - .into_iter() - .map(|path| { - let locator = path.to_string_lossy().into_owned(); - let observed = FileSnapshot::capture(&path)?; - let cached_fingerprint = if context.force { - None - } else { - cached_content_fingerprint( - context.connection, - InputKind::Nfcapd, - &locator, - &observed, - )? - }; - Ok::<_, PipelineError>(NfcapdRevisionProbe { - path, - observed, - cached_content_fingerprint: cached_fingerprint, - }) - }) - .collect::, _>>()?; - - let resolved = context.revision_pool.install(|| { - probes - .par_iter() - .map(|probe| { - let captured = match &probe.cached_content_fingerprint { - Some(content_fingerprint) => { - Ok((content_fingerprint.clone(), probe.observed.clone())) - } - None => capture_file_revision(&probe.path), - }; - captured - .map_err(PipelineError::from) - .and_then(|(content_fingerprint, snapshot)| { - let revision = InputRevision::create( - "nfcapd", - probe.path.to_string_lossy().into_owned(), - content_fingerprint, - &decoder_fingerprint, - )?; - Ok(PreparedRevision { - revision, - snapshot: Some(snapshot), - }) - }) - }) - .collect::>() - }); - - probes - .into_iter() - .zip(resolved) - .map(|(probe, result)| result.map(|revision| (probe.path, revision))) - .collect::, _>>() -} - -#[allow(clippy::too_many_arguments)] -fn prepare_nfcapd_tree_timestamp( - connection: &Connection, - root: &Path, - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - bucket_start: i64, - extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, - revisions: &BTreeMap, -) -> Result { - let mut revision_cache: BTreeMap = BTreeMap::new(); - let mut jobs = Vec::new(); - for source in sources { - if !source_has_candidate( - source, - bucket_start, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - ) { - continue; - } - let present = source - .members - .iter() - .filter_map(|member| { - by_member_and_start - .get(&(member.clone(), bucket_start)) - .map(|path| (member.clone(), path.clone())) - }) - .collect::>(); - let mut owners = Vec::new(); - for (member, path) in &present { - let owner = match revision_cache.get(member) { - Some(owner) => owner.clone(), - None => { - let owner = revisions - .get(path) - .cloned() - .expect("present member has a resolved revision"); - revision_cache.insert(member.clone(), owner.clone()); - owner - } - }; - owners.push(owner); - } - let mut absences = Vec::new(); - let mut evidence = Vec::with_capacity(source.members.len()); - for ((member, path), owner) in present.iter().zip(&owners) { - evidence.push(InputEvidenceRow::new( - &source.source_id, - member, - bucket_start, - bucket_start + FIVE_MINUTES, - path.to_string_lossy(), - InputEvidenceState::Observed, - Some(owner.revision.fingerprint.clone()), - )); + if let Some(path) = &request.config_path { + let mut config: PipelineConfigFile = serde_json::from_slice(&fs::read(path)?)?; + if let Some(path) = &request.database_path { + config.database_path = path.clone(); } - for member in &source.members { - if !present.iter().any(|(present, _)| present == member) { - let expected = - expected_nfcapd_path(root, member, bucket_start, &pipeline.timezone)?; - absences.push(ExpectedAbsence::capture(&expected)?); - evidence.push(InputEvidenceRow::new( - &source.source_id, - member, - bucket_start, - bucket_start + FIVE_MINUTES, - expected.to_string_lossy(), - InputEvidenceState::Missing, - None, + let configured_selection = selection_from_value(&config.selection)?; + let requested_selection = selection_from_value(&request.selection)?; + let selection = if requested_selection.is_unrestricted() { + configured_selection + } else { + requested_selection + }; + validate_selection_inputs(&selection, &config.inputs)?; + if request.force { + let tree_count = config + .inputs + .iter() + .filter(|input| matches!(input, InputSpec::NfcapdTree { .. })) + .count(); + if tree_count != 1 { + return Err(PipelineError::InvalidConfig( + "--force in config mode requires exactly one nfcapd_tree input".into(), )); } + for input in &mut config.inputs { + if let InputSpec::NfcapdTree { force, .. } = input { + *force = true; + } + } } - evidence.sort_unstable_by(|left, right| left.unit_id.cmp(&right.unit_id)); - let previous_evidence = query_input_evidence(connection, &source.source_id, bucket_start)?; - let observed_input_disappeared = previous_evidence.iter().any(|previous| { - previous.evidence_state == InputEvidenceState::Observed - && evidence.iter().any(|current| { - current.unit_id == previous.unit_id - && current.evidence_state == InputEvidenceState::Missing - }) - }); - if observed_input_disappeared { - tracing::warn!( - source_id = source.source_id, - bucket_start, - "preserving prior bucket because an observed input is now missing" - ); - report.skipped_inputs += 1; - continue; - } - let is_repair = !force && !previous_evidence.is_empty() && previous_evidence != evidence; - let revisions = owners - .iter() - .map(|owner| owner.revision.clone()) - .collect::>(); - if !force - && previous_evidence == evidence - && (revisions.is_empty() - || nfcapd_logical_bucket_processed( - connection, - &source.source_id, - bucket_start, - &revisions, - )?) - { - report.skipped_inputs += 1; - continue; + let configured_nfdump = config.nfdump.unwrap_or_else(|| request.nfdump.clone()); + let requires_nfdump = inputs_require_nfdump(&config.inputs); + let (nfdump, nfdump_revision) = if requires_nfdump { + let (path, revision) = resolve_nfdump_revision(&configured_nfdump)?; + (path, Some(revision)) + } else { + (PathBuf::from(configured_nfdump), None) + }; + let mut control_paths = vec![path.clone()]; + if requires_nfdump { + control_paths.push(nfdump.clone()); } - jobs.push(PreparedTreeJob { - source_id: source.source_id.clone(), - expected_units: source.members.len(), - present, - owners, - absences, - evidence, - is_repair, + return Ok(ResolvedPipeline { + database_path: config.database_path, + control_paths, + timezone: config.timezone, + run_maad: config.run_maad.unwrap_or(true) && request.run_maad, + nfdump, + nfdump_revision, + selection, + inputs: config.inputs, + datasets: config.datasets, + require_complete: request.require_complete, }); } - Ok(PreparedTreeTimestamp { - bucket_start, - revision_cache, - jobs, - }) -} -#[allow(clippy::too_many_arguments)] -enum PreparedExplicitNfcapdKind { - File(PreparedRevision), - Gap { expected_path: Option }, + let repository_root = std::env::current_dir()?; + let registry_path = request + .datasets_path + .clone() + .unwrap_or_else(|| DatasetRegistry::default_path(&repository_root)); + let registry = load_dataset_registry(®istry_path, &repository_root)?; + resolve_dataset_request(request, ®istry_path, ®istry, None) } -struct PreparedExplicitNfcapd { - path: PathBuf, - source_id: String, - bucket_start: i64, - kind: PreparedExplicitNfcapdKind, +fn load_dataset_registry( + registry_path: &Path, + repository_root: &Path, +) -> Result { + #[cfg(test)] + DATASET_REGISTRY_LOAD_CALLS.with(|calls| calls.set(calls.get() + 1)); + Ok(DatasetRegistry::load(registry_path, repository_root)?) } -fn process_explicit_nfcapd_inputs( - connection: &Connection, - inputs: &[InputSpec], - pipeline: &ResolvedPipeline, -) -> Result { - let mut prepared = Vec::new(); - for input in inputs { - let InputSpec::Nfcapd { - path, - source_id, - bucket_start, - gap, - expected_path, - } = input - else { - continue; - }; - let bucket_start = match bucket_start { - Some(start) => *start, - None if !gap => ingest::parse_nfcapd_bucket_start(path, &pipeline.timezone)?, - None => { - return Err(PipelineError::InvalidConfig( - "explicit nfcapd gap requires bucket_start".into(), - )); - } - }; - let kind = if *gap { - PreparedExplicitNfcapdKind::Gap { - expected_path: expected_path.clone(), - } - } else { - let (revision, snapshot) = prepare_file_revision( - connection, - path, - InputKind::Nfcapd, - nfcapd_decoder_fingerprint()?, - )?; - PreparedExplicitNfcapdKind::File(PreparedRevision { - revision, - snapshot: Some(snapshot), - }) - }; - prepared.push(PreparedExplicitNfcapd { - path: path.clone(), - source_id: source_id.clone(), - bucket_start, - kind, - }); - } - prepared.sort_unstable_by(|left, right| { - (left.bucket_start, &left.source_id, &left.path).cmp(&( - right.bucket_start, - &right.source_id, - &right.path, - )) - }); - if prepared.is_empty() { - return Ok(PipelineReport::default()); +fn resolve_dataset_request( + request: &PipelineRequest, + registry_path: &Path, + registry: &DatasetRegistry, + shared_nfdump: Option<(&Path, &ExecutableRevision)>, +) -> Result { + let dataset_id = request + .dataset_id + .as_deref() + .ok_or(PipelineError::MissingMode)?; + let dataset = registry.get(dataset_id)?.clone(); + let start_date = request.start_date.clone().ok_or_else(|| { + PipelineError::InvalidConfig("--start-date is required with --dataset".into()) + })?; + let configured_selection = selection_from_value(&dataset.selection)?; + let requested_selection = selection_from_value(&request.selection)?; + let selection = if requested_selection.is_unrestricted() { + configured_selection.clone() + } else { + requested_selection + }; + if selection != configured_selection && request.database_path.is_none() { + return Err(PipelineError::InvalidConfig( + "overriding a dataset's flow selection requires an explicit --database-path".into(), + )); } - process_atomic(connection, pipeline, |aggregates, report| { - for input in &prepared { - match &input.kind { - PreparedExplicitNfcapdKind::File(owner) => process_nfcapd( - connection, - &input.path, - &input.source_id, - input.bucket_start, - owner, - pipeline, - aggregates, - report, - )?, - PreparedExplicitNfcapdKind::Gap { expected_path } => process_nfcapd_gap( - connection, - &input.path, - expected_path.as_deref(), - &input.source_id, - input.bucket_start, - pipeline, - aggregates, - report, - )?, - } + let (nfdump, nfdump_revision) = match shared_nfdump { + Some((path, revision)) => (path.to_owned(), Some(revision.clone())), + None => { + let (path, revision) = resolve_nfdump_revision(&request.nfdump)?; + (path, Some(revision)) } - Ok(()) + }; + let mut control_paths = vec![registry_path.to_owned()]; + control_paths.push(nfdump.clone()); + Ok(ResolvedPipeline { + database_path: request + .database_path + .clone() + .unwrap_or_else(|| dataset.db_path.clone()), + control_paths, + timezone: DEFAULT_TIMEZONE.into(), + run_maad: request.run_maad, + nfdump, + nfdump_revision, + selection, + inputs: vec![InputSpec::NfcapdTree { + root_path: dataset.root_path.clone(), + source_ids: dataset.source_ids.clone(), + sources: dataset.sources.clone(), + start_date, + end_date: request.end_date.clone(), + start_time: request.start_time.clone(), + end_time: request.end_time.clone(), + force: request.force, + }], + datasets: vec![dataset], + require_complete: request.require_complete, }) } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd( - connection: &Connection, - path: &Path, - source_id: &str, - bucket_start: i64, - owner: &PreparedRevision, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - if nfcapd_logical_bucket_processed( - connection, - source_id, - bucket_start, - std::slice::from_ref(&owner.revision), - )? { - report.skipped_inputs += 1; - return Ok(()); - } - let bucket = ingest::read_nfcapd_bucket( - path, - source_id, - &pipeline.selection, - &pipeline.nfdump, - &pipeline.timezone, - )?; - let snapshot = owner - .snapshot - .as_ref() - .expect("explicit file input has a snapshot"); - verify_file_snapshot(path, snapshot)?; - aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; - publish_nfcapd_bucket( - connection, - &bucket, - std::slice::from_ref(owner), - &[], - &[InputEvidenceRow::new( - source_id, - source_id, - bucket_start, - bucket_start + FIVE_MINUTES, - &owner.revision.locator, - InputEvidenceState::Observed, - Some(owner.revision.fingerprint.clone()), - )], - false, - false, - pipeline.run_maad, - )?; - aggregates.include(&bucket, &pipeline.timezone)?; - report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; - report.five_minute_buckets += 1; - Ok(()) +fn selection_from_value(value: &Value) -> Result { + FlowSelection::from_payload((!value.is_null()).then_some(value)) } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_gap( - connection: &Connection, - _locator_path: &Path, - expected_path: Option<&Path>, - source_id: &str, - bucket_start: i64, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, +fn validate_selection_inputs( + selection: &FlowSelection, + inputs: &[InputSpec], ) -> Result<(), PipelineError> { - let expected_path = expected_path.ok_or_else(|| { - PipelineError::InvalidConfig( - "explicit nfcapd gap requires expected_path for absence verification".into(), - ) - })?; - let absence = ExpectedAbsence::capture(expected_path)?; - let evidence = [InputEvidenceRow::new( - source_id, - source_id, - bucket_start, - bucket_start + FIVE_MINUTES, - expected_path.to_string_lossy(), - InputEvidenceState::Missing, - None, - )]; - if query_input_evidence(connection, source_id, bucket_start)? == evidence { - report.skipped_inputs += 1; - return Ok(()); + if selection.selects_daily_active_sources() + && (inputs.len() != 1 || !matches!(inputs.first(), Some(InputSpec::NfcapdTree { .. }))) + { + return Err(PipelineError::InvalidConfig( + "daily_active_sources selection requires exactly one nfcapd_tree input".into(), + )); } - let bucket = StatisticalBucket::new(BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - )) - .with_coverage(BucketCoverage::new(1, 0, 0).map_err(DomainError::from)?) - .finish(); - aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; - publish_nfcapd_bucket( - connection, - &bucket, - &[], - &[absence], - &evidence, - false, - false, - pipeline.run_maad, - )?; - aggregates.include(&bucket, &pipeline.timezone)?; - report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; - report.five_minute_buckets += 1; Ok(()) } -fn normalize_sources( - root: &Path, - source_ids: &[String], - sources: &[DatasetSource], -) -> Result, PipelineError> { - if !source_ids.is_empty() && !sources.is_empty() { - return Err(PipelineError::InvalidConfig( - "nfcapd_tree cannot define both source_ids and sources".into(), - )); +fn execute(pipeline: ResolvedPipeline) -> Result { + let plan = plan_single_output(&pipeline)?; + if let Some(root) = plan.first_root() { + invoke_single_plan_hook(root); } - let mut normalized = if !sources.is_empty() { - sources.to_vec() - } else if !source_ids.is_empty() { - source_ids - .iter() - .map(|source_id| DatasetSource { - source_id: source_id.clone(), - members: vec![source_id.clone()], - }) - .collect() - } else { - let entries = fs::read_dir(root)?; - entries - .filter_map(Result::ok) - .filter_map(|entry| { - entry.file_type().ok()?.is_dir().then(|| { - let source_id = entry.file_name().to_string_lossy().into_owned(); - DatasetSource { - source_id: source_id.clone(), - members: vec![source_id], - } - }) - }) - .collect() - }; - normalized.sort_unstable_by(|left, right| left.source_id.cmp(&right.source_id)); - let mut ids = BTreeSet::new(); - for source in &normalized { - if !is_safe_path_component(&source.source_id) - || source.members.is_empty() - || !ids.insert(source.source_id.clone()) - { - return Err(PipelineError::InvalidConfig( - "logical sources require unique non-empty IDs and members".into(), - )); - } - let mut members = BTreeSet::new(); - for member in &source.members { - if !is_safe_path_component(member) || !members.insert(member) { - return Err(PipelineError::InvalidConfig(format!( - "source {:?} has an unsafe or duplicate member path component", - source.source_id - ))); - } - if !root.join(member).is_dir() { - return Err(PipelineError::InvalidConfig(format!( - "source {:?} references missing member directory {:?}", - source.source_id, member - ))); + verify_single_auto_source_layouts(&pipeline, &plan)?; + verify_nfdump_revision(&pipeline)?; + if let Some(parent) = pipeline.database_path.parent() { + fs::create_dir_all(parent)?; + } + let _lock = DatabaseOperationLock::acquire(&pipeline.database_path, "pipeline build")?; + let connection = connect_pipeline_writer(&pipeline.database_path)?; + init_schema(&connection)?; + initialize_metadata_with_plan(&connection, &pipeline, &plan)?; + + let mut report = PipelineReport::default(); + let mut csv_inputs = pipeline + .inputs + .iter() + .filter_map(|input| match input { + InputSpec::Csv { path, mapping_path } => Some(ingest::CsvInputSpec { + path: path.clone(), + mapping_path: mapping_path.clone(), + }), + _ => None, + }) + .collect::>(); + let explicit_nfcapd = pipeline + .inputs + .iter() + .filter(|input| matches!(input, InputSpec::Nfcapd { .. })) + .cloned() + .collect::>(); + for (input_index, input) in pipeline.inputs.iter().enumerate() { + match input { + InputSpec::Csv { .. } | InputSpec::Nfcapd { .. } => {} + InputSpec::CsvTree { + root_path, + mapping_path, + } => { + let mapping = CsvSourceConfig::load(mapping_path)?; + csv_inputs.extend(ingest::discover_csv_inputs( + root_path, + mapping_path, + &mapping, + )?); } + InputSpec::NfcapdTree { + start_date, + end_date, + start_time, + end_time, + force, + .. + } => process_nfcapd_tree( + &connection, + plan.trees + .get(&input_index) + .expect("every nfcapd_tree input has a frozen layout"), + start_date, + end_date.as_deref(), + start_time.as_deref(), + end_time.as_deref(), + *force, + &pipeline, + &plan.capture_snapshots, + &mut report, + )?, } } - Ok(normalized) -} - -fn merge_source_bucket( - source_id: &str, - bucket_start: i64, - expected_units: usize, - members: &[&CanonicalBucket], -) -> Result { - let key = BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, + merge_report( + &mut report, + process_csv_inputs(&connection, &csv_inputs, &pipeline)?, ); - let mut builder = if members.is_empty() { - StatisticalBucket::new(key) - } else { - StatisticalBucket::dense(key) + merge_report( + &mut report, + process_explicit_nfcapd_inputs(&connection, &explicit_nfcapd, &pipeline)?, + ); + infer_default_start_dates(&connection, &pipeline)?; + populate_coverage_summary(&connection, &mut report)?; + // This is the publication seam for the in-place pipeline product. Keep it before the strict + // coverage error so an incomplete-but-inspectable database also gets useful planner stats. + if let Err(error) = optimize_all_query_planner_statistics(&connection) { + tracing::warn!(%error, "could not refresh SQLite planner statistics"); } - .with_coverage(BucketCoverage::empty()); - for member in members { - builder.include(member)?; + if pipeline.require_complete { + let incomplete = + count_incomplete_requested_coverage_with_plan(&connection, &pipeline, &plan)?; + if incomplete != 0 { + return Err(PipelineError::IncompleteCoverage(incomplete)); + } } - let coverage = BucketCoverage::new( - u64::try_from(expected_units).unwrap_or(u64::MAX), - u64::try_from(members.len()).unwrap_or(u64::MAX), - 0, - ) - .map_err(DomainError::from)?; - Ok(builder.with_coverage(coverage).finish()) + Ok(report) } -fn logical_source_bucket<'a>( - source_id: &str, - bucket_start: i64, - expected_units: usize, - members: &[&'a CanonicalBucket], -) -> Result, PipelineError> { - let expected_key = BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - ); - if expected_units == 1 - && let [member] = members - && member.key == expected_key - { - return Ok(Cow::Borrowed(member)); - } - Ok(Cow::Owned(merge_source_bucket( - source_id, - bucket_start, - expected_units, - members, - )?)) +struct CoordinatedOutput { + pipeline: ResolvedPipeline, + sources: Vec, + connection: Connection, + _lock: DatabaseOperationLock, } -#[derive(Debug, Default)] -struct NfcapdBucketPublishProfile { - total_elapsed: Duration, - preflight_elapsed: Duration, - overlap_elapsed: Duration, - force_delete_elapsed: Duration, - owner_upsert_elapsed: Duration, - write: WriteBucketsProfile, - owner_status_elapsed: Duration, - postflight_elapsed: Duration, - owners: u64, - absences: u64, +type NfcapdFingerprintKey = (String, u64, u64, u64, i64, i64); + +/// Resume state for one output and one local day. +/// +/// Coordinated preparation revisits every logical source/bucket, but the state needed for those +/// decisions is limited to this day. Keeping the maps here avoids retaining prior days or loading +/// a product's complete provenance history into memory. +#[derive(Clone, Debug, Default)] +struct NfcapdDayResumeCache { + evidence: BTreeMap<(String, i64), Vec>, + processed: BTreeMap<(String, i64), BTreeSet<(String, String)>>, + fingerprints: BTreeMap, } -impl NfcapdBucketPublishProfile { - fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.preflight_elapsed += profile.preflight_elapsed; - self.overlap_elapsed += profile.overlap_elapsed; - self.force_delete_elapsed += profile.force_delete_elapsed; - self.owner_upsert_elapsed += profile.owner_upsert_elapsed; - self.write.include(profile.write); - self.owner_status_elapsed += profile.owner_status_elapsed; - self.postflight_elapsed += profile.postflight_elapsed; - self.owners += profile.owners; - self.absences += profile.absences; +impl NfcapdDayResumeCache { + fn load( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + ) -> Result { + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>() + .into_iter() + .collect::>(); + let mut cache = Self::default(); + for row in query_input_evidence_range(connection, &source_ids, start, end)? { + cache + .evidence + .entry((row.source_id.clone(), row.bucket_start)) + .or_default() + .push(row); + } + for row in query_processed_nfcapd_range(connection, &source_ids, start, end)? { + cache + .processed + .entry((row.source_id.clone(), row.bucket_start)) + .or_default() + .insert((row.input_locator.clone(), row.revision_fingerprint)); + if let Some(snapshot) = row.file_snapshot { + cache + .fingerprints + .entry(Self::fingerprint_key(&row.input_locator, &snapshot)) + .or_insert(row.content_fingerprint); + } + } + Ok(cache) } - fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.preflight_elapsed - + self.overlap_elapsed - + self.force_delete_elapsed - + self.owner_upsert_elapsed - + self.write.total_elapsed - + self.owner_status_elapsed - + self.postflight_elapsed, + fn fingerprint_key(locator: &str, snapshot: &FileSnapshot) -> NfcapdFingerprintKey { + ( + locator.to_owned(), + snapshot.device, + snapshot.inode, + snapshot.size, + snapshot.mtime_ns, + snapshot.ctime_ns, ) } -} -#[derive(Debug, Default)] -struct FinalRollupProfile { - total_elapsed: Duration, - finish_elapsed: Duration, - delete_elapsed: Duration, - write: WriteBucketsProfile, - incomplete_keys: u64, - rollup_buckets: u64, -} + fn evidence(&self, source_id: &str, bucket_start: i64) -> &[InputEvidenceRow] { + self.evidence + .get(&(source_id.to_owned(), bucket_start)) + .map(Vec::as_slice) + .unwrap_or(&[]) + } -impl FinalRollupProfile { - fn other_elapsed(&self) -> Duration { - self.total_elapsed - .saturating_sub(self.finish_elapsed + self.delete_elapsed + self.write.total_elapsed) + fn processed( + &self, + source_id: &str, + bucket_start: i64, + revisions: &[InputRevision], + ) -> Result { + if revisions.is_empty() { + return Ok(false); + } + let requested = revisions + .iter() + .map(|revision| (revision.locator.clone(), revision.fingerprint.clone())) + .collect::>(); + let stored = self + .processed + .get(&(source_id.to_owned(), bucket_start)) + .cloned() + .unwrap_or_default(); + let stored_locators = stored + .iter() + .map(|(locator, _)| locator) + .collect::>(); + let requested_locators = requested + .iter() + .map(|(locator, _)| locator) + .collect::>(); + if stored_locators == requested_locators && stored != requested { + return Err(PipelineError::Storage( + StorageError::InputRevisionConflict { + locator: format!("{source_id}:{bucket_start}"), + components: "nfcapd content or decoder; rerun with force to rewrite it" + .to_owned(), + }, + )); + } + Ok(stored == requested) + } + + fn cached_content_fingerprint(&self, path: &Path, snapshot: &FileSnapshot) -> Option { + self.fingerprints + .get(&Self::fingerprint_key(&path.to_string_lossy(), snapshot)) + .cloned() } } -#[derive(Debug, Default)] -struct AggregateGranularityProfile { - total_elapsed: Duration, - bounds_elapsed: Duration, - builder_elapsed: Duration, - bucket: StatisticalBucketIncludeProfile, +/// Worker pools shared by every day in a coordinated run. +/// +/// Revision hashing is needed while deciding whether a day has pending work, so it is built when +/// the run starts. Decode and activity work are both lazy: a complete no-op run never allocates +/// either pool, and a multi-day run reuses each pool after its first pending day. +struct CoordinatedPools { + revision: rayon::ThreadPool, + decode: Option, + activity: Option, } -impl AggregateGranularityProfile { - fn include( - &mut self, - total_elapsed: Duration, - bounds_elapsed: Duration, - builder_elapsed: Duration, - bucket: StatisticalBucketIncludeProfile, - ) { - self.total_elapsed += total_elapsed; - self.bounds_elapsed += bounds_elapsed; - self.builder_elapsed += builder_elapsed; - self.bucket.include(bucket); +impl CoordinatedPools { + fn new() -> Result { + Ok(Self { + revision: build_revision_hash_pool()?, + decode: None, + activity: None, + }) } - fn other_elapsed(&self) -> Duration { - self.total_elapsed - .saturating_sub(self.bounds_elapsed + self.builder_elapsed + self.bucket.total_elapsed) + fn decode(&mut self) -> Result<&rayon::ThreadPool, PipelineError> { + if self.decode.is_none() { + self.decode = Some(build_nfcapd_decode_pool()?); + } + Ok(self + .decode + .as_ref() + .expect("decode pool was just initialized")) + } + + fn activity(&mut self) -> Result<&rayon::ThreadPool, PipelineError> { + if self.activity.is_none() { + self.activity = Some(build_nfcapd_activity_pool()?); + } + Ok(self + .activity + .as_ref() + .expect("activity pool was just initialized")) } } +/// Prepared coordinated work retained between the day preflight and publication pass. +/// +/// Each entry is bounded by the existing nfcapd decode batch; the coordinated day retains one +/// such entry per batch. Publication takes ownership of each batch before decoding so prepared +/// evidence does not remain live alongside decoded buckets. +struct PreparedCoordinatedBatch { + prepared: BTreeMap>, +} + #[derive(Debug, Default)] -struct AggregateIncludeProfile { - total_elapsed: Duration, - thirty_minutes: AggregateGranularityProfile, - one_hour: AggregateGranularityProfile, - one_day: AggregateGranularityProfile, +struct CoordinatedDaySharedProfile { + revision_elapsed: Duration, + eligibility_elapsed: Duration, + activity_elapsed: Duration, + decode_elapsed: Duration, + revision_paths: u64, + activity_members: u64, + activity_inputs: u64, + active_set_counts: Vec, } -impl AggregateIncludeProfile { - fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.thirty_minutes.include( - profile.thirty_minutes.total_elapsed, - profile.thirty_minutes.bounds_elapsed, - profile.thirty_minutes.builder_elapsed, - profile.thirty_minutes.bucket, - ); - self.one_hour.include( - profile.one_hour.total_elapsed, - profile.one_hour.bounds_elapsed, - profile.one_hour.builder_elapsed, - profile.one_hour.bucket, - ); - self.one_day.include( - profile.one_day.total_elapsed, - profile.one_day.bounds_elapsed, - profile.one_day.builder_elapsed, - profile.one_day.bucket, +impl CoordinatedDaySharedProfile { + fn log(&self, day_start: i64, day_end: i64) { + tracing::info!( + target: "netflow_db::profile", + phase = "coordinated_day_shared", + day_start, + day_end, + revision_seconds = self.revision_elapsed.as_secs_f64(), + eligibility_seconds = self.eligibility_elapsed.as_secs_f64(), + activity_seconds = self.activity_elapsed.as_secs_f64(), + decode_seconds = self.decode_elapsed.as_secs_f64(), + revision_paths = self.revision_paths, + activity_members = self.activity_members, + activity_inputs = self.activity_inputs, + active_set_counts = ?self.active_set_counts, ); } +} - fn granularity_mut(&mut self, granularity: Granularity) -> &mut AggregateGranularityProfile { - match granularity { - Granularity::ThirtyMinutes => &mut self.thirty_minutes, - Granularity::OneHour => &mut self.one_hour, - Granularity::OneDay => &mut self.one_day, - Granularity::FiveMinutes => unreachable!("five-minute buckets are not rollups"), +struct CoordinatedPlan { + root_path: PathBuf, + sources: Vec, + dataset_sources: BTreeMap>, + physical_ids: Vec, + member_identities: BTreeMap, + auto_discovered_datasets: BTreeSet, + by_member_and_start: BTreeMap<(String, i64), PathBuf>, + capture_snapshots: BTreeMap, + member_bounds: BTreeMap, + start: i64, + end: i64, + extend_gaps_to_window: bool, + force: bool, + timezone: String, +} + +/// Resolve all read-only coordinated inputs before creating an output directory, lock, or +/// SQLite schema. The canonical root is also the shared locator root for every dataset output. +fn plan_coordinated(pipelines: &[ResolvedPipeline]) -> Result { + let first = pipelines + .first() + .ok_or_else(|| PipelineError::InvalidConfig("coordinated mode has no pipelines".into()))?; + let first_input = only_nfcapd_tree(first)?; + let first_config = nfcapd_tree_config(first_input)?; + let root_path = canonical_path(first_config.root_path)?; + let mut sources = None; + let mut dataset_sources = BTreeMap::new(); + let mut auto_discovered_datasets = BTreeSet::new(); + for pipeline in pipelines { + let input = only_nfcapd_tree(pipeline)?; + let config = nfcapd_tree_config(input)?; + if root_path != canonical_path(config.root_path)? { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfcapd root".into(), + )); + } + let resolved_sources = canonical_logical_sources(input)?; + if let Some(expected) = &sources { + if expected != &resolved_sources { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same logical source layout and membership" + .into(), + )); + } + } else { + sources = Some(resolved_sources.clone()); + } + let dataset_id = pipeline + .datasets + .first() + .map(|dataset| dataset.dataset_id.clone()) + .ok_or_else(|| { + PipelineError::InvalidConfig( + "coordinated datasets require registry-backed dataset metadata".into(), + ) + })?; + dataset_sources.insert(dataset_id.clone(), resolved_sources); + if matches!( + input, + InputSpec::NfcapdTree { + source_ids, + sources, + .. + } if source_ids.is_empty() && sources.is_empty() + ) { + auto_discovered_datasets.insert(dataset_id); } } + let sources = sources.expect("coordinated plan has at least one pipeline"); + let selected_start = parse_date_start(first_config.start_date, first.timezone.as_str())?; + let explicit_end = first_config + .end_date + .map(|date| next_date_start(date, first.timezone.as_str())) + .transpose()?; + let physical_ids = sources + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + validate_daily_active_source_layout(&sources, &physical_ids)?; + let member_identities = capture_member_directory_identities(&root_path, &physical_ids)?; - fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.thirty_minutes.total_elapsed - + self.one_hour.total_elapsed - + self.one_day.total_elapsed, - ) + // Parse explicit dates before discovery as well as before output setup. This keeps malformed + // finite windows side-effect free even when the source tree is large. + let discovery_started = Instant::now(); + let discovered = + ingest::discover_nfcapd_source_paths(&root_path, &physical_ids, first.timezone.as_str())?; + tracing::info!( + target: "netflow_db::profile", + phase = "coordinated_discovery", + elapsed_seconds = discovery_started.elapsed().as_secs_f64(), + physical_sources = physical_ids.len(), + discovered_inputs = discovered.len(), + ); + let mut by_member_and_start = BTreeMap::new(); + let mut member_bounds = BTreeMap::new(); + for input in discovered { + member_bounds + .entry(input.source_id.clone()) + .and_modify(|(first, last): &mut (i64, i64)| { + *first = (*first).min(input.bucket_start); + *last = (*last).max(input.bucket_start); + }) + .or_insert((input.bucket_start, input.bucket_start)); + by_member_and_start.insert((input.source_id, input.bucket_start), input.path); } -} + let discovered_end = by_member_and_start + .keys() + .map(|(_, bucket_start)| *bucket_start) + .max() + .map(|start| aggregate_bounds(start, Granularity::OneDay, first.timezone.as_str())) + .transpose()? + .map(|(_, end)| end) + .unwrap_or(selected_start); + let selected_end = explicit_end.unwrap_or(discovered_end); + let start = match first_config.start_time { + Some(value) => parse_local_datetime(value, first.timezone.as_str())?, + None => selected_start, + }; + let end = match first_config.end_time { + Some(value) => parse_local_datetime(value, first.timezone.as_str())?, + None => selected_end, + }; + validate_window( + selected_start, + selected_end, + start, + end, + first.timezone.as_str(), + )?; -#[derive(Debug, Default)] -struct NfcapdDayPublishProfile { - day_elapsed: Duration, - prepare_elapsed: Duration, - decode_elapsed: Duration, - batch_publish_elapsed: Duration, - logical_source_elapsed: Duration, - persisted_sibling_elapsed: Duration, - bucket_publish: NfcapdBucketPublishProfile, - aggregate_include: AggregateIncludeProfile, - completed_rollup_flush_elapsed: Duration, - completed_rollup_write: WriteBucketsProfile, - final_rollups: FinalRollupProfile, - logical_buckets: u64, - completed_rollup_flushes: u64, - nonempty_rollup_flushes: u64, + Ok(CoordinatedPlan { + root_path, + sources, + dataset_sources, + physical_ids, + member_identities, + auto_discovered_datasets, + by_member_and_start, + capture_snapshots: BTreeMap::new(), + member_bounds, + start, + end, + extend_gaps_to_window: first_config.end_date.is_some(), + force: first_config.force, + timezone: first.timezone.clone(), + }) } -impl NfcapdDayPublishProfile { - fn log(&self, day_start: i64, day_end: i64, transaction_elapsed: Duration) { - let mut rollup_write = self.completed_rollup_write.clone(); - rollup_write.include(self.final_rollups.write.clone()); - let publish_other = self.batch_publish_elapsed.saturating_sub( - self.logical_source_elapsed - + self.persisted_sibling_elapsed - + self.bucket_publish.total_elapsed - + self.aggregate_include.total_elapsed - + self.completed_rollup_flush_elapsed, - ); - let transaction_other = - transaction_elapsed.saturating_sub(self.day_elapsed + self.final_rollups.total_elapsed); - let completed_rollup_housekeeping = self - .completed_rollup_flush_elapsed - .saturating_sub(self.completed_rollup_write.total_elapsed); - tracing::info!( - target: "netflow_db::profile", - phase = "nfcapd_tree_day_publish_detail", - day_start, - day_end, - transaction_seconds = transaction_elapsed.as_secs_f64(), - transaction_other_seconds = transaction_other.as_secs_f64(), - day_seconds = self.day_elapsed.as_secs_f64(), - prepare_seconds = self.prepare_elapsed.as_secs_f64(), - decode_seconds = self.decode_elapsed.as_secs_f64(), - batch_publish_seconds = self.batch_publish_elapsed.as_secs_f64(), - publish_other_seconds = publish_other.as_secs_f64(), - logical_source_seconds = self.logical_source_elapsed.as_secs_f64(), - persisted_sibling_seconds = self.persisted_sibling_elapsed.as_secs_f64(), - bucket_publish_seconds = self.bucket_publish.total_elapsed.as_secs_f64(), - bucket_preflight_seconds = self.bucket_publish.preflight_elapsed.as_secs_f64(), - bucket_overlap_seconds = self.bucket_publish.overlap_elapsed.as_secs_f64(), - bucket_force_delete_seconds = self.bucket_publish.force_delete_elapsed.as_secs_f64(), - owner_upsert_seconds = self.bucket_publish.owner_upsert_elapsed.as_secs_f64(), - owner_status_seconds = self.bucket_publish.owner_status_elapsed.as_secs_f64(), - bucket_postflight_seconds = self.bucket_publish.postflight_elapsed.as_secs_f64(), - bucket_other_seconds = self.bucket_publish.other_elapsed().as_secs_f64(), - aggregate_include_seconds = self.aggregate_include.total_elapsed.as_secs_f64(), - aggregate_include_other_seconds = self.aggregate_include.other_elapsed().as_secs_f64(), - aggregate_30m_seconds = self.aggregate_include.thirty_minutes.total_elapsed.as_secs_f64(), - aggregate_30m_bounds_seconds = self.aggregate_include.thirty_minutes.bounds_elapsed.as_secs_f64(), - aggregate_30m_builder_seconds = self.aggregate_include.thirty_minutes.builder_elapsed.as_secs_f64(), - aggregate_30m_traffic_seconds = self.aggregate_include.thirty_minutes.bucket.traffic_elapsed.as_secs_f64(), - aggregate_30m_protocols_seconds = self.aggregate_include.thirty_minutes.bucket.protocols_elapsed.as_secs_f64(), - aggregate_30m_addresses_seconds = self.aggregate_include.thirty_minutes.bucket.addresses_elapsed.as_secs_f64(), - aggregate_30m_ports_seconds = self.aggregate_include.thirty_minutes.bucket.ports_elapsed.as_secs_f64(), - aggregate_30m_coverage_seconds = self.aggregate_include.thirty_minutes.bucket.coverage_elapsed.as_secs_f64(), - aggregate_30m_bucket_other_seconds = self.aggregate_include.thirty_minutes.bucket.other_elapsed().as_secs_f64(), - aggregate_30m_other_seconds = self.aggregate_include.thirty_minutes.other_elapsed().as_secs_f64(), - aggregate_1h_seconds = self.aggregate_include.one_hour.total_elapsed.as_secs_f64(), - aggregate_1h_bounds_seconds = self.aggregate_include.one_hour.bounds_elapsed.as_secs_f64(), - aggregate_1h_builder_seconds = self.aggregate_include.one_hour.builder_elapsed.as_secs_f64(), - aggregate_1h_traffic_seconds = self.aggregate_include.one_hour.bucket.traffic_elapsed.as_secs_f64(), - aggregate_1h_protocols_seconds = self.aggregate_include.one_hour.bucket.protocols_elapsed.as_secs_f64(), - aggregate_1h_addresses_seconds = self.aggregate_include.one_hour.bucket.addresses_elapsed.as_secs_f64(), - aggregate_1h_ports_seconds = self.aggregate_include.one_hour.bucket.ports_elapsed.as_secs_f64(), - aggregate_1h_coverage_seconds = self.aggregate_include.one_hour.bucket.coverage_elapsed.as_secs_f64(), - aggregate_1h_bucket_other_seconds = self.aggregate_include.one_hour.bucket.other_elapsed().as_secs_f64(), - aggregate_1h_other_seconds = self.aggregate_include.one_hour.other_elapsed().as_secs_f64(), - aggregate_1d_seconds = self.aggregate_include.one_day.total_elapsed.as_secs_f64(), - aggregate_1d_bounds_seconds = self.aggregate_include.one_day.bounds_elapsed.as_secs_f64(), - aggregate_1d_builder_seconds = self.aggregate_include.one_day.builder_elapsed.as_secs_f64(), - aggregate_1d_traffic_seconds = self.aggregate_include.one_day.bucket.traffic_elapsed.as_secs_f64(), - aggregate_1d_protocols_seconds = self.aggregate_include.one_day.bucket.protocols_elapsed.as_secs_f64(), - aggregate_1d_addresses_seconds = self.aggregate_include.one_day.bucket.addresses_elapsed.as_secs_f64(), - aggregate_1d_ports_seconds = self.aggregate_include.one_day.bucket.ports_elapsed.as_secs_f64(), - aggregate_1d_coverage_seconds = self.aggregate_include.one_day.bucket.coverage_elapsed.as_secs_f64(), - aggregate_1d_bucket_other_seconds = self.aggregate_include.one_day.bucket.other_elapsed().as_secs_f64(), - aggregate_1d_other_seconds = self.aggregate_include.one_day.other_elapsed().as_secs_f64(), - completed_rollup_flush_seconds = self.completed_rollup_flush_elapsed.as_secs_f64(), - completed_rollup_housekeeping_seconds = completed_rollup_housekeeping.as_secs_f64(), - final_rollup_seconds = self.final_rollups.total_elapsed.as_secs_f64(), - final_rollup_finish_seconds = self.final_rollups.finish_elapsed.as_secs_f64(), - final_rollup_delete_seconds = self.final_rollups.delete_elapsed.as_secs_f64(), - final_rollup_other_seconds = self.final_rollups.other_elapsed().as_secs_f64(), - five_minute_write_seconds = self.bucket_publish.write.total_elapsed.as_secs_f64(), - five_minute_delete_seconds = self.bucket_publish.write.delete_elapsed.as_secs_f64(), - five_minute_canonical_rows_seconds = self.bucket_publish.write.canonical_rows_elapsed.as_secs_f64(), - five_minute_scalar_rows_seconds = self.bucket_publish.write.scalar_rows_elapsed.as_secs_f64(), - five_minute_scalar_insert_seconds = scalar_insert_elapsed(&self.bucket_publish.write).as_secs_f64(), - five_minute_maad_seconds = self.bucket_publish.write.maad_elapsed.as_secs_f64(), - five_minute_address_structure_insert_seconds = self.bucket_publish.write.address_structure_insert_elapsed.as_secs_f64(), - five_minute_write_other_seconds = self.bucket_publish.write.other_elapsed().as_secs_f64(), - rollup_write_seconds = rollup_write.total_elapsed.as_secs_f64(), - rollup_delete_seconds = rollup_write.delete_elapsed.as_secs_f64(), - rollup_canonical_rows_seconds = rollup_write.canonical_rows_elapsed.as_secs_f64(), - rollup_scalar_rows_seconds = rollup_write.scalar_rows_elapsed.as_secs_f64(), - rollup_scalar_insert_seconds = scalar_insert_elapsed(&rollup_write).as_secs_f64(), - rollup_maad_seconds = rollup_write.maad_elapsed.as_secs_f64(), - rollup_address_structure_insert_seconds = rollup_write.address_structure_insert_elapsed.as_secs_f64(), - rollup_write_other_seconds = rollup_write.other_elapsed().as_secs_f64(), - logical_buckets = self.logical_buckets, - owners = self.bucket_publish.owners, - absences = self.bucket_publish.absences, - completed_rollup_flushes = self.completed_rollup_flushes, - nonempty_rollup_flushes = self.nonempty_rollup_flushes, - final_incomplete_keys = self.final_rollups.incomplete_keys, - final_rollup_buckets = self.final_rollups.rollup_buckets, - five_minute_write_calls = self.bucket_publish.write.write_calls, - rollup_write_calls = rollup_write.write_calls, - five_minute_bucket_keys = self.bucket_publish.write.bucket_keys, - rollup_bucket_keys = rollup_write.bucket_keys, - traffic_rows = self.bucket_publish.write.traffic_rows + rollup_write.traffic_rows, - protocol_rows = self.bucket_publish.write.protocol_rows + rollup_write.protocol_rows, - address_count_rows = self.bucket_publish.write.address_count_rows + rollup_write.address_count_rows, - port_count_rows = self.bucket_publish.write.port_count_rows + rollup_write.port_count_rows, - address_structure_rows = self.bucket_publish.write.address_structure_rows + rollup_write.address_structure_rows, - maad_address_sets = self.bucket_publish.write.maad_address_sets + rollup_write.maad_address_sets, - maad_addresses = self.bucket_publish.write.maad_addresses + rollup_write.maad_addresses, - address_structure_json_bytes = self.bucket_publish.write.address_structure_json_bytes + rollup_write.address_structure_json_bytes, - ); +/// Re-check every auto-discovered dataset after all coordinated read-only planning and before +/// creating any output parent, lock, or database. Explicit layouts were already identity-checked +/// by [`normalize_sources`] while the plan was built. +fn verify_coordinated_auto_source_layouts( + pipelines: &[ResolvedPipeline], + plan: &CoordinatedPlan, +) -> Result<(), PipelineError> { + for pipeline in pipelines { + let dataset = pipeline.datasets.first().ok_or_else(|| { + PipelineError::InvalidConfig( + "coordinated datasets require registry-backed dataset metadata".into(), + ) + })?; + if !plan.auto_discovered_datasets.contains(&dataset.dataset_id) { + continue; + } + let input = only_nfcapd_tree(pipeline)?; + let current = canonical_logical_sources(input)?; + let expected = plan + .dataset_sources + .get(&dataset.dataset_id) + .expect("every coordinated dataset has a frozen source layout"); + if current != *expected { + return Err(PipelineError::InvalidConfig(format!( + "auto-discovered source layout changed for dataset {:?} during coordinated planning", + dataset.dataset_id + ))); + } } + Ok(()) } -fn scalar_insert_elapsed(profile: &WriteBucketsProfile) -> Duration { - profile.traffic_insert_elapsed - + profile.protocol_insert_elapsed - + profile.address_count_insert_elapsed - + profile.port_count_insert_elapsed -} +/// Execute the shared physical nfcapd scan while keeping each logical product independent. +/// +/// Preparation is performed against every output so resume decisions remain output-local. Once a +/// batch is known to be needed, its physical files are decoded once and the canonical buckets are +/// fanned out to the outputs whose pending jobs reference them. +fn execute_many(pipelines: Vec) -> Result { + let output_paths = pipelines + .iter() + .map(|pipeline| pipeline.database_path.as_path()) + .collect::>(); + validate_output_input_separation( + &output_paths, + pipelines + .iter() + .flat_map(|pipeline| pipeline.control_paths.iter().map(PathBuf::as_path)), + "pipeline control path", + )?; + validate_database_path_separation(&output_paths)?; + let mut plan = plan_coordinated(&pipelines)?; + invoke_coordinated_plan_hook(&plan.root_path); + verify_member_directory_identities(&plan.root_path, &plan.member_identities)?; + for pipeline in &pipelines { + verify_nfdump_revision(pipeline)?; + } + let locator_namespaces = nfcapd_locator_namespaces(&plan.root_path, &plan.physical_ids)?; + if output_has_existing_identity(&output_paths)? { + let capture_paths = plan + .by_member_and_start + .values() + .cloned() + .collect::>(); + plan.capture_snapshots = capture_nfcapd_snapshots(&capture_paths)?; + validate_output_nfcapd_capture_separation_with_snapshots( + &output_paths, + &locator_namespaces, + plan.capture_snapshots + .iter() + .map(|(path, snapshot)| (path.as_path(), snapshot)), + )?; + } else { + validate_output_nfcapd_locator_separation(&output_paths, &locator_namespaces)?; + } + if !plan.auto_discovered_datasets.is_empty() { + validate_output_nfcapd_auto_namespace_separation( + &output_paths, + std::slice::from_ref(&plan.root_path), + )?; + } + if pipelines + .first() + .is_some_and(|pipeline| pipeline.nfdump_revision.is_some()) + { + ingest::probe_nfdump_compatibility(&pipelines[0].nfdump)?; + for pipeline in &pipelines { + verify_nfdump_revision(pipeline)?; + } + } + verify_coordinated_auto_source_layouts(&pipelines, &plan)?; + let mut lock_order = (0..pipelines.len()).collect::>(); + lock_order.sort_unstable_by_key(|index| normalized_path_key(&pipelines[*index].database_path)); + let mut locks: Vec> = + (0..pipelines.len()).map(|_| None).collect(); + for index in lock_order { + let pipeline = &pipelines[index]; + if let Some(parent) = pipeline.database_path.parent() { + fs::create_dir_all(parent)?; + } + locks[index] = Some(DatabaseOperationLock::acquire( + &pipeline.database_path, + "coordinated pipeline build", + )?); + } -fn profile_count(value: usize) -> u64 { - u64::try_from(value).unwrap_or(u64::MAX) -} + let mut outputs = Vec::with_capacity(pipelines.len()); + let mut initialization_transactions = vec![false; pipelines.len()]; + for (index, pipeline) in pipelines.into_iter().enumerate() { + let connection = match connect_pipeline_writer(&pipeline.database_path) { + Ok(connection) => connection, + Err(error) => { + rollback_coordinated_transactions(&outputs, &initialization_transactions); + return Err(coordinated_output_error(&pipeline, error.into())); + } + }; + outputs.push(CoordinatedOutput { + pipeline, + sources: plan.sources.clone(), + connection, + _lock: locks[index] + .take() + .expect("every coordinated output has a lock"), + }); + let initialization = (|| { + outputs[index] + .connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(StorageError::from)?; + initialization_transactions[index] = true; + init_schema(&outputs[index].connection)?; + initialize_coordinated_metadata_in_transaction( + &outputs[index].connection, + &outputs[index].pipeline, + &plan.sources, + &plan.dataset_sources, + ) + })(); + if let Err(error) = initialization { + rollback_coordinated_transactions(&outputs, &initialization_transactions); + return Err(coordinated_output_error(&outputs[index].pipeline, error)); + } + } + if let Err(error) = verify_coordinated_nfdump_revisions(&outputs) { + rollback_coordinated_transactions(&outputs, &initialization_transactions); + let pipeline = &outputs[0].pipeline; + return Err(coordinated_output_error(pipeline, error)); + } + for (index, output) in outputs.iter().enumerate() { + if let Err(error) = output.connection.execute_batch("COMMIT") { + rollback_coordinated_transactions(&outputs, &initialization_transactions); + return Err(coordinated_output_error( + &output.pipeline, + PipelineError::Storage(StorageError::from(error)), + )); + } + initialization_transactions[index] = false; + } -#[derive(Clone, Debug)] -struct PreparedRevision { - revision: InputRevision, - snapshot: Option, + let mut pools = CoordinatedPools::new()?; + let mut report = PipelineReport::default(); + let mut day_start = plan.start; + while day_start < plan.end { + verify_member_directory_identities(&plan.root_path, &plan.member_identities)?; + let day_end = aggregate_bounds(day_start, Granularity::OneDay, &plan.timezone)?.1; + let capture_complete = day_capture_is_complete( + &plan.sources, + &plan.by_member_and_start, + day_start, + day_end, + &plan.timezone, + )?; + let mut stale_outputs = Vec::new(); + let mut canonical_day_verified = vec![false; outputs.len()]; + let mut marker_needs_backfill = vec![false; outputs.len()]; + for (index, output) in outputs.iter().enumerate() { + let published_day = + day_was_published(&output.connection, &output.sources, day_start, day_end)?; + if !capture_complete { + if published_day { + stale_outputs.push(index); + } + } else if published_day && !plan.force { + match nfcapd_day_completion_state( + &output.connection, + &output.sources, + day_start, + day_end, + output.pipeline.run_maad, + )? { + DailyProductCompletionState::Clean => { + canonical_day_verified[index] = true; + } + DailyProductCompletionState::Dirty => { + return Err(PipelineError::InvalidConfig(format!( + "published local day {day_start}..{day_end} was mutated after completion for database {}; rerun that whole day with --force", + output.pipeline.database_path.display() + ))); + } + DailyProductCompletionState::Missing => { + if !nfcapd_day_has_canonical_topology( + &output.connection, + &output.sources, + day_start, + day_end, + &plan.timezone, + output.pipeline.run_maad, + )? { + return Err(PipelineError::InvalidConfig(format!( + "published local day {day_start}..{day_end} has damaged canonical topology for database {}; rerun that whole day with --force", + output.pipeline.database_path.display() + ))); + } + canonical_day_verified[index] = true; + marker_needs_backfill[index] = true; + } + } + } + } + let reset_outputs = if plan.force { + (0..outputs.len()).collect::>() + } else { + stale_outputs + }; + if !reset_outputs.is_empty() && !plan.force { + return Err(PipelineError::InvalidConfig(format!( + "published local day {day_start}..{day_end} no longer has complete nfcapd capture coverage; rerun that day with --force" + ))); + } + let missing = missing_physical_day_inputs( + &plan.physical_ids, + &plan.by_member_and_start, + day_start, + day_end, + &plan.timezone, + )?; + let missing_absences = build_missing_day_absences( + &plan.root_path, + &missing, + day_start, + day_end, + &plan.timezone, + )?; + invoke_missing_day_absence_hook(&plan.root_path, &missing, &plan.timezone); + if !missing.is_empty() { + let missing_details = + missing_day_warning_details(&plan.root_path, &missing, &plan.timezone)?; + tracing::warn!( + day_start, + day_end, + missing_inputs = missing.len(), + missing_details = %missing_details, + "skipping incomplete physical day for coordinated selections" + ); + report.skipped_inputs += missing.len(); + if reset_outputs.is_empty() { + day_start = day_end; + continue; + } + } + + let day_reports = process_coordinated_day( + &mut outputs, + &plan.root_path, + &plan.physical_ids, + &plan.member_identities, + &plan.by_member_and_start, + &plan.member_bounds, + day_start, + day_end, + plan.extend_gaps_to_window, + plan.force, + &reset_outputs, + !missing.is_empty(), + &missing_absences, + &canonical_day_verified, + &marker_needs_backfill, + &plan.capture_snapshots, + &mut pools, + )?; + for day_report in day_reports { + merge_report(&mut report, day_report); + } + day_start = day_end; + } + + for output in &outputs { + infer_default_start_dates(&output.connection, &output.pipeline)?; + let mut coverage_report = PipelineReport::default(); + populate_coverage_summary(&output.connection, &mut coverage_report)?; + report.complete_five_minute_buckets += coverage_report.complete_five_minute_buckets; + report.partial_five_minute_buckets += coverage_report.partial_five_minute_buckets; + report.unknown_five_minute_buckets += coverage_report.unknown_five_minute_buckets; + if let Err(error) = optimize_all_query_planner_statistics(&output.connection) { + tracing::warn!(%error, "could not refresh SQLite planner statistics"); + } + if output.pipeline.require_complete { + let incomplete = count_incomplete_coverage_for_layout( + &output.connection, + &plan.sources, + plan.start, + plan.end, + &plan.timezone, + )?; + if incomplete != 0 { + let dataset_id = output + .pipeline + .datasets + .first() + .map_or("", |dataset| dataset.dataset_id.as_str()); + return Err(PipelineError::InvalidConfig(format!( + "dataset {dataset_id:?} database {} has {incomplete} incomplete five-minute coverage buckets", + output.pipeline.database_path.display() + ))); + } + } + } + Ok(report) } -struct PreparedTreeJob { - source_id: String, - expected_units: usize, - present: Vec<(String, PathBuf)>, - owners: Vec, - absences: Vec, - evidence: Vec, - is_repair: bool, +fn verify_coordinated_postflight_snapshot( + path: &Path, + snapshot: &FileSnapshot, +) -> Result<(), ProvenanceError> { + #[cfg(test)] + COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS.with(|calls| calls.set(calls.get() + 1)); + verify_file_snapshot(path, snapshot) } -struct PreparedTreeTimestamp { - bucket_start: i64, - revision_cache: BTreeMap, - jobs: Vec, +fn verify_coordinated_nfdump_revisions(outputs: &[CoordinatedOutput]) -> Result<(), PipelineError> { + for output in outputs { + verify_nfdump_revision(&output.pipeline)?; + } + Ok(()) } +/// Check every external input guard while all coordinated output transactions are still open. +/// +/// This is deliberately the last fallible phase before the commit loop. Keeping the loop itself +/// to COMMIT and transaction bookkeeping prevents a late capture or decoder replacement from +/// making one output commit while another rolls back. #[allow(clippy::too_many_arguments)] -fn publish_nfcapd_bucket( - connection: &Connection, - bucket: &CanonicalBucket, - owners: &[PreparedRevision], - absences: &[ExpectedAbsence], - evidence: &[InputEvidenceRow], - allow_coverage_repair: bool, - force: bool, - run_maad: bool, +fn verify_coordinated_precommit_guards<'a, 'b>( + outputs: &[CoordinatedOutput], + root: &Path, + member_identities: &BTreeMap, + revisions: impl IntoIterator, + activity_snapshots: impl IntoIterator, + missing_absences: &[ExpectedAbsence], + start: i64, + end: i64, ) -> Result<(), PipelineError> { - publish_nfcapd_bucket_profiled( - connection, - bucket, - owners, - absences, - evidence, - allow_coverage_repair, - force, - run_maad, - ) - .map(|_| ()) + invoke_coordinated_commit_guard_hook(); + verify_member_directory_identities(root, member_identities)?; + for (path, snapshot) in revisions { + verify_coordinated_postflight_snapshot(path, snapshot)?; + } + for (path, snapshot) in activity_snapshots { + verify_coordinated_postflight_snapshot(path, snapshot)?; + } + verify_coordinated_nfdump_revisions(outputs)?; + verify_missing_day_absences(missing_absences, start, end) } #[allow(clippy::too_many_arguments)] -fn publish_nfcapd_bucket_profiled( - connection: &Connection, - bucket: &CanonicalBucket, - owners: &[PreparedRevision], - absences: &[ExpectedAbsence], - evidence: &[InputEvidenceRow], - allow_coverage_repair: bool, +fn process_coordinated_day( + outputs: &mut [CoordinatedOutput], + root: &Path, + physical_ids: &[String], + member_identities: &BTreeMap, + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + start: i64, + end: i64, + extend_gaps_to_window: bool, force: bool, - run_maad: bool, -) -> Result { - let total_started = Instant::now(); - let mut profile = NfcapdBucketPublishProfile { - owners: profile_count(owners.len()), - absences: profile_count(absences.len()), - ..NfcapdBucketPublishProfile::default() - }; - let preflight_started = Instant::now(); - for absence in absences { - absence.verify()?; - } - for owner in owners { - if let Some(snapshot) = &owner.snapshot { - verify_file_snapshot(&owner.revision.locator, snapshot)?; + reset_outputs: &[usize], + skip_incomplete_day: bool, + missing_absences: &[ExpectedAbsence], + canonical_day_verified: &[bool], + marker_needs_backfill: &[bool], + capture_snapshots: &BTreeMap, + pools: &mut CoordinatedPools, +) -> Result, PipelineError> { + verify_member_directory_identities(root, member_identities)?; + verify_coordinated_nfdump_revisions(outputs)?; + let reset_set = reset_outputs.iter().copied().collect::>(); + if skip_incomplete_day { + if reset_outputs.is_empty() { + return Ok((0..outputs.len()) + .map(|_| PipelineReport::default()) + .collect()); } - } - profile.preflight_elapsed += preflight_started.elapsed(); - let overlap_started = Instant::now(); - reject_overlapping_bucket( - connection, - bucket, - InputKind::Nfcapd, - "", - force || allow_coverage_repair, - )?; - profile.overlap_elapsed += overlap_started.elapsed(); - if force { - let force_delete_started = Instant::now(); - connection.execute( - "DELETE FROM processed_inputs WHERE input_kind = 'nfcapd' AND source_id = ?1 AND bucket_start = ?2", - params![bucket.key.source_id, bucket.key.bucket_start], - ).map_err(StorageError::from)?; - profile.force_delete_elapsed += force_delete_started.elapsed(); - } - let publication = (|| -> Result<(), PipelineError> { - let owner_upsert_started = Instant::now(); - for prepared in owners { - let revision = &prepared.revision; - let owner = InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: revision.locator.clone(), - scan_locator: revision.locator.clone(), - source_id: bucket.key.source_id.clone(), - bucket_start: bucket.key.bucket_start, - bucket_end: bucket.key.bucket_end, - revision: revision.clone(), - file_snapshot: prepared.snapshot.clone(), - }; - upsert_input_bucket(connection, &owner, force)?; + let mut transactions = (0..outputs.len()).map(|_| false).collect::>(); + for &index in reset_outputs { + if let Err(error) = outputs[index].connection.execute_batch("BEGIN IMMEDIATE") { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = true; + let source_ids = outputs[index] + .sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + if let Err(error) = + delete_stats_time_range(&outputs[index].connection, &source_ids, start, end) + { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(error)); + } } - profile.owner_upsert_elapsed += owner_upsert_started.elapsed(); - profile.write = write_buckets_profiled(connection, std::slice::from_ref(bucket), run_maad)?; - replace_input_evidence( - connection, - &bucket.key.source_id, - bucket.key.bucket_start, - evidence, - )?; - let owner_status_started = Instant::now(); - for prepared in owners { - let revision = &prepared.revision; - mark_input_bucket_status( - connection, - InputKind::Nfcapd, - &revision.locator, - &bucket.key.source_id, - bucket.key.bucket_start, - InputStatus::Processed, - revision, - None, - )?; + if let Err(error) = verify_coordinated_precommit_guards( + outputs, + root, + member_identities, + std::iter::empty(), + std::iter::empty(), + missing_absences, + start, + end, + ) { + rollback_coordinated_transactions(outputs, &transactions); + return Err(error); } - profile.owner_status_elapsed += owner_status_started.elapsed(); - let postflight_started = Instant::now(); - for absence in absences { - absence.verify()?; + for &index in reset_outputs { + if let Err(error) = outputs[index].connection.execute_batch("COMMIT") { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = false; } - profile.postflight_elapsed += postflight_started.elapsed(); - Ok(()) - })(); - publication?; - profile.total_elapsed = total_started.elapsed(); - Ok(profile) -} - -fn reject_overlapping_bucket( - connection: &Connection, - bucket: &CanonicalBucket, - input_kind: InputKind, - allowed_scan: &str, - replace_nfcapd: bool, -) -> Result<(), PipelineError> { - let conflict = connection - .query_row( - "SELECT input_kind, input_locator, scan_locator FROM processed_inputs - WHERE source_id = ?1 AND bucket_start = ?2 - AND NOT (input_kind = ?3 AND scan_locator = ?4) - ORDER BY input_kind, input_locator LIMIT 1", - params![ - bucket.key.source_id, - bucket.key.bucket_start, - input_kind.as_str(), - allowed_scan, - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .optional() - .map_err(StorageError::from)?; - if let Some((kind, locator, _)) = conflict - && !(replace_nfcapd && kind == InputKind::Nfcapd.as_str()) - { - return Err(PipelineError::InvalidConfig(format!( - "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", - bucket.key.source_id, bucket.key.bucket_start - ))); + return Ok((0..outputs.len()) + .map(|_| PipelineReport::default()) + .collect()); } - Ok(()) -} -#[derive(Default)] -struct AggregateBuckets { - builders: BTreeMap<(String, Granularity, i64, i64), StatisticalBucket>, - published_through: BTreeMap, - owned_keys: BTreeSet<(String, i64)>, - current_run_keys: BTreeSet<(String, i64)>, -} + let resume_caches = outputs + .iter() + .map(|output| NfcapdDayResumeCache::load(&output.connection, &output.sources, start, end)) + .collect::, _>>()?; -impl AggregateBuckets { - fn with_owned_keys(owned_keys: BTreeSet<(String, i64)>) -> Self { - Self { - owned_keys, - ..Self::default() + let mut owned_keys = BTreeSet::new(); + let mut bucket_start = start; + while bucket_start < end { + for source in &outputs[0].sources { + if (force || !reset_set.is_empty()) + && source_has_candidate( + source, + bucket_start, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + ) + { + owned_keys.insert((source.source_id.clone(), bucket_start)); + } } + bucket_start = next_local_five_minute_start(bucket_start, &outputs[0].pipeline.timezone)?; } - fn reject_persisted_siblings( - &self, - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, - ) -> Result<(), PipelineError> { - self.reject_persisted_siblings_inner(connection, child, timezone, false) - } - - fn reject_persisted_csv_siblings( - &self, - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, - ) -> Result<(), PipelineError> { - self.reject_persisted_siblings_inner(connection, child, timezone, true) - } - - fn reject_persisted_siblings_inner( - &self, - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, - allow_staged_csv_keys: bool, - ) -> Result<(), PipelineError> { - let (day_start, day_end) = - aggregate_bounds(child.key.bucket_start, Granularity::OneDay, timezone)?; - let mut statement = connection - .prepare( - "SELECT DISTINCT bucket_start FROM traffic_stats - WHERE source_id = ?1 AND granularity = '5m' - AND bucket_start >= ?2 AND bucket_start < ?3 - ORDER BY bucket_start", - ) - .map_err(StorageError::from)?; - let persisted = statement - .query_map(params![child.key.source_id, day_start, day_end], |row| { - row.get::<_, i64>(0) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - for bucket_start in persisted { - if bucket_start == child.key.bucket_start - || self - .owned_keys - .contains(&(child.key.source_id.clone(), bucket_start)) - || self - .current_run_keys - .contains(&(child.key.source_id.clone(), bucket_start)) - { - continue; + let mut reports = (0..outputs.len()) + .map(|_| PipelineReport::default()) + .collect::>(); + let mut profiles = (0..outputs.len()) + .map(|_| NfcapdDayPublishProfile::default()) + .collect::>(); + let mut shared_profile = CoordinatedDaySharedProfile { + active_set_counts: vec![0; outputs.len()], + ..CoordinatedDaySharedProfile::default() + }; + let mut pending_set = BTreeSet::new(); + let mut has_repair = false; + let mut batches = Vec::new(); + let mut all_revisions = BTreeMap::new(); + let mut next = start; + while next < end { + let batch_starts = nfcapd_batch_starts( + next, + end, + &outputs[0].pipeline.timezone, + &outputs[0].sources, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + )?; + next = batch_starts + .last() + .copied() + .map(|last| next_local_five_minute_start(last, &outputs[0].pipeline.timezone)) + .transpose()? + .expect("non-empty coordinated nfcapd batch while processing a non-empty window"); + let revision_started = Instant::now(); + let revisions = resolve_coordinated_batch_revisions_with_cache( + outputs, + &outputs[0].sources, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + force, + &pools.revision, + &batch_starts, + &resume_caches, + capture_snapshots, + )?; + shared_profile.revision_elapsed += revision_started.elapsed(); + shared_profile.revision_paths += profile_count(revisions.len()); + all_revisions.extend( + revisions + .iter() + .map(|(path, revision)| (path.clone(), revision.clone())), + ); + let mut prepared = BTreeMap::new(); + for (index, output) in outputs.iter().enumerate() { + let output_prepare_started = Instant::now(); + let mut output_batch = Vec::with_capacity(batch_starts.len()); + for &bucket_start in &batch_starts { + let mut preflight_report = PipelineReport::default(); + let timestamp = prepare_nfcapd_tree_timestamp_with_cache( + &output.connection, + root, + &output.sources, + by_member_and_start, + member_bounds, + bucket_start, + extend_gaps_to_window, + force || reset_set.contains(&index), + &output.pipeline, + &mut preflight_report, + &revisions, + canonical_day_verified[index], + Some(&resume_caches[index]), + )?; + reports[index].skipped_inputs += preflight_report.skipped_inputs; + if !timestamp.jobs.is_empty() { + pending_set.insert(index); + } + has_repair |= timestamp.jobs.iter().any(|job| job.is_repair); + output_batch.push(timestamp); } - if allow_staged_csv_keys - && connection - .query_row( - "SELECT 1 FROM csv_bucket_stage - WHERE source_id = ?1 AND bucket_start = ?2 LIMIT 1", - params![child.key.source_id, bucket_start], - |_| Ok(()), - ) - .optional() - .map_err(StorageError::from)? - .is_some() + let output_prepare_elapsed = output_prepare_started.elapsed(); + profiles[index].prepare_elapsed += output_prepare_elapsed; + shared_profile.eligibility_elapsed += output_prepare_elapsed; + if output_batch + .iter() + .any(|timestamp| !timestamp.jobs.is_empty()) { - continue; + prepared.insert(index, output_batch); } - return Err(PipelineError::InvalidConfig(format!( - "cannot reopen a persisted aggregate interval exactly: source={:?} bucket_start={} shares its local day with persisted five-minute bucket {bucket_start} from another transaction", - child.key.source_id, child.key.bucket_start - ))); } - Ok(()) + batches.push(PreparedCoordinatedBatch { prepared }); } - - fn include(&mut self, child: &CanonicalBucket, timezone: &str) -> Result<(), PipelineError> { - self.include_profiled(child, timezone).map(|_| ()) + if !force && has_repair { + return Err(PipelineError::InvalidConfig(format!( + "daily_active_sources input changed for local day {start}..{end}; rerun that whole day with --force" + ))); } - fn include_profiled( - &mut self, - child: &CanonicalBucket, - timezone: &str, - ) -> Result { - let total_started = Instant::now(); - let mut profile = AggregateIncludeProfile::default(); - if self - .published_through - .get(&child.key.source_id) - .is_some_and(|previous| child.key.bucket_start <= *previous) - { - return Err(PipelineError::InvalidConfig(format!( - "five-minute buckets must be unique and chronological for source {:?}: {} followed {}", - child.key.source_id, - self.published_through[&child.key.source_id], - child.key.bucket_start - ))); + let pending = pending_set.into_iter().collect::>(); + let mut activity_snapshots = Vec::new(); + let mut active_sources = (0..outputs.len()) + .map(|_| None) + .collect::>>>(); + if !pending.is_empty() { + verify_coordinated_nfdump_revisions(outputs)?; + let activity_started = Instant::now(); + let selections = pending + .iter() + .map(|index| outputs[*index].pipeline.selection.clone()) + .collect::>(); + let activity_pool = pools.activity()?; + let (resolved_active_sources, snapshots) = resolve_coordinated_daily_active_sources( + physical_ids, + by_member_and_start, + start, + end, + &outputs[0].pipeline.timezone, + &selections, + outputs[0].pipeline.nfdump.as_path(), + activity_pool, + capture_snapshots, + &all_revisions, + )?; + shared_profile.activity_elapsed += activity_started.elapsed(); + shared_profile.activity_members = profile_count(physical_ids.len()); + activity_snapshots = snapshots; + verify_coordinated_nfdump_revisions(outputs)?; + shared_profile.activity_inputs = profile_count(activity_snapshots.len()); + for (path, snapshot) in &activity_snapshots { + verify_file_snapshot(path, snapshot)?; } - for granularity in [ - Granularity::ThirtyMinutes, - Granularity::OneHour, - Granularity::OneDay, - ] { - let granularity_started = Instant::now(); - let bounds_started = Instant::now(); - let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; - let bounds_elapsed = bounds_started.elapsed(); - let key = (child.key.source_id.clone(), granularity, start, end); - let builder_started = Instant::now(); - let builder = self.builders.entry(key.clone()).or_insert_with(|| { - StatisticalBucket::new(BucketKey::new(&key.0, key.1, key.2, key.3)) - }); - let builder_elapsed = builder_started.elapsed(); - let bucket = builder.include_profiled(child)?; - profile.granularity_mut(granularity).include( - granularity_started.elapsed(), - bounds_elapsed, - builder_elapsed, - bucket, - ); + for (pending_index, active) in pending.iter().zip(resolved_active_sources) { + let active_count = profile_count(active.len()); + profiles[*pending_index].active_set_count = active_count; + shared_profile.active_set_counts[*pending_index] = active_count; + active_sources[*pending_index] = Some(active); } - self.published_through - .insert(child.key.source_id.clone(), child.key.bucket_start); - self.current_run_keys - .insert((child.key.source_id.clone(), child.key.bucket_start)); - profile.total_elapsed = total_started.elapsed(); - Ok(profile) } - fn flush_complete( - &mut self, - connection: &Connection, - run_maad: bool, - ) -> Result { - self.flush_complete_profiled(connection, run_maad) - .map(|(count, _)| count) + if pending.is_empty() + && reset_outputs.is_empty() + && !marker_needs_backfill.iter().copied().any(|needed| needed) + { + shared_profile.log(start, end); + return Ok(reports); } - fn flush_complete_profiled( - &mut self, - connection: &Connection, - run_maad: bool, - ) -> Result<(usize, WriteBucketsProfile), PipelineError> { - let complete_keys = self - .builders + let transaction_indices = (0..outputs.len()) + .filter(|index| { + reset_set.contains(index) || pending.contains(index) || marker_needs_backfill[*index] + }) + .collect::>(); + let mut transactions = (0..outputs.len()).map(|_| false).collect::>(); + let transaction_started = Instant::now(); + let mut aggregates = (0..outputs.len()) + .map(|index| { + pending + .contains(&index) + .then(|| AggregateBuckets::with_owned_keys(owned_keys.clone())) + }) + .collect::>(); + for &index in &transaction_indices { + if let Err(error) = outputs[index].connection.execute_batch("BEGIN IMMEDIATE") { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = true; + if reset_set.contains(&index) { + let source_ids = outputs[index] + .sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + if let Err(error) = + delete_stats_time_range(&outputs[index].connection, &source_ids, start, end) + { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(error)); + } + } + let source_ids = outputs[index] + .sources .iter() - .filter(|(_, builder)| builder.has_complete_five_minute_coverage()) - .map(|(key, _)| key.clone()) + .map(|source| source.source_id.clone()) .collect::>(); - let buckets = complete_keys - .into_iter() - .filter_map(|key| self.builders.remove(&key)) - .map(|builder| builder.finish()) - .collect::>(); - let count = buckets.len(); - let profile = write_buckets_profiled(connection, &buckets, run_maad)?; - Ok((count, profile)) + if let Err(error) = provision_daily_product_completion_bucket_guards( + &outputs[index].connection, + &source_ids, + start, + end, + ) { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(error)); + } } - fn finish(self) -> (Vec, Vec) { - ( - self.builders - .into_values() - .map(|builder| builder.finish()) - .collect(), - Vec::new(), + let result = if pending.is_empty() { + Ok(()) + } else { + verify_coordinated_nfdump_revisions(outputs)?; + let decode_pool = pools.decode()?; + process_coordinated_batches( + outputs, + &pending, + &mut reports, + &mut profiles, + &mut shared_profile, + &mut aggregates, + by_member_and_start, + force, + &mut batches, + &all_revisions, + &active_sources, + decode_pool, ) + }; + if let Err(error) = result { + rollback_coordinated_transactions(outputs, &transactions); + return Err(error); + } + let day_publish_elapsed = transaction_started.elapsed(); + for &index in &pending { + let aggregates = aggregates[index] + .take() + .expect("pending output has aggregate state"); + let final_profile = match publish_rollups_profiled( + &outputs[index].connection, + aggregates, + &outputs[index].pipeline, + &mut reports[index], + ) { + Ok(profile) => profile, + Err(error) => { + rollback_coordinated_transactions(outputs, &transactions); + return Err(error); + } + }; + profiles[index].final_rollups = final_profile; + } + let revision_snapshots = all_revisions.values().filter_map(|revision| { + revision + .snapshot + .as_ref() + .map(|snapshot| (Path::new(&revision.revision.locator), snapshot)) + }); + let activity_snapshot_refs = activity_snapshots + .iter() + .map(|(path, snapshot)| (path.as_path(), snapshot)); + if let Err(error) = verify_coordinated_precommit_guards( + outputs, + root, + member_identities, + revision_snapshots, + activity_snapshot_refs, + missing_absences, + start, + end, + ) { + rollback_coordinated_transactions(outputs, &transactions); + return Err(error); + } + if !skip_incomplete_day { + for &index in &transaction_indices { + if let Err(error) = mark_nfcapd_day_complete( + &outputs[index].connection, + &outputs[index].sources, + start, + end, + outputs[index].pipeline.run_maad, + ) { + rollback_coordinated_transactions(outputs, &transactions); + return Err(error); + } + } + } + for &index in &transaction_indices { + if let Err(error) = outputs[index].connection.execute_batch("COMMIT") { + rollback_coordinated_transactions(outputs, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = false; + } + let transaction_elapsed = transaction_started.elapsed(); + shared_profile.log(start, end); + for &index in &pending { + profiles[index].day_elapsed = day_publish_elapsed; + profiles[index].log_coordinated( + start, + end, + transaction_elapsed, + index, + &outputs[index].pipeline.database_path, + ); } + Ok(reports) } -fn publish_rollups( - connection: &Connection, - aggregates: AggregateBuckets, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - publish_rollups_profiled(connection, aggregates, pipeline, report).map(|_| ()) +#[allow(clippy::too_many_arguments)] +#[allow(dead_code)] +fn resolve_coordinated_batch_revisions( + outputs: &[CoordinatedOutput], + sources: &[DatasetSource], + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, + force: bool, + revision_pool: &rayon::ThreadPool, + batch_starts: &[i64], +) -> Result, PipelineError> { + resolve_coordinated_batch_revisions_with_cache( + outputs, + sources, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + force, + revision_pool, + batch_starts, + &[], + &BTreeMap::new(), + ) } -fn publish_rollups_profiled( - connection: &Connection, - aggregates: AggregateBuckets, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result { - let total_started = Instant::now(); - let finish_started = Instant::now(); - let (rollups, incomplete) = aggregates.finish(); - let finish_elapsed = finish_started.elapsed(); - let delete_started = Instant::now(); - delete_stats_bucket_keys(connection, &incomplete)?; - let delete_elapsed = delete_started.elapsed(); - let write = write_buckets_profiled(connection, &rollups, pipeline.run_maad)?; - report.rollup_buckets += rollups.len(); - Ok(FinalRollupProfile { - total_elapsed: total_started.elapsed(), - finish_elapsed, - delete_elapsed, - write, - incomplete_keys: profile_count(incomplete.len()), - rollup_buckets: profile_count(rollups.len()), - }) -} +#[allow(clippy::too_many_arguments)] +fn resolve_coordinated_batch_revisions_with_cache( + outputs: &[CoordinatedOutput], + sources: &[DatasetSource], + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, + force: bool, + revision_pool: &rayon::ThreadPool, + batch_starts: &[i64], + resume_caches: &[NfcapdDayResumeCache], + capture_snapshots: &BTreeMap, +) -> Result, PipelineError> { + let mut paths = BTreeSet::new(); + for &bucket_start in batch_starts { + for source in sources { + if !source_has_candidate( + source, + bucket_start, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + ) { + continue; + } + paths.extend(source.members.iter().filter_map(|member| { + by_member_and_start + .get(&(member.clone(), bucket_start)) + .cloned() + })); + } + } + let decoder_fingerprint = nfdump_decoder_fingerprint_for_pipeline(&outputs[0].pipeline)?; + let probes = paths + .into_iter() + .map(|path| { + let observed = capture_snapshots + .get(&path) + .cloned() + .map(Ok) + .unwrap_or_else(|| capture_nfcapd_snapshot(&path))?; + let cached = if force { + None + } else { + let mut shared = None; + let mut conflict = false; + for (index, output) in outputs.iter().enumerate() { + let fingerprint = match resume_caches.get(index) { + Some(cache) => cache.cached_content_fingerprint(&path, &observed), + None => cached_content_fingerprint( + &output.connection, + InputKind::Nfcapd, + &path.to_string_lossy(), + &observed, + )?, + }; + match (&shared, fingerprint) { + (None, Some(value)) => shared = Some(value), + (Some(previous), Some(value)) if previous == &value => {} + (Some(_), Some(_)) => conflict = true, + (None, None) => {} + (Some(_), None) => {} + } + } + (!conflict).then_some(shared).flatten() + }; + Ok::<_, PipelineError>((path, observed, cached)) + }) + .collect::, _>>()?; + revision_pool.install(|| { + probes + .par_iter() + .map(|(path, observed, cached)| { + let (content_fingerprint, snapshot) = match cached { + Some(content_fingerprint) => (content_fingerprint.clone(), observed.clone()), + None => capture_file_revision_with_snapshot(path, observed)?, + }; + let revision = InputRevision::create( + "nfcapd", + path.to_string_lossy().into_owned(), + content_fingerprint, + &decoder_fingerprint, + )?; + Ok::<_, PipelineError>(( + path.clone(), + PreparedRevision { + revision, + snapshot: Some(snapshot), + }, + )) + }) + .collect::, _>>() + }) +} -/// A repaired five-minute bucket is exact, but persisted coarse unique-count -/// and MAAD rows cannot be patched from scalar results. Keep additive capture -/// coverage current and remove only the affected derived metric rows. -fn refresh_rollups_after_five_minute_repair( - connection: &Connection, - child: &CanonicalBucket, +type CoordinatedActiveResolution = (Vec>, Vec<(PathBuf, FileSnapshot)>); +type DailyActiveResolution = (Arc, Vec<(PathBuf, FileSnapshot)>); + +/// Return only the capture paths on the publication grid for one physical local day. +/// +/// Discovery intentionally accepts every valid nfcapd timestamp, but daily eligibility must use +/// the same five-minute keys that publication reads. An off-grid capture can therefore never +/// contribute activity merely because it falls inside the day's lexical path range. +fn nfcapd_day_activity_paths( + paths: &BTreeMap<(String, i64), PathBuf>, + member: &str, + start: i64, + end: i64, timezone: &str, -) -> Result<(), PipelineError> { - const DERIVED_TABLES: [&str; 5] = [ - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - ]; +) -> Result, PipelineError> { + let mut member_paths = Vec::new(); + let mut bucket_start = start; + while bucket_start < end { + if let Some(path) = paths.get(&(member.to_owned(), bucket_start)) { + member_paths.push(path.clone()); + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(member_paths) +} - for granularity in [ - Granularity::ThirtyMinutes, - Granularity::OneHour, - Granularity::OneDay, - ] { - let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; - for table in DERIVED_TABLES { - connection - .execute( - &format!( - "DELETE FROM {table} - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3" - ), - params![child.key.source_id, granularity.as_str(), start], - ) - .map_err(StorageError::from)?; +fn daily_activity_scan_error( + member: &str, + start: i64, + end: i64, + paths: &[PathBuf], + error: impl std::fmt::Display, +) -> PipelineError { + let paths = if paths.is_empty() { + "".to_owned() + } else { + paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", ") + }; + PipelineError::InvalidConfig(format!( + "daily activity scan failed for member {member:?}, day {start}..{end}, paths [{paths}]: {error}" + )) +} + +fn nfcapd_decode_error( + member: &str, + bucket_start: i64, + path: &Path, + error: impl std::fmt::Display, +) -> PipelineError { + PipelineError::InvalidConfig(format!( + "nfcapd decode failed for member {member:?}, bucket {bucket_start}, path {}: {error}", + path.display() + )) +} + +#[allow(clippy::too_many_arguments)] +fn resolve_coordinated_daily_active_sources( + physical_ids: &[String], + paths: &BTreeMap<(String, i64), PathBuf>, + start: i64, + end: i64, + timezone: &str, + selections: &[FlowSelection], + executable: &Path, + activity_pool: &rayon::ThreadPool, + capture_snapshots: &BTreeMap, + revision_snapshots: &BTreeMap, +) -> Result { + let mut combined = (0..selections.len()) + .map(|_| HashMap::::new()) + .collect::>(); + let mut snapshots = Vec::new(); + for member_chunk in physical_ids.chunks(NFCAPD_DECODE_BATCH_SIZE) { + let requests = member_chunk + .iter() + .map(|member| { + nfcapd_day_activity_paths(paths, member, start, end, timezone) + .map(|member_paths| (member.clone(), member_paths)) + }) + .collect::, _>>()?; + let member_results = activity_pool.install(|| { + requests + .par_iter() + .map(|(member, member_paths)| { + let snapshots = member_paths + .iter() + .map(|path| { + let snapshot = capture_snapshots + .get(path) + .cloned() + .or_else(|| { + revision_snapshots + .get(path) + .and_then(|revision| revision.snapshot.clone()) + }) + .map(Ok) + .unwrap_or_else(|| capture_nfcapd_snapshot(path)); + snapshot + .map(|snapshot| (path.clone(), snapshot)) + .map_err(|error| { + daily_activity_scan_error( + member, + start, + end, + member_paths, + error, + ) + }) + }) + .collect::, PipelineError>>()?; + let activities = ingest::read_nfcapd_daily_source_activities( + member_paths, + selections, + executable, + ) + .map_err(|error| { + daily_activity_scan_error(member, start, end, member_paths, error) + })?; + if activities.len() != selections.len() { + return Err(daily_activity_scan_error( + member, + start, + end, + member_paths, + format!( + "daily activity decoder returned {} results for {} selections", + activities.len(), + selections.len() + ), + )); + } + Ok::<_, PipelineError>((activities, snapshots)) + }) + .collect::, _>>() + })?; + for (activities, member_snapshots) in member_results { + snapshots.extend(member_snapshots); + for (selection_index, activity) in activities.into_iter().enumerate() { + for (address, metrics) in activity { + combined[selection_index] + .entry(address) + .or_default() + .include(metrics); + } + } } + } + let active_sources = combined + .into_iter() + .map(|activity| { + Arc::new( + activity + .into_iter() + .filter_map(|(address, metrics)| { + FlowSelection::daily_activity_threshold_met( + metrics.flows, + metrics.packets, + metrics.bytes, + ) + .then_some(address) + }) + .collect(), + ) + }) + .collect(); + Ok((active_sources, snapshots)) +} - let children = query_bucket_coverage( - connection, - &child.key.source_id, - Granularity::FiveMinutes.as_str(), - start, - end, - )?; - let expected_children = - usize::try_from((end - start).div_euclid(FIVE_MINUTES)).unwrap_or(usize::MAX); - if children.len() != expected_children { - connection - .execute( - "DELETE FROM bucket_coverage - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3", - params![child.key.source_id, granularity.as_str(), start], - ) - .map_err(StorageError::from)?; - continue; +#[allow(clippy::too_many_arguments)] +fn process_coordinated_batches( + outputs: &[CoordinatedOutput], + pending: &[usize], + reports: &mut [PipelineReport], + profiles: &mut [NfcapdDayPublishProfile], + shared_profile: &mut CoordinatedDaySharedProfile, + aggregates: &mut [Option], + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + force: bool, + batches: &mut [PreparedCoordinatedBatch], + revisions: &BTreeMap, + active_sources: &[Option>], + decode_pool: &rayon::ThreadPool, +) -> Result<(), PipelineError> { + if pending.is_empty() { + return Ok(()); + } + let executable = outputs[pending[0]].pipeline.nfdump.clone(); + let timezone = outputs[pending[0]].pipeline.timezone.clone(); + for batch in batches { + verify_coordinated_nfdump_revisions(outputs)?; + let prepared = std::mem::take(&mut batch.prepared); + + let mut needed = BTreeMap::<(String, i64), BTreeSet>::new(); + for (&output_index, batch) in &prepared { + for timestamp in batch { + for job in ×tamp.jobs { + for (member, _) in &job.present { + needed + .entry((member.clone(), timestamp.bucket_start)) + .or_default() + .insert(output_index); + } + } + } } - let mut coverage = BucketCoverage::empty(); - for row in children { - coverage - .include(row.coverage()?) - .map_err(DomainError::from)?; + let decode_requests = needed + .iter() + .map(|((member, bucket_start), output_indices)| { + let path = by_member_and_start + .get(&(member.clone(), *bucket_start)) + .cloned() + .ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "coordinated decoder could not locate physical input {member}:{bucket_start}" + )) + })?; + let output_indices = output_indices.iter().copied().collect::>(); + let pairs = output_indices + .iter() + .map(|index| { + ( + outputs[*index].pipeline.selection.clone(), + active_sources[*index] + .clone() + .expect("pending daily selection has active sources"), + ) + }) + .collect::>(); + let snapshot = revisions + .get(&path) + .and_then(|owner| owner.snapshot.as_ref()) + .ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "coordinated decoder has no revision snapshot for {member}:{bucket_start}" + )) + })?; + Ok::<_, PipelineError>(( + member.clone(), + *bucket_start, + path, + output_indices, + pairs, + snapshot, + )) + }) + .collect::, _>>()?; + + // Keep at most twelve child processes active, even when one logical timestamp fans out to + // many physical members. The decoded map is keyed by physical request and retains the + // single fanout vector returned by nfdump rather than an output-expanded result list. + let decode_started = Instant::now(); + let mut decoded = BTreeMap::<(String, i64), Vec<(usize, CanonicalBucket)>>::new(); + for request_chunk in nfcapd_decode_request_chunks(&decode_requests) { + verify_coordinated_nfdump_revisions(outputs)?; + let decoded_results = decode_pool.install(|| { + request_chunk + .par_iter() + .map( + |(member, bucket_start, path, output_indices, pairs, snapshot)| { + let buckets = ingest::read_nfcapd_buckets_with_active_sources( + path, + member, + pairs, + &executable, + &timezone, + ) + .map_err(|error| { + nfcapd_decode_error(member, *bucket_start, path, error) + })?; + if buckets.len() != output_indices.len() { + return Err(nfcapd_decode_error( + member, + *bucket_start, + path, + format!( + "bucket decoder returned {} results for {} selections", + buckets.len(), + output_indices.len() + ), + )); + } + verify_file_snapshot(path, snapshot).map_err(|error| { + nfcapd_decode_error(member, *bucket_start, path, error) + })?; + Ok::<_, PipelineError>(( + member.clone(), + *bucket_start, + output_indices.clone(), + buckets, + )) + }, + ) + .collect::, _>>() + })?; + verify_coordinated_nfdump_revisions(outputs)?; + for (member, bucket_start, output_indices, buckets) in decoded_results { + decoded.insert( + (member, bucket_start), + output_indices.into_iter().zip(buckets).collect(), + ); + } + } + let decode_elapsed = decode_started.elapsed(); + shared_profile.decode_elapsed += decode_elapsed; + for &output_index in prepared.keys() { + profiles[output_index].decode_elapsed += decode_elapsed; + } + + for (&output_index, batch) in &prepared { + let aggregate = aggregates[output_index] + .as_mut() + .expect("pending output has aggregate state"); + let publish_started = Instant::now(); + for timestamp in batch { + for job in ×tamp.jobs { + let member_buckets = job + .present + .iter() + .map(|(member, _)| { + decoded + .get(&(member.clone(), timestamp.bucket_start)) + .and_then(|buckets| { + buckets.iter().find_map(|(index, bucket)| { + (*index == output_index).then_some(bucket) + }) + }) + .expect("requested physical member was decoded") + }) + .collect::>(); + let logical_started = Instant::now(); + let logical = logical_source_bucket( + &job.source_id, + timestamp.bucket_start, + job.expected_units, + &member_buckets, + )?; + profiles[output_index].logical_source_elapsed += logical_started.elapsed(); + let sibling_started = Instant::now(); + if !job.is_repair { + aggregate.reject_persisted_siblings( + &outputs[output_index].connection, + &logical, + &outputs[output_index].pipeline.timezone, + )?; + } + profiles[output_index].persisted_sibling_elapsed += sibling_started.elapsed(); + let bucket_profile = publish_nfcapd_bucket_profiled( + &outputs[output_index].connection, + &logical, + &job.owners, + &job.absences, + &job.evidence, + true, + force, + outputs[output_index].pipeline.run_maad, + )?; + profiles[output_index] + .bucket_publish + .include(bucket_profile); + let flushed = if job.is_repair { + refresh_rollups_after_five_minute_repair( + &outputs[output_index].connection, + &logical, + &outputs[output_index].pipeline.timezone, + )?; + 0 + } else { + let aggregate_profile = aggregate + .include_profiled(&logical, &outputs[output_index].pipeline.timezone)?; + profiles[output_index] + .aggregate_include + .include(aggregate_profile); + let flush_started = Instant::now(); + let (flushed, rollup_write) = aggregate.flush_complete_profiled( + &outputs[output_index].connection, + outputs[output_index].pipeline.run_maad, + )?; + profiles[output_index].completed_rollup_flush_elapsed += + flush_started.elapsed(); + profiles[output_index] + .completed_rollup_write + .include(rollup_write); + profiles[output_index].completed_rollup_flushes += 1; + if flushed > 0 { + profiles[output_index].nonempty_rollup_flushes += 1; + } + flushed + }; + profiles[output_index].logical_buckets += 1; + reports[output_index].rollup_buckets += flushed; + reports[output_index].five_minute_buckets += 1; + } + } + profiles[output_index].batch_publish_elapsed += publish_started.elapsed(); } - insert_bucket_coverage_rows( - connection, - &[BucketCoverageRow::new( - &child.key.source_id, - granularity.as_str(), - start, - end, - coverage, - )], - )?; } Ok(()) } -fn aggregate_bounds( - bucket_start: i64, - granularity: Granularity, - timezone: &str, -) -> Result<(i64, i64), PipelineError> { - let timestamp = Timestamp::from_second(bucket_start) - .map_err(|error| PipelineError::Time(error.to_string()))?; - let zoned = timestamp - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))?; - let start = match granularity { - Granularity::ThirtyMinutes => zoned - .round( - ZonedRound::new() - .smallest(Unit::Minute) - .increment(30) - .mode(RoundMode::Trunc), - ) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::OneHour => zoned - .round( - ZonedRound::new() - .smallest(Unit::Hour) - .mode(RoundMode::Trunc), - ) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::OneDay => zoned - .date() - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::FiveMinutes => { - return Err(PipelineError::InvalidConfig( - "five-minute input is not a rollup granularity".into(), - )); +fn rollback_coordinated_transactions(outputs: &[CoordinatedOutput], transactions: &[bool]) { + for (output, active) in outputs.iter().zip(transactions) { + if *active { + let _ = output.connection.execute_batch("ROLLBACK"); } - }; - let end = match granularity { - Granularity::ThirtyMinutes => start + 1_800, - Granularity::OneHour => start + 3_600, - Granularity::OneDay => zoned - .date() - .tomorrow() - .and_then(|date| date.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::FiveMinutes => unreachable!("rejected above"), - }; - Ok((start, end)) + } } -fn next_local_five_minute_start(bucket_start: i64, timezone: &str) -> Result { - let current = Timestamp::from_second(bucket_start) - .and_then(|timestamp| timestamp.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))?; - let next = current - .datetime() - .checked_add(5.minutes()) - .and_then(|datetime| datetime.in_tz(timezone)) +/// Give every dataset without a configured `default_start_date` the earliest ingested local day. +/// +/// This runs after ingestion so that newly ingested earlier days move the stored date back. Until +/// the database holds traffic, the row keeps the fallback that [`upsert_dataset_metadata`] wrote. +fn infer_default_start_dates( + connection: &Connection, + pipeline: &ResolvedPipeline, +) -> Result<(), PipelineError> { + let inferred = pipeline + .datasets + .iter() + .filter(|dataset| dataset.default_start_date.trim().is_empty()) + .collect::>(); + if inferred.is_empty() { + return Ok(()); + } + let Some(bucket_start) = earliest_traffic_bucket_start(connection)? else { + return Ok(()); + }; + let date = local_date(bucket_start, &pipeline.timezone)?; + with_transaction(connection, || { + for dataset in inferred { + set_dataset_default_start_date(connection, &dataset.dataset_id, &date)?; + } + Ok(()) + }) +} + +/// Local calendar day that contains `timestamp`, formatted as `YYYY-MM-DD`. +fn local_date(timestamp: i64, timezone: &str) -> Result { + Ok(Timestamp::from_second(timestamp) .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(); - if next <= bucket_start { - return Err(PipelineError::Time(format!( - "local five-minute clock did not advance after {bucket_start} in {timezone:?}" + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .date() + .to_string()) +} + +fn with_transaction( + connection: &Connection, + operation: impl FnOnce() -> Result, +) -> Result { + with_transaction_precommit(connection, operation, || Ok(())) +} + +/// Run a transaction with a final read-only guard immediately before COMMIT. +/// +/// The guard runs after all writes have completed while the transaction is still open. Any guard +/// failure therefore rolls back the writes instead of leaving a partially repaired product behind. +fn with_transaction_precommit( + connection: &Connection, + operation: impl FnOnce() -> Result, + precommit: impl FnOnce() -> Result<(), PipelineError>, +) -> Result { + with_transaction_precommit_value( + connection, + || operation().map(|value| (value, ())), + |_| precommit(), + ) +} + +fn with_transaction_precommit_value( + connection: &Connection, + operation: impl FnOnce() -> Result<(T, G), PipelineError>, + precommit: impl FnOnce(&G) -> Result<(), PipelineError>, +) -> Result { + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(StorageError::from)?; + let result = operation().and_then(|(value, guards)| { + precommit(&guards)?; + connection + .execute_batch("COMMIT") + .map_err(StorageError::from)?; + Ok(value) + }); + match result { + Ok(value) => Ok(value), + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + Err(error) + } + } +} + +fn coordinated_output_error(pipeline: &ResolvedPipeline, error: PipelineError) -> PipelineError { + let dataset_id = pipeline + .datasets + .first() + .map(|dataset| dataset.dataset_id.as_str()) + .unwrap_or(""); + PipelineError::InvalidConfig(format!( + "coordinated output initialization failed for dataset {dataset_id:?} at database {}: {error}", + pipeline.database_path.display() + )) +} + +#[cfg(test)] +fn initialize_metadata( + connection: &Connection, + pipeline: &ResolvedPipeline, +) -> Result<(), PipelineError> { + let plan = plan_single_output(pipeline)?; + initialize_metadata_with_plan(connection, pipeline, &plan) +} + +fn initialize_metadata_with_plan( + connection: &Connection, + pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, +) -> Result<(), PipelineError> { + let layouts = plan + .trees + .values() + .flat_map(|tree| tree.sources.iter().cloned()) + .collect::>(); + let mut source_ids = BTreeSet::new(); + if let Some(duplicate) = layouts + .iter() + .find(|source| !source_ids.insert(source.source_id.clone())) + { + return Err(PipelineError::InvalidConfig(format!( + "nfcapd_tree inputs define duplicate logical source ID {:?}", + duplicate.source_id ))); } - Ok(next) -} + with_transaction(connection, || { + initialize_metadata_in_transaction_with_layouts( + connection, + pipeline, + &layouts, + &plan.dataset_sources, + ) + }) +} + +fn initialize_metadata_in_transaction_with_layouts( + connection: &Connection, + pipeline: &ResolvedPipeline, + layouts: &[DatasetSource], + dataset_sources: &BTreeMap>, +) -> Result<(), PipelineError> { + bind_identity(connection, pipeline)?; + for dataset in &pipeline.datasets { + let sources = dataset_sources.get(&dataset.dataset_id).ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "single-output plan has no frozen source layout for dataset {:?}", + dataset.dataset_id + )) + })?; + upsert_dataset_with_sources(connection, dataset, sources)?; + } + if !layouts.is_empty() { + let layout = layouts + .iter() + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) + .collect::>(); + bind_nfcapd_source_layout(connection, &layout)?; + } + Ok(()) +} + +fn initialize_coordinated_metadata_in_transaction( + connection: &Connection, + pipeline: &ResolvedPipeline, + layout: &[DatasetSource], + dataset_layouts: &BTreeMap>, +) -> Result<(), PipelineError> { + bind_identity(connection, pipeline)?; + for dataset in &pipeline.datasets { + let sources = dataset_layouts.get(&dataset.dataset_id).ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "coordinated plan has no frozen source layout for dataset {:?}", + dataset.dataset_id + )) + })?; + upsert_dataset_with_sources(connection, dataset, sources)?; + } + if !layout.is_empty() { + let layout = layout + .iter() + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) + .collect::>(); + bind_nfcapd_source_layout(connection, &layout)?; + } + Ok(()) +} + +fn process_atomic( + connection: &Connection, + pipeline: &ResolvedPipeline, + operation: impl FnOnce(&mut AggregateBuckets, &mut PipelineReport) -> Result<(), PipelineError>, +) -> Result { + let mut aggregates = AggregateBuckets::default(); + let mut report = PipelineReport::default(); + with_transaction(connection, || { + operation(&mut aggregates, &mut report)?; + publish_rollups(connection, aggregates, pipeline, &mut report)?; + verify_nfdump_revision(pipeline) + })?; + Ok(report) +} + +fn merge_report(total: &mut PipelineReport, addition: PipelineReport) { + total.input_scans += addition.input_scans; + total.skipped_inputs += addition.skipped_inputs; + total.five_minute_buckets += addition.five_minute_buckets; + total.rollup_buckets += addition.rollup_buckets; +} + +fn populate_coverage_summary( + connection: &Connection, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + let mut statement = connection + .prepare( + "SELECT coverage_state, COUNT(*) + FROM bucket_coverage + WHERE granularity = '5m' + GROUP BY coverage_state", + ) + .map_err(StorageError::from)?; + let rows = statement + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + for (state, count) in rows { + let count = usize::try_from(count) + .map_err(|_| PipelineError::InvalidConfig("coverage summary count overflow".into()))?; + match state.as_str() { + "complete" => report.complete_five_minute_buckets = count, + "partial" => report.partial_five_minute_buckets = count, + "unknown" => report.unknown_five_minute_buckets = count, + _ => { + return Err(PipelineError::InvalidConfig(format!( + "invalid five-minute coverage state in database: {state:?}" + ))); + } + } + } + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct CoverageScope { + source_ids: Vec, + start: i64, + end: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct CoverageRange { + source_id: String, + start: i64, + end: i64, +} + +fn merged_requested_coverage_ranges(scopes: Vec) -> Vec { + let mut ranges = scopes + .into_iter() + .flat_map(|scope| { + scope + .source_ids + .into_iter() + .map(move |source_id| CoverageRange { + source_id, + start: scope.start, + end: scope.end, + }) + }) + .collect::>(); + ranges.sort_unstable_by(|left, right| { + (&left.source_id, left.start, left.end).cmp(&(&right.source_id, right.start, right.end)) + }); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(previous) = merged.last_mut() + && previous.source_id == range.source_id + && range.start <= previous.end + { + previous.end = previous.end.max(range.end); + } else { + merged.push(range); + } + } + merged +} + +/// A finite native request can be checked independently of incomplete data +/// already stored outside that request. CSV and literal-input configurations +/// have no separately declared time window, so their configured product is +/// the strict scope. +fn discovered_nfcapd_tree_end( + root_path: &Path, + source_ids: &[String], + configured_sources: &[DatasetSource], + selected_start: i64, + timezone: &str, +) -> Result { + let root = canonical_path(root_path)?; + let sources = normalize_sources(&root, source_ids, configured_sources)?; + let physical_ids = sources + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + let discovered = ingest::discover_nfcapd_source_paths(&root, &physical_ids, timezone)?; + discovered + .iter() + .map(|input| input.bucket_start) + .max() + .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) + .transpose()? + .map_or(Ok(selected_start), |(_, end)| Ok(end)) +} + +fn requested_coverage_scopes_with_plan( + pipeline: &ResolvedPipeline, + plan: Option<&SingleOutputPlan>, +) -> Result>, PipelineError> { + let mut scopes = Vec::new(); + for (input_index, input) in pipeline.inputs.iter().enumerate() { + let InputSpec::NfcapdTree { + root_path, + source_ids, + sources, + start_date, + end_date, + start_time, + end_time, + .. + } = input + else { + return Ok(None); + }; + let selected_start = parse_date_start(start_date, &pipeline.timezone)?; + let start = match start_time { + Some(value) => parse_local_datetime(value, &pipeline.timezone)?, + None => selected_start, + }; + let frozen = plan.and_then(|plan| plan.trees.get(&input_index)); + let end = match (end_time, end_date) { + (Some(value), _) => parse_local_datetime(value, &pipeline.timezone)?, + (None, Some(value)) => next_date_start(value, &pipeline.timezone)?, + (None, None) => match frozen { + Some(tree) => discovered_nfcapd_tree_end_with_sources( + &tree.root_path, + &tree.sources, + selected_start, + &pipeline.timezone, + )?, + None => discovered_nfcapd_tree_end( + root_path, + source_ids, + sources, + selected_start, + &pipeline.timezone, + )?, + }, + }; + let source_ids = match frozen { + Some(tree) => tree.sources.clone(), + None => normalize_sources(root_path, source_ids, sources)?, + } + .into_iter() + .map(|source| source.source_id) + .collect(); + scopes.push(CoverageScope { + source_ids, + start, + end, + }); + } + Ok(Some(scopes)) +} + +fn discovered_nfcapd_tree_end_with_sources( + root_path: &Path, + sources: &[DatasetSource], + selected_start: i64, + timezone: &str, +) -> Result { + let physical_ids = sources + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + let discovered = ingest::discover_nfcapd_source_paths(root_path, &physical_ids, timezone)?; + discovered + .iter() + .map(|input| input.bucket_start) + .max() + .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) + .transpose()? + .map_or(Ok(selected_start), |(_, end)| Ok(end)) +} + +#[cfg(test)] +fn count_incomplete_requested_coverage( + connection: &Connection, + pipeline: &ResolvedPipeline, +) -> Result { + count_incomplete_requested_coverage_with_plan( + connection, + pipeline, + &SingleOutputPlan::default(), + ) +} + +fn count_incomplete_requested_coverage_with_plan( + connection: &Connection, + pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, +) -> Result { + let plan = (!plan.trees.is_empty()).then_some(plan); + let Some(scopes) = requested_coverage_scopes_with_plan(pipeline, plan)? else { + return connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE granularity = '5m' AND coverage_state <> 'complete'", + [], + |row| row.get(0), + ) + .map_err(StorageError::from) + .map_err(PipelineError::from); + }; + + count_incomplete_coverage_ranges( + connection, + merged_requested_coverage_ranges(scopes), + &pipeline.timezone, + ) +} + +fn count_incomplete_coverage_for_layout( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + timezone: &str, +) -> Result { + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + let ranges = merged_requested_coverage_ranges(vec![CoverageScope { + source_ids, + start, + end, + }]); + count_incomplete_coverage_ranges(connection, ranges, timezone) +} + +fn count_incomplete_coverage_ranges( + connection: &Connection, + ranges: Vec, + timezone: &str, +) -> Result { + let mut incomplete = 0_i64; + for range in ranges { + let complete = connection + .prepare( + "SELECT bucket_start + FROM bucket_coverage + WHERE source_id = ?1 + AND granularity = '5m' + AND bucket_start >= ?2 + AND bucket_start < ?3 + AND coverage_state = 'complete' + ORDER BY bucket_start", + ) + .map_err(StorageError::from)? + .query_map(params![&range.source_id, range.start, range.end], |row| { + row.get::<_, i64>(0) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + let mut bucket_start = range.start; + while bucket_start < range.end { + if !complete.contains(&bucket_start) { + incomplete = incomplete.checked_add(1).ok_or_else(|| { + PipelineError::InvalidConfig( + "requested coverage count exceeds SQLite INTEGER range".into(), + ) + })?; + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + } + Ok(incomplete) +} + +fn bind_identity( + connection: &Connection, + pipeline: &ResolvedPipeline, +) -> Result<(), PipelineError> { + verify_nfdump_revision(pipeline)?; + let maad_config = serde_json::to_value(crate::maad::MaadConfig::default())?; + let schema = json!({ + "version": 3, + "tables": [ + {"name":"traffic_stats","version":2}, + {"name":"protocol_stats","version":1}, + {"name":"address_count_stats","version":1}, + {"name":"port_count_stats","version":1}, + {"name":"address_structure_stats","version":1}, + {"name":"bucket_coverage","version":1} + ] + }); + let nfdump_executable = pipeline.nfdump_revision.as_ref().map(|revision| { + json!({ + "locator": revision.locator, + "content_fingerprint": revision.content_fingerprint, + }) + }); + let result_config = json!({ + "version": 4, + "timezone": pipeline.timezone, + "nfcapd_decoder": { + "protocol_version": nfdump::CONTRACT_VERSION, + "input_contract": nfdump::INPUT_CONTRACT, + "output_contract": nfdump::OUTPUT_CONTRACT, + "contract_id": nfcapd_decoder_fingerprint()?, + "decoder_fingerprint": pipeline.nfdump_revision.as_ref().map(|revision| revision.decoder_fingerprint.clone()), + "executable": nfdump_executable, + }, + "maad": { + "enabled": pipeline.run_maad, + "backend": "in-process", + "contract_version": 2, + "config": maad_config + } + }); + let identity = ProductIdentity::create( + &schema, + &pipeline.selection.normalized_payload(), + &result_config, + )?; + bind_product_identity(connection, &identity, &crate::storage::STATS_TABLE_NAMES)?; + Ok(()) +} + +fn upsert_dataset_with_sources( + connection: &Connection, + dataset: &Dataset, + logical_sources: &[DatasetSource], +) -> Result<(), PipelineError> { + let sources = logical_sources + .iter() + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) + .collect::>(); + let mut metadata = DatasetMetadata::new(&dataset.dataset_id); + metadata.label = dataset.label.clone(); + metadata.default_start_date = dataset.default_start_date.clone(); + metadata.source_mode = dataset.source_mode.clone(); + metadata.discovery_mode = dataset.discovery_mode.clone(); + metadata.sort_order = dataset.sort_order; + metadata.sources = sources; + upsert_dataset_metadata(connection, &metadata)?; + Ok(()) +} + +struct PreparedCsvInput { + path: PathBuf, + mapping: CsvSourceConfig, + revision: InputRevision, + snapshot: FileSnapshot, +} + +fn prepare_file_revision( + connection: &Connection, + path: &Path, + input_kind: InputKind, + decoder_fingerprint: String, +) -> Result<(InputRevision, FileSnapshot), PipelineError> { + prepare_file_revision_with(connection, path, input_kind, decoder_fingerprint, || { + capture_file_revision(path) + }) +} + +fn prepare_file_revision_with( + connection: &Connection, + path: &Path, + input_kind: InputKind, + decoder_fingerprint: String, + hash_file: impl FnOnce() -> Result<(String, FileSnapshot), ProvenanceError>, +) -> Result<(InputRevision, FileSnapshot), PipelineError> { + let locator = path.to_string_lossy().into_owned(); + let observed = FileSnapshot::capture(path)?; + let (content_fingerprint, snapshot) = + match cached_content_fingerprint(connection, input_kind, &locator, &observed)? { + Some(content_fingerprint) => (content_fingerprint, observed), + None => hash_file()?, + }; + let revision = InputRevision::create( + input_kind.as_str(), + locator, + content_fingerprint, + decoder_fingerprint, + )?; + Ok((revision, snapshot)) +} + +fn process_csv_inputs( + connection: &Connection, + inputs: &[ingest::CsvInputSpec], + pipeline: &ResolvedPipeline, +) -> Result { + let mut prepared = Vec::new(); + let mut skipped_inputs = 0_usize; + let mut needs_rescan = false; + for input in inputs { + let mapping = CsvSourceConfig::load(&input.mapping_path)?; + let (revision, snapshot) = prepare_file_revision( + connection, + &input.path, + InputKind::Csv, + csv_decoder_fingerprint(&mapping)?, + )?; + if input_scan_fully_processed(connection, InputKind::Csv, &revision.locator, &revision)? { + skipped_inputs += 1; + } else { + needs_rescan = true; + } + prepared.push(PreparedCsvInput { + path: input.path.clone(), + mapping, + revision, + snapshot, + }); + } + if !needs_rescan { + return Ok(PipelineReport { + skipped_inputs, + ..PipelineReport::default() + }); + } + prepared.sort_unstable_by(|left, right| left.path.cmp(&right.path)); + + let mut aggregates = AggregateBuckets::default(); + let mut report = PipelineReport::default(); + with_transaction(connection, || { + connection + .execute_batch( + "CREATE TEMP TABLE csv_bucket_stage ( + source_id TEXT NOT NULL, + bucket_start INTEGER NOT NULL, + input_locator TEXT NOT NULL, + revision_fingerprint TEXT, + payload BLOB NOT NULL + ); + CREATE INDEX csv_bucket_stage_order + ON csv_bucket_stage(source_id, bucket_start);", + ) + .map_err(StorageError::from)?; + for input in &prepared { + process_csv( + connection, + &input.path, + &input.mapping, + &input.revision, + &input.snapshot, + pipeline, + &mut report, + )?; + } + publish_csv_stage(connection, pipeline, &mut aggregates, &mut report)?; + publish_rollups(connection, aggregates, pipeline, &mut report) + })?; + Ok(report) +} + +#[allow(clippy::too_many_arguments)] +fn process_csv( + connection: &Connection, + path: &Path, + mapping: &CsvSourceConfig, + revision: &InputRevision, + snapshot: &FileSnapshot, + pipeline: &ResolvedPipeline, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + connection + .execute( + "DELETE FROM processed_inputs + WHERE input_kind = 'csv' AND scan_locator = ?1", + params![revision.locator], + ) + .map_err(StorageError::from)?; + let completion = match ingest::scan_csv(path, mapping, &pipeline.selection, |event| { + let bucket_revision = revision_for_locator(revision, &event.input_locator)?; + let owner = InputBucket { + input_kind: InputKind::Csv, + input_locator: event.input_locator.clone(), + scan_locator: event.scan_locator, + source_id: event.bucket.key.source_id.clone(), + bucket_start: event.bucket.key.bucket_start, + bucket_end: event.bucket.key.bucket_end, + revision: bucket_revision.clone(), + file_snapshot: Some(snapshot.clone()), + }; + upsert_input_bucket(connection, &owner, false)?; + mark_input_bucket_status( + connection, + InputKind::Csv, + &event.input_locator, + &event.bucket.key.source_id, + event.bucket.key.bucket_start, + InputStatus::Processed, + &bucket_revision, + None, + )?; + let payload = serde_json::to_vec(&event.bucket)?; + connection + .execute( + "INSERT INTO csv_bucket_stage ( + source_id, bucket_start, input_locator, + revision_fingerprint, payload + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + event.bucket.key.source_id, + event.bucket.key.bucket_start, + event.input_locator, + bucket_revision.fingerprint, + payload, + ], + ) + .map_err(StorageError::from)?; + Ok::<_, PipelineError>(()) + }) { + Ok(completion) => completion, + Err(ProducerError::Input(error)) => return Err(error.into()), + Err(ProducerError::Sink(error)) => return Err(error), + }; + verify_file_snapshot(path, snapshot)?; + complete_input_scan( + connection, + InputKind::Csv, + &completion.scan_locator, + i64::try_from(completion.rejected_rows) + .map_err(|_| PipelineError::InvalidConfig("rejected row count overflow".into()))?, + i64::try_from(completion.skipped_bad_column_count).map_err(|_| { + PipelineError::InvalidConfig("skipped bad-column count overflow".into()) + })?, + revision, + Some(snapshot), + )?; + verify_file_snapshot(path, snapshot)?; + report.input_scans += 1; + Ok(()) +} + +struct CsvStageMember { + bucket: CanonicalBucket, + input_locator: String, + revision_fingerprint: Option, +} + +/// Merge all staged CSV buckets in source/time order. The stage is indexed on +/// disk, so only one overlapping bucket group is held in memory at a time. +fn publish_csv_stage( + connection: &Connection, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + let mut statement = connection + .prepare( + "SELECT source_id, bucket_start, input_locator, + revision_fingerprint, payload + FROM csv_bucket_stage + ORDER BY source_id, bucket_start, input_locator", + ) + .map_err(StorageError::from)?; + let mut rows = statement.query([]).map_err(StorageError::from)?; + let mut group: Option<(String, i64, Vec)> = None; + let mut current_source = None; + let mut next_expected = None; + loop { + let Some((source_id, bucket_start, input_locator, revision_fingerprint, payload)) = rows + .next() + .map_err(StorageError::from)? + .map(|row| { + Ok::<_, rusqlite::Error>(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Vec>(4)?, + )) + }) + .transpose() + .map_err(StorageError::from)? + else { + break; + }; + let member = CsvStageMember { + bucket: serde_json::from_slice(&payload)?, + input_locator, + revision_fingerprint, + }; + match group.as_mut() { + Some((group_source, group_start, members)) + if group_source == &source_id && *group_start == bucket_start => + { + members.push(member); + } + _ => { + if let Some((group_source, group_start, members)) = group.take() { + publish_csv_stage_group( + connection, + pipeline, + aggregates, + report, + &group_source, + group_start, + &members, + &mut current_source, + &mut next_expected, + )?; + } + group = Some((source_id, bucket_start, vec![member])); + } + } + } + drop(rows); + drop(statement); + if let Some((group_source, group_start, members)) = group { + publish_csv_stage_group( + connection, + pipeline, + aggregates, + report, + &group_source, + group_start, + &members, + &mut current_source, + &mut next_expected, + )?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn publish_csv_stage_group( + connection: &Connection, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, + source_id: &str, + bucket_start: i64, + members: &[CsvStageMember], + current_source: &mut Option, + next_expected: &mut Option, +) -> Result<(), PipelineError> { + if current_source.as_deref() != Some(source_id) { + *current_source = Some(source_id.to_owned()); + *next_expected = None; + } else { + let mut expected = next_expected.ok_or_else(|| { + PipelineError::InvalidConfig("CSV stage lost its source envelope".into()) + })?; + while expected < bucket_start { + let (bucket, evidence) = merged_csv_bucket(source_id, expected, &[])?; + publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; + expected = expected.checked_add(FIVE_MINUTES).ok_or_else(|| { + PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) + })?; + } + } + let (bucket, evidence) = merged_csv_bucket(source_id, bucket_start, members)?; + publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; + *next_expected = Some(bucket_start.checked_add(FIVE_MINUTES).ok_or_else(|| { + PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) + })?); + Ok(()) +} + +fn merged_csv_bucket( + source_id: &str, + bucket_start: i64, + members: &[CsvStageMember], +) -> Result<(CanonicalBucket, InputEvidenceRow), PipelineError> { + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + ); + let any_observed = members + .iter() + .any(|member| member.bucket.coverage.observed_units() != 0); + let any_rejected = members + .iter() + .any(|member| member.bucket.coverage.rejected_units() != 0); + let mut builder = if any_observed { + StatisticalBucket::dense(key) + } else { + StatisticalBucket::new(key) + } + .with_coverage(BucketCoverage::empty()); + for member in members { + builder.include(&member.bucket)?; + } + let coverage = BucketCoverage::new(1, u64::from(any_observed), u64::from(any_rejected)) + .map_err(DomainError::from)?; + let bucket = builder.with_coverage(coverage).finish_owned(); + let evidence_state = if any_rejected { + InputEvidenceState::Rejected + } else if any_observed { + InputEvidenceState::Observed + } else { + InputEvidenceState::Missing + }; + let (input_locator, revision_fingerprint) = match members { + [member] => ( + member.input_locator.clone(), + member.revision_fingerprint.clone(), + ), + [] => (format!("csv://{source_id}"), None), + _ => (format!("csv://{source_id}"), None), + }; + let evidence = InputEvidenceRow::new( + source_id, + source_id, + bucket_start, + bucket_start + FIVE_MINUTES, + input_locator, + evidence_state, + revision_fingerprint, + ); + Ok((bucket, evidence)) +} + +fn publish_csv_bucket( + connection: &Connection, + bucket: &CanonicalBucket, + evidence: &InputEvidenceRow, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + reject_cross_kind_overlap(connection, bucket, InputKind::Csv)?; + aggregates.reject_persisted_csv_siblings(connection, bucket, &pipeline.timezone)?; + let (day_start, day_end) = aggregate_bounds( + bucket.key.bucket_start, + Granularity::OneDay, + &pipeline.timezone, + )?; + ensure_daily_product_completion_bucket_guard( + connection, + &bucket.key.source_id, + bucket.key.bucket_start, + day_start, + day_end, + )?; + write_buckets(connection, std::slice::from_ref(bucket), pipeline.run_maad)?; + replace_input_evidence( + connection, + &bucket.key.source_id, + bucket.key.bucket_start, + std::slice::from_ref(evidence), + )?; + aggregates.include(bucket, &pipeline.timezone)?; + report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; + report.five_minute_buckets += 1; + Ok(()) +} + +fn reject_cross_kind_overlap( + connection: &Connection, + bucket: &CanonicalBucket, + input_kind: InputKind, +) -> Result<(), PipelineError> { + let conflict = connection + .query_row( + "SELECT input_kind, input_locator FROM processed_inputs + WHERE source_id = ?1 AND bucket_start = ?2 AND input_kind <> ?3 + ORDER BY input_kind, input_locator LIMIT 1", + params![ + bucket.key.source_id, + bucket.key.bucket_start, + input_kind.as_str(), + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional() + .map_err(StorageError::from)?; + if let Some((kind, locator)) = conflict { + return Err(PipelineError::InvalidConfig(format!( + "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", + bucket.key.source_id, bucket.key.bucket_start + ))); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn process_nfcapd_tree( + connection: &Connection, + tree: &FrozenNfcapdTreeLayout, + start_date: &str, + end_date: Option<&str>, + start_time: Option<&str>, + end_time: Option<&str>, + force: bool, + pipeline: &ResolvedPipeline, + capture_snapshots: &BTreeMap, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + if pipeline.selection.selects_daily_active_sources() + && (start_time.is_some() || end_time.is_some()) + { + return Err(PipelineError::InvalidConfig( + "daily_active_sources selection requires whole local calendar days; start_time and end_time are unsupported".into(), + )); + } + let root = &tree.root_path; + let sources = &tree.sources; + let physical_ids = &tree.physical_ids; + if pipeline.selection.selects_daily_active_sources() { + validate_daily_active_source_layout(sources, physical_ids)?; + } + let discovery_started = Instant::now(); + let discovered = ingest::discover_nfcapd_source_paths(root, physical_ids, &pipeline.timezone)?; + tracing::info!( + target: "netflow_db::profile", + phase = "discovery", + elapsed_seconds = discovery_started.elapsed().as_secs_f64(), + physical_sources = physical_ids.len(), + discovered_inputs = discovered.len(), + ); + let mut by_member_and_start = BTreeMap::new(); + let mut member_bounds = BTreeMap::new(); + for input in discovered { + member_bounds + .entry(input.source_id.clone()) + .and_modify(|(first, last): &mut (i64, i64)| { + *first = (*first).min(input.bucket_start); + *last = (*last).max(input.bucket_start); + }) + .or_insert((input.bucket_start, input.bucket_start)); + by_member_and_start.insert((input.source_id, input.bucket_start), input.path); + } + let window = resolve_nfcapd_tree_window( + start_date, + end_date, + start_time, + end_time, + by_member_and_start + .keys() + .map(|(_, bucket_start)| *bucket_start), + &pipeline.timezone, + )?; + + let mut day_start = window.start; + while day_start < window.end { + verify_member_directory_identities(root, &tree.member_identities)?; + let day_end = aggregate_bounds(day_start, Granularity::OneDay, &pipeline.timezone)?.1; + let capture_complete = day_capture_is_complete( + sources, + &by_member_and_start, + day_start, + day_end, + &pipeline.timezone, + )?; + let published_day = if pipeline.selection.selects_daily_active_sources() { + day_was_published(connection, sources, day_start, day_end)? + } else { + day_has_complete_coverage(connection, sources, day_start, day_end, &pipeline.timezone)? + }; + let stale_published_day = !capture_complete && published_day; + let force_replaces_day = force && pipeline.selection.selects_daily_active_sources(); + if stale_published_day && !force { + return Err(PipelineError::InvalidConfig(format!( + "published local day {day_start}..{day_end} no longer has complete nfcapd capture coverage; rerun that day with --force" + ))); + } + let mut marker_needs_backfill = false; + let canonical_day_verified = if pipeline.selection.selects_daily_active_sources() + && capture_complete + && published_day + && !force + { + match nfcapd_day_completion_state( + connection, + sources, + day_start, + day_end, + pipeline.run_maad, + )? { + DailyProductCompletionState::Clean => true, + DailyProductCompletionState::Dirty => { + return Err(PipelineError::InvalidConfig(format!( + "published local day {day_start}..{day_end} was mutated after completion; rerun that whole day with --force" + ))); + } + DailyProductCompletionState::Missing => { + if !nfcapd_day_has_canonical_topology( + connection, + sources, + day_start, + day_end, + &pipeline.timezone, + pipeline.run_maad, + )? { + return Err(PipelineError::InvalidConfig(format!( + "published local day {day_start}..{day_end} has damaged canonical topology; rerun that whole day with --force" + ))); + } + marker_needs_backfill = true; + true + } + } + } else { + false + }; + let missing = if pipeline.selection.selects_daily_active_sources() || stale_published_day { + missing_physical_day_inputs( + physical_ids, + &by_member_and_start, + day_start, + day_end, + &pipeline.timezone, + )? + } else { + Vec::new() + }; + let missing_absences = + build_missing_day_absences(root, &missing, day_start, day_end, &pipeline.timezone)?; + invoke_missing_day_absence_hook(root, &missing, &pipeline.timezone); + verify_member_directory_identities(root, &tree.member_identities)?; + if pipeline.selection.selects_daily_active_sources() && !missing.is_empty() { + let missing_details = missing_day_warning_details(root, &missing, &pipeline.timezone)?; + tracing::warn!( + day_start, + day_end, + missing_inputs = missing.len(), + missing_details = %missing_details, + "skipping incomplete physical day for daily_active_sources selection" + ); + report.skipped_inputs += missing.len(); + if stale_published_day || force_replaces_day { + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + let guards = NfcapdDayGuards { + nfdump_revision: pipeline.nfdump_revision.clone(), + ..NfcapdDayGuards::default() + }; + with_transaction_precommit_value( + connection, + || { + verify_missing_day_absences(&missing_absences, day_start, day_end)?; + delete_stats_time_range(connection, &source_ids, day_start, day_end)?; + Ok(((), guards)) + }, + |guards| { + verify_single_day_guards( + guards, + root, + &tree.member_identities, + &missing_absences, + day_start, + day_end, + ) + }, + )?; + } + day_start = day_end; + continue; + } + let mut owned_keys = BTreeSet::new(); + let mut bucket_start = day_start; + while bucket_start < day_end { + for source in sources { + if force + && source_has_candidate( + source, + bucket_start, + &by_member_and_start, + &member_bounds, + end_date.is_some(), + ) + { + owned_keys.insert((source.source_id.clone(), bucket_start)); + } + } + bucket_start = next_local_five_minute_start(bucket_start, &pipeline.timezone)?; + } + let transaction_started = Instant::now(); + let (day_report, day_profile) = with_transaction_precommit_value( + connection, + || { + if stale_published_day || force_replaces_day { + verify_missing_day_absences(&missing_absences, day_start, day_end)?; + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + delete_stats_time_range(connection, &source_ids, day_start, day_end)?; + } + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + provision_daily_product_completion_bucket_guards( + connection, + &source_ids, + day_start, + day_end, + )?; + let mut aggregates = AggregateBuckets::with_owned_keys(owned_keys); + let mut day_report = PipelineReport::default(); + let mut day_result = process_nfcapd_tree_day( + connection, + root, + sources, + &by_member_and_start, + &member_bounds, + day_start, + day_end, + end_date.is_some(), + force, + pipeline, + capture_snapshots, + &mut aggregates, + &mut day_report, + canonical_day_verified, + )?; + day_result.profile.final_rollups = + publish_rollups_profiled(connection, aggregates, pipeline, &mut day_report)?; + if !pipeline.selection.selects_daily_active_sources() { + verify_nfdump_revision(pipeline)?; + } + if pipeline.selection.selects_daily_active_sources() + && capture_complete + && missing.is_empty() + && (!published_day || force_replaces_day || marker_needs_backfill) + { + mark_nfcapd_day_complete( + connection, + sources, + day_start, + day_end, + pipeline.run_maad, + )?; + } + Ok(((day_report, day_result.profile), day_result.guards)) + }, + |guards| { + if pipeline.selection.selects_daily_active_sources() { + verify_single_day_guards( + guards, + root, + &tree.member_identities, + &missing_absences, + day_start, + day_end, + ) + } else { + invoke_single_commit_guard_hook(); + verify_member_directory_identities(root, &tree.member_identities)?; + verify_missing_day_absences(&missing_absences, day_start, day_end) + } + }, + )?; + day_profile.log(day_start, day_end, transaction_started.elapsed()); + merge_report(report, day_report); + day_start = day_end; + } + Ok(()) +} + +fn missing_physical_day_inputs( + physical_ids: &[String], + paths: &BTreeMap<(String, i64), PathBuf>, + start: i64, + end: i64, + timezone: &str, +) -> Result, PipelineError> { + let mut missing = Vec::new(); + let mut bucket_start = start; + while bucket_start < end { + for member in physical_ids { + if !paths.contains_key(&(member.clone(), bucket_start)) { + missing.push((member.clone(), bucket_start)); + } + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(missing) +} + +fn missing_day_absence_error( + start: i64, + end: i64, + path: &Path, + error: impl std::fmt::Display, +) -> PipelineError { + PipelineError::InvalidConfig(format!( + "nfcapd capture appeared while protecting missing inputs for local day {start}..{end} at {}; refusing to delete the existing product: {error}", + path.display() + )) +} + +fn build_missing_day_absences( + root: &Path, + missing: &[(String, i64)], + start: i64, + end: i64, + timezone: &str, +) -> Result, PipelineError> { + missing + .iter() + .map(|(member, bucket_start)| { + let path = expected_nfcapd_path(root, member, *bucket_start, timezone)?; + ExpectedAbsence::capture(&path) + .map_err(|error| missing_day_absence_error(start, end, &path, error)) + }) + .collect() +} + +fn verify_missing_day_absences( + absences: &[ExpectedAbsence], + start: i64, + end: i64, +) -> Result<(), PipelineError> { + for absence in absences { + absence + .verify() + .map_err(|error| missing_day_absence_error(start, end, absence.path(), error))?; + } + Ok(()) +} + +fn verify_single_day_guards( + guards: &NfcapdDayGuards, + root: &Path, + member_identities: &BTreeMap, + missing_absences: &[ExpectedAbsence], + start: i64, + end: i64, +) -> Result<(), PipelineError> { + invoke_single_commit_guard_hook(); + verify_member_directory_identities(root, member_identities)?; + for (path, revision) in &guards.capture_revisions { + if let Some(snapshot) = &revision.snapshot { + verify_file_snapshot(path, snapshot)?; + } + } + for (path, snapshot) in &guards.activity_snapshots { + if !guards.capture_revisions.contains_key(path) { + verify_file_snapshot(path, snapshot)?; + } + } + if let Some(revision) = &guards.nfdump_revision { + verify_nfdump_revision_snapshot(revision)?; + } + verify_missing_day_absences(missing_absences, start, end) +} + +fn missing_day_warning_details( + root: &Path, + missing: &[(String, i64)], + timezone: &str, +) -> Result { + let mut details = missing + .iter() + .take(MAX_MISSING_DAY_WARNING_DETAILS) + .map(|(member, bucket_start)| { + expected_nfcapd_path(root, member, *bucket_start, timezone).map(|expected_path| { + format!( + "member={member} timestamp={bucket_start} expected_path={}", + expected_path.display() + ) + }) + }) + .collect::, _>>()?; + let omitted = missing.len().saturating_sub(details.len()); + if omitted != 0 { + details.push(format!("… {omitted} more missing inputs")); + } + Ok(details.join("; ")) +} + +/// Whether every member of every logical source has a discovered capture in this local day. +fn day_capture_is_complete( + sources: &[DatasetSource], + paths: &BTreeMap<(String, i64), PathBuf>, + start: i64, + end: i64, + timezone: &str, +) -> Result { + let mut bucket_start = start; + while bucket_start < end { + if sources.iter().any(|source| { + source + .members + .iter() + .any(|member| !paths.contains_key(&(member.clone(), bucket_start))) + }) { + return Ok(false); + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(true) +} + +/// Non-daily selections may repair a partially published physical day as new members arrive. +/// Only treat that day as stale when its complete coverage envelope had already been committed; +/// daily-active selections use [`day_was_published`] below because their day cohort is atomic. +fn day_has_complete_coverage( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + timezone: &str, +) -> Result { + let mut expected_bucket_count = 0_i64; + let mut bucket_start = start; + while bucket_start < end { + expected_bucket_count = expected_bucket_count.checked_add(1).ok_or_else(|| { + PipelineError::InvalidConfig("local day contains too many five-minute buckets".into()) + })?; + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + if sources.is_empty() { + return Ok(false); + } + for source in sources { + let complete = connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE source_id = ?1 AND granularity = '5m' + AND bucket_start >= ?2 AND bucket_start < ?3 + AND coverage_state = 'complete'", + params![source.source_id, start, end], + |row| row.get::<_, i64>(0), + ) + .map_err(StorageError::from)?; + if complete != expected_bucket_count { + return Ok(false); + } + } + Ok(true) +} + +/// Any committed product, evidence, or processed-input provenance makes a day a prior +/// publication. Coverage is one part of that product, not the publication marker: if a coverage +/// row is damaged or missing while a capture also disappears, force must still remove the stale +/// day instead of treating it as a first run. +fn day_was_published( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, +) -> Result { + for source in sources { + let completion = connection + .query_row( + "SELECT EXISTS( + SELECT 1 FROM daily_product_completion + WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2 + ) OR EXISTS( + SELECT 1 FROM daily_product_completion_dirty + WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2 + )", + params![source.source_id, start, end], + |row| row.get::<_, i64>(0), + ) + .map_err(StorageError::from)?; + if completion != 0 { + return Ok(true); + } + for table in STATS_TABLE_NAMES { + let published = connection + .query_row( + &format!( + "SELECT EXISTS( + SELECT 1 FROM {table} + WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} + AND bucket_start >= ?2 AND bucket_start < ?3 + )" + ), + params![source.source_id, start, end], + |row| row.get::<_, i64>(0), + ) + .map_err(StorageError::from)?; + if published != 0 { + return Ok(true); + } + } + let evidence = connection + .query_row( + "SELECT EXISTS( + SELECT 1 FROM input_evidence + WHERE source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3 + )", + params![source.source_id, start, end], + |row| row.get::<_, i64>(0), + ) + .map_err(StorageError::from)?; + if evidence != 0 { + return Ok(true); + } + let provenance = connection + .query_row( + "SELECT EXISTS( + SELECT 1 FROM processed_inputs + WHERE input_kind = 'nfcapd' AND status = 'processed' + AND source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3 + )", + params![source.source_id, start, end], + |row| row.get::<_, i64>(0), + ) + .map_err(StorageError::from)?; + if provenance != 0 { + return Ok(true); + } + } + Ok(false) +} + +const CANONICAL_GRANULARITY_PREDICATE: &str = "granularity IN ('5m', '30m', '1h', '1d')"; + +const CANONICAL_SCOPE_PREDICATE: &str = "( + (src_visibility = 'all' AND dst_visibility = 'all') OR + (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR + (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR + (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR + (src_visibility = 'literal' AND dst_visibility = 'literal') +)"; + +fn canonical_row_family_predicate(table: &str) -> &'static str { + match table { + "traffic_stats" | "protocol_stats" => CANONICAL_SCOPE_PREDICATE, + "address_count_stats" => { + "( + address_side IN ('source', 'destination') AND + ((src_visibility = 'all' AND dst_visibility = 'all') OR + (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR + (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR + (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR + (src_visibility = 'literal' AND dst_visibility = 'literal')) + )" + } + "port_count_stats" => { + "( + port_side IN ('source', 'destination') AND + port_range IN ('low', 'high') AND + ((src_visibility = 'all' AND dst_visibility = 'all') OR + (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR + (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR + (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR + (src_visibility = 'literal' AND dst_visibility = 'literal')) + )" + } + "address_structure_stats" => { + "( + ip_version = 4 AND + address_side IN ('source', 'destination') AND + structure_kind IN ('structure', 'spectrum', 'dimension') AND + ((src_visibility = 'all' AND dst_visibility = 'all') OR + (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR + (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR + (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR + (src_visibility = 'literal' AND dst_visibility = 'literal')) + )" + } + _ => unreachable!("unknown canonical product table {table}"), + } +} + +fn canonical_row_family_count(table: &str, dense: bool, run_maad: bool) -> i64 { + if !dense || (table == "address_structure_stats" && !run_maad) { + return 0; + } + let scopes = nfcapd_dense_traffic_scope_count(); + match table { + "traffic_stats" | "protocol_stats" => scopes, + "address_count_stats" => scopes * 2, + "port_count_stats" => scopes * 4, + // MAAD is emitted only for IPv4 address sets. Dense traffic has one IPv4 and one IPv6 + // row for each visibility scope, while each IPv4 side gets three MAAD structures. + "address_structure_stats" => scopes * 3, + _ => unreachable!("unknown canonical product table {table}"), + } +} + +fn canonical_coverage_state( + observed_units: i64, + expected_units: i64, + rejected_units: i64, +) -> Option<&'static str> { + if expected_units <= 0 + || observed_units < 0 + || rejected_units < 0 + || observed_units > expected_units + || rejected_units > expected_units + { + return None; + } + Some(if observed_units == expected_units && rejected_units == 0 { + "complete" + } else if observed_units == 0 && rejected_units == 0 { + "unknown" + } else { + "partial" + }) +} + +fn canonical_bucket_coverage_matches( + connection: &Connection, + source_id: &str, + bucket_start: i64, + expected_end: i64, + observed_units: usize, + expected_units: usize, +) -> Result { + let expected_units = i64::try_from(expected_units) + .map_err(|_| PipelineError::InvalidConfig("nfcapd coverage unit count overflow".into()))?; + let observed_units = i64::try_from(observed_units) + .map_err(|_| PipelineError::InvalidConfig("nfcapd observed unit count overflow".into()))?; + let Some(expected_state) = canonical_coverage_state(observed_units, expected_units, 0) else { + return Ok(false); + }; + let row = connection + .query_row( + "SELECT bucket_end, coverage_state, observed_units, expected_units, rejected_units + FROM bucket_coverage + WHERE source_id = ?1 AND granularity = '5m' AND bucket_start = ?2", + params![source_id, bucket_start], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )) + }, + ) + .optional() + .map_err(StorageError::from)?; + Ok( + row.is_some_and(|(bucket_end, state, observed, expected, rejected)| { + bucket_end == expected_end + && state == expected_state + && observed == observed_units + && expected == expected_units + && rejected == 0 + }), + ) +} + +fn canonical_bucket_rows_match( + connection: &Connection, + source_id: &str, + granularity: Granularity, + bucket_start: i64, + bucket_end: i64, + dense: bool, + run_maad: bool, +) -> Result { + for table in [ + "traffic_stats", + "protocol_stats", + "address_count_stats", + "port_count_stats", + "address_structure_stats", + ] { + let expected = canonical_row_family_count(table, dense, run_maad); + let query = format!( + "SELECT COUNT(*), COALESCE(SUM(CASE WHEN bucket_end = ?4 AND ip_version IN (4, 6) AND ({predicate}) THEN 1 ELSE 0 END), 0) + FROM {table} + WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3", + predicate = canonical_row_family_predicate(table), + ); + let (total, canonical) = connection + .query_row( + &query, + params![source_id, granularity.as_str(), bucket_start, bucket_end], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + ) + .map_err(StorageError::from)?; + if total != expected || canonical != expected { + return Ok(false); + } + } + Ok(true) +} + +/// A matching evidence/provenance pair is resumable only when the committed canonical bucket +/// topology is still present. Observed logical buckets are dense; all-missing buckets are valid +/// sparse coverage-only rows. +fn nfcapd_logical_bucket_has_canonical_topology( + connection: &Connection, + source_id: &str, + bucket_start: i64, + observed_units: usize, + expected_units: usize, + run_maad: bool, +) -> Result { + #[cfg(test)] + NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS.with(|calls| calls.set(calls.get() + 1)); + let bucket_end = bucket_start + .checked_add(FIVE_MINUTES) + .ok_or_else(|| PipelineError::InvalidConfig("nfcapd bucket end overflow".into()))?; + if !canonical_bucket_coverage_matches( + connection, + source_id, + bucket_start, + bucket_end, + observed_units, + expected_units, + )? { + return Ok(false); + } + canonical_bucket_rows_match( + connection, + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_end, + observed_units != 0, + run_maad, + ) +} + +#[derive(Clone, Copy, Debug)] +struct ExpectedNfcapdTopologyBucket { + bucket_end: i64, + child_count: i64, +} + +type ExpectedNfcapdTopology = BTreeMap>; + +fn expected_nfcapd_day_topology( + start: i64, + end: i64, + timezone: &str, +) -> Result { + let mut topology = BTreeMap::new(); + let mut bucket_start = start; + while bucket_start < end { + let five_minute_end = bucket_start + .checked_add(FIVE_MINUTES) + .ok_or_else(|| PipelineError::InvalidConfig("nfcapd bucket end overflow".into()))?; + topology + .entry(Granularity::FiveMinutes) + .or_insert_with(BTreeMap::new) + .insert( + bucket_start, + ExpectedNfcapdTopologyBucket { + bucket_end: five_minute_end, + child_count: 1, + }, + ); + for granularity in [ + Granularity::ThirtyMinutes, + Granularity::OneHour, + Granularity::OneDay, + ] { + let (rollup_start, rollup_end) = aggregate_bounds(bucket_start, granularity, timezone)?; + topology + .entry(granularity) + .or_insert_with(BTreeMap::new) + .entry(rollup_start) + .and_modify(|bucket: &mut ExpectedNfcapdTopologyBucket| { + bucket.child_count += 1; + }) + .or_insert(ExpectedNfcapdTopologyBucket { + bucket_end: rollup_end, + child_count: 1, + }); + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(topology) +} + +fn nfcapd_day_coverage_is_canonical( + connection: &Connection, + source_id: &str, + source_units: usize, + topology: &ExpectedNfcapdTopology, + day_start: i64, + day_end: i64, +) -> Result { + let rows = connection + .prepare(&format!( + "SELECT granularity, bucket_start, bucket_end, coverage_state, + observed_units, expected_units, rejected_units + FROM bucket_coverage + WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} + AND bucket_start >= ?2 AND bucket_start < ?3 + ORDER BY granularity, bucket_start" + )) + .map_err(StorageError::from)? + .query_map(params![source_id, day_start, day_end], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + )) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + let actual = rows + .into_iter() + .map( + |(granularity, start, end, state, observed, expected, rejected)| { + ( + (granularity, start, end), + (state, observed, expected, rejected), + ) + }, + ) + .collect::>(); + let source_units = i64::try_from(source_units) + .map_err(|_| PipelineError::InvalidConfig("nfcapd source unit count overflow".into()))?; + let mut expected_keys = BTreeSet::new(); + for (granularity, buckets) in topology { + for (bucket_start, bucket) in buckets { + expected_keys.insert(( + granularity.as_str().to_owned(), + *bucket_start, + bucket.bucket_end, + )); + let expected_units = source_units + .checked_mul(bucket.child_count) + .ok_or_else(|| { + PipelineError::InvalidConfig("nfcapd coverage unit count overflow".into()) + })?; + let Some((state, observed, actual_expected, rejected)) = actual.get(&( + granularity.as_str().to_owned(), + *bucket_start, + bucket.bucket_end, + )) else { + return Ok(false); + }; + if *state != "complete" + || *observed != expected_units + || *actual_expected != expected_units + || *rejected != 0 + { + return Ok(false); + } + } + } + Ok(actual.keys().all(|key| expected_keys.contains(key)) && actual.len() == expected_keys.len()) +} + +fn nfcapd_day_rows_are_canonical( + connection: &Connection, + source_id: &str, + topology: &ExpectedNfcapdTopology, + run_maad: bool, + day_start: i64, + day_end: i64, +) -> Result { + for table in [ + "traffic_stats", + "protocol_stats", + "address_count_stats", + "port_count_stats", + "address_structure_stats", + ] { + let query = format!( + "SELECT granularity, bucket_start, MIN(bucket_end), MAX(bucket_end), COUNT(*), + COALESCE(SUM(CASE WHEN ip_version IN (4, 6) AND ({predicate}) THEN 1 ELSE 0 END), 0) + FROM {table} + WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} + AND bucket_start >= ?2 AND bucket_start < ?3 + GROUP BY granularity, bucket_start", + predicate = canonical_row_family_predicate(table), + ); + let rows = connection + .prepare(&query) + .map_err(StorageError::from)? + .query_map(params![source_id, day_start, day_end], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + )) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + let actual = rows + .into_iter() + .map( + |(granularity, start, minimum_end, maximum_end, total, canonical)| { + ( + (granularity, start), + (minimum_end, maximum_end, total, canonical), + ) + }, + ) + .collect::>(); + let mut expected_keys = BTreeSet::new(); + for (granularity, buckets) in topology { + let expected = canonical_row_family_count(table, true, run_maad); + for (bucket_start, bucket) in buckets { + let key = (granularity.as_str().to_owned(), *bucket_start); + if expected == 0 { + if actual.contains_key(&key) { + return Ok(false); + } + continue; + } + expected_keys.insert(key.clone()); + let Some((minimum_end, maximum_end, total, canonical)) = actual.get(&key) else { + return Ok(false); + }; + if minimum_end != maximum_end + || *minimum_end != bucket.bucket_end + || *total != expected + || *canonical != expected + { + return Ok(false); + } + } + } + if actual.len() != expected_keys.len() + || actual.keys().any(|key| !expected_keys.contains(key)) + { + return Ok(false); + } + } + Ok(true) +} + +/// Validate the complete topology once before a daily-active day is treated as resumable. The +/// grouped queries inspect all row families and all four granularities in the day, so a single +/// damaged bucket or rollup cannot be hidden by a matching total elsewhere. +fn nfcapd_day_has_canonical_topology( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + timezone: &str, + run_maad: bool, +) -> Result { + #[cfg(test)] + NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS.with(|calls| calls.set(calls.get() + 1)); + let topology = expected_nfcapd_day_topology(start, end, timezone)?; + if topology + .get(&Granularity::FiveMinutes) + .is_none_or(BTreeMap::is_empty) + { + return Ok(false); + } + let range_end = topology + .get(&Granularity::OneDay) + .and_then(|buckets| buckets.values().map(|bucket| bucket.bucket_end).max()) + .unwrap_or(end); + for source in sources { + if !nfcapd_day_coverage_is_canonical( + connection, + &source.source_id, + source.members.len(), + &topology, + start, + range_end, + )? { + return Ok(false); + } + if !nfcapd_day_rows_are_canonical( + connection, + &source.source_id, + &topology, + run_maad, + start, + range_end, + )? { + return Ok(false); + } + } + Ok(!sources.is_empty()) +} + +/// Classify completion evidence for every logical source in a daily-active product. +/// +/// A dirty tombstone is intentionally distinct from a missing marker. Missing markers (including +/// databases created before completion markers existed) may use the legacy topology audit once; +/// dirty days must be rebuilt with `--force` before any normal resume mutation is attempted. +fn nfcapd_day_completion_state( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + run_maad: bool, +) -> Result { + let Some(product_fingerprint) = current_product_fingerprint(connection)? else { + return Ok(DailyProductCompletionState::Missing); + }; + if sources.is_empty() { + return Ok(DailyProductCompletionState::Missing); + } + let mut missing = false; + for source in sources { + match daily_product_completion_state( + connection, + &source.source_id, + start, + end, + &product_fingerprint, + run_maad, + )? { + DailyProductCompletionState::Clean => {} + DailyProductCompletionState::Dirty => return Ok(DailyProductCompletionState::Dirty), + DailyProductCompletionState::Missing => missing = true, + } + } + Ok(if missing { + DailyProductCompletionState::Missing + } else { + DailyProductCompletionState::Clean + }) +} + +/// Publish completion markers after a complete daily-active transaction has written every +/// canonical family, rollup, and evidence/provenance row. +fn mark_nfcapd_day_complete( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + run_maad: bool, +) -> Result<(), PipelineError> { + let product_fingerprint = current_product_fingerprint(connection)?.ok_or_else(|| { + PipelineError::InvalidConfig( + "cannot publish a daily product completion marker before product identity binding" + .into(), + ) + })?; + for source in sources { + upsert_daily_product_completion( + connection, + &source.source_id, + start, + end, + &product_fingerprint, + run_maad, + )?; + } + Ok(()) +} + +fn source_has_candidate( + source: &DatasetSource, + bucket_start: i64, + paths: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, +) -> bool { + let has_file = source + .members + .iter() + .any(|member| paths.contains_key(&(member.clone(), bucket_start))); + has_file + || extend_gaps_to_window + || source.members.iter().any(|member| { + member_bounds + .get(member) + .is_some_and(|(first, last)| *first <= bucket_start && bucket_start <= *last) + }) +} + +/// Group local five-minute starts so each decode batch has at most twelve physical requests when +/// possible. A timestamp with more than twelve members is kept as one batch and drained in +/// physical-request chunks by the decode caller. +fn nfcapd_batch_starts( + start: i64, + end: i64, + timezone: &str, + sources: &[DatasetSource], + paths: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, +) -> Result, PipelineError> { + let mut starts = Vec::with_capacity(NFCAPD_DECODE_BATCH_SIZE); + let mut physical_requests = BTreeSet::new(); + let mut next = start; + while next < end { + let timestamp_requests = sources + .iter() + .filter(|source| { + source_has_candidate(source, next, paths, member_bounds, extend_gaps_to_window) + }) + .flat_map(|source| { + source.members.iter().filter_map(|member| { + paths + .contains_key(&(member.clone(), next)) + .then_some((member.clone(), next)) + }) + }) + .collect::>(); + let would_exceed_physical_limit = !starts.is_empty() + && physical_requests.len() + timestamp_requests.len() > NFCAPD_DECODE_BATCH_SIZE; + if would_exceed_physical_limit || starts.len() == NFCAPD_DECODE_BATCH_SIZE { + break; + } + starts.push(next); + physical_requests.extend(timestamp_requests); + next = next_local_five_minute_start(next, timezone)?; + } + Ok(starts) +} + +fn nfcapd_decode_request_chunks(requests: &[T]) -> impl Iterator { + requests.chunks(NFCAPD_DECODE_BATCH_SIZE) +} + +/// External inputs observed while preparing one local day. The maps stay bounded to that day and +/// let the transaction's final guard verify the exact files used by the day before COMMIT. +#[derive(Clone, Debug, Default)] +struct NfcapdDayGuards { + capture_revisions: BTreeMap, + activity_snapshots: BTreeMap, + nfdump_revision: Option, +} + +struct NfcapdDayResult { + profile: NfcapdDayPublishProfile, + guards: NfcapdDayGuards, +} + +#[allow(clippy::too_many_arguments)] +fn process_nfcapd_tree_day( + connection: &Connection, + root: &Path, + sources: &[DatasetSource], + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + start: i64, + end: i64, + extend_gaps_to_window: bool, + force: bool, + pipeline: &ResolvedPipeline, + capture_snapshots: &BTreeMap, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, + canonical_day_verified: bool, +) -> Result { + let day_started = Instant::now(); + let mut prepare_elapsed = Duration::ZERO; + let mut decode_elapsed = Duration::ZERO; + let mut publish_elapsed = Duration::ZERO; + let mut publish_profile = NfcapdDayPublishProfile::default(); + let mut guards = NfcapdDayGuards { + nfdump_revision: pipeline + .selection + .selects_daily_active_sources() + .then(|| pipeline.nfdump_revision.clone()) + .flatten(), + ..NfcapdDayGuards::default() + }; + let revision_hash_workers = std::thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .min(NFCAPD_REVISION_HASH_MAX_WORKERS); + let revision_pool = rayon::ThreadPoolBuilder::new() + .num_threads(revision_hash_workers) + .thread_name(|index| format!("nfcapd-revision-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build revision hash pool: {error}")) + })?; + let decoder_fingerprint = nfdump_decoder_fingerprint_for_pipeline(pipeline)?; + let revision_context = NfcapdRevisionContext { + connection, + sources, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + force, + decoder_fingerprint, + capture_snapshots, + revision_pool: &revision_pool, + }; + let mut bucket_start = start; + let mut batches = Vec::new(); + while bucket_start < end { + let prepare_started = Instant::now(); + let batch_starts = nfcapd_batch_starts( + bucket_start, + end, + &pipeline.timezone, + sources, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + )?; + bucket_start = batch_starts + .last() + .copied() + .map(|last| next_local_five_minute_start(last, &pipeline.timezone)) + .transpose()? + .expect("non-empty nfcapd batch while processing a non-empty window"); + let revisions = resolve_nfcapd_batch_revisions(&revision_context, &batch_starts)?; + if pipeline.selection.selects_daily_active_sources() { + guards.capture_revisions.extend( + revisions + .iter() + .map(|(path, revision)| (path.clone(), revision.clone())), + ); + } + let mut batch = Vec::with_capacity(batch_starts.len()); + for bucket_start in batch_starts { + batch.push(prepare_nfcapd_tree_timestamp( + connection, + root, + sources, + by_member_and_start, + member_bounds, + bucket_start, + extend_gaps_to_window, + force, + pipeline, + report, + &revisions, + canonical_day_verified, + )?); + } + prepare_elapsed += prepare_started.elapsed(); + batches.push(batch); + } + + if pipeline.selection.selects_daily_active_sources() + && !force + && batches + .iter() + .flatten() + .flat_map(|timestamp| ×tamp.jobs) + .any(|job| job.is_repair) + { + return Err(PipelineError::InvalidConfig(format!( + "daily_active_sources input changed for local day {start}..{end}; rerun that whole day with --force" + ))); + } + let has_jobs = batches + .iter() + .flatten() + .any(|timestamp| !timestamp.jobs.is_empty()); + let active_resolution = if pipeline.selection.selects_daily_active_sources() && has_jobs { + verify_nfdump_revision(pipeline)?; + Some(resolve_daily_active_sources( + sources, + by_member_and_start, + start, + end, + pipeline, + capture_snapshots, + &guards.capture_revisions, + )?) + } else { + None + }; + if active_resolution.is_some() { + verify_nfdump_revision(pipeline)?; + } + if let Some((active_sources, _)) = &active_resolution { + publish_profile.active_set_count = profile_count(active_sources.len()); + } + let decode_pool = if has_jobs { + Some(build_nfcapd_decode_pool()?) + } else { + None + }; + + for batch in batches { + verify_nfdump_revision(pipeline)?; + let decode_started = Instant::now(); + let needed = batch + .iter() + .flat_map(|timestamp| { + timestamp.jobs.iter().flat_map(|job| { + job.present.iter().map(|(member, path)| { + let snapshot = timestamp + .revision_cache + .get(member) + .and_then(|owner| owner.snapshot.clone()) + .expect("present member has a snapshot"); + ( + (member.clone(), timestamp.bucket_start), + (path.clone(), snapshot), + ) + }) + }) + }) + .collect::>(); + let requests = needed.into_iter().collect::>(); + let mut decoded_cache = BTreeMap::new(); + for request_chunk in nfcapd_decode_request_chunks(&requests) { + let decoded = decode_pool + .as_ref() + .expect("pending nfcapd work has a decode pool") + .install(|| { + request_chunk + .par_iter() + .map(|((member, bucket_start), (path, snapshot))| { + let bucket = (|| -> Result { + let bucket = match &active_resolution { + Some((active_sources, _)) => { + ingest::read_nfcapd_bucket_with_active_sources( + path, + member, + &pipeline.selection, + active_sources.clone(), + &pipeline.nfdump, + &pipeline.timezone, + )? + } + None => ingest::read_nfcapd_bucket( + path, + member, + &pipeline.selection, + &pipeline.nfdump, + &pipeline.timezone, + )?, + }; + verify_file_snapshot(path, snapshot)?; + Ok(bucket) + })() + .map_err(|error| { + nfcapd_decode_error(member, *bucket_start, path, error) + })?; + Ok::<_, PipelineError>(((member.clone(), *bucket_start), bucket)) + }) + .collect::, _>>() + })?; + decoded_cache.extend(decoded); + verify_nfdump_revision(pipeline)?; + } + decode_elapsed += decode_started.elapsed(); + + let publish_started = Instant::now(); + for timestamp in batch { + for job in timestamp.jobs { + let member_buckets = job + .present + .iter() + .map(|(member, _)| { + decoded_cache + .get(&(member.clone(), timestamp.bucket_start)) + .expect("requested physical member was decoded") + }) + .collect::>(); + let logical_started = Instant::now(); + let logical = logical_source_bucket( + &job.source_id, + timestamp.bucket_start, + job.expected_units, + &member_buckets, + )?; + publish_profile.logical_source_elapsed += logical_started.elapsed(); + let sibling_started = Instant::now(); + if !job.is_repair { + aggregates.reject_persisted_siblings( + connection, + &logical, + &pipeline.timezone, + )?; + } + publish_profile.persisted_sibling_elapsed += sibling_started.elapsed(); + let bucket_profile = publish_nfcapd_bucket_profiled( + connection, + &logical, + &job.owners, + &job.absences, + &job.evidence, + true, + force, + pipeline.run_maad, + )?; + publish_profile.bucket_publish.include(bucket_profile); + let flushed = if job.is_repair { + refresh_rollups_after_five_minute_repair( + connection, + &logical, + &pipeline.timezone, + )?; + 0 + } else { + let aggregate_profile = + aggregates.include_profiled(&logical, &pipeline.timezone)?; + publish_profile.aggregate_include.include(aggregate_profile); + let flush_started = Instant::now(); + let (flushed, rollup_write) = + aggregates.flush_complete_profiled(connection, pipeline.run_maad)?; + publish_profile.completed_rollup_flush_elapsed += flush_started.elapsed(); + publish_profile.completed_rollup_write.include(rollup_write); + publish_profile.completed_rollup_flushes += 1; + if flushed > 0 { + publish_profile.nonempty_rollup_flushes += 1; + } + flushed + }; + publish_profile.logical_buckets += 1; + report.rollup_buckets += flushed; + report.five_minute_buckets += 1; + } + decoded_cache.retain(|(_, start), _| *start != timestamp.bucket_start); + } + publish_elapsed += publish_started.elapsed(); + } + if let Some((_, snapshots)) = &active_resolution { + for (path, snapshot) in snapshots { + guards + .activity_snapshots + .insert(path.clone(), snapshot.clone()); + verify_file_snapshot(path, snapshot)?; + } + } + verify_nfdump_revision(pipeline)?; + publish_profile.day_elapsed = day_started.elapsed(); + publish_profile.prepare_elapsed = prepare_elapsed; + publish_profile.decode_elapsed = decode_elapsed; + publish_profile.batch_publish_elapsed = publish_elapsed; + tracing::info!( + target: "netflow_db::profile", + phase = "nfcapd_tree_day", + day_start = start, + day_end = end, + elapsed_seconds = publish_profile.day_elapsed.as_secs_f64(), + prepare_seconds = prepare_elapsed.as_secs_f64(), + decode_seconds = decode_elapsed.as_secs_f64(), + publish_seconds = publish_elapsed.as_secs_f64(), + ); + Ok(NfcapdDayResult { + profile: publish_profile, + guards, + }) +} + +fn resolve_daily_active_sources( + sources: &[DatasetSource], + paths: &BTreeMap<(String, i64), PathBuf>, + start: i64, + end: i64, + pipeline: &ResolvedPipeline, + capture_snapshots: &BTreeMap, + revision_snapshots: &BTreeMap, +) -> Result { + let physical_ids = sources + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>(); + let physical_ids = physical_ids.into_iter().collect::>(); + let activity_pool = build_nfcapd_activity_pool()?; + let mut combined = HashMap::::new(); + let mut snapshots = Vec::new(); + for member_chunk in physical_ids.chunks(NFCAPD_DECODE_BATCH_SIZE) { + let requests = member_chunk + .iter() + .map(|member| { + nfcapd_day_activity_paths(paths, member, start, end, &pipeline.timezone) + .map(|member_paths| (member.clone(), member_paths)) + }) + .collect::, _>>()?; + let member_results = activity_pool.install(|| { + requests + .par_iter() + .map(|(member, member_paths)| { + let member_snapshots = member_paths + .iter() + .map(|path| { + let snapshot = capture_snapshots + .get(path) + .cloned() + .or_else(|| { + revision_snapshots + .get(path) + .and_then(|revision| revision.snapshot.clone()) + }) + .map(Ok) + .unwrap_or_else(|| capture_nfcapd_snapshot(path)); + snapshot + .map(|snapshot| (path.clone(), snapshot)) + .map_err(|error| { + daily_activity_scan_error( + member, + start, + end, + member_paths, + error, + ) + }) + }) + .collect::, PipelineError>>()?; + let activity = ingest::read_nfcapd_daily_source_activity( + member_paths, + &pipeline.selection, + &pipeline.nfdump, + ) + .map_err(|error| { + daily_activity_scan_error(member, start, end, member_paths, error) + })?; + Ok::<_, PipelineError>((activity, member_snapshots)) + }) + .collect::, _>>() + })?; + for (activity, member_snapshots) in member_results { + snapshots.extend(member_snapshots); + for (address, metrics) in activity { + combined.entry(address).or_default().include(metrics); + } + } + } + for (path, snapshot) in &snapshots { + verify_file_snapshot(path, snapshot)?; + } + + let mut active = AddressSet::default(); + active.extend(combined.into_iter().filter_map(|(address, metrics)| { + FlowSelection::daily_activity_threshold_met(metrics.flows, metrics.packets, metrics.bytes) + .then_some(address) + })); + tracing::info!( + day_start = start, + day_end = end, + active_sources = active.len(), + "resolved daily active sources" + ); + Ok((Arc::new(active), snapshots)) +} + +struct NfcapdRevisionProbe { + path: PathBuf, + observed: FileSnapshot, + cached_content_fingerprint: Option, +} + +/// Hash a capture after an already-captured observation, retaining the usual before/after +/// stability check without taking a redundant pre-hash snapshot. +fn capture_file_revision_with_snapshot( + path: &Path, + observed: &FileSnapshot, +) -> Result<(String, FileSnapshot), ProvenanceError> { + let content_fingerprint = file_sha256(path)?; + let after = FileSnapshot::capture(path)?; + if &after != observed { + return Err(ProvenanceError::InputContentChanged(format!( + "Input changed while its revision was being captured: {:?}", + path + ))); + } + Ok((content_fingerprint, after)) +} + +struct NfcapdRevisionContext<'a> { + connection: &'a Connection, + sources: &'a [DatasetSource], + by_member_and_start: &'a BTreeMap<(String, i64), PathBuf>, + member_bounds: &'a BTreeMap, + extend_gaps_to_window: bool, + force: bool, + decoder_fingerprint: String, + capture_snapshots: &'a BTreeMap, + revision_pool: &'a rayon::ThreadPool, +} + +/// Resolve the physical files needed by a decode batch before making any job decisions. +/// SQLite access stays on the pipeline thread; only exact hashes run in parallel. +fn resolve_nfcapd_batch_revisions( + context: &NfcapdRevisionContext<'_>, + batch_starts: &[i64], +) -> Result, PipelineError> { + let mut paths = BTreeSet::new(); + for &bucket_start in batch_starts { + for source in context.sources { + if !source_has_candidate( + source, + bucket_start, + context.by_member_and_start, + context.member_bounds, + context.extend_gaps_to_window, + ) { + continue; + } + paths.extend(source.members.iter().filter_map(|member| { + context + .by_member_and_start + .get(&(member.clone(), bucket_start)) + .cloned() + })); + } + } + + let probes = paths + .into_iter() + .map(|path| { + let locator = path.to_string_lossy().into_owned(); + let observed = context + .capture_snapshots + .get(&path) + .cloned() + .map(Ok) + .unwrap_or_else(|| capture_nfcapd_snapshot(&path))?; + let cached_fingerprint = if context.force { + None + } else { + cached_content_fingerprint( + context.connection, + InputKind::Nfcapd, + &locator, + &observed, + )? + }; + Ok::<_, PipelineError>(NfcapdRevisionProbe { + path, + observed, + cached_content_fingerprint: cached_fingerprint, + }) + }) + .collect::, _>>()?; + let decoder_fingerprint = context.decoder_fingerprint.clone(); + + let resolved = context.revision_pool.install(|| { + probes + .par_iter() + .map(|probe| { + let captured = match &probe.cached_content_fingerprint { + Some(content_fingerprint) => { + Ok((content_fingerprint.clone(), probe.observed.clone())) + } + None => capture_file_revision_with_snapshot(&probe.path, &probe.observed), + }; + captured + .map_err(PipelineError::from) + .and_then(|(content_fingerprint, snapshot)| { + let revision = InputRevision::create( + "nfcapd", + probe.path.to_string_lossy().into_owned(), + content_fingerprint, + &decoder_fingerprint, + )?; + Ok(PreparedRevision { + revision, + snapshot: Some(snapshot), + }) + }) + }) + .collect::>() + }); + + probes + .into_iter() + .zip(resolved) + .map(|(probe, result)| result.map(|revision| (probe.path, revision))) + .collect::, _>>() +} + +#[allow(clippy::too_many_arguments)] +fn prepare_nfcapd_tree_timestamp( + connection: &Connection, + root: &Path, + sources: &[DatasetSource], + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + bucket_start: i64, + extend_gaps_to_window: bool, + force: bool, + pipeline: &ResolvedPipeline, + report: &mut PipelineReport, + revisions: &BTreeMap, + canonical_day_verified: bool, +) -> Result { + prepare_nfcapd_tree_timestamp_with_cache( + connection, + root, + sources, + by_member_and_start, + member_bounds, + bucket_start, + extend_gaps_to_window, + force, + pipeline, + report, + revisions, + canonical_day_verified, + None, + ) +} + +#[allow(clippy::too_many_arguments)] +fn prepare_nfcapd_tree_timestamp_with_cache( + connection: &Connection, + root: &Path, + sources: &[DatasetSource], + by_member_and_start: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + bucket_start: i64, + extend_gaps_to_window: bool, + force: bool, + pipeline: &ResolvedPipeline, + report: &mut PipelineReport, + revisions: &BTreeMap, + canonical_day_verified: bool, + resume_cache: Option<&NfcapdDayResumeCache>, +) -> Result { + #[cfg(test)] + PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS.with(|calls| calls.set(calls.get() + 1)); + + let mut revision_cache: BTreeMap = BTreeMap::new(); + let mut jobs = Vec::new(); + for source in sources { + if !source_has_candidate( + source, + bucket_start, + by_member_and_start, + member_bounds, + extend_gaps_to_window, + ) { + continue; + } + let present = source + .members + .iter() + .filter_map(|member| { + by_member_and_start + .get(&(member.clone(), bucket_start)) + .map(|path| (member.clone(), path.clone())) + }) + .collect::>(); + let mut owners = Vec::new(); + for (member, path) in &present { + let owner = match revision_cache.get(member) { + Some(owner) => owner.clone(), + None => { + let owner = revisions + .get(path) + .cloned() + .expect("present member has a resolved revision"); + revision_cache.insert(member.clone(), owner.clone()); + owner + } + }; + owners.push(owner); + } + let mut absences = Vec::new(); + let mut evidence = Vec::with_capacity(source.members.len()); + for ((member, path), owner) in present.iter().zip(&owners) { + evidence.push(InputEvidenceRow::new( + &source.source_id, + member, + bucket_start, + bucket_start + FIVE_MINUTES, + path.to_string_lossy(), + InputEvidenceState::Observed, + Some(owner.revision.fingerprint.clone()), + )); + } + for member in &source.members { + if !present.iter().any(|(present, _)| present == member) { + let expected = + expected_nfcapd_path(root, member, bucket_start, &pipeline.timezone)?; + absences.push(ExpectedAbsence::capture(&expected)?); + evidence.push(InputEvidenceRow::new( + &source.source_id, + member, + bucket_start, + bucket_start + FIVE_MINUTES, + expected.to_string_lossy(), + InputEvidenceState::Missing, + None, + )); + } + } + evidence.sort_unstable_by(|left, right| left.unit_id.cmp(&right.unit_id)); + let previous_evidence = match resume_cache { + Some(cache) => Cow::Borrowed(cache.evidence(&source.source_id, bucket_start)), + None => Cow::Owned(query_input_evidence( + connection, + &source.source_id, + bucket_start, + )?), + }; + let observed_input_disappeared = previous_evidence.iter().any(|previous| { + previous.evidence_state == InputEvidenceState::Observed + && evidence.iter().any(|current| { + current.unit_id == previous.unit_id + && current.evidence_state == InputEvidenceState::Missing + }) + }); + if observed_input_disappeared { + tracing::warn!( + source_id = source.source_id, + bucket_start, + "preserving prior bucket because an observed input is now missing" + ); + report.skipped_inputs += 1; + continue; + } + let revisions = owners + .iter() + .map(|owner| owner.revision.clone()) + .collect::>(); + let persisted_processed = if force || revisions.is_empty() { + false + } else { + match resume_cache { + Some(cache) => cache.processed(&source.source_id, bucket_start, &revisions)?, + None => nfcapd_logical_bucket_processed( + connection, + &source.source_id, + bucket_start, + &revisions, + )?, + } + }; + let mut topology_corruption = false; + if !force && previous_evidence == evidence { + let provenance_complete = revisions.is_empty() || persisted_processed; + if provenance_complete { + let topology_matches = canonical_day_verified + || nfcapd_logical_bucket_has_canonical_topology( + connection, + &source.source_id, + bucket_start, + present.len(), + source.members.len(), + pipeline.run_maad, + )?; + if topology_matches { + report.skipped_inputs += 1; + continue; + } + topology_corruption = true; + } + } + let orphaned_provenance = !force + && previous_evidence.is_empty() + && persisted_processed + && (canonical_day_verified + || nfcapd_logical_bucket_has_canonical_topology( + connection, + &source.source_id, + bucket_start, + present.len(), + source.members.len(), + pipeline.run_maad, + )?); + let is_repair = !force + && (orphaned_provenance + || (!previous_evidence.is_empty() + && (previous_evidence != evidence || topology_corruption))); + jobs.push(PreparedTreeJob { + source_id: source.source_id.clone(), + expected_units: source.members.len(), + present, + owners, + absences, + evidence, + is_repair, + }); + } + Ok(PreparedTreeTimestamp { + bucket_start, + revision_cache, + jobs, + }) +} + +#[allow(clippy::too_many_arguments)] +enum PreparedExplicitNfcapdKind { + File(PreparedRevision), + Gap { expected_path: Option }, +} + +struct PreparedExplicitNfcapd { + path: PathBuf, + source_id: String, + bucket_start: i64, + kind: PreparedExplicitNfcapdKind, +} + +fn process_explicit_nfcapd_inputs( + connection: &Connection, + inputs: &[InputSpec], + pipeline: &ResolvedPipeline, +) -> Result { + let mut prepared = Vec::new(); + for input in inputs { + let InputSpec::Nfcapd { + path, + source_id, + bucket_start, + gap, + expected_path, + } = input + else { + continue; + }; + let bucket_start = match bucket_start { + Some(start) => *start, + None if !gap => ingest::parse_nfcapd_bucket_start(path, &pipeline.timezone)?, + None => { + return Err(PipelineError::InvalidConfig( + "explicit nfcapd gap requires bucket_start".into(), + )); + } + }; + let kind = if *gap { + PreparedExplicitNfcapdKind::Gap { + expected_path: expected_path.clone(), + } + } else { + let (revision, snapshot) = prepare_file_revision( + connection, + path, + InputKind::Nfcapd, + nfdump_decoder_fingerprint_for_pipeline(pipeline)?, + )?; + PreparedExplicitNfcapdKind::File(PreparedRevision { + revision, + snapshot: Some(snapshot), + }) + }; + prepared.push(PreparedExplicitNfcapd { + path: path.clone(), + source_id: source_id.clone(), + bucket_start, + kind, + }); + } + prepared.sort_unstable_by(|left, right| { + (left.bucket_start, &left.source_id, &left.path).cmp(&( + right.bucket_start, + &right.source_id, + &right.path, + )) + }); + if prepared.is_empty() { + return Ok(PipelineReport::default()); + } + process_atomic(connection, pipeline, |aggregates, report| { + for input in &prepared { + match &input.kind { + PreparedExplicitNfcapdKind::File(owner) => process_nfcapd( + connection, + &input.path, + &input.source_id, + input.bucket_start, + owner, + pipeline, + aggregates, + report, + )?, + PreparedExplicitNfcapdKind::Gap { expected_path } => process_nfcapd_gap( + connection, + &input.path, + expected_path.as_deref(), + &input.source_id, + input.bucket_start, + pipeline, + aggregates, + report, + )?, + } + } + Ok(()) + }) +} + +#[allow(clippy::too_many_arguments)] +fn process_nfcapd( + connection: &Connection, + path: &Path, + source_id: &str, + bucket_start: i64, + owner: &PreparedRevision, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + if nfcapd_logical_bucket_processed( + connection, + source_id, + bucket_start, + std::slice::from_ref(&owner.revision), + )? { + report.skipped_inputs += 1; + return Ok(()); + } + verify_nfdump_revision(pipeline)?; + let bucket = ingest::read_nfcapd_bucket( + path, + source_id, + &pipeline.selection, + &pipeline.nfdump, + &pipeline.timezone, + ) + .map_err(|error| nfcapd_decode_error(source_id, bucket_start, path, error))?; + verify_nfdump_revision(pipeline)?; + let snapshot = owner + .snapshot + .as_ref() + .expect("explicit file input has a snapshot"); + verify_file_snapshot(path, snapshot) + .map_err(|error| nfcapd_decode_error(source_id, bucket_start, path, error))?; + aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; + publish_nfcapd_bucket( + connection, + &bucket, + std::slice::from_ref(owner), + &[], + &[InputEvidenceRow::new( + source_id, + source_id, + bucket_start, + bucket_start + FIVE_MINUTES, + &owner.revision.locator, + InputEvidenceState::Observed, + Some(owner.revision.fingerprint.clone()), + )], + false, + false, + pipeline.run_maad, + )?; + aggregates.include(&bucket, &pipeline.timezone)?; + report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; + report.five_minute_buckets += 1; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn process_nfcapd_gap( + connection: &Connection, + _locator_path: &Path, + expected_path: Option<&Path>, + source_id: &str, + bucket_start: i64, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + let expected_path = expected_path.ok_or_else(|| { + PipelineError::InvalidConfig( + "explicit nfcapd gap requires expected_path for absence verification".into(), + ) + })?; + let absence = ExpectedAbsence::capture(expected_path)?; + let evidence = [InputEvidenceRow::new( + source_id, + source_id, + bucket_start, + bucket_start + FIVE_MINUTES, + expected_path.to_string_lossy(), + InputEvidenceState::Missing, + None, + )]; + if query_input_evidence(connection, source_id, bucket_start)? == evidence { + report.skipped_inputs += 1; + return Ok(()); + } + let bucket = StatisticalBucket::new(BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + )) + .with_coverage(BucketCoverage::new(1, 0, 0).map_err(DomainError::from)?) + .finish_owned(); + aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; + publish_nfcapd_bucket( + connection, + &bucket, + &[], + &[absence], + &evidence, + false, + false, + pipeline.run_maad, + )?; + aggregates.include(&bucket, &pipeline.timezone)?; + report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; + report.five_minute_buckets += 1; + Ok(()) +} + +fn normalize_sources( + root: &Path, + source_ids: &[String], + sources: &[DatasetSource], +) -> Result, PipelineError> { + if !source_ids.is_empty() && !sources.is_empty() { + return Err(PipelineError::InvalidConfig( + "nfcapd_tree cannot define both source_ids and sources".into(), + )); + } + let mut normalized = if !sources.is_empty() { + sources.to_vec() + } else if !source_ids.is_empty() { + source_ids + .iter() + .map(|source_id| DatasetSource { + source_id: source_id.clone(), + members: vec![source_id.clone()], + }) + .collect() + } else { + let entries = fs::read_dir(root)?; + entries + .filter_map(Result::ok) + .filter_map(|entry| { + entry.file_type().ok()?.is_dir().then(|| { + let source_id = entry.file_name().to_string_lossy().into_owned(); + DatasetSource { + source_id: source_id.clone(), + members: vec![source_id], + } + }) + }) + .collect() + }; + normalized.sort_unstable_by(|left, right| left.source_id.cmp(&right.source_id)); + let mut ids = BTreeSet::new(); + for source in &normalized { + if !is_safe_path_component(&source.source_id) + || source.members.is_empty() + || !ids.insert(source.source_id.clone()) + { + return Err(PipelineError::InvalidConfig( + "logical sources require unique non-empty IDs and members".into(), + )); + } + let mut members = BTreeSet::new(); + for member in &source.members { + if !is_safe_path_component(member) || !members.insert(member) { + return Err(PipelineError::InvalidConfig(format!( + "source {:?} has an unsafe or duplicate member path component", + source.source_id + ))); + } + if !root.join(member).is_dir() { + return Err(PipelineError::InvalidConfig(format!( + "source {:?} references missing member directory {:?}", + source.source_id, member + ))); + } + } + } + + let member_ids = normalized + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>(); + let mut member_paths = BTreeMap::::new(); + #[cfg(unix)] + let mut member_identities = BTreeMap::<(u64, u64), (String, PathBuf)>::new(); + for member in member_ids { + let member_path = root.join(&member); + let canonical_member_path = canonical_path(&member_path)?; + if let Some(previous_member) = member_paths.get(&canonical_member_path) { + return Err(PipelineError::InvalidConfig(format!( + "nfcapd_tree member IDs {:?} and {:?} resolve to the same directory {}", + previous_member, + member, + canonical_member_path.display() + ))); + } + member_paths.insert(canonical_member_path.clone(), member.clone()); + + #[cfg(unix)] + { + let identity = existing_path_identity(&member_path)?.ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "source member directory {:?} disappeared during normalization", + member + )) + })?; + if let Some((previous_member, previous_path)) = member_identities.get(&identity) { + return Err(PipelineError::InvalidConfig(format!( + "nfcapd_tree member IDs {:?} and {:?} resolve to the same physical directory via device/inode {:?} ({} and {})", + previous_member, + member, + identity, + previous_path.display(), + canonical_member_path.display() + ))); + } + member_identities.insert(identity, (member, canonical_member_path)); + } + } + Ok(normalized) +} + +fn merge_source_bucket( + source_id: &str, + bucket_start: i64, + expected_units: usize, + members: &[&CanonicalBucket], +) -> Result { + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + ); + let mut builder = if members.is_empty() { + StatisticalBucket::new(key) + } else { + StatisticalBucket::dense(key) + } + .with_coverage(BucketCoverage::empty()); + for member in members { + builder.include(member)?; + } + let coverage = BucketCoverage::new( + u64::try_from(expected_units).unwrap_or(u64::MAX), + u64::try_from(members.len()).unwrap_or(u64::MAX), + 0, + ) + .map_err(DomainError::from)?; + Ok(builder.with_coverage(coverage).finish_owned()) +} + +fn logical_source_bucket<'a>( + source_id: &str, + bucket_start: i64, + expected_units: usize, + members: &[&'a CanonicalBucket], +) -> Result, PipelineError> { + let expected_key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + ); + if expected_units == 1 + && let [member] = members + && member.key == expected_key + { + return Ok(Cow::Borrowed(member)); + } + Ok(Cow::Owned(merge_source_bucket( + source_id, + bucket_start, + expected_units, + members, + )?)) +} + +#[derive(Debug, Default)] +struct NfcapdBucketPublishProfile { + total_elapsed: Duration, + preflight_elapsed: Duration, + overlap_elapsed: Duration, + force_delete_elapsed: Duration, + owner_upsert_elapsed: Duration, + write: WriteBucketsProfile, + owner_status_elapsed: Duration, + postflight_elapsed: Duration, + owners: u64, + absences: u64, +} + +impl NfcapdBucketPublishProfile { + fn include(&mut self, profile: Self) { + self.total_elapsed += profile.total_elapsed; + self.preflight_elapsed += profile.preflight_elapsed; + self.overlap_elapsed += profile.overlap_elapsed; + self.force_delete_elapsed += profile.force_delete_elapsed; + self.owner_upsert_elapsed += profile.owner_upsert_elapsed; + self.write.include(profile.write); + self.owner_status_elapsed += profile.owner_status_elapsed; + self.postflight_elapsed += profile.postflight_elapsed; + self.owners += profile.owners; + self.absences += profile.absences; + } + + fn other_elapsed(&self) -> Duration { + self.total_elapsed.saturating_sub( + self.preflight_elapsed + + self.overlap_elapsed + + self.force_delete_elapsed + + self.owner_upsert_elapsed + + self.write.total_elapsed + + self.owner_status_elapsed + + self.postflight_elapsed, + ) + } +} + +#[derive(Debug, Default)] +struct FinalRollupProfile { + total_elapsed: Duration, + finish_elapsed: Duration, + delete_elapsed: Duration, + write: WriteBucketsProfile, + incomplete_keys: u64, + rollup_buckets: u64, +} + +impl FinalRollupProfile { + fn other_elapsed(&self) -> Duration { + self.total_elapsed + .saturating_sub(self.finish_elapsed + self.delete_elapsed + self.write.total_elapsed) + } +} + +#[derive(Debug, Default)] +struct AggregateGranularityProfile { + total_elapsed: Duration, + bounds_elapsed: Duration, + builder_elapsed: Duration, + bucket: StatisticalBucketIncludeProfile, +} + +impl AggregateGranularityProfile { + fn include( + &mut self, + total_elapsed: Duration, + bounds_elapsed: Duration, + builder_elapsed: Duration, + bucket: StatisticalBucketIncludeProfile, + ) { + self.total_elapsed += total_elapsed; + self.bounds_elapsed += bounds_elapsed; + self.builder_elapsed += builder_elapsed; + self.bucket.include(bucket); + } + + fn other_elapsed(&self) -> Duration { + self.total_elapsed + .saturating_sub(self.bounds_elapsed + self.builder_elapsed + self.bucket.total_elapsed) + } +} + +#[derive(Debug, Default)] +struct AggregateIncludeProfile { + total_elapsed: Duration, + thirty_minutes: AggregateGranularityProfile, + one_hour: AggregateGranularityProfile, + one_day: AggregateGranularityProfile, +} + +impl AggregateIncludeProfile { + fn include(&mut self, profile: Self) { + self.total_elapsed += profile.total_elapsed; + self.thirty_minutes.include( + profile.thirty_minutes.total_elapsed, + profile.thirty_minutes.bounds_elapsed, + profile.thirty_minutes.builder_elapsed, + profile.thirty_minutes.bucket, + ); + self.one_hour.include( + profile.one_hour.total_elapsed, + profile.one_hour.bounds_elapsed, + profile.one_hour.builder_elapsed, + profile.one_hour.bucket, + ); + self.one_day.include( + profile.one_day.total_elapsed, + profile.one_day.bounds_elapsed, + profile.one_day.builder_elapsed, + profile.one_day.bucket, + ); + } + + fn granularity_mut(&mut self, granularity: Granularity) -> &mut AggregateGranularityProfile { + match granularity { + Granularity::ThirtyMinutes => &mut self.thirty_minutes, + Granularity::OneHour => &mut self.one_hour, + Granularity::OneDay => &mut self.one_day, + Granularity::FiveMinutes => unreachable!("five-minute buckets are not rollups"), + } + } + + fn other_elapsed(&self) -> Duration { + self.total_elapsed.saturating_sub( + self.thirty_minutes.total_elapsed + + self.one_hour.total_elapsed + + self.one_day.total_elapsed, + ) + } +} + +#[derive(Debug, Default)] +struct NfcapdDayPublishProfile { + day_elapsed: Duration, + prepare_elapsed: Duration, + decode_elapsed: Duration, + batch_publish_elapsed: Duration, + logical_source_elapsed: Duration, + persisted_sibling_elapsed: Duration, + bucket_publish: NfcapdBucketPublishProfile, + aggregate_include: AggregateIncludeProfile, + completed_rollup_flush_elapsed: Duration, + completed_rollup_write: WriteBucketsProfile, + final_rollups: FinalRollupProfile, + logical_buckets: u64, + completed_rollup_flushes: u64, + nonempty_rollup_flushes: u64, + active_set_count: u64, +} + +impl NfcapdDayPublishProfile { + fn log(&self, day_start: i64, day_end: i64, transaction_elapsed: Duration) { + self.log_with_context( + day_start, + day_end, + transaction_elapsed, + "single", + None, + None, + ); + } + + fn log_coordinated( + &self, + day_start: i64, + day_end: i64, + transaction_elapsed: Duration, + output_index: usize, + output_path: &Path, + ) { + self.log_with_context( + day_start, + day_end, + transaction_elapsed, + "coordinated", + Some(output_index), + Some(output_path), + ); + } + + fn log_with_context( + &self, + day_start: i64, + day_end: i64, + transaction_elapsed: Duration, + mode: &'static str, + output_index: Option, + output_path: Option<&Path>, + ) { + let mut rollup_write = self.completed_rollup_write.clone(); + rollup_write.include(self.final_rollups.write.clone()); + let publish_other = self.batch_publish_elapsed.saturating_sub( + self.logical_source_elapsed + + self.persisted_sibling_elapsed + + self.bucket_publish.total_elapsed + + self.aggregate_include.total_elapsed + + self.completed_rollup_flush_elapsed, + ); + let transaction_other = + transaction_elapsed.saturating_sub(self.day_elapsed + self.final_rollups.total_elapsed); + let completed_rollup_housekeeping = self + .completed_rollup_flush_elapsed + .saturating_sub(self.completed_rollup_write.total_elapsed); + tracing::info!( + target: "netflow_db::profile", + phase = "nfcapd_tree_day_publish_detail", + mode, + output_index = ?output_index, + output_path = ?output_path, + day_start, + day_end, + transaction_seconds = transaction_elapsed.as_secs_f64(), + transaction_other_seconds = transaction_other.as_secs_f64(), + day_seconds = self.day_elapsed.as_secs_f64(), + prepare_seconds = self.prepare_elapsed.as_secs_f64(), + decode_seconds = self.decode_elapsed.as_secs_f64(), + batch_publish_seconds = self.batch_publish_elapsed.as_secs_f64(), + publish_other_seconds = publish_other.as_secs_f64(), + logical_source_seconds = self.logical_source_elapsed.as_secs_f64(), + persisted_sibling_seconds = self.persisted_sibling_elapsed.as_secs_f64(), + bucket_publish_seconds = self.bucket_publish.total_elapsed.as_secs_f64(), + bucket_preflight_seconds = self.bucket_publish.preflight_elapsed.as_secs_f64(), + bucket_overlap_seconds = self.bucket_publish.overlap_elapsed.as_secs_f64(), + bucket_force_delete_seconds = self.bucket_publish.force_delete_elapsed.as_secs_f64(), + owner_upsert_seconds = self.bucket_publish.owner_upsert_elapsed.as_secs_f64(), + owner_status_seconds = self.bucket_publish.owner_status_elapsed.as_secs_f64(), + bucket_postflight_seconds = self.bucket_publish.postflight_elapsed.as_secs_f64(), + bucket_other_seconds = self.bucket_publish.other_elapsed().as_secs_f64(), + aggregate_include_seconds = self.aggregate_include.total_elapsed.as_secs_f64(), + aggregate_include_other_seconds = self.aggregate_include.other_elapsed().as_secs_f64(), + aggregate_30m_seconds = self.aggregate_include.thirty_minutes.total_elapsed.as_secs_f64(), + aggregate_30m_bounds_seconds = self.aggregate_include.thirty_minutes.bounds_elapsed.as_secs_f64(), + aggregate_30m_builder_seconds = self.aggregate_include.thirty_minutes.builder_elapsed.as_secs_f64(), + aggregate_30m_traffic_seconds = self.aggregate_include.thirty_minutes.bucket.traffic_elapsed.as_secs_f64(), + aggregate_30m_protocols_seconds = self.aggregate_include.thirty_minutes.bucket.protocols_elapsed.as_secs_f64(), + aggregate_30m_addresses_seconds = self.aggregate_include.thirty_minutes.bucket.addresses_elapsed.as_secs_f64(), + aggregate_30m_ports_seconds = self.aggregate_include.thirty_minutes.bucket.ports_elapsed.as_secs_f64(), + aggregate_30m_coverage_seconds = self.aggregate_include.thirty_minutes.bucket.coverage_elapsed.as_secs_f64(), + aggregate_30m_bucket_other_seconds = self.aggregate_include.thirty_minutes.bucket.other_elapsed().as_secs_f64(), + aggregate_30m_other_seconds = self.aggregate_include.thirty_minutes.other_elapsed().as_secs_f64(), + aggregate_1h_seconds = self.aggregate_include.one_hour.total_elapsed.as_secs_f64(), + aggregate_1h_bounds_seconds = self.aggregate_include.one_hour.bounds_elapsed.as_secs_f64(), + aggregate_1h_builder_seconds = self.aggregate_include.one_hour.builder_elapsed.as_secs_f64(), + aggregate_1h_traffic_seconds = self.aggregate_include.one_hour.bucket.traffic_elapsed.as_secs_f64(), + aggregate_1h_protocols_seconds = self.aggregate_include.one_hour.bucket.protocols_elapsed.as_secs_f64(), + aggregate_1h_addresses_seconds = self.aggregate_include.one_hour.bucket.addresses_elapsed.as_secs_f64(), + aggregate_1h_ports_seconds = self.aggregate_include.one_hour.bucket.ports_elapsed.as_secs_f64(), + aggregate_1h_coverage_seconds = self.aggregate_include.one_hour.bucket.coverage_elapsed.as_secs_f64(), + aggregate_1h_bucket_other_seconds = self.aggregate_include.one_hour.bucket.other_elapsed().as_secs_f64(), + aggregate_1h_other_seconds = self.aggregate_include.one_hour.other_elapsed().as_secs_f64(), + aggregate_1d_seconds = self.aggregate_include.one_day.total_elapsed.as_secs_f64(), + aggregate_1d_bounds_seconds = self.aggregate_include.one_day.bounds_elapsed.as_secs_f64(), + aggregate_1d_builder_seconds = self.aggregate_include.one_day.builder_elapsed.as_secs_f64(), + aggregate_1d_traffic_seconds = self.aggregate_include.one_day.bucket.traffic_elapsed.as_secs_f64(), + aggregate_1d_protocols_seconds = self.aggregate_include.one_day.bucket.protocols_elapsed.as_secs_f64(), + aggregate_1d_addresses_seconds = self.aggregate_include.one_day.bucket.addresses_elapsed.as_secs_f64(), + aggregate_1d_ports_seconds = self.aggregate_include.one_day.bucket.ports_elapsed.as_secs_f64(), + aggregate_1d_coverage_seconds = self.aggregate_include.one_day.bucket.coverage_elapsed.as_secs_f64(), + aggregate_1d_bucket_other_seconds = self.aggregate_include.one_day.bucket.other_elapsed().as_secs_f64(), + aggregate_1d_other_seconds = self.aggregate_include.one_day.other_elapsed().as_secs_f64(), + completed_rollup_flush_seconds = self.completed_rollup_flush_elapsed.as_secs_f64(), + completed_rollup_housekeeping_seconds = completed_rollup_housekeeping.as_secs_f64(), + final_rollup_seconds = self.final_rollups.total_elapsed.as_secs_f64(), + final_rollup_finish_seconds = self.final_rollups.finish_elapsed.as_secs_f64(), + final_rollup_delete_seconds = self.final_rollups.delete_elapsed.as_secs_f64(), + final_rollup_other_seconds = self.final_rollups.other_elapsed().as_secs_f64(), + five_minute_write_seconds = self.bucket_publish.write.total_elapsed.as_secs_f64(), + five_minute_delete_seconds = self.bucket_publish.write.delete_elapsed.as_secs_f64(), + five_minute_canonical_rows_seconds = self.bucket_publish.write.canonical_rows_elapsed.as_secs_f64(), + five_minute_scalar_rows_seconds = self.bucket_publish.write.scalar_rows_elapsed.as_secs_f64(), + five_minute_scalar_insert_seconds = scalar_insert_elapsed(&self.bucket_publish.write).as_secs_f64(), + five_minute_maad_seconds = self.bucket_publish.write.maad_elapsed.as_secs_f64(), + five_minute_address_structure_insert_seconds = self.bucket_publish.write.address_structure_insert_elapsed.as_secs_f64(), + five_minute_write_other_seconds = self.bucket_publish.write.other_elapsed().as_secs_f64(), + rollup_write_seconds = rollup_write.total_elapsed.as_secs_f64(), + rollup_delete_seconds = rollup_write.delete_elapsed.as_secs_f64(), + rollup_canonical_rows_seconds = rollup_write.canonical_rows_elapsed.as_secs_f64(), + rollup_scalar_rows_seconds = rollup_write.scalar_rows_elapsed.as_secs_f64(), + rollup_scalar_insert_seconds = scalar_insert_elapsed(&rollup_write).as_secs_f64(), + rollup_maad_seconds = rollup_write.maad_elapsed.as_secs_f64(), + rollup_address_structure_insert_seconds = rollup_write.address_structure_insert_elapsed.as_secs_f64(), + rollup_write_other_seconds = rollup_write.other_elapsed().as_secs_f64(), + logical_buckets = self.logical_buckets, + owners = self.bucket_publish.owners, + absences = self.bucket_publish.absences, + completed_rollup_flushes = self.completed_rollup_flushes, + nonempty_rollup_flushes = self.nonempty_rollup_flushes, + active_set_count = self.active_set_count, + final_incomplete_keys = self.final_rollups.incomplete_keys, + final_rollup_buckets = self.final_rollups.rollup_buckets, + five_minute_write_calls = self.bucket_publish.write.write_calls, + rollup_write_calls = rollup_write.write_calls, + five_minute_bucket_keys = self.bucket_publish.write.bucket_keys, + rollup_bucket_keys = rollup_write.bucket_keys, + traffic_rows = self.bucket_publish.write.traffic_rows + rollup_write.traffic_rows, + protocol_rows = self.bucket_publish.write.protocol_rows + rollup_write.protocol_rows, + address_count_rows = self.bucket_publish.write.address_count_rows + rollup_write.address_count_rows, + port_count_rows = self.bucket_publish.write.port_count_rows + rollup_write.port_count_rows, + address_structure_rows = self.bucket_publish.write.address_structure_rows + rollup_write.address_structure_rows, + maad_address_sets = self.bucket_publish.write.maad_address_sets + rollup_write.maad_address_sets, + maad_addresses = self.bucket_publish.write.maad_addresses + rollup_write.maad_addresses, + address_structure_json_bytes = self.bucket_publish.write.address_structure_json_bytes + rollup_write.address_structure_json_bytes, + ); + } +} + +fn scalar_insert_elapsed(profile: &WriteBucketsProfile) -> Duration { + profile.traffic_insert_elapsed + + profile.protocol_insert_elapsed + + profile.address_count_insert_elapsed + + profile.port_count_insert_elapsed +} + +fn profile_count(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +#[derive(Clone, Debug)] +struct PreparedRevision { + revision: InputRevision, + snapshot: Option, +} + +struct PreparedTreeJob { + source_id: String, + expected_units: usize, + present: Vec<(String, PathBuf)>, + owners: Vec, + absences: Vec, + evidence: Vec, + is_repair: bool, +} + +struct PreparedTreeTimestamp { + bucket_start: i64, + revision_cache: BTreeMap, + jobs: Vec, +} + +#[allow(clippy::too_many_arguments)] +fn publish_nfcapd_bucket( + connection: &Connection, + bucket: &CanonicalBucket, + owners: &[PreparedRevision], + absences: &[ExpectedAbsence], + evidence: &[InputEvidenceRow], + allow_coverage_repair: bool, + force: bool, + run_maad: bool, +) -> Result<(), PipelineError> { + publish_nfcapd_bucket_profiled( + connection, + bucket, + owners, + absences, + evidence, + allow_coverage_repair, + force, + run_maad, + ) + .map(|_| ()) +} + +#[allow(clippy::too_many_arguments)] +fn publish_nfcapd_bucket_profiled( + connection: &Connection, + bucket: &CanonicalBucket, + owners: &[PreparedRevision], + absences: &[ExpectedAbsence], + evidence: &[InputEvidenceRow], + allow_coverage_repair: bool, + force: bool, + run_maad: bool, +) -> Result { + let total_started = Instant::now(); + let mut profile = NfcapdBucketPublishProfile { + owners: profile_count(owners.len()), + absences: profile_count(absences.len()), + ..NfcapdBucketPublishProfile::default() + }; + let preflight_started = Instant::now(); + for absence in absences { + absence.verify()?; + } + for owner in owners { + if let Some(snapshot) = &owner.snapshot { + verify_file_snapshot(&owner.revision.locator, snapshot)?; + } + } + profile.preflight_elapsed += preflight_started.elapsed(); + let overlap_started = Instant::now(); + reject_overlapping_bucket( + connection, + bucket, + InputKind::Nfcapd, + "", + force || allow_coverage_repair, + )?; + profile.overlap_elapsed += overlap_started.elapsed(); + if force { + let force_delete_started = Instant::now(); + connection.execute( + "DELETE FROM processed_inputs WHERE input_kind = 'nfcapd' AND source_id = ?1 AND bucket_start = ?2", + params![bucket.key.source_id, bucket.key.bucket_start], + ).map_err(StorageError::from)?; + profile.force_delete_elapsed += force_delete_started.elapsed(); + } + let publication = (|| -> Result<(), PipelineError> { + let owner_upsert_started = Instant::now(); + for prepared in owners { + let revision = &prepared.revision; + let owner = InputBucket { + input_kind: InputKind::Nfcapd, + input_locator: revision.locator.clone(), + scan_locator: revision.locator.clone(), + source_id: bucket.key.source_id.clone(), + bucket_start: bucket.key.bucket_start, + bucket_end: bucket.key.bucket_end, + revision: revision.clone(), + file_snapshot: prepared.snapshot.clone(), + }; + upsert_input_bucket(connection, &owner, force)?; + } + profile.owner_upsert_elapsed += owner_upsert_started.elapsed(); + profile.write = write_buckets_profiled(connection, std::slice::from_ref(bucket), run_maad)?; + replace_input_evidence( + connection, + &bucket.key.source_id, + bucket.key.bucket_start, + evidence, + )?; + let owner_status_started = Instant::now(); + for prepared in owners { + let revision = &prepared.revision; + mark_input_bucket_status( + connection, + InputKind::Nfcapd, + &revision.locator, + &bucket.key.source_id, + bucket.key.bucket_start, + InputStatus::Processed, + revision, + None, + )?; + } + profile.owner_status_elapsed += owner_status_started.elapsed(); + let postflight_started = Instant::now(); + for absence in absences { + absence.verify()?; + } + profile.postflight_elapsed += postflight_started.elapsed(); + Ok(()) + })(); + publication?; + profile.total_elapsed = total_started.elapsed(); + Ok(profile) +} + +fn reject_overlapping_bucket( + connection: &Connection, + bucket: &CanonicalBucket, + input_kind: InputKind, + allowed_scan: &str, + replace_nfcapd: bool, +) -> Result<(), PipelineError> { + let conflict = connection + .query_row( + "SELECT input_kind, input_locator, scan_locator FROM processed_inputs + WHERE source_id = ?1 AND bucket_start = ?2 + AND NOT (input_kind = ?3 AND scan_locator = ?4) + ORDER BY input_kind, input_locator LIMIT 1", + params![ + bucket.key.source_id, + bucket.key.bucket_start, + input_kind.as_str(), + allowed_scan, + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .optional() + .map_err(StorageError::from)?; + if let Some((kind, locator, _)) = conflict + && !(replace_nfcapd && kind == InputKind::Nfcapd.as_str()) + { + return Err(PipelineError::InvalidConfig(format!( + "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", + bucket.key.source_id, bucket.key.bucket_start + ))); + } + Ok(()) +} + +#[derive(Default)] +struct AggregateBuckets { + builders: BTreeMap<(String, Granularity, i64, i64), StatisticalBucket>, + published_through: BTreeMap, + owned_keys: BTreeSet<(String, i64)>, + current_run_keys: BTreeSet<(String, i64)>, + persisted_sibling_validations: BTreeSet<(String, i64, bool)>, + #[cfg(test)] + persisted_sibling_queries: usize, +} + +impl AggregateBuckets { + fn with_owned_keys(owned_keys: BTreeSet<(String, i64)>) -> Self { + Self { + owned_keys, + ..Self::default() + } + } + + fn reject_persisted_siblings( + &mut self, + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, + ) -> Result<(), PipelineError> { + self.reject_persisted_siblings_inner(connection, child, timezone, false) + } + + fn reject_persisted_csv_siblings( + &mut self, + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, + ) -> Result<(), PipelineError> { + self.reject_persisted_siblings_inner(connection, child, timezone, true) + } + + fn reject_persisted_siblings_inner( + &mut self, + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, + allow_staged_csv_keys: bool, + ) -> Result<(), PipelineError> { + let (day_start, day_end) = + aggregate_bounds(child.key.bucket_start, Granularity::OneDay, timezone)?; + let validation_key = ( + child.key.source_id.clone(), + day_start, + allow_staged_csv_keys, + ); + if self.persisted_sibling_validations.contains(&validation_key) { + return Ok(()); + } + #[cfg(test)] + { + self.persisted_sibling_queries += 1; + } + let mut statement = connection + .prepare( + "SELECT DISTINCT bucket_start FROM traffic_stats + WHERE source_id = ?1 AND granularity = '5m' + AND bucket_start >= ?2 AND bucket_start < ?3 + ORDER BY bucket_start", + ) + .map_err(StorageError::from)?; + let persisted = statement + .query_map(params![child.key.source_id, day_start, day_end], |row| { + row.get::<_, i64>(0) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + for bucket_start in persisted { + if bucket_start == child.key.bucket_start + || self + .owned_keys + .contains(&(child.key.source_id.clone(), bucket_start)) + || self + .current_run_keys + .contains(&(child.key.source_id.clone(), bucket_start)) + { + continue; + } + if allow_staged_csv_keys + && connection + .query_row( + "SELECT 1 FROM csv_bucket_stage + WHERE source_id = ?1 AND bucket_start = ?2 LIMIT 1", + params![child.key.source_id, bucket_start], + |_| Ok(()), + ) + .optional() + .map_err(StorageError::from)? + .is_some() + { + continue; + } + return Err(PipelineError::InvalidConfig(format!( + "cannot reopen a persisted aggregate interval exactly: source={:?} bucket_start={} shares its local day with persisted five-minute bucket {bucket_start} from another transaction", + child.key.source_id, child.key.bucket_start + ))); + } + // This validation is intentionally local to one aggregate transaction/output. Persisted + // siblings cannot change except through keys owned by this run, which are already excluded + // above, so later children in the same source/day can reuse the successful result. + self.persisted_sibling_validations.insert(validation_key); + Ok(()) + } + + fn include(&mut self, child: &CanonicalBucket, timezone: &str) -> Result<(), PipelineError> { + self.include_profiled(child, timezone).map(|_| ()) + } + + fn include_profiled( + &mut self, + child: &CanonicalBucket, + timezone: &str, + ) -> Result { + let total_started = Instant::now(); + let mut profile = AggregateIncludeProfile::default(); + if self + .published_through + .get(&child.key.source_id) + .is_some_and(|previous| child.key.bucket_start <= *previous) + { + return Err(PipelineError::InvalidConfig(format!( + "five-minute buckets must be unique and chronological for source {:?}: {} followed {}", + child.key.source_id, + self.published_through[&child.key.source_id], + child.key.bucket_start + ))); + } + for granularity in [ + Granularity::ThirtyMinutes, + Granularity::OneHour, + Granularity::OneDay, + ] { + let granularity_started = Instant::now(); + let bounds_started = Instant::now(); + let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; + let bounds_elapsed = bounds_started.elapsed(); + let key = (child.key.source_id.clone(), granularity, start, end); + let builder_started = Instant::now(); + let builder = self.builders.entry(key.clone()).or_insert_with(|| { + StatisticalBucket::new(BucketKey::new(&key.0, key.1, key.2, key.3)) + }); + let builder_elapsed = builder_started.elapsed(); + let bucket = builder.include_profiled(child)?; + profile.granularity_mut(granularity).include( + granularity_started.elapsed(), + bounds_elapsed, + builder_elapsed, + bucket, + ); + } + self.published_through + .insert(child.key.source_id.clone(), child.key.bucket_start); + self.current_run_keys + .insert((child.key.source_id.clone(), child.key.bucket_start)); + profile.total_elapsed = total_started.elapsed(); + Ok(profile) + } + + fn flush_complete( + &mut self, + connection: &Connection, + run_maad: bool, + ) -> Result { + self.flush_complete_profiled(connection, run_maad) + .map(|(count, _)| count) + } + + fn flush_complete_profiled( + &mut self, + connection: &Connection, + run_maad: bool, + ) -> Result<(usize, WriteBucketsProfile), PipelineError> { + let complete_keys = self + .builders + .iter() + .filter(|(_, builder)| builder.has_complete_five_minute_coverage()) + .map(|(key, _)| key.clone()) + .collect::>(); + let buckets = complete_keys + .into_iter() + .filter_map(|key| self.builders.remove(&key)) + .map(StatisticalBucket::finish_owned) + .collect::>(); + let count = buckets.len(); + let profile = write_buckets_profiled(connection, &buckets, run_maad)?; + Ok((count, profile)) + } + + fn finish(self) -> (Vec, Vec) { + ( + self.builders + .into_values() + .map(StatisticalBucket::finish_owned) + .collect(), + Vec::new(), + ) + } +} + +fn publish_rollups( + connection: &Connection, + aggregates: AggregateBuckets, + pipeline: &ResolvedPipeline, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + publish_rollups_profiled(connection, aggregates, pipeline, report).map(|_| ()) +} + +fn publish_rollups_profiled( + connection: &Connection, + aggregates: AggregateBuckets, + pipeline: &ResolvedPipeline, + report: &mut PipelineReport, +) -> Result { + let total_started = Instant::now(); + let finish_started = Instant::now(); + let (rollups, incomplete) = aggregates.finish(); + let finish_elapsed = finish_started.elapsed(); + let delete_started = Instant::now(); + delete_stats_bucket_keys(connection, &incomplete)?; + let delete_elapsed = delete_started.elapsed(); + let write = write_buckets_profiled(connection, &rollups, pipeline.run_maad)?; + report.rollup_buckets += rollups.len(); + Ok(FinalRollupProfile { + total_elapsed: total_started.elapsed(), + finish_elapsed, + delete_elapsed, + write, + incomplete_keys: profile_count(incomplete.len()), + rollup_buckets: profile_count(rollups.len()), + }) +} + +/// A repaired five-minute bucket is exact, but persisted coarse unique-count +/// and MAAD rows cannot be patched from scalar results. Keep additive capture +/// coverage current and remove only the affected derived metric rows. +fn refresh_rollups_after_five_minute_repair( + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, +) -> Result<(), PipelineError> { + const DERIVED_TABLES: [&str; 5] = [ + "traffic_stats", + "protocol_stats", + "address_count_stats", + "port_count_stats", + "address_structure_stats", + ]; + + for granularity in [ + Granularity::ThirtyMinutes, + Granularity::OneHour, + Granularity::OneDay, + ] { + let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; + for table in DERIVED_TABLES { + connection + .execute( + &format!( + "DELETE FROM {table} + WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3" + ), + params![child.key.source_id, granularity.as_str(), start], + ) + .map_err(StorageError::from)?; + } + + let children = query_bucket_coverage( + connection, + &child.key.source_id, + Granularity::FiveMinutes.as_str(), + start, + end, + )?; + let expected_children = + usize::try_from((end - start).div_euclid(FIVE_MINUTES)).unwrap_or(usize::MAX); + if children.len() != expected_children { + connection + .execute( + "DELETE FROM bucket_coverage + WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3", + params![child.key.source_id, granularity.as_str(), start], + ) + .map_err(StorageError::from)?; + continue; + } + + let mut coverage = BucketCoverage::empty(); + for row in children { + coverage + .include(row.coverage()?) + .map_err(DomainError::from)?; + } + insert_bucket_coverage_rows( + connection, + &[BucketCoverageRow::new( + &child.key.source_id, + granularity.as_str(), + start, + end, + coverage, + )], + )?; + } + Ok(()) +} + +fn aggregate_bounds( + bucket_start: i64, + granularity: Granularity, + timezone: &str, +) -> Result<(i64, i64), PipelineError> { + let timestamp = Timestamp::from_second(bucket_start) + .map_err(|error| PipelineError::Time(error.to_string()))?; + let zoned = timestamp + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))?; + let start = match granularity { + Granularity::ThirtyMinutes => zoned + .round( + ZonedRound::new() + .smallest(Unit::Minute) + .increment(30) + .mode(RoundMode::Trunc), + ) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::OneHour => zoned + .round( + ZonedRound::new() + .smallest(Unit::Hour) + .mode(RoundMode::Trunc), + ) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::OneDay => zoned + .date() + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::FiveMinutes => { + return Err(PipelineError::InvalidConfig( + "five-minute input is not a rollup granularity".into(), + )); + } + }; + let end = match granularity { + Granularity::ThirtyMinutes => start + 1_800, + Granularity::OneHour => start + 3_600, + Granularity::OneDay => zoned + .date() + .tomorrow() + .and_then(|date| date.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::FiveMinutes => unreachable!("rejected above"), + }; + Ok((start, end)) +} + +fn next_local_five_minute_start(bucket_start: i64, timezone: &str) -> Result { + let current = Timestamp::from_second(bucket_start) + .and_then(|timestamp| timestamp.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))?; + let next = current + .datetime() + .checked_add(5.minutes()) + .and_then(|datetime| datetime.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(); + if next <= bucket_start { + return Err(PipelineError::Time(format!( + "local five-minute clock did not advance after {bucket_start} in {timezone:?}" + ))); + } + Ok(next) +} + +#[derive(Clone, Copy, Debug)] +struct NfcapdTreeWindow { + start: i64, + end: i64, +} + +/// Resolve the selected and requested nfcapd window using the same date, timezone, and alignment +/// rules for preflight, single-output processing, and coordinated planning. +fn resolve_nfcapd_tree_window( + start_date: &str, + end_date: Option<&str>, + start_time: Option<&str>, + end_time: Option<&str>, + discovered_bucket_starts: impl IntoIterator, + timezone: &str, +) -> Result { + let selected_start = parse_date_start(start_date, timezone)?; + let explicit_end = end_date + .map(|date| next_date_start(date, timezone)) + .transpose()?; + let explicit_start_time = start_time + .map(|value| parse_local_datetime(value, timezone)) + .transpose()?; + let explicit_end_time = end_time + .map(|value| parse_local_datetime(value, timezone)) + .transpose()?; + let discovered_end = discovered_bucket_starts + .into_iter() + .max() + .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) + .transpose()? + .map(|(_, end)| end) + .unwrap_or(selected_start); + let selected_end = explicit_end.unwrap_or(discovered_end); + let start = explicit_start_time.unwrap_or(selected_start); + let end = explicit_end_time.unwrap_or(selected_end); + validate_window(selected_start, selected_end, start, end, timezone)?; + Ok(NfcapdTreeWindow { start, end }) +} + +fn parse_date_start(raw: &str, timezone: &str) -> Result { + let date: Date = raw + .parse() + .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; + Ok(date + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second()) +} + +fn next_date_start(raw: &str, timezone: &str) -> Result { + let date: Date = raw + .parse() + .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; + Ok(date + .tomorrow() + .and_then(|date| date.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second()) +} + +fn parse_local_datetime(raw: &str, timezone: &str) -> Result { + let normalized = if raw.len() == 16 { + format!("{raw}:00") + } else { + raw.to_owned() + }; + let datetime = normalized + .parse::() + .map_err(|error| PipelineError::Time(error.to_string()))?; + Ok(datetime + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second()) +} + +fn validate_window( + selected_start: i64, + selected_end: i64, + start: i64, + end: i64, + timezone: &str, +) -> Result<(), PipelineError> { + if start < selected_start { + return Err(PipelineError::InvalidConfig( + "start_time must be on or after the selected start_date".into(), + )); + } + if end > selected_end { + return Err(PipelineError::InvalidConfig( + "end_time must be on or before the selected end_date window".into(), + )); + } + if start >= end { + return Err(PipelineError::InvalidConfig( + "input time window must be non-empty".into(), + )); + } + for (label, value) in [("start_time", start), ("end_time", end)] { + if aggregate_bounds(value, Granularity::OneDay, timezone)?.0 != value { + return Err(PipelineError::InvalidConfig(format!( + "{label} must align to a local-day boundary so aggregate rows stay complete" + ))); + } + } + Ok(()) +} + +fn expected_nfcapd_path( + root: &Path, + member: &str, + bucket_start: i64, + timezone: &str, +) -> Result { + let timestamp = Timestamp::from_second(bucket_start) + .and_then(|timestamp| timestamp.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))?; + Ok(root + .join(member) + .join(timestamp.strftime("%Y").to_string()) + .join(timestamp.strftime("%m").to_string()) + .join(timestamp.strftime("%d").to_string()) + .join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M")))) +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + net::{IpAddr, Ipv4Addr}, + }; + + use rusqlite::{Connection, types::ValueRef}; + use serde_json::json; + use tempfile::tempdir; + + use super::*; + use crate::{ + coverage::CoverageState, + domain::{AddressSide, FlowObservation, IpVersion, Scope, Visibility}, + storage::database_operation_lock_path, + }; + + fn write_fake_nfdump(executable: &Path, setup: &str) { + let stream = executable.with_extension("stream"); + let empty_stream = executable.with_extension("empty.stream"); + fs::write(&stream, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); + fs::write( + &empty_stream, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + executable, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\n{setup}\ncat '{}'\n", + empty_stream.display(), + stream.display() + ), + ) + .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); + } + } + + #[cfg(unix)] + fn write_nfcapd_day(root: &Path) { + write_nfcapd_day_for_date(root, "2025-06-01"); + } + + #[cfg(unix)] + fn write_nfcapd_day_for_date(root: &Path, date: &str) { + let date_path = date.replace('-', "/"); + let day = root.join(format!("edge/{date_path}")); + fs::create_dir_all(&day).unwrap(); + let day_start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + } + + #[cfg(unix)] + fn replace_member_directory(root: &Path) -> PathBuf { + let member = root.join("edge"); + let previous = root.join("edge-before-replacement"); + fs::rename(&member, &previous).unwrap(); + fs::create_dir(&member).unwrap(); + previous + } + + #[cfg(unix)] + fn repeated_dataset_request( + temporary: &tempfile::TempDir, + nfdump: PathBuf, + ) -> (PipelineRequest, PathBuf, PathBuf, PathBuf, PathBuf) { + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output_directory = temporary.path().join("outputs"); + let first_database = output_directory.join("first.sqlite"); + let second_database = output_directory.join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + let sentinel = temporary.path().join("sentinel.txt"); + fs::write(&sentinel, b"leave this alone").unwrap(); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": temporary.path().join("captures"), + "db_path": second_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + ( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + output_directory, + first_database, + second_database, + sentinel, + ) + } + + #[test] + fn daily_active_sources_rejects_inputs_without_a_tree_day_cohort() { + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16" + }))) + .unwrap(); + let csv = InputSpec::Csv { + path: "flows.csv".into(), + mapping_path: "mapping.json".into(), + }; + let explicit = InputSpec::Nfcapd { + path: "nfcapd.202501010000".into(), + source_id: "edge".into(), + bucket_start: None, + gap: false, + expected_path: None, + }; + + assert!(validate_selection_inputs(&selection, &[csv]).is_err()); + assert!(validate_selection_inputs(&selection, &[explicit]).is_err()); + assert!( + validate_selection_inputs( + &selection, + &[ + InputSpec::NfcapdTree { + root_path: "captures".into(), + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-01-01".into(), + end_date: Some("2025-01-01".into()), + start_time: None, + end_time: None, + force: false, + }, + InputSpec::CsvTree { + root_path: "csv".into(), + mapping_path: "mapping.json".into(), + }, + ], + ) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn auto_discovered_nfcapd_root_rejects_output_that_would_create_a_member_directory() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir(&root).unwrap(); + let database = root.join("new-member/netflow.sqlite"); + let config = temporary.path().join("pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&json!({ + "database_path": database, + "timezone": DEFAULT_TIMEZONE, + "inputs": [{ + "input_kind": "nfcapd_tree", + "root_path": root, + "start_date": "2025-06-01", + "end_date": "2025-06-01" + }] + })) + .unwrap(), + ) + .unwrap(); + + let error = run(PipelineRequest::config(&config)).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("direct-child directory"), "{message}"); + assert!(!database.parent().unwrap().exists()); + } + + #[test] + fn dataset_mode_applies_its_persisted_selection() { + let temporary = tempdir().unwrap(); + let registry = temporary.path().join("datasets.json"); + let database = temporary.path().join("active.sqlite"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": temporary.path(), + "db_path": database, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "72.5.0.0/16" + } + }])) + .unwrap(), + ) + .unwrap(); + let resolved = resolve_request(&PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: "nfdump".into(), + force: false, + run_maad: true, + require_complete: false, + }) + .unwrap(); + + assert!(resolved.selection.selects_daily_active_sources()); + assert_eq!(resolved.database_path, database); + } + + #[cfg(unix)] + #[test] + fn nfcapd_member_aliases_are_rejected_before_output_mutation() { + use std::os::unix::fs::symlink; + + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let real_member = root.join("real-member"); + fs::create_dir_all(&real_member).unwrap(); + let capture = real_member.join("2025/06/01/nfcapd.202506010000"); + fs::create_dir_all(capture.parent().unwrap()).unwrap(); + fs::write(&capture, b"capture bytes").unwrap(); + symlink("real-member", root.join("edge-a")).unwrap(); + symlink("real-member", root.join("edge-b")).unwrap(); + let output = temporary.path().join("output.sqlite"); + + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root, + source_ids: vec!["edge-a".into(), "edge-b".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let before = fs::read(&capture).unwrap(); + let error = execute(pipeline).unwrap_err(); + + assert!(error.to_string().contains("same directory")); + assert_eq!(fs::read(capture).unwrap(), before); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[cfg(unix)] + #[test] + fn repeated_dataset_missing_explicit_nfdump_is_side_effect_free() { + let temporary = tempdir().unwrap(); + let missing = temporary.path().join("tools/missing-nfdump"); + let (request, output_directory, first_database, second_database, sentinel) = + repeated_dataset_request(&temporary, missing); + let sentinel_before = fs::read(&sentinel).unwrap(); + + let error = run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); + + let message = error.to_string(); + assert!(message.contains("explicit nfdump executable"), "{message}"); + assert!(message.contains("missing-nfdump"), "{message}"); + assert!(!output_directory.exists()); + for database in [&first_database, &second_database] { + assert!(!database.exists()); + assert!(!database_operation_lock_path(database).unwrap().exists()); + } + assert_eq!(fs::read(sentinel).unwrap(), sentinel_before); + } + + #[cfg(unix)] + #[test] + fn repeated_dataset_non_executable_explicit_nfdump_is_side_effect_free() { + use std::os::unix::fs::PermissionsExt; + + let temporary = tempdir().unwrap(); + let non_executable = temporary.path().join("nfdump-not-executable"); + fs::write(&non_executable, b"#!/bin/sh\n").unwrap(); + fs::set_permissions(&non_executable, fs::Permissions::from_mode(0o644)).unwrap(); + let (request, output_directory, first_database, second_database, sentinel) = + repeated_dataset_request(&temporary, non_executable); + let sentinel_before = fs::read(&sentinel).unwrap(); + + let error = run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); + + let message = error.to_string(); + assert!( + message.contains("not executable by this process"), + "{message}" + ); + assert!(!output_directory.exists()); + for database in [&first_database, &second_database] { + assert!(!database.exists()); + assert!(!database_operation_lock_path(database).unwrap().exists()); + } + assert_eq!(fs::read(sentinel).unwrap(), sentinel_before); + } + + #[cfg(unix)] + #[test] + fn coordinated_mode_loads_one_registry_snapshot_for_all_datasets() { + let temporary = tempdir().unwrap(); + let first_root = temporary.path().join("first-captures"); + let second_root = temporary.path().join("second-captures"); + fs::create_dir_all(first_root.join("edge")).unwrap(); + fs::create_dir_all(second_root.join("edge")).unwrap(); + let first_database = temporary.path().join("first.sqlite"); + let second_database = temporary.path().join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": first_root, + "db_path": first_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": second_root, + "db_path": second_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + let nfdump = temporary.path().join("nfdump"); + write_fake_nfdump(&nfdump, ""); + let request = PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }; + + reset_dataset_registry_load_calls(); + let error = run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); + + assert!(error.to_string().contains("same nfcapd root"), "{error}"); + assert_eq!(dataset_registry_load_calls(), 1); + } + + #[test] + fn coordinated_mode_rejects_duplicate_and_incompatible_datasets() { + let temporary = tempdir().unwrap(); + let first_root = temporary.path().join("first"); + let second_root = temporary.path().join("second"); + fs::create_dir_all(first_root.join("edge")).unwrap(); + fs::create_dir_all(second_root.join("edge")).unwrap(); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": first_root, + "db_path": temporary.path().join("first.sqlite"), + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "72.5.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": second_root, + "db_path": temporary.path().join("second.sqlite"), + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "72.6.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + let request = PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: "nfdump".into(), + force: false, + run_maad: true, + require_complete: false, + }; + + assert!(run_many(request.clone(), vec!["first".into(), "first".into()]).is_err()); + assert!(run_many(request, vec!["first".into(), "second".into()]).is_err()); + } + + fn empty_coordinated_pipeline(database_path: PathBuf) -> ResolvedPipeline { + ResolvedPipeline { + database_path, + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: Vec::new(), + datasets: Vec::new(), + require_complete: false, + } + } + + fn semantic_table_rows(connection: &Connection, table: &str) -> Vec> { + let mut columns = connection + .prepare(&format!("PRAGMA table_info({table})")) + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + columns.sort_unstable_by_key(|(index, _)| *index); + let semantic = columns + .into_iter() + .filter(|(_, column)| { + !matches!( + column.as_str(), + "bound_at" | "discovered_at" | "processed_at" + ) + }) + .collect::>(); + let selected = semantic + .iter() + .map(|(_, column)| column.as_str()) + .collect::>() + .join(", "); + let order = semantic + .iter() + .map(|(_, column)| column.as_str()) + .collect::>() + .join(", "); + let mut statement = connection + .prepare(&format!("SELECT {selected} FROM {table} ORDER BY {order}")) + .unwrap(); + statement + .query_map([], |row| { + (0..semantic.len()) + .map(|index| match row.get_ref(index)? { + ValueRef::Null => Ok("NULL".into()), + ValueRef::Integer(value) => Ok(value.to_string()), + ValueRef::Real(value) => Ok(value.to_string()), + ValueRef::Text(value) => Ok(String::from_utf8_lossy(value).into_owned()), + ValueRef::Blob(value) => Ok(value + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::()), + }) + .collect() + }) + .unwrap() + .collect::>>() + .unwrap() + } + + fn coordinated_semantic_snapshot(connection: &Connection) -> Vec>> { + [ + "pipeline_product", + "nfcapd_source_layout", + "datasets", + "source_members", + "bucket_coverage", + "traffic_stats", + "protocol_stats", + "address_count_stats", + "port_count_stats", + "address_structure_stats", + "processed_inputs", + "input_evidence", + ] + .into_iter() + .map(|table| semantic_table_rows(connection, table)) + .collect() + } + + #[test] + fn coordinated_output_aliases_are_rejected_before_filesystem_mutation() { + let temporary = tempdir().unwrap(); + let target = temporary.path().join("target.sqlite"); + let operation_lock = database_operation_lock_path(&target).unwrap(); + for alias in [ + target.with_file_name("target.sqlite-wal"), + operation_lock.clone(), + ] { + let error = execute_many(vec![ + empty_coordinated_pipeline(target.clone()), + empty_coordinated_pipeline(alias), + ]) + .unwrap_err(); + assert!(error.to_string().contains("must be distinct")); + assert!(!target.exists()); + assert!(!operation_lock.exists()); + } + + let normalized_parent = temporary.path().join("normalized"); + let normalized = normalized_parent.join("..").join("normalized.sqlite"); + let normalized_target = temporary.path().join("normalized.sqlite"); + let error = execute_many(vec![ + empty_coordinated_pipeline(normalized_target.clone()), + empty_coordinated_pipeline(normalized), + ]) + .unwrap_err(); + assert!(error.to_string().contains("must be distinct")); + assert!(!normalized_parent.exists()); + assert!(!normalized_target.exists()); + + let target = temporary.path().join("nested-target.sqlite"); + let target_related = [ + target.clone(), + target.with_file_name("nested-target.sqlite-wal"), + database_operation_lock_path(&target).unwrap(), + ]; + for ancestor in target_related { + let descendant = ancestor.join("second.sqlite"); + let error = execute_many(vec![ + empty_coordinated_pipeline(target.clone()), + empty_coordinated_pipeline(descendant.clone()), + ]) + .unwrap_err(); + assert!(error.to_string().contains("must be distinct"), "{error}"); + assert!(!target.exists()); + assert!(!descendant.exists()); + assert!(!ancestor.exists()); + } + } + + #[cfg(unix)] + #[test] + fn coordinated_output_symlink_and_hard_link_aliases_are_rejected_before_mutation() { + use std::{fs::hard_link, os::unix::fs::symlink}; + + let temporary = tempdir().unwrap(); + let target = temporary.path().join("target.sqlite"); + fs::write(&target, b"existing database bytes").unwrap(); + let target_lock = database_operation_lock_path(&target).unwrap(); + + let symlink_alias = temporary.path().join("symlink.sqlite"); + symlink(&target, &symlink_alias).unwrap(); + let error = execute_many(vec![ + empty_coordinated_pipeline(target.clone()), + empty_coordinated_pipeline(symlink_alias.clone()), + ]) + .unwrap_err(); + assert!(error.to_string().contains("must be distinct")); + assert_eq!(fs::read(&target).unwrap(), b"existing database bytes"); + assert!(!target_lock.exists()); + assert!(symlink_alias.is_symlink()); + + let hard_link_alias = temporary.path().join("hard-link.sqlite"); + hard_link(&target, &hard_link_alias).unwrap(); + let error = execute_many(vec![ + empty_coordinated_pipeline(target.clone()), + empty_coordinated_pipeline(hard_link_alias.clone()), + ]) + .unwrap_err(); + assert!(error.to_string().contains("must be distinct")); + assert_eq!(fs::read(&target).unwrap(), b"existing database bytes"); + assert_eq!( + fs::read(&hard_link_alias).unwrap(), + b"existing database bytes" + ); + assert!(!target_lock.exists()); + } + + #[cfg(unix)] + #[test] + fn output_capture_aliases_are_rejected_without_mutating_capture_bytes() { + use std::{fs::hard_link, os::unix::fs::symlink}; + + let temporary = tempdir().unwrap(); + let capture = temporary.path().join("nfcapd.202506010000"); + fs::write(&capture, b"capture bytes").unwrap(); + let output = temporary.path().join("output.sqlite"); + let sidecar = output.with_file_name("output.sqlite-wal"); + symlink(&capture, &sidecar).unwrap(); + + for alias in [capture.clone(), sidecar.clone()] { + let error = + validate_output_capture_separation(&[&alias], std::iter::once(capture.as_path())) + .unwrap_err(); + assert!( + error + .to_string() + .contains("aliases discovered nfcapd capture") + ); + assert_eq!(fs::read(&capture).unwrap(), b"capture bytes"); + } + + let hard_link_path = temporary.path().join("hard-link.sqlite"); + hard_link(&capture, &hard_link_path).unwrap(); + let error = validate_output_capture_separation( + &[&hard_link_path], + std::iter::once(capture.as_path()), + ) + .unwrap_err(); + assert!(error.to_string().contains("device/inode")); + assert_eq!(fs::read(&capture).unwrap(), b"capture bytes"); + } + + #[test] + fn output_separation_streams_input_paths_until_a_conflict() { + struct OneInputThenPanic<'a> { + input: Option<&'a Path>, + } + + impl<'a> Iterator for OneInputThenPanic<'a> { + type Item = &'a Path; + + fn next(&mut self) -> Option { + self.input + .take() + .or_else(|| panic!("input separation collected the entire iterator")) + } + } + + let temporary = tempdir().unwrap(); + let capture = temporary.path().join("nfcapd.202506010000"); + fs::write(&capture, b"capture bytes").unwrap(); + let error = validate_output_capture_separation( + &[&capture], + OneInputThenPanic { + input: Some(capture.as_path()), + }, + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("aliases discovered nfcapd capture") + ); + } + + #[test] + fn nfcapd_tree_output_validation_skips_capture_metadata_for_new_outputs() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let namespaces = nfcapd_locator_namespaces(&root, &["edge".into()]).unwrap(); + let output = temporary.path().join("pipeline.sqlite"); + let captures = (0..4_096) + .map(|index| { + root.join(format!( + "edge/2025/06/01/nfcapd.202506{:06}", + index % 100_000 + )) + }) + .collect::>(); + + reset_nfcapd_capture_identity_calls(); + validate_output_nfcapd_capture_separation( + &[&output], + &namespaces, + "UTC", + captures.iter().map(PathBuf::as_path), + ) + .unwrap(); + assert_eq!( + nfcapd_capture_identity_calls(), + 0, + "a new output has no inode that can require a physical capture scan" + ); + } + + #[cfg(unix)] + #[test] + fn nfcapd_tree_output_validation_scans_capture_inodes_only_for_existing_output() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let namespaces = nfcapd_locator_namespaces(&root, &["edge".into()]).unwrap(); + let output = temporary.path().join("pipeline.sqlite"); + fs::write(&output, b"existing output").unwrap(); + let captures = (0..64) + .map(|index| { + let path = temporary.path().join(format!("capture-{index}.nfcapd")); + fs::write(&path, format!("capture-{index}")).unwrap(); + path + }) + .collect::>(); + + reset_nfcapd_capture_identity_calls(); + validate_output_nfcapd_capture_separation( + &[&output], + &namespaces, + "UTC", + captures.iter().map(PathBuf::as_path), + ) + .unwrap(); + assert_eq!(nfcapd_capture_identity_calls(), captures.len()); + + let hard_link_output = temporary.path().join("hard-link.sqlite"); + fs::hard_link(&captures[0], &hard_link_output).unwrap(); + reset_nfcapd_capture_identity_calls(); + let error = validate_output_nfcapd_capture_separation( + &[&hard_link_output], + &namespaces, + "UTC", + captures.iter().map(PathBuf::as_path), + ) + .unwrap_err(); + assert!(error.to_string().contains("device/inode")); + assert_eq!(nfcapd_capture_identity_calls(), 1); + } + + #[cfg(unix)] + #[test] + fn existing_output_reuses_one_bounded_capture_snapshot_for_alias_and_revision_work() { + let temporary = tempdir().unwrap(); + let output = temporary.path().join("pipeline.sqlite"); + fs::write(&output, b"existing output").unwrap(); + let captures = (0..4_096) + .map(|index| { + let path = temporary.path().join(format!("capture-{index}.nfcapd")); + fs::write(&path, format!("capture-{index}")).unwrap(); + path + }) + .collect::>(); + + reset_nfcapd_capture_identity_calls(); + let snapshot_calls = AtomicUsize::new(0); + let snapshots = capture_nfcapd_snapshots_counted(&captures, &snapshot_calls).unwrap(); + assert_eq!(snapshot_calls.load(Ordering::Relaxed), captures.len()); + validate_output_nfcapd_capture_separation_with_snapshots( + &[&output], + &[], + snapshots + .iter() + .map(|(path, snapshot)| (path.as_path(), snapshot)), + ) + .unwrap(); + assert_eq!( + nfcapd_capture_identity_calls(), + 0, + "snapshot-backed alias validation must not run a second serial metadata pass" + ); + + let capture = captures.iter().next().unwrap().clone(); + let sources = [DatasetSource { + source_id: "r1".into(), + members: vec!["r1".into()], + }]; + let paths = BTreeMap::from([(("r1".into(), 0), capture.clone())]); + let bounds = BTreeMap::from([("r1".into(), (0, 0))]); + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .unwrap(); + let context = NfcapdRevisionContext { + connection: &connection, + sources: &sources, + by_member_and_start: &paths, + member_bounds: &bounds, + extend_gaps_to_window: false, + force: false, + decoder_fingerprint: "decoder".into(), + capture_snapshots: &snapshots, + revision_pool: &pool, + }; + resolve_nfcapd_batch_revisions(&context, &[0]).unwrap(); + + let hard_link_output = temporary.path().join("hard-link.sqlite"); + fs::hard_link(&capture, &hard_link_output).unwrap(); + let error = validate_output_nfcapd_capture_separation_with_snapshots( + &[&hard_link_output], + &[], + snapshots + .iter() + .map(|(path, snapshot)| (path.as_path(), snapshot)), + ) + .unwrap_err(); + assert!(error.to_string().contains("device/inode"), "{error}"); + fs::remove_file(hard_link_output).unwrap(); + + fs::write(&capture, b"capture changed").unwrap(); + let error = resolve_nfcapd_batch_revisions(&context, &[0]).unwrap_err(); + assert!( + error.to_string().contains("changed while"), + "revision preparation must reuse the original snapshot: {error}" + ); + } + + #[test] + fn auto_discovered_tree_rejects_a_new_member_locator_before_mutation() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(&root).unwrap(); + let output = root.join("new-member/2025/06/01/nfcapd.202506010000"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: Vec::new(), + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = execute(pipeline).unwrap_err(); + assert!( + error + .to_string() + .contains("auto-discovered member namespace") + ); + assert!(!output.exists()); + assert!(!root.join("new-member").exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn configured_nfcapd_member_rejects_output_in_a_year_directory() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output = root.join("edge/2025"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = execute(pipeline).unwrap_err(); + let message = error.to_string(); + assert!( + message.contains("configured nfcapd member namespace"), + "{message}" + ); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn auto_discovered_tree_rejects_a_future_member_directory() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(&root).unwrap(); + let output = root.join("future-member"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: Vec::new(), + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = execute(pipeline).unwrap_err(); + let message = error.to_string(); + assert!( + message.contains("auto-discovered member namespace"), + "{message}" + ); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[cfg(unix)] + #[test] + fn nfcapd_namespace_aliases_are_rejected_before_output_mutation() { + use std::os::unix::fs::symlink; + + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let configured_target = root.join("edge/2025"); + let configured_alias = temporary.path().join("configured-alias.sqlite"); + symlink(&configured_target, &configured_alias).unwrap(); + + let configured_pipeline = ResolvedPipeline { + database_path: configured_alias.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + let error = execute(configured_pipeline).unwrap_err(); + assert!( + error + .to_string() + .contains("configured nfcapd member namespace") + ); + assert!(configured_alias.is_symlink()); + assert!( + !database_operation_lock_path(&configured_alias) + .unwrap() + .exists() + ); + + let auto_target = root.join("future-member"); + let auto_alias = temporary.path().join("auto-alias.sqlite"); + symlink(&auto_target, &auto_alias).unwrap(); + let auto_pipeline = ResolvedPipeline { + database_path: auto_alias.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: Vec::new(), + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + let error = execute(auto_pipeline).unwrap_err(); + assert!( + error + .to_string() + .contains("auto-discovered member namespace") + ); + assert!(auto_alias.is_symlink()); + assert!(!database_operation_lock_path(&auto_alias).unwrap().exists()); + } + + #[cfg(unix)] + #[test] + fn csv_and_mapping_aliases_are_rejected_before_mutation() { + use std::{fs::hard_link, os::unix::fs::symlink}; + + let temporary = tempdir().unwrap(); + let input = temporary.path().join("flows.csv"); + let mapping = temporary.path().join("mapping.json"); + fs::write(&input, b"CSV input bytes").unwrap(); + fs::write(&mapping, b"mapping bytes").unwrap(); + + let mut aliases = vec![input.clone(), mapping.clone()]; + for (index, target) in [(&input, "input"), (&mapping, "mapping")] { + let symlink_alias = temporary.path().join(format!("{target}-symlink.sqlite")); + symlink(index, &symlink_alias).unwrap(); + aliases.push(symlink_alias); + let hard_link_alias = temporary.path().join(format!("{target}-hard-link.sqlite")); + hard_link(index, &hard_link_alias).unwrap(); + aliases.push(hard_link_alias); + } + + for output in aliases { + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::Csv { + path: input.clone(), + mapping_path: mapping.clone(), + }], + datasets: Vec::new(), + require_complete: false, + }; + let error = execute(pipeline).unwrap_err(); + assert!(error.to_string().contains("aliases discovered CSV input")); + assert_eq!(fs::read(&input).unwrap(), b"CSV input bytes"); + assert_eq!(fs::read(&mapping).unwrap(), b"mapping bytes"); + for suffix in ["-journal", "-wal", "-shm"] { + assert!( + !output + .with_file_name(format!( + "{}{}", + output.file_name().unwrap().to_string_lossy(), + suffix + )) + .exists() + ); + } + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + } + + #[test] + fn csv_tree_rejects_an_output_that_would_be_discovered_after_creation() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("csv-tree"); + fs::create_dir_all(&root).unwrap(); + let mapping = temporary.path().join("mapping.json"); + fs::write( + &mapping, + serde_json::to_vec(&json!({ + "has_header": true, + "timestamp_format": "datetime", + "timestamp_timezone": "UTC", + "columns": { + "time_end": "time", + "src_ip": "src", + "dst_ip": "dst" + }, + "source_id": {"value": "r1"}, + "discovery": {"include_suffixes": [".csv"]} + })) + .unwrap(), + ) + .unwrap(); + let output = root.join("new-output.csv"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::CsvTree { + root_path: root.clone(), + mapping_path: mapping.clone(), + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = execute(pipeline).unwrap_err(); + assert!( + error + .to_string() + .contains("would be discovered as a CSV tree input") + ); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + assert_eq!( + fs::read(&mapping).unwrap(), + serde_json::to_vec(&json!({ + "has_header": true, + "timestamp_format": "datetime", + "timestamp_timezone": "UTC", + "columns": { + "time_end": "time", + "src_ip": "src", + "dst_ip": "dst" + }, + "source_id": {"value": "r1"}, + "discovery": {"include_suffixes": [".csv"]} + })) + .unwrap() + ); + } + + #[test] + fn single_daily_active_preflight_rejects_an_absent_expected_capture_alias() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output = root.join("edge/2025/06/01/nfcapd.202506010000"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + }))) + .unwrap(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root, + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = preflight_single_output(&pipeline).unwrap_err(); + assert!( + error + .to_string() + .contains("aliases discovered nfcapd capture") + ); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn single_tree_preflight_rejects_missing_locator_in_finite_window() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output = root.join("edge/2025/06/01/nfcapd.202506010000"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root, + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = preflight_single_output(&pipeline).unwrap_err(); + assert!(error.to_string().contains("nfcapd capture locator")); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn single_tree_preflight_rejects_open_ended_future_successor_locator() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output = root.join("edge/2025/06/02/nfcapd.202506020000"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root, + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: None, + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = preflight_single_output(&pipeline).unwrap_err(); + assert!(error.to_string().contains("nfcapd capture locator")); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn explicit_gap_expected_locator_is_checked_before_output_setup() { + let temporary = tempdir().unwrap(); + let expected_path = temporary + .path() + .join("captures/edge/2025/06/01/nfcapd.202506010000"); + let output = expected_path.clone(); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::Nfcapd { + path: temporary.path().join("gap-marker"), + source_id: "edge".into(), + bucket_start: Some(1_735_689_600), + gap: true, + expected_path: Some(expected_path), + }], + datasets: Vec::new(), + require_complete: false, + }; + + let error = preflight_single_output(&pipeline).unwrap_err(); + assert!( + error + .to_string() + .contains("aliases discovered nfcapd capture") + ); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn one_year_ten_member_preflight_does_not_materialize_capture_paths() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let members = (0..10).map(|index| format!("edge-{index}")); + for member in members.clone() { + fs::create_dir_all(root.join(member)).unwrap(); + } + let output = temporary.path().join("pipeline.sqlite"); + let pipeline = ResolvedPipeline { + database_path: output.clone(), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + }))) + .unwrap(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root, + source_ids: members.collect(), + sources: Vec::new(), + start_date: "2025-01-01".into(), + end_date: Some("2026-01-01".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }; + + preflight_single_output(&pipeline).unwrap(); + assert!(!output.exists()); + assert!(!database_operation_lock_path(&output).unwrap().exists()); + } + + #[test] + fn coordinated_metadata_conflict_rolls_back_earlier_output_initialization() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let first_database = temporary.path().join("first.sqlite"); + let second_database = temporary.path().join("second.sqlite"); + let selection_a = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + }))) + .unwrap(); + let selection_b = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16" + }))) + .unwrap(); + let pipeline_for = + |database_path: PathBuf, dataset_id: &str, label: &str, selection: FlowSelection| { + let dataset = Dataset { + dataset_id: dataset_id.into(), + label: label.into(), + root_path: root.clone(), + db_path: database_path.clone(), + default_start_date: String::new(), + source_mode: "static".into(), + discovery_mode: "static".into(), + sort_order: 0, + source_ids: vec!["edge".into()], + sources: Vec::new(), + selection: selection.normalized_payload(), + }; + ResolvedPipeline { + database_path, + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection, + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "1970-01-01".into(), + end_date: Some("1970-01-01".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: vec![dataset], + require_complete: false, + } + }; + let first_existing = pipeline_for( + first_database.clone(), + "first", + "before", + selection_a.clone(), + ); + let second_existing = pipeline_for( + second_database.clone(), + "second", + "second", + selection_a.clone(), + ); + for pipeline in [&first_existing, &second_existing] { + let lock = DatabaseOperationLock::acquire(&pipeline.database_path, "test").unwrap(); + let connection = connect_pipeline_writer(&pipeline.database_path).unwrap(); + init_schema(&connection).unwrap(); + initialize_metadata(&connection, pipeline).unwrap(); + drop(connection); + drop(lock); + } + let before = coordinated_semantic_snapshot(&Connection::open(&first_database).unwrap()); + + let first_requested = pipeline_for(first_database.clone(), "first", "after", selection_a); + let second_requested = + pipeline_for(second_database.clone(), "second", "second", selection_b); + let error = execute_many(vec![first_requested, second_requested]).unwrap_err(); + assert!(error.to_string().contains("second")); + assert!( + error + .to_string() + .contains(second_database.to_string_lossy().as_ref()) + ); + assert_eq!( + coordinated_semantic_snapshot(&Connection::open(first_database).unwrap()), + before + ); + } + + #[test] + fn coordinated_invalid_finite_windows_leave_output_paths_uncreated() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output_directory = temporary.path().join("outputs"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": output_directory.join("first.sqlite"), + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": output_directory.join("second.sqlite"), + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + for (start_date, end_date) in [("2025-99-01", "2025-10-01"), ("2025-10-02", "2025-10-01")] { + let error = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry.clone()), + start_date: Some(start_date.into()), + end_date: Some(end_date.into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: "./missing-nfdump".into(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["first".into(), "second".into()], + ) + .unwrap_err(); + assert!(matches!( + error, + PipelineError::Time(_) | PipelineError::InvalidConfig(_) + )); + assert!(!output_directory.exists()); + } + } + + #[test] + fn coordinated_auto_discovery_protects_root_regardless_of_dataset_order() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let configured_database = temporary.path().join("configured.sqlite"); + let auto_database = root.join("new-member/netflow.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "configured", + "root_path": root, + "db_path": configured_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "auto", + "root_path": root, + "db_path": auto_database, + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + for order in [ + vec!["configured".to_owned(), "auto".to_owned()], + vec!["auto".to_owned(), "configured".to_owned()], + ] { + let error = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + order, + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("auto-discovered member namespace") + ); + assert!(!configured_database.exists()); + assert!(!auto_database.exists()); + assert!(!auto_database.parent().unwrap().exists()); + assert!( + !database_operation_lock_path(&configured_database) + .unwrap() + .exists() + ); + assert!( + !database_operation_lock_path(&auto_database) + .unwrap() + .exists() + ); + } + } + + #[cfg(unix)] + #[test] + fn coordinated_auto_layout_change_after_planning_is_side_effect_free_in_both_orders() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output_directory = temporary.path().join("outputs"); + let first_database = output_directory.join("first.sqlite"); + let second_database = output_directory.join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "configured", + "root_path": root, + "db_path": first_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "auto", + "root_path": temporary.path().join("captures"), + "db_path": second_database, + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + let request = || PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }; + + for order in [ + vec!["configured".to_owned(), "auto".to_owned()], + vec!["auto".to_owned(), "configured".to_owned()], + ] { + let added_member = root.join("edge-b"); + set_coordinated_plan_hook(move |planned_root| { + fs::create_dir_all(planned_root.join("edge-b")).unwrap(); + }); + let error = run_many(request(), order).unwrap_err(); + clear_coordinated_plan_hook(); + + assert!( + error + .to_string() + .contains("auto-discovered source layout changed"), + "{error}" + ); + assert!(!output_directory.exists()); + for database in [&first_database, &second_database] { + assert!(!database.exists()); + assert!(!database_operation_lock_path(database).unwrap().exists()); + } + fs::remove_dir(added_member).unwrap(); + } + } + + #[test] + fn coordinated_revision_resolution_reuses_a_digest_from_one_output() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let member = root.join("edge"); + fs::create_dir_all(&member).unwrap(); + let capture = member.join("nfcapd.197001010000"); + fs::write(&capture, b"capture").unwrap(); + let sources = vec![DatasetSource { + source_id: "edge".into(), + members: vec!["edge".into()], + }]; + let mut paths = BTreeMap::new(); + paths.insert(("edge".into(), 0), capture.clone()); + let mut outputs = Vec::new(); + for name in ["first.sqlite", "second.sqlite"] { + let database_path = temporary.path().join(name); + let lock = DatabaseOperationLock::acquire(&database_path, "test").unwrap(); + let connection = connect_pipeline_writer(&database_path).unwrap(); + init_schema(&connection).unwrap(); + outputs.push(CoordinatedOutput { + pipeline: ResolvedPipeline { + database_path, + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "1970-01-01".into(), + end_date: Some("1970-01-01".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + }, + sources: sources.clone(), + connection, + _lock: lock, + }); + } + + let observed = FileSnapshot::capture(&capture).unwrap(); + let cached_revision = InputRevision::create( + "nfcapd", + capture.to_string_lossy(), + "cached-content", + "decoder", + ) + .unwrap(); + upsert_input_bucket( + &outputs[0].connection, + &InputBucket { + input_kind: InputKind::Nfcapd, + input_locator: cached_revision.locator.clone(), + scan_locator: cached_revision.locator.clone(), + source_id: "edge".into(), + bucket_start: 0, + bucket_end: FIVE_MINUTES, + revision: cached_revision.clone(), + file_snapshot: Some(observed), + }, + false, + ) + .unwrap(); + mark_input_bucket_status( + &outputs[0].connection, + InputKind::Nfcapd, + &cached_revision.locator, + "edge", + 0, + InputStatus::Processed, + &cached_revision, + None, + ) + .unwrap(); + + let revision_pool = build_revision_hash_pool().unwrap(); + let revisions = resolve_coordinated_batch_revisions( + &outputs, + &sources, + &paths, + &BTreeMap::from([("edge".into(), (0, 0))]), + false, + false, + &revision_pool, + &[0], + ) + .unwrap(); + assert_eq!(revisions.len(), 1); + assert_eq!( + revisions[&capture].revision.content_fingerprint, + "cached-content" + ); + } + + #[cfg(unix)] + #[test] + fn coordinated_run_decodes_each_capture_once_and_publishes_distinct_products() { + use std::os::unix::fs::{PermissionsExt, symlink}; + + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + let executable = temporary.path().join("fake-nfdump"); + let stream_path = temporary.path().join("stream.bin"); + let empty_stream_path = temporary.path().join("empty-stream.bin"); + let invocation_log = temporary.path().join("invocations.log"); + let mut stream = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); + let record = 16; + stream[record + 32..record + 40].copy_from_slice(&20_u64.to_le_bytes()); + stream[record + 40..record + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + stream[record + 48..record + 56].copy_from_slice(&3_u64.to_le_bytes()); + stream[record + 64..record + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + stream[record + 69] = 0b010; + fs::write(&stream_path, stream).unwrap(); + fs::write( + &empty_stream_path, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + empty_stream_path.display(), + invocation_log.display(), + stream_path.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + let path = day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))); + fs::write(path, b"capture").unwrap(); + } + let registry = temporary.path().join("datasets.json"); + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + reset_prepare_nfcapd_tree_timestamp_calls(); + reset_nfcapd_pool_builds(); + reset_dataset_registry_load_calls(); + reset_coordinated_postflight_snapshot_verifications(); + let report = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["first".into(), "second".into()], + ) + .unwrap(); + + assert_eq!(report.five_minute_buckets, 576); + assert_eq!( + dataset_registry_load_calls(), + 1, + "coordinated resolution must use one registry snapshot" + ); + assert_eq!(nfcapd_pool_builds(), (1, 1, 1)); + assert_eq!( + coordinated_postflight_snapshot_verifications(), + 288 + 288, + "cold coordinated publication must verify every input snapshot once before commit" + ); + assert_eq!( + prepare_nfcapd_tree_timestamp_calls(), + 2 * 288, + "coordinated publication must consume preflight preparation" + ); + assert_eq!( + fs::read_to_string(&invocation_log).unwrap().lines().count(), + 289 + ); + let first_identity: String = Connection::open(&first_db) + .unwrap() + .query_row( + "SELECT selection_json FROM pipeline_product WHERE singleton = 1", + [], + |row| row.get(0), + ) + .unwrap(); + let second_identity: String = Connection::open(&second_db) + .unwrap() + .query_row( + "SELECT selection_json FROM pipeline_product WHERE singleton = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_ne!(first_identity, second_identity); + + let canonical_root = fs::canonicalize(&root).unwrap(); + let root_alias = temporary.path().join("captures-alias"); + symlink(&root, &root_alias).unwrap(); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root_alias, + "db_path": first_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root_alias, + "db_path": second_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + reset_nfcapd_logical_bucket_topology_calls(); + reset_nfcapd_day_topology_audit_calls(); + reset_nfcapd_pool_builds(); + reset_dataset_registry_load_calls(); + reset_coordinated_postflight_snapshot_verifications(); + crate::storage::reset_resume_query_counters(); + let resumed = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["second".into(), "first".into()], + ) + .unwrap(); + assert_eq!(resumed.five_minute_buckets, 0); + assert_eq!(dataset_registry_load_calls(), 1); + assert_eq!(nfcapd_logical_bucket_topology_calls(), 0); + assert_eq!( + nfcapd_day_topology_audit_calls(), + 0, + "a healthy marker-backed no-op must not scan stats-family topology" + ); + assert_eq!(nfcapd_pool_builds(), (1, 0, 0)); + assert_eq!( + crate::storage::resume_query_counters(), + crate::storage::ResumeQueryCounters { + input_evidence: 2, + processed_nfcapd: 2, + content_fingerprint: 0, + }, + "coordinated no-op resume state should load once per output" + ); + assert_eq!( + coordinated_postflight_snapshot_verifications(), + 0, + "a complete coordinated no-op must not repeat postflight snapshot verification" + ); + assert_eq!( + fs::read_to_string(&invocation_log).unwrap().lines().count(), + 289 + ); + let locator: String = Connection::open(&first_db) + .unwrap() + .query_row( + "SELECT input_locator FROM processed_inputs WHERE input_kind = 'nfcapd' LIMIT 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(locator.starts_with(canonical_root.to_string_lossy().as_ref())); + assert!(!locator.contains("captures-alias")); + + let connection = Connection::open(&first_db).unwrap(); + connection + .execute( + "DELETE FROM daily_product_completion + WHERE source_id = 'edge' AND day_start = ?1", + params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], + ) + .unwrap(); + drop(connection); + reset_nfcapd_day_topology_audit_calls(); + let legacy_resumed = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["first".into(), "second".into()], + ) + .unwrap(); + assert_eq!(legacy_resumed.five_minute_buckets, 0); + assert_eq!( + nfcapd_day_topology_audit_calls(), + 1, + "a missing marker without a dirty tombstone must use the legacy topology audit" + ); + let connection = Connection::open(&first_db).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'edge' AND day_start = ?1", + params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "legacy recovery must backfill the completion marker" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'edge' AND day_start = ?1", + params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + } + + #[cfg(unix)] + #[test] + fn coordinated_resume_keeps_a_complete_output_unchanged_while_catching_up_an_empty_one() { + use std::os::unix::fs::PermissionsExt; + + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second( + parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap() + bucket * FIVE_MINUTES, + ) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + let executable = temporary.path().join("fake-nfdump"); + let stream_path = temporary.path().join("stream.bin"); + let empty_stream_path = temporary.path().join("empty.stream"); + fs::write(&stream_path, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); + fs::write( + &empty_stream_path, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then cat '{}'; exit 0; fi\ncat '{}'\n", + empty_stream_path.display(), + stream_path.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + let single_request = PipelineRequest { + config_path: None, + dataset_id: Some("first".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }; + run(single_request).unwrap(); + let before = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); + + let report = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["first".into(), "second".into()], + ) + .unwrap(); + + assert_eq!(report.five_minute_buckets, 288); + assert_eq!(report.skipped_inputs, 288); + assert_eq!( + coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()), + before, + "a complete coordinated output must remain semantically unchanged" + ); + assert_eq!( + Connection::open(&second_db) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE granularity = '5m' AND coverage_state = 'complete'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 288 + ); + } + + #[cfg(unix)] + #[test] + fn coordinated_resume_handles_an_output_needed_only_in_a_later_decode_batch() { + use std::os::unix::fs::PermissionsExt; + + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second( + parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap() + bucket * FIVE_MINUTES, + ) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + let executable = temporary.path().join("fake-nfdump"); + let stream_path = temporary.path().join("stream.bin"); + let empty_stream_path = temporary.path().join("empty.stream"); + fs::write(&stream_path, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); + fs::write( + &empty_stream_path, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then cat '{}'; exit 0; fi\ncat '{}'\n", + empty_stream_path.display(), + stream_path.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "203.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + run(PipelineRequest { + config_path: None, + dataset_id: Some("first".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }) + .unwrap(); + + let before = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); + let first_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); + let later_batch_start = first_start + 12 * FIVE_MINUTES; + let later_batch_end = later_batch_start + 12 * FIVE_MINUTES; + let first_connection = Connection::open(&first_db).unwrap(); + first_connection + .execute( + "DELETE FROM input_evidence + WHERE source_id = 'edge' AND bucket_start >= ?1 AND bucket_start < ?2", + params![later_batch_start, later_batch_end], + ) + .unwrap(); + drop(first_connection); + + let after_fixture = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); + let error = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["first".into(), "second".into()], + ) + .unwrap_err(); + + let message = error.to_string(); + assert!( + message.contains("rerun that whole day with --force"), + "normal resume must reject orphaned provenance: {message}" + ); + let after_error = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); + assert_eq!( + &after_error[4..], + &after_fixture[4..], + "normal resume must not mutate the partially orphaned output" + ); + + let report = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: true, + run_maad: false, + require_complete: false, + }, + vec!["first".into(), "second".into()], + ) + .unwrap(); + + assert_eq!(report.five_minute_buckets, 576); + for database in [&first_db, &second_db] { + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE granularity = '5m' AND coverage_state = 'complete'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 288 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(DISTINCT bucket_start) FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 288, + "resume must restore every distinct five-minute traffic bucket" + ); + let five_minute_totals = connection + .query_row( + "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) + FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m' + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + )) + }, + ) + .unwrap(); + let daily_totals = connection + .query_row( + "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) + FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '1d' + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + )) + }, + ) + .unwrap(); + assert_eq!( + daily_totals, five_minute_totals, + "daily traffic must match 5m parity" + ); + } + let after_force = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); + assert_eq!( + &after_force[4..], + &before[4..], + "force rebuild must restore the original first output" + ); + } + + #[cfg(unix)] + #[test] + fn daily_active_resume_accepts_complete_maad_and_rejects_missing_maad_until_force_rebuilds_the_day() + { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let database = temporary.path().join("pipeline.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "default_start_date": "2025-06-01", + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let request = |force: bool| PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force, + run_maad: true, + require_complete: false, + }; + + run(request(false)).unwrap(); + let resumed = run(request(false)).unwrap(); + assert_eq!(resumed.five_minute_buckets, 0); + + // Coverage is part of the certified daily product. A direct mutation must dirty the + // marker so a normal resume cannot silently accept the stale canonical output; force + // rebuilds the complete day and restores the coverage envelope. + let connection = Connection::open(&database).unwrap(); + connection + .execute( + "UPDATE bucket_coverage + SET coverage_state = 'partial', observed_units = 1, expected_units = 2 + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![day_start], + ) + .unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'edge' AND day_start = ?1", + params![day_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "coverage updates must dirty a completed daily-active marker" + ); + drop(connection); + let error = run(request(false)).unwrap_err(); + assert!( + error + .to_string() + .contains("rerun that whole day with --force"), + "normal resume must reject post-certification coverage mutation: {error}" + ); + run(request(true)).unwrap(); + + let connection = Connection::open(&database).unwrap(); + connection + .execute( + "DELETE FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![day_start], + ) + .unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'edge' AND day_start = ?1", + params![day_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "coverage deletes must dirty a completed daily-active marker" + ); + drop(connection); + let error = run(request(false)).unwrap_err(); + assert!( + error + .to_string() + .contains("rerun that whole day with --force"), + "normal resume must reject post-certification coverage deletion: {error}" + ); + run(request(true)).unwrap(); + assert_eq!( + Connection::open(&database) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' + AND coverage_state = 'complete'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 288, + "force rebuild must republish deleted coverage" + ); + + let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); + let connection = Connection::open(&database).unwrap(); + connection + .execute( + "UPDATE traffic_stats SET flows = flows + 1 + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1 + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + params![day_start], + ) + .unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'edge' AND day_start = ?1", + params![day_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + drop(connection); + reset_nfcapd_day_topology_audit_calls(); + let error = run(request(false)).unwrap_err(); + assert!( + error + .to_string() + .contains("rerun that whole day with --force"), + "normal resume must reject a post-certification metric mutation: {error}" + ); + assert_eq!( + nfcapd_day_topology_audit_calls(), + 0, + "dirty completion evidence must not fall through to legacy topology auditing" + ); + let connection = Connection::open(&database).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'edge' AND day_start = ?1", + params![day_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "normal resume must retain the prior marker while refusing the dirty day" + ); + drop(connection); + run(request(true)).unwrap(); + let connection = Connection::open(&database).unwrap(); + assert_eq!( + coordinated_semantic_snapshot(&connection), + before, + "force rebuild must restore exact metrics after a direct mutation" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'edge' AND day_start = ?1", + params![day_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "force rebuild must leave a clean completion marker" + ); + drop(connection); + let mixed_end_bucket = day_start + 8 * FIVE_MINUTES; + let connection = Connection::open(&database).unwrap(); + connection + .execute( + "UPDATE traffic_stats SET bucket_end = ?2 + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1 + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + params![mixed_end_bucket, mixed_end_bucket + FIVE_MINUTES + 1], + ) + .unwrap(); + drop(connection); + let error = run(request(false)).unwrap_err(); + assert!( + error + .to_string() + .contains("rerun that whole day with --force"), + "mixed bucket_end corruption must fail full-day validation: {error}" + ); + let connection = Connection::open(&database).unwrap(); + connection + .execute( + "UPDATE traffic_stats SET bucket_end = bucket_start + 300 + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![mixed_end_bucket], + ) + .unwrap(); + let corrupted_bucket = day_start + 12 * FIVE_MINUTES; + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM address_structure_stats + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![corrupted_bucket], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 30, + "a healthy dense MAAD bucket has five IPv4 scopes, two sides, and three structures" + ); + connection + .execute( + "DELETE FROM address_structure_stats + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1 + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all' + AND address_side = 'source' AND structure_kind = 'structure'", + params![corrupted_bucket], + ) + .unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM address_structure_stats + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![corrupted_bucket], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 29, + "the corruption fixture must remove one of the 30 IPv4 MAAD rows" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = 'edge' AND bucket_start = ?1 + AND status = 'processed'", + params![corrupted_bucket], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "the corruption fixture must retain provenance" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![corrupted_bucket], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "the corruption fixture must retain coverage" + ); + drop(connection); + + let error = run(request(false)).unwrap_err(); + let message = error.to_string(); + assert!( + message.contains("rerun that whole day with --force"), + "normal resume must reject canonical topology corruption: {message}" + ); + assert_eq!( + Connection::open(&database) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM address_structure_stats + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![corrupted_bucket], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 29, + "normal resume must not partially repair the product" + ); + + run(request(true)).unwrap(); + let connection = Connection::open(&database).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(DISTINCT bucket_start) FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 288 + ); + let five_minute_totals = connection + .query_row( + "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) + FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m' + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + )) + }, + ) + .unwrap(); + let daily_totals = connection + .query_row( + "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) + FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '1d' + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + )) + }, + ) + .unwrap(); + assert_eq!( + daily_totals, five_minute_totals, + "force rebuild must restore daily parity" + ); + assert_eq!( + coordinated_semantic_snapshot(&connection), + before, + "force rebuild must restore the semantic product" + ); + } + + #[cfg(unix)] + #[test] + fn daily_active_force_rebuild_removes_surplus_day_rows() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let database = temporary.path().join("pipeline.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "default_start_date": "2025-06-01", + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let request = |force: bool| PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force, + run_maad: false, + require_complete: false, + }; + + run(request(false)).unwrap(); + let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); + let surplus_start = day_start + 60; + let surplus_end = surplus_start + FIVE_MINUTES; + let connection = Connection::open(&database).unwrap(); + connection + .execute( + "INSERT INTO bucket_coverage ( + source_id, granularity, bucket_start, bucket_end, coverage_state, + observed_units, expected_units, rejected_units + ) VALUES ('edge', '5m', ?1, ?2, 'complete', 1, 1, 0)", + params![surplus_start, surplus_end], + ) + .unwrap(); + connection + .execute( + "INSERT INTO traffic_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_visibility, dst_visibility, flows, flows_tcp, flows_udp, + flows_icmp, flows_other, packets, packets_tcp, packets_udp, + packets_icmp, packets_other, bytes, bytes_tcp, bytes_udp, + bytes_icmp, bytes_other, duration_sum_ms, duration_count, + average_duration_ms, min_ttl_sum, min_ttl_count, average_min_ttl, + max_ttl_sum, max_ttl_count, average_max_ttl + ) + SELECT source_id, granularity, ?1, ?2, ip_version, + src_visibility, dst_visibility, flows, flows_tcp, flows_udp, + flows_icmp, flows_other, packets, packets_tcp, packets_udp, + packets_icmp, packets_other, bytes, bytes_tcp, bytes_udp, + bytes_icmp, bytes_other, duration_sum_ms, duration_count, + average_duration_ms, min_ttl_sum, min_ttl_count, average_min_ttl, + max_ttl_sum, max_ttl_count, average_max_ttl + FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m' + LIMIT 1", + params![surplus_start, surplus_end], + ) + .unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![surplus_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![surplus_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + drop(connection); + + let error = run(request(false)).unwrap_err(); + assert!( + error + .to_string() + .contains("rerun that whole day with --force"), + "normal resume must reject a valid surplus row: {error}" + ); + + run(request(true)).unwrap(); + let connection = Connection::open(&database).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![surplus_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "force must remove surplus coverage inside the requested day" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![surplus_start], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "force must remove surplus product rows inside the requested day" + ); + assert_eq!(coordinated_semantic_snapshot(&connection), before); + drop(connection); + + let resumed = run(request(false)).unwrap(); + assert_eq!(resumed.five_minute_buckets, 0); + assert_eq!( + coordinated_semantic_snapshot(&Connection::open(&database).unwrap()), + before, + "a clean normal resume must be a no-op after force rebuild" + ); + } + + #[test] + fn coordinated_run_skips_an_incomplete_physical_day_for_every_output() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + fs::write(day.join("nfcapd.202506010000"), b"capture").unwrap(); + let registry = temporary.path().join("datasets.json"); + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + let error = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: true, + }, + vec!["first".into(), "second".into()], + ) + .unwrap_err(); + + let message = error.to_string(); + assert!(message.contains("dataset \"first\""), "{message}"); + assert!( + message.contains(&first_db.to_string_lossy().to_string()), + "{message}" + ); + assert!( + message.contains("576 incomplete five-minute coverage buckets"), + "{message}" + ); + for database in [first_db, second_db] { + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + } + } + + #[test] + fn canonical_day_resume_queries_seek_bounded_source_granularity_ranges() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + + let explain = |query: String| { + connection + .prepare(&query) + .unwrap() + .query_map(params!["r1", 0_i64, 86_400_i64], |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + }; + let assert_bounded_seek = |label: &str, plan: &[String]| { + let uses_range_seek = plan.iter().any(|detail| { + let compact = detail.replace(' ', ""); + compact.contains("SEARCH") + && compact.contains("source_id=?") + && compact.contains("granularity=?") + && compact.contains("bucket_start>?") + && compact.contains("bucket_start= ?2 AND bucket_start < ?3 + ORDER BY granularity, bucket_start" + )), + ); + for table in [ + "traffic_stats", + "protocol_stats", + "address_count_stats", + "port_count_stats", + "address_structure_stats", + ] { + assert_bounded_seek( + table, + &explain(format!( + "EXPLAIN QUERY PLAN + SELECT granularity, bucket_start, MIN(bucket_end), MAX(bucket_end), COUNT(*) + FROM {table} + WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} + AND bucket_start >= ?2 AND bucket_start < ?3 + GROUP BY granularity, bucket_start" + )), + ); + } + } + + #[cfg(unix)] + #[test] + fn coordinated_mixed_explicit_and_auto_layouts_publish_identical_source_metadata() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + write_nfcapd_day(&root); + let first_database = temporary.path().join("first.sqlite"); + let second_database = temporary.path().join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "configured", + "root_path": root, + "db_path": first_database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "auto", + "root_path": temporary.path().join("captures"), + "db_path": second_database, + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + let report = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["configured".into(), "auto".into()], + ) + .unwrap(); + assert_eq!(report.five_minute_buckets, 576); + + let source_members = |database: &Path| { + let connection = Connection::open(database).unwrap(); + connection + .prepare( + "SELECT source_id, member_id FROM source_members + ORDER BY dataset_id, source_id, member_id", + ) + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap() + }; + let source_layout = |database: &Path| { + let connection = Connection::open(database).unwrap(); + connection + .query_row( + "SELECT layout_json FROM nfcapd_source_layout WHERE singleton = 1", + [], + |row| row.get::<_, String>(0), + ) + .unwrap() + }; + let coverage_layout = |database: &Path| { + let connection = Connection::open(database).unwrap(); + connection + .prepare( + "SELECT source_id, granularity, bucket_start, bucket_end + FROM bucket_coverage ORDER BY source_id, granularity, bucket_start", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap() + }; + + assert_eq!( + source_members(&first_database), + source_members(&second_database) + ); + assert_eq!( + source_layout(&first_database), + source_layout(&second_database) + ); + assert_eq!( + coverage_layout(&first_database), + coverage_layout(&second_database) + ); + } + + #[cfg(unix)] + #[test] + fn single_auto_layout_change_after_planning_is_side_effect_free() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let output_directory = temporary.path().join("outputs"); + let database = output_directory.join("active.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + set_single_plan_hook(move |planned_root| { + fs::create_dir_all(planned_root.join("edge-b")).unwrap(); + }); + let error = run(PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }) + .unwrap_err(); + clear_single_plan_hook(); + + assert!( + error + .to_string() + .contains("auto-discovered source layout changed"), + "{error}" + ); + assert!(!output_directory.exists()); + assert!(!database.exists()); + assert!(!database_operation_lock_path(&database).unwrap().exists()); + fs::remove_dir_all(root.join("edge-b")).unwrap(); + } + + #[cfg(unix)] + #[test] + fn member_directory_replacement_after_planning_is_side_effect_free_in_single_and_coordinated_modes() + { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let database = temporary.path().join("single.sqlite"); + let registry = temporary.path().join("single-datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let single_backup = root.join("edge-before-single-replacement"); + let single_backup_for_hook = single_backup.clone(); + set_single_plan_hook(move |planned_root| { + let member = planned_root.join("edge"); + fs::rename(&member, &single_backup_for_hook).unwrap(); + fs::create_dir(&member).unwrap(); + }); + let single_error = run(PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }) + .unwrap_err(); + clear_single_plan_hook(); + assert!(single_error.to_string().contains("member directory")); + assert!(!database.exists()); + assert!(!database_operation_lock_path(&database).unwrap().exists()); + fs::remove_dir(root.join("edge")).unwrap(); + fs::rename(single_backup, root.join("edge")).unwrap(); + + let (request, output_directory, first_database, second_database, sentinel) = + repeated_dataset_request(&temporary, nfdump.clone()); + let coordinated_backup = root.join("edge-before-coordinated-replacement"); + set_coordinated_plan_hook(move |planned_root| { + let member = planned_root.join("edge"); + fs::rename(&member, &coordinated_backup).unwrap(); + fs::create_dir(&member).unwrap(); + }); + let coordinated_error = + run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); + clear_coordinated_plan_hook(); + assert!(coordinated_error.to_string().contains("member directory")); + assert!(!output_directory.exists()); + for database in [&first_database, &second_database] { + assert!(!database.exists()); + assert!(!database_operation_lock_path(database).unwrap().exists()); + } + assert_eq!(fs::read(sentinel).unwrap(), b"leave this alone"); + } + + #[cfg(unix)] + #[test] + fn member_directory_replacement_between_days_does_not_mix_single_product_days() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + write_nfcapd_day_for_date(&root, "2025-06-01"); + write_nfcapd_day_for_date(&root, "2025-06-02"); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let database = temporary.path().join("active.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let calls = std::rc::Rc::new(std::cell::Cell::new(0)); + let hook_calls = std::rc::Rc::clone(&calls); + set_missing_day_absence_hook(move |planned_root, _, _| { + let call = hook_calls.get(); + hook_calls.set(call + 1); + if call == 1 { + replace_member_directory(planned_root); + } + }); + let error = run(PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-03".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }) + .unwrap_err(); + clear_missing_day_absence_hook(); + + assert!(error.to_string().contains("member directory"), "{error}"); + assert_eq!(calls.get(), 2); + let connection = Connection::open(database).unwrap(); + let day_count = |start_date: &str| { + let start = parse_date_start(start_date, DEFAULT_TIMEZONE).unwrap(); + connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' + AND bucket_start >= ?1 AND bucket_start < ?2", + params![start, start + 86_400], + |row| row.get::<_, i64>(0), + ) + .unwrap() + }; + assert_eq!(day_count("2025-06-01"), 288); + assert_eq!(day_count("2025-06-02"), 0); + } + + #[cfg(unix)] + #[test] + fn member_directory_replacement_at_precommit_rolls_back_single_and_coordinated_days() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root); + let database = temporary.path().join("single.sqlite"); + let registry = temporary.path().join("single-datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let single_backup = root.join("edge-before-single-precommit-replacement"); + let root_for_hook = root.clone(); + let single_backup_for_hook = single_backup.clone(); + set_single_commit_guard_hook(move || { + let member = root_for_hook.join("edge"); + fs::rename(&member, &single_backup_for_hook).unwrap(); + fs::create_dir(&member).unwrap(); + }); + let single_error = run(PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-02".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }) + .unwrap_err(); + clear_single_commit_guard_hook(); + assert!(single_error.to_string().contains("member directory")); + let single_connection = Connection::open(&database).unwrap(); + assert_eq!( + single_connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 0 + ); + fs::remove_dir(root.join("edge")).unwrap(); + fs::rename(single_backup, root.join("edge")).unwrap(); + + let (mut request, _, first_database, second_database, _) = + repeated_dataset_request(&temporary, nfdump.clone()); + request.end_date = Some("2025-06-01".into()); + write_nfcapd_day(&root); + let coordinated_backup = root.join("edge-before-coordinated-precommit-replacement"); + set_coordinated_commit_guard_hook(move || { + let member = root.join("edge"); + fs::rename(&member, &coordinated_backup).unwrap(); + fs::create_dir(&member).unwrap(); + }); + let coordinated_error = + run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); + clear_coordinated_commit_guard_hook(); + assert!(coordinated_error.to_string().contains("member directory")); + for database in [first_database, second_database] { + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 0 + ); + } + } + + #[cfg(unix)] + #[test] + fn single_daily_active_capture_rewrite_before_commit_rolls_back_day() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let database = temporary.path().join("active.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "default_start_date": "2025-06-01", + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let request = |force| PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force, + run_maad: false, + require_complete: false, + }; + + run(request(false)).unwrap(); + let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); + let marker_count_before = Connection::open(&database) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'edge'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + let capture = root.join("edge/2025/06/01/nfcapd.202506010000"); + let original = fs::read(&capture).unwrap(); + let rewritten = capture.clone(); + set_single_commit_guard_hook(move || { + fs::write(&rewritten, b"rewritten after processing").unwrap(); + }); + + let error = run(request(true)).unwrap_err(); + clear_single_commit_guard_hook(); + + assert!(error.to_string().contains("Input changed"), "{error}"); + assert_eq!( + coordinated_semantic_snapshot(&Connection::open(&database).unwrap()), + before, + "a capture rewrite at the precommit seam must roll back all day rows" + ); + assert_eq!( + Connection::open(&database) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'edge'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + marker_count_before, + "the completion marker must roll back with the day rows" + ); + assert_ne!(fs::read(&capture).unwrap(), original); + fs::write(capture, original).unwrap(); + } + + #[cfg(unix)] + #[test] + fn single_tree_rejects_stale_complete_day_and_force_removes_it_before_strict_failure() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second( + parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap() + bucket * FIVE_MINUTES, + ) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let registry = temporary.path().join("datasets.json"); + let database = temporary.path().join("pipeline.sqlite"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let request = |force: bool, require_complete: bool| PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force, + run_maad: false, + require_complete, + }; + + run(request(false, false)).unwrap(); + let removed = day.join("nfcapd.202506010000"); + fs::remove_file(&removed).unwrap(); + Connection::open(&database) + .unwrap() + .execute( + "DELETE FROM bucket_coverage + WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", + params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], + ) + .unwrap(); + let before = Connection::open(&database) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + assert!(matches!( + run(request(false, false)), + Err(PipelineError::InvalidConfig(_)) + )); + assert_eq!( + Connection::open(&database) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + before + ); + + let error = run(request(true, true)).unwrap_err(); + assert!(matches!(error, PipelineError::IncompleteCoverage(288))); + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM input_evidence", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 0 + ); + } + + #[cfg(unix)] + #[test] + fn single_force_stale_day_rejects_a_late_capture_without_product_loss() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let registry = temporary.path().join("datasets.json"); + let database = temporary.path().join("pipeline.sqlite"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + let request = |force: bool| PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry.clone()), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force, + run_maad: false, + require_complete: false, + }; + + run(request(false)).unwrap(); + let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); + let removed = root.join("edge/2025/06/01/nfcapd.202506010000"); + fs::remove_file(&removed).unwrap(); + let restored = removed.clone(); + set_missing_day_absence_hook(move |_, missing, _| { + assert!(!missing.is_empty()); + fs::write(&restored, b"late capture").unwrap(); + }); + + let error = run(request(true)).unwrap_err(); + clear_missing_day_absence_hook(); + + let message = error.to_string(); + assert!( + message.contains("refusing to delete the existing product"), + "{message}" + ); + assert!(message.contains("nfcapd.202506010000"), "{message}"); + assert!(removed.is_file()); + assert_eq!( + &coordinated_semantic_snapshot(&Connection::open(database).unwrap())[4..], + &before[4..], + "late capture detection must roll back the stale-day deletion" + ); + } -fn parse_date_start(raw: &str, timezone: &str) -> Result { - let date: Date = raw - .parse() - .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; - Ok(date - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second()) -} + #[cfg(unix)] + #[test] + fn coordinated_force_stale_day_rejects_a_late_capture_without_product_loss() { + let temporary = tempdir().unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let (mut request, _output_directory, first_database, second_database, _sentinel) = + repeated_dataset_request(&temporary, executable.clone()); + request.end_date = Some("2025-06-01".into()); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root); + + run_many(request.clone(), vec!["first".into(), "second".into()]).unwrap(); + let before_first = + coordinated_semantic_snapshot(&Connection::open(&first_database).unwrap()); + let before_second = + coordinated_semantic_snapshot(&Connection::open(&second_database).unwrap()); + let removed = root.join("edge/2025/06/01/nfcapd.202506010000"); + fs::remove_file(&removed).unwrap(); + let restored = removed.clone(); + set_missing_day_absence_hook(move |_, missing, _| { + assert!(!missing.is_empty()); + fs::write(&restored, b"late capture").unwrap(); + }); -fn next_date_start(raw: &str, timezone: &str) -> Result { - let date: Date = raw - .parse() - .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; - Ok(date - .tomorrow() - .and_then(|date| date.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second()) -} + let mut forced = request; + forced.force = true; + let error = run_many(forced, vec!["first".into(), "second".into()]).unwrap_err(); + clear_missing_day_absence_hook(); -fn parse_local_datetime(raw: &str, timezone: &str) -> Result { - let normalized = if raw.len() == 16 { - format!("{raw}:00") - } else { - raw.to_owned() - }; - let datetime = normalized - .parse::() - .map_err(|error| PipelineError::Time(error.to_string()))?; - Ok(datetime - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second()) -} + let message = error.to_string(); + assert!( + message.contains("refusing to delete the existing product"), + "{message}" + ); + assert!(message.contains("nfcapd.202506010000"), "{message}"); + assert!(removed.is_file()); + assert_eq!( + &coordinated_semantic_snapshot(&Connection::open(first_database).unwrap())[4..], + &before_first[4..], + "first coordinated output must survive a late capture" + ); + assert_eq!( + &coordinated_semantic_snapshot(&Connection::open(second_database).unwrap())[4..], + &before_second[4..], + "second coordinated output must survive a late capture" + ); + } -fn validate_window( - selected_start: i64, - selected_end: i64, - start: i64, - end: i64, - timezone: &str, -) -> Result<(), PipelineError> { - if start < selected_start { - return Err(PipelineError::InvalidConfig( - "start_time must be on or after the selected start_date".into(), - )); + #[cfg(unix)] + #[test] + fn coordinated_force_stale_day_does_not_guard_again_after_the_first_commit() { + let temporary = tempdir().unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let (mut request, _output_directory, first_database, second_database, _sentinel) = + repeated_dataset_request(&temporary, executable.clone()); + request.end_date = Some("2025-06-01".into()); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root); + + run_many(request.clone(), vec!["first".into(), "second".into()]).unwrap(); + let removed = root.join("edge/2025/06/01/nfcapd.202506010000"); + fs::remove_file(&removed).unwrap(); + let calls = std::rc::Rc::new(std::cell::Cell::new(0)); + let hook_calls = std::rc::Rc::clone(&calls); + let late_capture = removed.clone(); + set_coordinated_commit_guard_hook(move || { + let call = hook_calls.get(); + hook_calls.set(call + 1); + if call == 1 { + fs::write(&late_capture, b"late capture").unwrap(); + } + }); + + let mut forced = request; + forced.force = true; + let result = run_many(forced, vec!["first".into(), "second".into()]); + clear_coordinated_commit_guard_hook(); + result.unwrap(); + + assert_eq!(calls.get(), 1); + assert!(!removed.exists()); + assert_eq!( + &coordinated_semantic_snapshot(&Connection::open(first_database).unwrap())[4..], + &coordinated_semantic_snapshot(&Connection::open(second_database).unwrap())[4..], + "a late-capture hook at the former second-commit seam must not split outputs" + ); } - if end > selected_end { - return Err(PipelineError::InvalidConfig( - "end_time must be on or before the selected end_date window".into(), - )); + + #[cfg(unix)] + #[test] + fn coordinated_publication_does_not_guard_again_after_the_first_commit() { + let temporary = tempdir().unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, ""); + let (request, _output_directory, first_database, second_database, _sentinel) = + repeated_dataset_request(&temporary, executable.clone()); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root); + + let calls = std::rc::Rc::new(std::cell::Cell::new(0)); + let hook_calls = std::rc::Rc::clone(&calls); + let changed_decoder = executable.clone(); + set_coordinated_commit_guard_hook(move || { + let call = hook_calls.get(); + hook_calls.set(call + 1); + if call == 1 { + let mut contents = fs::read(&changed_decoder).unwrap(); + contents.push(b'\n'); + fs::write(&changed_decoder, contents).unwrap(); + } + }); + + let result = run_many(request, vec!["first".into(), "second".into()]); + clear_coordinated_commit_guard_hook(); + result.unwrap(); + + assert_eq!(calls.get(), 1); + let mut states = Vec::new(); + for database in [first_database, second_database] { + let connection = Connection::open(database).unwrap(); + let processed_inputs = connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + let bucket_coverage = connection + .query_row("SELECT COUNT(*) FROM bucket_coverage", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + assert!(processed_inputs > 0); + assert!(bucket_coverage > 0); + states.push((processed_inputs, bucket_coverage)); + } + assert_eq!(states[0], states[1]); } - if start >= end { - return Err(PipelineError::InvalidConfig( - "input time window must be non-empty".into(), - )); + + #[test] + fn coordinated_empty_finite_request_is_incomplete_without_publishing_traffic() { + let temporary = tempdir().unwrap(); + let nfdump = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&nfdump, ""); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} + } + ])) + .unwrap(), + ) + .unwrap(); + + let error = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: nfdump.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: true, + }, + vec!["first".into(), "second".into()], + ) + .unwrap_err(); + + let message = error.to_string(); + assert!(message.contains("dataset \"first\""), "{message}"); + assert!( + message.contains(&first_db.to_string_lossy().to_string()), + "{message}" + ); + assert!( + message.contains("288 incomplete five-minute coverage buckets"), + "{message}" + ); + for database in [first_db, second_db] { + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + } } - for (label, value) in [("start_time", start), ("end_time", end)] { - if aggregate_bounds(value, Granularity::OneDay, timezone)?.0 != value { - return Err(PipelineError::InvalidConfig(format!( - "{label} must align to a local-day boundary so aggregate rows stay complete" - ))); + + #[test] + fn complete_physical_day_uses_local_dst_bucket_boundaries() { + let start = parse_date_start("2025-03-09", "America/Los_Angeles").unwrap(); + let end = next_date_start("2025-03-09", "America/Los_Angeles").unwrap(); + let members = vec!["cc".to_owned(), "oh".to_owned()]; + let mut paths = BTreeMap::new(); + let mut bucket_start = start; + let mut count = 0; + while bucket_start < end { + for member in &members { + paths.insert( + (member.clone(), bucket_start), + PathBuf::from(format!("{member}/{bucket_start}")), + ); + } + count += 1; + bucket_start = + next_local_five_minute_start(bucket_start, "America/Los_Angeles").unwrap(); } + + assert_eq!(count, 276); + assert!( + missing_physical_day_inputs(&members, &paths, start, end, "America/Los_Angeles") + .unwrap() + .is_empty() + ); + paths.remove(&("oh".to_owned(), start)); + assert_eq!( + missing_physical_day_inputs(&members, &paths, start, end, "America/Los_Angeles") + .unwrap(), + [("oh".to_owned(), start)] + ); } - Ok(()) -} -fn expected_nfcapd_path( - root: &Path, - member: &str, - bucket_start: i64, - timezone: &str, -) -> Result { - let timestamp = Timestamp::from_second(bucket_start) - .and_then(|timestamp| timestamp.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))?; - Ok(root - .join(member) - .join(timestamp.strftime("%Y").to_string()) - .join(timestamp.strftime("%m").to_string()) - .join(timestamp.strftime("%d").to_string()) - .join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M")))) -} + #[test] + fn nfcapd_decode_chunks_cap_a_timestamp_with_more_than_twelve_members() { + let members = (0..13).map(|index| format!("member-{index:02}")); + let members = members.collect::>(); + let sources = [DatasetSource { + source_id: "logical".into(), + members: members.clone(), + }]; + let paths = members + .iter() + .map(|member| { + ( + (member.clone(), 0), + PathBuf::from(format!("{member}/nfcapd.197001010000")), + ) + }) + .collect::>(); -#[cfg(test)] -mod tests { - use std::{ - fs, - net::{IpAddr, Ipv4Addr}, - }; + assert_eq!( + nfcapd_batch_starts( + 0, + FIVE_MINUTES, + "UTC", + &sources, + &paths, + &BTreeMap::new(), + false, + ) + .unwrap(), + [0] + ); + let requests = members + .iter() + .map(|member| (member.clone(), 0_i64)) + .collect::>(); + let chunk_lengths = nfcapd_decode_request_chunks(&requests) + .map(<[_]>::len) + .collect::>(); + assert_eq!(chunk_lengths, [12, 1]); + assert!( + chunk_lengths + .iter() + .all(|length| *length <= NFCAPD_DECODE_BATCH_SIZE) + ); + } - use rusqlite::Connection; - use serde_json::json; - use tempfile::tempdir; + #[cfg(unix)] + #[test] + fn daily_activity_unions_each_physical_member_once() { + use std::os::unix::fs::PermissionsExt; - use super::*; - use crate::{ - coverage::CoverageState, - domain::{AddressSide, FlowObservation, IpVersion, Scope, Visibility}, - }; + let temporary = tempdir().unwrap(); + let executable = temporary.path().join("fake-nfdump"); + let stream_path = temporary.path().join("activity.stream"); + let empty_stream_path = temporary.path().join("empty.stream"); + let invocation_log = temporary.path().join("invocations.log"); + let mut stream = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); + let record = 16; + stream[record + 32..record + 40].copy_from_slice(&10_u64.to_le_bytes()); + stream[record + 40..record + 48].copy_from_slice(&1_000_u64.to_le_bytes()); + stream[record + 48..record + 56].copy_from_slice(&2_u64.to_le_bytes()); + stream[record + 64..record + 66].copy_from_slice(&1_024_u16.to_le_bytes()); + stream[record + 69] = 0b010; + fs::write(&stream_path, stream).unwrap(); + fs::write( + &empty_stream_path, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + empty_stream_path.display(), + invocation_log.display(), + stream_path.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let mut paths = BTreeMap::new(); + for member in ["cc", "oh"] { + let directory = temporary.path().join(member); + fs::create_dir(&directory).unwrap(); + let path = directory.join("nfcapd.197001010000"); + fs::write(&path, "capture").unwrap(); + paths.insert((member.to_owned(), 0), path); + } + let sources = vec![ + DatasetSource { + source_id: "cc".into(), + members: vec!["cc".into()], + }, + DatasetSource { + source_id: "oh".into(), + members: vec!["oh".into()], + }, + DatasetSource { + source_id: "all".into(), + members: vec!["cc".into(), "oh".into()], + }, + ]; + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + }))) + .unwrap(); + let pipeline = ResolvedPipeline { + database_path: temporary.path().join("unused.sqlite"), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: executable.to_string_lossy().into_owned().into(), + nfdump_revision: None, + selection, + inputs: Vec::new(), + datasets: Vec::new(), + require_complete: false, + }; + + let (active, _) = resolve_daily_active_sources( + &sources, + &paths, + 0, + FIVE_MINUTES, + &pipeline, + &BTreeMap::new(), + &BTreeMap::new(), + ) + .unwrap(); + + assert!(active.contains(&IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)))); + assert_eq!( + fs::read_to_string(invocation_log).unwrap().lines().count(), + 2 + ); + } #[cfg(unix)] - fn write_fake_nfdump(executable: &Path, setup: &str) { + #[test] + fn daily_active_eligibility_ignores_off_grid_capture_keys() { use std::os::unix::fs::PermissionsExt; - let stream = executable.with_extension("stream"); - fs::write(&stream, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); + for bucket in 0..288 { + let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + fs::write( + day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), + b"capture", + ) + .unwrap(); + } + let off_grid = day.join("nfcapd.202506010001"); + fs::write(&off_grid, b"off-grid capture").unwrap(); + + let executable = temporary.path().join("fake-nfdump"); + let active_stream = temporary.path().join("active.stream"); + let empty_stream = temporary.path().join("empty.stream"); + let mut active_bytes = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); + active_bytes[16 + 32..16 + 40].copy_from_slice(&20_u64.to_le_bytes()); + active_bytes[16 + 40..16 + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + active_bytes[16 + 48..16 + 56].copy_from_slice(&3_u64.to_le_bytes()); + fs::write(&active_stream, active_bytes).unwrap(); fs::write( - executable, - format!("#!/bin/sh\n{setup}\ncat '{}'\n", stream.display()), + &empty_stream, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh +if [ \"$1\" = \"-R\" ]; then + if [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then + cat '{}' + exit 0 + fi + for path in \"$2\"/*; do + target=$(readlink \"$path\") + case \"$target\" in + *nfcapd.202506010001) cat '{}'; exit 0 ;; + esac + done + cat '{}' +else + cat '{}' +fi +", + empty_stream.display(), + active_stream.display(), + empty_stream.display(), + active_stream.display() + ), ) .unwrap(); - fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let database = temporary.path().join("pipeline.sqlite"); + let registry = temporary.path().join("datasets.json"); + fs::write( + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} + }])) + .unwrap(), + ) + .unwrap(); + + run(PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + }) + .unwrap(); + + let connection = Connection::open(&database).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COALESCE(SUM(flows), 0) FROM traffic_stats + WHERE source_id = 'edge' AND granularity = '5m' + AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "an off-grid-only threshold hit must not qualify the active source" + ); + for table in [ + "bucket_coverage", + "traffic_stats", + "processed_inputs", + "input_evidence", + ] { + assert_eq!( + connection + .query_row( + &format!( + "SELECT COUNT(*) FROM {table} + WHERE source_id = 'edge' AND bucket_start = ?1" + ), + params![day_start + 60], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "off-grid capture must not appear in {table}" + ); + } } #[test] @@ -3085,9 +12188,11 @@ mod tests { } let pipeline = ResolvedPipeline { database_path: temporary.path().join("netflow.sqlite"), + control_paths: Vec::new(), timezone: "UTC".into(), run_maad: false, nfdump: "nfdump".into(), + nfdump_revision: None, selection: FlowSelection::default(), inputs: vec![InputSpec::NfcapdTree { root_path: capture_root, @@ -3105,6 +12210,219 @@ mod tests { assert_eq!( count_incomplete_requested_coverage(&connection, &pipeline).unwrap(), + 288 + ); + } + + #[test] + fn open_ended_native_strict_coverage_includes_the_discovered_latest_day() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + let day = root.join("edge/2025/06/01"); + fs::create_dir_all(&day).unwrap(); + fs::write(day.join("nfcapd.202506010000"), b"capture").unwrap(); + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let start = parse_date_start("2025-06-01", "UTC").unwrap(); + connection + .execute( + "INSERT INTO bucket_coverage ( + source_id, granularity, bucket_start, bucket_end, + coverage_state, observed_units, expected_units, rejected_units + ) VALUES ('edge', '5m', ?1, ?2, 'complete', 1, 1, 0)", + params![start, start + FIVE_MINUTES], + ) + .unwrap(); + let pipeline = ResolvedPipeline { + database_path: temporary.path().join("unused.sqlite"), + control_paths: Vec::new(), + timezone: "UTC".into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root, + source_ids: vec!["edge".into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: None, + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: true, + }; + + assert_eq!( + count_incomplete_requested_coverage(&connection, &pipeline).unwrap(), + 287 + ); + } + + #[test] + fn persisted_sibling_validation_is_cached_per_source_day_but_rejects_foreign_rows() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let bucket = |bucket_start| { + StatisticalBucket::dense(BucketKey::new( + "r1", + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + )) + .finish_owned() + }; + + let mut aggregates = AggregateBuckets::default(); + for index in 0..288_i64 { + let child = bucket(index * FIVE_MINUTES); + aggregates + .reject_persisted_siblings(&connection, &child, "UTC") + .unwrap(); + aggregates.include(&child, "UTC").unwrap(); + } + assert_eq!(aggregates.persisted_sibling_queries, 1); + + let foreign = bucket(FIVE_MINUTES); + write_buckets(&connection, std::slice::from_ref(&foreign), false).unwrap(); + let error = AggregateBuckets::default() + .reject_persisted_siblings(&connection, &bucket(0), "UTC") + .unwrap_err(); + assert!(error.to_string().contains("cannot reopen")); + } + + #[test] + fn strict_coverage_counts_missing_partial_and_dst_successor_rows() { + let temporary = tempdir().unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("r1")).unwrap(); + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + + let pipeline_for = |date: &str, timezone: &str| ResolvedPipeline { + database_path: temporary.path().join("unused.sqlite"), + control_paths: Vec::new(), + timezone: timezone.into(), + run_maad: false, + nfdump: "nfdump".into(), + nfdump_revision: None, + selection: FlowSelection::default(), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.clone(), + source_ids: vec!["r1".into()], + sources: Vec::new(), + start_date: date.into(), + end_date: Some(date.into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: true, + }; + let local_starts = |date: &str, timezone: &str| { + let start = parse_date_start(date, timezone).unwrap(); + let end = next_date_start(date, timezone).unwrap(); + let mut starts = Vec::new(); + let mut current = start; + while current < end { + starts.push(current); + current = next_local_five_minute_start(current, timezone).unwrap(); + } + starts + }; + let insert = |start: i64, state: &str, observed: i64, expected: i64| { + connection + .execute( + "INSERT INTO bucket_coverage ( + source_id, granularity, bucket_start, bucket_end, + coverage_state, observed_units, expected_units, rejected_units + ) VALUES ('r1', '5m', ?1, ?2, ?3, ?4, ?5, 0)", + params![start, start + FIVE_MINUTES, state, observed, expected], + ) + .unwrap(); + }; + + let normal = local_starts("2025-01-01", "UTC"); + assert_eq!(normal.len(), 288); + for start in &normal { + insert(*start, "complete", 1, 1); + } + assert_eq!( + count_incomplete_requested_coverage(&connection, &pipeline_for("2025-01-01", "UTC")) + .unwrap(), + 0 + ); + connection + .execute( + "DELETE FROM bucket_coverage WHERE source_id = 'r1' AND bucket_start = ?1", + params![normal[0]], + ) + .unwrap(); + assert_eq!( + count_incomplete_requested_coverage(&connection, &pipeline_for("2025-01-01", "UTC")) + .unwrap(), + 1 + ); + insert(normal[0], "partial", 1, 2); + assert_eq!( + count_incomplete_requested_coverage(&connection, &pipeline_for("2025-01-01", "UTC")) + .unwrap(), + 1 + ); + + connection + .execute("DELETE FROM bucket_coverage", []) + .unwrap(); + let spring = local_starts("2025-03-09", DEFAULT_TIMEZONE); + assert_eq!(spring.len(), 276); + for start in &spring { + insert(*start, "complete", 1, 1); + } + assert_eq!( + count_incomplete_requested_coverage( + &connection, + &pipeline_for("2025-03-09", DEFAULT_TIMEZONE) + ) + .unwrap(), + 0 + ); + + connection + .execute("DELETE FROM bucket_coverage", []) + .unwrap(); + let fall = local_starts("2025-11-02", DEFAULT_TIMEZONE); + assert_eq!( + fall.len(), + 288, + "preserve the current fall-back successor contract" + ); + for start in &fall { + insert(*start, "complete", 1, 1); + } + assert_eq!( + count_incomplete_requested_coverage( + &connection, + &pipeline_for("2025-11-02", DEFAULT_TIMEZONE) + ) + .unwrap(), + 0 + ); + connection + .execute( + "UPDATE bucket_coverage SET coverage_state = 'partial', observed_units = 1, expected_units = 2 + WHERE source_id = 'r1' AND bucket_start = ?1", + params![fall[0]], + ) + .unwrap(); + assert_eq!( + count_incomplete_requested_coverage( + &connection, + &pipeline_for("2025-11-02", DEFAULT_TIMEZONE) + ) + .unwrap(), 1 ); } @@ -3959,6 +13277,8 @@ mod tests { member_bounds: &bounds, extend_gaps_to_window: false, force: false, + decoder_fingerprint: "decoder".into(), + capture_snapshots: &BTreeMap::new(), revision_pool: &pool, }; let cached = resolve_nfcapd_batch_revisions(&normal_context, &[0]) @@ -4013,7 +13333,14 @@ mod tests { ) .unwrap(); - assert!(run(PipelineRequest::config(config)).is_err()); + let error = run(PipelineRequest::config(config)).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("nfcapd decode failed"), "{message}"); + assert!(message.contains("member \"r1\""), "{message}"); + assert!( + message.contains(&second.to_string_lossy().to_string()), + "{message}" + ); let connection = Connection::open(database).unwrap(); let starts = connection .prepare("SELECT bucket_start FROM processed_inputs ORDER BY bucket_start") @@ -4356,4 +13683,174 @@ mod tests { assert_eq!(fs::read_to_string(calls).unwrap().lines().count(), 1); } + + #[cfg(unix)] + #[test] + fn nfdump_revision_conflicts_before_a_second_binary_can_mix_product_rows() { + let temporary = tempdir().unwrap(); + let capture = temporary.path().join("nfcapd.202504151200"); + let first_decoder = temporary.path().join("nfdump-a"); + let second_decoder = temporary.path().join("nfdump-b"); + let database = temporary.path().join("netflow.sqlite"); + let config = temporary.path().join("pipeline.json"); + fs::write(&capture, "fixture").unwrap(); + write_fake_nfdump(&first_decoder, ""); + write_fake_nfdump(&second_decoder, ""); + let write_config = |decoder: &Path| { + fs::write( + &config, + serde_json::to_vec(&json!({ + "database_path": database, + "timezone": "America/Los_Angeles", + "nfdump": decoder, + "run_maad": false, + "inputs": [{"input_kind": "nfcapd", "path": capture, "source_id": "r1"}] + })) + .unwrap(), + ) + .unwrap(); + }; + + write_config(&first_decoder); + run(PipelineRequest::config(&config)).unwrap(); + let connection = Connection::open(&database).unwrap(); + let before_inputs: i64 = connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + row.get(0) + }) + .unwrap(); + let before_traffic: i64 = connection + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row.get(0)) + .unwrap(); + drop(connection); + + write_config(&second_decoder); + let error = run(PipelineRequest::config(&config)).unwrap_err(); + assert!( + error + .to_string() + .contains("Pipeline product identity mismatch") + ); + + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + before_inputs + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + before_traffic + ); + } + + #[cfg(unix)] + #[test] + fn nfdump_replacement_after_decode_rolls_back_the_native_transaction() { + use std::os::unix::fs::PermissionsExt; + + let temporary = tempdir().unwrap(); + let capture = temporary.path().join("nfcapd.202504151200"); + let decoder = temporary.path().join("nfdump"); + let replacement = temporary.path().join("nfdump-replacement"); + let stream = temporary.path().join("stream.bin"); + let empty_stream = temporary.path().join("empty.stream"); + let database = temporary.path().join("netflow.sqlite"); + let config = temporary.path().join("pipeline.json"); + fs::write(&capture, "fixture").unwrap(); + fs::write(&stream, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); + fs::write( + &empty_stream, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &replacement, + format!("#!/bin/sh\ncat '{}'\n", stream.display()), + ) + .unwrap(); + fs::set_permissions(&replacement, fs::Permissions::from_mode(0o755)).unwrap(); + fs::write( + &decoder, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ]; then cat '{}'; exit 0; fi\ncat '{}'\ncp '{}' \"$0\"\n", + empty_stream.display(), + stream.display(), + replacement.display() + ), + ) + .unwrap(); + fs::set_permissions(&decoder, fs::Permissions::from_mode(0o755)).unwrap(); + fs::write( + &config, + serde_json::to_vec(&json!({ + "database_path": database, + "timezone": "America/Los_Angeles", + "nfdump": decoder, + "run_maad": false, + "inputs": [{"input_kind": "nfcapd", "path": capture, "source_id": "r1"}] + })) + .unwrap(), + ) + .unwrap(); + + let error = run(PipelineRequest::config(&config)).unwrap_err(); + assert!( + error.to_string().contains("nfdump executable changed"), + "{error}" + ); + let connection = Connection::open(database).unwrap(); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + } + + #[cfg(unix)] + #[test] + fn incompatible_nfdump_probe_fails_before_output_setup() { + use std::os::unix::fs::PermissionsExt; + + let temporary = tempdir().unwrap(); + let capture = temporary.path().join("nfcapd.202504151200"); + let decoder = temporary.path().join("incompatible-nfdump"); + let output_directory = temporary.path().join("outputs"); + let database = output_directory.join("netflow.sqlite"); + let config = temporary.path().join("pipeline.json"); + fs::write(&capture, "fixture").unwrap(); + fs::write(&decoder, "#!/bin/sh\nexit 0\n").unwrap(); + fs::set_permissions(&decoder, fs::Permissions::from_mode(0o755)).unwrap(); + fs::write( + &config, + serde_json::to_vec(&json!({ + "database_path": database, + "timezone": "America/Los_Angeles", + "nfdump": decoder, + "run_maad": false, + "inputs": [{"input_kind": "nfcapd", "path": capture, "source_id": "r1"}] + })) + .unwrap(), + ) + .unwrap(); + + let error = run(PipelineRequest::config(&config)).unwrap_err(); + assert!(error.to_string().contains("compatibility probe"), "{error}"); + assert!(!output_directory.exists()); + assert!(!database_operation_lock_path(&database).unwrap().exists()); + } } diff --git a/tools/netflow-db/src/provenance.rs b/tools/netflow-db/src/provenance.rs index 880a845..867d12d 100644 --- a/tools/netflow-db/src/provenance.rs +++ b/tools/netflow-db/src/provenance.rs @@ -355,6 +355,48 @@ pub fn nfcapd_decoder_fingerprint() -> Result { })) } +/// The executable identity is part of the native decoder identity. The contract fingerprint +/// describes the stream decoder in this crate; the executable fingerprint binds that contract to +/// the exact nfdump implementation that produced the stream. +pub fn nfcapd_decoder_fingerprint_for_executable( + executable_locator: &str, + executable_content_fingerprint: &str, +) -> Result { + fingerprint(&json!({ + "version": 1, + "contract_id": nfcapd_decoder_fingerprint()?, + "executable": { + "locator": executable_locator, + "content_fingerprint": executable_content_fingerprint, + }, + })) +} + +/// One read-only snapshot of the executable used by native decoding. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExecutableRevision { + pub locator: String, + pub content_fingerprint: String, + pub snapshot: FileSnapshot, + pub decoder_fingerprint: String, +} + +impl ExecutableRevision { + pub fn capture(path: impl AsRef) -> Result { + let path = path.as_ref(); + let locator = path.to_string_lossy().into_owned(); + let (content_fingerprint, snapshot) = capture_file_revision(path)?; + let decoder_fingerprint = + nfcapd_decoder_fingerprint_for_executable(&locator, &content_fingerprint)?; + Ok(Self { + locator, + content_fingerprint, + snapshot, + decoder_fingerprint, + }) + } +} + pub fn capture_csv_input_revision( path: impl AsRef, config: &CsvSourceConfig, diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index c42ba5a..1fb161c 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -3,6 +3,7 @@ use std::{ collections::BTreeMap, net::IpAddr, + sync::OnceLock, time::{Duration, Instant}, }; @@ -33,10 +34,15 @@ pub enum PublishError { Storage(#[from] StorageError), #[error("unable to serialize MAAD rows: {0}")] Json(#[from] serde_json::Error), + #[error("unable to build MAAD worker pool: {0}")] + MaadPool(String), #[error("aggregate bucket lacks complete five-minute coverage: {0:?}")] IncompleteCoverage(BucketKey), } +const MAAD_WORKERS: usize = 2; +static MAAD_POOL: OnceLock> = OnceLock::new(); + /// Aggregate timings and work counts for one or more `write_buckets` calls. /// /// Timers wrap batch boundaries rather than individual rows so profiling remains @@ -334,46 +340,72 @@ fn insert_rows( fn maad_rows( address_sets: &[AddressSetRow<'_>], ) -> Result, PublishError> { - Ok(address_sets - .par_iter() + // Filter before entering the pool so the indexed collection below keeps + // canonical input order while still allowing independent scopes to run in + // parallel. + let address_sets = address_sets + .iter() .filter(|addresses| addresses.scope.ip_version == IpVersion::V4) - .map(|addresses| { - let result = maad::compute(addresses.addresses.iter().filter_map( - |address| match address { - IpAddr::V4(address) => Some(*address), - IpAddr::V6(_) => None, - }, - )); - let metadata_json = serde_json::to_string(&result.metadata)?; - let dimensions = dimensions(&addresses.key, addresses.scope); - Ok::<_, serde_json::Error>([ - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "structure".into(), - values_json: serde_json::to_string(&result.structure)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "spectrum".into(), - values_json: serde_json::to_string(&result.spectrum)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions, - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "dimension".into(), - values_json: serde_json::to_string(&result.dimensions)?, - metadata_json, - }, - ]) - }) - .collect::, _>>()? - .into_iter() - .flatten() - .collect()) + .collect::>(); + if address_sets.is_empty() { + return Ok(Vec::new()); + } + + let pool = maad_pool()?; + let rows = + pool.install(|| { + address_sets + .par_iter() + .map(|addresses| { + let result = + maad::compute(addresses.addresses.iter().filter_map( + |address| match address { + IpAddr::V4(address) => Some(*address), + IpAddr::V6(_) => None, + }, + )); + let metadata_json = serde_json::to_string(&result.metadata)?; + let dimensions = dimensions(&addresses.key, addresses.scope); + Ok::<_, serde_json::Error>([ + AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + structure_kind: "structure".into(), + values_json: serde_json::to_string(&result.structure)?, + metadata_json: metadata_json.clone(), + }, + AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + structure_kind: "spectrum".into(), + values_json: serde_json::to_string(&result.spectrum)?, + metadata_json: metadata_json.clone(), + }, + AddressStructureStatsRow { + dimensions, + address_side: addresses.address_side.as_str().to_owned(), + structure_kind: "dimension".into(), + values_json: serde_json::to_string(&result.dimensions)?, + metadata_json, + }, + ]) + }) + .collect::, _>>() + })?; + Ok(rows.into_iter().flatten().collect()) +} + +fn maad_pool() -> Result<&'static rayon::ThreadPool, PublishError> { + match MAAD_POOL.get_or_init(|| { + rayon::ThreadPoolBuilder::new() + .num_threads(MAAD_WORKERS) + .thread_name(|index| format!("maad-{index}")) + .build() + .map_err(|error| error.to_string()) + }) { + Ok(pool) => Ok(pool), + Err(error) => Err(PublishError::MaadPool(error.clone())), + } } fn dimensions(key: &BucketKey, scope: crate::domain::Scope) -> StatsDimensions { @@ -401,7 +433,10 @@ mod tests { use super::*; use crate::{ coverage::{BucketCoverage, CoverageState}, - domain::{AddressSide, FlowObservation, IpVersion, Scope, ScopedAddressesFact, Visibility}, + domain::{ + AddressSet, AddressSide, FlowObservation, IpVersion, Scope, ScopedAddressesFact, + Visibility, + }, storage::init_stats_tables, }; @@ -507,6 +542,63 @@ mod tests { assert_eq!(product_rows(&forward), product_rows(&reverse)); } + #[test] + fn maad_rows_preserve_scope_order_and_bytes() { + let key = BucketKey::new("r1", Granularity::FiveMinutes, 0, 300); + let first_addresses = AddressSet::from_iter([ + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 3)), + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 4)), + ]); + let second_addresses = AddressSet::from_iter([ + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 3)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 4)), + ]); + let rows = [ + AddressSetRow { + key: key.clone(), + scope: Scope::new(IpVersion::V4, Visibility::All, Visibility::All), + address_side: AddressSide::Source, + addresses: &first_addresses, + }, + AddressSetRow { + key, + scope: Scope::new(IpVersion::V4, Visibility::Literal, Visibility::All), + address_side: AddressSide::Destination, + addresses: &second_addresses, + }, + ]; + + let first = maad_rows(&rows).unwrap(); + let second = maad_rows(&rows).unwrap(); + + assert_eq!(first, second); + assert_eq!(first.len(), 6); + assert_eq!( + first + .chunks_exact(3) + .map(|rows| ( + rows[0].dimensions.src_visibility.as_str(), + rows[0].address_side.as_str(), + rows.iter() + .map(|row| row.structure_kind.as_str()) + .collect::>(), + )) + .collect::>(), + vec![ + ("all", "source", vec!["structure", "spectrum", "dimension"]), + ( + "literal", + "destination", + vec!["structure", "spectrum", "dimension"] + ), + ] + ); + } + #[test] fn rollups_keep_touched_edges_without_extending_the_input_envelope() { let raw = (0..6) diff --git a/tools/netflow-db/src/registry.rs b/tools/netflow-db/src/registry.rs index 9b00100..0c89f12 100644 --- a/tools/netflow-db/src/registry.rs +++ b/tools/netflow-db/src/registry.rs @@ -7,6 +7,7 @@ use std::{ }; use serde::{Deserialize, Serialize}; +use serde_json::Value; use thiserror::Error; #[derive(Debug, Error)] @@ -51,13 +52,19 @@ pub struct Dataset { pub source_ids: Vec, #[serde(default)] pub sources: Vec, + /// Optional product selection applied automatically by dataset-mode pipeline runs. + #[serde(default)] + pub selection: Value, } impl Dataset { pub fn validate(&mut self, repository_root: &Path) -> Result<(), RegistryError> { self.dataset_id = self.dataset_id.trim().to_owned(); - if self.dataset_id.is_empty() { - return Err(RegistryError::Invalid("dataset_id cannot be empty".into())); + if !is_safe_path_component(&self.dataset_id) { + return Err(RegistryError::Invalid(format!( + "dataset_id {:?} must be exactly one normal path component", + self.dataset_id + ))); } if self.label.trim().is_empty() { self.label = title(&self.dataset_id); @@ -167,10 +174,14 @@ impl DatasetRegistry { } pub fn load_default(repository_root: &Path) -> Result { - let configured = env::var_os("DATASETS_CONFIG_PATH") + Self::load(Self::default_path(repository_root), repository_root) + } + + /// Return the registry path selected by the environment, or the repository default. + pub fn default_path(repository_root: &Path) -> PathBuf { + env::var_os("DATASETS_CONFIG_PATH") .map(PathBuf::from) - .unwrap_or_else(|| repository_root.join("datasets.json")); - Self::load(configured, repository_root) + .unwrap_or_else(|| repository_root.join("datasets.json")) } pub fn get(&self, dataset_id: &str) -> Result<&Dataset, RegistryError> { @@ -318,4 +329,54 @@ mod tests { root.path().join("data/sample_data/netflow.sqlite") ); } + + #[test] + fn registry_rejects_dataset_ids_that_are_not_safe_path_components() { + for dataset_id in [ + "", + ".", + "..", + "../outside", + "/outside", + "nested/id", + r"nested\id", + ] { + let root = tempdir().unwrap(); + let list = root.path().join("datasets.json"); + fs::write( + &list, + serde_json::json!([{ + "dataset_id": dataset_id, + "root_path": "/captures" + }]) + .to_string(), + ) + .unwrap(); + + let error = DatasetRegistry::load(&list, root.path()).unwrap_err(); + assert!( + error.to_string().contains("dataset_id") + && error.to_string().contains("one normal path component"), + "dataset_id {dataset_id:?}: {error}" + ); + } + } + + #[test] + fn registry_accepts_hyphenated_dataset_ids() { + let root = tempdir().unwrap(); + let list = root.path().join("datasets.json"); + fs::write( + &list, + r#"[{"dataset_id":"uoregon-active-0-220","root_path":"/captures"}]"#, + ) + .unwrap(); + + let registry = DatasetRegistry::load(&list, root.path()).unwrap(); + + assert_eq!( + registry.get("uoregon-active-0-220").unwrap().db_path, + root.path().join("data/uoregon-active-0-220/netflow.sqlite") + ); + } } diff --git a/tools/netflow-db/src/storage.rs b/tools/netflow-db/src/storage.rs index d3fc8e9..f52fc96 100644 --- a/tools/netflow-db/src/storage.rs +++ b/tools/netflow-db/src/storage.rs @@ -11,7 +11,7 @@ use std::{ use fs2::FileExt; use rusqlite::{ Connection, OpenFlags, OptionalExtension, Transaction, TransactionBehavior, backup::Backup, - params, + params, params_from_iter, types::ToSql, }; use serde::Serialize; #[cfg(unix)] @@ -34,6 +34,18 @@ pub const STATS_TABLE_NAMES: [&str; 6] = [ "address_structure_stats", "bucket_coverage", ]; +const STATS_GRANULARITIES: [&str; 4] = ["5m", "30m", "1h", "1d"]; +const DAILY_PRODUCT_COMPLETION_BUCKET_SECONDS: i64 = 300; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DailyProductCompletionState { + /// The day has no completion marker or mutation tombstone and may use legacy recovery. + Missing, + /// The day has a matching completion marker with no post-certification mutation. + Clean, + /// A canonical row changed after the completion marker was written. + Dirty, +} #[derive(Debug, Error)] pub enum StorageError { @@ -157,6 +169,14 @@ pub fn database_operation_lock_path(path: impl AsRef) -> Result) -> Result { + absolute_path(path.as_ref()) +} + pub fn connect_pipeline_writer(path: impl AsRef) -> Result { connect_pipeline_writer_with_timeout(path, BUSY_TIMEOUT_MS) } @@ -239,34 +259,111 @@ fn absolute_path(path: &Path) -> Result { } else { std::env::current_dir()?.join(path) }; - let mut normalized = PathBuf::new(); - for component in expanded.components() { + + // Resolve components in operating-system order. In particular, `link/..` must apply `..` + // to the link target, rather than to the directory containing the link. Resolving the longest + // existing prefix and normalizing the rest lexically gets that case wrong. This resolver also + // expands dangling symlinks, which is needed before output alias checks can safely derive + // SQLite sidecar and operation-lock paths. + let mut pending = expanded + .components() + .map(|component| match component { + Component::Prefix(prefix) => OwnedPathComponent::Prefix(prefix.as_os_str().to_owned()), + Component::RootDir => OwnedPathComponent::Root, + Component::CurDir => OwnedPathComponent::CurDir, + Component::ParentDir => OwnedPathComponent::ParentDir, + Component::Normal(name) => OwnedPathComponent::Normal(name.to_owned()), + }) + .collect::>(); + let mut resolved = PathBuf::new(); + let mut symlink_count = 0_u8; + + while let Some(component) = pending.pop_front() { match component { - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); + OwnedPathComponent::Prefix(prefix) => resolved.push(prefix), + OwnedPathComponent::Root => resolved.push(Path::new(std::path::MAIN_SEPARATOR_STR)), + OwnedPathComponent::CurDir => {} + OwnedPathComponent::ParentDir => { + // `PathBuf::pop` already keeps an absolute path at its root. + resolved.pop(); + } + OwnedPathComponent::Normal(name) => { + let candidate = resolved.join(&name); + match fs::symlink_metadata(&candidate) { + Ok(metadata) if metadata.file_type().is_symlink() => { + symlink_count = symlink_count.checked_add(1).ok_or_else(|| { + StorageError::InvalidInput(format!( + "too many symlink components while resolving {}", + path.display() + )) + })?; + let target = fs::read_link(&candidate)?; + let parent = candidate.parent().ok_or_else(|| { + StorageError::InvalidInput(format!( + "cannot resolve symlink component in {}", + path.display() + )) + })?; + if target.is_absolute() { + resolved.clear(); + let target_components = target + .components() + .map(OwnedPathComponent::from) + .collect::>(); + for component in target_components.into_iter().rev() { + pending.push_front(component); + } + } else { + resolved = parent.to_path_buf(); + let target_components = target + .components() + .map(OwnedPathComponent::from) + .collect::>(); + for component in target_components.into_iter().rev() { + pending.push_front(component); + } + } + } + Ok(_) => resolved.push(name), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + // A missing suffix is allowed for output databases. Preserve it as an + // absolute path while still resolving any symlink components before it. + resolved.push(name); + } + Err(error) => return Err(error.into()), + } } - other => normalized.push(other.as_os_str()), } } - let mut existing = normalized.as_path(); - let mut suffix = Vec::new(); - while !existing.exists() { - let name = existing.file_name().ok_or_else(|| { - StorageError::InvalidInput(format!("cannot resolve path {}", path.display())) - })?; - suffix.push(name.to_owned()); - existing = existing.parent().ok_or_else(|| { - StorageError::InvalidInput(format!("cannot resolve path {}", path.display())) - })?; - } - let mut resolved = existing.canonicalize()?; - for component in suffix.into_iter().rev() { - resolved.push(component); - } Ok(resolved) } +#[derive(Debug)] +enum OwnedPathComponent { + Prefix(std::ffi::OsString), + Root, + CurDir, + ParentDir, + Normal(std::ffi::OsString), +} + +impl From> for OwnedPathComponent { + fn from(component: Component<'_>) -> Self { + match component { + Component::Prefix(prefix) => Self::Prefix(prefix.as_os_str().to_owned()), + Component::RootDir => Self::Root, + Component::CurDir => Self::CurDir, + Component::ParentDir => Self::ParentDir, + Component::Normal(name) => Self::Normal(name.to_owned()), + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum InputKind { Nfcapd, @@ -346,6 +443,59 @@ pub struct InputEvidenceRow { pub revision_fingerprint: Option, } +/// A processed native capture row used to rebuild bounded resume caches. +/// +/// The row carries both logical-bucket provenance and the file identity used by the content +/// digest cache. Callers should keep these rows scoped to one local day. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ProcessedNfcapdInput { + pub source_id: String, + pub bucket_start: i64, + pub input_locator: String, + pub content_fingerprint: String, + pub revision_fingerprint: String, + pub file_snapshot: Option, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct ResumeQueryCounters { + pub input_evidence: usize, + pub processed_nfcapd: usize, + pub content_fingerprint: usize, +} + +#[cfg(test)] +thread_local! { + static RESUME_QUERY_COUNTERS: std::cell::Cell = + const { std::cell::Cell::new(ResumeQueryCounters { + input_evidence: 0, + processed_nfcapd: 0, + content_fingerprint: 0, + }) }; +} + +#[cfg(test)] +fn count_resume_query(field: impl FnOnce(&mut ResumeQueryCounters)) { + RESUME_QUERY_COUNTERS.with(|counters| { + let mut value = counters.get(); + field(&mut value); + counters.set(value); + }); +} + +#[cfg(not(test))] +fn count_resume_query(_field: impl FnOnce(&mut ResumeQueryCounters)) {} + +#[cfg(test)] +pub(crate) fn reset_resume_query_counters() { + RESUME_QUERY_COUNTERS.with(|counters| counters.set(ResumeQueryCounters::default())); +} + +#[cfg(test)] +pub(crate) fn resume_query_counters() -> ResumeQueryCounters { + RESUME_QUERY_COUNTERS.with(std::cell::Cell::get) +} + impl InputEvidenceRow { #[must_use] pub fn new( @@ -479,6 +629,7 @@ pub fn query_input_evidence( source_id: &str, bucket_start: i64, ) -> Result, StorageError> { + count_resume_query(|counters| counters.input_evidence += 1); let mut statement = connection.prepare( " SELECT source_id, unit_id, bucket_start, bucket_end, input_locator, @@ -526,6 +677,162 @@ pub fn query_input_evidence( .collect() } +fn source_range_query( + source_ids: &[String], + start: i64, + end: i64, +) -> (String, Vec>) { + let placeholders = (1..=source_ids.len()) + .map(|index| format!("?{index}")) + .collect::>() + .join(", "); + let start_parameter = source_ids.len() + 1; + let end_parameter = source_ids.len() + 2; + let values = source_ids + .iter() + .map(|source_id| Box::new(source_id.clone()) as Box) + .chain([ + Box::new(start) as Box, + Box::new(end) as Box, + ]) + .collect(); + ( + format!( + "source_id IN ({placeholders}) AND bucket_start >= ?{start_parameter} AND bucket_start < ?{end_parameter}" + ), + values, + ) +} + +/// Load all native input evidence for one output's local day in one indexed range query. +pub(crate) fn query_input_evidence_range( + connection: &Connection, + source_ids: &[String], + start: i64, + end: i64, +) -> Result, StorageError> { + if source_ids.is_empty() || start >= end { + return Ok(Vec::new()); + } + count_resume_query(|counters| counters.input_evidence += 1); + let (predicate, values) = source_range_query(source_ids, start, end); + let mut statement = connection.prepare(&format!( + "SELECT source_id, unit_id, bucket_start, bucket_end, input_locator, + evidence_state, revision_fingerprint + FROM input_evidence + WHERE {predicate} + ORDER BY source_id, bucket_start, unit_id" + ))?; + let rows = statement + .query_map( + params_from_iter(values.iter().map(|value| value.as_ref())), + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, Option>(6)?, + )) + }, + )? + .collect::>>()?; + rows.into_iter() + .map( + |( + source_id, + unit_id, + bucket_start, + bucket_end, + input_locator, + evidence_state, + revision_fingerprint, + )| { + Ok(InputEvidenceRow { + source_id, + unit_id, + bucket_start, + bucket_end, + input_locator, + evidence_state: input_evidence_state(&evidence_state)?, + revision_fingerprint, + }) + }, + ) + .collect() +} + +fn file_snapshot_from_columns( + device: Option, + inode: Option, + size: Option, + mtime_ns: Option, + ctime_ns: Option, +) -> Option { + let (Some(device), Some(inode), Some(size), Some(mtime_ns), Some(ctime_ns)) = + (device, inode, size, mtime_ns, ctime_ns) + else { + return None; + }; + (size >= 0).then_some(FileSnapshot { + device: u64::from_ne_bytes(device.to_ne_bytes()), + inode: u64::from_ne_bytes(inode.to_ne_bytes()), + size: u64::try_from(size).ok()?, + mtime_ns, + ctime_ns, + }) +} + +/// Load processed nfcapd provenance and reusable file identities for one output's local day. +/// +/// The query keeps the existing `(source_id, bucket_start)` and source/bucket indexes in use. +/// Callers can derive both logical-bucket revision sets and locator/snapshot digest lookups from +/// the returned rows without issuing one query per bucket or capture. +pub(crate) fn query_processed_nfcapd_range( + connection: &Connection, + source_ids: &[String], + start: i64, + end: i64, +) -> Result, StorageError> { + if source_ids.is_empty() || start >= end { + return Ok(Vec::new()); + } + count_resume_query(|counters| counters.processed_nfcapd += 1); + let (predicate, values) = source_range_query(source_ids, start, end); + let mut statement = connection.prepare(&format!( + "SELECT source_id, bucket_start, input_locator, + content_fingerprint, revision_fingerprint, + file_device, file_inode, file_size, file_mtime_ns, file_ctime_ns + FROM processed_inputs + WHERE input_kind = 'nfcapd' AND status = 'processed' AND {predicate} + ORDER BY source_id, bucket_start, input_locator" + ))?; + statement + .query_map( + params_from_iter(values.iter().map(|value| value.as_ref())), + |row| { + Ok(ProcessedNfcapdInput { + source_id: row.get(0)?, + bucket_start: row.get(1)?, + input_locator: row.get(2)?, + content_fingerprint: row.get(3)?, + revision_fingerprint: row.get(4)?, + file_snapshot: file_snapshot_from_columns( + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + ), + }) + }, + )? + .collect::>>() + .map_err(StorageError::from) +} + pub fn init_processed_inputs_table(connection: &Connection) -> Result<(), StorageError> { connection.execute_batch( " @@ -895,6 +1202,7 @@ pub fn cached_content_fingerprint( input_locator: &str, file_snapshot: &FileSnapshot, ) -> Result, StorageError> { + count_resume_query(|counters| counters.content_fingerprint += 1); let table = if input_kind == InputKind::Csv { "processed_input_scans" } else { @@ -984,6 +1292,7 @@ pub fn nfcapd_logical_bucket_processed( bucket_start: i64, revisions: &[InputRevision], ) -> Result { + count_resume_query(|counters| counters.processed_nfcapd += 1); if revisions.is_empty() { return Ok(false); } @@ -1542,6 +1851,481 @@ pub fn init_stats_tables(connection: &Connection) -> Result<(), StorageError> { DROP INDEX IF EXISTS idx_port_count_stats_query; ", )?; + init_daily_product_completion_table(connection)?; + Ok(()) +} + +/// Initialize the transactionally maintained completion marker for native daily-active products. +/// +/// The marker is deliberately separate from the canonical product tables. Every canonical table +/// has row-level invalidation triggers so direct SQL edits, as well as normal pipeline writes, +/// cannot leave a stale marker looking complete. +pub fn init_daily_product_completion_table(connection: &Connection) -> Result<(), StorageError> { + connection.execute_batch( + " + CREATE TABLE IF NOT EXISTS daily_product_completion ( + source_id TEXT NOT NULL, + day_start INTEGER NOT NULL, + day_end INTEGER NOT NULL CHECK (day_end > day_start), + product_fingerprint TEXT NOT NULL, + run_maad INTEGER NOT NULL CHECK (run_maad IN (0, 1)), + completed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source_id, day_start) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_daily_product_completion_source_range + ON daily_product_completion(source_id, day_start, day_end); + CREATE TABLE IF NOT EXISTS daily_product_completion_dirty ( + source_id TEXT NOT NULL, + day_start INTEGER NOT NULL, + day_end INTEGER NOT NULL CHECK (day_end > day_start), + dirtied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source_id, day_start) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_daily_product_completion_dirty_source_range + ON daily_product_completion_dirty(source_id, day_start, day_end); + CREATE TABLE IF NOT EXISTS daily_product_completion_bucket_guard ( + source_id TEXT NOT NULL, + bucket_start INTEGER NOT NULL, + day_start INTEGER NOT NULL, + day_end INTEGER NOT NULL CHECK (day_end > day_start), + PRIMARY KEY (source_id, bucket_start) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_daily_product_completion_bucket_guard_day + ON daily_product_completion_bucket_guard(source_id, day_start, day_end); + ", + )?; + + // Databases created before the exact-bucket guard existed may already have completion + // markers. Only expand days whose indexed guard count is incomplete. A complete legacy day + // is therefore absent from the recursive CTE entirely, while a partially migrated day still + // gets every missing ownership row. The recursive depth is bounded by one local day. + seed_daily_product_completion_bucket_guards(connection)?; + + for table in STATS_TABLE_NAMES { + connection.execute_batch(&format!( + " + DROP TRIGGER IF EXISTS daily_product_completion_{table}_insert; + DROP TRIGGER IF EXISTS daily_product_completion_{table}_insert_fallback; + DROP TRIGGER IF EXISTS daily_product_completion_{table}_update; + DROP TRIGGER IF EXISTS daily_product_completion_{table}_update_old_fallback; + DROP TRIGGER IF EXISTS daily_product_completion_{table}_update_new_fallback; + DROP TRIGGER IF EXISTS daily_product_completion_{table}_delete; + DROP TRIGGER IF EXISTS daily_product_completion_{table}_delete_fallback; + + CREATE TRIGGER daily_product_completion_{table}_insert + AFTER INSERT ON {table} + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT guard.source_id, guard.day_start, guard.day_end + FROM daily_product_completion_bucket_guard AS guard + JOIN daily_product_completion AS completion + ON completion.source_id = guard.source_id + AND completion.day_start = guard.day_start + AND completion.day_end = guard.day_end + WHERE guard.source_id = NEW.source_id + AND guard.bucket_start = NEW.bucket_start; + END; + + CREATE TRIGGER daily_product_completion_{table}_insert_fallback + AFTER INSERT ON {table} + WHEN NOT EXISTS ( + SELECT 1 + FROM daily_product_completion_bucket_guard + WHERE source_id = NEW.source_id + AND bucket_start = NEW.bucket_start + ) + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT source_id, day_start, day_end + FROM daily_product_completion + WHERE source_id = NEW.source_id + AND day_start <= NEW.bucket_start + AND day_end > NEW.bucket_start; + END; + + CREATE TRIGGER daily_product_completion_{table}_update + AFTER UPDATE ON {table} + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT guard.source_id, guard.day_start, guard.day_end + FROM daily_product_completion_bucket_guard AS guard + JOIN daily_product_completion AS completion + ON completion.source_id = guard.source_id + AND completion.day_start = guard.day_start + AND completion.day_end = guard.day_end + WHERE guard.source_id = OLD.source_id + AND guard.bucket_start = OLD.bucket_start; + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT guard.source_id, guard.day_start, guard.day_end + FROM daily_product_completion_bucket_guard AS guard + JOIN daily_product_completion AS completion + ON completion.source_id = guard.source_id + AND completion.day_start = guard.day_start + AND completion.day_end = guard.day_end + WHERE guard.source_id = NEW.source_id + AND guard.bucket_start = NEW.bucket_start; + END; + + CREATE TRIGGER daily_product_completion_{table}_update_old_fallback + AFTER UPDATE ON {table} + WHEN NOT EXISTS ( + SELECT 1 + FROM daily_product_completion_bucket_guard + WHERE source_id = OLD.source_id + AND bucket_start = OLD.bucket_start + ) + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT source_id, day_start, day_end + FROM daily_product_completion + WHERE source_id = OLD.source_id + AND day_start <= OLD.bucket_start + AND day_end > OLD.bucket_start; + END; + + CREATE TRIGGER daily_product_completion_{table}_update_new_fallback + AFTER UPDATE ON {table} + WHEN NOT EXISTS ( + SELECT 1 + FROM daily_product_completion_bucket_guard + WHERE source_id = NEW.source_id + AND bucket_start = NEW.bucket_start + ) + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT source_id, day_start, day_end + FROM daily_product_completion + WHERE source_id = NEW.source_id + AND day_start <= NEW.bucket_start + AND day_end > NEW.bucket_start; + END; + + CREATE TRIGGER daily_product_completion_{table}_delete + AFTER DELETE ON {table} + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT guard.source_id, guard.day_start, guard.day_end + FROM daily_product_completion_bucket_guard AS guard + JOIN daily_product_completion AS completion + ON completion.source_id = guard.source_id + AND completion.day_start = guard.day_start + AND completion.day_end = guard.day_end + WHERE guard.source_id = OLD.source_id + AND guard.bucket_start = OLD.bucket_start; + END; + + CREATE TRIGGER daily_product_completion_{table}_delete_fallback + AFTER DELETE ON {table} + WHEN NOT EXISTS ( + SELECT 1 + FROM daily_product_completion_bucket_guard + WHERE source_id = OLD.source_id + AND bucket_start = OLD.bucket_start + ) + BEGIN + INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) + SELECT source_id, day_start, day_end + FROM daily_product_completion + WHERE source_id = OLD.source_id + AND day_start <= OLD.bucket_start + AND day_end > OLD.bucket_start; + END; + " + ))?; + } + Ok(()) +} + +/// Backfill exact five-minute guard ownership for legacy completion markers. +/// +/// The returned count is useful to callers that need to profile initialization. In particular, +/// a repeat initializer should return zero after all completion days have their expected guards. +fn seed_daily_product_completion_bucket_guards( + connection: &Connection, +) -> Result { + connection.execute( + "WITH RECURSIVE completion_days(source_id, day_start, day_end) AS ( + SELECT completion.source_id, completion.day_start, completion.day_end + FROM daily_product_completion AS completion + WHERE ( + SELECT COUNT(*) + FROM daily_product_completion_bucket_guard AS guard + WHERE guard.source_id = completion.source_id + AND guard.day_start = completion.day_start + AND guard.day_end = completion.day_end + ) < (completion.day_end - completion.day_start) / ?1 + ), + completion_buckets(source_id, day_start, day_end, bucket_start) AS ( + SELECT source_id, day_start, day_end, day_start + FROM completion_days + UNION ALL + SELECT source_id, day_start, day_end, + bucket_start + ?1 + FROM completion_buckets + WHERE bucket_start + ?1 < day_end + ) + INSERT OR IGNORE INTO daily_product_completion_bucket_guard ( + source_id, bucket_start, day_start, day_end + ) + SELECT source_id, bucket_start, day_start, day_end + FROM completion_buckets", + [DAILY_PRODUCT_COMPLETION_BUCKET_SECONDS], + )?; + Ok(usize::try_from(connection.changes()).unwrap_or(usize::MAX)) +} + +/// Provision exact five-minute bucket ownership before publishing canonical rows for one day. +/// +/// Rollups begin on the same five-minute grid, so the ownership rows cover every expected +/// canonical bucket start in the day while keeping trigger invalidation point-lookups bounded. +pub fn provision_daily_product_completion_bucket_guards( + connection: &Connection, + source_ids: &[String], + day_start: i64, + day_end: i64, +) -> Result<(), StorageError> { + if day_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion guard requires a non-empty day range".into(), + )); + } + let mut statement = connection.prepare_cached( + "INSERT INTO daily_product_completion_bucket_guard ( + source_id, bucket_start, day_start, day_end + ) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(source_id, bucket_start) DO UPDATE SET + day_start = excluded.day_start, + day_end = excluded.day_end", + )?; + let mut bucket_start = day_start; + while bucket_start < day_end { + let next_bucket_start = bucket_start + .checked_add(DAILY_PRODUCT_COMPLETION_BUCKET_SECONDS) + .ok_or_else(|| { + StorageError::InvalidInput( + "daily product completion guard exceeds SQLite INTEGER range".into(), + ) + })?; + for source_id in source_ids { + statement.execute(params![source_id, bucket_start, day_start, day_end])?; + } + bucket_start = next_bucket_start; + } + Ok(()) +} + +/// Add one exact ownership row for a canonical bucket that is being published outside a full-day +/// nfcapd transaction, such as a staged CSV bucket. +pub fn ensure_daily_product_completion_bucket_guard( + connection: &Connection, + source_id: &str, + bucket_start: i64, + day_start: i64, + day_end: i64, +) -> Result<(), StorageError> { + if day_start >= day_end || bucket_start < day_start || bucket_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion guard bucket must be inside a non-empty day range".into(), + )); + } + connection.execute( + "INSERT INTO daily_product_completion_bucket_guard ( + source_id, bucket_start, day_start, day_end + ) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(source_id, bucket_start) DO UPDATE SET + day_start = excluded.day_start, + day_end = excluded.day_end", + params![source_id, bucket_start, day_start, day_end], + )?; + Ok(()) +} + +/// Return the product identity currently bound to this database, if one exists. +pub fn current_product_fingerprint( + connection: &Connection, +) -> Result, StorageError> { + Ok(connection + .query_row( + "SELECT product_fingerprint FROM pipeline_product WHERE singleton = 1", + [], + |row| row.get(0), + ) + .optional()?) +} + +/// Test whether one source/day has a marker for the current product identity and MAAD setting. +pub fn daily_product_completion_matches( + connection: &Connection, + source_id: &str, + day_start: i64, + day_end: i64, + product_fingerprint: &str, + run_maad: bool, +) -> Result { + if day_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion requires a non-empty day range".into(), + )); + } + Ok(connection.query_row( + "SELECT EXISTS( + SELECT 1 FROM daily_product_completion + WHERE source_id = ?1 AND day_start = ?2 AND day_end = ?3 + AND product_fingerprint = ?4 AND run_maad = ?5 + AND NOT EXISTS( + SELECT 1 FROM daily_product_completion_dirty + WHERE source_id = ?1 AND day_start = ?2 + ) + )", + params![ + source_id, + day_start, + day_end, + product_fingerprint, + i64::from(run_maad) + ], + |row| row.get::<_, i64>(0), + )? != 0) +} + +/// Return whether a source/day is clean, dirty, or missing a completion marker. +/// +/// A dirty tombstone is checked independently of the marker so an interrupted or manually +/// altered cleanup cannot turn a previously certified day into a legacy-looking day. +pub fn daily_product_completion_state( + connection: &Connection, + source_id: &str, + day_start: i64, + day_end: i64, + product_fingerprint: &str, + run_maad: bool, +) -> Result { + if day_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion requires a non-empty day range".into(), + )); + } + let state = connection.query_row( + "SELECT CASE + WHEN EXISTS( + SELECT 1 FROM daily_product_completion_dirty + WHERE source_id = ?1 AND day_start = ?2 + ) THEN 'dirty' + WHEN EXISTS( + SELECT 1 FROM daily_product_completion + WHERE source_id = ?1 AND day_start = ?2 AND day_end = ?3 + AND product_fingerprint = ?4 AND run_maad = ?5 + ) THEN 'clean' + ELSE 'missing' + END", + params![ + source_id, + day_start, + day_end, + product_fingerprint, + i64::from(run_maad) + ], + |row| row.get::<_, String>(0), + )?; + match state.as_str() { + "clean" => Ok(DailyProductCompletionState::Clean), + "dirty" => Ok(DailyProductCompletionState::Dirty), + "missing" => Ok(DailyProductCompletionState::Missing), + _ => Err(StorageError::InvalidInput(format!( + "invalid daily product completion state: {state:?}" + ))), + } +} + +/// Publish or refresh one source/day marker. Callers must invoke this inside the same transaction +/// that publishes the day's canonical rows, rollups, and evidence/provenance. +pub fn upsert_daily_product_completion( + connection: &Connection, + source_id: &str, + day_start: i64, + day_end: i64, + product_fingerprint: &str, + run_maad: bool, +) -> Result<(), StorageError> { + if day_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion requires a non-empty day range".into(), + )); + } + // Legacy marker backfills may not pass through the day publisher. Keep their exact ownership + // map coherent before the marker becomes visible to trigger invalidation, while avoiding a + // second full-day write when the publisher already provisioned the map in this transaction. + let guard_exists = connection.query_row( + "SELECT EXISTS( + SELECT 1 FROM daily_product_completion_bucket_guard + WHERE source_id = ?1 AND bucket_start = ?2 + AND day_start = ?2 AND day_end = ?3 + )", + params![source_id, day_start, day_end], + |row| row.get::<_, i64>(0), + )? != 0; + if !guard_exists { + provision_daily_product_completion_bucket_guards( + connection, + &[source_id.to_owned()], + day_start, + day_end, + )?; + } + connection.execute( + "INSERT INTO daily_product_completion ( + source_id, day_start, day_end, product_fingerprint, run_maad + ) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(source_id, day_start) DO UPDATE SET + day_end = excluded.day_end, + product_fingerprint = excluded.product_fingerprint, + run_maad = excluded.run_maad, + completed_at = CURRENT_TIMESTAMP", + params![ + source_id, + day_start, + day_end, + product_fingerprint, + i64::from(run_maad), + ], + )?; + connection.execute( + "DELETE FROM daily_product_completion_dirty + WHERE source_id = ?1 AND day_start = ?2", + params![source_id, day_start], + )?; + Ok(()) +} + +/// Remove completion markers overlapping a deleted source/time range. +pub fn delete_daily_product_completion( + connection: &Connection, + source_ids: &[String], + start: i64, + end: i64, +) -> Result<(), StorageError> { + if start >= end { + return Err(StorageError::InvalidInput( + "daily product completion deletion requires a non-empty range".into(), + )); + } + for source_id in source_ids { + connection.execute( + "DELETE FROM daily_product_completion + WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2", + params![source_id, start, end], + )?; + connection.execute( + "DELETE FROM daily_product_completion_dirty + WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2", + params![source_id, start, end], + )?; + connection.execute( + "DELETE FROM daily_product_completion_bucket_guard + WHERE source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3", + params![source_id, start, end], + )?; + } Ok(()) } @@ -2101,18 +2885,63 @@ pub fn delete_stats_bucket_keys( Ok(()) } -#[cfg(test)] -impl StatsDimensions { - fn example() -> Self { - Self { - source_id: "r1".into(), - granularity: "5m".into(), - bucket_start: 0, - bucket_end: 300, - ip_version: 4, - src_visibility: "all".into(), - dst_visibility: "all".into(), - } +/// Remove every persisted product row and input-evidence row for a local time range. +/// +/// Callers use this inside their day transaction when a previously complete capture day must be +/// invalidated. Keeping input state with the stats deletion prevents a later resume from treating +/// stale revisions as proof that the deleted day is still published. +pub(crate) fn delete_stats_time_range( + connection: &Connection, + source_ids: &[String], + start: i64, + end: i64, +) -> Result<(), StorageError> { + if start >= end { + return Err(StorageError::InvalidInput( + "time-range deletion requires a non-empty range".into(), + )); + } + delete_daily_product_completion(connection, source_ids, start, end)?; + for table in STATS_TABLE_NAMES { + let mut statement = connection.prepare_cached(&format!( + "DELETE FROM {table} + WHERE source_id = ?1 AND granularity = ?2 + AND bucket_start >= ?3 AND bucket_start < ?4" + ))?; + for source_id in source_ids { + for granularity in STATS_GRANULARITIES { + statement.execute(params![source_id, granularity, start, end])?; + } + } + } + for source_id in source_ids { + connection.execute( + "DELETE FROM input_evidence + WHERE source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3", + params![source_id, start, end], + )?; + connection.execute( + "DELETE FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = ?1 + AND bucket_start >= ?2 AND bucket_start < ?3", + params![source_id, start, end], + )?; + } + Ok(()) +} + +#[cfg(test)] +impl StatsDimensions { + fn example() -> Self { + Self { + source_id: "r1".into(), + granularity: "5m".into(), + bucket_start: 0, + bucket_end: 300, + ip_version: 4, + src_visibility: "all".into(), + dst_visibility: "all".into(), + } } } @@ -2404,30 +3233,65 @@ fn resolved_backup_paths( Ok((source_path, target_path)) } -fn validate_database_path_separation(paths: &[&Path]) -> Result<(), StorageError> { - let mut claimed = BTreeMap::new(); +/// Reject database paths whose files, SQLite sidecars, or operation locks alias one another. +/// +/// Callers that are about to create directories, locks, or SQLite databases should invoke this +/// before their first mutation. Existing path aliases are compared by both resolved path and, +/// on Unix, device/inode identity so hard links cannot bypass the lexical checks. +pub(crate) fn validate_database_path_separation(paths: &[&Path]) -> Result<(), StorageError> { + let mut claimed = Vec::<(PathBuf, PathBuf)>::new(); + #[cfg(unix)] + let mut claimed_identities = BTreeMap::new(); for path in paths { for related in database_related_paths(path)? { - if let Some(owner) = claimed.insert(related.clone(), (*path).to_owned()) { + if let Some((owner_related, owner)) = claimed.iter().find(|(owner_related, _)| { + owner_related == &related + || owner_related.starts_with(&related) + || related.starts_with(owner_related) + }) { return Err(StorageError::InvalidInput(format!( - "database paths and their SQLite sidecar/operation-lock paths must be distinct: {} aliases {} through {}", + "database paths and their SQLite sidecar/operation-lock paths must be distinct: {} and {} overlap through {} and {}", owner.display(), path.display(), + owner_related.display(), related.display() ))); } + claimed.push((related.clone(), (*path).to_owned())); + #[cfg(unix)] + if let Some(identity) = existing_path_identity(&related)? + && let Some(owner) = claimed_identities.insert(identity, (*path).to_owned()) + { + return Err(StorageError::InvalidInput(format!( + "database paths and their SQLite sidecar/operation-lock paths must be distinct: {} aliases {} through device/inode {:?}", + owner.display(), + path.display(), + identity + ))); + } } } Ok(()) } +#[cfg(unix)] +fn existing_path_identity(path: &Path) -> Result, StorageError> { + use std::os::unix::fs::MetadataExt; + + match fs::metadata(path) { + Ok(metadata) => Ok(Some((metadata.dev(), metadata.ino()))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } +} + fn database_related_paths(path: &Path) -> Result, StorageError> { - let mut paths = vec![path.to_owned(), database_operation_lock_path(path)?]; - paths.extend(["-journal", "-wal", "-shm"].map(|suffix| sidecar_path(path, suffix))); - paths - .into_iter() - .map(|related| absolute_path(&related)) - .collect() + // Resolve the database itself before deriving related names. For a dangling `alias.sqlite` + // symlink, deriving `alias.sqlite-wal` would miss the real database's sidecar path. + let database = absolute_path(path)?; + let mut paths = vec![database.clone(), database_operation_lock_path(&database)?]; + paths.extend(["-journal", "-wal", "-shm"].map(|suffix| sidecar_path(&database, suffix))); + paths.into_iter().map(|path| absolute_path(&path)).collect() } fn acquire_database_operation_locks<'a>( @@ -2805,6 +3669,780 @@ mod tests { } } + #[test] + fn daily_product_completion_markers_are_invalidated_by_every_canonical_family() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let identity = ProductIdentity::create( + &json!({"version": 1}), + &json!({"kind": "daily_active_sources"}), + &json!({"run_maad": false}), + ) + .unwrap(); + bind_product_identity(&connection, &identity, &STATS_TABLE_NAMES).unwrap(); + + let marker = || { + upsert_daily_product_completion( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(); + assert!( + daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + }; + + let mut traffic = TrafficStatsRow::example(); + traffic.dimensions.source_id = "r1".into(); + insert_traffic_stats_rows(&connection, &[traffic.clone()]).unwrap(); + marker(); + connection.execute_batch("BEGIN IMMEDIATE").unwrap(); + connection + .execute( + "UPDATE traffic_stats SET flows = flows + 1 + WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", + [], + ) + .unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Dirty + ); + connection.execute_batch("ROLLBACK").unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Clean, + "rolling back a canonical mutation must roll back its dirty tombstone" + ); + connection + .execute( + "UPDATE traffic_stats SET flows = flows + 1 + WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", + [], + ) + .unwrap(); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + + marker(); + connection + .execute( + "DELETE FROM traffic_stats + WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", + [], + ) + .unwrap(); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + + let mut protocol = ProtocolStatsRow::example(); + protocol.dimensions.source_id = "r1".into(); + insert_protocol_stats_rows(&connection, &[protocol]).unwrap(); + marker(); + let mut address_count = AddressCountStatsRow::example(); + address_count.dimensions.source_id = "r1".into(); + insert_address_count_stats_rows(&connection, &[address_count]).unwrap(); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + + marker(); + let mut port_count = PortCountStatsRow::example(); + port_count.dimensions.source_id = "r1".into(); + insert_port_count_stats_rows(&connection, &[port_count]).unwrap(); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + + marker(); + let mut address_structure = AddressStructureStatsRow::example(); + address_structure.dimensions.source_id = "r1".into(); + insert_address_structure_stats_rows(&connection, &[address_structure]).unwrap(); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + + marker(); + insert_bucket_coverage_rows( + &connection, + &[BucketCoverageRow::new( + "r1", + "5m", + 300, + 600, + BucketCoverage::complete_unit(), + )], + ) + .unwrap(); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Dirty + ); + + upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) + .unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Clean + ); + + let mut extra = TrafficStatsRow::example(); + extra.dimensions.source_id = "r1".into(); + extra.dimensions.bucket_start = 300; + extra.dimensions.bucket_end = 600; + insert_traffic_stats_rows(&connection, &[extra]).unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Dirty + ); + + upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) + .unwrap(); + connection + .execute( + "DELETE FROM traffic_stats + WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 300", + [], + ) + .unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Dirty + ); + + upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) + .unwrap(); + connection + .execute( + "DELETE FROM daily_product_completion + WHERE source_id = 'r1' AND day_start = 0", + [], + ) + .unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Missing, + "a deliberately removed marker without a dirty tombstone is legacy evidence" + ); + + upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) + .unwrap(); + delete_daily_product_completion(&connection, &["r1".into()], 0, 86_400).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'r1' AND day_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "day deletion must clear dirty evidence with the completion marker" + ); + } + + #[test] + fn canonical_mutation_preserves_completion_marker_as_dirty_evidence() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let identity = ProductIdentity::create( + &json!({"version": 1}), + &json!({"kind": "daily_active_sources"}), + &json!({"run_maad": false}), + ) + .unwrap(); + bind_product_identity(&connection, &identity, &STATS_TABLE_NAMES).unwrap(); + + let mut traffic = TrafficStatsRow::example(); + traffic.dimensions.source_id = "r1".into(); + insert_traffic_stats_rows(&connection, &[traffic]).unwrap(); + upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) + .unwrap(); + + connection + .execute( + "UPDATE traffic_stats SET flows = flows + 1 + WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", + [], + ) + .unwrap(); + + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'r1' AND day_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "a mutation must retain the completion marker as evidence of prior certification" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_dirty + WHERE source_id = 'r1' AND day_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "a mutation must leave a dirty tombstone" + ); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + &identity.fingerprint, + false, + ) + .unwrap() + ); + } + + #[test] + fn daily_product_completion_guards_keep_normal_writes_point_lookups_and_fallback_off_grid() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let identity = ProductIdentity::create( + &json!({"version": 1}), + &json!({"kind": "daily_active_sources"}), + &json!({"run_maad": false}), + ) + .unwrap(); + bind_product_identity(&connection, &identity, &STATS_TABLE_NAMES).unwrap(); + + // Seed a cold-build-shaped history without calling the marker helper for every day. The + // idempotent schema initializer must backfill its exact ownership rows in one pass. + for source_id in ["r1", "r2"] { + for day in 0..394_i64 { + let day_start = day * 86_400; + connection + .execute( + "INSERT INTO daily_product_completion ( + source_id, day_start, day_end, product_fingerprint, run_maad + ) VALUES (?1, ?2, ?3, ?4, 0)", + params![ + source_id, + day_start, + day_start + 86_400, + identity.fingerprint + ], + ) + .unwrap(); + } + } + init_daily_product_completion_table(&connection).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_bucket_guard", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 2 * 394 * 288 + ); + + // A repeat initializer must not replay the legacy INSERT for any completed day. The + // SQLite total-change counter is sampled around the actual initializer, so trigger DDL + // and planner work cannot hide a second guard write. + let existing_guard = connection + .query_row( + "SELECT day_end FROM daily_product_completion_bucket_guard + WHERE source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + let changes_before = connection.total_changes(); + init_daily_product_completion_table(&connection).unwrap(); + let changes_after = connection.total_changes(); + assert_eq!(changes_after, changes_before); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_bucket_guard", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 2 * 394 * 288 + ); + assert_eq!( + connection + .query_row( + "SELECT day_end FROM daily_product_completion_bucket_guard + WHERE source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + existing_guard + ); + + // A partially migrated legacy day remains eligible for the bounded recursive backfill. + connection + .execute( + "INSERT INTO daily_product_completion ( + source_id, day_start, day_end, product_fingerprint, run_maad + ) VALUES ('legacy-partial', ?1, ?2, ?3, 0)", + params![394 * 86_400, 395 * 86_400, identity.fingerprint], + ) + .unwrap(); + connection + .execute( + "INSERT INTO daily_product_completion_bucket_guard ( + source_id, bucket_start, day_start, day_end + ) VALUES ('legacy-partial', ?1, ?2, ?3)", + params![394 * 86_400, 394 * 86_400, 395 * 86_400], + ) + .unwrap(); + init_daily_product_completion_table(&connection).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion_bucket_guard + WHERE source_id = 'legacy-partial'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 288 + ); + + let exact_plan = connection + .prepare( + "EXPLAIN QUERY PLAN + SELECT guard.source_id, guard.day_start, guard.day_end + FROM daily_product_completion_bucket_guard AS guard + JOIN daily_product_completion AS completion + ON completion.source_id = guard.source_id + AND completion.day_start = guard.day_start + AND completion.day_end = guard.day_end + WHERE guard.source_id = ?1 AND guard.bucket_start = ?2", + ) + .unwrap() + .query_map(params!["r1", 393 * 86_400 + 300], |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + assert!(exact_plan.contains("SEARCH guard USING PRIMARY KEY")); + assert!(exact_plan.contains("SEARCH completion USING COVERING INDEX")); + assert!(!exact_plan.contains("SCAN daily_product_completion")); + + let mut normal = TrafficStatsRow::example(); + normal.dimensions.source_id = "r1".into(); + normal.dimensions.bucket_start = 393 * 86_400 + 300; + normal.dimensions.bucket_end = normal.dimensions.bucket_start + 300; + insert_traffic_stats_rows(&connection, &[normal]).unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 393 * 86_400, + 394 * 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Dirty + ); + + upsert_daily_product_completion( + &connection, + "r1", + 393 * 86_400, + 394 * 86_400, + &identity.fingerprint, + false, + ) + .unwrap(); + let mut off_grid = TrafficStatsRow::example(); + off_grid.dimensions.source_id = "r1".into(); + off_grid.dimensions.bucket_start = 393 * 86_400 + 301; + off_grid.dimensions.bucket_end = off_grid.dimensions.bucket_start + 300; + insert_traffic_stats_rows(&connection, &[off_grid]).unwrap(); + assert_eq!( + daily_product_completion_state( + &connection, + "r1", + 393 * 86_400, + 394 * 86_400, + &identity.fingerprint, + false, + ) + .unwrap(), + DailyProductCompletionState::Dirty + ); + } + + #[test] + fn time_range_deletion_is_granularity_bounded_and_preserves_other_rows() { + let connection = Connection::open_in_memory().unwrap(); + init_stats_tables(&connection).unwrap(); + init_input_evidence_table(&connection).unwrap(); + init_processed_inputs_table(&connection).unwrap(); + + for table in STATS_TABLE_NAMES { + let mut statement = connection + .prepare(&format!( + "EXPLAIN QUERY PLAN DELETE FROM {table} + WHERE source_id = ?1 AND granularity = ?2 + AND bucket_start >= ?3 AND bucket_start < ?4" + )) + .unwrap(); + let plan = statement + .query_map(params!["r1", "5m", 0_i64, 86_400_i64], |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + for clause in [ + "USING PRIMARY KEY", + "source_id=?", + "granularity=?", + "bucket_start>?", + "bucket_start Result<(), StorageError> { + let locator = format!( + "/captures/{}-{source_id}-{bucket_start}", + input_kind.as_str() + ); + let revision = + InputRevision::create(input_kind.as_str(), &locator, "content", "decoder")?; + upsert_input_bucket( + &connection, + &InputBucket { + input_kind, + input_locator: locator.clone(), + scan_locator: locator.clone(), + source_id: source_id.into(), + bucket_start, + bucket_end: bucket_start + 300, + revision: revision.clone(), + file_snapshot: None, + }, + false, + )?; + mark_input_bucket_status( + &connection, + input_kind, + &locator, + source_id, + bucket_start, + InputStatus::Processed, + &revision, + None, + ) + }; + + for &(source_id, bucket_start) in + &[("r1", 0_i64), ("r1", 86_400), ("r2", 0), ("r2", 86_400)] + { + insert_processed(source_id, bucket_start, InputKind::Nfcapd).unwrap(); + } + insert_processed("r1", 0, InputKind::Csv).unwrap(); + + upsert_daily_product_completion(&connection, "r1", 0, 86_400, "product", false).unwrap(); + upsert_daily_product_completion(&connection, "r2", 0, 86_400, "product", false).unwrap(); + + delete_stats_time_range(&connection, &["r1".into()], 0, 86_400).unwrap(); + + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'r1'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "deleting a day must remove its completion marker" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'r2'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "deleting one source/day must preserve another source marker" + ); + + for table in STATS_TABLE_NAMES { + let count = connection + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + assert_eq!(count, 12, "{table}"); + assert_eq!( + connection + .query_row( + &format!( + "SELECT COUNT(*) FROM {table} + WHERE source_id = 'r1' AND bucket_start = 0" + ), + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "the requested source and range should be deleted from {table}" + ); + } + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM input_evidence", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 3 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM input_evidence + WHERE source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 4 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM processed_inputs + WHERE input_kind = 'csv' AND source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + } + #[test] fn stats_timeseries_indexes_apply_equality_filters_before_bucket_range() { let connection = Connection::open_in_memory().unwrap(); @@ -3254,6 +4892,95 @@ mod tests { assert_eq!(fs::read(source).unwrap(), original); } + #[cfg(unix)] + #[test] + fn canonical_path_applies_parent_components_after_symlink_targets() { + use std::os::unix::fs::symlink; + + let directory = tempdir().unwrap(); + let real = directory.path().join("real"); + fs::create_dir_all(real.join("child")).unwrap(); + let alias = directory.path().join("alias"); + symlink(real.join("child"), &alias).unwrap(); + + assert_eq!(canonical_path(alias.join("..").as_path()).unwrap(), real); + } + + #[cfg(unix)] + #[test] + fn separation_resolves_dangling_database_sidecar_and_lock_aliases() { + use std::os::unix::fs::symlink; + + let directory = tempdir().unwrap(); + let database = directory.path().join("target.sqlite"); + let database_alias = directory.path().join("database-alias.sqlite"); + symlink(&database, &database_alias).unwrap(); + assert!( + validate_database_path_separation(&[database.as_path(), database_alias.as_path()]) + .is_err() + ); + + let sidecar = sidecar_path(&database, "-wal"); + let sidecar_alias = directory.path().join("sidecar-alias.sqlite"); + symlink(&sidecar, &sidecar_alias).unwrap(); + assert!( + validate_database_path_separation(&[database.as_path(), sidecar_alias.as_path()]) + .is_err() + ); + + let lock = database_operation_lock_path(&database).unwrap(); + let lock_alias = directory.path().join("lock-alias.sqlite"); + symlink(&lock, &lock_alias).unwrap(); + assert!( + validate_database_path_separation(&[database.as_path(), lock_alias.as_path()]).is_err() + ); + } + + #[test] + fn separation_rejects_database_ancestor_and_descendant_paths() { + let directory = tempdir().unwrap(); + let database = directory.path().join("first.sqlite"); + let nested_database = database.join("second.sqlite"); + assert!( + validate_database_path_separation(&[database.as_path(), nested_database.as_path()]) + .unwrap_err() + .to_string() + .contains("must be distinct") + ); + + let sidecar = sidecar_path(&database, "-wal"); + let nested_lock = sidecar.join("operation.lock"); + assert!( + validate_database_path_separation(&[database.as_path(), nested_lock.as_path()]) + .unwrap_err() + .to_string() + .contains("must be distinct") + ); + assert!(!database.exists()); + assert!(!nested_database.exists()); + assert!(!nested_lock.exists()); + } + + #[cfg(unix)] + #[test] + fn separation_rejects_ancestor_paths_through_dangling_symlinks() { + use std::os::unix::fs::symlink; + + let directory = tempdir().unwrap(); + let real_database = directory.path().join("real.sqlite"); + let alias = directory.path().join("alias.sqlite"); + symlink(&real_database, &alias).unwrap(); + let nested = real_database.join("second.sqlite"); + + let error = + validate_database_path_separation(&[alias.as_path(), nested.as_path()]).unwrap_err(); + + assert!(error.to_string().contains("must be distinct")); + assert!(alias.is_symlink()); + assert!(!real_database.exists()); + assert!(!nested.exists()); + } + #[test] fn failed_transaction_rolls_back_all_persistence_changes() { let mut connection = Connection::open_in_memory().unwrap(); diff --git a/tools/netflow-db/tests/pipeline_cli_help.rs b/tools/netflow-db/tests/pipeline_cli_help.rs new file mode 100644 index 0000000..0da0c38 --- /dev/null +++ b/tools/netflow-db/tests/pipeline_cli_help.rs @@ -0,0 +1,317 @@ +use std::{fs, process::Command}; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +#[cfg(unix)] +use std::os::unix::fs::symlink; + +use rusqlite::Connection; +use tempfile::tempdir; + +#[test] +fn pipeline_help_explains_repeated_dataset_mode() { + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args(["pipeline", "--help"]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let help = String::from_utf8(output.stdout).unwrap(); + assert!( + help.contains("Repeat --dataset for two or more values"), + "{help}" + ); + assert!( + help.contains("coordinated fixed daily-active subset run"), + "{help}" + ); + assert!( + help.contains("one value keeps the normal single-dataset path"), + "{help}" + ); +} + +#[test] +fn pipeline_repeated_dataset_uses_isolated_registry_and_outputs() { + let temporary = tempdir().unwrap(); + let capture_root = temporary.path().join("captures"); + fs::create_dir_all(capture_root.join("shared")).unwrap(); + let registry_path = temporary.path().join("registry.json"); + let first_database = temporary.path().join("first.sqlite"); + let second_database = temporary.path().join("second.sqlite"); + let nfdump = temporary.path().join("nfdump"); + let empty_stream = temporary.path().join("empty.stream"); + fs::write( + &empty_stream, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &nfdump, + format!("#!/bin/sh\ncat '{}'\n", empty_stream.display()), + ) + .unwrap(); + #[cfg(unix)] + fs::set_permissions(&nfdump, fs::Permissions::from_mode(0o755)).unwrap(); + let registry = serde_json::json!({ + "datasets": [ + { + "dataset_id": "first", + "root_path": capture_root, + "db_path": first_database, + "source_ids": ["shared"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "10.0.0.0/16" + } + }, + { + "dataset_id": "second", + "root_path": capture_root, + "db_path": second_database, + "source_ids": ["shared"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "10.0.0.0/16" + } + } + ] + }); + fs::write(®istry_path, serde_json::to_vec(®istry).unwrap()).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--dataset", + "first", + "--dataset", + "second", + "--start-date", + "2025-01-01", + "--end-date", + "2025-01-02", + "--datasets", + registry_path.to_str().unwrap(), + "--no-maad", + "--nfdump", + nfdump.to_str().unwrap(), + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + for database in [&first_database, &second_database] { + assert!( + database.is_file(), + "missing coordinated output {database:?}" + ); + let connection = Connection::open(database).unwrap(); + let selection: String = connection + .query_row( + "SELECT selection_json FROM pipeline_product WHERE singleton = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(selection.contains("daily_active_sources"), "{selection}"); + } +} + +#[cfg(unix)] +#[test] +fn pipeline_rejects_an_inaccessible_path_candidate_before_output_setup() { + let temporary = tempdir().unwrap(); + let first_path = temporary.path().join("first-path"); + let second_path = temporary.path().join("second-path"); + fs::create_dir_all(&first_path).unwrap(); + fs::create_dir_all(&second_path).unwrap(); + + // The owner has no execute permission, while another class does. Checking mode bits alone + // incorrectly treats this as the first PATH candidate for the current process. + let inaccessible = first_path.join("nfdump"); + fs::write(&inaccessible, b"inaccessible candidate").unwrap(); + fs::set_permissions(&inaccessible, fs::Permissions::from_mode(0o001)).unwrap(); + + let database = temporary.path().join("pipeline.sqlite"); + let sidecar = database.with_file_name("pipeline.sqlite-wal"); + let sentinel = b"do not overwrite this executable-related sentinel"; + fs::write(&sidecar, sentinel).unwrap(); + fs::set_permissions(&sidecar, fs::Permissions::from_mode(0o755)).unwrap(); + symlink(&sidecar, second_path.join("nfdump")).unwrap(); + + let capture_root = temporary.path().join("captures"); + fs::create_dir_all(capture_root.join("edge")).unwrap(); + let config = temporary.path().join("pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "nfdump": "nfdump", + "inputs": [{ + "input_kind": "nfcapd_tree", + "root_path": capture_root, + "source_ids": ["edge"], + "start_date": "2025-01-01", + "end_date": "2025-01-02" + }] + })) + .unwrap(), + ) + .unwrap(); + let path = std::env::join_paths([&first_path, &second_path]).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("PATH", path) + .output() + .unwrap(); + + assert!(!output.status.success(), "pipeline unexpectedly succeeded"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("aliases pipeline control path"), + "stderr={stderr}" + ); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel); + assert!( + !database.exists(), + "output database was created after rejection" + ); + assert!( + !temporary + .path() + .join(".pipeline.sqlite.operation.lock") + .exists(), + "output operation lock was created after rejection" + ); +} + +#[test] +fn csv_pipeline_does_not_require_nfdump_from_path() { + let temporary = tempdir().unwrap(); + let empty_path = temporary.path().join("empty-path"); + fs::create_dir(&empty_path).unwrap(); + let csv = temporary.path().join("flows.csv"); + let mapping = temporary.path().join("mapping.json"); + let database = temporary.path().join("csv.sqlite"); + fs::write(&csv, "received,src,dst\n0,192.0.2.1,198.51.100.1\n").unwrap(); + fs::write( + &mapping, + serde_json::to_vec(&serde_json::json!({ + "timestamp_format": "unix", + "timestamp_timezone": "UTC", + "columns": { + "time_received": "received", + "src_ip": "src", + "dst_ip": "dst" + }, + "source_id": {"value": "edge"} + })) + .unwrap(), + ) + .unwrap(); + let config = temporary.path().join("csv-pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "inputs": [{ + "input_kind": "csv", + "path": csv, + "mapping_path": mapping + }] + })) + .unwrap(), + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("PATH", empty_path) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("Published five-minute buckets: 1\n"), + "stdout={stdout}" + ); + assert!(database.is_file()); +} + +#[test] +fn native_pipeline_requires_nfdump_before_output_setup() { + let temporary = tempdir().unwrap(); + let empty_path = temporary.path().join("empty-path"); + fs::create_dir(&empty_path).unwrap(); + let capture_root = temporary.path().join("captures"); + fs::create_dir_all(capture_root.join("edge")).unwrap(); + let database = temporary.path().join("native.sqlite"); + let config = temporary.path().join("native-pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "inputs": [{ + "input_kind": "nfcapd_tree", + "root_path": capture_root, + "source_ids": ["edge"], + "start_date": "2025-01-01", + "end_date": "2025-01-02" + }] + })) + .unwrap(), + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("PATH", empty_path) + .output() + .unwrap(); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("cannot resolve bare nfdump executable"), + "stderr={stderr}" + ); + assert!(!database.exists()); +} From 825c38cc3364546a502ba3aabffc6b803a347957 Mon Sep 17 00:00:00 2001 From: flamboh Date: Fri, 28 Aug 2026 00:40:26 -0700 Subject: [PATCH 2/2] refactor(netflow-db): unify coordinated runs on the single-output day loop Collapse the duplicated coordinated day path onto the shared per-day publication machinery (coordinated mode drives N product sinks; single output is N=1), enforce compatibility once via a CompatiblePlan witness, reduce resume to per-day completion markers, and replace the alias machinery with canonical-path overlap rejection. Fix the CI-only test failures with self-contained nfdump fixtures, add strict registry deserialization, and trim the setup docs to the essentials. --- docs/code/pipeline-contract.md | 4 + docs/user/setup-pipeline.md | 612 +- shell.nix | 2 - tools/netflow-db/src/domain.rs | 60 +- tools/netflow-db/src/ingest.rs | 135 +- tools/netflow-db/src/maad.rs | 38 +- tools/netflow-db/src/nfdump.rs | 23 +- tools/netflow-db/src/pipeline.rs | 15295 +++--------------- tools/netflow-db/src/provenance.rs | 33 - tools/netflow-db/src/publish.rs | 28 +- tools/netflow-db/src/registry.rs | 29 + tools/netflow-db/src/storage.rs | 1489 +- tools/netflow-db/tests/pipeline_cli_help.rs | 106 - 13 files changed, 2806 insertions(+), 15048 deletions(-) diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 72f4da3..ac47884 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -65,6 +65,10 @@ Each subset keeps its own immutable product database, product identity, transact and MAAD configuration. Active sets are still resolved independently. Overlapping subsets may both receive the same qualifying flow. +Outputs commit sequentially rather than as one cross-database transaction. If the process stops +between product commits, sibling databases can differ by at most the local day that was in flight. +The next run sees the missing completion marker and rebuilds that day for each unfinished product. + ## Input identity Each input records an exact revision. The revision contains a SHA-256 content identity and a canonical decoder fingerprint. diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 9298eb9..291b230 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -29,11 +29,6 @@ On macOS, bind mounts over large capture trees are slower than native filesystem `scripts/netflow-db.sh` runs the crate with `cargo run --locked --release`, which compiles it when necessary. Set `NETFLOW_DB_BIN` to run a prebuilt binary instead. -The native gate below also needs `jq`, Python 3, GNU `time` with verbose (`-v`) support, and a -running per-user `systemd` manager on a cgroup-v2 host. `jq`, Python, and GNU `time` are supplied -on `PATH` by `shell.nix`; `systemd-run` and `systemctl` normally come from the host. Run the gate -from `nix-shell shell.nix` (or an equivalent environment). - nfcapd input also needs the pinned ATLANTIS nfdump fork. A system nfdump installation does not work: the pipeline uses an output mode that only the fork has. CSV input does not need nfdump. 1. Initialize the Git submodules. @@ -71,596 +66,41 @@ If a command fails, read [Troubleshooting](troubleshooting.md). ## Process coordinated subsets -Repeat `--dataset` for two or more registry entries that describe subsets of one nfcapd tree: +Repeat `--dataset` for two or more registry entries that select subsets of one nfcapd tree. +Native runs must name the pinned ATLANTIS nfdump fork explicitly: ```bash ./scripts/netflow-db.sh pipeline \ + --nfdump target/nfdump/libexec/nfdump \ --dataset campus-a \ --dataset campus-b \ --start-date \ --end-date ``` -Multi mode supports `daily_active_sources` only. Put that selection in every registry entry. The -command infers each root and source configuration from its entry. It does not use a parent or -`source_dataset` relation. - -Use the same root and logical source layout for all entries. Use whole local-day dates. Do not pass -`--config`, `--database-path`, partial time bounds, or selection flags in multi mode. The command -uses each registry entry's database path. - -The pipeline performs one shared daily eligibility scan and one shared publication scan. It still -needs two physical phases because it must qualify sources over the whole local day first. A missing -required capture makes that local day incomplete for every subset. - -Each subset keeps its own product database, identity, transactions, resume state, and MAAD settings. -The active set is independent for each subset, so an overlapping subset may receive the same flow. - -### Gate a coordinated run - -Before a full MAAD run, gate one local day with temporary output paths. Keep the gate directory -under `data/`: the Docker wrapper mounts that directory at `/workspace/data`. Copy the registry and -rewrite only its `db_path` values; keep the roots, sources, and selections unchanged: - -```bash -mkdir -p data -gate_dir=$(mktemp -d data/netflow-gate.XXXXXX) -gate_root=$(realpath "$gate_dir") -gate_date=2025-06-01 # replace with one complete local day -jq --arg dir "$gate_dir" \ - '(if type == "array" then {datasets: .} - elif type == "object" then . - else error("registry must be an array or an object with a datasets array") - end) - | (.datasets // .) as $entries - | if ($entries | type) != "array" then - error("registry must be an array or an object with a datasets array") - elif ($entries | length) == 0 then - error("registry cannot be empty") - elif any($entries[]; type != "object") then - error("registry entries must be objects") - else - $entries - | to_entries - | map(.value.db_path = ($dir + "/" + ((.key + 1) | tostring) + ".sqlite") | .value) - end' \ - datasets.json >"$gate_dir/datasets.json" - -registry_db_paths() { - local registry=$1 - jq -er ' - (if type == "array" then {datasets: .} - elif type == "object" then . - else error("registry must be an array or an object with a datasets array") - end) - | (.datasets // .) as $entries - | if ($entries | type) != "array" then - error("registry must be an array or an object with a datasets array") - elif any($entries[]; type != "object") then - error("registry entries must be objects") - else - ["campus-a", "campus-b"][] as $dataset_id - | ($entries | map(select(.dataset_id == $dataset_id))) as $matches - | if ($matches | length) != 1 then - error("registry must contain exactly one entry for " + $dataset_id) - elif (($matches[0].db_path | type) != "string") then - error("db_path for " + $dataset_id + " must be a string") - elif ($matches[0].db_path | length) == 0 then - error("db_path for " + $dataset_id + " cannot be empty") - else - $matches[0].db_path - end - end - ' "$registry" -} - -mapfile -t gate_db_paths < <(registry_db_paths "$gate_dir/datasets.json") -if (( ${#gate_db_paths[@]} != 2 )); then - echo "the gate requires campus-a and campus-b database paths" >&2 - exit 1 -fi -for path in "${gate_db_paths[@]}"; do - resolved=$(realpath -m -- "$path") - case "$resolved" in - "$gate_root"/*.sqlite) ;; - *) echo "temporary database escaped gate directory: $path" >&2; exit 1 ;; - esac -done -``` - -Run the same MAAD-enabled native command twice for one complete local day, saving separate cold and -no-op resource/profile logs. The cold invocation checks positive publication cardinality; the -second invocation checks the resume path and zero new publications. Record the cgroup aggregate -peak, elapsed wall time, every `netflow_db::profile` phase, and the byte sizes of each temporary -SQLite output and its `-wal` file. GNU `time -v` still writes its per-process -`Maximum resident set size` for diagnostics, but that value is not a gate: it is not the sum of -the pipeline's concurrent nfdump children. - -The native gate puts the whole pipeline process tree in a transient per-user systemd scope. Its -16 GiB `MemoryMax` (`16777216` KiB) is an aggregate cgroup-v2 limit, and the gate reads the scope's -`memory.peak` and fails closed if peak accounting or the cgroup's OOM counters cannot be read. -This leaves a safe margin below roughly 20 GiB available on Barbera without requiring root. Set -`NETFLOW_GATE_MAX_MEMORY_KIB`, `NETFLOW_GATE_COLD_MAX_ELAPSED_SECONDS`, -`NETFLOW_GATE_FULL_COLD_MAX_ELAPSED_SECONDS`, `NETFLOW_GATE_FULL_NOOP_MAX_ELAPSED_SECONDS`, or -`NETFLOW_GATE_SPACE_HEADROOM_PERCENT` before running the block to change the ceilings. The -full-cold budget defaults to 30 days (`2592000` seconds); the gate conservatively projects two -times the measured one-day cold elapsed time across 394 days before launching it. The space -headroom defaults to 100% (a 2x projection). These positive-integer defaults are fail-closed: -raise them explicitly only when the host and run window justify it. -The gate refuses hosts without cgroup v2 or a usable user systemd manager; it does not silently -fall back to GNU `time` or an incomplete process-tree RSS sample. - -Run this Bash block from that Nix shell; it keeps the one-day and full-history phases in the same -gated session: - -```bash -set -euo pipefail -pipeline=(./scripts/netflow-db.sh) -time_bin="$(type -P time || true)" -if [[ -z "$time_bin" ]] || ! "$time_bin" -v true >/dev/null 2>&1; then - echo "GNU time with -v is required on PATH; enter nix-shell shell.nix" >&2 - exit 1 -fi -systemd_run_bin="$(type -P systemd-run || true)" -systemctl_bin="$(type -P systemctl || true)" -true_bin="$(type -P true || true)" -sleep_bin="$(type -P sleep || true)" -if [[ -z "$systemd_run_bin" || -z "$systemctl_bin" || -z "$true_bin" ]] || - ! [[ "$(stat -fc %T -- /sys/fs/cgroup 2>/dev/null || true)" == cgroup2fs ]] || - ! "$systemd_run_bin" --user --scope --quiet -p MemoryMax=64M "$true_bin" >/dev/null 2>&1; then - echo "a running user systemd manager with cgroup v2 is required for the aggregate memory gate" >&2 - exit 1 -fi -runtime_probe_unit="netflow-gate-runtime-probe-$$.scope" -if [[ -z "$sleep_bin" ]] || - "$systemd_run_bin" --user --scope --quiet --collect --unit="$runtime_probe_unit" \ - -p MemoryMax=64M -p RuntimeMaxSec=1s -- "$sleep_bin" 5 >/dev/null 2>&1; then - echo "a user-systemd scope with RuntimeMaxSec is required for the elapsed-time gate" >&2 - exit 1 -fi -max_memory_kib_limit="${NETFLOW_GATE_MAX_MEMORY_KIB:-16777216}" -cold_elapsed_limit_seconds="${NETFLOW_GATE_COLD_MAX_ELAPSED_SECONDS:-1800}" -full_cold_elapsed_limit_seconds="${NETFLOW_GATE_FULL_COLD_MAX_ELAPSED_SECONDS:-2592000}" -full_noop_elapsed_limit_seconds="${NETFLOW_GATE_FULL_NOOP_MAX_ELAPSED_SECONDS:-1800}" -space_headroom_percent="${NETFLOW_GATE_SPACE_HEADROOM_PERCENT:-100}" -if ! [[ "$max_memory_kib_limit" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$cold_elapsed_limit_seconds" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$full_cold_elapsed_limit_seconds" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$full_noop_elapsed_limit_seconds" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$space_headroom_percent" =~ ^[1-9][0-9]*$ ]]; then - echo "native gate ceilings must be positive integers" >&2 - exit 1 -fi - -parse_elapsed_seconds() { - awk ' - /^[[:space:]]*Elapsed \(wall clock\) time \(h:mm:ss or m:ss\):[[:space:]]*/ { - value = $0 - sub(/^.*\):[[:space:]]*/, "", value) - gsub(/[[:space:]]/, "", value) - matches++ - if (value !~ /^[0-9]+:[0-9][0-9](:[0-9][0-9])?([.][0-9]+)?$/) { - invalid = 1 - next - } - fields = split(value, part, ":") - if (fields == 2) { - if (part[2] + 0 >= 60) invalid = 1 - else seconds = (part[1] * 60) + part[2] - } else if (fields == 3) { - if (part[2] + 0 >= 60 || part[3] + 0 >= 60) invalid = 1 - else seconds = (part[1] * 3600) + (part[2] * 60) + part[3] - } else { - invalid = 1 - } - } - END { - if (matches != 1 || invalid) exit 1 - printf "%.6f\n", seconds - } - ' "$1" -} - -assert_resources() { - local log_name=$1 - local elapsed_limit_seconds=$2 - local log_path="$gate_dir/$log_name" - local peak_bytes peak_kib elapsed_seconds - if ! peak_bytes="$(cat "$gate_dir/$log_name.cgroup-peak-bytes" 2>/dev/null)" || - ! [[ "$peak_bytes" =~ ^[1-9][0-9]*$ ]]; then - echo "missing or malformed aggregate cgroup peak in $log_path" >&2 - return 1 - fi - if ! elapsed_seconds="$(parse_elapsed_seconds "$log_path")"; then - echo "missing or malformed elapsed wall time in $log_path" >&2 - return 1 - fi - peak_kib=$(( (peak_bytes + 1023) / 1024 )) - if (( peak_bytes > max_memory_kib_limit * 1024 )); then - echo "aggregate memory limit exceeded in $log_path: ${peak_kib} KiB > ${max_memory_kib_limit} KiB" >&2 - return 1 - fi - if ! awk -v actual="$elapsed_seconds" -v limit="$elapsed_limit_seconds" ' - BEGIN { - if (actual !~ /^[0-9]+([.][0-9]+)?$/ || actual > limit) exit 1 - } - '; then - echo "elapsed limit exceeded in $log_path: ${elapsed_seconds}s > ${elapsed_limit_seconds}s" >&2 - return 1 - fi - printf 'Resource gate %s: %s KiB aggregate cgroup peak, %ss elapsed\n' \ - "$log_name" "$peak_kib" "$elapsed_seconds" -} - -stop_gate_unit() { - local unit=$1 - "$systemctl_bin" --user stop "$unit" >/dev/null 2>&1 || true -} - -gate_pid_is_running() { - local pid=$1 - local state - if ! [[ "$pid" =~ ^[1-9][0-9]*$ ]] || - ! state="$(awk '$1 == "State:" { print $2; exit }' "/proc/$pid/status" 2>/dev/null)"; then - return 1 - fi - [[ "$state" =~ ^[A-Z]$ && "$state" != Z ]] -} - -monitor_cgroup_peak() { - local cgroup_dir=$1 - local peak_path=$2 - local launch_pid=$3 - local unit=$4 - local peak_bytes events_values oom oom_kill oom_group_kill saw_sample=0 - while true; do - if [[ ! -r "$cgroup_dir/memory.peak" || ! -r "$cgroup_dir/memory.events" ]]; then - if gate_pid_is_running "$launch_pid"; then - stop_gate_unit "$unit" - return 1 - fi - break - fi - if ! peak_bytes="$(cat "$cgroup_dir/memory.peak")" || - ! [[ "$peak_bytes" =~ ^[0-9]+$ ]]; then - if gate_pid_is_running "$launch_pid"; then - stop_gate_unit "$unit" - return 1 - fi - break - fi - if ! events_values="$(awk ' - $1 == "oom" { - oom_count++ - if (NF != 2 || $2 !~ /^[0-9]+$/) invalid = 1 - else oom_value = $2 - next - } - $1 == "oom_kill" { - oom_kill_count++ - if (NF != 2 || $2 !~ /^[0-9]+$/) invalid = 1 - else oom_kill_value = $2 - next - } - $1 == "oom_group_kill" { - oom_group_kill_count++ - if (NF != 2 || $2 !~ /^[0-9]+$/) invalid = 1 - else oom_group_kill_value = $2 - next - } - END { - if (invalid || oom_count != 1 || oom_kill_count != 1 || oom_group_kill_count != 1) exit 1 - printf "%s %s %s\n", oom_value, oom_kill_value, oom_group_kill_value - } - ' "$cgroup_dir/memory.events")"; then - if gate_pid_is_running "$launch_pid"; then - stop_gate_unit "$unit" - return 1 - fi - break - fi - read -r oom oom_kill oom_group_kill <<<"$events_values" - if [[ "$oom" != 0 || "$oom_kill" != 0 || "$oom_group_kill" != 0 ]]; then - if gate_pid_is_running "$launch_pid"; then - stop_gate_unit "$unit" - fi - return 1 - fi - printf '%s\n' "$peak_bytes" >"$peak_path" - saw_sample=1 - gate_pid_is_running "$launch_pid" || break - sleep 0.1 - done - (( saw_sample == 1 )) -} - -gate_active_unit= -gate_active_launch_pid= -gate_active_monitor_pid= -cleanup_gate_processes() { - local unit=$gate_active_unit - local launch_pid=$gate_active_launch_pid - local monitor_pid=$gate_active_monitor_pid - gate_active_unit= - gate_active_launch_pid= - gate_active_monitor_pid= - - if [[ -n "$unit" ]]; then - stop_gate_unit "$unit" - fi - if [[ -n "$monitor_pid" ]] && gate_pid_is_running "$monitor_pid"; then - kill "$monitor_pid" >/dev/null 2>&1 || true - fi - if [[ -n "$monitor_pid" ]]; then - wait "$monitor_pid" >/dev/null 2>&1 || true - fi - if [[ -n "$launch_pid" ]]; then - wait "$launch_pid" >/dev/null 2>&1 || true - fi -} - -gate_error() { - local status=$? - cleanup_gate_processes - trap - ERR - exit "$status" -} - -gate_signal() { - local status=$1 - cleanup_gate_processes - trap - ERR - exit "$status" -} - -trap gate_error ERR -trap 'gate_signal 129' HUP -trap 'gate_signal 130' INT -trap 'gate_signal 143' TERM -trap cleanup_gate_processes EXIT - -run_gate() { - local log_name=$1 - local published_pattern=$2 - local start_date="${3:-$gate_date}" - local end_date="${4:-$gate_date}" - local registry="${5:-$gate_dir/datasets.json}" - local elapsed_limit_seconds="${6:-$cold_elapsed_limit_seconds}" - local unit="netflow-gate-${log_name%.log}-$$.scope" - local launch_pid pipeline_status monitor_pid monitor_status control_group cgroup_dir= - gate_active_unit=$unit - RUST_LOG=netflow_db::profile=info "$time_bin" -v \ - "$systemd_run_bin" --user --scope --quiet --collect --unit="$unit" \ - -p "MemoryMax=${max_memory_kib_limit}K" \ - -p "RuntimeMaxSec=${elapsed_limit_seconds}s" -- \ - "${pipeline[@]}" pipeline \ - --datasets "$registry" \ - --dataset campus-a --dataset campus-b \ - --start-date "$start_date" --end-date "$end_date" \ - --require-complete \ - >"$gate_dir/$log_name" 2>&1 & - launch_pid=$! - gate_active_launch_pid=$launch_pid - for _ in {1..300}; do - control_group="$("$systemctl_bin" --user show "$unit" --property=ControlGroup --value 2>/dev/null || true)" - if [[ "$control_group" == /* && "$control_group" != *..* && - "$control_group" != *$'\n'* && "$control_group" != *$'\t'* ]]; then - cgroup_dir="/sys/fs/cgroup$control_group" - if [[ -r "$cgroup_dir/memory.peak" && -r "$cgroup_dir/memory.events" ]]; then - break - fi - cgroup_dir= - fi - gate_pid_is_running "$launch_pid" || break - sleep 0.1 - done - if [[ -z "$cgroup_dir" ]]; then - echo "could not locate aggregate cgroup for $unit" >&2 - cleanup_gate_processes - return 1 - fi - monitor_cgroup_peak "$cgroup_dir" "$gate_dir/$log_name.cgroup-peak-bytes" "$launch_pid" "$unit" & - monitor_pid=$! - gate_active_monitor_pid=$monitor_pid - if wait "$monitor_pid"; then monitor_status=0; else monitor_status=$?; fi - if (( monitor_status != 0 )); then - stop_gate_unit "$unit" - fi - if wait "$launch_pid"; then pipeline_status=0; else pipeline_status=$?; fi - gate_active_unit= - gate_active_launch_pid= - gate_active_monitor_pid= - if (( pipeline_status != 0 || monitor_status != 0 )); then - echo "aggregate memory gate failed for $log_name (pipeline=$pipeline_status monitor=$monitor_status)" >&2 - return 1 - fi - cat "$gate_dir/$log_name" - grep -Eq '^Five-minute coverage: [1-9][0-9]* complete' "$gate_dir/$log_name" - grep -Eq '^Five-minute coverage: [1-9][0-9]* complete, 0 partial, 0 unknown$' \ - "$gate_dir/$log_name" - grep -Eq "$published_pattern" "$gate_dir/$log_name" - assert_resources "$log_name" "$elapsed_limit_seconds" -} -run_gate one-day-cold.log '^Published five-minute buckets: [1-9][0-9]*$' \ - "$gate_date" "$gate_date" "$gate_dir/datasets.json" "$cold_elapsed_limit_seconds" - -mapfile -t destination_db_paths < <(registry_db_paths datasets.json) -if (( ${#destination_db_paths[@]} != 2 )); then - echo "the gate requires campus-a and campus-b database paths" >&2 - exit 1 -fi - -destination_related_paths() { - local database=$1 - local resolved path parent name suffix - if ! resolved="$(realpath -m -- "$database")"; then - echo "could not resolve final database path: $database" >&2 - return 1 - fi - for path in "$database" "$resolved"; do - parent=${path%/*} - [[ "$parent" == "$path" ]] && parent=. - name=${path##*/} - printf '%s\n' "$path" - for suffix in -journal -wal -shm; do - printf '%s%s\n' "$path" "$suffix" - done - printf '%s/.%s.operation.lock\n' "$parent" "$name" - done -} - -assert_fresh_destination_paths() { - local database=$1 - local related - while IFS= read -r related; do - if [[ -e "$related" || -L "$related" ]]; then - echo "full-cold requires a fresh final database path; found existing database, SQLite sidecar, or operation lock: $related" >&2 - echo "Remove that path and retry; the final destination must be fresh before full-cold: $database" >&2 - return 1 - fi - done < <(destination_related_paths "$database") -} - -for db_path in "${destination_db_paths[@]}"; do - assert_fresh_destination_paths "$db_path" -done - -for db_path in "${gate_db_paths[@]}"; do - python3 - "$db_path" <<'PY' -import sqlite3 -import sys - -database = sys.argv[1] -with sqlite3.connect(database, timeout=30) as connection: - result = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() -if result is None or result[0] != 0: - raise SystemExit(f"WAL checkpoint was busy for {database!r}: {result!r}") -PY - if [[ -e "$db_path-wal" && "$(stat -c %s -- "$db_path-wal")" != 0 ]]; then - echo "WAL was not truncated for $db_path" >&2 - exit 1 - fi -done - -python3 - "$space_headroom_percent" \ - "${gate_db_paths[0]}" "${gate_db_paths[1]}" \ - "${destination_db_paths[0]}" "${destination_db_paths[1]}" <<'PY' | tee "$gate_dir/space.log" -import os -import shutil -import sys -from collections import defaultdict - -headroom_percent = int(sys.argv[1]) -source_paths = sys.argv[2:4] -destination_paths = sys.argv[4:6] -if headroom_percent < 1 or len(source_paths) != 2 or len(destination_paths) != 2: - raise SystemExit("invalid space-gate arguments") - -history_days = 394 -factor = history_days * (100 + headroom_percent) -projections = [] -for source, destination in zip(source_paths, destination_paths): - source_size = os.stat(source).st_size - if source_size < 1: - raise SystemExit(f"one-day database is empty: {source!r}") - destination_realpath = os.path.realpath(destination) - destination_parent = os.path.dirname(destination_realpath) or "." - if not os.path.isdir(destination_parent): - raise SystemExit(f"destination directory does not exist: {destination_parent!r}") - projected = (source_size * factor + 99) // 100 - projections.append((destination_parent, destination, source_size, projected)) - -by_filesystem = defaultdict(list) -for projection in projections: - by_filesystem[os.stat(projection[0]).st_dev].append(projection) - -for device, files in by_filesystem.items(): - free = shutil.disk_usage(files[0][0]).free - required = sum(item[3] for item in files) - print(f"filesystem {device}: {required} projected bytes required, {free} bytes free") - for _, destination, source_size, projected in files: - print(f" {destination}: {source_size} one-day bytes -> {projected} projected bytes") - if required > free: - raise SystemExit(f"insufficient free space on filesystem {device}") -PY -run_gate one-day-noop.log '^Published five-minute buckets: 0$' \ - "$gate_date" "$gate_date" "$gate_dir/datasets.json" "$cold_elapsed_limit_seconds" - -for path in "$gate_dir"/*.sqlite "$gate_dir"/*.sqlite-wal; do - if [[ -e "$path" ]]; then - stat --printf='%n %s bytes\n' "$path" - fi -done | tee "$gate_dir/sizes.log" - -one_day_cold_elapsed_seconds="$(parse_elapsed_seconds "$gate_dir/one-day-cold.log")" -full_cold_projection_seconds="$(awk -v one_day="$one_day_cold_elapsed_seconds" ' - BEGIN { - if (one_day !~ /^[0-9]+([.][0-9]+)?$/ || one_day <= 0) exit 1 - projected = one_day * 394 * 2 - rounded = int(projected) - if (projected > rounded) rounded++ - printf "%d\n", rounded - } -')" -if ! [[ "$full_cold_projection_seconds" =~ ^[1-9][0-9]*$ ]] || - ! awk -v projected="$full_cold_projection_seconds" -v limit="$full_cold_elapsed_limit_seconds" ' - BEGIN { - if (projected !~ /^[0-9]+$/ || limit !~ /^[0-9]+$/ || projected > limit) exit 1 - } - '; then - echo "projected full-cold elapsed time exceeds its configured budget: ${full_cold_projection_seconds}s > ${full_cold_elapsed_limit_seconds}s" >&2 - exit 1 -fi -printf 'Full cold elapsed projection: %ss (2x %s-day one-day cold elapsed of %ss); budget: %ss\n' \ - "$full_cold_projection_seconds" 394 "$one_day_cold_elapsed_seconds" \ - "$full_cold_elapsed_limit_seconds" - -run_gate full-cold.log '^Published five-minute buckets: [1-9][0-9]*$' \ - 2025-06-01 2026-06-29 datasets.json "$full_cold_elapsed_limit_seconds" -run_gate full-noop.log '^Published five-minute buckets: 0$' \ - 2025-06-01 2026-06-29 datasets.json "$full_noop_elapsed_limit_seconds" -``` - -The block runs four distinct assertions: `one-day-cold.log` requires positive publication from -the temporary databases, `one-day-noop.log` requires zero new publication while retaining complete -coverage, `full-cold.log` requires positive publication from the real registry, and -`full-noop.log` requires zero new publication over the complete history. Every run is covered by -the same aggregate cgroup memory monitor and process-tree-safe `RuntimeMaxSec` limit. The full -cold run is launched only after the one-day elapsed time has been parsed and its conservative -2x/394-day projection fits the explicit full-cold budget; the full no-op remains after that build. - -The full-history no-op asserts zero published buckets, positive complete coverage with no partial -or unknown buckets, the same 16 GiB-by-default aggregate cgroup memory ceiling, and its separate -elapsed ceiling. The one-day space projection is deliberately conservative: it scales each -checkpointed output by 394 days and adds the configured headroom for SQLite growth and WAL space, -then compares the combined requirement with free bytes on the actual destination filesystems -before the full build is launched. - -The Docker wrapper is suitable for a functional smoke check, not this RSS gate. It does not expose -a container memory limit or cgroup peak sampler, and timing the wrapper measures the local Docker -client rather than the process inside the container. If you use Docker for the smoke check, keep -the cold and no-op logs separate and keep the capture root absolute: - -```bash -set -euo pipefail -docker_gate() { - local log_name=$1 - local published_pattern=$2 - ./scripts/netflow-db-docker.sh --capture-root /absolute/path/to/captures pipeline \ - --datasets "$gate_dir/datasets.json" \ - --dataset campus-a --dataset campus-b \ - --start-date "$gate_date" --end-date "$gate_date" \ - --require-complete \ - 2>&1 | tee "$gate_dir/$log_name" - grep -Eq '^Five-minute coverage: [1-9][0-9]* complete' "$gate_dir/$log_name" - grep -Eq "$published_pattern" "$gate_dir/$log_name" -} -docker_gate docker-cold.log '^Published five-minute buckets: [1-9][0-9]*$' -docker_gate docker-noop.log '^Published five-minute buckets: 0$' -``` - -Both the temporary registry and SQLite outputs work in this Docker smoke check because their paths -are relative to the mounted repository `data/` directory. +Coordinated mode accepts only registry-backed `daily_active_sources` products. It rejects a run +unless all selected entries have compatible inputs and execution settings: + +- Dataset IDs and output database paths must be unique. +- Every entry must use the same canonical nfcapd root, logical source layout, and timezone. +- Every entry must use the same whole local-day window, force setting, MAAD setting, coverage + setting, and nfdump executable revision. +- Each entry supplies its own `daily_active_sources` prefix and `db_path`. +- Output databases, locks, and sidecars must not overlap one another or the capture tree. + +Do not combine repeated `--dataset` with `--config`, `--database-path`, `--start-time`, `--end-time`, +or command-line selection flags. Configure selection and output paths in `datasets.json`. The +command has no parent dataset or `source_dataset` relation. + +The pipeline discovers the capture plan once, scans each day once, and fans the decoded flow stream +out to the selected products. A missing required capture leaves that day unpublished for every +product. Each product still has its own identity, active-source set, transaction, and completion +marker, so overlapping prefixes may contain the same qualifying flow. + +After a successful run, repeating the exact command is a no-op. The report says +`Published five-minute buckets: 0`, and the pipeline does not rewrite completed days. If a previous +run stopped between product commits, the next run rebuilds only the unfinished day for the affected +product. ## Select flows diff --git a/shell.nix b/shell.nix index ce49730..4e27246 100644 --- a/shell.nix +++ b/shell.nix @@ -15,13 +15,11 @@ mkShell { pkgs.git pkgs.gnumake pkgs.gnutar - pkgs.jq pkgs.libtool pkgs.nodejs pkgs.pkg-config pkgs.python3 pkgs.rustup - pkgs.time pkgs.playwright-driver.browsers ]; diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index 6fb0e11..27d7d49 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -3,7 +3,6 @@ use std::{ collections::{BTreeMap, BTreeSet, HashSet}, net::IpAddr, - time::{Duration, Instant}, }; use fixedbitset::FixedBitSet; @@ -1107,38 +1106,6 @@ pub struct StatisticalBucket { five_minute_starts: BTreeSet, } -/// Aggregate timings for merging canonical children into one rollup builder. -#[derive(Clone, Debug, Default)] -pub(crate) struct StatisticalBucketIncludeProfile { - pub(crate) total_elapsed: Duration, - pub(crate) traffic_elapsed: Duration, - pub(crate) protocols_elapsed: Duration, - pub(crate) addresses_elapsed: Duration, - pub(crate) ports_elapsed: Duration, - pub(crate) coverage_elapsed: Duration, -} - -impl StatisticalBucketIncludeProfile { - pub(crate) fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.traffic_elapsed += profile.traffic_elapsed; - self.protocols_elapsed += profile.protocols_elapsed; - self.addresses_elapsed += profile.addresses_elapsed; - self.ports_elapsed += profile.ports_elapsed; - self.coverage_elapsed += profile.coverage_elapsed; - } - - pub(crate) fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.traffic_elapsed - + self.protocols_elapsed - + self.addresses_elapsed - + self.ports_elapsed - + self.coverage_elapsed, - ) - } -} - impl StatisticalBucket { #[must_use] pub fn new(key: BucketKey) -> Self { @@ -1198,15 +1165,6 @@ impl StatisticalBucket { } pub fn include(&mut self, child: &CanonicalBucket) -> Result<(), DomainError> { - self.include_profiled(child).map(|_| ()) - } - - pub(crate) fn include_profiled( - &mut self, - child: &CanonicalBucket, - ) -> Result { - let total_started = Instant::now(); - let traffic_started = Instant::now(); let mut updates = Vec::with_capacity(child.traffic.len()); for entry in &child.traffic { let mut metrics = self.traffic.get(&entry.scope).cloned().unwrap_or_default(); @@ -1216,44 +1174,28 @@ impl StatisticalBucket { for (scope, metrics) in updates { self.traffic.insert(scope, metrics); } - let traffic_elapsed = traffic_started.elapsed(); - let protocols_started = Instant::now(); for entry in &child.protocols { self.protocols .entry(entry.scope) .or_default() .extend(entry.protocols.iter().cloned()); } - let protocols_elapsed = protocols_started.elapsed(); - let addresses_started = Instant::now(); for entry in &child.addresses { self.addresses .entry((entry.scope, entry.address_side)) .or_default() .extend(entry.addresses.iter().copied()); } - let addresses_elapsed = addresses_started.elapsed(); - let ports_started = Instant::now(); for entry in &child.ports { self.ports .entry((entry.scope, entry.port_side)) .or_insert_with(empty_ports) .union_with(&entry.ports); } - let ports_elapsed = ports_started.elapsed(); - let coverage_started = Instant::now(); self.five_minute_starts .extend(child.five_minute_starts.iter().copied()); self.coverage.include(child.coverage)?; - let coverage_elapsed = coverage_started.elapsed(); - Ok(StatisticalBucketIncludeProfile { - total_elapsed: total_started.elapsed(), - traffic_elapsed, - protocols_elapsed, - addresses_elapsed, - ports_elapsed, - coverage_elapsed, - }) + Ok(()) } /// Whether every five-minute child in this bucket's interval was included. diff --git a/tools/netflow-db/src/ingest.rs b/tools/netflow-db/src/ingest.rs index 2a9da68..7eca752 100644 --- a/tools/netflow-db/src/ingest.rs +++ b/tools/netflow-db/src/ingest.rs @@ -1161,58 +1161,6 @@ pub fn build_nfdump_command_for_selections( Ok(command) } -/// Build one range command for a complete physical member day. -pub fn build_nfdump_range_command( - paths: &[PathBuf], - selection: &FlowSelection, - executable: impl AsRef, -) -> Result, IngestError> { - build_nfdump_range_command_for_selections(paths, std::slice::from_ref(selection), executable) -} - -/// Build one range command for a complete physical member day and several selections. -pub fn build_nfdump_range_command_for_selections( - paths: &[PathBuf], - selections: &[FlowSelection], - executable: impl AsRef, -) -> Result, IngestError> { - let first = paths - .first() - .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; - let last = paths.last().expect("nonempty range has a last path"); - if paths.iter().any(|path| path.parent() != first.parent()) { - return Err(IngestError::InvalidInput(format!( - "nfcapd day range spans directories, including {} and {}", - first.display(), - last.display() - ))); - } - if paths.windows(2).any(|pair| pair[0] >= pair[1]) { - return Err(IngestError::InvalidInput( - "nfcapd day range paths must be strictly chronological".into(), - )); - } - first.file_name().ok_or_else(|| { - IngestError::InvalidInput(format!("invalid nfcapd path: {}", first.display())) - })?; - let last_name = last.file_name().ok_or_else(|| { - IngestError::InvalidInput(format!("invalid nfcapd path: {}", last.display())) - })?; - let mut range = first.to_path_buf().into_os_string(); - range.push(":"); - range.push(last_name); - let mut command = vec![ - executable.as_ref().to_owned(), - "-R".into(), - range, - "-q".into(), - "-o".into(), - nfdump::OUTPUT_MODE.into(), - ]; - command.push(daily_active_union_filter(selections)?.into()); - Ok(command) -} - fn daily_active_union_filter(selections: &[FlowSelection]) -> Result { if selections.is_empty() { return Err(IngestError::InvalidInput( @@ -1302,21 +1250,10 @@ fn prepare_nfcapd_manifest(paths: &[PathBuf]) -> Result<(tempfile::TempDir, Path Ok((manifest, manifest_path)) } -#[cfg(unix)] fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { std::os::unix::fs::symlink(target, link) } -#[cfg(windows)] -fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { - std::os::windows::fs::symlink_file(target, link) -} - -#[cfg(not(any(unix, windows)))] -fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { - fs::hard_link(target, link) -} - fn build_nfdump_manifest_command_for_selections( manifest: &Path, selections: &[FlowSelection], @@ -1494,26 +1431,6 @@ pub(crate) fn read_nfcapd_daily_source_activities( }) } -pub(crate) fn read_nfcapd_daily_source_activity( - paths: &[PathBuf], - selection: &FlowSelection, - executable: impl AsRef, -) -> Result, IngestError> { - let context = paths - .first() - .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; - let (_manifest, manifest_path) = prepare_nfcapd_manifest(paths)?; - let command = build_nfdump_manifest_command_for_selections( - &manifest_path, - std::slice::from_ref(selection), - executable, - )?; - let selection = selection.clone(); - run_nfdump(command, context, NFDUMP_DAY_TIMEOUT, move |stdout| { - nfdump::reduce_to_daily_source_activity(stdout, &selection) - }) -} - fn run_nfdump( command: Vec, path: &Path, @@ -1966,38 +1883,6 @@ mod tests { ); } - #[test] - fn daily_activity_command_uses_one_source_only_member_range() { - let selection = FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "0.220.0.0/16" - }))) - .unwrap(); - let paths = [ - PathBuf::from("/captures/edge/nfcapd.202506010000"), - PathBuf::from("/captures/edge/nfcapd.202506012355"), - ]; - - let command = build_nfdump_range_command(&paths, &selection, "nfdump").unwrap(); - - assert_eq!( - command, - [ - "nfdump", - "-R", - "/captures/edge/nfcapd.202506010000:nfcapd.202506012355", - "-q", - "-o", - "atlantis", - "(src net 0.220.0.0/16 and ipv4 and (proto tcp or proto udp) and src port > 1023) or (src tun net 0.220.0.0/16 and (tun proto tcp or tun proto udp) and src port > 1023)", - ] - .map(OsString::from) - ); - let filter = command.last().unwrap().to_string_lossy(); - assert!(!filter.contains("dst port")); - assert!(!filter.contains("flags")); - } - #[test] fn multi_daily_activity_command_unions_prefixes_and_keeps_fixed_filter() { let selections = [ @@ -2017,13 +1902,8 @@ mod tests { }))) .unwrap(), ]; - let paths = [ - PathBuf::from("/captures/edge/nfcapd.202506010000"), - PathBuf::from("/captures/edge/nfcapd.202506012355"), - ]; - let command = - build_nfdump_range_command_for_selections(&paths, &selections, "nfdump").unwrap(); + build_nfdump_command_for_selections("capture", &selections, "nfdump").unwrap(); assert_eq!( command.last().unwrap().to_string_lossy(), @@ -2033,9 +1913,8 @@ mod tests { #[test] fn multi_nfdump_commands_reject_empty_and_non_daily_selection_sets() { - let paths = [PathBuf::from("/captures/edge/nfcapd.202506010000")]; assert!(matches!( - build_nfdump_range_command_for_selections(&paths, &[], "nfdump"), + build_nfdump_command_for_selections("capture", &[], "nfdump"), Err(IngestError::InvalidInput(message)) if message.contains("empty") )); assert!(matches!( @@ -2296,12 +2175,13 @@ mod tests { "ip_prefix": "192.0.0.0/16", }))) .unwrap(); - let activity = read_nfcapd_daily_source_activity( + let mut activities = read_nfcapd_daily_source_activities( &[first.clone(), second.clone()], - &selection, + std::slice::from_ref(&selection), &executable, ) .unwrap(); + let activity = activities.pop().unwrap(); let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); assert_eq!(activity[&source].flows, 3); @@ -2371,12 +2251,13 @@ mod tests { }))) .unwrap(); - let activity = read_nfcapd_daily_source_activity( + let mut activities = read_nfcapd_daily_source_activities( std::slice::from_ref(&capture), - &selection, + std::slice::from_ref(&selection), &executable, ) .unwrap(); + let activity = activities.pop().unwrap(); let source = IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 1)); assert_eq!(activity.len(), 1); diff --git a/tools/netflow-db/src/maad.rs b/tools/netflow-db/src/maad.rs index 5c07458..0331bf9 100644 --- a/tools/netflow-db/src/maad.rs +++ b/tools/netflow-db/src/maad.rs @@ -136,7 +136,7 @@ pub fn compute_with_config( if addresses.len() < MIN_MAAD_ADDRESSES { return Ok(empty_result(addresses.len())); } - let counts = build_prefix_counts(&addresses, config.max_prefix_length); + let counts = build_prefix_counts(&addresses); let prepared = prepare_valid_moments(&counts, &config); if prepared.is_empty() { return Ok(empty_result(addresses.len())); @@ -182,13 +182,9 @@ fn empty_result(total_addrs: usize) -> MaadResult { } } -fn build_prefix_counts(addresses: &[u32], max_prefix_length: u8) -> Vec> { - // Moment preparation needs each configured parent level and its children; - // dimensions only read the configured parent levels. Do not retain the - // unused /32 level range for the default /24 analysis. - let count_levels = usize::from(max_prefix_length) + 2; - let mut counts = Vec::with_capacity(count_levels); - for prefix_length in 0..=max_prefix_length + 1 { +fn build_prefix_counts(addresses: &[u32]) -> Vec> { + let mut counts = Vec::with_capacity(33); + for prefix_length in 0..=32_u8 { let mut prefixes = Vec::new(); for &address in addresses { let prefix = prefix_of(address, prefix_length); @@ -934,32 +930,6 @@ mod tests { assert_eq!(result.structure.len(), 33); } - #[test] - fn prefix_counts_stop_after_the_configured_parent_and_child_levels() { - let addresses = [ - u32::from(Ipv4Addr::new(192, 0, 2, 1)), - u32::from(Ipv4Addr::new(192, 0, 2, 2)), - ]; - - let default_counts = - build_prefix_counts(&addresses, MaadConfig::default().max_prefix_length); - assert_eq!(default_counts.len(), 26); - assert_eq!( - default_counts.last().unwrap(), - &vec![(prefix_of(addresses[0], 25), 2)] - ); - - let deepest_counts = build_prefix_counts(&addresses, 31); - assert_eq!(deepest_counts.len(), 33); - assert_eq!( - deepest_counts.last().unwrap(), - &vec![ - (u32::from(Ipv4Addr::new(192, 0, 2, 1)), 1), - (u32::from(Ipv4Addr::new(192, 0, 2, 2)), 1), - ] - ); - } - #[test] fn configurable_q_grid_is_uniform_and_includes_dimension_qs() { let config = MaadConfig { diff --git a/tools/netflow-db/src/nfdump.rs b/tools/netflow-db/src/nfdump.rs index 403cbe8..4a8a211 100644 --- a/tools/netflow-db/src/nfdump.rs +++ b/tools/netflow-db/src/nfdump.rs @@ -434,24 +434,6 @@ pub(crate) fn reduce_to_buckets_with_active_sources( } } -pub(crate) fn reduce_to_daily_source_activity( - mut input: R, - selection: &FlowSelection, -) -> Result, NfdumpError> { - if !selection.selects_daily_active_sources() { - return Err(NfdumpError::new( - Phase::Aggregate, - Field::SourceAddress, - ErrorReason::DailyActivityRequiresDailyActiveSourceSelection, - )); - } - let mut activities = - reduce_to_daily_source_activities(&mut input, std::slice::from_ref(selection))?; - Ok(activities - .pop() - .expect("one daily selection produces one activity map")) -} - pub(crate) fn reduce_to_daily_source_activities( mut input: R, selections: &[FlowSelection], @@ -1379,11 +1361,12 @@ mod tests { inactive[40..48].copy_from_slice(&1_999_u64.to_le_bytes()); inactive[48..56].copy_from_slice(&2_u64.to_le_bytes()); - let activity = reduce_to_daily_source_activity( + let mut activities = reduce_to_daily_source_activities( Cursor::new(stream(&[first, second, inactive])), - &selection, + std::slice::from_ref(&selection), ) .unwrap(); + let activity = activities.pop().unwrap(); assert_eq!( activity[&IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))], SourceActivity { diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index fb3a2e9..5f805e0 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -6,8 +6,7 @@ use std::{ fs, net::IpAddr, path::{Path, PathBuf}, - sync::{Arc, OnceLock}, - time::{Duration, Instant}, + sync::Arc, }; use jiff::{RoundMode, Timestamp, ToSpan, Unit, ZonedRound, civil::Date}; @@ -17,40 +16,33 @@ use serde::Deserialize; use serde_json::{Value, json}; use thiserror::Error; -#[cfg(test)] -use std::sync::atomic::{AtomicUsize, Ordering}; - use crate::{ config::{ConfigError, CsvSourceConfig}, coverage::BucketCoverage, domain::{ AddressSet, BucketKey, CanonicalBucket, DomainError, FlowSelection, Granularity, - StatisticalBucket, StatisticalBucketIncludeProfile, + StatisticalBucket, }, ingest::{self, IngestError, ProducerError}, nfdump, provenance::{ ExecutableRevision, ExpectedAbsence, FileSnapshot, InputRevision, ProvenanceError, - capture_file_revision, csv_decoder_fingerprint, file_sha256, nfcapd_decoder_fingerprint, + capture_file_revision, csv_decoder_fingerprint, nfcapd_decoder_fingerprint, revision_for_locator, verify_file_snapshot, }, - publish::{PublishError, WriteBucketsProfile, write_buckets, write_buckets_profiled}, + publish::{PublishError, write_buckets}, registry::{Dataset, DatasetRegistry, DatasetSource, RegistryError, is_safe_path_component}, storage::{ - BucketCoverageRow, DailyProductCompletionState, DatabaseOperationLock, DatasetMetadata, - InputBucket, InputEvidenceRow, InputEvidenceState, InputKind, InputStatus, ProductIdentity, - STATS_TABLE_NAMES, SourceDefinition, StatsBucketKey, StorageError, + DatabaseOperationLock, DatasetMetadata, InputBucket, InputEvidenceRow, InputEvidenceState, + InputKind, InputStatus, ProductIdentity, SourceDefinition, StorageError, bind_nfcapd_source_layout, bind_product_identity, cached_content_fingerprint, canonical_path, complete_input_scan, connect_pipeline_writer, current_product_fingerprint, - daily_product_completion_state, database_operation_lock_path, delete_stats_bucket_keys, - delete_stats_time_range, earliest_traffic_bucket_start, - ensure_daily_product_completion_bucket_guard, init_schema, input_scan_fully_processed, - insert_bucket_coverage_rows, mark_input_bucket_status, nfcapd_logical_bucket_processed, - optimize_all_query_planner_statistics, provision_daily_product_completion_bucket_guards, - query_bucket_coverage, query_input_evidence, query_input_evidence_range, - query_processed_nfcapd_range, replace_input_evidence, set_dataset_default_start_date, - upsert_daily_product_completion, upsert_dataset_metadata, upsert_input_bucket, - validate_database_path_separation, + daily_product_completion_matches, database_related_paths, delete_stats_time_range, + earliest_traffic_bucket_start, init_schema, input_scan_fully_processed, + mark_input_bucket_status, nfcapd_logical_bucket_processed, + optimize_all_query_planner_statistics, query_input_evidence, replace_input_evidence, + set_dataset_default_start_date, upsert_daily_product_completion, upsert_dataset_metadata, + upsert_input_bucket, validate_database_path_separation, }, }; @@ -60,281 +52,7 @@ const NFCAPD_REVISION_HASH_MAX_WORKERS: usize = NFCAPD_DECODE_BATCH_SIZE * 2; const MAX_MISSING_DAY_WARNING_DETAILS: usize = 8; const DEFAULT_TIMEZONE: &str = "America/Los_Angeles"; -static NFCAPD_DENSE_TRAFFIC_SCOPE_COUNT: OnceLock = OnceLock::new(); - -fn nfcapd_dense_traffic_scope_count() -> i64 { - *NFCAPD_DENSE_TRAFFIC_SCOPE_COUNT.get_or_init(|| { - let key = BucketKey::new("", Granularity::FiveMinutes, 0, FIVE_MINUTES); - i64::try_from(StatisticalBucket::dense(key).finish_owned().traffic.len()) - .expect("dense traffic scope count fits SQLite INTEGER") - }) -} - -#[cfg(test)] -type MissingDayAbsenceHook = Box; - -#[cfg(test)] -type CoordinatedCommitGuardHook = Box; - -#[cfg(test)] -type CoordinatedPlanHook = Box; - -#[cfg(test)] -type SinglePlanHook = Box; - -#[cfg(test)] -type SingleCommitGuardHook = Box; - -#[cfg(test)] -thread_local! { - static PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static NFCAPD_CAPTURE_IDENTITY_CALLS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static NFCAPD_REVISION_POOL_BUILDS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static NFCAPD_DECODE_POOL_BUILDS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static NFCAPD_ACTIVITY_POOL_BUILDS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static DATASET_REGISTRY_LOAD_CALLS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS: std::cell::Cell = const { - std::cell::Cell::new(0) - }; - static MISSING_DAY_ABSENCE_HOOK: std::cell::RefCell> = - const { std::cell::RefCell::new(None) }; - static COORDINATED_COMMIT_GUARD_HOOK: std::cell::RefCell> = - const { std::cell::RefCell::new(None) }; - static COORDINATED_PLAN_HOOK: std::cell::RefCell> = - const { std::cell::RefCell::new(None) }; - static SINGLE_COMMIT_GUARD_HOOK: std::cell::RefCell> = - const { std::cell::RefCell::new(None) }; - static SINGLE_PLAN_HOOK: std::cell::RefCell> = - const { std::cell::RefCell::new(None) }; -} - -#[cfg(test)] -fn reset_prepare_nfcapd_tree_timestamp_calls() { - PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS.with(|calls| calls.set(0)); -} - -#[cfg(test)] -fn prepare_nfcapd_tree_timestamp_calls() -> usize { - PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS.with(std::cell::Cell::get) -} - -#[cfg(test)] -fn reset_nfcapd_logical_bucket_topology_calls() { - NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS.with(|calls| calls.set(0)); -} - -#[cfg(test)] -fn nfcapd_logical_bucket_topology_calls() -> usize { - NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS.with(std::cell::Cell::get) -} - -#[cfg(test)] -fn reset_nfcapd_day_topology_audit_calls() { - NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS.with(|calls| calls.set(0)); -} - -#[cfg(test)] -fn nfcapd_day_topology_audit_calls() -> usize { - NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS.with(std::cell::Cell::get) -} - -#[cfg(test)] -fn reset_nfcapd_capture_identity_calls() { - NFCAPD_CAPTURE_IDENTITY_CALLS.with(|calls| calls.set(0)); -} - -#[cfg(test)] -fn nfcapd_capture_identity_calls() -> usize { - NFCAPD_CAPTURE_IDENTITY_CALLS.with(std::cell::Cell::get) -} - -#[cfg(test)] -fn reset_nfcapd_pool_builds() { - NFCAPD_REVISION_POOL_BUILDS.with(|builds| builds.set(0)); - NFCAPD_DECODE_POOL_BUILDS.with(|builds| builds.set(0)); - NFCAPD_ACTIVITY_POOL_BUILDS.with(|builds| builds.set(0)); -} - -#[cfg(test)] -fn nfcapd_pool_builds() -> (usize, usize, usize) { - ( - NFCAPD_REVISION_POOL_BUILDS.with(std::cell::Cell::get), - NFCAPD_DECODE_POOL_BUILDS.with(std::cell::Cell::get), - NFCAPD_ACTIVITY_POOL_BUILDS.with(std::cell::Cell::get), - ) -} - -#[cfg(test)] -fn reset_dataset_registry_load_calls() { - DATASET_REGISTRY_LOAD_CALLS.with(|calls| calls.set(0)); -} - -#[cfg(test)] -fn dataset_registry_load_calls() -> usize { - DATASET_REGISTRY_LOAD_CALLS.with(std::cell::Cell::get) -} - -#[cfg(test)] -fn reset_coordinated_postflight_snapshot_verifications() { - COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS.with(|calls| calls.set(0)); -} - -#[cfg(test)] -fn coordinated_postflight_snapshot_verifications() -> usize { - COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS.with(std::cell::Cell::get) -} - -#[cfg(test)] -fn set_missing_day_absence_hook(hook: impl FnMut(&Path, &[(String, i64)], &str) + 'static) { - MISSING_DAY_ABSENCE_HOOK.with(|current| { - *current.borrow_mut() = Some(Box::new(hook)); - }); -} - -#[cfg(test)] -fn clear_missing_day_absence_hook() { - MISSING_DAY_ABSENCE_HOOK.with(|current| { - *current.borrow_mut() = None; - }); -} - -#[cfg(test)] -fn set_coordinated_commit_guard_hook(hook: impl FnMut() + 'static) { - COORDINATED_COMMIT_GUARD_HOOK.with(|current| { - *current.borrow_mut() = Some(Box::new(hook)); - }); -} - -#[cfg(test)] -fn clear_coordinated_commit_guard_hook() { - COORDINATED_COMMIT_GUARD_HOOK.with(|current| { - *current.borrow_mut() = None; - }); -} - -#[cfg(test)] -fn invoke_coordinated_commit_guard_hook() { - COORDINATED_COMMIT_GUARD_HOOK.with(|current| { - if let Some(hook) = current.borrow_mut().as_mut() { - hook(); - } - }); -} - -#[cfg(test)] -fn set_coordinated_plan_hook(hook: impl FnMut(&Path) + 'static) { - COORDINATED_PLAN_HOOK.with(|current| { - *current.borrow_mut() = Some(Box::new(hook)); - }); -} - -#[cfg(test)] -fn clear_coordinated_plan_hook() { - COORDINATED_PLAN_HOOK.with(|current| { - *current.borrow_mut() = None; - }); -} - -#[cfg(test)] -fn invoke_coordinated_plan_hook(root: &Path) { - COORDINATED_PLAN_HOOK.with(|current| { - if let Some(hook) = current.borrow_mut().as_mut() { - hook(root); - } - }); -} - -#[cfg(test)] -fn set_single_commit_guard_hook(hook: impl FnMut() + 'static) { - SINGLE_COMMIT_GUARD_HOOK.with(|current| { - *current.borrow_mut() = Some(Box::new(hook)); - }); -} - -#[cfg(test)] -fn clear_single_commit_guard_hook() { - SINGLE_COMMIT_GUARD_HOOK.with(|current| { - *current.borrow_mut() = None; - }); -} - -#[cfg(test)] -fn invoke_single_commit_guard_hook() { - SINGLE_COMMIT_GUARD_HOOK.with(|current| { - if let Some(hook) = current.borrow_mut().as_mut() { - hook(); - } - }); -} - -#[cfg(test)] -fn set_single_plan_hook(hook: impl FnMut(&Path) + 'static) { - SINGLE_PLAN_HOOK.with(|current| { - *current.borrow_mut() = Some(Box::new(hook)); - }); -} - -#[cfg(test)] -fn clear_single_plan_hook() { - SINGLE_PLAN_HOOK.with(|current| { - *current.borrow_mut() = None; - }); -} - -#[cfg(test)] -fn invoke_single_plan_hook(root: &Path) { - SINGLE_PLAN_HOOK.with(|current| { - if let Some(hook) = current.borrow_mut().as_mut() { - hook(root); - } - }); -} - -#[cfg(not(test))] -fn invoke_coordinated_commit_guard_hook() {} - -#[cfg(not(test))] -fn invoke_coordinated_plan_hook(_root: &Path) {} - -#[cfg(not(test))] -fn invoke_single_commit_guard_hook() {} - -#[cfg(not(test))] -fn invoke_single_plan_hook(_root: &Path) {} - -#[cfg(test)] -fn invoke_missing_day_absence_hook(root: &Path, missing: &[(String, i64)], timezone: &str) { - MISSING_DAY_ABSENCE_HOOK.with(|current| { - if let Some(hook) = current.borrow_mut().as_mut() { - hook(root, missing, timezone); - } - }); -} - -#[cfg(not(test))] -fn invoke_missing_day_absence_hook(_root: &Path, _missing: &[(String, i64)], _timezone: &str) {} - fn build_revision_hash_pool() -> Result { - #[cfg(test)] - NFCAPD_REVISION_POOL_BUILDS.with(|builds| builds.set(builds.get() + 1)); let revision_hash_workers = std::thread::available_parallelism() .map_or(1, std::num::NonZeroUsize::get) .min(NFCAPD_REVISION_HASH_MAX_WORKERS); @@ -347,22 +65,7 @@ fn build_revision_hash_pool() -> Result { }) } -fn build_nfcapd_snapshot_pool() -> Result { - let snapshot_workers = std::thread::available_parallelism() - .map_or(1, std::num::NonZeroUsize::get) - .min(NFCAPD_REVISION_HASH_MAX_WORKERS); - rayon::ThreadPoolBuilder::new() - .num_threads(snapshot_workers) - .thread_name(|index| format!("nfcapd-snapshot-{index}")) - .build() - .map_err(|error| { - PipelineError::InvalidConfig(format!("failed to build nfcapd snapshot pool: {error}")) - }) -} - fn build_nfcapd_decode_pool() -> Result { - #[cfg(test)] - NFCAPD_DECODE_POOL_BUILDS.with(|builds| builds.set(builds.get() + 1)); rayon::ThreadPoolBuilder::new() .num_threads(NFCAPD_DECODE_BATCH_SIZE) .thread_name(|index| format!("nfcapd-decode-{index}")) @@ -373,8 +76,6 @@ fn build_nfcapd_decode_pool() -> Result { } fn build_nfcapd_activity_pool() -> Result { - #[cfg(test)] - NFCAPD_ACTIVITY_POOL_BUILDS.with(|builds| builds.set(builds.get() + 1)); rayon::ThreadPoolBuilder::new() .num_threads(NFCAPD_DECODE_BATCH_SIZE) .thread_name(|index| format!("nfcapd-activity-{index}")) @@ -523,8 +224,6 @@ enum InputSpec { #[derive(Clone, Debug)] struct ResolvedPipeline { database_path: PathBuf, - /// Files that configure or execute the pipeline and must remain read-only during output setup. - control_paths: Vec, timezone: String, run_maad: bool, nfdump: PathBuf, @@ -545,30 +244,14 @@ fn nfdump_control_path(value: &str) -> Option { .then(|| path.to_owned()) } -#[cfg(unix)] fn has_effective_execute_access(path: &Path) -> bool { - #[cfg(all(not(target_os = "android"), not(target_os = "redox")))] - { - nix::unistd::faccessat( - None, - path, - nix::unistd::AccessFlags::X_OK, - nix::fcntl::AtFlags::AT_EACCESS, - ) - .is_ok() - } - - #[cfg(any(target_os = "android", target_os = "redox"))] - { - // These targets do not expose AT_EACCESS. Their normal process lookup uses the same - // credentials as access(2) for this non-set-id pipeline. - nix::unistd::access(path, nix::unistd::AccessFlags::X_OK).is_ok() - } -} - -#[cfg(not(unix))] -fn has_effective_execute_access(_path: &Path) -> bool { - true + nix::unistd::faccessat( + None, + path, + nix::unistd::AccessFlags::X_OK, + nix::fcntl::AtFlags::AT_EACCESS, + ) + .is_ok() } /// Resolve the executable that a nfdump command will select. @@ -692,9 +375,8 @@ pub fn run( /// Run several registry datasets as coordinated daily-active-source products. /// -/// The datasets share discovery and nfdump work, while each output retains its own product -/// identity, provenance, transactions, and resume state. This deliberately stays separate from -/// [`run`] so the established single-dataset path remains unchanged. +/// The datasets share the same frozen discovery plan, day loop, and nfdump work used by [`run`], +/// while each output retains its own product identity, transaction, and completion markers. pub fn run_many( request: impl std::borrow::Borrow, dataset_ids: Vec, @@ -746,13 +428,11 @@ pub fn run_many( single.dataset_id = Some(dataset_id.clone()); pipelines.push(resolve_dataset_request( &single, - ®istry_path, ®istry, Some((&shared_nfdump.0, &shared_nfdump.1)), )?); } - validate_compatible_pipelines(&pipelines)?; - execute_many(pipelines) + execute_many(validate_compatible_pipelines(pipelines)?) } fn selection_override_requested(value: &Value) -> bool { @@ -761,7 +441,14 @@ fn selection_override_requested(value: &Value) -> bool { .is_some_and(|object| object.values().any(|entry| !entry.is_null())) } -fn validate_compatible_pipelines(pipelines: &[ResolvedPipeline]) -> Result<(), PipelineError> { +struct CompatiblePlan { + pipelines: Vec, + tree: FrozenNfcapdTreeLayout, +} + +fn validate_compatible_pipelines( + pipelines: Vec, +) -> Result { let Some(first) = pipelines.first() else { return Err(PipelineError::InvalidConfig( "coordinated dataset mode requires at least two datasets".into(), @@ -775,7 +462,17 @@ fn validate_compatible_pipelines(pipelines: &[ResolvedPipeline]) -> Result<(), P } let first_input = only_nfcapd_tree(first)?; let first_config = nfcapd_tree_config(first_input)?; - let first_root = canonical_path(first_config.root_path)?; + let output_paths = pipelines + .iter() + .map(|pipeline| pipeline.database_path.as_path()) + .collect::>(); + validate_database_path_separation(&output_paths)?; + let tree = freeze_nfcapd_tree( + first_input, + &first.selection, + &first.timezone, + &output_paths, + )?; for pipeline in pipelines.iter().skip(1) { if !pipeline.selection.selects_daily_active_sources() { return Err(PipelineError::InvalidConfig(format!( @@ -789,7 +486,7 @@ fn validate_compatible_pipelines(pipelines: &[ResolvedPipeline]) -> Result<(), P } let input = only_nfcapd_tree(pipeline)?; let config = nfcapd_tree_config(input)?; - if first_root != canonical_path(config.root_path)? { + if tree.root_path != fs::canonicalize(config.root_path)? { return Err(PipelineError::InvalidConfig( "coordinated datasets must use the same nfcapd root".into(), )); @@ -838,13 +535,17 @@ fn validate_compatible_pipelines(pipelines: &[ResolvedPipeline]) -> Result<(), P "coordinated datasets must use the same coverage settings".into(), )); } + if tree.sources != canonical_logical_sources(input)? { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same logical source layout and membership" + .into(), + )); + } } - let output_paths = pipelines - .iter() - .map(|pipeline| pipeline.database_path.as_path()) - .collect::>(); - validate_database_path_separation(&output_paths)?; - Ok(()) + if first.nfdump_revision.is_some() { + ingest::probe_nfdump_compatibility(&first.nfdump)?; + } + Ok(CompatiblePlan { pipelines, tree }) } fn only_nfcapd_tree(pipeline: &ResolvedPipeline) -> Result<&InputSpec, PipelineError> { @@ -914,720 +615,162 @@ fn canonical_logical_sources(input: &InputSpec) -> Result, Pi Ok(sources) } -fn normalized_path_key(path: &Path) -> PathBuf { - let mut result = PathBuf::new(); - for component in path.components() { - match component { - std::path::Component::CurDir => {} - std::path::Component::ParentDir => { - result.pop(); - } - _ => result.push(component.as_os_str()), - } - } - result -} - -fn absolute_lexical_path(path: &Path) -> Result { - let absolute = if path.is_absolute() { - path.to_owned() - } else { - std::env::current_dir()?.join(path) - }; - Ok(normalized_path_key(&absolute)) -} - -fn sqlite_related_path(path: &Path, suffix: &str) -> Result { - let parent = path.parent().ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "database path has no parent directory: {}", - path.display() - )) - })?; - let name = path.file_name().ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "database path has no file name: {}", - path.display() - )) - })?; - Ok(parent.join(format!("{}{}", name.to_string_lossy(), suffix))) +fn paths_overlap(left: &Path, right: &Path) -> bool { + left.starts_with(right) || right.starts_with(left) } -/// Return every path SQLite or the pipeline lock can touch for an output path. -/// -/// Keep both the caller spelling and the resolved spelling here. SQLite receives the caller -/// spelling, while the operation lock resolves the database first; a symlink can therefore make -/// those two sets differ even when the database itself is absent. -fn output_related_paths(path: &Path) -> Result, PipelineError> { - let raw = absolute_lexical_path(path)?; - let resolved = canonical_path(path)?; - let mut candidates = BTreeSet::new(); - let mut add = |candidate: PathBuf| -> Result<(), PipelineError> { - let candidate = absolute_lexical_path(&candidate)?; - candidates.insert(candidate.clone()); - candidates.insert(canonical_path(candidate)?); - Ok(()) - }; - - for database in [&raw, &resolved] { - add(database.to_owned())?; - for suffix in ["-journal", "-wal", "-shm"] { - add(sqlite_related_path(database, suffix)?)?; - } +fn validate_output_capture_separation( + output_paths: &[&Path], + capture_root: &Path, + member_ids: &[String], +) -> Result<(), PipelineError> { + let capture_root = fs::canonicalize(capture_root)?; + let mut capture_paths = vec![capture_root.clone()]; + for member in member_ids { + capture_paths.push(fs::canonicalize(capture_root.join(member))?); } - add(database_operation_lock_path(&resolved)?)?; - add(raw.with_file_name(format!( - ".{}.operation.lock", - raw.file_name() - .ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "database path has no file name: {}", - raw.display() - )) - })? - .to_string_lossy() - )))?; - Ok(candidates.into_iter().collect()) -} - -#[cfg(unix)] -fn existing_path_identity(path: &Path) -> Result, PipelineError> { - use std::os::unix::fs::MetadataExt; - - match fs::metadata(path) { - Ok(metadata) => Ok(Some((metadata.dev(), metadata.ino()))), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(error) => Err(error.into()), + for path in output_paths { + for output in database_related_paths(path)? { + if capture_paths + .iter() + .any(|capture| paths_overlap(&output, capture)) + { + return Err(PipelineError::InvalidConfig(format!( + "output database {} overlaps the nfcapd capture tree {}", + path.display(), + capture_root.display() + ))); + } + } } + Ok(()) } -#[cfg(unix)] -#[cfg(test)] -fn nfcapd_capture_identity(path: &Path) -> Result, PipelineError> { - #[cfg(test)] - NFCAPD_CAPTURE_IDENTITY_CALLS.with(|calls| calls.set(calls.get() + 1)); - existing_path_identity(path) -} - -fn capture_nfcapd_snapshot(path: &Path) -> Result { - FileSnapshot::capture(path).map_err(PipelineError::from) -} - -/// Capture the cheap identities for a discovered capture set with bounded parallelism. -/// -/// The caller keeps the resulting metadata alongside the already-discovered paths. Hashing and -/// decode work can then reuse the same observation instead of doing a serial alias pass followed -/// by another metadata walk. -fn capture_nfcapd_snapshots( - paths: &BTreeSet, -) -> Result, PipelineError> { - capture_nfcapd_snapshots_with(paths, capture_nfcapd_snapshot) -} - -fn capture_nfcapd_snapshots_with( - paths: &BTreeSet, - capture: F, -) -> Result, PipelineError> -where - F: Fn(&Path) -> Result + Sync, -{ - if paths.is_empty() { - return Ok(BTreeMap::new()); +fn validate_daily_active_source_layout( + sources: &[DatasetSource], + physical_ids: &[String], +) -> Result<(), PipelineError> { + if sources.is_empty() || physical_ids.is_empty() { + return Err(PipelineError::InvalidConfig( + "daily_active_sources requires at least one logical and physical source".into(), + )); } - let pool = build_nfcapd_snapshot_pool()?; - pool.install(|| { - paths - .par_iter() - .map(|path| capture(path).map(|snapshot| (path.clone(), snapshot))) - .collect::, _>>() - }) -} - -#[cfg(test)] -fn capture_nfcapd_snapshots_counted( - paths: &BTreeSet, - calls: &AtomicUsize, -) -> Result, PipelineError> { - capture_nfcapd_snapshots_with(paths, |path| { - calls.fetch_add(1, Ordering::Relaxed); - capture_nfcapd_snapshot(path) - }) + Ok(()) } -#[cfg(unix)] -fn output_has_existing_identity(output_paths: &[&Path]) -> Result { - for output in output_paths { - for related in output_related_paths(output)? { - if existing_path_identity(&related)?.is_some() { - return Ok(true); - } - } - } - Ok(false) +#[derive(Clone, Debug)] +struct FrozenNfcapdTreeLayout { + root_path: PathBuf, + sources: Vec, + physical_ids: Vec, + by_member_and_start: BTreeMap<(String, i64), PathBuf>, + member_bounds: BTreeMap, + start: i64, + end: i64, + extend_gaps_to_window: bool, + force: bool, } -#[cfg(not(unix))] -fn output_has_existing_identity(_output_paths: &[&Path]) -> Result { - Ok(false) +#[derive(Clone, Debug, Default)] +struct SingleOutputPlan { + trees: BTreeMap, + dataset_sources: BTreeMap>, } -/// Reject an output database, its SQLite sidecars, or its operation lock when any aliases an -/// input file. This must run after input discovery and before output setup. -fn validate_output_input_separation<'a, I>( - output_paths: &[&Path], - input_paths: I, - input_label: &str, -) -> Result<(), PipelineError> -where - I: IntoIterator, -{ - // Outputs are few, while a discovered capture tree can contain hundreds of thousands of - // paths. Index the small output side and stream the input side so validation does not retain a - // second copy of the discovered corpus. - let mut outputs_by_path = BTreeMap::::new(); - #[cfg(unix)] - let mut outputs_by_identity = BTreeMap::<(u64, u64), (usize, PathBuf)>::new(); - for (output_index, output) in output_paths.iter().enumerate() { - for related in output_related_paths(output)? { - let resolved = canonical_path(&related)?; - outputs_by_path - .entry(resolved) - .or_insert_with(|| (output_index, related.clone())); - #[cfg(unix)] - if let Some(identity) = existing_path_identity(&related)? { - outputs_by_identity - .entry(identity) - .or_insert_with(|| (output_index, related)); - } - } - } - - for input in input_paths { - let resolved = canonical_path(input)?; - if let Some((output_index, related)) = outputs_by_path.get(&resolved) { - return Err(PipelineError::InvalidConfig(format!( - "output database {} aliases {input_label} {} through {}", - output_paths[*output_index].display(), - input.display(), - related.display() - ))); - } - #[cfg(unix)] - if let Some(identity) = existing_path_identity(input)? - && let Some((output_index, related)) = outputs_by_identity.get(&identity) - { - return Err(PipelineError::InvalidConfig(format!( - "output database {} aliases {input_label} {} through device/inode {:?} at {}", - output_paths[*output_index].display(), - input.display(), - identity, - related.display() - ))); - } - } - Ok(()) -} - -/// Reject an output database, its SQLite sidecars, or its operation lock when any aliases a -/// discovered nfcapd capture. This must run after capture discovery and before output setup. -fn validate_output_capture_separation<'a, I>( - output_paths: &[&Path], - capture_paths: I, -) -> Result<(), PipelineError> -where - I: IntoIterator, -{ - validate_output_input_separation(output_paths, capture_paths, "discovered nfcapd capture") -} - -/// Protect a discovered nfcapd tree without resolving every capture path. -/// -/// Capture discovery already walks the configured member namespaces, so a capture's lexical -/// locator is known. Output aliases are checked against the configured and resolved namespace -/// spellings, including locators that do not exist yet. Only an existing output-side inode needs -/// a physical capture scan for hard-link aliases; the common new-output case performs no metadata -/// call per capture. -#[cfg(test)] -fn validate_output_nfcapd_capture_separation<'a, I>( - output_paths: &[&Path], - namespaces: &[PathBuf], - _timezone: &str, - capture_paths: I, -) -> Result<(), PipelineError> -where - I: IntoIterator, -{ - validate_output_nfcapd_locator_separation(output_paths, namespaces)?; - - #[cfg(unix)] - let mut outputs_by_identity = BTreeMap::<(u64, u64), (usize, PathBuf)>::new(); - #[cfg(unix)] - for (output_index, output) in output_paths.iter().enumerate() { - for related in output_related_paths(output)? { - if let Some(identity) = existing_path_identity(&related)? { - outputs_by_identity - .entry(identity) - .or_insert_with(|| (output_index, related)); - } - } - } - - #[cfg(unix)] - if outputs_by_identity.is_empty() { - return Ok(()); - } - - #[cfg(unix)] - for input in capture_paths { - if let Some(identity) = nfcapd_capture_identity(input)? - && let Some((output_index, related)) = outputs_by_identity.get(&identity) - { - return Err(PipelineError::InvalidConfig(format!( - "output database {} aliases discovered nfcapd capture {} through device/inode {:?} at {}", - output_paths[*output_index].display(), - input.display(), - identity, - related.display() - ))); - } - } - - #[cfg(not(unix))] - let _ = capture_paths; - Ok(()) -} - -/// Validate discovered nfcapd captures using identities captured by the bounded snapshot pass. -/// -/// This is the existing-output path: the snapshot map is also consumed by revision preparation, -/// so hard-link protection does not require a second serial metadata walk. -fn validate_output_nfcapd_capture_separation_with_snapshots<'a, I>( - output_paths: &[&Path], - namespaces: &[PathBuf], - capture_snapshots: I, -) -> Result<(), PipelineError> -where - I: IntoIterator, -{ - validate_output_nfcapd_locator_separation(output_paths, namespaces)?; - - #[cfg(unix)] - let mut outputs_by_identity = BTreeMap::<(u64, u64), (usize, PathBuf)>::new(); - #[cfg(unix)] - for (output_index, output) in output_paths.iter().enumerate() { - for related in output_related_paths(output)? { - if let Some(identity) = existing_path_identity(&related)? { - outputs_by_identity - .entry(identity) - .or_insert_with(|| (output_index, related)); - } - } - } - - #[cfg(unix)] - if outputs_by_identity.is_empty() { - return Ok(()); - } - - #[cfg(unix)] - for (input, snapshot) in capture_snapshots { - let identity = (snapshot.device, snapshot.inode); - if let Some((output_index, related)) = outputs_by_identity.get(&identity) { - return Err(PipelineError::InvalidConfig(format!( - "output database {} aliases discovered nfcapd capture {} through device/inode {:?} at {}", - output_paths[*output_index].display(), - input.display(), - identity, - related.display() - ))); - } - } - - #[cfg(not(unix))] - let _ = capture_snapshots; - Ok(()) -} - -/// Return the canonical and configured spellings of each member's locator namespace. -/// -/// The output preflight uses these prefixes instead of materializing every possible capture -/// path in the selected window. Missing future captures are still protected because the output -/// path itself is checked against the namespace shape. -fn nfcapd_locator_namespaces( - root: &Path, - physical_ids: &[String], -) -> Result, PipelineError> { - let mut namespaces = BTreeSet::new(); - for member in physical_ids { - let configured = absolute_lexical_path(&root.join(member))?; - namespaces.insert(configured.clone()); - namespaces.insert(canonical_path(configured)?); - } - Ok(namespaces.into_iter().collect()) -} - -/// Return whether two paths overlap as a namespace and a path. -/// -/// The equality and ancestor cases matter when the output or one of its sidecars is the -/// namespace itself, or when it would replace a root/ancestor needed to discover captures. -fn paths_overlap_namespace(path: &Path, namespace: &Path) -> bool { - path == namespace || path.starts_with(namespace) || namespace.starts_with(path) -} - -/// Reject output databases, sidecars, and operation locks anywhere in a configured member -/// namespace, even when the output is not itself a valid nfcapd capture locator yet. -fn validate_output_nfcapd_locator_separation( - output_paths: &[&Path], - namespaces: &[PathBuf], -) -> Result<(), PipelineError> { - for output in output_paths { - for related in output_related_paths(output)? { - for namespace in namespaces { - if paths_overlap_namespace(&related, namespace) { - return Err(PipelineError::InvalidConfig(format!( - "output database {} aliases discovered nfcapd capture locator in configured nfcapd member namespace {} through {}", - output.display(), - namespace.display(), - related.display() - ))); - } - } - } - } - Ok(()) -} - -/// Reject output databases, sidecars, and operation locks anywhere below an auto-discovered -/// nfcapd root. The member directory may not exist during preflight, so checking only discovered -/// captures would leave a future member namespace writable by output setup. -fn validate_output_nfcapd_auto_namespace_separation( - output_paths: &[&Path], - roots: &[PathBuf], -) -> Result<(), PipelineError> { - for output in output_paths { - for related in output_related_paths(output)? { - for root in roots { - if paths_overlap_namespace(&related, root) { - return Err(PipelineError::InvalidConfig(format!( - "output database {} aliases discovered nfcapd capture locator in the auto-discovered member namespace under nfcapd root {} (including direct-child directory paths) through {}", - output.display(), - root.display(), - related.display() - ))); - } - } - } - } - Ok(()) -} - -/// Reject output paths that a CSV tree would discover after SQLite creates them. Discovery is -/// intentionally flat, so only a direct child of the configured tree root can change its input -/// set. -fn validate_output_csv_tree_separation( +fn freeze_nfcapd_tree( + input: &InputSpec, + selection: &FlowSelection, + timezone: &str, output_paths: &[&Path], - trees: &[(PathBuf, CsvSourceConfig)], -) -> Result<(), PipelineError> { - for output in output_paths { - for related in output_related_paths(output)? { - for (root, mapping) in trees { - if related.parent() != Some(root.as_path()) { - continue; - } - let Some(name) = related.file_name().and_then(|name| name.to_str()) else { - continue; - }; - let lowercase_name = name.to_ascii_lowercase(); - let excluded = mapping - .discovery_exclude_suffixes - .iter() - .any(|suffix| lowercase_name.ends_with(&suffix.to_ascii_lowercase())); - if !excluded && ingest::matches_csv_discovery(&lowercase_name, mapping) { - return Err(PipelineError::InvalidConfig(format!( - "output database {} would be discovered as a CSV tree input at {} under {}", - output.display(), - related.display(), - root.display() - ))); - } - } - } - } - Ok(()) -} - -fn validate_daily_active_source_layout( - sources: &[DatasetSource], - physical_ids: &[String], -) -> Result<(), PipelineError> { - if sources.is_empty() { +) -> Result { + let InputSpec::NfcapdTree { + root_path, + source_ids, + sources, + start_date, + end_date, + start_time, + end_time, + force, + } = input + else { return Err(PipelineError::InvalidConfig( - "daily_active_sources requires at least one logical source".into(), + "expected an nfcapd_tree input".into(), )); - } - if physical_ids.is_empty() { + }; + if selection.selects_daily_active_sources() && (start_time.is_some() || end_time.is_some()) { return Err(PipelineError::InvalidConfig( - "daily_active_sources requires at least one physical source member".into(), + "daily_active_sources selection requires whole local calendar days; start_time and end_time are unsupported".into(), )); } - Ok(()) -} - -/// Identity of a planned physical member directory. -/// -/// The canonical path catches a retargeted root/member symlink or a renamed directory. On Unix, -/// the device/inode pair also catches a replacement at the same configured path. Directory -/// timestamps are intentionally not part of this identity: normal capture creation changes them. -#[derive(Clone, Debug, PartialEq, Eq)] -struct MemberDirectoryIdentity { - canonical_path: PathBuf, - #[cfg(unix)] - device: u64, - #[cfg(unix)] - inode: u64, -} - -fn capture_member_directory_identity( - root: &Path, - member: &str, -) -> Result { - let member_path = root.join(member); - let canonical_member_path = canonical_path(&member_path)?; - #[cfg(unix)] - { - let (device, inode) = existing_path_identity(&member_path)?.ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "source member directory {:?} disappeared while its identity was being captured", - member - )) - })?; - Ok(MemberDirectoryIdentity { - canonical_path: canonical_member_path, - device, - inode, - }) - } - #[cfg(not(unix))] - { - Ok(MemberDirectoryIdentity { - canonical_path: canonical_member_path, - }) - } -} -fn capture_member_directory_identities( - root: &Path, - physical_ids: &[String], -) -> Result, PipelineError> { - physical_ids + let root_path = fs::canonicalize(root_path)?; + let sources = normalize_sources(&root_path, source_ids, sources)?; + let physical_ids = sources .iter() - .map(|member| { - capture_member_directory_identity(root, member) - .map(|identity| (member.clone(), identity)) - }) - .collect() -} - -fn verify_member_directory_identities( - root: &Path, - expected: &BTreeMap, -) -> Result<(), PipelineError> { - for (member, expected_identity) in expected { - let current = capture_member_directory_identity(root, member)?; - if current != *expected_identity { - return Err(PipelineError::InvalidConfig(format!( - "nfcapd member directory {:?} changed during the pipeline: planned {} but found {}", - member, - expected_identity.canonical_path.display(), - current.canonical_path.display() - ))); - } + .flat_map(|source| source.members.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + if selection.selects_daily_active_sources() { + validate_daily_active_source_layout(&sources, &physical_ids)?; } - Ok(()) -} - -#[derive(Clone, Debug)] -struct FrozenNfcapdTreeLayout { - root_path: PathBuf, - sources: Vec, - physical_ids: Vec, - member_identities: BTreeMap, - auto_discovered: bool, -} - -#[derive(Clone, Debug, Default)] -struct SingleOutputPlan { - trees: BTreeMap, - dataset_sources: BTreeMap>, - capture_snapshots: BTreeMap, -} + validate_output_capture_separation(output_paths, &root_path, &physical_ids)?; -impl SingleOutputPlan { - fn first_root(&self) -> Option<&Path> { - self.trees - .values() - .next() - .map(|tree| tree.root_path.as_path()) + let discovered = ingest::discover_nfcapd_source_paths(&root_path, &physical_ids, timezone)?; + let mut by_member_and_start = BTreeMap::new(); + let mut member_bounds = BTreeMap::new(); + for input in discovered { + member_bounds + .entry(input.source_id.clone()) + .and_modify(|(first, last): &mut (i64, i64)| { + *first = (*first).min(input.bucket_start); + *last = (*last).max(input.bucket_start); + }) + .or_insert((input.bucket_start, input.bucket_start)); + by_member_and_start.insert((input.source_id, input.bucket_start), input.path); } + let window = resolve_nfcapd_tree_window( + start_date, + end_date.as_deref(), + start_time.as_deref(), + end_time.as_deref(), + by_member_and_start.keys().map(|(_, start)| *start), + timezone, + )?; + + Ok(FrozenNfcapdTreeLayout { + root_path, + sources, + physical_ids, + by_member_and_start, + member_bounds, + start: window.start, + end: window.end, + extend_gaps_to_window: end_date.is_some(), + force: *force, + }) } -/// Perform the read-only nfcapd discovery needed to protect a single output before opening it. -/// -/// The returned layout is the only source membership snapshot used by output initialization, -/// processing, and strict coverage checks. Auto-discovered layouts are revalidated by -/// [`verify_single_auto_source_layouts`] after all read-only planning and immediately before -/// output setup. fn plan_single_output(pipeline: &ResolvedPipeline) -> Result { - verify_nfdump_revision(pipeline)?; - let mut locator_namespaces = BTreeSet::new(); - let mut auto_discovery_roots = BTreeSet::new(); - let mut csv_tree_configs = Vec::new(); - let mut nfcapd_windows = Vec::new(); - let mut nfcapd_capture_paths = BTreeSet::new(); - let mut trees = BTreeMap::new(); let output_path = pipeline.database_path.as_path(); let output_paths = std::slice::from_ref(&output_path); - validate_output_input_separation( - output_paths, - pipeline.control_paths.iter().map(PathBuf::as_path), - "pipeline control path", - )?; + let mut trees = BTreeMap::new(); for (input_index, input) in pipeline.inputs.iter().enumerate() { - match input { - InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - start_date, - end_date, - start_time, - end_time, - .. - } => { - let root = canonical_path(root_path)?; - let auto_discovered = source_ids.is_empty() && sources.is_empty(); - if auto_discovered { - auto_discovery_roots.insert(root.clone()); - } - let sources = normalize_sources(&root, source_ids, sources)?; - let physical_ids = sources - .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>() - .into_iter() - .collect::>(); - let member_identities = capture_member_directory_identities(&root, &physical_ids)?; - trees.insert( - input_index, - FrozenNfcapdTreeLayout { - root_path: root.clone(), - sources: sources.clone(), - physical_ids: physical_ids.clone(), - member_identities, - auto_discovered, - }, - ); - locator_namespaces.extend(nfcapd_locator_namespaces(&root, &physical_ids)?); - if pipeline.selection.selects_daily_active_sources() { - validate_daily_active_source_layout(&sources, &physical_ids)?; - if start_time.is_some() || end_time.is_some() { - return Err(PipelineError::InvalidConfig( - "daily_active_sources selection requires whole local calendar days; start_time and end_time are unsupported".into(), - )); - } - } - let discovered = - ingest::discover_nfcapd_source_paths(&root, &physical_ids, &pipeline.timezone)?; - nfcapd_capture_paths.extend(discovered.iter().map(|input| input.path.clone())); - nfcapd_windows.push(( - start_date.clone(), - end_date.clone(), - start_time.clone(), - end_time.clone(), - discovered.iter().map(|input| input.bucket_start).max(), - )); - } - InputSpec::Nfcapd { - path, - gap, - expected_path, - .. - } => { - if *gap { - if let Some(expected_path) = expected_path { - validate_output_capture_separation( - output_paths, - std::iter::once(expected_path.as_path()), - )?; - } - } else { - validate_output_capture_separation( - output_paths, - std::iter::once(path.as_path()), - )?; - } - } - InputSpec::Csv { path, mapping_path } => { - validate_output_input_separation( - output_paths, - [path.as_path(), mapping_path.as_path()], - "discovered CSV input", - )?; - } - InputSpec::CsvTree { - root_path, - mapping_path, - } => { - let mapping = CsvSourceConfig::load(mapping_path)?; - let discovered = ingest::discover_csv_inputs(root_path, mapping_path, &mapping)?; - validate_output_input_separation( - output_paths, - std::iter::once(mapping_path.as_path()) - .chain(discovered.iter().map(|input| input.path.as_path())), - "discovered CSV input", - )?; - csv_tree_configs.push((canonical_path(root_path)?, mapping)); - } + if matches!(input, InputSpec::NfcapdTree { .. }) { + trees.insert( + input_index, + freeze_nfcapd_tree(input, &pipeline.selection, &pipeline.timezone, output_paths)?, + ); } } - let locator_namespaces = locator_namespaces.into_iter().collect::>(); - validate_output_nfcapd_locator_separation(output_paths, &locator_namespaces)?; - validate_output_nfcapd_auto_namespace_separation( - output_paths, - &auto_discovery_roots.into_iter().collect::>(), - )?; - validate_output_csv_tree_separation(output_paths, &csv_tree_configs)?; - let capture_snapshots = if output_has_existing_identity(output_paths)? { - let capture_snapshots = capture_nfcapd_snapshots(&nfcapd_capture_paths)?; - validate_output_nfcapd_capture_separation_with_snapshots( - output_paths, - &locator_namespaces, - capture_snapshots - .iter() - .map(|(path, snapshot)| (path.as_path(), snapshot)), - )?; - capture_snapshots - } else { - BTreeMap::new() - }; - for (start_date, end_date, start_time, end_time, discovered_end) in nfcapd_windows { - resolve_nfcapd_tree_window( - &start_date, - end_date.as_deref(), - start_time.as_deref(), - end_time.as_deref(), - discovered_end, - &pipeline.timezone, - )?; - } - if let Some(revision) = &pipeline.nfdump_revision { + if pipeline.nfdump_revision.is_some() { ingest::probe_nfdump_compatibility(&pipeline.nfdump)?; - verify_file_snapshot(&pipeline.nfdump, &revision.snapshot)?; } - verify_nfdump_revision(pipeline)?; + let mut dataset_sources = BTreeMap::new(); for dataset in &pipeline.datasets { - let dataset_root = canonical_path(&dataset.root_path)?; + let dataset_root = fs::canonicalize(&dataset.root_path)?; let sources = trees .values() .find(|tree| tree.root_path == dataset_root) @@ -1638,36 +781,9 @@ fn plan_single_output(pipeline: &ResolvedPipeline) -> Result Result<(), PipelineError> { - for tree in plan.trees.values() { - verify_member_directory_identities(&tree.root_path, &tree.member_identities)?; - if tree.auto_discovered { - let current = normalize_sources(&tree.root_path, &[], &[])?; - if current != tree.sources { - return Err(PipelineError::InvalidConfig( - "auto-discovered source layout changed during single-output planning".into(), - )); - } - } - } - // Keep this parameter in the validation seam so a future pipeline with multiple roots can - // report the owning dataset without rediscovering its metadata. - let _ = pipeline; - Ok(()) -} - -#[cfg(test)] -fn preflight_single_output(pipeline: &ResolvedPipeline) -> Result<(), PipelineError> { - plan_single_output(pipeline).map(|_| ()) -} - fn resolve_request(request: &PipelineRequest) -> Result { match (&request.config_path, &request.dataset_id) { (Some(_), Some(_)) => return Err(PipelineError::ConflictingModes), @@ -1712,13 +828,8 @@ fn resolve_request(request: &PipelineRequest) -> Result Result Result { - #[cfg(test)] - DATASET_REGISTRY_LOAD_CALLS.with(|calls| calls.set(calls.get() + 1)); Ok(DatasetRegistry::load(registry_path, repository_root)?) } fn resolve_dataset_request( request: &PipelineRequest, - registry_path: &Path, registry: &DatasetRegistry, shared_nfdump: Option<(&Path, &ExecutableRevision)>, ) -> Result { @@ -1781,14 +889,11 @@ fn resolve_dataset_request( (path, Some(revision)) } }; - let mut control_paths = vec![registry_path.to_owned()]; - control_paths.push(nfdump.clone()); Ok(ResolvedPipeline { database_path: request .database_path .clone() .unwrap_or_else(|| dataset.db_path.clone()), - control_paths, timezone: DEFAULT_TIMEZONE.into(), run_maad: request.run_maad, nfdump, @@ -1829,11 +934,6 @@ fn validate_selection_inputs( fn execute(pipeline: ResolvedPipeline) -> Result { let plan = plan_single_output(&pipeline)?; - if let Some(root) = plan.first_root() { - invoke_single_plan_hook(root); - } - verify_single_auto_source_layouts(&pipeline, &plan)?; - verify_nfdump_revision(&pipeline)?; if let Some(parent) = pipeline.database_path.parent() { fs::create_dir_all(parent)?; } @@ -1874,27 +974,19 @@ fn execute(pipeline: ResolvedPipeline) -> Result &mapping, )?); } - InputSpec::NfcapdTree { - start_date, - end_date, - start_time, - end_time, - force, - .. - } => process_nfcapd_tree( - &connection, - plan.trees - .get(&input_index) - .expect("every nfcapd_tree input has a frozen layout"), - start_date, - end_date.as_deref(), - start_time.as_deref(), - end_time.as_deref(), - *force, - &pipeline, - &plan.capture_snapshots, - &mut report, - )?, + InputSpec::NfcapdTree { .. } => { + let mut sinks = [ProductSink { + pipeline: &pipeline, + connection: &connection, + report: &mut report, + }]; + process_nfcapd_tree( + plan.trees + .get(&input_index) + .expect("every nfcapd_tree input has a frozen layout"), + &mut sinks, + )?; + } } } merge_report( @@ -1924,11933 +1016,2892 @@ fn execute(pipeline: ResolvedPipeline) -> Result struct CoordinatedOutput { pipeline: ResolvedPipeline, - sources: Vec, connection: Connection, + report: PipelineReport, _lock: DatabaseOperationLock, } -type NfcapdFingerprintKey = (String, u64, u64, u64, i64, i64); +struct ProductSink<'a> { + pipeline: &'a ResolvedPipeline, + connection: &'a Connection, + report: &'a mut PipelineReport, +} -/// Resume state for one output and one local day. -/// -/// Coordinated preparation revisits every logical source/bucket, but the state needed for those -/// decisions is limited to this day. Keeping the maps here avoids retaining prior days or loading -/// a product's complete provenance history into memory. -#[derive(Clone, Debug, Default)] -struct NfcapdDayResumeCache { - evidence: BTreeMap<(String, i64), Vec>, - processed: BTreeMap<(String, i64), BTreeSet<(String, String)>>, - fingerprints: BTreeMap, -} - -impl NfcapdDayResumeCache { - fn load( - connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, - ) -> Result { - let source_ids = sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>() - .into_iter() - .collect::>(); - let mut cache = Self::default(); - for row in query_input_evidence_range(connection, &source_ids, start, end)? { - cache - .evidence - .entry((row.source_id.clone(), row.bucket_start)) - .or_default() - .push(row); - } - for row in query_processed_nfcapd_range(connection, &source_ids, start, end)? { - cache - .processed - .entry((row.source_id.clone(), row.bucket_start)) - .or_default() - .insert((row.input_locator.clone(), row.revision_fingerprint)); - if let Some(snapshot) = row.file_snapshot { - cache - .fingerprints - .entry(Self::fingerprint_key(&row.input_locator, &snapshot)) - .or_insert(row.content_fingerprint); - } - } - Ok(cache) - } - - fn fingerprint_key(locator: &str, snapshot: &FileSnapshot) -> NfcapdFingerprintKey { - ( - locator.to_owned(), - snapshot.device, - snapshot.inode, - snapshot.size, - snapshot.mtime_ns, - snapshot.ctime_ns, - ) - } - - fn evidence(&self, source_id: &str, bucket_start: i64) -> &[InputEvidenceRow] { - self.evidence - .get(&(source_id.to_owned(), bucket_start)) - .map(Vec::as_slice) - .unwrap_or(&[]) - } - - fn processed( - &self, - source_id: &str, - bucket_start: i64, - revisions: &[InputRevision], - ) -> Result { - if revisions.is_empty() { - return Ok(false); - } - let requested = revisions - .iter() - .map(|revision| (revision.locator.clone(), revision.fingerprint.clone())) - .collect::>(); - let stored = self - .processed - .get(&(source_id.to_owned(), bucket_start)) - .cloned() - .unwrap_or_default(); - let stored_locators = stored - .iter() - .map(|(locator, _)| locator) - .collect::>(); - let requested_locators = requested - .iter() - .map(|(locator, _)| locator) - .collect::>(); - if stored_locators == requested_locators && stored != requested { - return Err(PipelineError::Storage( - StorageError::InputRevisionConflict { - locator: format!("{source_id}:{bucket_start}"), - components: "nfcapd content or decoder; rerun with force to rewrite it" - .to_owned(), - }, - )); - } - Ok(stored == requested) - } - - fn cached_content_fingerprint(&self, path: &Path, snapshot: &FileSnapshot) -> Option { - self.fingerprints - .get(&Self::fingerprint_key(&path.to_string_lossy(), snapshot)) - .cloned() - } -} - -/// Worker pools shared by every day in a coordinated run. -/// -/// Revision hashing is needed while deciding whether a day has pending work, so it is built when -/// the run starts. Decode and activity work are both lazy: a complete no-op run never allocates -/// either pool, and a multi-day run reuses each pool after its first pending day. -struct CoordinatedPools { - revision: rayon::ThreadPool, - decode: Option, - activity: Option, -} - -impl CoordinatedPools { - fn new() -> Result { - Ok(Self { - revision: build_revision_hash_pool()?, - decode: None, - activity: None, - }) - } - - fn decode(&mut self) -> Result<&rayon::ThreadPool, PipelineError> { - if self.decode.is_none() { - self.decode = Some(build_nfcapd_decode_pool()?); - } - Ok(self - .decode - .as_ref() - .expect("decode pool was just initialized")) - } - - fn activity(&mut self) -> Result<&rayon::ThreadPool, PipelineError> { - if self.activity.is_none() { - self.activity = Some(build_nfcapd_activity_pool()?); - } - Ok(self - .activity - .as_ref() - .expect("activity pool was just initialized")) - } -} - -/// Prepared coordinated work retained between the day preflight and publication pass. -/// -/// Each entry is bounded by the existing nfcapd decode batch; the coordinated day retains one -/// such entry per batch. Publication takes ownership of each batch before decoding so prepared -/// evidence does not remain live alongside decoded buckets. -struct PreparedCoordinatedBatch { - prepared: BTreeMap>, -} - -#[derive(Debug, Default)] -struct CoordinatedDaySharedProfile { - revision_elapsed: Duration, - eligibility_elapsed: Duration, - activity_elapsed: Duration, - decode_elapsed: Duration, - revision_paths: u64, - activity_members: u64, - activity_inputs: u64, - active_set_counts: Vec, -} - -impl CoordinatedDaySharedProfile { - fn log(&self, day_start: i64, day_end: i64) { - tracing::info!( - target: "netflow_db::profile", - phase = "coordinated_day_shared", - day_start, - day_end, - revision_seconds = self.revision_elapsed.as_secs_f64(), - eligibility_seconds = self.eligibility_elapsed.as_secs_f64(), - activity_seconds = self.activity_elapsed.as_secs_f64(), - decode_seconds = self.decode_elapsed.as_secs_f64(), - revision_paths = self.revision_paths, - activity_members = self.activity_members, - activity_inputs = self.activity_inputs, - active_set_counts = ?self.active_set_counts, - ); - } -} - -struct CoordinatedPlan { - root_path: PathBuf, - sources: Vec, - dataset_sources: BTreeMap>, - physical_ids: Vec, - member_identities: BTreeMap, - auto_discovered_datasets: BTreeSet, - by_member_and_start: BTreeMap<(String, i64), PathBuf>, - capture_snapshots: BTreeMap, - member_bounds: BTreeMap, - start: i64, - end: i64, - extend_gaps_to_window: bool, - force: bool, - timezone: String, -} - -/// Resolve all read-only coordinated inputs before creating an output directory, lock, or -/// SQLite schema. The canonical root is also the shared locator root for every dataset output. -fn plan_coordinated(pipelines: &[ResolvedPipeline]) -> Result { - let first = pipelines - .first() - .ok_or_else(|| PipelineError::InvalidConfig("coordinated mode has no pipelines".into()))?; - let first_input = only_nfcapd_tree(first)?; - let first_config = nfcapd_tree_config(first_input)?; - let root_path = canonical_path(first_config.root_path)?; - let mut sources = None; +fn execute_many(plan: CompatiblePlan) -> Result { + let CompatiblePlan { pipelines, tree } = plan; let mut dataset_sources = BTreeMap::new(); - let mut auto_discovered_datasets = BTreeSet::new(); - for pipeline in pipelines { - let input = only_nfcapd_tree(pipeline)?; - let config = nfcapd_tree_config(input)?; - if root_path != canonical_path(config.root_path)? { - return Err(PipelineError::InvalidConfig( - "coordinated datasets must use the same nfcapd root".into(), - )); - } - let resolved_sources = canonical_logical_sources(input)?; - if let Some(expected) = &sources { - if expected != &resolved_sources { - return Err(PipelineError::InvalidConfig( - "coordinated datasets must use the same logical source layout and membership" - .into(), - )); - } - } else { - sources = Some(resolved_sources.clone()); - } - let dataset_id = pipeline - .datasets - .first() - .map(|dataset| dataset.dataset_id.clone()) - .ok_or_else(|| { - PipelineError::InvalidConfig( - "coordinated datasets require registry-backed dataset metadata".into(), - ) - })?; - dataset_sources.insert(dataset_id.clone(), resolved_sources); - if matches!( - input, - InputSpec::NfcapdTree { - source_ids, - sources, - .. - } if source_ids.is_empty() && sources.is_empty() - ) { - auto_discovered_datasets.insert(dataset_id); - } - } - let sources = sources.expect("coordinated plan has at least one pipeline"); - let selected_start = parse_date_start(first_config.start_date, first.timezone.as_str())?; - let explicit_end = first_config - .end_date - .map(|date| next_date_start(date, first.timezone.as_str())) - .transpose()?; - let physical_ids = sources - .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>() - .into_iter() - .collect::>(); - validate_daily_active_source_layout(&sources, &physical_ids)?; - let member_identities = capture_member_directory_identities(&root_path, &physical_ids)?; - - // Parse explicit dates before discovery as well as before output setup. This keeps malformed - // finite windows side-effect free even when the source tree is large. - let discovery_started = Instant::now(); - let discovered = - ingest::discover_nfcapd_source_paths(&root_path, &physical_ids, first.timezone.as_str())?; - tracing::info!( - target: "netflow_db::profile", - phase = "coordinated_discovery", - elapsed_seconds = discovery_started.elapsed().as_secs_f64(), - physical_sources = physical_ids.len(), - discovered_inputs = discovered.len(), - ); - let mut by_member_and_start = BTreeMap::new(); - let mut member_bounds = BTreeMap::new(); - for input in discovered { - member_bounds - .entry(input.source_id.clone()) - .and_modify(|(first, last): &mut (i64, i64)| { - *first = (*first).min(input.bucket_start); - *last = (*last).max(input.bucket_start); - }) - .or_insert((input.bucket_start, input.bucket_start)); - by_member_and_start.insert((input.source_id, input.bucket_start), input.path); - } - let discovered_end = by_member_and_start - .keys() - .map(|(_, bucket_start)| *bucket_start) - .max() - .map(|start| aggregate_bounds(start, Granularity::OneDay, first.timezone.as_str())) - .transpose()? - .map(|(_, end)| end) - .unwrap_or(selected_start); - let selected_end = explicit_end.unwrap_or(discovered_end); - let start = match first_config.start_time { - Some(value) => parse_local_datetime(value, first.timezone.as_str())?, - None => selected_start, - }; - let end = match first_config.end_time { - Some(value) => parse_local_datetime(value, first.timezone.as_str())?, - None => selected_end, - }; - validate_window( - selected_start, - selected_end, - start, - end, - first.timezone.as_str(), - )?; - - Ok(CoordinatedPlan { - root_path, - sources, - dataset_sources, - physical_ids, - member_identities, - auto_discovered_datasets, - by_member_and_start, - capture_snapshots: BTreeMap::new(), - member_bounds, - start, - end, - extend_gaps_to_window: first_config.end_date.is_some(), - force: first_config.force, - timezone: first.timezone.clone(), - }) -} - -/// Re-check every auto-discovered dataset after all coordinated read-only planning and before -/// creating any output parent, lock, or database. Explicit layouts were already identity-checked -/// by [`normalize_sources`] while the plan was built. -fn verify_coordinated_auto_source_layouts( - pipelines: &[ResolvedPipeline], - plan: &CoordinatedPlan, -) -> Result<(), PipelineError> { - for pipeline in pipelines { + for pipeline in &pipelines { let dataset = pipeline.datasets.first().ok_or_else(|| { PipelineError::InvalidConfig( "coordinated datasets require registry-backed dataset metadata".into(), ) })?; - if !plan.auto_discovered_datasets.contains(&dataset.dataset_id) { - continue; - } - let input = only_nfcapd_tree(pipeline)?; - let current = canonical_logical_sources(input)?; - let expected = plan - .dataset_sources - .get(&dataset.dataset_id) - .expect("every coordinated dataset has a frozen source layout"); - if current != *expected { - return Err(PipelineError::InvalidConfig(format!( - "auto-discovered source layout changed for dataset {:?} during coordinated planning", - dataset.dataset_id - ))); - } + dataset_sources.insert(dataset.dataset_id.clone(), tree.sources.clone()); } - Ok(()) -} -/// Execute the shared physical nfcapd scan while keeping each logical product independent. -/// -/// Preparation is performed against every output so resume decisions remain output-local. Once a -/// batch is known to be needed, its physical files are decoded once and the canonical buckets are -/// fanned out to the outputs whose pending jobs reference them. -fn execute_many(pipelines: Vec) -> Result { - let output_paths = pipelines - .iter() - .map(|pipeline| pipeline.database_path.as_path()) - .collect::>(); - validate_output_input_separation( - &output_paths, - pipelines - .iter() - .flat_map(|pipeline| pipeline.control_paths.iter().map(PathBuf::as_path)), - "pipeline control path", - )?; - validate_database_path_separation(&output_paths)?; - let mut plan = plan_coordinated(&pipelines)?; - invoke_coordinated_plan_hook(&plan.root_path); - verify_member_directory_identities(&plan.root_path, &plan.member_identities)?; - for pipeline in &pipelines { - verify_nfdump_revision(pipeline)?; - } - let locator_namespaces = nfcapd_locator_namespaces(&plan.root_path, &plan.physical_ids)?; - if output_has_existing_identity(&output_paths)? { - let capture_paths = plan - .by_member_and_start - .values() - .cloned() - .collect::>(); - plan.capture_snapshots = capture_nfcapd_snapshots(&capture_paths)?; - validate_output_nfcapd_capture_separation_with_snapshots( - &output_paths, - &locator_namespaces, - plan.capture_snapshots - .iter() - .map(|(path, snapshot)| (path.as_path(), snapshot)), - )?; - } else { - validate_output_nfcapd_locator_separation(&output_paths, &locator_namespaces)?; - } - if !plan.auto_discovered_datasets.is_empty() { - validate_output_nfcapd_auto_namespace_separation( - &output_paths, - std::slice::from_ref(&plan.root_path), - )?; - } - if pipelines - .first() - .is_some_and(|pipeline| pipeline.nfdump_revision.is_some()) - { - ingest::probe_nfdump_compatibility(&pipelines[0].nfdump)?; - for pipeline in &pipelines { - verify_nfdump_revision(pipeline)?; - } - } - verify_coordinated_auto_source_layouts(&pipelines, &plan)?; - let mut lock_order = (0..pipelines.len()).collect::>(); - lock_order.sort_unstable_by_key(|index| normalized_path_key(&pipelines[*index].database_path)); - let mut locks: Vec> = - (0..pipelines.len()).map(|_| None).collect(); - for index in lock_order { - let pipeline = &pipelines[index]; + let mut outputs = Vec::with_capacity(pipelines.len()); + for pipeline in pipelines { if let Some(parent) = pipeline.database_path.parent() { fs::create_dir_all(parent)?; } - locks[index] = Some(DatabaseOperationLock::acquire( - &pipeline.database_path, - "coordinated pipeline build", - )?); - } - - let mut outputs = Vec::with_capacity(pipelines.len()); - let mut initialization_transactions = vec![false; pipelines.len()]; - for (index, pipeline) in pipelines.into_iter().enumerate() { - let connection = match connect_pipeline_writer(&pipeline.database_path) { - Ok(connection) => connection, - Err(error) => { - rollback_coordinated_transactions(&outputs, &initialization_transactions); - return Err(coordinated_output_error(&pipeline, error.into())); - } - }; + let lock = + DatabaseOperationLock::acquire(&pipeline.database_path, "coordinated pipeline build")?; + let connection = connect_pipeline_writer(&pipeline.database_path)?; + init_schema(&connection)?; + with_transaction(&connection, || { + initialize_coordinated_metadata_in_transaction( + &connection, + &pipeline, + &tree.sources, + &dataset_sources, + ) + })?; outputs.push(CoordinatedOutput { pipeline, - sources: plan.sources.clone(), connection, - _lock: locks[index] - .take() - .expect("every coordinated output has a lock"), + report: PipelineReport::default(), + _lock: lock, }); - let initialization = (|| { - outputs[index] - .connection - .execute_batch("BEGIN IMMEDIATE") - .map_err(StorageError::from)?; - initialization_transactions[index] = true; - init_schema(&outputs[index].connection)?; - initialize_coordinated_metadata_in_transaction( - &outputs[index].connection, - &outputs[index].pipeline, - &plan.sources, - &plan.dataset_sources, - ) - })(); - if let Err(error) = initialization { - rollback_coordinated_transactions(&outputs, &initialization_transactions); - return Err(coordinated_output_error(&outputs[index].pipeline, error)); - } - } - if let Err(error) = verify_coordinated_nfdump_revisions(&outputs) { - rollback_coordinated_transactions(&outputs, &initialization_transactions); - let pipeline = &outputs[0].pipeline; - return Err(coordinated_output_error(pipeline, error)); - } - for (index, output) in outputs.iter().enumerate() { - if let Err(error) = output.connection.execute_batch("COMMIT") { - rollback_coordinated_transactions(&outputs, &initialization_transactions); - return Err(coordinated_output_error( - &output.pipeline, - PipelineError::Storage(StorageError::from(error)), - )); - } - initialization_transactions[index] = false; } - let mut pools = CoordinatedPools::new()?; - let mut report = PipelineReport::default(); - let mut day_start = plan.start; - while day_start < plan.end { - verify_member_directory_identities(&plan.root_path, &plan.member_identities)?; - let day_end = aggregate_bounds(day_start, Granularity::OneDay, &plan.timezone)?.1; - let capture_complete = day_capture_is_complete( - &plan.sources, - &plan.by_member_and_start, - day_start, - day_end, - &plan.timezone, - )?; - let mut stale_outputs = Vec::new(); - let mut canonical_day_verified = vec![false; outputs.len()]; - let mut marker_needs_backfill = vec![false; outputs.len()]; - for (index, output) in outputs.iter().enumerate() { - let published_day = - day_was_published(&output.connection, &output.sources, day_start, day_end)?; - if !capture_complete { - if published_day { - stale_outputs.push(index); - } - } else if published_day && !plan.force { - match nfcapd_day_completion_state( - &output.connection, - &output.sources, - day_start, - day_end, - output.pipeline.run_maad, - )? { - DailyProductCompletionState::Clean => { - canonical_day_verified[index] = true; - } - DailyProductCompletionState::Dirty => { - return Err(PipelineError::InvalidConfig(format!( - "published local day {day_start}..{day_end} was mutated after completion for database {}; rerun that whole day with --force", - output.pipeline.database_path.display() - ))); - } - DailyProductCompletionState::Missing => { - if !nfcapd_day_has_canonical_topology( - &output.connection, - &output.sources, - day_start, - day_end, - &plan.timezone, - output.pipeline.run_maad, - )? { - return Err(PipelineError::InvalidConfig(format!( - "published local day {day_start}..{day_end} has damaged canonical topology for database {}; rerun that whole day with --force", - output.pipeline.database_path.display() - ))); - } - canonical_day_verified[index] = true; - marker_needs_backfill[index] = true; - } - } - } - } - let reset_outputs = if plan.force { - (0..outputs.len()).collect::>() - } else { - stale_outputs - }; - if !reset_outputs.is_empty() && !plan.force { - return Err(PipelineError::InvalidConfig(format!( - "published local day {day_start}..{day_end} no longer has complete nfcapd capture coverage; rerun that day with --force" - ))); - } - let missing = missing_physical_day_inputs( - &plan.physical_ids, - &plan.by_member_and_start, - day_start, - day_end, - &plan.timezone, - )?; - let missing_absences = build_missing_day_absences( - &plan.root_path, - &missing, - day_start, - day_end, - &plan.timezone, - )?; - invoke_missing_day_absence_hook(&plan.root_path, &missing, &plan.timezone); - if !missing.is_empty() { - let missing_details = - missing_day_warning_details(&plan.root_path, &missing, &plan.timezone)?; - tracing::warn!( - day_start, - day_end, - missing_inputs = missing.len(), - missing_details = %missing_details, - "skipping incomplete physical day for coordinated selections" - ); - report.skipped_inputs += missing.len(); - if reset_outputs.is_empty() { - day_start = day_end; - continue; - } - } - - let day_reports = process_coordinated_day( - &mut outputs, - &plan.root_path, - &plan.physical_ids, - &plan.member_identities, - &plan.by_member_and_start, - &plan.member_bounds, - day_start, - day_end, - plan.extend_gaps_to_window, - plan.force, - &reset_outputs, - !missing.is_empty(), - &missing_absences, - &canonical_day_verified, - &marker_needs_backfill, - &plan.capture_snapshots, - &mut pools, - )?; - for day_report in day_reports { - merge_report(&mut report, day_report); - } - day_start = day_end; + { + let mut sinks = outputs + .iter_mut() + .map(|output| ProductSink { + pipeline: &output.pipeline, + connection: &output.connection, + report: &mut output.report, + }) + .collect::>(); + process_nfcapd_tree(&tree, &mut sinks)?; } - for output in &outputs { + let mut report = PipelineReport::default(); + for output in &mut outputs { infer_default_start_dates(&output.connection, &output.pipeline)?; - let mut coverage_report = PipelineReport::default(); - populate_coverage_summary(&output.connection, &mut coverage_report)?; - report.complete_five_minute_buckets += coverage_report.complete_five_minute_buckets; - report.partial_five_minute_buckets += coverage_report.partial_five_minute_buckets; - report.unknown_five_minute_buckets += coverage_report.unknown_five_minute_buckets; + populate_coverage_summary(&output.connection, &mut output.report)?; if let Err(error) = optimize_all_query_planner_statistics(&output.connection) { tracing::warn!(%error, "could not refresh SQLite planner statistics"); } if output.pipeline.require_complete { let incomplete = count_incomplete_coverage_for_layout( &output.connection, - &plan.sources, - plan.start, - plan.end, - &plan.timezone, + &tree.sources, + tree.start, + tree.end, + &output.pipeline.timezone, )?; if incomplete != 0 { - let dataset_id = output - .pipeline - .datasets - .first() - .map_or("", |dataset| dataset.dataset_id.as_str()); - return Err(PipelineError::InvalidConfig(format!( - "dataset {dataset_id:?} database {} has {incomplete} incomplete five-minute coverage buckets", - output.pipeline.database_path.display() - ))); + return Err(PipelineError::IncompleteCoverage(incomplete)); } } + merge_report(&mut report, std::mem::take(&mut output.report)); } Ok(report) } - -fn verify_coordinated_postflight_snapshot( - path: &Path, - snapshot: &FileSnapshot, -) -> Result<(), ProvenanceError> { - #[cfg(test)] - COORDINATED_POSTFLIGHT_SNAPSHOT_VERIFICATIONS.with(|calls| calls.set(calls.get() + 1)); - verify_file_snapshot(path, snapshot) -} - -fn verify_coordinated_nfdump_revisions(outputs: &[CoordinatedOutput]) -> Result<(), PipelineError> { - for output in outputs { - verify_nfdump_revision(&output.pipeline)?; - } - Ok(()) -} - -/// Check every external input guard while all coordinated output transactions are still open. -/// -/// This is deliberately the last fallible phase before the commit loop. Keeping the loop itself -/// to COMMIT and transaction bookkeeping prevents a late capture or decoder replacement from -/// making one output commit while another rolls back. -#[allow(clippy::too_many_arguments)] -fn verify_coordinated_precommit_guards<'a, 'b>( - outputs: &[CoordinatedOutput], - root: &Path, - member_identities: &BTreeMap, - revisions: impl IntoIterator, - activity_snapshots: impl IntoIterator, - missing_absences: &[ExpectedAbsence], - start: i64, - end: i64, +fn infer_default_start_dates( + connection: &Connection, + pipeline: &ResolvedPipeline, ) -> Result<(), PipelineError> { - invoke_coordinated_commit_guard_hook(); - verify_member_directory_identities(root, member_identities)?; - for (path, snapshot) in revisions { - verify_coordinated_postflight_snapshot(path, snapshot)?; - } - for (path, snapshot) in activity_snapshots { - verify_coordinated_postflight_snapshot(path, snapshot)?; + let inferred = pipeline + .datasets + .iter() + .filter(|dataset| dataset.default_start_date.trim().is_empty()) + .collect::>(); + if inferred.is_empty() { + return Ok(()); } - verify_coordinated_nfdump_revisions(outputs)?; - verify_missing_day_absences(missing_absences, start, end) -} - -#[allow(clippy::too_many_arguments)] -fn process_coordinated_day( - outputs: &mut [CoordinatedOutput], - root: &Path, - physical_ids: &[String], - member_identities: &BTreeMap, - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - start: i64, - end: i64, - extend_gaps_to_window: bool, - force: bool, - reset_outputs: &[usize], - skip_incomplete_day: bool, - missing_absences: &[ExpectedAbsence], - canonical_day_verified: &[bool], - marker_needs_backfill: &[bool], - capture_snapshots: &BTreeMap, - pools: &mut CoordinatedPools, -) -> Result, PipelineError> { - verify_member_directory_identities(root, member_identities)?; - verify_coordinated_nfdump_revisions(outputs)?; - let reset_set = reset_outputs.iter().copied().collect::>(); - if skip_incomplete_day { - if reset_outputs.is_empty() { - return Ok((0..outputs.len()) - .map(|_| PipelineReport::default()) - .collect()); - } - let mut transactions = (0..outputs.len()).map(|_| false).collect::>(); - for &index in reset_outputs { - if let Err(error) = outputs[index].connection.execute_batch("BEGIN IMMEDIATE") { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(StorageError::from(error))); - } - transactions[index] = true; - let source_ids = outputs[index] - .sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>(); - if let Err(error) = - delete_stats_time_range(&outputs[index].connection, &source_ids, start, end) - { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(error)); - } - } - if let Err(error) = verify_coordinated_precommit_guards( - outputs, - root, - member_identities, - std::iter::empty(), - std::iter::empty(), - missing_absences, - start, - end, - ) { - rollback_coordinated_transactions(outputs, &transactions); - return Err(error); - } - for &index in reset_outputs { - if let Err(error) = outputs[index].connection.execute_batch("COMMIT") { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(StorageError::from(error))); - } - transactions[index] = false; + let Some(bucket_start) = earliest_traffic_bucket_start(connection)? else { + return Ok(()); + }; + let date = local_date(bucket_start, &pipeline.timezone)?; + with_transaction(connection, || { + for dataset in inferred { + set_dataset_default_start_date(connection, &dataset.dataset_id, &date)?; } - return Ok((0..outputs.len()) - .map(|_| PipelineReport::default()) - .collect()); - } + Ok(()) + }) +} - let resume_caches = outputs - .iter() - .map(|output| NfcapdDayResumeCache::load(&output.connection, &output.sources, start, end)) - .collect::, _>>()?; +/// Local calendar day that contains `timestamp`, formatted as `YYYY-MM-DD`. +fn local_date(timestamp: i64, timezone: &str) -> Result { + Ok(Timestamp::from_second(timestamp) + .map_err(|error| PipelineError::Time(error.to_string()))? + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .date() + .to_string()) +} - let mut owned_keys = BTreeSet::new(); - let mut bucket_start = start; - while bucket_start < end { - for source in &outputs[0].sources { - if (force || !reset_set.is_empty()) - && source_has_candidate( - source, - bucket_start, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - ) - { - owned_keys.insert((source.source_id.clone(), bucket_start)); - } +fn with_transaction( + connection: &Connection, + operation: impl FnOnce() -> Result, +) -> Result { + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(StorageError::from)?; + let result = operation().and_then(|value| { + connection + .execute_batch("COMMIT") + .map_err(StorageError::from)?; + Ok(value) + }); + match result { + Ok(value) => Ok(value), + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + Err(error) } - bucket_start = next_local_five_minute_start(bucket_start, &outputs[0].pipeline.timezone)?; } +} - let mut reports = (0..outputs.len()) - .map(|_| PipelineReport::default()) - .collect::>(); - let mut profiles = (0..outputs.len()) - .map(|_| NfcapdDayPublishProfile::default()) +fn initialize_metadata_with_plan( + connection: &Connection, + pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, +) -> Result<(), PipelineError> { + let layouts = plan + .trees + .values() + .flat_map(|tree| tree.sources.iter().cloned()) .collect::>(); - let mut shared_profile = CoordinatedDaySharedProfile { - active_set_counts: vec![0; outputs.len()], - ..CoordinatedDaySharedProfile::default() - }; - let mut pending_set = BTreeSet::new(); - let mut has_repair = false; - let mut batches = Vec::new(); - let mut all_revisions = BTreeMap::new(); - let mut next = start; - while next < end { - let batch_starts = nfcapd_batch_starts( - next, - end, - &outputs[0].pipeline.timezone, - &outputs[0].sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - )?; - next = batch_starts - .last() - .copied() - .map(|last| next_local_five_minute_start(last, &outputs[0].pipeline.timezone)) - .transpose()? - .expect("non-empty coordinated nfcapd batch while processing a non-empty window"); - let revision_started = Instant::now(); - let revisions = resolve_coordinated_batch_revisions_with_cache( - outputs, - &outputs[0].sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - force, - &pools.revision, - &batch_starts, - &resume_caches, - capture_snapshots, - )?; - shared_profile.revision_elapsed += revision_started.elapsed(); - shared_profile.revision_paths += profile_count(revisions.len()); - all_revisions.extend( - revisions - .iter() - .map(|(path, revision)| (path.clone(), revision.clone())), - ); - let mut prepared = BTreeMap::new(); - for (index, output) in outputs.iter().enumerate() { - let output_prepare_started = Instant::now(); - let mut output_batch = Vec::with_capacity(batch_starts.len()); - for &bucket_start in &batch_starts { - let mut preflight_report = PipelineReport::default(); - let timestamp = prepare_nfcapd_tree_timestamp_with_cache( - &output.connection, - root, - &output.sources, - by_member_and_start, - member_bounds, - bucket_start, - extend_gaps_to_window, - force || reset_set.contains(&index), - &output.pipeline, - &mut preflight_report, - &revisions, - canonical_day_verified[index], - Some(&resume_caches[index]), - )?; - reports[index].skipped_inputs += preflight_report.skipped_inputs; - if !timestamp.jobs.is_empty() { - pending_set.insert(index); - } - has_repair |= timestamp.jobs.iter().any(|job| job.is_repair); - output_batch.push(timestamp); - } - let output_prepare_elapsed = output_prepare_started.elapsed(); - profiles[index].prepare_elapsed += output_prepare_elapsed; - shared_profile.eligibility_elapsed += output_prepare_elapsed; - if output_batch - .iter() - .any(|timestamp| !timestamp.jobs.is_empty()) - { - prepared.insert(index, output_batch); - } - } - batches.push(PreparedCoordinatedBatch { prepared }); - } - if !force && has_repair { + let mut source_ids = BTreeSet::new(); + if let Some(duplicate) = layouts + .iter() + .find(|source| !source_ids.insert(source.source_id.clone())) + { return Err(PipelineError::InvalidConfig(format!( - "daily_active_sources input changed for local day {start}..{end}; rerun that whole day with --force" + "nfcapd_tree inputs define duplicate logical source ID {:?}", + duplicate.source_id ))); } + with_transaction(connection, || { + initialize_metadata_in_transaction_with_layouts( + connection, + pipeline, + &layouts, + &plan.dataset_sources, + ) + }) +} - let pending = pending_set.into_iter().collect::>(); - let mut activity_snapshots = Vec::new(); - let mut active_sources = (0..outputs.len()) - .map(|_| None) - .collect::>>>(); - if !pending.is_empty() { - verify_coordinated_nfdump_revisions(outputs)?; - let activity_started = Instant::now(); - let selections = pending +fn initialize_metadata_in_transaction_with_layouts( + connection: &Connection, + pipeline: &ResolvedPipeline, + layouts: &[DatasetSource], + dataset_sources: &BTreeMap>, +) -> Result<(), PipelineError> { + bind_identity(connection, pipeline)?; + for dataset in &pipeline.datasets { + let sources = dataset_sources.get(&dataset.dataset_id).ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "single-output plan has no frozen source layout for dataset {:?}", + dataset.dataset_id + )) + })?; + upsert_dataset_with_sources(connection, dataset, sources)?; + } + if !layouts.is_empty() { + let layout = layouts .iter() - .map(|index| outputs[*index].pipeline.selection.clone()) + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) .collect::>(); - let activity_pool = pools.activity()?; - let (resolved_active_sources, snapshots) = resolve_coordinated_daily_active_sources( - physical_ids, - by_member_and_start, - start, - end, - &outputs[0].pipeline.timezone, - &selections, - outputs[0].pipeline.nfdump.as_path(), - activity_pool, - capture_snapshots, - &all_revisions, - )?; - shared_profile.activity_elapsed += activity_started.elapsed(); - shared_profile.activity_members = profile_count(physical_ids.len()); - activity_snapshots = snapshots; - verify_coordinated_nfdump_revisions(outputs)?; - shared_profile.activity_inputs = profile_count(activity_snapshots.len()); - for (path, snapshot) in &activity_snapshots { - verify_file_snapshot(path, snapshot)?; - } - for (pending_index, active) in pending.iter().zip(resolved_active_sources) { - let active_count = profile_count(active.len()); - profiles[*pending_index].active_set_count = active_count; - shared_profile.active_set_counts[*pending_index] = active_count; - active_sources[*pending_index] = Some(active); - } + bind_nfcapd_source_layout(connection, &layout)?; } + Ok(()) +} - if pending.is_empty() - && reset_outputs.is_empty() - && !marker_needs_backfill.iter().copied().any(|needed| needed) - { - shared_profile.log(start, end); - return Ok(reports); +fn initialize_coordinated_metadata_in_transaction( + connection: &Connection, + pipeline: &ResolvedPipeline, + layout: &[DatasetSource], + dataset_layouts: &BTreeMap>, +) -> Result<(), PipelineError> { + bind_identity(connection, pipeline)?; + for dataset in &pipeline.datasets { + let sources = dataset_layouts.get(&dataset.dataset_id).ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "coordinated plan has no frozen source layout for dataset {:?}", + dataset.dataset_id + )) + })?; + upsert_dataset_with_sources(connection, dataset, sources)?; } - - let transaction_indices = (0..outputs.len()) - .filter(|index| { - reset_set.contains(index) || pending.contains(index) || marker_needs_backfill[*index] - }) - .collect::>(); - let mut transactions = (0..outputs.len()).map(|_| false).collect::>(); - let transaction_started = Instant::now(); - let mut aggregates = (0..outputs.len()) - .map(|index| { - pending - .contains(&index) - .then(|| AggregateBuckets::with_owned_keys(owned_keys.clone())) - }) - .collect::>(); - for &index in &transaction_indices { - if let Err(error) = outputs[index].connection.execute_batch("BEGIN IMMEDIATE") { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(StorageError::from(error))); - } - transactions[index] = true; - if reset_set.contains(&index) { - let source_ids = outputs[index] - .sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>(); - if let Err(error) = - delete_stats_time_range(&outputs[index].connection, &source_ids, start, end) - { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(error)); - } - } - let source_ids = outputs[index] - .sources + if !layout.is_empty() { + let layout = layout .iter() - .map(|source| source.source_id.clone()) + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) .collect::>(); - if let Err(error) = provision_daily_product_completion_bucket_guards( - &outputs[index].connection, - &source_ids, - start, - end, - ) { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(error)); - } + bind_nfcapd_source_layout(connection, &layout)?; } + Ok(()) +} - let result = if pending.is_empty() { - Ok(()) - } else { - verify_coordinated_nfdump_revisions(outputs)?; - let decode_pool = pools.decode()?; - process_coordinated_batches( - outputs, - &pending, - &mut reports, - &mut profiles, - &mut shared_profile, - &mut aggregates, - by_member_and_start, - force, - &mut batches, - &all_revisions, - &active_sources, - decode_pool, - ) - }; - if let Err(error) = result { - rollback_coordinated_transactions(outputs, &transactions); - return Err(error); - } - let day_publish_elapsed = transaction_started.elapsed(); - for &index in &pending { - let aggregates = aggregates[index] - .take() - .expect("pending output has aggregate state"); - let final_profile = match publish_rollups_profiled( - &outputs[index].connection, - aggregates, - &outputs[index].pipeline, - &mut reports[index], - ) { - Ok(profile) => profile, - Err(error) => { - rollback_coordinated_transactions(outputs, &transactions); - return Err(error); - } - }; - profiles[index].final_rollups = final_profile; - } - let revision_snapshots = all_revisions.values().filter_map(|revision| { - revision - .snapshot - .as_ref() - .map(|snapshot| (Path::new(&revision.revision.locator), snapshot)) - }); - let activity_snapshot_refs = activity_snapshots - .iter() - .map(|(path, snapshot)| (path.as_path(), snapshot)); - if let Err(error) = verify_coordinated_precommit_guards( - outputs, - root, - member_identities, - revision_snapshots, - activity_snapshot_refs, - missing_absences, - start, - end, - ) { - rollback_coordinated_transactions(outputs, &transactions); - return Err(error); - } - if !skip_incomplete_day { - for &index in &transaction_indices { - if let Err(error) = mark_nfcapd_day_complete( - &outputs[index].connection, - &outputs[index].sources, - start, - end, - outputs[index].pipeline.run_maad, - ) { - rollback_coordinated_transactions(outputs, &transactions); - return Err(error); - } - } - } - for &index in &transaction_indices { - if let Err(error) = outputs[index].connection.execute_batch("COMMIT") { - rollback_coordinated_transactions(outputs, &transactions); - return Err(PipelineError::Storage(StorageError::from(error))); - } - transactions[index] = false; - } - let transaction_elapsed = transaction_started.elapsed(); - shared_profile.log(start, end); - for &index in &pending { - profiles[index].day_elapsed = day_publish_elapsed; - profiles[index].log_coordinated( - start, - end, - transaction_elapsed, - index, - &outputs[index].pipeline.database_path, - ); - } - Ok(reports) +fn process_atomic( + connection: &Connection, + pipeline: &ResolvedPipeline, + operation: impl FnOnce(&mut AggregateBuckets, &mut PipelineReport) -> Result<(), PipelineError>, +) -> Result { + let mut aggregates = AggregateBuckets::default(); + let mut report = PipelineReport::default(); + with_transaction(connection, || { + operation(&mut aggregates, &mut report)?; + publish_rollups(connection, aggregates, pipeline, &mut report)?; + verify_nfdump_revision(pipeline) + })?; + Ok(report) } -#[allow(clippy::too_many_arguments)] -#[allow(dead_code)] -fn resolve_coordinated_batch_revisions( - outputs: &[CoordinatedOutput], - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - extend_gaps_to_window: bool, - force: bool, - revision_pool: &rayon::ThreadPool, - batch_starts: &[i64], -) -> Result, PipelineError> { - resolve_coordinated_batch_revisions_with_cache( - outputs, - sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - force, - revision_pool, - batch_starts, - &[], - &BTreeMap::new(), - ) +fn merge_report(total: &mut PipelineReport, addition: PipelineReport) { + total.input_scans += addition.input_scans; + total.skipped_inputs += addition.skipped_inputs; + total.five_minute_buckets += addition.five_minute_buckets; + total.rollup_buckets += addition.rollup_buckets; } -#[allow(clippy::too_many_arguments)] -fn resolve_coordinated_batch_revisions_with_cache( - outputs: &[CoordinatedOutput], - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - extend_gaps_to_window: bool, - force: bool, - revision_pool: &rayon::ThreadPool, - batch_starts: &[i64], - resume_caches: &[NfcapdDayResumeCache], - capture_snapshots: &BTreeMap, -) -> Result, PipelineError> { - let mut paths = BTreeSet::new(); - for &bucket_start in batch_starts { - for source in sources { - if !source_has_candidate( - source, - bucket_start, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - ) { - continue; +fn populate_coverage_summary( + connection: &Connection, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + let mut statement = connection + .prepare( + "SELECT coverage_state, COUNT(*) + FROM bucket_coverage + WHERE granularity = '5m' + GROUP BY coverage_state", + ) + .map_err(StorageError::from)?; + let rows = statement + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + for (state, count) in rows { + let count = usize::try_from(count) + .map_err(|_| PipelineError::InvalidConfig("coverage summary count overflow".into()))?; + match state.as_str() { + "complete" => report.complete_five_minute_buckets = count, + "partial" => report.partial_five_minute_buckets = count, + "unknown" => report.unknown_five_minute_buckets = count, + _ => { + return Err(PipelineError::InvalidConfig(format!( + "invalid five-minute coverage state in database: {state:?}" + ))); } - paths.extend(source.members.iter().filter_map(|member| { - by_member_and_start - .get(&(member.clone(), bucket_start)) - .cloned() - })); } } - let decoder_fingerprint = nfdump_decoder_fingerprint_for_pipeline(&outputs[0].pipeline)?; - let probes = paths - .into_iter() - .map(|path| { - let observed = capture_snapshots - .get(&path) - .cloned() - .map(Ok) - .unwrap_or_else(|| capture_nfcapd_snapshot(&path))?; - let cached = if force { - None - } else { - let mut shared = None; - let mut conflict = false; - for (index, output) in outputs.iter().enumerate() { - let fingerprint = match resume_caches.get(index) { - Some(cache) => cache.cached_content_fingerprint(&path, &observed), - None => cached_content_fingerprint( - &output.connection, - InputKind::Nfcapd, - &path.to_string_lossy(), - &observed, - )?, - }; - match (&shared, fingerprint) { - (None, Some(value)) => shared = Some(value), - (Some(previous), Some(value)) if previous == &value => {} - (Some(_), Some(_)) => conflict = true, - (None, None) => {} - (Some(_), None) => {} - } - } - (!conflict).then_some(shared).flatten() - }; - Ok::<_, PipelineError>((path, observed, cached)) - }) - .collect::, _>>()?; - revision_pool.install(|| { - probes - .par_iter() - .map(|(path, observed, cached)| { - let (content_fingerprint, snapshot) = match cached { - Some(content_fingerprint) => (content_fingerprint.clone(), observed.clone()), - None => capture_file_revision_with_snapshot(path, observed)?, - }; - let revision = InputRevision::create( - "nfcapd", - path.to_string_lossy().into_owned(), - content_fingerprint, - &decoder_fingerprint, - )?; - Ok::<_, PipelineError>(( - path.clone(), - PreparedRevision { - revision, - snapshot: Some(snapshot), - }, - )) - }) - .collect::, _>>() - }) + Ok(()) } -type CoordinatedActiveResolution = (Vec>, Vec<(PathBuf, FileSnapshot)>); -type DailyActiveResolution = (Arc, Vec<(PathBuf, FileSnapshot)>); +#[derive(Clone, Debug, PartialEq, Eq)] +struct CoverageScope { + source_ids: Vec, + start: i64, + end: i64, +} -/// Return only the capture paths on the publication grid for one physical local day. -/// -/// Discovery intentionally accepts every valid nfcapd timestamp, but daily eligibility must use -/// the same five-minute keys that publication reads. An off-grid capture can therefore never -/// contribute activity merely because it falls inside the day's lexical path range. -fn nfcapd_day_activity_paths( - paths: &BTreeMap<(String, i64), PathBuf>, - member: &str, +#[derive(Clone, Debug, PartialEq, Eq)] +struct CoverageRange { + source_id: String, start: i64, end: i64, - timezone: &str, -) -> Result, PipelineError> { - let mut member_paths = Vec::new(); - let mut bucket_start = start; - while bucket_start < end { - if let Some(path) = paths.get(&(member.to_owned(), bucket_start)) { - member_paths.push(path.clone()); +} + +fn merged_requested_coverage_ranges(scopes: Vec) -> Vec { + let mut ranges = scopes + .into_iter() + .flat_map(|scope| { + scope + .source_ids + .into_iter() + .map(move |source_id| CoverageRange { + source_id, + start: scope.start, + end: scope.end, + }) + }) + .collect::>(); + ranges.sort_unstable_by(|left, right| { + (&left.source_id, left.start, left.end).cmp(&(&right.source_id, right.start, right.end)) + }); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(previous) = merged.last_mut() + && previous.source_id == range.source_id + && range.start <= previous.end + { + previous.end = previous.end.max(range.end); + } else { + merged.push(range); } - bucket_start = next_local_five_minute_start(bucket_start, timezone)?; } - Ok(member_paths) + merged } -fn daily_activity_scan_error( - member: &str, - start: i64, - end: i64, - paths: &[PathBuf], - error: impl std::fmt::Display, -) -> PipelineError { - let paths = if paths.is_empty() { - "".to_owned() - } else { - paths - .iter() - .map(|path| path.display().to_string()) - .collect::>() - .join(", ") - }; - PipelineError::InvalidConfig(format!( - "daily activity scan failed for member {member:?}, day {start}..{end}, paths [{paths}]: {error}" - )) +/// A finite native request is checked against the same frozen window and source layout that was +/// used for publication. CSV and literal-input configurations have no separately declared window, +/// so their configured product remains the strict scope. +fn requested_coverage_scopes_with_plan( + pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, +) -> Result>, PipelineError> { + let mut scopes = Vec::new(); + for (input_index, input) in pipeline.inputs.iter().enumerate() { + if !matches!(input, InputSpec::NfcapdTree { .. }) { + return Ok(None); + } + let tree = plan + .trees + .get(&input_index) + .expect("every nfcapd_tree input has a frozen layout"); + scopes.push(CoverageScope { + source_ids: tree + .sources + .iter() + .map(|source| source.source_id.clone()) + .collect(), + start: tree.start, + end: tree.end, + }); + } + Ok(Some(scopes)) } -fn nfcapd_decode_error( - member: &str, - bucket_start: i64, - path: &Path, - error: impl std::fmt::Display, -) -> PipelineError { - PipelineError::InvalidConfig(format!( - "nfcapd decode failed for member {member:?}, bucket {bucket_start}, path {}: {error}", - path.display() - )) +fn count_incomplete_requested_coverage_with_plan( + connection: &Connection, + pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, +) -> Result { + let Some(scopes) = requested_coverage_scopes_with_plan(pipeline, plan)? else { + return connection + .query_row( + "SELECT COUNT(*) FROM bucket_coverage + WHERE granularity = '5m' AND coverage_state <> 'complete'", + [], + |row| row.get(0), + ) + .map_err(StorageError::from) + .map_err(PipelineError::from); + }; + + count_incomplete_coverage_ranges( + connection, + merged_requested_coverage_ranges(scopes), + &pipeline.timezone, + ) } -#[allow(clippy::too_many_arguments)] -fn resolve_coordinated_daily_active_sources( - physical_ids: &[String], - paths: &BTreeMap<(String, i64), PathBuf>, +fn count_incomplete_coverage_for_layout( + connection: &Connection, + sources: &[DatasetSource], start: i64, end: i64, timezone: &str, - selections: &[FlowSelection], - executable: &Path, - activity_pool: &rayon::ThreadPool, - capture_snapshots: &BTreeMap, - revision_snapshots: &BTreeMap, -) -> Result { - let mut combined = (0..selections.len()) - .map(|_| HashMap::::new()) +) -> Result { + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) .collect::>(); - let mut snapshots = Vec::new(); - for member_chunk in physical_ids.chunks(NFCAPD_DECODE_BATCH_SIZE) { - let requests = member_chunk - .iter() - .map(|member| { - nfcapd_day_activity_paths(paths, member, start, end, timezone) - .map(|member_paths| (member.clone(), member_paths)) + let ranges = merged_requested_coverage_ranges(vec![CoverageScope { + source_ids, + start, + end, + }]); + count_incomplete_coverage_ranges(connection, ranges, timezone) +} + +fn count_incomplete_coverage_ranges( + connection: &Connection, + ranges: Vec, + timezone: &str, +) -> Result { + let mut incomplete = 0_i64; + for range in ranges { + let complete = connection + .prepare( + "SELECT bucket_start + FROM bucket_coverage + WHERE source_id = ?1 + AND granularity = '5m' + AND bucket_start >= ?2 + AND bucket_start < ?3 + AND coverage_state = 'complete' + ORDER BY bucket_start", + ) + .map_err(StorageError::from)? + .query_map(params![&range.source_id, range.start, range.end], |row| { + row.get::<_, i64>(0) }) - .collect::, _>>()?; - let member_results = activity_pool.install(|| { - requests - .par_iter() - .map(|(member, member_paths)| { - let snapshots = member_paths - .iter() - .map(|path| { - let snapshot = capture_snapshots - .get(path) - .cloned() - .or_else(|| { - revision_snapshots - .get(path) - .and_then(|revision| revision.snapshot.clone()) - }) - .map(Ok) - .unwrap_or_else(|| capture_nfcapd_snapshot(path)); - snapshot - .map(|snapshot| (path.clone(), snapshot)) - .map_err(|error| { - daily_activity_scan_error( - member, - start, - end, - member_paths, - error, - ) - }) - }) - .collect::, PipelineError>>()?; - let activities = ingest::read_nfcapd_daily_source_activities( - member_paths, - selections, - executable, + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + let mut bucket_start = range.start; + while bucket_start < range.end { + if !complete.contains(&bucket_start) { + incomplete = incomplete.checked_add(1).ok_or_else(|| { + PipelineError::InvalidConfig( + "requested coverage count exceeds SQLite INTEGER range".into(), ) - .map_err(|error| { - daily_activity_scan_error(member, start, end, member_paths, error) - })?; - if activities.len() != selections.len() { - return Err(daily_activity_scan_error( - member, - start, - end, - member_paths, - format!( - "daily activity decoder returned {} results for {} selections", - activities.len(), - selections.len() - ), - )); - } - Ok::<_, PipelineError>((activities, snapshots)) - }) - .collect::, _>>() - })?; - for (activities, member_snapshots) in member_results { - snapshots.extend(member_snapshots); - for (selection_index, activity) in activities.into_iter().enumerate() { - for (address, metrics) in activity { - combined[selection_index] - .entry(address) - .or_default() - .include(metrics); - } + })?; } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; } } - let active_sources = combined - .into_iter() - .map(|activity| { - Arc::new( - activity - .into_iter() - .filter_map(|(address, metrics)| { - FlowSelection::daily_activity_threshold_met( - metrics.flows, - metrics.packets, - metrics.bytes, - ) - .then_some(address) - }) - .collect(), - ) - }) - .collect(); - Ok((active_sources, snapshots)) + Ok(incomplete) } -#[allow(clippy::too_many_arguments)] -fn process_coordinated_batches( - outputs: &[CoordinatedOutput], - pending: &[usize], - reports: &mut [PipelineReport], - profiles: &mut [NfcapdDayPublishProfile], - shared_profile: &mut CoordinatedDaySharedProfile, - aggregates: &mut [Option], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - force: bool, - batches: &mut [PreparedCoordinatedBatch], - revisions: &BTreeMap, - active_sources: &[Option>], - decode_pool: &rayon::ThreadPool, +fn bind_identity( + connection: &Connection, + pipeline: &ResolvedPipeline, ) -> Result<(), PipelineError> { - if pending.is_empty() { - return Ok(()); - } - let executable = outputs[pending[0]].pipeline.nfdump.clone(); - let timezone = outputs[pending[0]].pipeline.timezone.clone(); - for batch in batches { - verify_coordinated_nfdump_revisions(outputs)?; - let prepared = std::mem::take(&mut batch.prepared); - - let mut needed = BTreeMap::<(String, i64), BTreeSet>::new(); - for (&output_index, batch) in &prepared { - for timestamp in batch { - for job in ×tamp.jobs { - for (member, _) in &job.present { - needed - .entry((member.clone(), timestamp.bucket_start)) - .or_default() - .insert(output_index); - } - } - } - } - - let decode_requests = needed - .iter() - .map(|((member, bucket_start), output_indices)| { - let path = by_member_and_start - .get(&(member.clone(), *bucket_start)) - .cloned() - .ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "coordinated decoder could not locate physical input {member}:{bucket_start}" - )) - })?; - let output_indices = output_indices.iter().copied().collect::>(); - let pairs = output_indices - .iter() - .map(|index| { - ( - outputs[*index].pipeline.selection.clone(), - active_sources[*index] - .clone() - .expect("pending daily selection has active sources"), - ) - }) - .collect::>(); - let snapshot = revisions - .get(&path) - .and_then(|owner| owner.snapshot.as_ref()) - .ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "coordinated decoder has no revision snapshot for {member}:{bucket_start}" - )) - })?; - Ok::<_, PipelineError>(( - member.clone(), - *bucket_start, - path, - output_indices, - pairs, - snapshot, - )) - }) - .collect::, _>>()?; - - // Keep at most twelve child processes active, even when one logical timestamp fans out to - // many physical members. The decoded map is keyed by physical request and retains the - // single fanout vector returned by nfdump rather than an output-expanded result list. - let decode_started = Instant::now(); - let mut decoded = BTreeMap::<(String, i64), Vec<(usize, CanonicalBucket)>>::new(); - for request_chunk in nfcapd_decode_request_chunks(&decode_requests) { - verify_coordinated_nfdump_revisions(outputs)?; - let decoded_results = decode_pool.install(|| { - request_chunk - .par_iter() - .map( - |(member, bucket_start, path, output_indices, pairs, snapshot)| { - let buckets = ingest::read_nfcapd_buckets_with_active_sources( - path, - member, - pairs, - &executable, - &timezone, - ) - .map_err(|error| { - nfcapd_decode_error(member, *bucket_start, path, error) - })?; - if buckets.len() != output_indices.len() { - return Err(nfcapd_decode_error( - member, - *bucket_start, - path, - format!( - "bucket decoder returned {} results for {} selections", - buckets.len(), - output_indices.len() - ), - )); - } - verify_file_snapshot(path, snapshot).map_err(|error| { - nfcapd_decode_error(member, *bucket_start, path, error) - })?; - Ok::<_, PipelineError>(( - member.clone(), - *bucket_start, - output_indices.clone(), - buckets, - )) - }, - ) - .collect::, _>>() - })?; - verify_coordinated_nfdump_revisions(outputs)?; - for (member, bucket_start, output_indices, buckets) in decoded_results { - decoded.insert( - (member, bucket_start), - output_indices.into_iter().zip(buckets).collect(), - ); - } - } - let decode_elapsed = decode_started.elapsed(); - shared_profile.decode_elapsed += decode_elapsed; - for &output_index in prepared.keys() { - profiles[output_index].decode_elapsed += decode_elapsed; - } - - for (&output_index, batch) in &prepared { - let aggregate = aggregates[output_index] - .as_mut() - .expect("pending output has aggregate state"); - let publish_started = Instant::now(); - for timestamp in batch { - for job in ×tamp.jobs { - let member_buckets = job - .present - .iter() - .map(|(member, _)| { - decoded - .get(&(member.clone(), timestamp.bucket_start)) - .and_then(|buckets| { - buckets.iter().find_map(|(index, bucket)| { - (*index == output_index).then_some(bucket) - }) - }) - .expect("requested physical member was decoded") - }) - .collect::>(); - let logical_started = Instant::now(); - let logical = logical_source_bucket( - &job.source_id, - timestamp.bucket_start, - job.expected_units, - &member_buckets, - )?; - profiles[output_index].logical_source_elapsed += logical_started.elapsed(); - let sibling_started = Instant::now(); - if !job.is_repair { - aggregate.reject_persisted_siblings( - &outputs[output_index].connection, - &logical, - &outputs[output_index].pipeline.timezone, - )?; - } - profiles[output_index].persisted_sibling_elapsed += sibling_started.elapsed(); - let bucket_profile = publish_nfcapd_bucket_profiled( - &outputs[output_index].connection, - &logical, - &job.owners, - &job.absences, - &job.evidence, - true, - force, - outputs[output_index].pipeline.run_maad, - )?; - profiles[output_index] - .bucket_publish - .include(bucket_profile); - let flushed = if job.is_repair { - refresh_rollups_after_five_minute_repair( - &outputs[output_index].connection, - &logical, - &outputs[output_index].pipeline.timezone, - )?; - 0 - } else { - let aggregate_profile = aggregate - .include_profiled(&logical, &outputs[output_index].pipeline.timezone)?; - profiles[output_index] - .aggregate_include - .include(aggregate_profile); - let flush_started = Instant::now(); - let (flushed, rollup_write) = aggregate.flush_complete_profiled( - &outputs[output_index].connection, - outputs[output_index].pipeline.run_maad, - )?; - profiles[output_index].completed_rollup_flush_elapsed += - flush_started.elapsed(); - profiles[output_index] - .completed_rollup_write - .include(rollup_write); - profiles[output_index].completed_rollup_flushes += 1; - if flushed > 0 { - profiles[output_index].nonempty_rollup_flushes += 1; - } - flushed - }; - profiles[output_index].logical_buckets += 1; - reports[output_index].rollup_buckets += flushed; - reports[output_index].five_minute_buckets += 1; - } - } - profiles[output_index].batch_publish_elapsed += publish_started.elapsed(); + verify_nfdump_revision(pipeline)?; + let maad_config = serde_json::to_value(crate::maad::MaadConfig::default())?; + let schema = json!({ + "version": 3, + "tables": [ + {"name":"traffic_stats","version":2}, + {"name":"protocol_stats","version":1}, + {"name":"address_count_stats","version":1}, + {"name":"port_count_stats","version":1}, + {"name":"address_structure_stats","version":1}, + {"name":"bucket_coverage","version":1} + ] + }); + let nfdump_executable = pipeline.nfdump_revision.as_ref().map(|revision| { + json!({ + "locator": revision.locator, + "content_fingerprint": revision.content_fingerprint, + }) + }); + let result_config = json!({ + "version": 4, + "timezone": pipeline.timezone, + "nfcapd_decoder": { + "protocol_version": nfdump::CONTRACT_VERSION, + "input_contract": nfdump::INPUT_CONTRACT, + "output_contract": nfdump::OUTPUT_CONTRACT, + "contract_id": nfcapd_decoder_fingerprint()?, + "decoder_fingerprint": pipeline.nfdump_revision.as_ref().map(|revision| revision.decoder_fingerprint.clone()), + "executable": nfdump_executable, + }, + "maad": { + "enabled": pipeline.run_maad, + "backend": "in-process", + "contract_version": 2, + "config": maad_config } - } + }); + let identity = ProductIdentity::create( + &schema, + &pipeline.selection.normalized_payload(), + &result_config, + )?; + bind_product_identity(connection, &identity, &crate::storage::STATS_TABLE_NAMES)?; Ok(()) } -fn rollback_coordinated_transactions(outputs: &[CoordinatedOutput], transactions: &[bool]) { - for (output, active) in outputs.iter().zip(transactions) { - if *active { - let _ = output.connection.execute_batch("ROLLBACK"); - } - } -} - -/// Give every dataset without a configured `default_start_date` the earliest ingested local day. -/// -/// This runs after ingestion so that newly ingested earlier days move the stored date back. Until -/// the database holds traffic, the row keeps the fallback that [`upsert_dataset_metadata`] wrote. -fn infer_default_start_dates( +fn upsert_dataset_with_sources( connection: &Connection, - pipeline: &ResolvedPipeline, + dataset: &Dataset, + logical_sources: &[DatasetSource], ) -> Result<(), PipelineError> { - let inferred = pipeline - .datasets + let sources = logical_sources .iter() - .filter(|dataset| dataset.default_start_date.trim().is_empty()) + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) .collect::>(); - if inferred.is_empty() { - return Ok(()); - } - let Some(bucket_start) = earliest_traffic_bucket_start(connection)? else { - return Ok(()); - }; - let date = local_date(bucket_start, &pipeline.timezone)?; - with_transaction(connection, || { - for dataset in inferred { - set_dataset_default_start_date(connection, &dataset.dataset_id, &date)?; - } - Ok(()) - }) + let mut metadata = DatasetMetadata::new(&dataset.dataset_id); + metadata.label = dataset.label.clone(); + metadata.default_start_date = dataset.default_start_date.clone(); + metadata.source_mode = dataset.source_mode.clone(); + metadata.discovery_mode = dataset.discovery_mode.clone(); + metadata.sort_order = dataset.sort_order; + metadata.sources = sources; + upsert_dataset_metadata(connection, &metadata)?; + Ok(()) } -/// Local calendar day that contains `timestamp`, formatted as `YYYY-MM-DD`. -fn local_date(timestamp: i64, timezone: &str) -> Result { - Ok(Timestamp::from_second(timestamp) - .map_err(|error| PipelineError::Time(error.to_string()))? - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .date() - .to_string()) +struct PreparedCsvInput { + path: PathBuf, + mapping: CsvSourceConfig, + revision: InputRevision, + snapshot: FileSnapshot, } -fn with_transaction( +fn prepare_file_revision( connection: &Connection, - operation: impl FnOnce() -> Result, -) -> Result { - with_transaction_precommit(connection, operation, || Ok(())) + path: &Path, + input_kind: InputKind, + decoder_fingerprint: String, +) -> Result<(InputRevision, FileSnapshot), PipelineError> { + prepare_file_revision_with(connection, path, input_kind, decoder_fingerprint, || { + capture_file_revision(path) + }) } -/// Run a transaction with a final read-only guard immediately before COMMIT. -/// -/// The guard runs after all writes have completed while the transaction is still open. Any guard -/// failure therefore rolls back the writes instead of leaving a partially repaired product behind. -fn with_transaction_precommit( +fn prepare_file_revision_with( connection: &Connection, - operation: impl FnOnce() -> Result, - precommit: impl FnOnce() -> Result<(), PipelineError>, -) -> Result { - with_transaction_precommit_value( - connection, - || operation().map(|value| (value, ())), - |_| precommit(), - ) + path: &Path, + input_kind: InputKind, + decoder_fingerprint: String, + hash_file: impl FnOnce() -> Result<(String, FileSnapshot), ProvenanceError>, +) -> Result<(InputRevision, FileSnapshot), PipelineError> { + let locator = path.to_string_lossy().into_owned(); + let observed = FileSnapshot::capture(path)?; + let (content_fingerprint, snapshot) = + match cached_content_fingerprint(connection, input_kind, &locator, &observed)? { + Some(content_fingerprint) => (content_fingerprint, observed), + None => hash_file()?, + }; + let revision = InputRevision::create( + input_kind.as_str(), + locator, + content_fingerprint, + decoder_fingerprint, + )?; + Ok((revision, snapshot)) } -fn with_transaction_precommit_value( +fn process_csv_inputs( connection: &Connection, - operation: impl FnOnce() -> Result<(T, G), PipelineError>, - precommit: impl FnOnce(&G) -> Result<(), PipelineError>, -) -> Result { - connection - .execute_batch("BEGIN IMMEDIATE") - .map_err(StorageError::from)?; - let result = operation().and_then(|(value, guards)| { - precommit(&guards)?; - connection - .execute_batch("COMMIT") - .map_err(StorageError::from)?; - Ok(value) - }); - match result { - Ok(value) => Ok(value), - Err(error) => { - let _ = connection.execute_batch("ROLLBACK"); - Err(error) + inputs: &[ingest::CsvInputSpec], + pipeline: &ResolvedPipeline, +) -> Result { + let mut prepared = Vec::new(); + let mut skipped_inputs = 0_usize; + let mut needs_rescan = false; + for input in inputs { + let mapping = CsvSourceConfig::load(&input.mapping_path)?; + let (revision, snapshot) = prepare_file_revision( + connection, + &input.path, + InputKind::Csv, + csv_decoder_fingerprint(&mapping)?, + )?; + if input_scan_fully_processed(connection, InputKind::Csv, &revision.locator, &revision)? { + skipped_inputs += 1; + } else { + needs_rescan = true; } + prepared.push(PreparedCsvInput { + path: input.path.clone(), + mapping, + revision, + snapshot, + }); } -} - -fn coordinated_output_error(pipeline: &ResolvedPipeline, error: PipelineError) -> PipelineError { - let dataset_id = pipeline - .datasets - .first() - .map(|dataset| dataset.dataset_id.as_str()) - .unwrap_or(""); - PipelineError::InvalidConfig(format!( - "coordinated output initialization failed for dataset {dataset_id:?} at database {}: {error}", - pipeline.database_path.display() - )) -} - -#[cfg(test)] -fn initialize_metadata( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result<(), PipelineError> { - let plan = plan_single_output(pipeline)?; - initialize_metadata_with_plan(connection, pipeline, &plan) -} + if !needs_rescan { + return Ok(PipelineReport { + skipped_inputs, + ..PipelineReport::default() + }); + } + prepared.sort_unstable_by(|left, right| left.path.cmp(&right.path)); -fn initialize_metadata_with_plan( - connection: &Connection, - pipeline: &ResolvedPipeline, - plan: &SingleOutputPlan, -) -> Result<(), PipelineError> { - let layouts = plan - .trees - .values() - .flat_map(|tree| tree.sources.iter().cloned()) - .collect::>(); - let mut source_ids = BTreeSet::new(); - if let Some(duplicate) = layouts - .iter() - .find(|source| !source_ids.insert(source.source_id.clone())) - { - return Err(PipelineError::InvalidConfig(format!( - "nfcapd_tree inputs define duplicate logical source ID {:?}", - duplicate.source_id - ))); - } + let mut aggregates = AggregateBuckets::default(); + let mut report = PipelineReport::default(); with_transaction(connection, || { - initialize_metadata_in_transaction_with_layouts( - connection, - pipeline, - &layouts, - &plan.dataset_sources, - ) - }) + connection + .execute_batch( + "CREATE TEMP TABLE csv_bucket_stage ( + source_id TEXT NOT NULL, + bucket_start INTEGER NOT NULL, + input_locator TEXT NOT NULL, + revision_fingerprint TEXT, + payload BLOB NOT NULL + ); + CREATE INDEX csv_bucket_stage_order + ON csv_bucket_stage(source_id, bucket_start);", + ) + .map_err(StorageError::from)?; + for input in &prepared { + process_csv( + connection, + &input.path, + &input.mapping, + &input.revision, + &input.snapshot, + pipeline, + &mut report, + )?; + } + publish_csv_stage(connection, pipeline, &mut aggregates, &mut report)?; + publish_rollups(connection, aggregates, pipeline, &mut report) + })?; + Ok(report) } -fn initialize_metadata_in_transaction_with_layouts( +#[allow(clippy::too_many_arguments)] +fn process_csv( connection: &Connection, + path: &Path, + mapping: &CsvSourceConfig, + revision: &InputRevision, + snapshot: &FileSnapshot, pipeline: &ResolvedPipeline, - layouts: &[DatasetSource], - dataset_sources: &BTreeMap>, + report: &mut PipelineReport, ) -> Result<(), PipelineError> { - bind_identity(connection, pipeline)?; - for dataset in &pipeline.datasets { - let sources = dataset_sources.get(&dataset.dataset_id).ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "single-output plan has no frozen source layout for dataset {:?}", - dataset.dataset_id - )) - })?; - upsert_dataset_with_sources(connection, dataset, sources)?; - } - if !layouts.is_empty() { - let layout = layouts - .iter() - .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) - .collect::>(); - bind_nfcapd_source_layout(connection, &layout)?; - } + connection + .execute( + "DELETE FROM processed_inputs + WHERE input_kind = 'csv' AND scan_locator = ?1", + params![revision.locator], + ) + .map_err(StorageError::from)?; + let completion = match ingest::scan_csv(path, mapping, &pipeline.selection, |event| { + let bucket_revision = revision_for_locator(revision, &event.input_locator)?; + let owner = InputBucket { + input_kind: InputKind::Csv, + input_locator: event.input_locator.clone(), + scan_locator: event.scan_locator, + source_id: event.bucket.key.source_id.clone(), + bucket_start: event.bucket.key.bucket_start, + bucket_end: event.bucket.key.bucket_end, + revision: bucket_revision.clone(), + file_snapshot: Some(snapshot.clone()), + }; + upsert_input_bucket(connection, &owner, false)?; + mark_input_bucket_status( + connection, + InputKind::Csv, + &event.input_locator, + &event.bucket.key.source_id, + event.bucket.key.bucket_start, + InputStatus::Processed, + &bucket_revision, + None, + )?; + let payload = serde_json::to_vec(&event.bucket)?; + connection + .execute( + "INSERT INTO csv_bucket_stage ( + source_id, bucket_start, input_locator, + revision_fingerprint, payload + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + event.bucket.key.source_id, + event.bucket.key.bucket_start, + event.input_locator, + bucket_revision.fingerprint, + payload, + ], + ) + .map_err(StorageError::from)?; + Ok::<_, PipelineError>(()) + }) { + Ok(completion) => completion, + Err(ProducerError::Input(error)) => return Err(error.into()), + Err(ProducerError::Sink(error)) => return Err(error), + }; + verify_file_snapshot(path, snapshot)?; + complete_input_scan( + connection, + InputKind::Csv, + &completion.scan_locator, + i64::try_from(completion.rejected_rows) + .map_err(|_| PipelineError::InvalidConfig("rejected row count overflow".into()))?, + i64::try_from(completion.skipped_bad_column_count).map_err(|_| { + PipelineError::InvalidConfig("skipped bad-column count overflow".into()) + })?, + revision, + Some(snapshot), + )?; + verify_file_snapshot(path, snapshot)?; + report.input_scans += 1; Ok(()) } -fn initialize_coordinated_metadata_in_transaction( - connection: &Connection, - pipeline: &ResolvedPipeline, - layout: &[DatasetSource], - dataset_layouts: &BTreeMap>, -) -> Result<(), PipelineError> { - bind_identity(connection, pipeline)?; - for dataset in &pipeline.datasets { - let sources = dataset_layouts.get(&dataset.dataset_id).ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "coordinated plan has no frozen source layout for dataset {:?}", - dataset.dataset_id - )) - })?; - upsert_dataset_with_sources(connection, dataset, sources)?; - } - if !layout.is_empty() { - let layout = layout - .iter() - .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) - .collect::>(); - bind_nfcapd_source_layout(connection, &layout)?; - } - Ok(()) +struct CsvStageMember { + bucket: CanonicalBucket, + input_locator: String, + revision_fingerprint: Option, } -fn process_atomic( +/// Merge all staged CSV buckets in source/time order. The stage is indexed on +/// disk, so only one overlapping bucket group is held in memory at a time. +fn publish_csv_stage( connection: &Connection, pipeline: &ResolvedPipeline, - operation: impl FnOnce(&mut AggregateBuckets, &mut PipelineReport) -> Result<(), PipelineError>, -) -> Result { - let mut aggregates = AggregateBuckets::default(); - let mut report = PipelineReport::default(); - with_transaction(connection, || { - operation(&mut aggregates, &mut report)?; - publish_rollups(connection, aggregates, pipeline, &mut report)?; - verify_nfdump_revision(pipeline) - })?; - Ok(report) -} - -fn merge_report(total: &mut PipelineReport, addition: PipelineReport) { - total.input_scans += addition.input_scans; - total.skipped_inputs += addition.skipped_inputs; - total.five_minute_buckets += addition.five_minute_buckets; - total.rollup_buckets += addition.rollup_buckets; -} - -fn populate_coverage_summary( - connection: &Connection, + aggregates: &mut AggregateBuckets, report: &mut PipelineReport, ) -> Result<(), PipelineError> { let mut statement = connection .prepare( - "SELECT coverage_state, COUNT(*) - FROM bucket_coverage - WHERE granularity = '5m' - GROUP BY coverage_state", + "SELECT source_id, bucket_start, input_locator, + revision_fingerprint, payload + FROM csv_bucket_stage + ORDER BY source_id, bucket_start, input_locator", ) .map_err(StorageError::from)?; - let rows = statement - .query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - for (state, count) in rows { - let count = usize::try_from(count) - .map_err(|_| PipelineError::InvalidConfig("coverage summary count overflow".into()))?; - match state.as_str() { - "complete" => report.complete_five_minute_buckets = count, - "partial" => report.partial_five_minute_buckets = count, - "unknown" => report.unknown_five_minute_buckets = count, - _ => { - return Err(PipelineError::InvalidConfig(format!( - "invalid five-minute coverage state in database: {state:?}" - ))); + let mut rows = statement.query([]).map_err(StorageError::from)?; + let mut group: Option<(String, i64, Vec)> = None; + let mut current_source = None; + let mut next_expected = None; + loop { + let Some((source_id, bucket_start, input_locator, revision_fingerprint, payload)) = rows + .next() + .map_err(StorageError::from)? + .map(|row| { + Ok::<_, rusqlite::Error>(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Vec>(4)?, + )) + }) + .transpose() + .map_err(StorageError::from)? + else { + break; + }; + let member = CsvStageMember { + bucket: serde_json::from_slice(&payload)?, + input_locator, + revision_fingerprint, + }; + match group.as_mut() { + Some((group_source, group_start, members)) + if group_source == &source_id && *group_start == bucket_start => + { + members.push(member); + } + _ => { + if let Some((group_source, group_start, members)) = group.take() { + publish_csv_stage_group( + connection, + pipeline, + aggregates, + report, + &group_source, + group_start, + &members, + &mut current_source, + &mut next_expected, + )?; + } + group = Some((source_id, bucket_start, vec![member])); } } } - Ok(()) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct CoverageScope { - source_ids: Vec, - start: i64, - end: i64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct CoverageRange { - source_id: String, - start: i64, - end: i64, -} - -fn merged_requested_coverage_ranges(scopes: Vec) -> Vec { - let mut ranges = scopes - .into_iter() - .flat_map(|scope| { - scope - .source_ids - .into_iter() - .map(move |source_id| CoverageRange { - source_id, - start: scope.start, - end: scope.end, - }) - }) - .collect::>(); - ranges.sort_unstable_by(|left, right| { - (&left.source_id, left.start, left.end).cmp(&(&right.source_id, right.start, right.end)) - }); - let mut merged: Vec = Vec::with_capacity(ranges.len()); - for range in ranges { - if let Some(previous) = merged.last_mut() - && previous.source_id == range.source_id - && range.start <= previous.end - { - previous.end = previous.end.max(range.end); - } else { - merged.push(range); - } - } - merged -} - -/// A finite native request can be checked independently of incomplete data -/// already stored outside that request. CSV and literal-input configurations -/// have no separately declared time window, so their configured product is -/// the strict scope. -fn discovered_nfcapd_tree_end( - root_path: &Path, - source_ids: &[String], - configured_sources: &[DatasetSource], - selected_start: i64, - timezone: &str, -) -> Result { - let root = canonical_path(root_path)?; - let sources = normalize_sources(&root, source_ids, configured_sources)?; - let physical_ids = sources - .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>() - .into_iter() - .collect::>(); - let discovered = ingest::discover_nfcapd_source_paths(&root, &physical_ids, timezone)?; - discovered - .iter() - .map(|input| input.bucket_start) - .max() - .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) - .transpose()? - .map_or(Ok(selected_start), |(_, end)| Ok(end)) + drop(rows); + drop(statement); + if let Some((group_source, group_start, members)) = group { + publish_csv_stage_group( + connection, + pipeline, + aggregates, + report, + &group_source, + group_start, + &members, + &mut current_source, + &mut next_expected, + )?; + } + Ok(()) } -fn requested_coverage_scopes_with_plan( +#[allow(clippy::too_many_arguments)] +fn publish_csv_stage_group( + connection: &Connection, pipeline: &ResolvedPipeline, - plan: Option<&SingleOutputPlan>, -) -> Result>, PipelineError> { - let mut scopes = Vec::new(); - for (input_index, input) in pipeline.inputs.iter().enumerate() { - let InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - start_date, - end_date, - start_time, - end_time, - .. - } = input - else { - return Ok(None); - }; - let selected_start = parse_date_start(start_date, &pipeline.timezone)?; - let start = match start_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_start, - }; - let frozen = plan.and_then(|plan| plan.trees.get(&input_index)); - let end = match (end_time, end_date) { - (Some(value), _) => parse_local_datetime(value, &pipeline.timezone)?, - (None, Some(value)) => next_date_start(value, &pipeline.timezone)?, - (None, None) => match frozen { - Some(tree) => discovered_nfcapd_tree_end_with_sources( - &tree.root_path, - &tree.sources, - selected_start, - &pipeline.timezone, - )?, - None => discovered_nfcapd_tree_end( - root_path, - source_ids, - sources, - selected_start, - &pipeline.timezone, - )?, - }, - }; - let source_ids = match frozen { - Some(tree) => tree.sources.clone(), - None => normalize_sources(root_path, source_ids, sources)?, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, + source_id: &str, + bucket_start: i64, + members: &[CsvStageMember], + current_source: &mut Option, + next_expected: &mut Option, +) -> Result<(), PipelineError> { + if current_source.as_deref() != Some(source_id) { + *current_source = Some(source_id.to_owned()); + *next_expected = None; + } else { + let mut expected = next_expected.ok_or_else(|| { + PipelineError::InvalidConfig("CSV stage lost its source envelope".into()) + })?; + while expected < bucket_start { + let (bucket, evidence) = merged_csv_bucket(source_id, expected, &[])?; + publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; + expected = expected.checked_add(FIVE_MINUTES).ok_or_else(|| { + PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) + })?; } - .into_iter() - .map(|source| source.source_id) - .collect(); - scopes.push(CoverageScope { - source_ids, - start, - end, - }); } - Ok(Some(scopes)) + let (bucket, evidence) = merged_csv_bucket(source_id, bucket_start, members)?; + publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; + *next_expected = Some(bucket_start.checked_add(FIVE_MINUTES).ok_or_else(|| { + PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) + })?); + Ok(()) } -fn discovered_nfcapd_tree_end_with_sources( - root_path: &Path, - sources: &[DatasetSource], - selected_start: i64, - timezone: &str, -) -> Result { - let physical_ids = sources +fn merged_csv_bucket( + source_id: &str, + bucket_start: i64, + members: &[CsvStageMember], +) -> Result<(CanonicalBucket, InputEvidenceRow), PipelineError> { + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + ); + let any_observed = members .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>() - .into_iter() - .collect::>(); - let discovered = ingest::discover_nfcapd_source_paths(root_path, &physical_ids, timezone)?; - discovered + .any(|member| member.bucket.coverage.observed_units() != 0); + let any_rejected = members .iter() - .map(|input| input.bucket_start) - .max() - .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) - .transpose()? - .map_or(Ok(selected_start), |(_, end)| Ok(end)) -} - -#[cfg(test)] -fn count_incomplete_requested_coverage( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result { - count_incomplete_requested_coverage_with_plan( - connection, - pipeline, - &SingleOutputPlan::default(), - ) + .any(|member| member.bucket.coverage.rejected_units() != 0); + let mut builder = if any_observed { + StatisticalBucket::dense(key) + } else { + StatisticalBucket::new(key) + } + .with_coverage(BucketCoverage::empty()); + for member in members { + builder.include(&member.bucket)?; + } + let coverage = BucketCoverage::new(1, u64::from(any_observed), u64::from(any_rejected)) + .map_err(DomainError::from)?; + let bucket = builder.with_coverage(coverage).finish_owned(); + let evidence_state = if any_rejected { + InputEvidenceState::Rejected + } else if any_observed { + InputEvidenceState::Observed + } else { + InputEvidenceState::Missing + }; + let (input_locator, revision_fingerprint) = match members { + [member] => ( + member.input_locator.clone(), + member.revision_fingerprint.clone(), + ), + [] => (format!("csv://{source_id}"), None), + _ => (format!("csv://{source_id}"), None), + }; + let evidence = InputEvidenceRow::new( + source_id, + source_id, + bucket_start, + bucket_start + FIVE_MINUTES, + input_locator, + evidence_state, + revision_fingerprint, + ); + Ok((bucket, evidence)) } -fn count_incomplete_requested_coverage_with_plan( +fn publish_csv_bucket( connection: &Connection, + bucket: &CanonicalBucket, + evidence: &InputEvidenceRow, pipeline: &ResolvedPipeline, - plan: &SingleOutputPlan, -) -> Result { - let plan = (!plan.trees.is_empty()).then_some(plan); - let Some(scopes) = requested_coverage_scopes_with_plan(pipeline, plan)? else { - return connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state <> 'complete'", - [], - |row| row.get(0), - ) - .map_err(StorageError::from) - .map_err(PipelineError::from); - }; - - count_incomplete_coverage_ranges( + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + reject_cross_kind_overlap(connection, bucket, InputKind::Csv)?; + aggregates.reject_persisted_csv_siblings(connection, bucket, &pipeline.timezone)?; + write_buckets(connection, std::slice::from_ref(bucket), pipeline.run_maad)?; + replace_input_evidence( connection, - merged_requested_coverage_ranges(scopes), - &pipeline.timezone, - ) + &bucket.key.source_id, + bucket.key.bucket_start, + std::slice::from_ref(evidence), + )?; + aggregates.include(bucket, &pipeline.timezone)?; + report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; + report.five_minute_buckets += 1; + Ok(()) } -fn count_incomplete_coverage_for_layout( +fn reject_cross_kind_overlap( connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, - timezone: &str, -) -> Result { - let source_ids = sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>(); - let ranges = merged_requested_coverage_ranges(vec![CoverageScope { - source_ids, - start, - end, - }]); - count_incomplete_coverage_ranges(connection, ranges, timezone) + bucket: &CanonicalBucket, + input_kind: InputKind, +) -> Result<(), PipelineError> { + let conflict = connection + .query_row( + "SELECT input_kind, input_locator FROM processed_inputs + WHERE source_id = ?1 AND bucket_start = ?2 AND input_kind <> ?3 + ORDER BY input_kind, input_locator LIMIT 1", + params![ + bucket.key.source_id, + bucket.key.bucket_start, + input_kind.as_str(), + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional() + .map_err(StorageError::from)?; + if let Some((kind, locator)) = conflict { + return Err(PipelineError::InvalidConfig(format!( + "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", + bucket.key.source_id, bucket.key.bucket_start + ))); + } + Ok(()) } -fn count_incomplete_coverage_ranges( - connection: &Connection, - ranges: Vec, - timezone: &str, -) -> Result { - let mut incomplete = 0_i64; - for range in ranges { - let complete = connection - .prepare( - "SELECT bucket_start - FROM bucket_coverage - WHERE source_id = ?1 - AND granularity = '5m' - AND bucket_start >= ?2 - AND bucket_start < ?3 - AND coverage_state = 'complete' - ORDER BY bucket_start", - ) - .map_err(StorageError::from)? - .query_map(params![&range.source_id, range.start, range.end], |row| { - row.get::<_, i64>(0) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - let mut bucket_start = range.start; - while bucket_start < range.end { - if !complete.contains(&bucket_start) { - incomplete = incomplete.checked_add(1).ok_or_else(|| { - PipelineError::InvalidConfig( - "requested coverage count exceeds SQLite INTEGER range".into(), - ) - })?; - } - bucket_start = next_local_five_minute_start(bucket_start, timezone)?; +fn nfcapd_day_is_complete( + sink: &ProductSink<'_>, + sources: &[DatasetSource], + start: i64, + end: i64, +) -> Result { + let Some(product_fingerprint) = current_product_fingerprint(sink.connection)? else { + return Ok(false); + }; + if sources.is_empty() { + return Ok(false); + } + for source in sources { + if !daily_product_completion_matches( + sink.connection, + &source.source_id, + start, + end, + &product_fingerprint, + sink.pipeline.run_maad, + )? { + return Ok(false); } } - Ok(incomplete) + Ok(true) } -fn bind_identity( - connection: &Connection, - pipeline: &ResolvedPipeline, -) -> Result<(), PipelineError> { - verify_nfdump_revision(pipeline)?; - let maad_config = serde_json::to_value(crate::maad::MaadConfig::default())?; - let schema = json!({ - "version": 3, - "tables": [ - {"name":"traffic_stats","version":2}, - {"name":"protocol_stats","version":1}, - {"name":"address_count_stats","version":1}, - {"name":"port_count_stats","version":1}, - {"name":"address_structure_stats","version":1}, - {"name":"bucket_coverage","version":1} - ] - }); - let nfdump_executable = pipeline.nfdump_revision.as_ref().map(|revision| { - json!({ - "locator": revision.locator, - "content_fingerprint": revision.content_fingerprint, - }) - }); - let result_config = json!({ - "version": 4, - "timezone": pipeline.timezone, - "nfcapd_decoder": { - "protocol_version": nfdump::CONTRACT_VERSION, - "input_contract": nfdump::INPUT_CONTRACT, - "output_contract": nfdump::OUTPUT_CONTRACT, - "contract_id": nfcapd_decoder_fingerprint()?, - "decoder_fingerprint": pipeline.nfdump_revision.as_ref().map(|revision| revision.decoder_fingerprint.clone()), - "executable": nfdump_executable, - }, - "maad": { - "enabled": pipeline.run_maad, - "backend": "in-process", - "contract_version": 2, - "config": maad_config +fn rollback_sink_transactions(sinks: &[ProductSink<'_>], transactions: &[bool]) { + for (sink, active) in sinks.iter().zip(transactions) { + if *active { + let _ = sink.connection.execute_batch("ROLLBACK"); } - }); - let identity = ProductIdentity::create( - &schema, - &pipeline.selection.normalized_payload(), - &result_config, - )?; - bind_product_identity(connection, &identity, &crate::storage::STATS_TABLE_NAMES)?; - Ok(()) + } } -fn upsert_dataset_with_sources( - connection: &Connection, - dataset: &Dataset, - logical_sources: &[DatasetSource], +fn process_nfcapd_tree( + tree: &FrozenNfcapdTreeLayout, + sinks: &mut [ProductSink<'_>], ) -> Result<(), PipelineError> { - let sources = logical_sources + let Some(first) = sinks.first() else { + return Ok(()); + }; + let timezone = first.pipeline.timezone.clone(); + let daily_active = first.pipeline.selection.selects_daily_active_sources(); + let source_ids = tree + .sources .iter() - .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) + .map(|source| source.source_id.clone()) .collect::>(); - let mut metadata = DatasetMetadata::new(&dataset.dataset_id); - metadata.label = dataset.label.clone(); - metadata.default_start_date = dataset.default_start_date.clone(); - metadata.source_mode = dataset.source_mode.clone(); - metadata.discovery_mode = dataset.discovery_mode.clone(); - metadata.sort_order = dataset.sort_order; - metadata.sources = sources; - upsert_dataset_metadata(connection, &metadata)?; - Ok(()) -} + let mut day_start = tree.start; -struct PreparedCsvInput { - path: PathBuf, - mapping: CsvSourceConfig, - revision: InputRevision, - snapshot: FileSnapshot, -} + while day_start < tree.end { + let day_end = aggregate_bounds(day_start, Granularity::OneDay, &timezone)? + .1 + .min(tree.end); + let mut pending = Vec::new(); + for (index, sink) in sinks.iter().enumerate() { + if tree.force || !nfcapd_day_is_complete(sink, &tree.sources, day_start, day_end)? { + pending.push(index); + } + } + if pending.is_empty() { + day_start = day_end; + continue; + } -fn prepare_file_revision( - connection: &Connection, - path: &Path, - input_kind: InputKind, - decoder_fingerprint: String, -) -> Result<(InputRevision, FileSnapshot), PipelineError> { - prepare_file_revision_with(connection, path, input_kind, decoder_fingerprint, || { - capture_file_revision(path) - }) -} + let missing = daily_active + .then(|| { + missing_physical_day_inputs( + &tree.physical_ids, + &tree.by_member_and_start, + day_start, + day_end, + &timezone, + ) + }) + .transpose()? + .unwrap_or_default(); -fn prepare_file_revision_with( - connection: &Connection, - path: &Path, - input_kind: InputKind, - decoder_fingerprint: String, - hash_file: impl FnOnce() -> Result<(String, FileSnapshot), ProvenanceError>, -) -> Result<(InputRevision, FileSnapshot), PipelineError> { - let locator = path.to_string_lossy().into_owned(); - let observed = FileSnapshot::capture(path)?; - let (content_fingerprint, snapshot) = - match cached_content_fingerprint(connection, input_kind, &locator, &observed)? { - Some(content_fingerprint) => (content_fingerprint, observed), - None => hash_file()?, - }; - let revision = InputRevision::create( - input_kind.as_str(), - locator, - content_fingerprint, - decoder_fingerprint, - )?; - Ok((revision, snapshot)) -} + let mut transactions = vec![false; sinks.len()]; + for &index in &pending { + if let Err(error) = sinks[index].connection.execute_batch("BEGIN IMMEDIATE") { + rollback_sink_transactions(sinks, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = true; + if let Err(error) = + delete_stats_time_range(sinks[index].connection, &source_ids, day_start, day_end) + { + rollback_sink_transactions(sinks, &transactions); + return Err(PipelineError::Storage(error)); + } + } -fn process_csv_inputs( - connection: &Connection, - inputs: &[ingest::CsvInputSpec], - pipeline: &ResolvedPipeline, -) -> Result { - let mut prepared = Vec::new(); - let mut skipped_inputs = 0_usize; - let mut needs_rescan = false; - for input in inputs { - let mapping = CsvSourceConfig::load(&input.mapping_path)?; - let (revision, snapshot) = prepare_file_revision( - connection, - &input.path, - InputKind::Csv, - csv_decoder_fingerprint(&mapping)?, - )?; - if input_scan_fully_processed(connection, InputKind::Csv, &revision.locator, &revision)? { - skipped_inputs += 1; + if daily_active && !missing.is_empty() { + let details = missing_day_warning_details(&tree.root_path, &missing, &timezone)?; + tracing::warn!( + day_start, + day_end, + missing_inputs = missing.len(), + missing_details = %details, + "skipping incomplete physical day for daily_active_sources selection" + ); + sinks[pending[0]].report.skipped_inputs += missing.len(); } else { - needs_rescan = true; + let mut owned_keys = BTreeSet::new(); + let mut bucket_start = day_start; + while bucket_start < day_end { + for source in &tree.sources { + if source_has_candidate( + source, + bucket_start, + &tree.by_member_and_start, + &tree.member_bounds, + tree.extend_gaps_to_window, + ) { + owned_keys.insert((source.source_id.clone(), bucket_start)); + } + } + bucket_start = next_local_five_minute_start(bucket_start, &timezone)?; + } + let mut aggregates = (0..sinks.len()) + .map(|index| { + pending + .contains(&index) + .then(|| AggregateBuckets::with_owned_keys(owned_keys.clone())) + }) + .collect::>(); + let result = + process_nfcapd_tree_day(tree, day_start, day_end, sinks, &pending, &mut aggregates); + if let Err(error) = result { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + for &index in &pending { + let aggregate = aggregates[index] + .take() + .expect("pending output has aggregate state"); + if let Err(error) = publish_rollups( + sinks[index].connection, + aggregate, + sinks[index].pipeline, + sinks[index].report, + ) { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + if let Err(error) = mark_nfcapd_day_complete( + sinks[index].connection, + &tree.sources, + day_start, + day_end, + sinks[index].pipeline.run_maad, + ) { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + } } - prepared.push(PreparedCsvInput { - path: input.path.clone(), - mapping, - revision, - snapshot, - }); - } - if !needs_rescan { - return Ok(PipelineReport { - skipped_inputs, - ..PipelineReport::default() - }); + + if let Err(error) = verify_nfdump_revision(sinks[pending[0]].pipeline) { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + + for &index in &pending { + if let Err(error) = sinks[index].connection.execute_batch("COMMIT") { + rollback_sink_transactions(sinks, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = false; + } + day_start = day_end; } - prepared.sort_unstable_by(|left, right| left.path.cmp(&right.path)); + Ok(()) +} - let mut aggregates = AggregateBuckets::default(); - let mut report = PipelineReport::default(); - with_transaction(connection, || { - connection - .execute_batch( - "CREATE TEMP TABLE csv_bucket_stage ( - source_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - input_locator TEXT NOT NULL, - revision_fingerprint TEXT, - payload BLOB NOT NULL - ); - CREATE INDEX csv_bucket_stage_order - ON csv_bucket_stage(source_id, bucket_start);", - ) - .map_err(StorageError::from)?; - for input in &prepared { - process_csv( - connection, - &input.path, - &input.mapping, - &input.revision, - &input.snapshot, - pipeline, - &mut report, - )?; +fn missing_physical_day_inputs( + physical_ids: &[String], + paths: &BTreeMap<(String, i64), PathBuf>, + start: i64, + end: i64, + timezone: &str, +) -> Result, PipelineError> { + let mut missing = Vec::new(); + let mut bucket_start = start; + while bucket_start < end { + for member in physical_ids { + if !paths.contains_key(&(member.clone(), bucket_start)) { + missing.push((member.clone(), bucket_start)); + } } - publish_csv_stage(connection, pipeline, &mut aggregates, &mut report)?; - publish_rollups(connection, aggregates, pipeline, &mut report) - })?; - Ok(report) + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(missing) } -#[allow(clippy::too_many_arguments)] -fn process_csv( +fn missing_day_warning_details( + root: &Path, + missing: &[(String, i64)], + timezone: &str, +) -> Result { + let mut details = missing + .iter() + .take(MAX_MISSING_DAY_WARNING_DETAILS) + .map(|(member, bucket_start)| { + expected_nfcapd_path(root, member, *bucket_start, timezone).map(|expected_path| { + format!( + "member={member} timestamp={bucket_start} expected_path={}", + expected_path.display() + ) + }) + }) + .collect::, _>>()?; + let omitted = missing.len().saturating_sub(details.len()); + if omitted != 0 { + details.push(format!("… {omitted} more missing inputs")); + } + Ok(details.join("; ")) +} + +/// Publish completion markers in the same transaction as the day's product rows. +fn mark_nfcapd_day_complete( connection: &Connection, - path: &Path, - mapping: &CsvSourceConfig, - revision: &InputRevision, - snapshot: &FileSnapshot, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, + sources: &[DatasetSource], + start: i64, + end: i64, + run_maad: bool, ) -> Result<(), PipelineError> { - connection - .execute( - "DELETE FROM processed_inputs - WHERE input_kind = 'csv' AND scan_locator = ?1", - params![revision.locator], + let product_fingerprint = current_product_fingerprint(connection)?.ok_or_else(|| { + PipelineError::InvalidConfig( + "cannot publish a daily product completion marker before product identity binding" + .into(), ) - .map_err(StorageError::from)?; - let completion = match ingest::scan_csv(path, mapping, &pipeline.selection, |event| { - let bucket_revision = revision_for_locator(revision, &event.input_locator)?; - let owner = InputBucket { - input_kind: InputKind::Csv, - input_locator: event.input_locator.clone(), - scan_locator: event.scan_locator, - source_id: event.bucket.key.source_id.clone(), - bucket_start: event.bucket.key.bucket_start, - bucket_end: event.bucket.key.bucket_end, - revision: bucket_revision.clone(), - file_snapshot: Some(snapshot.clone()), - }; - upsert_input_bucket(connection, &owner, false)?; - mark_input_bucket_status( + })?; + for source in sources { + upsert_daily_product_completion( connection, - InputKind::Csv, - &event.input_locator, - &event.bucket.key.source_id, - event.bucket.key.bucket_start, - InputStatus::Processed, - &bucket_revision, - None, + &source.source_id, + start, + end, + &product_fingerprint, + run_maad, )?; - let payload = serde_json::to_vec(&event.bucket)?; - connection - .execute( - "INSERT INTO csv_bucket_stage ( - source_id, bucket_start, input_locator, - revision_fingerprint, payload - ) VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - event.bucket.key.source_id, - event.bucket.key.bucket_start, - event.input_locator, - bucket_revision.fingerprint, - payload, - ], - ) - .map_err(StorageError::from)?; - Ok::<_, PipelineError>(()) - }) { - Ok(completion) => completion, - Err(ProducerError::Input(error)) => return Err(error.into()), - Err(ProducerError::Sink(error)) => return Err(error), - }; - verify_file_snapshot(path, snapshot)?; - complete_input_scan( - connection, - InputKind::Csv, - &completion.scan_locator, - i64::try_from(completion.rejected_rows) - .map_err(|_| PipelineError::InvalidConfig("rejected row count overflow".into()))?, - i64::try_from(completion.skipped_bad_column_count).map_err(|_| { - PipelineError::InvalidConfig("skipped bad-column count overflow".into()) - })?, - revision, - Some(snapshot), - )?; - verify_file_snapshot(path, snapshot)?; - report.input_scans += 1; + } Ok(()) } -struct CsvStageMember { - bucket: CanonicalBucket, - input_locator: String, - revision_fingerprint: Option, +fn source_has_candidate( + source: &DatasetSource, + bucket_start: i64, + paths: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, +) -> bool { + let has_file = source + .members + .iter() + .any(|member| paths.contains_key(&(member.clone(), bucket_start))); + has_file + || extend_gaps_to_window + || source.members.iter().any(|member| { + member_bounds + .get(member) + .is_some_and(|(first, last)| *first <= bucket_start && bucket_start <= *last) + }) } -/// Merge all staged CSV buckets in source/time order. The stage is indexed on -/// disk, so only one overlapping bucket group is held in memory at a time. -fn publish_csv_stage( - connection: &Connection, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - let mut statement = connection - .prepare( - "SELECT source_id, bucket_start, input_locator, - revision_fingerprint, payload - FROM csv_bucket_stage - ORDER BY source_id, bucket_start, input_locator", - ) - .map_err(StorageError::from)?; - let mut rows = statement.query([]).map_err(StorageError::from)?; - let mut group: Option<(String, i64, Vec)> = None; - let mut current_source = None; - let mut next_expected = None; - loop { - let Some((source_id, bucket_start, input_locator, revision_fingerprint, payload)) = rows - .next() - .map_err(StorageError::from)? - .map(|row| { - Ok::<_, rusqlite::Error>(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, String>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Vec>(4)?, - )) - }) - .transpose() - .map_err(StorageError::from)? - else { - break; - }; - let member = CsvStageMember { - bucket: serde_json::from_slice(&payload)?, - input_locator, - revision_fingerprint, - }; - match group.as_mut() { - Some((group_source, group_start, members)) - if group_source == &source_id && *group_start == bucket_start => - { - members.push(member); - } - _ => { - if let Some((group_source, group_start, members)) = group.take() { - publish_csv_stage_group( - connection, - pipeline, - aggregates, - report, - &group_source, - group_start, - &members, - &mut current_source, - &mut next_expected, - )?; - } - group = Some((source_id, bucket_start, vec![member])); - } +/// Group local five-minute starts so each decode batch has at most twelve physical requests when +/// possible. A timestamp with more than twelve members is kept as one batch and drained in +/// physical-request chunks by the decode caller. +fn nfcapd_batch_starts( + start: i64, + end: i64, + timezone: &str, + sources: &[DatasetSource], + paths: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, +) -> Result, PipelineError> { + let mut starts = Vec::with_capacity(NFCAPD_DECODE_BATCH_SIZE); + let mut physical_requests = BTreeSet::new(); + let mut next = start; + while next < end { + let timestamp_requests = sources + .iter() + .filter(|source| { + source_has_candidate(source, next, paths, member_bounds, extend_gaps_to_window) + }) + .flat_map(|source| { + source.members.iter().filter_map(|member| { + paths + .contains_key(&(member.clone(), next)) + .then_some((member.clone(), next)) + }) + }) + .collect::>(); + let would_exceed_physical_limit = !starts.is_empty() + && physical_requests.len() + timestamp_requests.len() > NFCAPD_DECODE_BATCH_SIZE; + if would_exceed_physical_limit || starts.len() == NFCAPD_DECODE_BATCH_SIZE { + break; } + starts.push(next); + physical_requests.extend(timestamp_requests); + next = next_local_five_minute_start(next, timezone)?; } - drop(rows); - drop(statement); - if let Some((group_source, group_start, members)) = group { - publish_csv_stage_group( - connection, - pipeline, - aggregates, - report, - &group_source, - group_start, - &members, - &mut current_source, - &mut next_expected, - )?; - } - Ok(()) + Ok(starts) } -#[allow(clippy::too_many_arguments)] -fn publish_csv_stage_group( - connection: &Connection, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, - source_id: &str, - bucket_start: i64, - members: &[CsvStageMember], - current_source: &mut Option, - next_expected: &mut Option, -) -> Result<(), PipelineError> { - if current_source.as_deref() != Some(source_id) { - *current_source = Some(source_id.to_owned()); - *next_expected = None; - } else { - let mut expected = next_expected.ok_or_else(|| { - PipelineError::InvalidConfig("CSV stage lost its source envelope".into()) - })?; - while expected < bucket_start { - let (bucket, evidence) = merged_csv_bucket(source_id, expected, &[])?; - publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; - expected = expected.checked_add(FIVE_MINUTES).ok_or_else(|| { - PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) - })?; +fn nfcapd_day_activity_paths( + paths: &BTreeMap<(String, i64), PathBuf>, + member: &str, + start: i64, + end: i64, + timezone: &str, +) -> Result, PipelineError> { + let mut result = Vec::new(); + let mut bucket_start = start; + while bucket_start < end { + if let Some(path) = paths.get(&(member.to_owned(), bucket_start)) { + result.push(path.clone()); } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; } - let (bucket, evidence) = merged_csv_bucket(source_id, bucket_start, members)?; - publish_csv_bucket(connection, &bucket, &evidence, pipeline, aggregates, report)?; - *next_expected = Some(bucket_start.checked_add(FIVE_MINUTES).ok_or_else(|| { - PipelineError::InvalidConfig("CSV source envelope exceeds time range".into()) - })?); - Ok(()) + Ok(result) } -fn merged_csv_bucket( - source_id: &str, +fn nfcapd_decode_error( + member: &str, bucket_start: i64, - members: &[CsvStageMember], -) -> Result<(CanonicalBucket, InputEvidenceRow), PipelineError> { - let key = BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - ); - let any_observed = members - .iter() - .any(|member| member.bucket.coverage.observed_units() != 0); - let any_rejected = members + path: &Path, + error: impl std::fmt::Display, +) -> PipelineError { + PipelineError::InvalidConfig(format!( + "nfcapd decode failed for member {member:?}, bucket {bucket_start}, path {}: {error}", + path.display() + )) +} + +fn resolve_daily_active_sources( + tree: &FrozenNfcapdTreeLayout, + start: i64, + end: i64, + timezone: &str, + selections: &[FlowSelection], + executable: &Path, +) -> Result>, PipelineError> { + let pool = build_nfcapd_activity_pool()?; + let requests = tree + .physical_ids .iter() - .any(|member| member.bucket.coverage.rejected_units() != 0); - let mut builder = if any_observed { - StatisticalBucket::dense(key) - } else { - StatisticalBucket::new(key) - } - .with_coverage(BucketCoverage::empty()); - for member in members { - builder.include(&member.bucket)?; + .map(|member| { + nfcapd_day_activity_paths(&tree.by_member_and_start, member, start, end, timezone) + .map(|paths| (member.clone(), paths)) + }) + .collect::, _>>()?; + let results = pool.install(|| { + requests + .par_iter() + .map(|(member, paths)| { + ingest::read_nfcapd_daily_source_activities(paths, selections, executable) + .map_err(|error| { + PipelineError::InvalidConfig(format!( + "daily activity scan failed for member {member:?}, day {start}..{end}: {error}" + )) + }) + }) + .collect::, _>>() + })?; + + let mut combined = (0..selections.len()) + .map(|_| HashMap::::new()) + .collect::>(); + for member_results in results { + for (selection_index, activity) in member_results.into_iter().enumerate() { + for (address, metrics) in activity { + combined[selection_index] + .entry(address) + .or_default() + .include(metrics); + } + } } - let coverage = BucketCoverage::new(1, u64::from(any_observed), u64::from(any_rejected)) - .map_err(DomainError::from)?; - let bucket = builder.with_coverage(coverage).finish_owned(); - let evidence_state = if any_rejected { - InputEvidenceState::Rejected - } else if any_observed { - InputEvidenceState::Observed - } else { - InputEvidenceState::Missing - }; - let (input_locator, revision_fingerprint) = match members { - [member] => ( - member.input_locator.clone(), - member.revision_fingerprint.clone(), - ), - [] => (format!("csv://{source_id}"), None), - _ => (format!("csv://{source_id}"), None), - }; - let evidence = InputEvidenceRow::new( - source_id, - source_id, - bucket_start, - bucket_start + FIVE_MINUTES, - input_locator, - evidence_state, - revision_fingerprint, - ); - Ok((bucket, evidence)) + Ok(combined + .into_iter() + .map(|activity| { + Arc::new( + activity + .into_iter() + .filter_map(|(address, metrics)| { + FlowSelection::daily_activity_threshold_met( + metrics.flows, + metrics.packets, + metrics.bytes, + ) + .then_some(address) + }) + .collect(), + ) + }) + .collect()) } -fn publish_csv_bucket( - connection: &Connection, - bucket: &CanonicalBucket, - evidence: &InputEvidenceRow, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, +fn resolve_nfcapd_batch_revisions( + tree: &FrozenNfcapdTreeLayout, + batch_starts: &[i64], + decoder_fingerprint: &str, + pool: &rayon::ThreadPool, +) -> Result, PipelineError> { + let paths = batch_starts + .iter() + .flat_map(|bucket_start| { + tree.sources.iter().flat_map(move |source| { + source.members.iter().filter_map(move |member| { + tree.by_member_and_start + .get(&(member.clone(), *bucket_start)) + .cloned() + }) + }) + }) + .collect::>(); + pool.install(|| { + paths + .par_iter() + .map(|path| { + let (content_fingerprint, snapshot) = capture_file_revision(path)?; + let revision = InputRevision::create( + "nfcapd", + path.to_string_lossy().into_owned(), + content_fingerprint, + decoder_fingerprint, + )?; + Ok(( + path.clone(), + PreparedRevision { + revision, + snapshot: Some(snapshot), + }, + )) + }) + .collect::, PipelineError>>() + }) +} + +fn verify_prepared_revision_snapshots( + revisions: &BTreeMap, ) -> Result<(), PipelineError> { - reject_cross_kind_overlap(connection, bucket, InputKind::Csv)?; - aggregates.reject_persisted_csv_siblings(connection, bucket, &pipeline.timezone)?; - let (day_start, day_end) = aggregate_bounds( - bucket.key.bucket_start, - Granularity::OneDay, - &pipeline.timezone, - )?; - ensure_daily_product_completion_bucket_guard( - connection, - &bucket.key.source_id, - bucket.key.bucket_start, - day_start, - day_end, - )?; - write_buckets(connection, std::slice::from_ref(bucket), pipeline.run_maad)?; - replace_input_evidence( - connection, - &bucket.key.source_id, - bucket.key.bucket_start, - std::slice::from_ref(evidence), - )?; - aggregates.include(bucket, &pipeline.timezone)?; - report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; - report.five_minute_buckets += 1; - Ok(()) -} - -fn reject_cross_kind_overlap( - connection: &Connection, - bucket: &CanonicalBucket, - input_kind: InputKind, -) -> Result<(), PipelineError> { - let conflict = connection - .query_row( - "SELECT input_kind, input_locator FROM processed_inputs - WHERE source_id = ?1 AND bucket_start = ?2 AND input_kind <> ?3 - ORDER BY input_kind, input_locator LIMIT 1", - params![ - bucket.key.source_id, - bucket.key.bucket_start, - input_kind.as_str(), - ], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional() - .map_err(StorageError::from)?; - if let Some((kind, locator)) = conflict { - return Err(PipelineError::InvalidConfig(format!( - "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", - bucket.key.source_id, bucket.key.bucket_start - ))); + for (path, prepared) in revisions { + if let Some(snapshot) = &prepared.snapshot { + verify_file_snapshot(path, snapshot)?; + } } Ok(()) } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_tree( - connection: &Connection, +fn prepare_nfcapd_tree_timestamp( tree: &FrozenNfcapdTreeLayout, - start_date: &str, - end_date: Option<&str>, - start_time: Option<&str>, - end_time: Option<&str>, - force: bool, - pipeline: &ResolvedPipeline, - capture_snapshots: &BTreeMap, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - if pipeline.selection.selects_daily_active_sources() - && (start_time.is_some() || end_time.is_some()) - { - return Err(PipelineError::InvalidConfig( - "daily_active_sources selection requires whole local calendar days; start_time and end_time are unsupported".into(), - )); - } - let root = &tree.root_path; - let sources = &tree.sources; - let physical_ids = &tree.physical_ids; - if pipeline.selection.selects_daily_active_sources() { - validate_daily_active_source_layout(sources, physical_ids)?; - } - let discovery_started = Instant::now(); - let discovered = ingest::discover_nfcapd_source_paths(root, physical_ids, &pipeline.timezone)?; - tracing::info!( - target: "netflow_db::profile", - phase = "discovery", - elapsed_seconds = discovery_started.elapsed().as_secs_f64(), - physical_sources = physical_ids.len(), - discovered_inputs = discovered.len(), - ); - let mut by_member_and_start = BTreeMap::new(); - let mut member_bounds = BTreeMap::new(); - for input in discovered { - member_bounds - .entry(input.source_id.clone()) - .and_modify(|(first, last): &mut (i64, i64)| { - *first = (*first).min(input.bucket_start); - *last = (*last).max(input.bucket_start); - }) - .or_insert((input.bucket_start, input.bucket_start)); - by_member_and_start.insert((input.source_id, input.bucket_start), input.path); - } - let window = resolve_nfcapd_tree_window( - start_date, - end_date, - start_time, - end_time, - by_member_and_start - .keys() - .map(|(_, bucket_start)| *bucket_start), - &pipeline.timezone, - )?; - - let mut day_start = window.start; - while day_start < window.end { - verify_member_directory_identities(root, &tree.member_identities)?; - let day_end = aggregate_bounds(day_start, Granularity::OneDay, &pipeline.timezone)?.1; - let capture_complete = day_capture_is_complete( - sources, - &by_member_and_start, - day_start, - day_end, - &pipeline.timezone, - )?; - let published_day = if pipeline.selection.selects_daily_active_sources() { - day_was_published(connection, sources, day_start, day_end)? - } else { - day_has_complete_coverage(connection, sources, day_start, day_end, &pipeline.timezone)? - }; - let stale_published_day = !capture_complete && published_day; - let force_replaces_day = force && pipeline.selection.selects_daily_active_sources(); - if stale_published_day && !force { - return Err(PipelineError::InvalidConfig(format!( - "published local day {day_start}..{day_end} no longer has complete nfcapd capture coverage; rerun that day with --force" - ))); - } - let mut marker_needs_backfill = false; - let canonical_day_verified = if pipeline.selection.selects_daily_active_sources() - && capture_complete - && published_day - && !force - { - match nfcapd_day_completion_state( - connection, - sources, - day_start, - day_end, - pipeline.run_maad, - )? { - DailyProductCompletionState::Clean => true, - DailyProductCompletionState::Dirty => { - return Err(PipelineError::InvalidConfig(format!( - "published local day {day_start}..{day_end} was mutated after completion; rerun that whole day with --force" - ))); - } - DailyProductCompletionState::Missing => { - if !nfcapd_day_has_canonical_topology( - connection, - sources, - day_start, - day_end, - &pipeline.timezone, - pipeline.run_maad, - )? { - return Err(PipelineError::InvalidConfig(format!( - "published local day {day_start}..{day_end} has damaged canonical topology; rerun that whole day with --force" - ))); - } - marker_needs_backfill = true; - true - } - } - } else { - false - }; - let missing = if pipeline.selection.selects_daily_active_sources() || stale_published_day { - missing_physical_day_inputs( - physical_ids, - &by_member_and_start, - day_start, - day_end, - &pipeline.timezone, - )? - } else { - Vec::new() - }; - let missing_absences = - build_missing_day_absences(root, &missing, day_start, day_end, &pipeline.timezone)?; - invoke_missing_day_absence_hook(root, &missing, &pipeline.timezone); - verify_member_directory_identities(root, &tree.member_identities)?; - if pipeline.selection.selects_daily_active_sources() && !missing.is_empty() { - let missing_details = missing_day_warning_details(root, &missing, &pipeline.timezone)?; - tracing::warn!( - day_start, - day_end, - missing_inputs = missing.len(), - missing_details = %missing_details, - "skipping incomplete physical day for daily_active_sources selection" - ); - report.skipped_inputs += missing.len(); - if stale_published_day || force_replaces_day { - let source_ids = sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>(); - let guards = NfcapdDayGuards { - nfdump_revision: pipeline.nfdump_revision.clone(), - ..NfcapdDayGuards::default() - }; - with_transaction_precommit_value( - connection, - || { - verify_missing_day_absences(&missing_absences, day_start, day_end)?; - delete_stats_time_range(connection, &source_ids, day_start, day_end)?; - Ok(((), guards)) - }, - |guards| { - verify_single_day_guards( - guards, - root, - &tree.member_identities, - &missing_absences, - day_start, - day_end, - ) - }, - )?; - } - day_start = day_end; + bucket_start: i64, + timezone: &str, + revisions: &BTreeMap, +) -> Result { + let mut jobs = Vec::new(); + for source in &tree.sources { + if !source_has_candidate( + source, + bucket_start, + &tree.by_member_and_start, + &tree.member_bounds, + tree.extend_gaps_to_window, + ) { continue; } - let mut owned_keys = BTreeSet::new(); - let mut bucket_start = day_start; - while bucket_start < day_end { - for source in sources { - if force - && source_has_candidate( - source, - bucket_start, - &by_member_and_start, - &member_bounds, - end_date.is_some(), - ) - { - owned_keys.insert((source.source_id.clone(), bucket_start)); - } - } - bucket_start = next_local_five_minute_start(bucket_start, &pipeline.timezone)?; + let present = source + .members + .iter() + .filter_map(|member| { + tree.by_member_and_start + .get(&(member.clone(), bucket_start)) + .map(|path| (member.clone(), path.clone())) + }) + .collect::>(); + let owners = present + .iter() + .map(|(_, path)| { + revisions + .get(path) + .cloned() + .expect("present capture has a prepared revision") + }) + .collect::>(); + let mut absences = Vec::new(); + let mut evidence = Vec::with_capacity(source.members.len()); + for ((member, path), owner) in present.iter().zip(&owners) { + evidence.push(InputEvidenceRow::new( + &source.source_id, + member, + bucket_start, + bucket_start + FIVE_MINUTES, + path.to_string_lossy(), + InputEvidenceState::Observed, + Some(owner.revision.fingerprint.clone()), + )); } - let transaction_started = Instant::now(); - let (day_report, day_profile) = with_transaction_precommit_value( - connection, - || { - if stale_published_day || force_replaces_day { - verify_missing_day_absences(&missing_absences, day_start, day_end)?; - let source_ids = sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>(); - delete_stats_time_range(connection, &source_ids, day_start, day_end)?; - } - let source_ids = sources - .iter() - .map(|source| source.source_id.clone()) - .collect::>(); - provision_daily_product_completion_bucket_guards( - connection, - &source_ids, - day_start, - day_end, - )?; - let mut aggregates = AggregateBuckets::with_owned_keys(owned_keys); - let mut day_report = PipelineReport::default(); - let mut day_result = process_nfcapd_tree_day( - connection, - root, - sources, - &by_member_and_start, - &member_bounds, - day_start, - day_end, - end_date.is_some(), - force, - pipeline, - capture_snapshots, - &mut aggregates, - &mut day_report, - canonical_day_verified, - )?; - day_result.profile.final_rollups = - publish_rollups_profiled(connection, aggregates, pipeline, &mut day_report)?; - if !pipeline.selection.selects_daily_active_sources() { - verify_nfdump_revision(pipeline)?; - } - if pipeline.selection.selects_daily_active_sources() - && capture_complete - && missing.is_empty() - && (!published_day || force_replaces_day || marker_needs_backfill) - { - mark_nfcapd_day_complete( - connection, - sources, - day_start, - day_end, - pipeline.run_maad, - )?; - } - Ok(((day_report, day_result.profile), day_result.guards)) - }, - |guards| { - if pipeline.selection.selects_daily_active_sources() { - verify_single_day_guards( - guards, - root, - &tree.member_identities, - &missing_absences, - day_start, - day_end, - ) - } else { - invoke_single_commit_guard_hook(); - verify_member_directory_identities(root, &tree.member_identities)?; - verify_missing_day_absences(&missing_absences, day_start, day_end) - } - }, - )?; - day_profile.log(day_start, day_end, transaction_started.elapsed()); - merge_report(report, day_report); - day_start = day_end; - } - Ok(()) -} - -fn missing_physical_day_inputs( - physical_ids: &[String], - paths: &BTreeMap<(String, i64), PathBuf>, - start: i64, - end: i64, - timezone: &str, -) -> Result, PipelineError> { - let mut missing = Vec::new(); - let mut bucket_start = start; - while bucket_start < end { - for member in physical_ids { - if !paths.contains_key(&(member.clone(), bucket_start)) { - missing.push((member.clone(), bucket_start)); + for member in &source.members { + if present.iter().all(|(present, _)| present != member) { + let expected = + expected_nfcapd_path(&tree.root_path, member, bucket_start, timezone)?; + absences.push(ExpectedAbsence::capture(&expected)?); + evidence.push(InputEvidenceRow::new( + &source.source_id, + member, + bucket_start, + bucket_start + FIVE_MINUTES, + expected.to_string_lossy(), + InputEvidenceState::Missing, + None, + )); } } - bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + evidence.sort_unstable_by(|left, right| left.unit_id.cmp(&right.unit_id)); + jobs.push(PreparedTreeJob { + source_id: source.source_id.clone(), + expected_units: source.members.len(), + present, + owners, + absences, + evidence, + }); } - Ok(missing) -} - -fn missing_day_absence_error( - start: i64, - end: i64, - path: &Path, - error: impl std::fmt::Display, -) -> PipelineError { - PipelineError::InvalidConfig(format!( - "nfcapd capture appeared while protecting missing inputs for local day {start}..{end} at {}; refusing to delete the existing product: {error}", - path.display() - )) + Ok(PreparedTreeTimestamp { bucket_start, jobs }) } -fn build_missing_day_absences( - root: &Path, - missing: &[(String, i64)], +fn process_nfcapd_tree_day( + tree: &FrozenNfcapdTreeLayout, start: i64, end: i64, - timezone: &str, -) -> Result, PipelineError> { - missing + sinks: &mut [ProductSink<'_>], + pending: &[usize], + aggregates: &mut [Option], +) -> Result<(), PipelineError> { + let first_pipeline = sinks[pending[0]].pipeline; + let timezone = first_pipeline.timezone.clone(); + let executable = first_pipeline.nfdump.clone(); + let daily_active = first_pipeline.selection.selects_daily_active_sources(); + let selections = pending .iter() - .map(|(member, bucket_start)| { - let path = expected_nfcapd_path(root, member, *bucket_start, timezone)?; - ExpectedAbsence::capture(&path) - .map_err(|error| missing_day_absence_error(start, end, &path, error)) - }) - .collect() -} + .map(|index| sinks[*index].pipeline.selection.clone()) + .collect::>(); + let decoder_fingerprint = nfdump_decoder_fingerprint_for_pipeline(first_pipeline)?; + let revision_pool = build_revision_hash_pool()?; + let mut revision_starts = Vec::new(); + let mut revision_start = start; + while revision_start < end { + revision_starts.push(revision_start); + revision_start = next_local_five_minute_start(revision_start, &timezone)?; + } + let revisions = resolve_nfcapd_batch_revisions( + tree, + &revision_starts, + &decoder_fingerprint, + &revision_pool, + )?; + verify_nfdump_revision(first_pipeline)?; + let active_sources = if daily_active { + Some(resolve_daily_active_sources( + tree, + start, + end, + &timezone, + &selections, + &executable, + )?) + } else { + None + }; + verify_nfdump_revision(first_pipeline)?; + verify_prepared_revision_snapshots(&revisions)?; + let active_pairs = active_sources.as_ref().map(|active| { + selections + .iter() + .cloned() + .zip(active.iter().cloned()) + .collect::>() + }); + let decode_pool = build_nfcapd_decode_pool()?; + let mut next = start; -fn verify_missing_day_absences( - absences: &[ExpectedAbsence], - start: i64, - end: i64, -) -> Result<(), PipelineError> { - for absence in absences { - absence - .verify() - .map_err(|error| missing_day_absence_error(start, end, absence.path(), error))?; - } - Ok(()) -} + while next < end { + let batch_starts = nfcapd_batch_starts( + next, + end, + &timezone, + &tree.sources, + &tree.by_member_and_start, + &tree.member_bounds, + tree.extend_gaps_to_window, + )?; + next = batch_starts + .last() + .copied() + .map(|last| next_local_five_minute_start(last, &timezone)) + .transpose()? + .expect("non-empty nfcapd batch"); + let prepared = batch_starts + .iter() + .map(|start| prepare_nfcapd_tree_timestamp(tree, *start, &timezone, &revisions)) + .collect::, _>>()?; + let requests = prepared + .iter() + .flat_map(|timestamp| { + timestamp.jobs.iter().flat_map(move |job| { + job.present.iter().map(move |(member, path)| { + ((member.clone(), timestamp.bucket_start), path.clone()) + }) + }) + }) + .collect::>() + .into_iter() + .collect::>(); + verify_nfdump_revision(first_pipeline)?; + let decoded = decode_pool.install(|| { + requests + .par_iter() + .map(|((member, bucket_start), path)| { + let buckets = match &active_pairs { + Some(pairs) => ingest::read_nfcapd_buckets_with_active_sources( + path, + member, + pairs, + &executable, + &timezone, + )?, + None => vec![ingest::read_nfcapd_bucket( + path, + member, + &selections[0], + &executable, + &timezone, + )?], + }; + Ok::<_, PipelineError>(((member.clone(), *bucket_start), buckets)) + }) + .collect::, _>>() + })?; + verify_nfdump_revision(first_pipeline)?; -fn verify_single_day_guards( - guards: &NfcapdDayGuards, - root: &Path, - member_identities: &BTreeMap, - missing_absences: &[ExpectedAbsence], - start: i64, - end: i64, -) -> Result<(), PipelineError> { - invoke_single_commit_guard_hook(); - verify_member_directory_identities(root, member_identities)?; - for (path, revision) in &guards.capture_revisions { - if let Some(snapshot) = &revision.snapshot { - verify_file_snapshot(path, snapshot)?; - } - } - for (path, snapshot) in &guards.activity_snapshots { - if !guards.capture_revisions.contains_key(path) { - verify_file_snapshot(path, snapshot)?; + for (selection_index, sink_index) in pending.iter().copied().enumerate() { + let aggregate = aggregates[sink_index] + .as_mut() + .expect("pending output has aggregate state"); + for timestamp in &prepared { + for job in ×tamp.jobs { + let member_buckets = job + .present + .iter() + .map(|(member, _)| { + &decoded[&(member.clone(), timestamp.bucket_start)][selection_index] + }) + .collect::>(); + let logical = logical_source_bucket( + &job.source_id, + timestamp.bucket_start, + job.expected_units, + &member_buckets, + )?; + aggregate.reject_persisted_siblings( + sinks[sink_index].connection, + &logical, + &timezone, + )?; + publish_nfcapd_bucket( + sinks[sink_index].connection, + &logical, + &job.owners, + &job.absences, + &job.evidence, + true, + sinks[sink_index].pipeline.run_maad, + )?; + aggregate.include(&logical, &timezone)?; + sinks[sink_index].report.rollup_buckets += aggregate.flush_complete( + sinks[sink_index].connection, + sinks[sink_index].pipeline.run_maad, + )?; + sinks[sink_index].report.five_minute_buckets += 1; + } + } } } - if let Some(revision) = &guards.nfdump_revision { - verify_nfdump_revision_snapshot(revision)?; - } - verify_missing_day_absences(missing_absences, start, end) + Ok(()) } - -fn missing_day_warning_details( - root: &Path, - missing: &[(String, i64)], - timezone: &str, -) -> Result { - let mut details = missing - .iter() - .take(MAX_MISSING_DAY_WARNING_DETAILS) - .map(|(member, bucket_start)| { - expected_nfcapd_path(root, member, *bucket_start, timezone).map(|expected_path| { - format!( - "member={member} timestamp={bucket_start} expected_path={}", - expected_path.display() - ) - }) - }) - .collect::, _>>()?; - let omitted = missing.len().saturating_sub(details.len()); - if omitted != 0 { - details.push(format!("… {omitted} more missing inputs")); - } - Ok(details.join("; ")) +enum PreparedExplicitNfcapdKind { + File(PreparedRevision), + Gap { expected_path: Option }, } -/// Whether every member of every logical source has a discovered capture in this local day. -fn day_capture_is_complete( - sources: &[DatasetSource], - paths: &BTreeMap<(String, i64), PathBuf>, - start: i64, - end: i64, - timezone: &str, -) -> Result { - let mut bucket_start = start; - while bucket_start < end { - if sources.iter().any(|source| { - source - .members - .iter() - .any(|member| !paths.contains_key(&(member.clone(), bucket_start))) - }) { - return Ok(false); - } - bucket_start = next_local_five_minute_start(bucket_start, timezone)?; - } - Ok(true) +struct PreparedExplicitNfcapd { + path: PathBuf, + source_id: String, + bucket_start: i64, + kind: PreparedExplicitNfcapdKind, } -/// Non-daily selections may repair a partially published physical day as new members arrive. -/// Only treat that day as stale when its complete coverage envelope had already been committed; -/// daily-active selections use [`day_was_published`] below because their day cohort is atomic. -fn day_has_complete_coverage( +fn process_explicit_nfcapd_inputs( connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, - timezone: &str, -) -> Result { - let mut expected_bucket_count = 0_i64; - let mut bucket_start = start; - while bucket_start < end { - expected_bucket_count = expected_bucket_count.checked_add(1).ok_or_else(|| { - PipelineError::InvalidConfig("local day contains too many five-minute buckets".into()) - })?; - bucket_start = next_local_five_minute_start(bucket_start, timezone)?; - } - if sources.is_empty() { - return Ok(false); + inputs: &[InputSpec], + pipeline: &ResolvedPipeline, +) -> Result { + let mut prepared = Vec::new(); + for input in inputs { + let InputSpec::Nfcapd { + path, + source_id, + bucket_start, + gap, + expected_path, + } = input + else { + continue; + }; + let bucket_start = match bucket_start { + Some(start) => *start, + None if !gap => ingest::parse_nfcapd_bucket_start(path, &pipeline.timezone)?, + None => { + return Err(PipelineError::InvalidConfig( + "explicit nfcapd gap requires bucket_start".into(), + )); + } + }; + let kind = if *gap { + PreparedExplicitNfcapdKind::Gap { + expected_path: expected_path.clone(), + } + } else { + let (revision, snapshot) = prepare_file_revision( + connection, + path, + InputKind::Nfcapd, + nfdump_decoder_fingerprint_for_pipeline(pipeline)?, + )?; + PreparedExplicitNfcapdKind::File(PreparedRevision { + revision, + snapshot: Some(snapshot), + }) + }; + prepared.push(PreparedExplicitNfcapd { + path: path.clone(), + source_id: source_id.clone(), + bucket_start, + kind, + }); } - for source in sources { - let complete = connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = ?1 AND granularity = '5m' - AND bucket_start >= ?2 AND bucket_start < ?3 - AND coverage_state = 'complete'", - params![source.source_id, start, end], - |row| row.get::<_, i64>(0), - ) - .map_err(StorageError::from)?; - if complete != expected_bucket_count { - return Ok(false); - } + prepared.sort_unstable_by(|left, right| { + (left.bucket_start, &left.source_id, &left.path).cmp(&( + right.bucket_start, + &right.source_id, + &right.path, + )) + }); + if prepared.is_empty() { + return Ok(PipelineReport::default()); } - Ok(true) -} - -/// Any committed product, evidence, or processed-input provenance makes a day a prior -/// publication. Coverage is one part of that product, not the publication marker: if a coverage -/// row is damaged or missing while a capture also disappears, force must still remove the stale -/// day instead of treating it as a first run. -fn day_was_published( - connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, -) -> Result { - for source in sources { - let completion = connection - .query_row( - "SELECT EXISTS( - SELECT 1 FROM daily_product_completion - WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2 - ) OR EXISTS( - SELECT 1 FROM daily_product_completion_dirty - WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2 - )", - params![source.source_id, start, end], - |row| row.get::<_, i64>(0), - ) - .map_err(StorageError::from)?; - if completion != 0 { - return Ok(true); - } - for table in STATS_TABLE_NAMES { - let published = connection - .query_row( - &format!( - "SELECT EXISTS( - SELECT 1 FROM {table} - WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} - AND bucket_start >= ?2 AND bucket_start < ?3 - )" - ), - params![source.source_id, start, end], - |row| row.get::<_, i64>(0), - ) - .map_err(StorageError::from)?; - if published != 0 { - return Ok(true); + process_atomic(connection, pipeline, |aggregates, report| { + for input in &prepared { + match &input.kind { + PreparedExplicitNfcapdKind::File(owner) => process_nfcapd( + connection, + &input.path, + &input.source_id, + input.bucket_start, + owner, + pipeline, + aggregates, + report, + )?, + PreparedExplicitNfcapdKind::Gap { expected_path } => process_nfcapd_gap( + connection, + &input.path, + expected_path.as_deref(), + &input.source_id, + input.bucket_start, + pipeline, + aggregates, + report, + )?, } } - let evidence = connection - .query_row( - "SELECT EXISTS( - SELECT 1 FROM input_evidence - WHERE source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3 - )", - params![source.source_id, start, end], - |row| row.get::<_, i64>(0), - ) - .map_err(StorageError::from)?; - if evidence != 0 { - return Ok(true); - } - let provenance = connection - .query_row( - "SELECT EXISTS( - SELECT 1 FROM processed_inputs - WHERE input_kind = 'nfcapd' AND status = 'processed' - AND source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3 - )", - params![source.source_id, start, end], - |row| row.get::<_, i64>(0), - ) - .map_err(StorageError::from)?; - if provenance != 0 { - return Ok(true); - } - } - Ok(false) -} - -const CANONICAL_GRANULARITY_PREDICATE: &str = "granularity IN ('5m', '30m', '1h', '1d')"; - -const CANONICAL_SCOPE_PREDICATE: &str = "( - (src_visibility = 'all' AND dst_visibility = 'all') OR - (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR - (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR - (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR - (src_visibility = 'literal' AND dst_visibility = 'literal') -)"; - -fn canonical_row_family_predicate(table: &str) -> &'static str { - match table { - "traffic_stats" | "protocol_stats" => CANONICAL_SCOPE_PREDICATE, - "address_count_stats" => { - "( - address_side IN ('source', 'destination') AND - ((src_visibility = 'all' AND dst_visibility = 'all') OR - (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR - (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR - (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR - (src_visibility = 'literal' AND dst_visibility = 'literal')) - )" - } - "port_count_stats" => { - "( - port_side IN ('source', 'destination') AND - port_range IN ('low', 'high') AND - ((src_visibility = 'all' AND dst_visibility = 'all') OR - (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR - (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR - (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR - (src_visibility = 'literal' AND dst_visibility = 'literal')) - )" - } - "address_structure_stats" => { - "( - ip_version = 4 AND - address_side IN ('source', 'destination') AND - structure_kind IN ('structure', 'spectrum', 'dimension') AND - ((src_visibility = 'all' AND dst_visibility = 'all') OR - (src_visibility = 'anonymized' AND dst_visibility = 'anonymized') OR - (src_visibility = 'anonymized' AND dst_visibility = 'literal') OR - (src_visibility = 'literal' AND dst_visibility = 'anonymized') OR - (src_visibility = 'literal' AND dst_visibility = 'literal')) - )" - } - _ => unreachable!("unknown canonical product table {table}"), - } -} - -fn canonical_row_family_count(table: &str, dense: bool, run_maad: bool) -> i64 { - if !dense || (table == "address_structure_stats" && !run_maad) { - return 0; - } - let scopes = nfcapd_dense_traffic_scope_count(); - match table { - "traffic_stats" | "protocol_stats" => scopes, - "address_count_stats" => scopes * 2, - "port_count_stats" => scopes * 4, - // MAAD is emitted only for IPv4 address sets. Dense traffic has one IPv4 and one IPv6 - // row for each visibility scope, while each IPv4 side gets three MAAD structures. - "address_structure_stats" => scopes * 3, - _ => unreachable!("unknown canonical product table {table}"), - } -} - -fn canonical_coverage_state( - observed_units: i64, - expected_units: i64, - rejected_units: i64, -) -> Option<&'static str> { - if expected_units <= 0 - || observed_units < 0 - || rejected_units < 0 - || observed_units > expected_units - || rejected_units > expected_units - { - return None; - } - Some(if observed_units == expected_units && rejected_units == 0 { - "complete" - } else if observed_units == 0 && rejected_units == 0 { - "unknown" - } else { - "partial" + Ok(()) }) } -fn canonical_bucket_coverage_matches( - connection: &Connection, - source_id: &str, - bucket_start: i64, - expected_end: i64, - observed_units: usize, - expected_units: usize, -) -> Result { - let expected_units = i64::try_from(expected_units) - .map_err(|_| PipelineError::InvalidConfig("nfcapd coverage unit count overflow".into()))?; - let observed_units = i64::try_from(observed_units) - .map_err(|_| PipelineError::InvalidConfig("nfcapd observed unit count overflow".into()))?; - let Some(expected_state) = canonical_coverage_state(observed_units, expected_units, 0) else { - return Ok(false); - }; - let row = connection - .query_row( - "SELECT bucket_end, coverage_state, observed_units, expected_units, rejected_units - FROM bucket_coverage - WHERE source_id = ?1 AND granularity = '5m' AND bucket_start = ?2", - params![source_id, bucket_start], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, i64>(4)?, - )) - }, - ) - .optional() - .map_err(StorageError::from)?; - Ok( - row.is_some_and(|(bucket_end, state, observed, expected, rejected)| { - bucket_end == expected_end - && state == expected_state - && observed == observed_units - && expected == expected_units - && rejected == 0 - }), - ) -} - -fn canonical_bucket_rows_match( +#[allow(clippy::too_many_arguments)] +fn process_nfcapd( connection: &Connection, + path: &Path, source_id: &str, - granularity: Granularity, bucket_start: i64, - bucket_end: i64, - dense: bool, - run_maad: bool, -) -> Result { - for table in [ - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - ] { - let expected = canonical_row_family_count(table, dense, run_maad); - let query = format!( - "SELECT COUNT(*), COALESCE(SUM(CASE WHEN bucket_end = ?4 AND ip_version IN (4, 6) AND ({predicate}) THEN 1 ELSE 0 END), 0) - FROM {table} - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3", - predicate = canonical_row_family_predicate(table), - ); - let (total, canonical) = connection - .query_row( - &query, - params![source_id, granularity.as_str(), bucket_start, bucket_end], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), - ) - .map_err(StorageError::from)?; - if total != expected || canonical != expected { - return Ok(false); - } + owner: &PreparedRevision, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + if nfcapd_logical_bucket_processed( + connection, + source_id, + bucket_start, + std::slice::from_ref(&owner.revision), + )? { + report.skipped_inputs += 1; + return Ok(()); } - Ok(true) + verify_nfdump_revision(pipeline)?; + let bucket = ingest::read_nfcapd_bucket( + path, + source_id, + &pipeline.selection, + &pipeline.nfdump, + &pipeline.timezone, + ) + .map_err(|error| nfcapd_decode_error(source_id, bucket_start, path, error))?; + verify_nfdump_revision(pipeline)?; + aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; + publish_nfcapd_bucket( + connection, + &bucket, + std::slice::from_ref(owner), + &[], + &[InputEvidenceRow::new( + source_id, + source_id, + bucket_start, + bucket_start + FIVE_MINUTES, + &owner.revision.locator, + InputEvidenceState::Observed, + Some(owner.revision.fingerprint.clone()), + )], + false, + pipeline.run_maad, + )?; + aggregates.include(&bucket, &pipeline.timezone)?; + report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; + report.five_minute_buckets += 1; + Ok(()) } -/// A matching evidence/provenance pair is resumable only when the committed canonical bucket -/// topology is still present. Observed logical buckets are dense; all-missing buckets are valid -/// sparse coverage-only rows. -fn nfcapd_logical_bucket_has_canonical_topology( +#[allow(clippy::too_many_arguments)] +fn process_nfcapd_gap( connection: &Connection, + _locator_path: &Path, + expected_path: Option<&Path>, source_id: &str, bucket_start: i64, - observed_units: usize, - expected_units: usize, - run_maad: bool, -) -> Result { - #[cfg(test)] - NFCAPD_LOGICAL_BUCKET_TOPOLOGY_CALLS.with(|calls| calls.set(calls.get() + 1)); - let bucket_end = bucket_start - .checked_add(FIVE_MINUTES) - .ok_or_else(|| PipelineError::InvalidConfig("nfcapd bucket end overflow".into()))?; - if !canonical_bucket_coverage_matches( - connection, + pipeline: &ResolvedPipeline, + aggregates: &mut AggregateBuckets, + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + let expected_path = expected_path.ok_or_else(|| { + PipelineError::InvalidConfig( + "explicit nfcapd gap requires expected_path for absence verification".into(), + ) + })?; + let absence = ExpectedAbsence::capture(expected_path)?; + let evidence = [InputEvidenceRow::new( + source_id, source_id, bucket_start, - bucket_end, - observed_units, - expected_units, - )? { - return Ok(false); + bucket_start + FIVE_MINUTES, + expected_path.to_string_lossy(), + InputEvidenceState::Missing, + None, + )]; + if query_input_evidence(connection, source_id, bucket_start)? == evidence { + report.skipped_inputs += 1; + return Ok(()); } - canonical_bucket_rows_match( - connection, + let bucket = StatisticalBucket::new(BucketKey::new( source_id, Granularity::FiveMinutes, bucket_start, - bucket_end, - observed_units != 0, - run_maad, - ) -} - -#[derive(Clone, Copy, Debug)] -struct ExpectedNfcapdTopologyBucket { - bucket_end: i64, - child_count: i64, + bucket_start + FIVE_MINUTES, + )) + .with_coverage(BucketCoverage::new(1, 0, 0).map_err(DomainError::from)?) + .finish_owned(); + aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; + publish_nfcapd_bucket( + connection, + &bucket, + &[], + &[absence], + &evidence, + false, + pipeline.run_maad, + )?; + aggregates.include(&bucket, &pipeline.timezone)?; + report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; + report.five_minute_buckets += 1; + Ok(()) } -type ExpectedNfcapdTopology = BTreeMap>; - -fn expected_nfcapd_day_topology( - start: i64, - end: i64, - timezone: &str, -) -> Result { - let mut topology = BTreeMap::new(); - let mut bucket_start = start; - while bucket_start < end { - let five_minute_end = bucket_start - .checked_add(FIVE_MINUTES) - .ok_or_else(|| PipelineError::InvalidConfig("nfcapd bucket end overflow".into()))?; - topology - .entry(Granularity::FiveMinutes) - .or_insert_with(BTreeMap::new) - .insert( - bucket_start, - ExpectedNfcapdTopologyBucket { - bucket_end: five_minute_end, - child_count: 1, - }, - ); - for granularity in [ - Granularity::ThirtyMinutes, - Granularity::OneHour, - Granularity::OneDay, - ] { - let (rollup_start, rollup_end) = aggregate_bounds(bucket_start, granularity, timezone)?; - topology - .entry(granularity) - .or_insert_with(BTreeMap::new) - .entry(rollup_start) - .and_modify(|bucket: &mut ExpectedNfcapdTopologyBucket| { - bucket.child_count += 1; +fn normalize_sources( + root: &Path, + source_ids: &[String], + sources: &[DatasetSource], +) -> Result, PipelineError> { + if !source_ids.is_empty() && !sources.is_empty() { + return Err(PipelineError::InvalidConfig( + "nfcapd_tree cannot define both source_ids and sources".into(), + )); + } + let mut normalized = if !sources.is_empty() { + sources.to_vec() + } else if !source_ids.is_empty() { + source_ids + .iter() + .map(|source_id| DatasetSource { + source_id: source_id.clone(), + members: vec![source_id.clone()], + }) + .collect() + } else { + let entries = fs::read_dir(root)?; + entries + .filter_map(Result::ok) + .filter_map(|entry| { + entry.file_type().ok()?.is_dir().then(|| { + let source_id = entry.file_name().to_string_lossy().into_owned(); + DatasetSource { + source_id: source_id.clone(), + members: vec![source_id], + } }) - .or_insert(ExpectedNfcapdTopologyBucket { - bucket_end: rollup_end, - child_count: 1, - }); + }) + .collect() + }; + normalized.sort_unstable_by(|left, right| left.source_id.cmp(&right.source_id)); + let mut ids = BTreeSet::new(); + for source in &normalized { + if !is_safe_path_component(&source.source_id) + || source.members.is_empty() + || !ids.insert(source.source_id.clone()) + { + return Err(PipelineError::InvalidConfig( + "logical sources require unique non-empty IDs and members".into(), + )); + } + let mut members = BTreeSet::new(); + for member in &source.members { + if !is_safe_path_component(member) || !members.insert(member) { + return Err(PipelineError::InvalidConfig(format!( + "source {:?} has an unsafe or duplicate member path component", + source.source_id + ))); + } + if !root.join(member).is_dir() { + return Err(PipelineError::InvalidConfig(format!( + "source {:?} references missing member directory {:?}", + source.source_id, member + ))); + } } - bucket_start = next_local_five_minute_start(bucket_start, timezone)?; } - Ok(topology) -} -fn nfcapd_day_coverage_is_canonical( - connection: &Connection, - source_id: &str, - source_units: usize, - topology: &ExpectedNfcapdTopology, - day_start: i64, - day_end: i64, -) -> Result { - let rows = connection - .prepare(&format!( - "SELECT granularity, bucket_start, bucket_end, coverage_state, - observed_units, expected_units, rejected_units - FROM bucket_coverage - WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} - AND bucket_start >= ?2 AND bucket_start < ?3 - ORDER BY granularity, bucket_start" - )) - .map_err(StorageError::from)? - .query_map(params![source_id, day_start, day_end], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, String>(3)?, - row.get::<_, i64>(4)?, - row.get::<_, i64>(5)?, - row.get::<_, i64>(6)?, - )) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - let actual = rows - .into_iter() - .map( - |(granularity, start, end, state, observed, expected, rejected)| { - ( - (granularity, start, end), - (state, observed, expected, rejected), - ) - }, - ) - .collect::>(); - let source_units = i64::try_from(source_units) - .map_err(|_| PipelineError::InvalidConfig("nfcapd source unit count overflow".into()))?; - let mut expected_keys = BTreeSet::new(); - for (granularity, buckets) in topology { - for (bucket_start, bucket) in buckets { - expected_keys.insert(( - granularity.as_str().to_owned(), - *bucket_start, - bucket.bucket_end, - )); - let expected_units = source_units - .checked_mul(bucket.child_count) - .ok_or_else(|| { - PipelineError::InvalidConfig("nfcapd coverage unit count overflow".into()) - })?; - let Some((state, observed, actual_expected, rejected)) = actual.get(&( - granularity.as_str().to_owned(), - *bucket_start, - bucket.bucket_end, - )) else { - return Ok(false); - }; - if *state != "complete" - || *observed != expected_units - || *actual_expected != expected_units - || *rejected != 0 - { - return Ok(false); - } + let member_ids = normalized + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>(); + let mut member_paths = BTreeMap::::new(); + for member in member_ids { + let member_path = root.join(&member); + let canonical_member_path = canonical_path(&member_path)?; + if let Some(previous_member) = member_paths.get(&canonical_member_path) { + return Err(PipelineError::InvalidConfig(format!( + "nfcapd_tree member IDs {:?} and {:?} resolve to the same directory {}", + previous_member, + member, + canonical_member_path.display() + ))); } + member_paths.insert(canonical_member_path, member); } - Ok(actual.keys().all(|key| expected_keys.contains(key)) && actual.len() == expected_keys.len()) + Ok(normalized) } -fn nfcapd_day_rows_are_canonical( - connection: &Connection, +fn merge_source_bucket( source_id: &str, - topology: &ExpectedNfcapdTopology, - run_maad: bool, - day_start: i64, - day_end: i64, -) -> Result { - for table in [ - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - ] { - let query = format!( - "SELECT granularity, bucket_start, MIN(bucket_end), MAX(bucket_end), COUNT(*), - COALESCE(SUM(CASE WHEN ip_version IN (4, 6) AND ({predicate}) THEN 1 ELSE 0 END), 0) - FROM {table} - WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} - AND bucket_start >= ?2 AND bucket_start < ?3 - GROUP BY granularity, bucket_start", - predicate = canonical_row_family_predicate(table), - ); - let rows = connection - .prepare(&query) - .map_err(StorageError::from)? - .query_map(params![source_id, day_start, day_end], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, i64>(4)?, - row.get::<_, i64>(5)?, - )) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - let actual = rows - .into_iter() - .map( - |(granularity, start, minimum_end, maximum_end, total, canonical)| { - ( - (granularity, start), - (minimum_end, maximum_end, total, canonical), - ) - }, - ) - .collect::>(); - let mut expected_keys = BTreeSet::new(); - for (granularity, buckets) in topology { - let expected = canonical_row_family_count(table, true, run_maad); - for (bucket_start, bucket) in buckets { - let key = (granularity.as_str().to_owned(), *bucket_start); - if expected == 0 { - if actual.contains_key(&key) { - return Ok(false); - } - continue; - } - expected_keys.insert(key.clone()); - let Some((minimum_end, maximum_end, total, canonical)) = actual.get(&key) else { - return Ok(false); - }; - if minimum_end != maximum_end - || *minimum_end != bucket.bucket_end - || *total != expected - || *canonical != expected - { - return Ok(false); - } - } - } - if actual.len() != expected_keys.len() - || actual.keys().any(|key| !expected_keys.contains(key)) - { - return Ok(false); - } + bucket_start: i64, + expected_units: usize, + members: &[&CanonicalBucket], +) -> Result { + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + ); + let mut builder = if members.is_empty() { + StatisticalBucket::new(key) + } else { + StatisticalBucket::dense(key) } - Ok(true) + .with_coverage(BucketCoverage::empty()); + for member in members { + builder.include(member)?; + } + let coverage = BucketCoverage::new( + u64::try_from(expected_units).unwrap_or(u64::MAX), + u64::try_from(members.len()).unwrap_or(u64::MAX), + 0, + ) + .map_err(DomainError::from)?; + Ok(builder.with_coverage(coverage).finish_owned()) } -/// Validate the complete topology once before a daily-active day is treated as resumable. The -/// grouped queries inspect all row families and all four granularities in the day, so a single -/// damaged bucket or rollup cannot be hidden by a matching total elsewhere. -fn nfcapd_day_has_canonical_topology( - connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, - timezone: &str, - run_maad: bool, -) -> Result { - #[cfg(test)] - NFCAPD_DAY_TOPOLOGY_AUDIT_CALLS.with(|calls| calls.set(calls.get() + 1)); - let topology = expected_nfcapd_day_topology(start, end, timezone)?; - if topology - .get(&Granularity::FiveMinutes) - .is_none_or(BTreeMap::is_empty) +fn logical_source_bucket<'a>( + source_id: &str, + bucket_start: i64, + expected_units: usize, + members: &[&'a CanonicalBucket], +) -> Result, PipelineError> { + let expected_key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + FIVE_MINUTES, + ); + if expected_units == 1 + && let [member] = members + && member.key == expected_key { - return Ok(false); - } - let range_end = topology - .get(&Granularity::OneDay) - .and_then(|buckets| buckets.values().map(|bucket| bucket.bucket_end).max()) - .unwrap_or(end); - for source in sources { - if !nfcapd_day_coverage_is_canonical( - connection, - &source.source_id, - source.members.len(), - &topology, - start, - range_end, - )? { - return Ok(false); - } - if !nfcapd_day_rows_are_canonical( - connection, - &source.source_id, - &topology, - run_maad, - start, - range_end, - )? { - return Ok(false); - } + return Ok(Cow::Borrowed(member)); } - Ok(!sources.is_empty()) + Ok(Cow::Owned(merge_source_bucket( + source_id, + bucket_start, + expected_units, + members, + )?)) } -/// Classify completion evidence for every logical source in a daily-active product. -/// -/// A dirty tombstone is intentionally distinct from a missing marker. Missing markers (including -/// databases created before completion markers existed) may use the legacy topology audit once; -/// dirty days must be rebuilt with `--force` before any normal resume mutation is attempted. -fn nfcapd_day_completion_state( +#[derive(Clone, Debug)] +struct PreparedRevision { + revision: InputRevision, + snapshot: Option, +} + +struct PreparedTreeJob { + source_id: String, + expected_units: usize, + present: Vec<(String, PathBuf)>, + owners: Vec, + absences: Vec, + evidence: Vec, +} + +struct PreparedTreeTimestamp { + bucket_start: i64, + jobs: Vec, +} + +#[allow(clippy::too_many_arguments)] +fn publish_nfcapd_bucket( connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, + bucket: &CanonicalBucket, + owners: &[PreparedRevision], + absences: &[ExpectedAbsence], + evidence: &[InputEvidenceRow], + replace_existing: bool, run_maad: bool, -) -> Result { - let Some(product_fingerprint) = current_product_fingerprint(connection)? else { - return Ok(DailyProductCompletionState::Missing); - }; - if sources.is_empty() { - return Ok(DailyProductCompletionState::Missing); +) -> Result<(), PipelineError> { + for prepared in owners { + if let Some(snapshot) = &prepared.snapshot { + verify_file_snapshot(Path::new(&prepared.revision.locator), snapshot)?; + } } - let mut missing = false; - for source in sources { - match daily_product_completion_state( + for absence in absences { + absence.verify()?; + } + reject_overlapping_bucket(connection, bucket, InputKind::Nfcapd, "", replace_existing)?; + if replace_existing { + connection + .execute( + "DELETE FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = ?1 AND bucket_start = ?2", + params![bucket.key.source_id, bucket.key.bucket_start], + ) + .map_err(StorageError::from)?; + } + for prepared in owners { + let revision = &prepared.revision; + upsert_input_bucket( connection, - &source.source_id, - start, - end, - &product_fingerprint, - run_maad, - )? { - DailyProductCompletionState::Clean => {} - DailyProductCompletionState::Dirty => return Ok(DailyProductCompletionState::Dirty), - DailyProductCompletionState::Missing => missing = true, - } + &InputBucket { + input_kind: InputKind::Nfcapd, + input_locator: revision.locator.clone(), + scan_locator: revision.locator.clone(), + source_id: bucket.key.source_id.clone(), + bucket_start: bucket.key.bucket_start, + bucket_end: bucket.key.bucket_end, + revision: revision.clone(), + file_snapshot: prepared.snapshot.clone(), + }, + replace_existing, + )?; } - Ok(if missing { - DailyProductCompletionState::Missing - } else { - DailyProductCompletionState::Clean - }) + write_buckets(connection, std::slice::from_ref(bucket), run_maad)?; + replace_input_evidence( + connection, + &bucket.key.source_id, + bucket.key.bucket_start, + evidence, + )?; + for prepared in owners { + let revision = &prepared.revision; + mark_input_bucket_status( + connection, + InputKind::Nfcapd, + &revision.locator, + &bucket.key.source_id, + bucket.key.bucket_start, + InputStatus::Processed, + revision, + None, + )?; + } + Ok(()) } -/// Publish completion markers after a complete daily-active transaction has written every -/// canonical family, rollup, and evidence/provenance row. -fn mark_nfcapd_day_complete( +fn reject_overlapping_bucket( connection: &Connection, - sources: &[DatasetSource], - start: i64, - end: i64, - run_maad: bool, + bucket: &CanonicalBucket, + input_kind: InputKind, + allowed_scan: &str, + replace_nfcapd: bool, ) -> Result<(), PipelineError> { - let product_fingerprint = current_product_fingerprint(connection)?.ok_or_else(|| { - PipelineError::InvalidConfig( - "cannot publish a daily product completion marker before product identity binding" - .into(), + let conflict = connection + .query_row( + "SELECT input_kind, input_locator, scan_locator FROM processed_inputs + WHERE source_id = ?1 AND bucket_start = ?2 + AND NOT (input_kind = ?3 AND scan_locator = ?4) + ORDER BY input_kind, input_locator LIMIT 1", + params![ + bucket.key.source_id, + bucket.key.bucket_start, + input_kind.as_str(), + allowed_scan, + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, ) - })?; - for source in sources { - upsert_daily_product_completion( - connection, - &source.source_id, - start, - end, - &product_fingerprint, - run_maad, - )?; + .optional() + .map_err(StorageError::from)?; + if let Some((kind, locator, _)) = conflict + && !(replace_nfcapd && kind == InputKind::Nfcapd.as_str()) + { + return Err(PipelineError::InvalidConfig(format!( + "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", + bucket.key.source_id, bucket.key.bucket_start + ))); } Ok(()) } -fn source_has_candidate( - source: &DatasetSource, - bucket_start: i64, - paths: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - extend_gaps_to_window: bool, -) -> bool { - let has_file = source - .members - .iter() - .any(|member| paths.contains_key(&(member.clone(), bucket_start))); - has_file - || extend_gaps_to_window - || source.members.iter().any(|member| { - member_bounds - .get(member) - .is_some_and(|(first, last)| *first <= bucket_start && bucket_start <= *last) - }) +#[derive(Default)] +struct AggregateBuckets { + builders: BTreeMap<(String, Granularity, i64, i64), StatisticalBucket>, + published_through: BTreeMap, + owned_keys: BTreeSet<(String, i64)>, + current_run_keys: BTreeSet<(String, i64)>, + persisted_sibling_validations: BTreeSet<(String, i64, bool)>, } -/// Group local five-minute starts so each decode batch has at most twelve physical requests when -/// possible. A timestamp with more than twelve members is kept as one batch and drained in -/// physical-request chunks by the decode caller. -fn nfcapd_batch_starts( - start: i64, - end: i64, - timezone: &str, - sources: &[DatasetSource], - paths: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - extend_gaps_to_window: bool, -) -> Result, PipelineError> { - let mut starts = Vec::with_capacity(NFCAPD_DECODE_BATCH_SIZE); - let mut physical_requests = BTreeSet::new(); - let mut next = start; - while next < end { - let timestamp_requests = sources - .iter() - .filter(|source| { - source_has_candidate(source, next, paths, member_bounds, extend_gaps_to_window) - }) - .flat_map(|source| { - source.members.iter().filter_map(|member| { - paths - .contains_key(&(member.clone(), next)) - .then_some((member.clone(), next)) - }) - }) - .collect::>(); - let would_exceed_physical_limit = !starts.is_empty() - && physical_requests.len() + timestamp_requests.len() > NFCAPD_DECODE_BATCH_SIZE; - if would_exceed_physical_limit || starts.len() == NFCAPD_DECODE_BATCH_SIZE { - break; +impl AggregateBuckets { + fn with_owned_keys(owned_keys: BTreeSet<(String, i64)>) -> Self { + Self { + owned_keys, + ..Self::default() } - starts.push(next); - physical_requests.extend(timestamp_requests); - next = next_local_five_minute_start(next, timezone)?; } - Ok(starts) -} - -fn nfcapd_decode_request_chunks(requests: &[T]) -> impl Iterator { - requests.chunks(NFCAPD_DECODE_BATCH_SIZE) -} -/// External inputs observed while preparing one local day. The maps stay bounded to that day and -/// let the transaction's final guard verify the exact files used by the day before COMMIT. -#[derive(Clone, Debug, Default)] -struct NfcapdDayGuards { - capture_revisions: BTreeMap, - activity_snapshots: BTreeMap, - nfdump_revision: Option, -} + fn reject_persisted_siblings( + &mut self, + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, + ) -> Result<(), PipelineError> { + self.reject_persisted_siblings_inner(connection, child, timezone, false) + } -struct NfcapdDayResult { - profile: NfcapdDayPublishProfile, - guards: NfcapdDayGuards, -} + fn reject_persisted_csv_siblings( + &mut self, + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, + ) -> Result<(), PipelineError> { + self.reject_persisted_siblings_inner(connection, child, timezone, true) + } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_tree_day( - connection: &Connection, - root: &Path, - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - start: i64, - end: i64, - extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - capture_snapshots: &BTreeMap, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, - canonical_day_verified: bool, -) -> Result { - let day_started = Instant::now(); - let mut prepare_elapsed = Duration::ZERO; - let mut decode_elapsed = Duration::ZERO; - let mut publish_elapsed = Duration::ZERO; - let mut publish_profile = NfcapdDayPublishProfile::default(); - let mut guards = NfcapdDayGuards { - nfdump_revision: pipeline - .selection - .selects_daily_active_sources() - .then(|| pipeline.nfdump_revision.clone()) - .flatten(), - ..NfcapdDayGuards::default() - }; - let revision_hash_workers = std::thread::available_parallelism() - .map_or(1, std::num::NonZeroUsize::get) - .min(NFCAPD_REVISION_HASH_MAX_WORKERS); - let revision_pool = rayon::ThreadPoolBuilder::new() - .num_threads(revision_hash_workers) - .thread_name(|index| format!("nfcapd-revision-{index}")) - .build() - .map_err(|error| { - PipelineError::InvalidConfig(format!("failed to build revision hash pool: {error}")) - })?; - let decoder_fingerprint = nfdump_decoder_fingerprint_for_pipeline(pipeline)?; - let revision_context = NfcapdRevisionContext { - connection, - sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - force, - decoder_fingerprint, - capture_snapshots, - revision_pool: &revision_pool, - }; - let mut bucket_start = start; - let mut batches = Vec::new(); - while bucket_start < end { - let prepare_started = Instant::now(); - let batch_starts = nfcapd_batch_starts( - bucket_start, - end, - &pipeline.timezone, - sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - )?; - bucket_start = batch_starts - .last() - .copied() - .map(|last| next_local_five_minute_start(last, &pipeline.timezone)) - .transpose()? - .expect("non-empty nfcapd batch while processing a non-empty window"); - let revisions = resolve_nfcapd_batch_revisions(&revision_context, &batch_starts)?; - if pipeline.selection.selects_daily_active_sources() { - guards.capture_revisions.extend( - revisions - .iter() - .map(|(path, revision)| (path.clone(), revision.clone())), - ); + fn reject_persisted_siblings_inner( + &mut self, + connection: &Connection, + child: &CanonicalBucket, + timezone: &str, + allow_staged_csv_keys: bool, + ) -> Result<(), PipelineError> { + let (day_start, day_end) = + aggregate_bounds(child.key.bucket_start, Granularity::OneDay, timezone)?; + let validation_key = ( + child.key.source_id.clone(), + day_start, + allow_staged_csv_keys, + ); + if self.persisted_sibling_validations.contains(&validation_key) { + return Ok(()); } - let mut batch = Vec::with_capacity(batch_starts.len()); - for bucket_start in batch_starts { - batch.push(prepare_nfcapd_tree_timestamp( - connection, - root, - sources, - by_member_and_start, - member_bounds, - bucket_start, - extend_gaps_to_window, - force, - pipeline, - report, - &revisions, - canonical_day_verified, - )?); + let mut statement = connection + .prepare( + "SELECT DISTINCT bucket_start FROM traffic_stats + WHERE source_id = ?1 AND granularity = '5m' + AND bucket_start >= ?2 AND bucket_start < ?3 + ORDER BY bucket_start", + ) + .map_err(StorageError::from)?; + let persisted = statement + .query_map(params![child.key.source_id, day_start, day_end], |row| { + row.get::<_, i64>(0) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + for bucket_start in persisted { + if bucket_start == child.key.bucket_start + || self + .owned_keys + .contains(&(child.key.source_id.clone(), bucket_start)) + || self + .current_run_keys + .contains(&(child.key.source_id.clone(), bucket_start)) + { + continue; + } + if allow_staged_csv_keys + && connection + .query_row( + "SELECT 1 FROM csv_bucket_stage + WHERE source_id = ?1 AND bucket_start = ?2 LIMIT 1", + params![child.key.source_id, bucket_start], + |_| Ok(()), + ) + .optional() + .map_err(StorageError::from)? + .is_some() + { + continue; + } + return Err(PipelineError::InvalidConfig(format!( + "cannot reopen a persisted aggregate interval exactly: source={:?} bucket_start={} shares its local day with persisted five-minute bucket {bucket_start} from another transaction", + child.key.source_id, child.key.bucket_start + ))); } - prepare_elapsed += prepare_started.elapsed(); - batches.push(batch); + // This validation is intentionally local to one aggregate transaction/output. Persisted + // siblings cannot change except through keys owned by this run, which are already excluded + // above, so later children in the same source/day can reuse the successful result. + self.persisted_sibling_validations.insert(validation_key); + Ok(()) } - if pipeline.selection.selects_daily_active_sources() - && !force - && batches - .iter() - .flatten() - .flat_map(|timestamp| ×tamp.jobs) - .any(|job| job.is_repair) - { - return Err(PipelineError::InvalidConfig(format!( - "daily_active_sources input changed for local day {start}..{end}; rerun that whole day with --force" - ))); - } - let has_jobs = batches - .iter() - .flatten() - .any(|timestamp| !timestamp.jobs.is_empty()); - let active_resolution = if pipeline.selection.selects_daily_active_sources() && has_jobs { - verify_nfdump_revision(pipeline)?; - Some(resolve_daily_active_sources( - sources, - by_member_and_start, - start, - end, - pipeline, - capture_snapshots, - &guards.capture_revisions, - )?) - } else { - None - }; - if active_resolution.is_some() { - verify_nfdump_revision(pipeline)?; - } - if let Some((active_sources, _)) = &active_resolution { - publish_profile.active_set_count = profile_count(active_sources.len()); + fn include(&mut self, child: &CanonicalBucket, timezone: &str) -> Result<(), PipelineError> { + if self + .published_through + .get(&child.key.source_id) + .is_some_and(|previous| child.key.bucket_start <= *previous) + { + return Err(PipelineError::InvalidConfig(format!( + "five-minute buckets must be unique and chronological for source {:?}: {} followed {}", + child.key.source_id, + self.published_through[&child.key.source_id], + child.key.bucket_start + ))); + } + for granularity in [ + Granularity::ThirtyMinutes, + Granularity::OneHour, + Granularity::OneDay, + ] { + let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; + let key = (child.key.source_id.clone(), granularity, start, end); + let builder = self.builders.entry(key.clone()).or_insert_with(|| { + StatisticalBucket::new(BucketKey::new(&key.0, key.1, key.2, key.3)) + }); + builder.include(child)?; + } + self.published_through + .insert(child.key.source_id.clone(), child.key.bucket_start); + self.current_run_keys + .insert((child.key.source_id.clone(), child.key.bucket_start)); + Ok(()) } - let decode_pool = if has_jobs { - Some(build_nfcapd_decode_pool()?) - } else { - None - }; - for batch in batches { - verify_nfdump_revision(pipeline)?; - let decode_started = Instant::now(); - let needed = batch + fn flush_complete( + &mut self, + connection: &Connection, + run_maad: bool, + ) -> Result { + let complete_keys = self + .builders .iter() - .flat_map(|timestamp| { - timestamp.jobs.iter().flat_map(|job| { - job.present.iter().map(|(member, path)| { - let snapshot = timestamp - .revision_cache - .get(member) - .and_then(|owner| owner.snapshot.clone()) - .expect("present member has a snapshot"); - ( - (member.clone(), timestamp.bucket_start), - (path.clone(), snapshot), - ) - }) - }) - }) - .collect::>(); - let requests = needed.into_iter().collect::>(); - let mut decoded_cache = BTreeMap::new(); - for request_chunk in nfcapd_decode_request_chunks(&requests) { - let decoded = decode_pool - .as_ref() - .expect("pending nfcapd work has a decode pool") - .install(|| { - request_chunk - .par_iter() - .map(|((member, bucket_start), (path, snapshot))| { - let bucket = (|| -> Result { - let bucket = match &active_resolution { - Some((active_sources, _)) => { - ingest::read_nfcapd_bucket_with_active_sources( - path, - member, - &pipeline.selection, - active_sources.clone(), - &pipeline.nfdump, - &pipeline.timezone, - )? - } - None => ingest::read_nfcapd_bucket( - path, - member, - &pipeline.selection, - &pipeline.nfdump, - &pipeline.timezone, - )?, - }; - verify_file_snapshot(path, snapshot)?; - Ok(bucket) - })() - .map_err(|error| { - nfcapd_decode_error(member, *bucket_start, path, error) - })?; - Ok::<_, PipelineError>(((member.clone(), *bucket_start), bucket)) - }) - .collect::, _>>() - })?; - decoded_cache.extend(decoded); - verify_nfdump_revision(pipeline)?; - } - decode_elapsed += decode_started.elapsed(); - - let publish_started = Instant::now(); - for timestamp in batch { - for job in timestamp.jobs { - let member_buckets = job - .present - .iter() - .map(|(member, _)| { - decoded_cache - .get(&(member.clone(), timestamp.bucket_start)) - .expect("requested physical member was decoded") - }) - .collect::>(); - let logical_started = Instant::now(); - let logical = logical_source_bucket( - &job.source_id, - timestamp.bucket_start, - job.expected_units, - &member_buckets, - )?; - publish_profile.logical_source_elapsed += logical_started.elapsed(); - let sibling_started = Instant::now(); - if !job.is_repair { - aggregates.reject_persisted_siblings( - connection, - &logical, - &pipeline.timezone, - )?; - } - publish_profile.persisted_sibling_elapsed += sibling_started.elapsed(); - let bucket_profile = publish_nfcapd_bucket_profiled( - connection, - &logical, - &job.owners, - &job.absences, - &job.evidence, - true, - force, - pipeline.run_maad, - )?; - publish_profile.bucket_publish.include(bucket_profile); - let flushed = if job.is_repair { - refresh_rollups_after_five_minute_repair( - connection, - &logical, - &pipeline.timezone, - )?; - 0 - } else { - let aggregate_profile = - aggregates.include_profiled(&logical, &pipeline.timezone)?; - publish_profile.aggregate_include.include(aggregate_profile); - let flush_started = Instant::now(); - let (flushed, rollup_write) = - aggregates.flush_complete_profiled(connection, pipeline.run_maad)?; - publish_profile.completed_rollup_flush_elapsed += flush_started.elapsed(); - publish_profile.completed_rollup_write.include(rollup_write); - publish_profile.completed_rollup_flushes += 1; - if flushed > 0 { - publish_profile.nonempty_rollup_flushes += 1; - } - flushed - }; - publish_profile.logical_buckets += 1; - report.rollup_buckets += flushed; - report.five_minute_buckets += 1; - } - decoded_cache.retain(|(_, start), _| *start != timestamp.bucket_start); - } - publish_elapsed += publish_started.elapsed(); + .filter(|(_, builder)| builder.has_complete_five_minute_coverage()) + .map(|(key, _)| key.clone()) + .collect::>(); + let buckets = complete_keys + .into_iter() + .filter_map(|key| self.builders.remove(&key)) + .map(StatisticalBucket::finish_owned) + .collect::>(); + let count = buckets.len(); + write_buckets(connection, &buckets, run_maad)?; + Ok(count) } - if let Some((_, snapshots)) = &active_resolution { - for (path, snapshot) in snapshots { - guards - .activity_snapshots - .insert(path.clone(), snapshot.clone()); - verify_file_snapshot(path, snapshot)?; - } + + fn finish(self) -> Vec { + self.builders + .into_values() + .map(StatisticalBucket::finish_owned) + .collect() } - verify_nfdump_revision(pipeline)?; - publish_profile.day_elapsed = day_started.elapsed(); - publish_profile.prepare_elapsed = prepare_elapsed; - publish_profile.decode_elapsed = decode_elapsed; - publish_profile.batch_publish_elapsed = publish_elapsed; - tracing::info!( - target: "netflow_db::profile", - phase = "nfcapd_tree_day", - day_start = start, - day_end = end, - elapsed_seconds = publish_profile.day_elapsed.as_secs_f64(), - prepare_seconds = prepare_elapsed.as_secs_f64(), - decode_seconds = decode_elapsed.as_secs_f64(), - publish_seconds = publish_elapsed.as_secs_f64(), - ); - Ok(NfcapdDayResult { - profile: publish_profile, - guards, - }) } -fn resolve_daily_active_sources( - sources: &[DatasetSource], - paths: &BTreeMap<(String, i64), PathBuf>, - start: i64, - end: i64, +fn publish_rollups( + connection: &Connection, + aggregates: AggregateBuckets, pipeline: &ResolvedPipeline, - capture_snapshots: &BTreeMap, - revision_snapshots: &BTreeMap, -) -> Result { - let physical_ids = sources - .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>(); - let physical_ids = physical_ids.into_iter().collect::>(); - let activity_pool = build_nfcapd_activity_pool()?; - let mut combined = HashMap::::new(); - let mut snapshots = Vec::new(); - for member_chunk in physical_ids.chunks(NFCAPD_DECODE_BATCH_SIZE) { - let requests = member_chunk - .iter() - .map(|member| { - nfcapd_day_activity_paths(paths, member, start, end, &pipeline.timezone) - .map(|member_paths| (member.clone(), member_paths)) - }) - .collect::, _>>()?; - let member_results = activity_pool.install(|| { - requests - .par_iter() - .map(|(member, member_paths)| { - let member_snapshots = member_paths - .iter() - .map(|path| { - let snapshot = capture_snapshots - .get(path) - .cloned() - .or_else(|| { - revision_snapshots - .get(path) - .and_then(|revision| revision.snapshot.clone()) - }) - .map(Ok) - .unwrap_or_else(|| capture_nfcapd_snapshot(path)); - snapshot - .map(|snapshot| (path.clone(), snapshot)) - .map_err(|error| { - daily_activity_scan_error( - member, - start, - end, - member_paths, - error, - ) - }) - }) - .collect::, PipelineError>>()?; - let activity = ingest::read_nfcapd_daily_source_activity( - member_paths, - &pipeline.selection, - &pipeline.nfdump, - ) - .map_err(|error| { - daily_activity_scan_error(member, start, end, member_paths, error) - })?; - Ok::<_, PipelineError>((activity, member_snapshots)) - }) - .collect::, _>>() - })?; - for (activity, member_snapshots) in member_results { - snapshots.extend(member_snapshots); - for (address, metrics) in activity { - combined.entry(address).or_default().include(metrics); - } - } - } - for (path, snapshot) in &snapshots { - verify_file_snapshot(path, snapshot)?; - } - - let mut active = AddressSet::default(); - active.extend(combined.into_iter().filter_map(|(address, metrics)| { - FlowSelection::daily_activity_threshold_met(metrics.flows, metrics.packets, metrics.bytes) - .then_some(address) - })); - tracing::info!( - day_start = start, - day_end = end, - active_sources = active.len(), - "resolved daily active sources" - ); - Ok((Arc::new(active), snapshots)) + report: &mut PipelineReport, +) -> Result<(), PipelineError> { + let rollups = aggregates.finish(); + write_buckets(connection, &rollups, pipeline.run_maad)?; + report.rollup_buckets += rollups.len(); + Ok(()) } - -struct NfcapdRevisionProbe { - path: PathBuf, - observed: FileSnapshot, - cached_content_fingerprint: Option, +fn aggregate_bounds( + bucket_start: i64, + granularity: Granularity, + timezone: &str, +) -> Result<(i64, i64), PipelineError> { + let timestamp = Timestamp::from_second(bucket_start) + .map_err(|error| PipelineError::Time(error.to_string()))?; + let zoned = timestamp + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))?; + let start = match granularity { + Granularity::ThirtyMinutes => zoned + .round( + ZonedRound::new() + .smallest(Unit::Minute) + .increment(30) + .mode(RoundMode::Trunc), + ) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::OneHour => zoned + .round( + ZonedRound::new() + .smallest(Unit::Hour) + .mode(RoundMode::Trunc), + ) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::OneDay => zoned + .date() + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::FiveMinutes => { + return Err(PipelineError::InvalidConfig( + "five-minute input is not a rollup granularity".into(), + )); + } + }; + let end = match granularity { + Granularity::ThirtyMinutes => start + 1_800, + Granularity::OneHour => start + 3_600, + Granularity::OneDay => zoned + .date() + .tomorrow() + .and_then(|date| date.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(), + Granularity::FiveMinutes => unreachable!("rejected above"), + }; + Ok((start, end)) } -/// Hash a capture after an already-captured observation, retaining the usual before/after -/// stability check without taking a redundant pre-hash snapshot. -fn capture_file_revision_with_snapshot( - path: &Path, - observed: &FileSnapshot, -) -> Result<(String, FileSnapshot), ProvenanceError> { - let content_fingerprint = file_sha256(path)?; - let after = FileSnapshot::capture(path)?; - if &after != observed { - return Err(ProvenanceError::InputContentChanged(format!( - "Input changed while its revision was being captured: {:?}", - path +fn next_local_five_minute_start(bucket_start: i64, timezone: &str) -> Result { + let current = Timestamp::from_second(bucket_start) + .and_then(|timestamp| timestamp.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))?; + let next = current + .datetime() + .checked_add(5.minutes()) + .and_then(|datetime| datetime.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second(); + if next <= bucket_start { + return Err(PipelineError::Time(format!( + "local five-minute clock did not advance after {bucket_start} in {timezone:?}" ))); } - Ok((content_fingerprint, after)) + Ok(next) } -struct NfcapdRevisionContext<'a> { - connection: &'a Connection, - sources: &'a [DatasetSource], - by_member_and_start: &'a BTreeMap<(String, i64), PathBuf>, - member_bounds: &'a BTreeMap, - extend_gaps_to_window: bool, - force: bool, - decoder_fingerprint: String, - capture_snapshots: &'a BTreeMap, - revision_pool: &'a rayon::ThreadPool, +#[derive(Clone, Copy, Debug)] +struct NfcapdTreeWindow { + start: i64, + end: i64, } -/// Resolve the physical files needed by a decode batch before making any job decisions. -/// SQLite access stays on the pipeline thread; only exact hashes run in parallel. -fn resolve_nfcapd_batch_revisions( - context: &NfcapdRevisionContext<'_>, - batch_starts: &[i64], -) -> Result, PipelineError> { - let mut paths = BTreeSet::new(); - for &bucket_start in batch_starts { - for source in context.sources { - if !source_has_candidate( - source, - bucket_start, - context.by_member_and_start, - context.member_bounds, - context.extend_gaps_to_window, - ) { - continue; - } - paths.extend(source.members.iter().filter_map(|member| { - context - .by_member_and_start - .get(&(member.clone(), bucket_start)) - .cloned() - })); - } - } - - let probes = paths - .into_iter() - .map(|path| { - let locator = path.to_string_lossy().into_owned(); - let observed = context - .capture_snapshots - .get(&path) - .cloned() - .map(Ok) - .unwrap_or_else(|| capture_nfcapd_snapshot(&path))?; - let cached_fingerprint = if context.force { - None - } else { - cached_content_fingerprint( - context.connection, - InputKind::Nfcapd, - &locator, - &observed, - )? - }; - Ok::<_, PipelineError>(NfcapdRevisionProbe { - path, - observed, - cached_content_fingerprint: cached_fingerprint, - }) - }) - .collect::, _>>()?; - let decoder_fingerprint = context.decoder_fingerprint.clone(); - - let resolved = context.revision_pool.install(|| { - probes - .par_iter() - .map(|probe| { - let captured = match &probe.cached_content_fingerprint { - Some(content_fingerprint) => { - Ok((content_fingerprint.clone(), probe.observed.clone())) - } - None => capture_file_revision_with_snapshot(&probe.path, &probe.observed), - }; - captured - .map_err(PipelineError::from) - .and_then(|(content_fingerprint, snapshot)| { - let revision = InputRevision::create( - "nfcapd", - probe.path.to_string_lossy().into_owned(), - content_fingerprint, - &decoder_fingerprint, - )?; - Ok(PreparedRevision { - revision, - snapshot: Some(snapshot), - }) - }) - }) - .collect::>() - }); - - probes +/// Resolve the selected and requested nfcapd window using the same date, timezone, and alignment +/// rules for preflight, single-output processing, and coordinated planning. +fn resolve_nfcapd_tree_window( + start_date: &str, + end_date: Option<&str>, + start_time: Option<&str>, + end_time: Option<&str>, + discovered_bucket_starts: impl IntoIterator, + timezone: &str, +) -> Result { + let selected_start = parse_date_start(start_date, timezone)?; + let explicit_end = end_date + .map(|date| next_date_start(date, timezone)) + .transpose()?; + let explicit_start_time = start_time + .map(|value| parse_local_datetime(value, timezone)) + .transpose()?; + let explicit_end_time = end_time + .map(|value| parse_local_datetime(value, timezone)) + .transpose()?; + let discovered_end = discovered_bucket_starts .into_iter() - .zip(resolved) - .map(|(probe, result)| result.map(|revision| (probe.path, revision))) - .collect::, _>>() + .max() + .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) + .transpose()? + .map(|(_, end)| end) + .unwrap_or(selected_start); + let selected_end = explicit_end.unwrap_or(discovered_end); + let start = explicit_start_time.unwrap_or(selected_start); + let end = explicit_end_time.unwrap_or(selected_end); + validate_window(selected_start, selected_end, start, end, timezone)?; + Ok(NfcapdTreeWindow { start, end }) } -#[allow(clippy::too_many_arguments)] -fn prepare_nfcapd_tree_timestamp( - connection: &Connection, - root: &Path, - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - bucket_start: i64, - extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, - revisions: &BTreeMap, - canonical_day_verified: bool, -) -> Result { - prepare_nfcapd_tree_timestamp_with_cache( - connection, - root, - sources, - by_member_and_start, - member_bounds, - bucket_start, - extend_gaps_to_window, - force, - pipeline, - report, - revisions, - canonical_day_verified, - None, - ) +fn parse_date_start(raw: &str, timezone: &str) -> Result { + let date: Date = raw + .parse() + .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; + Ok(date + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second()) } -#[allow(clippy::too_many_arguments)] -fn prepare_nfcapd_tree_timestamp_with_cache( - connection: &Connection, - root: &Path, - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - bucket_start: i64, - extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, - revisions: &BTreeMap, - canonical_day_verified: bool, - resume_cache: Option<&NfcapdDayResumeCache>, -) -> Result { - #[cfg(test)] - PREPARE_NFCAPD_TREE_TIMESTAMP_CALLS.with(|calls| calls.set(calls.get() + 1)); - - let mut revision_cache: BTreeMap = BTreeMap::new(); - let mut jobs = Vec::new(); - for source in sources { - if !source_has_candidate( - source, - bucket_start, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - ) { - continue; - } - let present = source - .members - .iter() - .filter_map(|member| { - by_member_and_start - .get(&(member.clone(), bucket_start)) - .map(|path| (member.clone(), path.clone())) - }) - .collect::>(); - let mut owners = Vec::new(); - for (member, path) in &present { - let owner = match revision_cache.get(member) { - Some(owner) => owner.clone(), - None => { - let owner = revisions - .get(path) - .cloned() - .expect("present member has a resolved revision"); - revision_cache.insert(member.clone(), owner.clone()); - owner - } - }; - owners.push(owner); - } - let mut absences = Vec::new(); - let mut evidence = Vec::with_capacity(source.members.len()); - for ((member, path), owner) in present.iter().zip(&owners) { - evidence.push(InputEvidenceRow::new( - &source.source_id, - member, - bucket_start, - bucket_start + FIVE_MINUTES, - path.to_string_lossy(), - InputEvidenceState::Observed, - Some(owner.revision.fingerprint.clone()), - )); - } - for member in &source.members { - if !present.iter().any(|(present, _)| present == member) { - let expected = - expected_nfcapd_path(root, member, bucket_start, &pipeline.timezone)?; - absences.push(ExpectedAbsence::capture(&expected)?); - evidence.push(InputEvidenceRow::new( - &source.source_id, - member, - bucket_start, - bucket_start + FIVE_MINUTES, - expected.to_string_lossy(), - InputEvidenceState::Missing, - None, - )); - } - } - evidence.sort_unstable_by(|left, right| left.unit_id.cmp(&right.unit_id)); - let previous_evidence = match resume_cache { - Some(cache) => Cow::Borrowed(cache.evidence(&source.source_id, bucket_start)), - None => Cow::Owned(query_input_evidence( - connection, - &source.source_id, - bucket_start, - )?), - }; - let observed_input_disappeared = previous_evidence.iter().any(|previous| { - previous.evidence_state == InputEvidenceState::Observed - && evidence.iter().any(|current| { - current.unit_id == previous.unit_id - && current.evidence_state == InputEvidenceState::Missing - }) - }); - if observed_input_disappeared { - tracing::warn!( - source_id = source.source_id, - bucket_start, - "preserving prior bucket because an observed input is now missing" - ); - report.skipped_inputs += 1; - continue; - } - let revisions = owners - .iter() - .map(|owner| owner.revision.clone()) - .collect::>(); - let persisted_processed = if force || revisions.is_empty() { - false - } else { - match resume_cache { - Some(cache) => cache.processed(&source.source_id, bucket_start, &revisions)?, - None => nfcapd_logical_bucket_processed( - connection, - &source.source_id, - bucket_start, - &revisions, - )?, - } - }; - let mut topology_corruption = false; - if !force && previous_evidence == evidence { - let provenance_complete = revisions.is_empty() || persisted_processed; - if provenance_complete { - let topology_matches = canonical_day_verified - || nfcapd_logical_bucket_has_canonical_topology( - connection, - &source.source_id, - bucket_start, - present.len(), - source.members.len(), - pipeline.run_maad, - )?; - if topology_matches { - report.skipped_inputs += 1; - continue; - } - topology_corruption = true; - } - } - let orphaned_provenance = !force - && previous_evidence.is_empty() - && persisted_processed - && (canonical_day_verified - || nfcapd_logical_bucket_has_canonical_topology( - connection, - &source.source_id, - bucket_start, - present.len(), - source.members.len(), - pipeline.run_maad, - )?); - let is_repair = !force - && (orphaned_provenance - || (!previous_evidence.is_empty() - && (previous_evidence != evidence || topology_corruption))); - jobs.push(PreparedTreeJob { - source_id: source.source_id.clone(), - expected_units: source.members.len(), - present, - owners, - absences, - evidence, - is_repair, - }); - } - Ok(PreparedTreeTimestamp { - bucket_start, - revision_cache, - jobs, - }) -} - -#[allow(clippy::too_many_arguments)] -enum PreparedExplicitNfcapdKind { - File(PreparedRevision), - Gap { expected_path: Option }, -} - -struct PreparedExplicitNfcapd { - path: PathBuf, - source_id: String, - bucket_start: i64, - kind: PreparedExplicitNfcapdKind, -} - -fn process_explicit_nfcapd_inputs( - connection: &Connection, - inputs: &[InputSpec], - pipeline: &ResolvedPipeline, -) -> Result { - let mut prepared = Vec::new(); - for input in inputs { - let InputSpec::Nfcapd { - path, - source_id, - bucket_start, - gap, - expected_path, - } = input - else { - continue; - }; - let bucket_start = match bucket_start { - Some(start) => *start, - None if !gap => ingest::parse_nfcapd_bucket_start(path, &pipeline.timezone)?, - None => { - return Err(PipelineError::InvalidConfig( - "explicit nfcapd gap requires bucket_start".into(), - )); - } - }; - let kind = if *gap { - PreparedExplicitNfcapdKind::Gap { - expected_path: expected_path.clone(), - } - } else { - let (revision, snapshot) = prepare_file_revision( - connection, - path, - InputKind::Nfcapd, - nfdump_decoder_fingerprint_for_pipeline(pipeline)?, - )?; - PreparedExplicitNfcapdKind::File(PreparedRevision { - revision, - snapshot: Some(snapshot), - }) - }; - prepared.push(PreparedExplicitNfcapd { - path: path.clone(), - source_id: source_id.clone(), - bucket_start, - kind, - }); - } - prepared.sort_unstable_by(|left, right| { - (left.bucket_start, &left.source_id, &left.path).cmp(&( - right.bucket_start, - &right.source_id, - &right.path, - )) - }); - if prepared.is_empty() { - return Ok(PipelineReport::default()); - } - process_atomic(connection, pipeline, |aggregates, report| { - for input in &prepared { - match &input.kind { - PreparedExplicitNfcapdKind::File(owner) => process_nfcapd( - connection, - &input.path, - &input.source_id, - input.bucket_start, - owner, - pipeline, - aggregates, - report, - )?, - PreparedExplicitNfcapdKind::Gap { expected_path } => process_nfcapd_gap( - connection, - &input.path, - expected_path.as_deref(), - &input.source_id, - input.bucket_start, - pipeline, - aggregates, - report, - )?, - } - } - Ok(()) - }) -} - -#[allow(clippy::too_many_arguments)] -fn process_nfcapd( - connection: &Connection, - path: &Path, - source_id: &str, - bucket_start: i64, - owner: &PreparedRevision, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - if nfcapd_logical_bucket_processed( - connection, - source_id, - bucket_start, - std::slice::from_ref(&owner.revision), - )? { - report.skipped_inputs += 1; - return Ok(()); - } - verify_nfdump_revision(pipeline)?; - let bucket = ingest::read_nfcapd_bucket( - path, - source_id, - &pipeline.selection, - &pipeline.nfdump, - &pipeline.timezone, - ) - .map_err(|error| nfcapd_decode_error(source_id, bucket_start, path, error))?; - verify_nfdump_revision(pipeline)?; - let snapshot = owner - .snapshot - .as_ref() - .expect("explicit file input has a snapshot"); - verify_file_snapshot(path, snapshot) - .map_err(|error| nfcapd_decode_error(source_id, bucket_start, path, error))?; - aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; - publish_nfcapd_bucket( - connection, - &bucket, - std::slice::from_ref(owner), - &[], - &[InputEvidenceRow::new( - source_id, - source_id, - bucket_start, - bucket_start + FIVE_MINUTES, - &owner.revision.locator, - InputEvidenceState::Observed, - Some(owner.revision.fingerprint.clone()), - )], - false, - false, - pipeline.run_maad, - )?; - aggregates.include(&bucket, &pipeline.timezone)?; - report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; - report.five_minute_buckets += 1; - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_gap( - connection: &Connection, - _locator_path: &Path, - expected_path: Option<&Path>, - source_id: &str, - bucket_start: i64, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - let expected_path = expected_path.ok_or_else(|| { - PipelineError::InvalidConfig( - "explicit nfcapd gap requires expected_path for absence verification".into(), - ) - })?; - let absence = ExpectedAbsence::capture(expected_path)?; - let evidence = [InputEvidenceRow::new( - source_id, - source_id, - bucket_start, - bucket_start + FIVE_MINUTES, - expected_path.to_string_lossy(), - InputEvidenceState::Missing, - None, - )]; - if query_input_evidence(connection, source_id, bucket_start)? == evidence { - report.skipped_inputs += 1; - return Ok(()); - } - let bucket = StatisticalBucket::new(BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - )) - .with_coverage(BucketCoverage::new(1, 0, 0).map_err(DomainError::from)?) - .finish_owned(); - aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; - publish_nfcapd_bucket( - connection, - &bucket, - &[], - &[absence], - &evidence, - false, - false, - pipeline.run_maad, - )?; - aggregates.include(&bucket, &pipeline.timezone)?; - report.rollup_buckets += aggregates.flush_complete(connection, pipeline.run_maad)?; - report.five_minute_buckets += 1; - Ok(()) -} - -fn normalize_sources( - root: &Path, - source_ids: &[String], - sources: &[DatasetSource], -) -> Result, PipelineError> { - if !source_ids.is_empty() && !sources.is_empty() { - return Err(PipelineError::InvalidConfig( - "nfcapd_tree cannot define both source_ids and sources".into(), - )); - } - let mut normalized = if !sources.is_empty() { - sources.to_vec() - } else if !source_ids.is_empty() { - source_ids - .iter() - .map(|source_id| DatasetSource { - source_id: source_id.clone(), - members: vec![source_id.clone()], - }) - .collect() - } else { - let entries = fs::read_dir(root)?; - entries - .filter_map(Result::ok) - .filter_map(|entry| { - entry.file_type().ok()?.is_dir().then(|| { - let source_id = entry.file_name().to_string_lossy().into_owned(); - DatasetSource { - source_id: source_id.clone(), - members: vec![source_id], - } - }) - }) - .collect() - }; - normalized.sort_unstable_by(|left, right| left.source_id.cmp(&right.source_id)); - let mut ids = BTreeSet::new(); - for source in &normalized { - if !is_safe_path_component(&source.source_id) - || source.members.is_empty() - || !ids.insert(source.source_id.clone()) - { - return Err(PipelineError::InvalidConfig( - "logical sources require unique non-empty IDs and members".into(), - )); - } - let mut members = BTreeSet::new(); - for member in &source.members { - if !is_safe_path_component(member) || !members.insert(member) { - return Err(PipelineError::InvalidConfig(format!( - "source {:?} has an unsafe or duplicate member path component", - source.source_id - ))); - } - if !root.join(member).is_dir() { - return Err(PipelineError::InvalidConfig(format!( - "source {:?} references missing member directory {:?}", - source.source_id, member - ))); - } - } - } - - let member_ids = normalized - .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>(); - let mut member_paths = BTreeMap::::new(); - #[cfg(unix)] - let mut member_identities = BTreeMap::<(u64, u64), (String, PathBuf)>::new(); - for member in member_ids { - let member_path = root.join(&member); - let canonical_member_path = canonical_path(&member_path)?; - if let Some(previous_member) = member_paths.get(&canonical_member_path) { - return Err(PipelineError::InvalidConfig(format!( - "nfcapd_tree member IDs {:?} and {:?} resolve to the same directory {}", - previous_member, - member, - canonical_member_path.display() - ))); - } - member_paths.insert(canonical_member_path.clone(), member.clone()); - - #[cfg(unix)] - { - let identity = existing_path_identity(&member_path)?.ok_or_else(|| { - PipelineError::InvalidConfig(format!( - "source member directory {:?} disappeared during normalization", - member - )) - })?; - if let Some((previous_member, previous_path)) = member_identities.get(&identity) { - return Err(PipelineError::InvalidConfig(format!( - "nfcapd_tree member IDs {:?} and {:?} resolve to the same physical directory via device/inode {:?} ({} and {})", - previous_member, - member, - identity, - previous_path.display(), - canonical_member_path.display() - ))); - } - member_identities.insert(identity, (member, canonical_member_path)); - } - } - Ok(normalized) -} - -fn merge_source_bucket( - source_id: &str, - bucket_start: i64, - expected_units: usize, - members: &[&CanonicalBucket], -) -> Result { - let key = BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - ); - let mut builder = if members.is_empty() { - StatisticalBucket::new(key) - } else { - StatisticalBucket::dense(key) - } - .with_coverage(BucketCoverage::empty()); - for member in members { - builder.include(member)?; - } - let coverage = BucketCoverage::new( - u64::try_from(expected_units).unwrap_or(u64::MAX), - u64::try_from(members.len()).unwrap_or(u64::MAX), - 0, - ) - .map_err(DomainError::from)?; - Ok(builder.with_coverage(coverage).finish_owned()) -} - -fn logical_source_bucket<'a>( - source_id: &str, - bucket_start: i64, - expected_units: usize, - members: &[&'a CanonicalBucket], -) -> Result, PipelineError> { - let expected_key = BucketKey::new( - source_id, - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - ); - if expected_units == 1 - && let [member] = members - && member.key == expected_key - { - return Ok(Cow::Borrowed(member)); - } - Ok(Cow::Owned(merge_source_bucket( - source_id, - bucket_start, - expected_units, - members, - )?)) -} - -#[derive(Debug, Default)] -struct NfcapdBucketPublishProfile { - total_elapsed: Duration, - preflight_elapsed: Duration, - overlap_elapsed: Duration, - force_delete_elapsed: Duration, - owner_upsert_elapsed: Duration, - write: WriteBucketsProfile, - owner_status_elapsed: Duration, - postflight_elapsed: Duration, - owners: u64, - absences: u64, -} - -impl NfcapdBucketPublishProfile { - fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.preflight_elapsed += profile.preflight_elapsed; - self.overlap_elapsed += profile.overlap_elapsed; - self.force_delete_elapsed += profile.force_delete_elapsed; - self.owner_upsert_elapsed += profile.owner_upsert_elapsed; - self.write.include(profile.write); - self.owner_status_elapsed += profile.owner_status_elapsed; - self.postflight_elapsed += profile.postflight_elapsed; - self.owners += profile.owners; - self.absences += profile.absences; - } - - fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.preflight_elapsed - + self.overlap_elapsed - + self.force_delete_elapsed - + self.owner_upsert_elapsed - + self.write.total_elapsed - + self.owner_status_elapsed - + self.postflight_elapsed, - ) - } -} - -#[derive(Debug, Default)] -struct FinalRollupProfile { - total_elapsed: Duration, - finish_elapsed: Duration, - delete_elapsed: Duration, - write: WriteBucketsProfile, - incomplete_keys: u64, - rollup_buckets: u64, -} - -impl FinalRollupProfile { - fn other_elapsed(&self) -> Duration { - self.total_elapsed - .saturating_sub(self.finish_elapsed + self.delete_elapsed + self.write.total_elapsed) - } -} - -#[derive(Debug, Default)] -struct AggregateGranularityProfile { - total_elapsed: Duration, - bounds_elapsed: Duration, - builder_elapsed: Duration, - bucket: StatisticalBucketIncludeProfile, -} - -impl AggregateGranularityProfile { - fn include( - &mut self, - total_elapsed: Duration, - bounds_elapsed: Duration, - builder_elapsed: Duration, - bucket: StatisticalBucketIncludeProfile, - ) { - self.total_elapsed += total_elapsed; - self.bounds_elapsed += bounds_elapsed; - self.builder_elapsed += builder_elapsed; - self.bucket.include(bucket); - } - - fn other_elapsed(&self) -> Duration { - self.total_elapsed - .saturating_sub(self.bounds_elapsed + self.builder_elapsed + self.bucket.total_elapsed) - } -} - -#[derive(Debug, Default)] -struct AggregateIncludeProfile { - total_elapsed: Duration, - thirty_minutes: AggregateGranularityProfile, - one_hour: AggregateGranularityProfile, - one_day: AggregateGranularityProfile, -} - -impl AggregateIncludeProfile { - fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.thirty_minutes.include( - profile.thirty_minutes.total_elapsed, - profile.thirty_minutes.bounds_elapsed, - profile.thirty_minutes.builder_elapsed, - profile.thirty_minutes.bucket, - ); - self.one_hour.include( - profile.one_hour.total_elapsed, - profile.one_hour.bounds_elapsed, - profile.one_hour.builder_elapsed, - profile.one_hour.bucket, - ); - self.one_day.include( - profile.one_day.total_elapsed, - profile.one_day.bounds_elapsed, - profile.one_day.builder_elapsed, - profile.one_day.bucket, - ); - } - - fn granularity_mut(&mut self, granularity: Granularity) -> &mut AggregateGranularityProfile { - match granularity { - Granularity::ThirtyMinutes => &mut self.thirty_minutes, - Granularity::OneHour => &mut self.one_hour, - Granularity::OneDay => &mut self.one_day, - Granularity::FiveMinutes => unreachable!("five-minute buckets are not rollups"), - } - } - - fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.thirty_minutes.total_elapsed - + self.one_hour.total_elapsed - + self.one_day.total_elapsed, - ) - } -} - -#[derive(Debug, Default)] -struct NfcapdDayPublishProfile { - day_elapsed: Duration, - prepare_elapsed: Duration, - decode_elapsed: Duration, - batch_publish_elapsed: Duration, - logical_source_elapsed: Duration, - persisted_sibling_elapsed: Duration, - bucket_publish: NfcapdBucketPublishProfile, - aggregate_include: AggregateIncludeProfile, - completed_rollup_flush_elapsed: Duration, - completed_rollup_write: WriteBucketsProfile, - final_rollups: FinalRollupProfile, - logical_buckets: u64, - completed_rollup_flushes: u64, - nonempty_rollup_flushes: u64, - active_set_count: u64, -} - -impl NfcapdDayPublishProfile { - fn log(&self, day_start: i64, day_end: i64, transaction_elapsed: Duration) { - self.log_with_context( - day_start, - day_end, - transaction_elapsed, - "single", - None, - None, - ); - } - - fn log_coordinated( - &self, - day_start: i64, - day_end: i64, - transaction_elapsed: Duration, - output_index: usize, - output_path: &Path, - ) { - self.log_with_context( - day_start, - day_end, - transaction_elapsed, - "coordinated", - Some(output_index), - Some(output_path), - ); - } - - fn log_with_context( - &self, - day_start: i64, - day_end: i64, - transaction_elapsed: Duration, - mode: &'static str, - output_index: Option, - output_path: Option<&Path>, - ) { - let mut rollup_write = self.completed_rollup_write.clone(); - rollup_write.include(self.final_rollups.write.clone()); - let publish_other = self.batch_publish_elapsed.saturating_sub( - self.logical_source_elapsed - + self.persisted_sibling_elapsed - + self.bucket_publish.total_elapsed - + self.aggregate_include.total_elapsed - + self.completed_rollup_flush_elapsed, - ); - let transaction_other = - transaction_elapsed.saturating_sub(self.day_elapsed + self.final_rollups.total_elapsed); - let completed_rollup_housekeeping = self - .completed_rollup_flush_elapsed - .saturating_sub(self.completed_rollup_write.total_elapsed); - tracing::info!( - target: "netflow_db::profile", - phase = "nfcapd_tree_day_publish_detail", - mode, - output_index = ?output_index, - output_path = ?output_path, - day_start, - day_end, - transaction_seconds = transaction_elapsed.as_secs_f64(), - transaction_other_seconds = transaction_other.as_secs_f64(), - day_seconds = self.day_elapsed.as_secs_f64(), - prepare_seconds = self.prepare_elapsed.as_secs_f64(), - decode_seconds = self.decode_elapsed.as_secs_f64(), - batch_publish_seconds = self.batch_publish_elapsed.as_secs_f64(), - publish_other_seconds = publish_other.as_secs_f64(), - logical_source_seconds = self.logical_source_elapsed.as_secs_f64(), - persisted_sibling_seconds = self.persisted_sibling_elapsed.as_secs_f64(), - bucket_publish_seconds = self.bucket_publish.total_elapsed.as_secs_f64(), - bucket_preflight_seconds = self.bucket_publish.preflight_elapsed.as_secs_f64(), - bucket_overlap_seconds = self.bucket_publish.overlap_elapsed.as_secs_f64(), - bucket_force_delete_seconds = self.bucket_publish.force_delete_elapsed.as_secs_f64(), - owner_upsert_seconds = self.bucket_publish.owner_upsert_elapsed.as_secs_f64(), - owner_status_seconds = self.bucket_publish.owner_status_elapsed.as_secs_f64(), - bucket_postflight_seconds = self.bucket_publish.postflight_elapsed.as_secs_f64(), - bucket_other_seconds = self.bucket_publish.other_elapsed().as_secs_f64(), - aggregate_include_seconds = self.aggregate_include.total_elapsed.as_secs_f64(), - aggregate_include_other_seconds = self.aggregate_include.other_elapsed().as_secs_f64(), - aggregate_30m_seconds = self.aggregate_include.thirty_minutes.total_elapsed.as_secs_f64(), - aggregate_30m_bounds_seconds = self.aggregate_include.thirty_minutes.bounds_elapsed.as_secs_f64(), - aggregate_30m_builder_seconds = self.aggregate_include.thirty_minutes.builder_elapsed.as_secs_f64(), - aggregate_30m_traffic_seconds = self.aggregate_include.thirty_minutes.bucket.traffic_elapsed.as_secs_f64(), - aggregate_30m_protocols_seconds = self.aggregate_include.thirty_minutes.bucket.protocols_elapsed.as_secs_f64(), - aggregate_30m_addresses_seconds = self.aggregate_include.thirty_minutes.bucket.addresses_elapsed.as_secs_f64(), - aggregate_30m_ports_seconds = self.aggregate_include.thirty_minutes.bucket.ports_elapsed.as_secs_f64(), - aggregate_30m_coverage_seconds = self.aggregate_include.thirty_minutes.bucket.coverage_elapsed.as_secs_f64(), - aggregate_30m_bucket_other_seconds = self.aggregate_include.thirty_minutes.bucket.other_elapsed().as_secs_f64(), - aggregate_30m_other_seconds = self.aggregate_include.thirty_minutes.other_elapsed().as_secs_f64(), - aggregate_1h_seconds = self.aggregate_include.one_hour.total_elapsed.as_secs_f64(), - aggregate_1h_bounds_seconds = self.aggregate_include.one_hour.bounds_elapsed.as_secs_f64(), - aggregate_1h_builder_seconds = self.aggregate_include.one_hour.builder_elapsed.as_secs_f64(), - aggregate_1h_traffic_seconds = self.aggregate_include.one_hour.bucket.traffic_elapsed.as_secs_f64(), - aggregate_1h_protocols_seconds = self.aggregate_include.one_hour.bucket.protocols_elapsed.as_secs_f64(), - aggregate_1h_addresses_seconds = self.aggregate_include.one_hour.bucket.addresses_elapsed.as_secs_f64(), - aggregate_1h_ports_seconds = self.aggregate_include.one_hour.bucket.ports_elapsed.as_secs_f64(), - aggregate_1h_coverage_seconds = self.aggregate_include.one_hour.bucket.coverage_elapsed.as_secs_f64(), - aggregate_1h_bucket_other_seconds = self.aggregate_include.one_hour.bucket.other_elapsed().as_secs_f64(), - aggregate_1h_other_seconds = self.aggregate_include.one_hour.other_elapsed().as_secs_f64(), - aggregate_1d_seconds = self.aggregate_include.one_day.total_elapsed.as_secs_f64(), - aggregate_1d_bounds_seconds = self.aggregate_include.one_day.bounds_elapsed.as_secs_f64(), - aggregate_1d_builder_seconds = self.aggregate_include.one_day.builder_elapsed.as_secs_f64(), - aggregate_1d_traffic_seconds = self.aggregate_include.one_day.bucket.traffic_elapsed.as_secs_f64(), - aggregate_1d_protocols_seconds = self.aggregate_include.one_day.bucket.protocols_elapsed.as_secs_f64(), - aggregate_1d_addresses_seconds = self.aggregate_include.one_day.bucket.addresses_elapsed.as_secs_f64(), - aggregate_1d_ports_seconds = self.aggregate_include.one_day.bucket.ports_elapsed.as_secs_f64(), - aggregate_1d_coverage_seconds = self.aggregate_include.one_day.bucket.coverage_elapsed.as_secs_f64(), - aggregate_1d_bucket_other_seconds = self.aggregate_include.one_day.bucket.other_elapsed().as_secs_f64(), - aggregate_1d_other_seconds = self.aggregate_include.one_day.other_elapsed().as_secs_f64(), - completed_rollup_flush_seconds = self.completed_rollup_flush_elapsed.as_secs_f64(), - completed_rollup_housekeeping_seconds = completed_rollup_housekeeping.as_secs_f64(), - final_rollup_seconds = self.final_rollups.total_elapsed.as_secs_f64(), - final_rollup_finish_seconds = self.final_rollups.finish_elapsed.as_secs_f64(), - final_rollup_delete_seconds = self.final_rollups.delete_elapsed.as_secs_f64(), - final_rollup_other_seconds = self.final_rollups.other_elapsed().as_secs_f64(), - five_minute_write_seconds = self.bucket_publish.write.total_elapsed.as_secs_f64(), - five_minute_delete_seconds = self.bucket_publish.write.delete_elapsed.as_secs_f64(), - five_minute_canonical_rows_seconds = self.bucket_publish.write.canonical_rows_elapsed.as_secs_f64(), - five_minute_scalar_rows_seconds = self.bucket_publish.write.scalar_rows_elapsed.as_secs_f64(), - five_minute_scalar_insert_seconds = scalar_insert_elapsed(&self.bucket_publish.write).as_secs_f64(), - five_minute_maad_seconds = self.bucket_publish.write.maad_elapsed.as_secs_f64(), - five_minute_address_structure_insert_seconds = self.bucket_publish.write.address_structure_insert_elapsed.as_secs_f64(), - five_minute_write_other_seconds = self.bucket_publish.write.other_elapsed().as_secs_f64(), - rollup_write_seconds = rollup_write.total_elapsed.as_secs_f64(), - rollup_delete_seconds = rollup_write.delete_elapsed.as_secs_f64(), - rollup_canonical_rows_seconds = rollup_write.canonical_rows_elapsed.as_secs_f64(), - rollup_scalar_rows_seconds = rollup_write.scalar_rows_elapsed.as_secs_f64(), - rollup_scalar_insert_seconds = scalar_insert_elapsed(&rollup_write).as_secs_f64(), - rollup_maad_seconds = rollup_write.maad_elapsed.as_secs_f64(), - rollup_address_structure_insert_seconds = rollup_write.address_structure_insert_elapsed.as_secs_f64(), - rollup_write_other_seconds = rollup_write.other_elapsed().as_secs_f64(), - logical_buckets = self.logical_buckets, - owners = self.bucket_publish.owners, - absences = self.bucket_publish.absences, - completed_rollup_flushes = self.completed_rollup_flushes, - nonempty_rollup_flushes = self.nonempty_rollup_flushes, - active_set_count = self.active_set_count, - final_incomplete_keys = self.final_rollups.incomplete_keys, - final_rollup_buckets = self.final_rollups.rollup_buckets, - five_minute_write_calls = self.bucket_publish.write.write_calls, - rollup_write_calls = rollup_write.write_calls, - five_minute_bucket_keys = self.bucket_publish.write.bucket_keys, - rollup_bucket_keys = rollup_write.bucket_keys, - traffic_rows = self.bucket_publish.write.traffic_rows + rollup_write.traffic_rows, - protocol_rows = self.bucket_publish.write.protocol_rows + rollup_write.protocol_rows, - address_count_rows = self.bucket_publish.write.address_count_rows + rollup_write.address_count_rows, - port_count_rows = self.bucket_publish.write.port_count_rows + rollup_write.port_count_rows, - address_structure_rows = self.bucket_publish.write.address_structure_rows + rollup_write.address_structure_rows, - maad_address_sets = self.bucket_publish.write.maad_address_sets + rollup_write.maad_address_sets, - maad_addresses = self.bucket_publish.write.maad_addresses + rollup_write.maad_addresses, - address_structure_json_bytes = self.bucket_publish.write.address_structure_json_bytes + rollup_write.address_structure_json_bytes, - ); - } -} - -fn scalar_insert_elapsed(profile: &WriteBucketsProfile) -> Duration { - profile.traffic_insert_elapsed - + profile.protocol_insert_elapsed - + profile.address_count_insert_elapsed - + profile.port_count_insert_elapsed -} - -fn profile_count(value: usize) -> u64 { - u64::try_from(value).unwrap_or(u64::MAX) -} - -#[derive(Clone, Debug)] -struct PreparedRevision { - revision: InputRevision, - snapshot: Option, -} - -struct PreparedTreeJob { - source_id: String, - expected_units: usize, - present: Vec<(String, PathBuf)>, - owners: Vec, - absences: Vec, - evidence: Vec, - is_repair: bool, -} - -struct PreparedTreeTimestamp { - bucket_start: i64, - revision_cache: BTreeMap, - jobs: Vec, -} - -#[allow(clippy::too_many_arguments)] -fn publish_nfcapd_bucket( - connection: &Connection, - bucket: &CanonicalBucket, - owners: &[PreparedRevision], - absences: &[ExpectedAbsence], - evidence: &[InputEvidenceRow], - allow_coverage_repair: bool, - force: bool, - run_maad: bool, -) -> Result<(), PipelineError> { - publish_nfcapd_bucket_profiled( - connection, - bucket, - owners, - absences, - evidence, - allow_coverage_repair, - force, - run_maad, - ) - .map(|_| ()) -} - -#[allow(clippy::too_many_arguments)] -fn publish_nfcapd_bucket_profiled( - connection: &Connection, - bucket: &CanonicalBucket, - owners: &[PreparedRevision], - absences: &[ExpectedAbsence], - evidence: &[InputEvidenceRow], - allow_coverage_repair: bool, - force: bool, - run_maad: bool, -) -> Result { - let total_started = Instant::now(); - let mut profile = NfcapdBucketPublishProfile { - owners: profile_count(owners.len()), - absences: profile_count(absences.len()), - ..NfcapdBucketPublishProfile::default() - }; - let preflight_started = Instant::now(); - for absence in absences { - absence.verify()?; - } - for owner in owners { - if let Some(snapshot) = &owner.snapshot { - verify_file_snapshot(&owner.revision.locator, snapshot)?; - } - } - profile.preflight_elapsed += preflight_started.elapsed(); - let overlap_started = Instant::now(); - reject_overlapping_bucket( - connection, - bucket, - InputKind::Nfcapd, - "", - force || allow_coverage_repair, - )?; - profile.overlap_elapsed += overlap_started.elapsed(); - if force { - let force_delete_started = Instant::now(); - connection.execute( - "DELETE FROM processed_inputs WHERE input_kind = 'nfcapd' AND source_id = ?1 AND bucket_start = ?2", - params![bucket.key.source_id, bucket.key.bucket_start], - ).map_err(StorageError::from)?; - profile.force_delete_elapsed += force_delete_started.elapsed(); - } - let publication = (|| -> Result<(), PipelineError> { - let owner_upsert_started = Instant::now(); - for prepared in owners { - let revision = &prepared.revision; - let owner = InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: revision.locator.clone(), - scan_locator: revision.locator.clone(), - source_id: bucket.key.source_id.clone(), - bucket_start: bucket.key.bucket_start, - bucket_end: bucket.key.bucket_end, - revision: revision.clone(), - file_snapshot: prepared.snapshot.clone(), - }; - upsert_input_bucket(connection, &owner, force)?; - } - profile.owner_upsert_elapsed += owner_upsert_started.elapsed(); - profile.write = write_buckets_profiled(connection, std::slice::from_ref(bucket), run_maad)?; - replace_input_evidence( - connection, - &bucket.key.source_id, - bucket.key.bucket_start, - evidence, - )?; - let owner_status_started = Instant::now(); - for prepared in owners { - let revision = &prepared.revision; - mark_input_bucket_status( - connection, - InputKind::Nfcapd, - &revision.locator, - &bucket.key.source_id, - bucket.key.bucket_start, - InputStatus::Processed, - revision, - None, - )?; - } - profile.owner_status_elapsed += owner_status_started.elapsed(); - let postflight_started = Instant::now(); - for absence in absences { - absence.verify()?; - } - profile.postflight_elapsed += postflight_started.elapsed(); - Ok(()) - })(); - publication?; - profile.total_elapsed = total_started.elapsed(); - Ok(profile) -} - -fn reject_overlapping_bucket( - connection: &Connection, - bucket: &CanonicalBucket, - input_kind: InputKind, - allowed_scan: &str, - replace_nfcapd: bool, -) -> Result<(), PipelineError> { - let conflict = connection - .query_row( - "SELECT input_kind, input_locator, scan_locator FROM processed_inputs - WHERE source_id = ?1 AND bucket_start = ?2 - AND NOT (input_kind = ?3 AND scan_locator = ?4) - ORDER BY input_kind, input_locator LIMIT 1", - params![ - bucket.key.source_id, - bucket.key.bucket_start, - input_kind.as_str(), - allowed_scan, - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .optional() - .map_err(StorageError::from)?; - if let Some((kind, locator, _)) = conflict - && !(replace_nfcapd && kind == InputKind::Nfcapd.as_str()) - { - return Err(PipelineError::InvalidConfig(format!( - "overlapping canonical five-minute input for source {:?} at {} conflicts with {kind}:{locator}", - bucket.key.source_id, bucket.key.bucket_start - ))); - } - Ok(()) -} - -#[derive(Default)] -struct AggregateBuckets { - builders: BTreeMap<(String, Granularity, i64, i64), StatisticalBucket>, - published_through: BTreeMap, - owned_keys: BTreeSet<(String, i64)>, - current_run_keys: BTreeSet<(String, i64)>, - persisted_sibling_validations: BTreeSet<(String, i64, bool)>, - #[cfg(test)] - persisted_sibling_queries: usize, -} - -impl AggregateBuckets { - fn with_owned_keys(owned_keys: BTreeSet<(String, i64)>) -> Self { - Self { - owned_keys, - ..Self::default() - } - } - - fn reject_persisted_siblings( - &mut self, - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, - ) -> Result<(), PipelineError> { - self.reject_persisted_siblings_inner(connection, child, timezone, false) - } - - fn reject_persisted_csv_siblings( - &mut self, - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, - ) -> Result<(), PipelineError> { - self.reject_persisted_siblings_inner(connection, child, timezone, true) - } - - fn reject_persisted_siblings_inner( - &mut self, - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, - allow_staged_csv_keys: bool, - ) -> Result<(), PipelineError> { - let (day_start, day_end) = - aggregate_bounds(child.key.bucket_start, Granularity::OneDay, timezone)?; - let validation_key = ( - child.key.source_id.clone(), - day_start, - allow_staged_csv_keys, - ); - if self.persisted_sibling_validations.contains(&validation_key) { - return Ok(()); - } - #[cfg(test)] - { - self.persisted_sibling_queries += 1; - } - let mut statement = connection - .prepare( - "SELECT DISTINCT bucket_start FROM traffic_stats - WHERE source_id = ?1 AND granularity = '5m' - AND bucket_start >= ?2 AND bucket_start < ?3 - ORDER BY bucket_start", - ) - .map_err(StorageError::from)?; - let persisted = statement - .query_map(params![child.key.source_id, day_start, day_end], |row| { - row.get::<_, i64>(0) - }) - .map_err(StorageError::from)? - .collect::>>() - .map_err(StorageError::from)?; - for bucket_start in persisted { - if bucket_start == child.key.bucket_start - || self - .owned_keys - .contains(&(child.key.source_id.clone(), bucket_start)) - || self - .current_run_keys - .contains(&(child.key.source_id.clone(), bucket_start)) - { - continue; - } - if allow_staged_csv_keys - && connection - .query_row( - "SELECT 1 FROM csv_bucket_stage - WHERE source_id = ?1 AND bucket_start = ?2 LIMIT 1", - params![child.key.source_id, bucket_start], - |_| Ok(()), - ) - .optional() - .map_err(StorageError::from)? - .is_some() - { - continue; - } - return Err(PipelineError::InvalidConfig(format!( - "cannot reopen a persisted aggregate interval exactly: source={:?} bucket_start={} shares its local day with persisted five-minute bucket {bucket_start} from another transaction", - child.key.source_id, child.key.bucket_start - ))); - } - // This validation is intentionally local to one aggregate transaction/output. Persisted - // siblings cannot change except through keys owned by this run, which are already excluded - // above, so later children in the same source/day can reuse the successful result. - self.persisted_sibling_validations.insert(validation_key); - Ok(()) - } - - fn include(&mut self, child: &CanonicalBucket, timezone: &str) -> Result<(), PipelineError> { - self.include_profiled(child, timezone).map(|_| ()) - } - - fn include_profiled( - &mut self, - child: &CanonicalBucket, - timezone: &str, - ) -> Result { - let total_started = Instant::now(); - let mut profile = AggregateIncludeProfile::default(); - if self - .published_through - .get(&child.key.source_id) - .is_some_and(|previous| child.key.bucket_start <= *previous) - { - return Err(PipelineError::InvalidConfig(format!( - "five-minute buckets must be unique and chronological for source {:?}: {} followed {}", - child.key.source_id, - self.published_through[&child.key.source_id], - child.key.bucket_start - ))); - } - for granularity in [ - Granularity::ThirtyMinutes, - Granularity::OneHour, - Granularity::OneDay, - ] { - let granularity_started = Instant::now(); - let bounds_started = Instant::now(); - let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; - let bounds_elapsed = bounds_started.elapsed(); - let key = (child.key.source_id.clone(), granularity, start, end); - let builder_started = Instant::now(); - let builder = self.builders.entry(key.clone()).or_insert_with(|| { - StatisticalBucket::new(BucketKey::new(&key.0, key.1, key.2, key.3)) - }); - let builder_elapsed = builder_started.elapsed(); - let bucket = builder.include_profiled(child)?; - profile.granularity_mut(granularity).include( - granularity_started.elapsed(), - bounds_elapsed, - builder_elapsed, - bucket, - ); - } - self.published_through - .insert(child.key.source_id.clone(), child.key.bucket_start); - self.current_run_keys - .insert((child.key.source_id.clone(), child.key.bucket_start)); - profile.total_elapsed = total_started.elapsed(); - Ok(profile) - } - - fn flush_complete( - &mut self, - connection: &Connection, - run_maad: bool, - ) -> Result { - self.flush_complete_profiled(connection, run_maad) - .map(|(count, _)| count) - } - - fn flush_complete_profiled( - &mut self, - connection: &Connection, - run_maad: bool, - ) -> Result<(usize, WriteBucketsProfile), PipelineError> { - let complete_keys = self - .builders - .iter() - .filter(|(_, builder)| builder.has_complete_five_minute_coverage()) - .map(|(key, _)| key.clone()) - .collect::>(); - let buckets = complete_keys - .into_iter() - .filter_map(|key| self.builders.remove(&key)) - .map(StatisticalBucket::finish_owned) - .collect::>(); - let count = buckets.len(); - let profile = write_buckets_profiled(connection, &buckets, run_maad)?; - Ok((count, profile)) - } - - fn finish(self) -> (Vec, Vec) { - ( - self.builders - .into_values() - .map(StatisticalBucket::finish_owned) - .collect(), - Vec::new(), - ) - } -} - -fn publish_rollups( - connection: &Connection, - aggregates: AggregateBuckets, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result<(), PipelineError> { - publish_rollups_profiled(connection, aggregates, pipeline, report).map(|_| ()) -} - -fn publish_rollups_profiled( - connection: &Connection, - aggregates: AggregateBuckets, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result { - let total_started = Instant::now(); - let finish_started = Instant::now(); - let (rollups, incomplete) = aggregates.finish(); - let finish_elapsed = finish_started.elapsed(); - let delete_started = Instant::now(); - delete_stats_bucket_keys(connection, &incomplete)?; - let delete_elapsed = delete_started.elapsed(); - let write = write_buckets_profiled(connection, &rollups, pipeline.run_maad)?; - report.rollup_buckets += rollups.len(); - Ok(FinalRollupProfile { - total_elapsed: total_started.elapsed(), - finish_elapsed, - delete_elapsed, - write, - incomplete_keys: profile_count(incomplete.len()), - rollup_buckets: profile_count(rollups.len()), - }) -} - -/// A repaired five-minute bucket is exact, but persisted coarse unique-count -/// and MAAD rows cannot be patched from scalar results. Keep additive capture -/// coverage current and remove only the affected derived metric rows. -fn refresh_rollups_after_five_minute_repair( - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, -) -> Result<(), PipelineError> { - const DERIVED_TABLES: [&str; 5] = [ - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - ]; - - for granularity in [ - Granularity::ThirtyMinutes, - Granularity::OneHour, - Granularity::OneDay, - ] { - let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; - for table in DERIVED_TABLES { - connection - .execute( - &format!( - "DELETE FROM {table} - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3" - ), - params![child.key.source_id, granularity.as_str(), start], - ) - .map_err(StorageError::from)?; - } - - let children = query_bucket_coverage( - connection, - &child.key.source_id, - Granularity::FiveMinutes.as_str(), - start, - end, - )?; - let expected_children = - usize::try_from((end - start).div_euclid(FIVE_MINUTES)).unwrap_or(usize::MAX); - if children.len() != expected_children { - connection - .execute( - "DELETE FROM bucket_coverage - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3", - params![child.key.source_id, granularity.as_str(), start], - ) - .map_err(StorageError::from)?; - continue; - } - - let mut coverage = BucketCoverage::empty(); - for row in children { - coverage - .include(row.coverage()?) - .map_err(DomainError::from)?; - } - insert_bucket_coverage_rows( - connection, - &[BucketCoverageRow::new( - &child.key.source_id, - granularity.as_str(), - start, - end, - coverage, - )], - )?; - } - Ok(()) -} - -fn aggregate_bounds( - bucket_start: i64, - granularity: Granularity, - timezone: &str, -) -> Result<(i64, i64), PipelineError> { - let timestamp = Timestamp::from_second(bucket_start) - .map_err(|error| PipelineError::Time(error.to_string()))?; - let zoned = timestamp - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))?; - let start = match granularity { - Granularity::ThirtyMinutes => zoned - .round( - ZonedRound::new() - .smallest(Unit::Minute) - .increment(30) - .mode(RoundMode::Trunc), - ) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::OneHour => zoned - .round( - ZonedRound::new() - .smallest(Unit::Hour) - .mode(RoundMode::Trunc), - ) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::OneDay => zoned - .date() - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::FiveMinutes => { - return Err(PipelineError::InvalidConfig( - "five-minute input is not a rollup granularity".into(), - )); - } - }; - let end = match granularity { - Granularity::ThirtyMinutes => start + 1_800, - Granularity::OneHour => start + 3_600, - Granularity::OneDay => zoned - .date() - .tomorrow() - .and_then(|date| date.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(), - Granularity::FiveMinutes => unreachable!("rejected above"), - }; - Ok((start, end)) -} - -fn next_local_five_minute_start(bucket_start: i64, timezone: &str) -> Result { - let current = Timestamp::from_second(bucket_start) - .and_then(|timestamp| timestamp.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))?; - let next = current - .datetime() - .checked_add(5.minutes()) - .and_then(|datetime| datetime.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second(); - if next <= bucket_start { - return Err(PipelineError::Time(format!( - "local five-minute clock did not advance after {bucket_start} in {timezone:?}" - ))); - } - Ok(next) -} - -#[derive(Clone, Copy, Debug)] -struct NfcapdTreeWindow { - start: i64, - end: i64, -} - -/// Resolve the selected and requested nfcapd window using the same date, timezone, and alignment -/// rules for preflight, single-output processing, and coordinated planning. -fn resolve_nfcapd_tree_window( - start_date: &str, - end_date: Option<&str>, - start_time: Option<&str>, - end_time: Option<&str>, - discovered_bucket_starts: impl IntoIterator, - timezone: &str, -) -> Result { - let selected_start = parse_date_start(start_date, timezone)?; - let explicit_end = end_date - .map(|date| next_date_start(date, timezone)) - .transpose()?; - let explicit_start_time = start_time - .map(|value| parse_local_datetime(value, timezone)) - .transpose()?; - let explicit_end_time = end_time - .map(|value| parse_local_datetime(value, timezone)) - .transpose()?; - let discovered_end = discovered_bucket_starts - .into_iter() - .max() - .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) - .transpose()? - .map(|(_, end)| end) - .unwrap_or(selected_start); - let selected_end = explicit_end.unwrap_or(discovered_end); - let start = explicit_start_time.unwrap_or(selected_start); - let end = explicit_end_time.unwrap_or(selected_end); - validate_window(selected_start, selected_end, start, end, timezone)?; - Ok(NfcapdTreeWindow { start, end }) -} - -fn parse_date_start(raw: &str, timezone: &str) -> Result { - let date: Date = raw - .parse() - .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; - Ok(date - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second()) -} - -fn next_date_start(raw: &str, timezone: &str) -> Result { - let date: Date = raw - .parse() - .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; - Ok(date - .tomorrow() - .and_then(|date| date.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second()) -} - -fn parse_local_datetime(raw: &str, timezone: &str) -> Result { - let normalized = if raw.len() == 16 { - format!("{raw}:00") - } else { - raw.to_owned() - }; - let datetime = normalized - .parse::() - .map_err(|error| PipelineError::Time(error.to_string()))?; - Ok(datetime - .in_tz(timezone) - .map_err(|error| PipelineError::Time(error.to_string()))? - .timestamp() - .as_second()) -} - -fn validate_window( - selected_start: i64, - selected_end: i64, - start: i64, - end: i64, - timezone: &str, -) -> Result<(), PipelineError> { - if start < selected_start { - return Err(PipelineError::InvalidConfig( - "start_time must be on or after the selected start_date".into(), - )); - } - if end > selected_end { - return Err(PipelineError::InvalidConfig( - "end_time must be on or before the selected end_date window".into(), - )); - } - if start >= end { - return Err(PipelineError::InvalidConfig( - "input time window must be non-empty".into(), - )); - } - for (label, value) in [("start_time", start), ("end_time", end)] { - if aggregate_bounds(value, Granularity::OneDay, timezone)?.0 != value { - return Err(PipelineError::InvalidConfig(format!( - "{label} must align to a local-day boundary so aggregate rows stay complete" - ))); - } - } - Ok(()) -} - -fn expected_nfcapd_path( - root: &Path, - member: &str, - bucket_start: i64, - timezone: &str, -) -> Result { - let timestamp = Timestamp::from_second(bucket_start) - .and_then(|timestamp| timestamp.in_tz(timezone)) - .map_err(|error| PipelineError::Time(error.to_string()))?; - Ok(root - .join(member) - .join(timestamp.strftime("%Y").to_string()) - .join(timestamp.strftime("%m").to_string()) - .join(timestamp.strftime("%d").to_string()) - .join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M")))) -} - -#[cfg(test)] -mod tests { - use std::{ - fs, - net::{IpAddr, Ipv4Addr}, - }; - - use rusqlite::{Connection, types::ValueRef}; - use serde_json::json; - use tempfile::tempdir; - - use super::*; - use crate::{ - coverage::CoverageState, - domain::{AddressSide, FlowObservation, IpVersion, Scope, Visibility}, - storage::database_operation_lock_path, - }; - - fn write_fake_nfdump(executable: &Path, setup: &str) { - let stream = executable.with_extension("stream"); - let empty_stream = executable.with_extension("empty.stream"); - fs::write(&stream, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); - fs::write( - &empty_stream, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - executable, - format!( - "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\n{setup}\ncat '{}'\n", - empty_stream.display(), - stream.display() - ), - ) - .unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); - } - } - - #[cfg(unix)] - fn write_nfcapd_day(root: &Path) { - write_nfcapd_day_for_date(root, "2025-06-01"); - } - - #[cfg(unix)] - fn write_nfcapd_day_for_date(root: &Path, date: &str) { - let date_path = date.replace('-', "/"); - let day = root.join(format!("edge/{date_path}")); - fs::create_dir_all(&day).unwrap(); - let day_start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - } - - #[cfg(unix)] - fn replace_member_directory(root: &Path) -> PathBuf { - let member = root.join("edge"); - let previous = root.join("edge-before-replacement"); - fs::rename(&member, &previous).unwrap(); - fs::create_dir(&member).unwrap(); - previous - } - - #[cfg(unix)] - fn repeated_dataset_request( - temporary: &tempfile::TempDir, - nfdump: PathBuf, - ) -> (PipelineRequest, PathBuf, PathBuf, PathBuf, PathBuf) { - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output_directory = temporary.path().join("outputs"); - let first_database = output_directory.join("first.sqlite"); - let second_database = output_directory.join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - let sentinel = temporary.path().join("sentinel.txt"); - fs::write(&sentinel, b"leave this alone").unwrap(); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": first_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": temporary.path().join("captures"), - "db_path": second_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - ( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - output_directory, - first_database, - second_database, - sentinel, - ) - } - - #[test] - fn daily_active_sources_rejects_inputs_without_a_tree_day_cohort() { - let selection = FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "0.220.0.0/16" - }))) - .unwrap(); - let csv = InputSpec::Csv { - path: "flows.csv".into(), - mapping_path: "mapping.json".into(), - }; - let explicit = InputSpec::Nfcapd { - path: "nfcapd.202501010000".into(), - source_id: "edge".into(), - bucket_start: None, - gap: false, - expected_path: None, - }; - - assert!(validate_selection_inputs(&selection, &[csv]).is_err()); - assert!(validate_selection_inputs(&selection, &[explicit]).is_err()); - assert!( - validate_selection_inputs( - &selection, - &[ - InputSpec::NfcapdTree { - root_path: "captures".into(), - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-01-01".into(), - end_date: Some("2025-01-01".into()), - start_time: None, - end_time: None, - force: false, - }, - InputSpec::CsvTree { - root_path: "csv".into(), - mapping_path: "mapping.json".into(), - }, - ], - ) - .is_err() - ); - } - - #[cfg(unix)] - #[test] - fn auto_discovered_nfcapd_root_rejects_output_that_would_create_a_member_directory() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir(&root).unwrap(); - let database = root.join("new-member/netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": DEFAULT_TIMEZONE, - "inputs": [{ - "input_kind": "nfcapd_tree", - "root_path": root, - "start_date": "2025-06-01", - "end_date": "2025-06-01" - }] - })) - .unwrap(), - ) - .unwrap(); - - let error = run(PipelineRequest::config(&config)).unwrap_err(); - let message = error.to_string(); - assert!(message.contains("direct-child directory"), "{message}"); - assert!(!database.parent().unwrap().exists()); - } - - #[test] - fn dataset_mode_applies_its_persisted_selection() { - let temporary = tempdir().unwrap(); - let registry = temporary.path().join("datasets.json"); - let database = temporary.path().join("active.sqlite"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": temporary.path(), - "db_path": database, - "source_ids": ["edge"], - "selection": { - "kind": "daily_active_sources", - "ip_prefix": "72.5.0.0/16" - } - }])) - .unwrap(), - ) - .unwrap(); - let resolved = resolve_request(&PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: "nfdump".into(), - force: false, - run_maad: true, - require_complete: false, - }) - .unwrap(); - - assert!(resolved.selection.selects_daily_active_sources()); - assert_eq!(resolved.database_path, database); - } - - #[cfg(unix)] - #[test] - fn nfcapd_member_aliases_are_rejected_before_output_mutation() { - use std::os::unix::fs::symlink; - - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let real_member = root.join("real-member"); - fs::create_dir_all(&real_member).unwrap(); - let capture = real_member.join("2025/06/01/nfcapd.202506010000"); - fs::create_dir_all(capture.parent().unwrap()).unwrap(); - fs::write(&capture, b"capture bytes").unwrap(); - symlink("real-member", root.join("edge-a")).unwrap(); - symlink("real-member", root.join("edge-b")).unwrap(); - let output = temporary.path().join("output.sqlite"); - - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root, - source_ids: vec!["edge-a".into(), "edge-b".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let before = fs::read(&capture).unwrap(); - let error = execute(pipeline).unwrap_err(); - - assert!(error.to_string().contains("same directory")); - assert_eq!(fs::read(capture).unwrap(), before); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[cfg(unix)] - #[test] - fn repeated_dataset_missing_explicit_nfdump_is_side_effect_free() { - let temporary = tempdir().unwrap(); - let missing = temporary.path().join("tools/missing-nfdump"); - let (request, output_directory, first_database, second_database, sentinel) = - repeated_dataset_request(&temporary, missing); - let sentinel_before = fs::read(&sentinel).unwrap(); - - let error = run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); - - let message = error.to_string(); - assert!(message.contains("explicit nfdump executable"), "{message}"); - assert!(message.contains("missing-nfdump"), "{message}"); - assert!(!output_directory.exists()); - for database in [&first_database, &second_database] { - assert!(!database.exists()); - assert!(!database_operation_lock_path(database).unwrap().exists()); - } - assert_eq!(fs::read(sentinel).unwrap(), sentinel_before); - } - - #[cfg(unix)] - #[test] - fn repeated_dataset_non_executable_explicit_nfdump_is_side_effect_free() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let non_executable = temporary.path().join("nfdump-not-executable"); - fs::write(&non_executable, b"#!/bin/sh\n").unwrap(); - fs::set_permissions(&non_executable, fs::Permissions::from_mode(0o644)).unwrap(); - let (request, output_directory, first_database, second_database, sentinel) = - repeated_dataset_request(&temporary, non_executable); - let sentinel_before = fs::read(&sentinel).unwrap(); - - let error = run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); - - let message = error.to_string(); - assert!( - message.contains("not executable by this process"), - "{message}" - ); - assert!(!output_directory.exists()); - for database in [&first_database, &second_database] { - assert!(!database.exists()); - assert!(!database_operation_lock_path(database).unwrap().exists()); - } - assert_eq!(fs::read(sentinel).unwrap(), sentinel_before); - } - - #[cfg(unix)] - #[test] - fn coordinated_mode_loads_one_registry_snapshot_for_all_datasets() { - let temporary = tempdir().unwrap(); - let first_root = temporary.path().join("first-captures"); - let second_root = temporary.path().join("second-captures"); - fs::create_dir_all(first_root.join("edge")).unwrap(); - fs::create_dir_all(second_root.join("edge")).unwrap(); - let first_database = temporary.path().join("first.sqlite"); - let second_database = temporary.path().join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": first_root, - "db_path": first_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": second_root, - "db_path": second_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - let nfdump = temporary.path().join("nfdump"); - write_fake_nfdump(&nfdump, ""); - let request = PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }; - - reset_dataset_registry_load_calls(); - let error = run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); - - assert!(error.to_string().contains("same nfcapd root"), "{error}"); - assert_eq!(dataset_registry_load_calls(), 1); - } - - #[test] - fn coordinated_mode_rejects_duplicate_and_incompatible_datasets() { - let temporary = tempdir().unwrap(); - let first_root = temporary.path().join("first"); - let second_root = temporary.path().join("second"); - fs::create_dir_all(first_root.join("edge")).unwrap(); - fs::create_dir_all(second_root.join("edge")).unwrap(); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": first_root, - "db_path": temporary.path().join("first.sqlite"), - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "72.5.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": second_root, - "db_path": temporary.path().join("second.sqlite"), - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "72.6.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - let request = PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: "nfdump".into(), - force: false, - run_maad: true, - require_complete: false, - }; - - assert!(run_many(request.clone(), vec!["first".into(), "first".into()]).is_err()); - assert!(run_many(request, vec!["first".into(), "second".into()]).is_err()); - } - - fn empty_coordinated_pipeline(database_path: PathBuf) -> ResolvedPipeline { - ResolvedPipeline { - database_path, - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: Vec::new(), - datasets: Vec::new(), - require_complete: false, - } - } - - fn semantic_table_rows(connection: &Connection, table: &str) -> Vec> { - let mut columns = connection - .prepare(&format!("PRAGMA table_info({table})")) - .unwrap() - .query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - }) - .unwrap() - .collect::>>() - .unwrap(); - columns.sort_unstable_by_key(|(index, _)| *index); - let semantic = columns - .into_iter() - .filter(|(_, column)| { - !matches!( - column.as_str(), - "bound_at" | "discovered_at" | "processed_at" - ) - }) - .collect::>(); - let selected = semantic - .iter() - .map(|(_, column)| column.as_str()) - .collect::>() - .join(", "); - let order = semantic - .iter() - .map(|(_, column)| column.as_str()) - .collect::>() - .join(", "); - let mut statement = connection - .prepare(&format!("SELECT {selected} FROM {table} ORDER BY {order}")) - .unwrap(); - statement - .query_map([], |row| { - (0..semantic.len()) - .map(|index| match row.get_ref(index)? { - ValueRef::Null => Ok("NULL".into()), - ValueRef::Integer(value) => Ok(value.to_string()), - ValueRef::Real(value) => Ok(value.to_string()), - ValueRef::Text(value) => Ok(String::from_utf8_lossy(value).into_owned()), - ValueRef::Blob(value) => Ok(value - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::()), - }) - .collect() - }) - .unwrap() - .collect::>>() - .unwrap() - } - - fn coordinated_semantic_snapshot(connection: &Connection) -> Vec>> { - [ - "pipeline_product", - "nfcapd_source_layout", - "datasets", - "source_members", - "bucket_coverage", - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - "processed_inputs", - "input_evidence", - ] - .into_iter() - .map(|table| semantic_table_rows(connection, table)) - .collect() - } - - #[test] - fn coordinated_output_aliases_are_rejected_before_filesystem_mutation() { - let temporary = tempdir().unwrap(); - let target = temporary.path().join("target.sqlite"); - let operation_lock = database_operation_lock_path(&target).unwrap(); - for alias in [ - target.with_file_name("target.sqlite-wal"), - operation_lock.clone(), - ] { - let error = execute_many(vec![ - empty_coordinated_pipeline(target.clone()), - empty_coordinated_pipeline(alias), - ]) - .unwrap_err(); - assert!(error.to_string().contains("must be distinct")); - assert!(!target.exists()); - assert!(!operation_lock.exists()); - } - - let normalized_parent = temporary.path().join("normalized"); - let normalized = normalized_parent.join("..").join("normalized.sqlite"); - let normalized_target = temporary.path().join("normalized.sqlite"); - let error = execute_many(vec![ - empty_coordinated_pipeline(normalized_target.clone()), - empty_coordinated_pipeline(normalized), - ]) - .unwrap_err(); - assert!(error.to_string().contains("must be distinct")); - assert!(!normalized_parent.exists()); - assert!(!normalized_target.exists()); - - let target = temporary.path().join("nested-target.sqlite"); - let target_related = [ - target.clone(), - target.with_file_name("nested-target.sqlite-wal"), - database_operation_lock_path(&target).unwrap(), - ]; - for ancestor in target_related { - let descendant = ancestor.join("second.sqlite"); - let error = execute_many(vec![ - empty_coordinated_pipeline(target.clone()), - empty_coordinated_pipeline(descendant.clone()), - ]) - .unwrap_err(); - assert!(error.to_string().contains("must be distinct"), "{error}"); - assert!(!target.exists()); - assert!(!descendant.exists()); - assert!(!ancestor.exists()); - } - } - - #[cfg(unix)] - #[test] - fn coordinated_output_symlink_and_hard_link_aliases_are_rejected_before_mutation() { - use std::{fs::hard_link, os::unix::fs::symlink}; - - let temporary = tempdir().unwrap(); - let target = temporary.path().join("target.sqlite"); - fs::write(&target, b"existing database bytes").unwrap(); - let target_lock = database_operation_lock_path(&target).unwrap(); - - let symlink_alias = temporary.path().join("symlink.sqlite"); - symlink(&target, &symlink_alias).unwrap(); - let error = execute_many(vec![ - empty_coordinated_pipeline(target.clone()), - empty_coordinated_pipeline(symlink_alias.clone()), - ]) - .unwrap_err(); - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(&target).unwrap(), b"existing database bytes"); - assert!(!target_lock.exists()); - assert!(symlink_alias.is_symlink()); - - let hard_link_alias = temporary.path().join("hard-link.sqlite"); - hard_link(&target, &hard_link_alias).unwrap(); - let error = execute_many(vec![ - empty_coordinated_pipeline(target.clone()), - empty_coordinated_pipeline(hard_link_alias.clone()), - ]) - .unwrap_err(); - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(&target).unwrap(), b"existing database bytes"); - assert_eq!( - fs::read(&hard_link_alias).unwrap(), - b"existing database bytes" - ); - assert!(!target_lock.exists()); - } - - #[cfg(unix)] - #[test] - fn output_capture_aliases_are_rejected_without_mutating_capture_bytes() { - use std::{fs::hard_link, os::unix::fs::symlink}; - - let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202506010000"); - fs::write(&capture, b"capture bytes").unwrap(); - let output = temporary.path().join("output.sqlite"); - let sidecar = output.with_file_name("output.sqlite-wal"); - symlink(&capture, &sidecar).unwrap(); - - for alias in [capture.clone(), sidecar.clone()] { - let error = - validate_output_capture_separation(&[&alias], std::iter::once(capture.as_path())) - .unwrap_err(); - assert!( - error - .to_string() - .contains("aliases discovered nfcapd capture") - ); - assert_eq!(fs::read(&capture).unwrap(), b"capture bytes"); - } - - let hard_link_path = temporary.path().join("hard-link.sqlite"); - hard_link(&capture, &hard_link_path).unwrap(); - let error = validate_output_capture_separation( - &[&hard_link_path], - std::iter::once(capture.as_path()), - ) - .unwrap_err(); - assert!(error.to_string().contains("device/inode")); - assert_eq!(fs::read(&capture).unwrap(), b"capture bytes"); - } - - #[test] - fn output_separation_streams_input_paths_until_a_conflict() { - struct OneInputThenPanic<'a> { - input: Option<&'a Path>, - } - - impl<'a> Iterator for OneInputThenPanic<'a> { - type Item = &'a Path; - - fn next(&mut self) -> Option { - self.input - .take() - .or_else(|| panic!("input separation collected the entire iterator")) - } - } - - let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202506010000"); - fs::write(&capture, b"capture bytes").unwrap(); - let error = validate_output_capture_separation( - &[&capture], - OneInputThenPanic { - input: Some(capture.as_path()), - }, - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("aliases discovered nfcapd capture") - ); - } - - #[test] - fn nfcapd_tree_output_validation_skips_capture_metadata_for_new_outputs() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let namespaces = nfcapd_locator_namespaces(&root, &["edge".into()]).unwrap(); - let output = temporary.path().join("pipeline.sqlite"); - let captures = (0..4_096) - .map(|index| { - root.join(format!( - "edge/2025/06/01/nfcapd.202506{:06}", - index % 100_000 - )) - }) - .collect::>(); - - reset_nfcapd_capture_identity_calls(); - validate_output_nfcapd_capture_separation( - &[&output], - &namespaces, - "UTC", - captures.iter().map(PathBuf::as_path), - ) - .unwrap(); - assert_eq!( - nfcapd_capture_identity_calls(), - 0, - "a new output has no inode that can require a physical capture scan" - ); - } - - #[cfg(unix)] - #[test] - fn nfcapd_tree_output_validation_scans_capture_inodes_only_for_existing_output() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let namespaces = nfcapd_locator_namespaces(&root, &["edge".into()]).unwrap(); - let output = temporary.path().join("pipeline.sqlite"); - fs::write(&output, b"existing output").unwrap(); - let captures = (0..64) - .map(|index| { - let path = temporary.path().join(format!("capture-{index}.nfcapd")); - fs::write(&path, format!("capture-{index}")).unwrap(); - path - }) - .collect::>(); - - reset_nfcapd_capture_identity_calls(); - validate_output_nfcapd_capture_separation( - &[&output], - &namespaces, - "UTC", - captures.iter().map(PathBuf::as_path), - ) - .unwrap(); - assert_eq!(nfcapd_capture_identity_calls(), captures.len()); - - let hard_link_output = temporary.path().join("hard-link.sqlite"); - fs::hard_link(&captures[0], &hard_link_output).unwrap(); - reset_nfcapd_capture_identity_calls(); - let error = validate_output_nfcapd_capture_separation( - &[&hard_link_output], - &namespaces, - "UTC", - captures.iter().map(PathBuf::as_path), - ) - .unwrap_err(); - assert!(error.to_string().contains("device/inode")); - assert_eq!(nfcapd_capture_identity_calls(), 1); - } - - #[cfg(unix)] - #[test] - fn existing_output_reuses_one_bounded_capture_snapshot_for_alias_and_revision_work() { - let temporary = tempdir().unwrap(); - let output = temporary.path().join("pipeline.sqlite"); - fs::write(&output, b"existing output").unwrap(); - let captures = (0..4_096) - .map(|index| { - let path = temporary.path().join(format!("capture-{index}.nfcapd")); - fs::write(&path, format!("capture-{index}")).unwrap(); - path - }) - .collect::>(); - - reset_nfcapd_capture_identity_calls(); - let snapshot_calls = AtomicUsize::new(0); - let snapshots = capture_nfcapd_snapshots_counted(&captures, &snapshot_calls).unwrap(); - assert_eq!(snapshot_calls.load(Ordering::Relaxed), captures.len()); - validate_output_nfcapd_capture_separation_with_snapshots( - &[&output], - &[], - snapshots - .iter() - .map(|(path, snapshot)| (path.as_path(), snapshot)), - ) - .unwrap(); - assert_eq!( - nfcapd_capture_identity_calls(), - 0, - "snapshot-backed alias validation must not run a second serial metadata pass" - ); - - let capture = captures.iter().next().unwrap().clone(); - let sources = [DatasetSource { - source_id: "r1".into(), - members: vec!["r1".into()], - }]; - let paths = BTreeMap::from([(("r1".into(), 0), capture.clone())]); - let bounds = BTreeMap::from([("r1".into(), (0, 0))]); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(1) - .build() - .unwrap(); - let context = NfcapdRevisionContext { - connection: &connection, - sources: &sources, - by_member_and_start: &paths, - member_bounds: &bounds, - extend_gaps_to_window: false, - force: false, - decoder_fingerprint: "decoder".into(), - capture_snapshots: &snapshots, - revision_pool: &pool, - }; - resolve_nfcapd_batch_revisions(&context, &[0]).unwrap(); - - let hard_link_output = temporary.path().join("hard-link.sqlite"); - fs::hard_link(&capture, &hard_link_output).unwrap(); - let error = validate_output_nfcapd_capture_separation_with_snapshots( - &[&hard_link_output], - &[], - snapshots - .iter() - .map(|(path, snapshot)| (path.as_path(), snapshot)), - ) - .unwrap_err(); - assert!(error.to_string().contains("device/inode"), "{error}"); - fs::remove_file(hard_link_output).unwrap(); - - fs::write(&capture, b"capture changed").unwrap(); - let error = resolve_nfcapd_batch_revisions(&context, &[0]).unwrap_err(); - assert!( - error.to_string().contains("changed while"), - "revision preparation must reuse the original snapshot: {error}" - ); - } - - #[test] - fn auto_discovered_tree_rejects_a_new_member_locator_before_mutation() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(&root).unwrap(); - let output = root.join("new-member/2025/06/01/nfcapd.202506010000"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: Vec::new(), - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = execute(pipeline).unwrap_err(); - assert!( - error - .to_string() - .contains("auto-discovered member namespace") - ); - assert!(!output.exists()); - assert!(!root.join("new-member").exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn configured_nfcapd_member_rejects_output_in_a_year_directory() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output = root.join("edge/2025"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = execute(pipeline).unwrap_err(); - let message = error.to_string(); - assert!( - message.contains("configured nfcapd member namespace"), - "{message}" - ); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn auto_discovered_tree_rejects_a_future_member_directory() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(&root).unwrap(); - let output = root.join("future-member"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: Vec::new(), - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = execute(pipeline).unwrap_err(); - let message = error.to_string(); - assert!( - message.contains("auto-discovered member namespace"), - "{message}" - ); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[cfg(unix)] - #[test] - fn nfcapd_namespace_aliases_are_rejected_before_output_mutation() { - use std::os::unix::fs::symlink; - - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let configured_target = root.join("edge/2025"); - let configured_alias = temporary.path().join("configured-alias.sqlite"); - symlink(&configured_target, &configured_alias).unwrap(); - - let configured_pipeline = ResolvedPipeline { - database_path: configured_alias.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - let error = execute(configured_pipeline).unwrap_err(); - assert!( - error - .to_string() - .contains("configured nfcapd member namespace") - ); - assert!(configured_alias.is_symlink()); - assert!( - !database_operation_lock_path(&configured_alias) - .unwrap() - .exists() - ); - - let auto_target = root.join("future-member"); - let auto_alias = temporary.path().join("auto-alias.sqlite"); - symlink(&auto_target, &auto_alias).unwrap(); - let auto_pipeline = ResolvedPipeline { - database_path: auto_alias.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: Vec::new(), - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - let error = execute(auto_pipeline).unwrap_err(); - assert!( - error - .to_string() - .contains("auto-discovered member namespace") - ); - assert!(auto_alias.is_symlink()); - assert!(!database_operation_lock_path(&auto_alias).unwrap().exists()); - } - - #[cfg(unix)] - #[test] - fn csv_and_mapping_aliases_are_rejected_before_mutation() { - use std::{fs::hard_link, os::unix::fs::symlink}; - - let temporary = tempdir().unwrap(); - let input = temporary.path().join("flows.csv"); - let mapping = temporary.path().join("mapping.json"); - fs::write(&input, b"CSV input bytes").unwrap(); - fs::write(&mapping, b"mapping bytes").unwrap(); - - let mut aliases = vec![input.clone(), mapping.clone()]; - for (index, target) in [(&input, "input"), (&mapping, "mapping")] { - let symlink_alias = temporary.path().join(format!("{target}-symlink.sqlite")); - symlink(index, &symlink_alias).unwrap(); - aliases.push(symlink_alias); - let hard_link_alias = temporary.path().join(format!("{target}-hard-link.sqlite")); - hard_link(index, &hard_link_alias).unwrap(); - aliases.push(hard_link_alias); - } - - for output in aliases { - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::Csv { - path: input.clone(), - mapping_path: mapping.clone(), - }], - datasets: Vec::new(), - require_complete: false, - }; - let error = execute(pipeline).unwrap_err(); - assert!(error.to_string().contains("aliases discovered CSV input")); - assert_eq!(fs::read(&input).unwrap(), b"CSV input bytes"); - assert_eq!(fs::read(&mapping).unwrap(), b"mapping bytes"); - for suffix in ["-journal", "-wal", "-shm"] { - assert!( - !output - .with_file_name(format!( - "{}{}", - output.file_name().unwrap().to_string_lossy(), - suffix - )) - .exists() - ); - } - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - } - - #[test] - fn csv_tree_rejects_an_output_that_would_be_discovered_after_creation() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("csv-tree"); - fs::create_dir_all(&root).unwrap(); - let mapping = temporary.path().join("mapping.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": { - "time_end": "time", - "src_ip": "src", - "dst_ip": "dst" - }, - "source_id": {"value": "r1"}, - "discovery": {"include_suffixes": [".csv"]} - })) - .unwrap(), - ) - .unwrap(); - let output = root.join("new-output.csv"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::CsvTree { - root_path: root.clone(), - mapping_path: mapping.clone(), - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = execute(pipeline).unwrap_err(); - assert!( - error - .to_string() - .contains("would be discovered as a CSV tree input") - ); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - assert_eq!( - fs::read(&mapping).unwrap(), - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": { - "time_end": "time", - "src_ip": "src", - "dst_ip": "dst" - }, - "source_id": {"value": "r1"}, - "discovery": {"include_suffixes": [".csv"]} - })) - .unwrap() - ); - } - - #[test] - fn single_daily_active_preflight_rejects_an_absent_expected_capture_alias() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output = root.join("edge/2025/06/01/nfcapd.202506010000"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "192.0.0.0/16" - }))) - .unwrap(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root, - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = preflight_single_output(&pipeline).unwrap_err(); - assert!( - error - .to_string() - .contains("aliases discovered nfcapd capture") - ); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn single_tree_preflight_rejects_missing_locator_in_finite_window() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output = root.join("edge/2025/06/01/nfcapd.202506010000"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root, - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = preflight_single_output(&pipeline).unwrap_err(); - assert!(error.to_string().contains("nfcapd capture locator")); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn single_tree_preflight_rejects_open_ended_future_successor_locator() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output = root.join("edge/2025/06/02/nfcapd.202506020000"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root, - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: None, - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = preflight_single_output(&pipeline).unwrap_err(); - assert!(error.to_string().contains("nfcapd capture locator")); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn explicit_gap_expected_locator_is_checked_before_output_setup() { - let temporary = tempdir().unwrap(); - let expected_path = temporary - .path() - .join("captures/edge/2025/06/01/nfcapd.202506010000"); - let output = expected_path.clone(); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::Nfcapd { - path: temporary.path().join("gap-marker"), - source_id: "edge".into(), - bucket_start: Some(1_735_689_600), - gap: true, - expected_path: Some(expected_path), - }], - datasets: Vec::new(), - require_complete: false, - }; - - let error = preflight_single_output(&pipeline).unwrap_err(); - assert!( - error - .to_string() - .contains("aliases discovered nfcapd capture") - ); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn one_year_ten_member_preflight_does_not_materialize_capture_paths() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let members = (0..10).map(|index| format!("edge-{index}")); - for member in members.clone() { - fs::create_dir_all(root.join(member)).unwrap(); - } - let output = temporary.path().join("pipeline.sqlite"); - let pipeline = ResolvedPipeline { - database_path: output.clone(), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "192.0.0.0/16" - }))) - .unwrap(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root, - source_ids: members.collect(), - sources: Vec::new(), - start_date: "2025-01-01".into(), - end_date: Some("2026-01-01".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }; - - preflight_single_output(&pipeline).unwrap(); - assert!(!output.exists()); - assert!(!database_operation_lock_path(&output).unwrap().exists()); - } - - #[test] - fn coordinated_metadata_conflict_rolls_back_earlier_output_initialization() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let first_database = temporary.path().join("first.sqlite"); - let second_database = temporary.path().join("second.sqlite"); - let selection_a = FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "192.0.0.0/16" - }))) - .unwrap(); - let selection_b = FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "198.51.0.0/16" - }))) - .unwrap(); - let pipeline_for = - |database_path: PathBuf, dataset_id: &str, label: &str, selection: FlowSelection| { - let dataset = Dataset { - dataset_id: dataset_id.into(), - label: label.into(), - root_path: root.clone(), - db_path: database_path.clone(), - default_start_date: String::new(), - source_mode: "static".into(), - discovery_mode: "static".into(), - sort_order: 0, - source_ids: vec!["edge".into()], - sources: Vec::new(), - selection: selection.normalized_payload(), - }; - ResolvedPipeline { - database_path, - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection, - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "1970-01-01".into(), - end_date: Some("1970-01-01".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: vec![dataset], - require_complete: false, - } - }; - let first_existing = pipeline_for( - first_database.clone(), - "first", - "before", - selection_a.clone(), - ); - let second_existing = pipeline_for( - second_database.clone(), - "second", - "second", - selection_a.clone(), - ); - for pipeline in [&first_existing, &second_existing] { - let lock = DatabaseOperationLock::acquire(&pipeline.database_path, "test").unwrap(); - let connection = connect_pipeline_writer(&pipeline.database_path).unwrap(); - init_schema(&connection).unwrap(); - initialize_metadata(&connection, pipeline).unwrap(); - drop(connection); - drop(lock); - } - let before = coordinated_semantic_snapshot(&Connection::open(&first_database).unwrap()); - - let first_requested = pipeline_for(first_database.clone(), "first", "after", selection_a); - let second_requested = - pipeline_for(second_database.clone(), "second", "second", selection_b); - let error = execute_many(vec![first_requested, second_requested]).unwrap_err(); - assert!(error.to_string().contains("second")); - assert!( - error - .to_string() - .contains(second_database.to_string_lossy().as_ref()) - ); - assert_eq!( - coordinated_semantic_snapshot(&Connection::open(first_database).unwrap()), - before - ); - } - - #[test] - fn coordinated_invalid_finite_windows_leave_output_paths_uncreated() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output_directory = temporary.path().join("outputs"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": output_directory.join("first.sqlite"), - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root, - "db_path": output_directory.join("second.sqlite"), - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - for (start_date, end_date) in [("2025-99-01", "2025-10-01"), ("2025-10-02", "2025-10-01")] { - let error = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry.clone()), - start_date: Some(start_date.into()), - end_date: Some(end_date.into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: "./missing-nfdump".into(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["first".into(), "second".into()], - ) - .unwrap_err(); - assert!(matches!( - error, - PipelineError::Time(_) | PipelineError::InvalidConfig(_) - )); - assert!(!output_directory.exists()); - } - } - - #[test] - fn coordinated_auto_discovery_protects_root_regardless_of_dataset_order() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let configured_database = temporary.path().join("configured.sqlite"); - let auto_database = root.join("new-member/netflow.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "configured", - "root_path": root, - "db_path": configured_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "auto", - "root_path": root, - "db_path": auto_database, - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - for order in [ - vec!["configured".to_owned(), "auto".to_owned()], - vec!["auto".to_owned(), "configured".to_owned()], - ] { - let error = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - order, - ) - .unwrap_err(); - - assert!( - error - .to_string() - .contains("auto-discovered member namespace") - ); - assert!(!configured_database.exists()); - assert!(!auto_database.exists()); - assert!(!auto_database.parent().unwrap().exists()); - assert!( - !database_operation_lock_path(&configured_database) - .unwrap() - .exists() - ); - assert!( - !database_operation_lock_path(&auto_database) - .unwrap() - .exists() - ); - } - } - - #[cfg(unix)] - #[test] - fn coordinated_auto_layout_change_after_planning_is_side_effect_free_in_both_orders() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output_directory = temporary.path().join("outputs"); - let first_database = output_directory.join("first.sqlite"); - let second_database = output_directory.join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "configured", - "root_path": root, - "db_path": first_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "auto", - "root_path": temporary.path().join("captures"), - "db_path": second_database, - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - let request = || PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }; - - for order in [ - vec!["configured".to_owned(), "auto".to_owned()], - vec!["auto".to_owned(), "configured".to_owned()], - ] { - let added_member = root.join("edge-b"); - set_coordinated_plan_hook(move |planned_root| { - fs::create_dir_all(planned_root.join("edge-b")).unwrap(); - }); - let error = run_many(request(), order).unwrap_err(); - clear_coordinated_plan_hook(); - - assert!( - error - .to_string() - .contains("auto-discovered source layout changed"), - "{error}" - ); - assert!(!output_directory.exists()); - for database in [&first_database, &second_database] { - assert!(!database.exists()); - assert!(!database_operation_lock_path(database).unwrap().exists()); - } - fs::remove_dir(added_member).unwrap(); - } - } - - #[test] - fn coordinated_revision_resolution_reuses_a_digest_from_one_output() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let member = root.join("edge"); - fs::create_dir_all(&member).unwrap(); - let capture = member.join("nfcapd.197001010000"); - fs::write(&capture, b"capture").unwrap(); - let sources = vec![DatasetSource { - source_id: "edge".into(), - members: vec!["edge".into()], - }]; - let mut paths = BTreeMap::new(); - paths.insert(("edge".into(), 0), capture.clone()); - let mut outputs = Vec::new(); - for name in ["first.sqlite", "second.sqlite"] { - let database_path = temporary.path().join(name); - let lock = DatabaseOperationLock::acquire(&database_path, "test").unwrap(); - let connection = connect_pipeline_writer(&database_path).unwrap(); - init_schema(&connection).unwrap(); - outputs.push(CoordinatedOutput { - pipeline: ResolvedPipeline { - database_path, - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "1970-01-01".into(), - end_date: Some("1970-01-01".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: false, - }, - sources: sources.clone(), - connection, - _lock: lock, - }); - } - - let observed = FileSnapshot::capture(&capture).unwrap(); - let cached_revision = InputRevision::create( - "nfcapd", - capture.to_string_lossy(), - "cached-content", - "decoder", - ) - .unwrap(); - upsert_input_bucket( - &outputs[0].connection, - &InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: cached_revision.locator.clone(), - scan_locator: cached_revision.locator.clone(), - source_id: "edge".into(), - bucket_start: 0, - bucket_end: FIVE_MINUTES, - revision: cached_revision.clone(), - file_snapshot: Some(observed), - }, - false, - ) - .unwrap(); - mark_input_bucket_status( - &outputs[0].connection, - InputKind::Nfcapd, - &cached_revision.locator, - "edge", - 0, - InputStatus::Processed, - &cached_revision, - None, - ) - .unwrap(); - - let revision_pool = build_revision_hash_pool().unwrap(); - let revisions = resolve_coordinated_batch_revisions( - &outputs, - &sources, - &paths, - &BTreeMap::from([("edge".into(), (0, 0))]), - false, - false, - &revision_pool, - &[0], - ) - .unwrap(); - assert_eq!(revisions.len(), 1); - assert_eq!( - revisions[&capture].revision.content_fingerprint, - "cached-content" - ); - } - - #[cfg(unix)] - #[test] - fn coordinated_run_decodes_each_capture_once_and_publishes_distinct_products() { - use std::os::unix::fs::{PermissionsExt, symlink}; - - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - let executable = temporary.path().join("fake-nfdump"); - let stream_path = temporary.path().join("stream.bin"); - let empty_stream_path = temporary.path().join("empty-stream.bin"); - let invocation_log = temporary.path().join("invocations.log"); - let mut stream = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); - let record = 16; - stream[record + 32..record + 40].copy_from_slice(&20_u64.to_le_bytes()); - stream[record + 40..record + 48].copy_from_slice(&2_000_u64.to_le_bytes()); - stream[record + 48..record + 56].copy_from_slice(&3_u64.to_le_bytes()); - stream[record + 64..record + 66].copy_from_slice(&55_000_u16.to_le_bytes()); - stream[record + 69] = 0b010; - fs::write(&stream_path, stream).unwrap(); - fs::write( - &empty_stream_path, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - &executable, - format!( - "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\nprintf 'x\\n' >> '{}'\ncat '{}'\n", - empty_stream_path.display(), - invocation_log.display(), - stream_path.display() - ), - ) - .unwrap(); - fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); - - let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - let path = day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))); - fs::write(path, b"capture").unwrap(); - } - let registry = temporary.path().join("datasets.json"); - let first_db = temporary.path().join("first.sqlite"); - let second_db = temporary.path().join("second.sqlite"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": first_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root, - "db_path": second_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - reset_prepare_nfcapd_tree_timestamp_calls(); - reset_nfcapd_pool_builds(); - reset_dataset_registry_load_calls(); - reset_coordinated_postflight_snapshot_verifications(); - let report = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["first".into(), "second".into()], - ) - .unwrap(); - - assert_eq!(report.five_minute_buckets, 576); - assert_eq!( - dataset_registry_load_calls(), - 1, - "coordinated resolution must use one registry snapshot" - ); - assert_eq!(nfcapd_pool_builds(), (1, 1, 1)); - assert_eq!( - coordinated_postflight_snapshot_verifications(), - 288 + 288, - "cold coordinated publication must verify every input snapshot once before commit" - ); - assert_eq!( - prepare_nfcapd_tree_timestamp_calls(), - 2 * 288, - "coordinated publication must consume preflight preparation" - ); - assert_eq!( - fs::read_to_string(&invocation_log).unwrap().lines().count(), - 289 - ); - let first_identity: String = Connection::open(&first_db) - .unwrap() - .query_row( - "SELECT selection_json FROM pipeline_product WHERE singleton = 1", - [], - |row| row.get(0), - ) - .unwrap(); - let second_identity: String = Connection::open(&second_db) - .unwrap() - .query_row( - "SELECT selection_json FROM pipeline_product WHERE singleton = 1", - [], - |row| row.get(0), - ) - .unwrap(); - assert_ne!(first_identity, second_identity); - - let canonical_root = fs::canonicalize(&root).unwrap(); - let root_alias = temporary.path().join("captures-alias"); - symlink(&root, &root_alias).unwrap(); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root_alias, - "db_path": first_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root_alias, - "db_path": second_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - reset_nfcapd_logical_bucket_topology_calls(); - reset_nfcapd_day_topology_audit_calls(); - reset_nfcapd_pool_builds(); - reset_dataset_registry_load_calls(); - reset_coordinated_postflight_snapshot_verifications(); - crate::storage::reset_resume_query_counters(); - let resumed = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["second".into(), "first".into()], - ) - .unwrap(); - assert_eq!(resumed.five_minute_buckets, 0); - assert_eq!(dataset_registry_load_calls(), 1); - assert_eq!(nfcapd_logical_bucket_topology_calls(), 0); - assert_eq!( - nfcapd_day_topology_audit_calls(), - 0, - "a healthy marker-backed no-op must not scan stats-family topology" - ); - assert_eq!(nfcapd_pool_builds(), (1, 0, 0)); - assert_eq!( - crate::storage::resume_query_counters(), - crate::storage::ResumeQueryCounters { - input_evidence: 2, - processed_nfcapd: 2, - content_fingerprint: 0, - }, - "coordinated no-op resume state should load once per output" - ); - assert_eq!( - coordinated_postflight_snapshot_verifications(), - 0, - "a complete coordinated no-op must not repeat postflight snapshot verification" - ); - assert_eq!( - fs::read_to_string(&invocation_log).unwrap().lines().count(), - 289 - ); - let locator: String = Connection::open(&first_db) - .unwrap() - .query_row( - "SELECT input_locator FROM processed_inputs WHERE input_kind = 'nfcapd' LIMIT 1", - [], - |row| row.get(0), - ) - .unwrap(); - assert!(locator.starts_with(canonical_root.to_string_lossy().as_ref())); - assert!(!locator.contains("captures-alias")); - - let connection = Connection::open(&first_db).unwrap(); - connection - .execute( - "DELETE FROM daily_product_completion - WHERE source_id = 'edge' AND day_start = ?1", - params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], - ) - .unwrap(); - drop(connection); - reset_nfcapd_day_topology_audit_calls(); - let legacy_resumed = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["first".into(), "second".into()], - ) - .unwrap(); - assert_eq!(legacy_resumed.five_minute_buckets, 0); - assert_eq!( - nfcapd_day_topology_audit_calls(), - 1, - "a missing marker without a dirty tombstone must use the legacy topology audit" - ); - let connection = Connection::open(&first_db).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion - WHERE source_id = 'edge' AND day_start = ?1", - params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "legacy recovery must backfill the completion marker" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'edge' AND day_start = ?1", - params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0 - ); - } - - #[cfg(unix)] - #[test] - fn coordinated_resume_keeps_a_complete_output_unchanged_while_catching_up_an_empty_one() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second( - parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap() + bucket * FIVE_MINUTES, - ) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - let executable = temporary.path().join("fake-nfdump"); - let stream_path = temporary.path().join("stream.bin"); - let empty_stream_path = temporary.path().join("empty.stream"); - fs::write(&stream_path, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); - fs::write( - &empty_stream_path, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - &executable, - format!( - "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then cat '{}'; exit 0; fi\ncat '{}'\n", - empty_stream_path.display(), - stream_path.display() - ), - ) - .unwrap(); - fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); - - let first_db = temporary.path().join("first.sqlite"); - let second_db = temporary.path().join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": first_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root, - "db_path": second_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - let single_request = PipelineRequest { - config_path: None, - dataset_id: Some("first".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }; - run(single_request).unwrap(); - let before = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); - - let report = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["first".into(), "second".into()], - ) - .unwrap(); - - assert_eq!(report.five_minute_buckets, 288); - assert_eq!(report.skipped_inputs, 288); - assert_eq!( - coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()), - before, - "a complete coordinated output must remain semantically unchanged" - ); - assert_eq!( - Connection::open(&second_db) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state = 'complete'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288 - ); - } - - #[cfg(unix)] - #[test] - fn coordinated_resume_handles_an_output_needed_only_in_a_later_decode_batch() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second( - parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap() + bucket * FIVE_MINUTES, - ) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - let executable = temporary.path().join("fake-nfdump"); - let stream_path = temporary.path().join("stream.bin"); - let empty_stream_path = temporary.path().join("empty.stream"); - fs::write(&stream_path, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); - fs::write( - &empty_stream_path, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - &executable, - format!( - "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then cat '{}'; exit 0; fi\ncat '{}'\n", - empty_stream_path.display(), - stream_path.display() - ), - ) - .unwrap(); - fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); - - let first_db = temporary.path().join("first.sqlite"); - let second_db = temporary.path().join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": first_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "203.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root, - "db_path": second_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - run(PipelineRequest { - config_path: None, - dataset_id: Some("first".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }) - .unwrap(); - - let before = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); - let first_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); - let later_batch_start = first_start + 12 * FIVE_MINUTES; - let later_batch_end = later_batch_start + 12 * FIVE_MINUTES; - let first_connection = Connection::open(&first_db).unwrap(); - first_connection - .execute( - "DELETE FROM input_evidence - WHERE source_id = 'edge' AND bucket_start >= ?1 AND bucket_start < ?2", - params![later_batch_start, later_batch_end], - ) - .unwrap(); - drop(first_connection); - - let after_fixture = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); - let error = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["first".into(), "second".into()], - ) - .unwrap_err(); - - let message = error.to_string(); - assert!( - message.contains("rerun that whole day with --force"), - "normal resume must reject orphaned provenance: {message}" - ); - let after_error = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); - assert_eq!( - &after_error[4..], - &after_fixture[4..], - "normal resume must not mutate the partially orphaned output" - ); - - let report = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: true, - run_maad: false, - require_complete: false, - }, - vec!["first".into(), "second".into()], - ) - .unwrap(); - - assert_eq!(report.five_minute_buckets, 576); - for database in [&first_db, &second_db] { - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state = 'complete'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(DISTINCT bucket_start) FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288, - "resume must restore every distinct five-minute traffic bucket" - ); - let five_minute_totals = connection - .query_row( - "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) - FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m' - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }, - ) - .unwrap(); - let daily_totals = connection - .query_row( - "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) - FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '1d' - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }, - ) - .unwrap(); - assert_eq!( - daily_totals, five_minute_totals, - "daily traffic must match 5m parity" - ); - } - let after_force = coordinated_semantic_snapshot(&Connection::open(&first_db).unwrap()); - assert_eq!( - &after_force[4..], - &before[4..], - "force rebuild must restore the original first output" - ); - } - - #[cfg(unix)] - #[test] - fn daily_active_resume_accepts_complete_maad_and_rejects_missing_maad_until_force_rebuilds_the_day() - { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let database = temporary.path().join("pipeline.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "default_start_date": "2025-06-01", - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let request = |force: bool| PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force, - run_maad: true, - require_complete: false, - }; - - run(request(false)).unwrap(); - let resumed = run(request(false)).unwrap(); - assert_eq!(resumed.five_minute_buckets, 0); - - // Coverage is part of the certified daily product. A direct mutation must dirty the - // marker so a normal resume cannot silently accept the stale canonical output; force - // rebuilds the complete day and restores the coverage envelope. - let connection = Connection::open(&database).unwrap(); - connection - .execute( - "UPDATE bucket_coverage - SET coverage_state = 'partial', observed_units = 1, expected_units = 2 - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![day_start], - ) - .unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'edge' AND day_start = ?1", - params![day_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "coverage updates must dirty a completed daily-active marker" - ); - drop(connection); - let error = run(request(false)).unwrap_err(); - assert!( - error - .to_string() - .contains("rerun that whole day with --force"), - "normal resume must reject post-certification coverage mutation: {error}" - ); - run(request(true)).unwrap(); - - let connection = Connection::open(&database).unwrap(); - connection - .execute( - "DELETE FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![day_start], - ) - .unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'edge' AND day_start = ?1", - params![day_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "coverage deletes must dirty a completed daily-active marker" - ); - drop(connection); - let error = run(request(false)).unwrap_err(); - assert!( - error - .to_string() - .contains("rerun that whole day with --force"), - "normal resume must reject post-certification coverage deletion: {error}" - ); - run(request(true)).unwrap(); - assert_eq!( - Connection::open(&database) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' - AND coverage_state = 'complete'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288, - "force rebuild must republish deleted coverage" - ); - - let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); - let connection = Connection::open(&database).unwrap(); - connection - .execute( - "UPDATE traffic_stats SET flows = flows + 1 - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1 - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - params![day_start], - ) - .unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'edge' AND day_start = ?1", - params![day_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1 - ); - drop(connection); - reset_nfcapd_day_topology_audit_calls(); - let error = run(request(false)).unwrap_err(); - assert!( - error - .to_string() - .contains("rerun that whole day with --force"), - "normal resume must reject a post-certification metric mutation: {error}" - ); - assert_eq!( - nfcapd_day_topology_audit_calls(), - 0, - "dirty completion evidence must not fall through to legacy topology auditing" - ); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion - WHERE source_id = 'edge' AND day_start = ?1", - params![day_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "normal resume must retain the prior marker while refusing the dirty day" - ); - drop(connection); - run(request(true)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - coordinated_semantic_snapshot(&connection), - before, - "force rebuild must restore exact metrics after a direct mutation" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'edge' AND day_start = ?1", - params![day_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "force rebuild must leave a clean completion marker" - ); - drop(connection); - let mixed_end_bucket = day_start + 8 * FIVE_MINUTES; - let connection = Connection::open(&database).unwrap(); - connection - .execute( - "UPDATE traffic_stats SET bucket_end = ?2 - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1 - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - params![mixed_end_bucket, mixed_end_bucket + FIVE_MINUTES + 1], - ) - .unwrap(); - drop(connection); - let error = run(request(false)).unwrap_err(); - assert!( - error - .to_string() - .contains("rerun that whole day with --force"), - "mixed bucket_end corruption must fail full-day validation: {error}" - ); - let connection = Connection::open(&database).unwrap(); - connection - .execute( - "UPDATE traffic_stats SET bucket_end = bucket_start + 300 - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![mixed_end_bucket], - ) - .unwrap(); - let corrupted_bucket = day_start + 12 * FIVE_MINUTES; - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM address_structure_stats - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![corrupted_bucket], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 30, - "a healthy dense MAAD bucket has five IPv4 scopes, two sides, and three structures" - ); - connection - .execute( - "DELETE FROM address_structure_stats - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1 - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all' - AND address_side = 'source' AND structure_kind = 'structure'", - params![corrupted_bucket], - ) - .unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM address_structure_stats - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![corrupted_bucket], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 29, - "the corruption fixture must remove one of the 30 IPv4 MAAD rows" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM processed_inputs - WHERE input_kind = 'nfcapd' AND source_id = 'edge' AND bucket_start = ?1 - AND status = 'processed'", - params![corrupted_bucket], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "the corruption fixture must retain provenance" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![corrupted_bucket], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "the corruption fixture must retain coverage" - ); - drop(connection); - - let error = run(request(false)).unwrap_err(); - let message = error.to_string(); - assert!( - message.contains("rerun that whole day with --force"), - "normal resume must reject canonical topology corruption: {message}" - ); - assert_eq!( - Connection::open(&database) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM address_structure_stats - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![corrupted_bucket], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 29, - "normal resume must not partially repair the product" - ); - - run(request(true)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(DISTINCT bucket_start) FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288 - ); - let five_minute_totals = connection - .query_row( - "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) - FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m' - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }, - ) - .unwrap(); - let daily_totals = connection - .query_row( - "SELECT COALESCE(SUM(flows), 0), COALESCE(SUM(packets), 0), COALESCE(SUM(bytes), 0) - FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '1d' - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }, - ) - .unwrap(); - assert_eq!( - daily_totals, five_minute_totals, - "force rebuild must restore daily parity" - ); - assert_eq!( - coordinated_semantic_snapshot(&connection), - before, - "force rebuild must restore the semantic product" - ); - } - - #[cfg(unix)] - #[test] - fn daily_active_force_rebuild_removes_surplus_day_rows() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let database = temporary.path().join("pipeline.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "default_start_date": "2025-06-01", - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let request = |force: bool| PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force, - run_maad: false, - require_complete: false, - }; - - run(request(false)).unwrap(); - let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); - let surplus_start = day_start + 60; - let surplus_end = surplus_start + FIVE_MINUTES; - let connection = Connection::open(&database).unwrap(); - connection - .execute( - "INSERT INTO bucket_coverage ( - source_id, granularity, bucket_start, bucket_end, coverage_state, - observed_units, expected_units, rejected_units - ) VALUES ('edge', '5m', ?1, ?2, 'complete', 1, 1, 0)", - params![surplus_start, surplus_end], - ) - .unwrap(); - connection - .execute( - "INSERT INTO traffic_stats ( - source_id, granularity, bucket_start, bucket_end, ip_version, - src_visibility, dst_visibility, flows, flows_tcp, flows_udp, - flows_icmp, flows_other, packets, packets_tcp, packets_udp, - packets_icmp, packets_other, bytes, bytes_tcp, bytes_udp, - bytes_icmp, bytes_other, duration_sum_ms, duration_count, - average_duration_ms, min_ttl_sum, min_ttl_count, average_min_ttl, - max_ttl_sum, max_ttl_count, average_max_ttl - ) - SELECT source_id, granularity, ?1, ?2, ip_version, - src_visibility, dst_visibility, flows, flows_tcp, flows_udp, - flows_icmp, flows_other, packets, packets_tcp, packets_udp, - packets_icmp, packets_other, bytes, bytes_tcp, bytes_udp, - bytes_icmp, bytes_other, duration_sum_ms, duration_count, - average_duration_ms, min_ttl_sum, min_ttl_count, average_min_ttl, - max_ttl_sum, max_ttl_count, average_max_ttl - FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m' - LIMIT 1", - params![surplus_start, surplus_end], - ) - .unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![surplus_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![surplus_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1 - ); - drop(connection); - - let error = run(request(false)).unwrap_err(); - assert!( - error - .to_string() - .contains("rerun that whole day with --force"), - "normal resume must reject a valid surplus row: {error}" - ); - - run(request(true)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![surplus_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "force must remove surplus coverage inside the requested day" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![surplus_start], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "force must remove surplus product rows inside the requested day" - ); - assert_eq!(coordinated_semantic_snapshot(&connection), before); - drop(connection); - - let resumed = run(request(false)).unwrap(); - assert_eq!(resumed.five_minute_buckets, 0); - assert_eq!( - coordinated_semantic_snapshot(&Connection::open(&database).unwrap()), - before, - "a clean normal resume must be a no-op after force rebuild" - ); - } - - #[test] - fn coordinated_run_skips_an_incomplete_physical_day_for_every_output() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - fs::write(day.join("nfcapd.202506010000"), b"capture").unwrap(); - let registry = temporary.path().join("datasets.json"); - let first_db = temporary.path().join("first.sqlite"); - let second_db = temporary.path().join("second.sqlite"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": first_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root, - "db_path": second_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - let error = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: true, - }, - vec!["first".into(), "second".into()], - ) - .unwrap_err(); - - let message = error.to_string(); - assert!(message.contains("dataset \"first\""), "{message}"); - assert!( - message.contains(&first_db.to_string_lossy().to_string()), - "{message}" - ); - assert!( - message.contains("576 incomplete five-minute coverage buckets"), - "{message}" - ); - for database in [first_db, second_db] { - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 0 - ); - } - } - - #[test] - fn canonical_day_resume_queries_seek_bounded_source_granularity_ranges() { - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - - let explain = |query: String| { - connection - .prepare(&query) - .unwrap() - .query_map(params!["r1", 0_i64, 86_400_i64], |row| { - row.get::<_, String>(3) - }) - .unwrap() - .collect::>>() - .unwrap() - }; - let assert_bounded_seek = |label: &str, plan: &[String]| { - let uses_range_seek = plan.iter().any(|detail| { - let compact = detail.replace(' ', ""); - compact.contains("SEARCH") - && compact.contains("source_id=?") - && compact.contains("granularity=?") - && compact.contains("bucket_start>?") - && compact.contains("bucket_start= ?2 AND bucket_start < ?3 - ORDER BY granularity, bucket_start" - )), - ); - for table in [ - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - ] { - assert_bounded_seek( - table, - &explain(format!( - "EXPLAIN QUERY PLAN - SELECT granularity, bucket_start, MIN(bucket_end), MAX(bucket_end), COUNT(*) - FROM {table} - WHERE source_id = ?1 AND {CANONICAL_GRANULARITY_PREDICATE} - AND bucket_start >= ?2 AND bucket_start < ?3 - GROUP BY granularity, bucket_start" - )), - ); - } - } - - #[cfg(unix)] - #[test] - fn coordinated_mixed_explicit_and_auto_layouts_publish_identical_source_metadata() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - write_nfcapd_day(&root); - let first_database = temporary.path().join("first.sqlite"); - let second_database = temporary.path().join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "configured", - "root_path": root, - "db_path": first_database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "auto", - "root_path": temporary.path().join("captures"), - "db_path": second_database, - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - let report = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }, - vec!["configured".into(), "auto".into()], - ) - .unwrap(); - assert_eq!(report.five_minute_buckets, 576); - - let source_members = |database: &Path| { - let connection = Connection::open(database).unwrap(); - connection - .prepare( - "SELECT source_id, member_id FROM source_members - ORDER BY dataset_id, source_id, member_id", - ) - .unwrap() - .query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) - }) - .unwrap() - .collect::>>() - .unwrap() - }; - let source_layout = |database: &Path| { - let connection = Connection::open(database).unwrap(); - connection - .query_row( - "SELECT layout_json FROM nfcapd_source_layout WHERE singleton = 1", - [], - |row| row.get::<_, String>(0), - ) - .unwrap() - }; - let coverage_layout = |database: &Path| { - let connection = Connection::open(database).unwrap(); - connection - .prepare( - "SELECT source_id, granularity, bucket_start, bucket_end - FROM bucket_coverage ORDER BY source_id, granularity, bucket_start", - ) - .unwrap() - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - )) - }) - .unwrap() - .collect::>>() - .unwrap() - }; - - assert_eq!( - source_members(&first_database), - source_members(&second_database) - ); - assert_eq!( - source_layout(&first_database), - source_layout(&second_database) - ); - assert_eq!( - coverage_layout(&first_database), - coverage_layout(&second_database) - ); - } - - #[cfg(unix)] - #[test] - fn single_auto_layout_change_after_planning_is_side_effect_free() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let output_directory = temporary.path().join("outputs"); - let database = output_directory.join("active.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - set_single_plan_hook(move |planned_root| { - fs::create_dir_all(planned_root.join("edge-b")).unwrap(); - }); - let error = run(PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }) - .unwrap_err(); - clear_single_plan_hook(); - - assert!( - error - .to_string() - .contains("auto-discovered source layout changed"), - "{error}" - ); - assert!(!output_directory.exists()); - assert!(!database.exists()); - assert!(!database_operation_lock_path(&database).unwrap().exists()); - fs::remove_dir_all(root.join("edge-b")).unwrap(); - } - - #[cfg(unix)] - #[test] - fn member_directory_replacement_after_planning_is_side_effect_free_in_single_and_coordinated_modes() - { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let database = temporary.path().join("single.sqlite"); - let registry = temporary.path().join("single-datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let single_backup = root.join("edge-before-single-replacement"); - let single_backup_for_hook = single_backup.clone(); - set_single_plan_hook(move |planned_root| { - let member = planned_root.join("edge"); - fs::rename(&member, &single_backup_for_hook).unwrap(); - fs::create_dir(&member).unwrap(); - }); - let single_error = run(PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }) - .unwrap_err(); - clear_single_plan_hook(); - assert!(single_error.to_string().contains("member directory")); - assert!(!database.exists()); - assert!(!database_operation_lock_path(&database).unwrap().exists()); - fs::remove_dir(root.join("edge")).unwrap(); - fs::rename(single_backup, root.join("edge")).unwrap(); - - let (request, output_directory, first_database, second_database, sentinel) = - repeated_dataset_request(&temporary, nfdump.clone()); - let coordinated_backup = root.join("edge-before-coordinated-replacement"); - set_coordinated_plan_hook(move |planned_root| { - let member = planned_root.join("edge"); - fs::rename(&member, &coordinated_backup).unwrap(); - fs::create_dir(&member).unwrap(); - }); - let coordinated_error = - run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); - clear_coordinated_plan_hook(); - assert!(coordinated_error.to_string().contains("member directory")); - assert!(!output_directory.exists()); - for database in [&first_database, &second_database] { - assert!(!database.exists()); - assert!(!database_operation_lock_path(database).unwrap().exists()); - } - assert_eq!(fs::read(sentinel).unwrap(), b"leave this alone"); - } - - #[cfg(unix)] - #[test] - fn member_directory_replacement_between_days_does_not_mix_single_product_days() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - write_nfcapd_day_for_date(&root, "2025-06-01"); - write_nfcapd_day_for_date(&root, "2025-06-02"); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let database = temporary.path().join("active.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let calls = std::rc::Rc::new(std::cell::Cell::new(0)); - let hook_calls = std::rc::Rc::clone(&calls); - set_missing_day_absence_hook(move |planned_root, _, _| { - let call = hook_calls.get(); - hook_calls.set(call + 1); - if call == 1 { - replace_member_directory(planned_root); - } - }); - let error = run(PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-03".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }) - .unwrap_err(); - clear_missing_day_absence_hook(); - - assert!(error.to_string().contains("member directory"), "{error}"); - assert_eq!(calls.get(), 2); - let connection = Connection::open(database).unwrap(); - let day_count = |start_date: &str| { - let start = parse_date_start(start_date, DEFAULT_TIMEZONE).unwrap(); - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' - AND bucket_start >= ?1 AND bucket_start < ?2", - params![start, start + 86_400], - |row| row.get::<_, i64>(0), - ) - .unwrap() - }; - assert_eq!(day_count("2025-06-01"), 288); - assert_eq!(day_count("2025-06-02"), 0); - } - - #[cfg(unix)] - #[test] - fn member_directory_replacement_at_precommit_rolls_back_single_and_coordinated_days() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - write_nfcapd_day(&root); - let database = temporary.path().join("single.sqlite"); - let registry = temporary.path().join("single-datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let single_backup = root.join("edge-before-single-precommit-replacement"); - let root_for_hook = root.clone(); - let single_backup_for_hook = single_backup.clone(); - set_single_commit_guard_hook(move || { - let member = root_for_hook.join("edge"); - fs::rename(&member, &single_backup_for_hook).unwrap(); - fs::create_dir(&member).unwrap(); - }); - let single_error = run(PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-02".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }) - .unwrap_err(); - clear_single_commit_guard_hook(); - assert!(single_error.to_string().contains("member directory")); - let single_connection = Connection::open(&database).unwrap(); - assert_eq!( - single_connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 0 - ); - fs::remove_dir(root.join("edge")).unwrap(); - fs::rename(single_backup, root.join("edge")).unwrap(); - - let (mut request, _, first_database, second_database, _) = - repeated_dataset_request(&temporary, nfdump.clone()); - request.end_date = Some("2025-06-01".into()); - write_nfcapd_day(&root); - let coordinated_backup = root.join("edge-before-coordinated-precommit-replacement"); - set_coordinated_commit_guard_hook(move || { - let member = root.join("edge"); - fs::rename(&member, &coordinated_backup).unwrap(); - fs::create_dir(&member).unwrap(); - }); - let coordinated_error = - run_many(request, vec!["first".into(), "second".into()]).unwrap_err(); - clear_coordinated_commit_guard_hook(); - assert!(coordinated_error.to_string().contains("member directory")); - for database in [first_database, second_database] { - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 0 - ); - } - } - - #[cfg(unix)] - #[test] - fn single_daily_active_capture_rewrite_before_commit_rolls_back_day() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - write_nfcapd_day(&root); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let database = temporary.path().join("active.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "default_start_date": "2025-06-01", - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let request = |force| PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force, - run_maad: false, - require_complete: false, - }; - - run(request(false)).unwrap(); - let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); - let marker_count_before = Connection::open(&database) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM daily_product_completion - WHERE source_id = 'edge'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(); - let capture = root.join("edge/2025/06/01/nfcapd.202506010000"); - let original = fs::read(&capture).unwrap(); - let rewritten = capture.clone(); - set_single_commit_guard_hook(move || { - fs::write(&rewritten, b"rewritten after processing").unwrap(); - }); - - let error = run(request(true)).unwrap_err(); - clear_single_commit_guard_hook(); - - assert!(error.to_string().contains("Input changed"), "{error}"); - assert_eq!( - coordinated_semantic_snapshot(&Connection::open(&database).unwrap()), - before, - "a capture rewrite at the precommit seam must roll back all day rows" - ); - assert_eq!( - Connection::open(&database) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM daily_product_completion - WHERE source_id = 'edge'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - marker_count_before, - "the completion marker must roll back with the day rows" - ); - assert_ne!(fs::read(&capture).unwrap(), original); - fs::write(capture, original).unwrap(); - } - - #[cfg(unix)] - #[test] - fn single_tree_rejects_stale_complete_day_and_force_removes_it_before_strict_failure() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second( - parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap() + bucket * FIVE_MINUTES, - ) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let registry = temporary.path().join("datasets.json"); - let database = temporary.path().join("pipeline.sqlite"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let request = |force: bool, require_complete: bool| PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force, - run_maad: false, - require_complete, - }; - - run(request(false, false)).unwrap(); - let removed = day.join("nfcapd.202506010000"); - fs::remove_file(&removed).unwrap(); - Connection::open(&database) - .unwrap() - .execute( - "DELETE FROM bucket_coverage - WHERE source_id = 'edge' AND granularity = '5m' AND bucket_start = ?1", - params![parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap()], - ) - .unwrap(); - let before = Connection::open(&database) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(); - assert!(matches!( - run(request(false, false)), - Err(PipelineError::InvalidConfig(_)) - )); - assert_eq!( - Connection::open(&database) - .unwrap() - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - before - ); - - let error = run(request(true, true)).unwrap_err(); - assert!(matches!(error, PipelineError::IncompleteCoverage(288))); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0 - ); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM input_evidence", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 0 - ); - } - - #[cfg(unix)] - #[test] - fn single_force_stale_day_rejects_a_late_capture_without_product_loss() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - write_nfcapd_day(&root); - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let registry = temporary.path().join("datasets.json"); - let database = temporary.path().join("pipeline.sqlite"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - let request = |force: bool| PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry.clone()), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force, - run_maad: false, - require_complete: false, - }; - - run(request(false)).unwrap(); - let before = coordinated_semantic_snapshot(&Connection::open(&database).unwrap()); - let removed = root.join("edge/2025/06/01/nfcapd.202506010000"); - fs::remove_file(&removed).unwrap(); - let restored = removed.clone(); - set_missing_day_absence_hook(move |_, missing, _| { - assert!(!missing.is_empty()); - fs::write(&restored, b"late capture").unwrap(); - }); - - let error = run(request(true)).unwrap_err(); - clear_missing_day_absence_hook(); - - let message = error.to_string(); - assert!( - message.contains("refusing to delete the existing product"), - "{message}" - ); - assert!(message.contains("nfcapd.202506010000"), "{message}"); - assert!(removed.is_file()); - assert_eq!( - &coordinated_semantic_snapshot(&Connection::open(database).unwrap())[4..], - &before[4..], - "late capture detection must roll back the stale-day deletion" - ); - } - - #[cfg(unix)] - #[test] - fn coordinated_force_stale_day_rejects_a_late_capture_without_product_loss() { - let temporary = tempdir().unwrap(); - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let (mut request, _output_directory, first_database, second_database, _sentinel) = - repeated_dataset_request(&temporary, executable.clone()); - request.end_date = Some("2025-06-01".into()); - let root = temporary.path().join("captures"); - write_nfcapd_day(&root); - - run_many(request.clone(), vec!["first".into(), "second".into()]).unwrap(); - let before_first = - coordinated_semantic_snapshot(&Connection::open(&first_database).unwrap()); - let before_second = - coordinated_semantic_snapshot(&Connection::open(&second_database).unwrap()); - let removed = root.join("edge/2025/06/01/nfcapd.202506010000"); - fs::remove_file(&removed).unwrap(); - let restored = removed.clone(); - set_missing_day_absence_hook(move |_, missing, _| { - assert!(!missing.is_empty()); - fs::write(&restored, b"late capture").unwrap(); - }); - - let mut forced = request; - forced.force = true; - let error = run_many(forced, vec!["first".into(), "second".into()]).unwrap_err(); - clear_missing_day_absence_hook(); - - let message = error.to_string(); - assert!( - message.contains("refusing to delete the existing product"), - "{message}" - ); - assert!(message.contains("nfcapd.202506010000"), "{message}"); - assert!(removed.is_file()); - assert_eq!( - &coordinated_semantic_snapshot(&Connection::open(first_database).unwrap())[4..], - &before_first[4..], - "first coordinated output must survive a late capture" - ); - assert_eq!( - &coordinated_semantic_snapshot(&Connection::open(second_database).unwrap())[4..], - &before_second[4..], - "second coordinated output must survive a late capture" - ); - } - - #[cfg(unix)] - #[test] - fn coordinated_force_stale_day_does_not_guard_again_after_the_first_commit() { - let temporary = tempdir().unwrap(); - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let (mut request, _output_directory, first_database, second_database, _sentinel) = - repeated_dataset_request(&temporary, executable.clone()); - request.end_date = Some("2025-06-01".into()); - let root = temporary.path().join("captures"); - write_nfcapd_day(&root); - - run_many(request.clone(), vec!["first".into(), "second".into()]).unwrap(); - let removed = root.join("edge/2025/06/01/nfcapd.202506010000"); - fs::remove_file(&removed).unwrap(); - let calls = std::rc::Rc::new(std::cell::Cell::new(0)); - let hook_calls = std::rc::Rc::clone(&calls); - let late_capture = removed.clone(); - set_coordinated_commit_guard_hook(move || { - let call = hook_calls.get(); - hook_calls.set(call + 1); - if call == 1 { - fs::write(&late_capture, b"late capture").unwrap(); - } - }); - - let mut forced = request; - forced.force = true; - let result = run_many(forced, vec!["first".into(), "second".into()]); - clear_coordinated_commit_guard_hook(); - result.unwrap(); - - assert_eq!(calls.get(), 1); - assert!(!removed.exists()); - assert_eq!( - &coordinated_semantic_snapshot(&Connection::open(first_database).unwrap())[4..], - &coordinated_semantic_snapshot(&Connection::open(second_database).unwrap())[4..], - "a late-capture hook at the former second-commit seam must not split outputs" - ); - } - - #[cfg(unix)] - #[test] - fn coordinated_publication_does_not_guard_again_after_the_first_commit() { - let temporary = tempdir().unwrap(); - let executable = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&executable, ""); - let (request, _output_directory, first_database, second_database, _sentinel) = - repeated_dataset_request(&temporary, executable.clone()); - let root = temporary.path().join("captures"); - write_nfcapd_day(&root); - - let calls = std::rc::Rc::new(std::cell::Cell::new(0)); - let hook_calls = std::rc::Rc::clone(&calls); - let changed_decoder = executable.clone(); - set_coordinated_commit_guard_hook(move || { - let call = hook_calls.get(); - hook_calls.set(call + 1); - if call == 1 { - let mut contents = fs::read(&changed_decoder).unwrap(); - contents.push(b'\n'); - fs::write(&changed_decoder, contents).unwrap(); - } - }); - - let result = run_many(request, vec!["first".into(), "second".into()]); - clear_coordinated_commit_guard_hook(); - result.unwrap(); - - assert_eq!(calls.get(), 1); - let mut states = Vec::new(); - for database in [first_database, second_database] { - let connection = Connection::open(database).unwrap(); - let processed_inputs = connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(); - let bucket_coverage = connection - .query_row("SELECT COUNT(*) FROM bucket_coverage", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(); - assert!(processed_inputs > 0); - assert!(bucket_coverage > 0); - states.push((processed_inputs, bucket_coverage)); - } - assert_eq!(states[0], states[1]); - } - - #[test] - fn coordinated_empty_finite_request_is_incomplete_without_publishing_traffic() { - let temporary = tempdir().unwrap(); - let nfdump = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&nfdump, ""); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("edge")).unwrap(); - let first_db = temporary.path().join("first.sqlite"); - let second_db = temporary.path().join("second.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([ - { - "dataset_id": "first", - "root_path": root, - "db_path": first_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }, - { - "dataset_id": "second", - "root_path": root, - "db_path": second_db, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "198.51.0.0/16"} - } - ])) - .unwrap(), - ) - .unwrap(); - - let error = run_many( - PipelineRequest { - config_path: None, - dataset_id: None, - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: nfdump.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: true, - }, - vec!["first".into(), "second".into()], - ) - .unwrap_err(); - - let message = error.to_string(); - assert!(message.contains("dataset \"first\""), "{message}"); - assert!( - message.contains(&first_db.to_string_lossy().to_string()), - "{message}" - ); - assert!( - message.contains("288 incomplete five-minute coverage buckets"), - "{message}" - ); - for database in [first_db, second_db] { - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 0 - ); - } - } - - #[test] - fn complete_physical_day_uses_local_dst_bucket_boundaries() { - let start = parse_date_start("2025-03-09", "America/Los_Angeles").unwrap(); - let end = next_date_start("2025-03-09", "America/Los_Angeles").unwrap(); - let members = vec!["cc".to_owned(), "oh".to_owned()]; - let mut paths = BTreeMap::new(); - let mut bucket_start = start; - let mut count = 0; - while bucket_start < end { - for member in &members { - paths.insert( - (member.clone(), bucket_start), - PathBuf::from(format!("{member}/{bucket_start}")), - ); - } - count += 1; - bucket_start = - next_local_five_minute_start(bucket_start, "America/Los_Angeles").unwrap(); - } - - assert_eq!(count, 276); - assert!( - missing_physical_day_inputs(&members, &paths, start, end, "America/Los_Angeles") - .unwrap() - .is_empty() - ); - paths.remove(&("oh".to_owned(), start)); - assert_eq!( - missing_physical_day_inputs(&members, &paths, start, end, "America/Los_Angeles") - .unwrap(), - [("oh".to_owned(), start)] - ); - } - - #[test] - fn nfcapd_decode_chunks_cap_a_timestamp_with_more_than_twelve_members() { - let members = (0..13).map(|index| format!("member-{index:02}")); - let members = members.collect::>(); - let sources = [DatasetSource { - source_id: "logical".into(), - members: members.clone(), - }]; - let paths = members - .iter() - .map(|member| { - ( - (member.clone(), 0), - PathBuf::from(format!("{member}/nfcapd.197001010000")), - ) - }) - .collect::>(); - - assert_eq!( - nfcapd_batch_starts( - 0, - FIVE_MINUTES, - "UTC", - &sources, - &paths, - &BTreeMap::new(), - false, - ) - .unwrap(), - [0] - ); - let requests = members - .iter() - .map(|member| (member.clone(), 0_i64)) - .collect::>(); - let chunk_lengths = nfcapd_decode_request_chunks(&requests) - .map(<[_]>::len) - .collect::>(); - assert_eq!(chunk_lengths, [12, 1]); - assert!( - chunk_lengths - .iter() - .all(|length| *length <= NFCAPD_DECODE_BATCH_SIZE) - ); - } - - #[cfg(unix)] - #[test] - fn daily_activity_unions_each_physical_member_once() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let executable = temporary.path().join("fake-nfdump"); - let stream_path = temporary.path().join("activity.stream"); - let empty_stream_path = temporary.path().join("empty.stream"); - let invocation_log = temporary.path().join("invocations.log"); - let mut stream = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); - let record = 16; - stream[record + 32..record + 40].copy_from_slice(&10_u64.to_le_bytes()); - stream[record + 40..record + 48].copy_from_slice(&1_000_u64.to_le_bytes()); - stream[record + 48..record + 56].copy_from_slice(&2_u64.to_le_bytes()); - stream[record + 64..record + 66].copy_from_slice(&1_024_u16.to_le_bytes()); - stream[record + 69] = 0b010; - fs::write(&stream_path, stream).unwrap(); - fs::write( - &empty_stream_path, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - &executable, - format!( - "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\nprintf 'x\\n' >> '{}'\ncat '{}'\n", - empty_stream_path.display(), - invocation_log.display(), - stream_path.display() - ), - ) - .unwrap(); - fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); - - let mut paths = BTreeMap::new(); - for member in ["cc", "oh"] { - let directory = temporary.path().join(member); - fs::create_dir(&directory).unwrap(); - let path = directory.join("nfcapd.197001010000"); - fs::write(&path, "capture").unwrap(); - paths.insert((member.to_owned(), 0), path); - } - let sources = vec![ - DatasetSource { - source_id: "cc".into(), - members: vec!["cc".into()], - }, - DatasetSource { - source_id: "oh".into(), - members: vec!["oh".into()], - }, - DatasetSource { - source_id: "all".into(), - members: vec!["cc".into(), "oh".into()], - }, - ]; - let selection = FlowSelection::from_payload(Some(&json!({ - "kind": "daily_active_sources", - "ip_prefix": "192.0.0.0/16" - }))) - .unwrap(); - let pipeline = ResolvedPipeline { - database_path: temporary.path().join("unused.sqlite"), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: executable.to_string_lossy().into_owned().into(), - nfdump_revision: None, - selection, - inputs: Vec::new(), - datasets: Vec::new(), - require_complete: false, - }; - - let (active, _) = resolve_daily_active_sources( - &sources, - &paths, - 0, - FIVE_MINUTES, - &pipeline, - &BTreeMap::new(), - &BTreeMap::new(), - ) - .unwrap(); - - assert!(active.contains(&IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)))); - assert_eq!( - fs::read_to_string(invocation_log).unwrap().lines().count(), - 2 - ); - } - - #[cfg(unix)] - #[test] - fn daily_active_eligibility_ignores_off_grid_capture_keys() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - let day_start = parse_date_start("2025-06-01", DEFAULT_TIMEZONE).unwrap(); - for bucket in 0..288 { - let timestamp = Timestamp::from_second(day_start + bucket * FIVE_MINUTES) - .unwrap() - .in_tz(DEFAULT_TIMEZONE) - .unwrap(); - fs::write( - day.join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))), - b"capture", - ) - .unwrap(); - } - let off_grid = day.join("nfcapd.202506010001"); - fs::write(&off_grid, b"off-grid capture").unwrap(); - - let executable = temporary.path().join("fake-nfdump"); - let active_stream = temporary.path().join("active.stream"); - let empty_stream = temporary.path().join("empty.stream"); - let mut active_bytes = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); - active_bytes[16 + 32..16 + 40].copy_from_slice(&20_u64.to_le_bytes()); - active_bytes[16 + 40..16 + 48].copy_from_slice(&2_000_u64.to_le_bytes()); - active_bytes[16 + 48..16 + 56].copy_from_slice(&3_u64.to_le_bytes()); - fs::write(&active_stream, active_bytes).unwrap(); - fs::write( - &empty_stream, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - &executable, - format!( - "#!/bin/sh -if [ \"$1\" = \"-R\" ]; then - if [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then - cat '{}' - exit 0 - fi - for path in \"$2\"/*; do - target=$(readlink \"$path\") - case \"$target\" in - *nfcapd.202506010001) cat '{}'; exit 0 ;; - esac - done - cat '{}' -else - cat '{}' -fi -", - empty_stream.display(), - active_stream.display(), - empty_stream.display(), - active_stream.display() - ), - ) - .unwrap(); - fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); - - let database = temporary.path().join("pipeline.sqlite"); - let registry = temporary.path().join("datasets.json"); - fs::write( - ®istry, - serde_json::to_vec(&json!([{ - "dataset_id": "active", - "root_path": root, - "db_path": database, - "source_ids": ["edge"], - "selection": {"kind": "daily_active_sources", "ip_prefix": "192.0.0.0/16"} - }])) - .unwrap(), - ) - .unwrap(); - - run(PipelineRequest { - config_path: None, - dataset_id: Some("active".into()), - datasets_path: Some(registry), - start_date: Some("2025-06-01".into()), - end_date: Some("2025-06-01".into()), - start_time: None, - end_time: None, - database_path: None, - selection: Value::Null, - nfdump: executable.to_string_lossy().into_owned(), - force: false, - run_maad: false, - require_complete: false, - }) - .unwrap(); - - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COALESCE(SUM(flows), 0) FROM traffic_stats - WHERE source_id = 'edge' AND granularity = '5m' - AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "an off-grid-only threshold hit must not qualify the active source" - ); - for table in [ - "bucket_coverage", - "traffic_stats", - "processed_inputs", - "input_evidence", - ] { - assert_eq!( - connection - .query_row( - &format!( - "SELECT COUNT(*) FROM {table} - WHERE source_id = 'edge' AND bucket_start = ?1" - ), - params![day_start + 60], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "off-grid capture must not appear in {table}" - ); - } - } - - #[test] - fn logical_sources_borrow_singletons_and_merge_overlapping_members() { - let build = |source_id: &str, destination: [u8; 4]| { - let mut bucket = StatisticalBucket::dense(BucketKey::new( - source_id, - Granularity::FiveMinutes, - 0, - FIVE_MINUTES, - )); - bucket - .add( - FlowObservation::new( - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), - IpAddr::V4(Ipv4Addr::from(destination)), - 6, - 2, - 128, - 0, - ) - .unwrap(), - ) - .unwrap(); - bucket.finish() - }; - let cc = build("cc_ir1_gw", [198, 51, 100, 1]); - let oh = build("oh_ir1_gw", [198, 51, 100, 2]); - - let singleton = logical_source_bucket("cc_ir1_gw", 0, 1, &[&cc]).unwrap(); - assert!(matches!(singleton, Cow::Borrowed(_))); - - let combined = logical_source_bucket("uoregon_all", 0, 2, &[&cc, &oh]).unwrap(); - assert!(matches!(combined, Cow::Owned(_))); - let all_v4 = Scope::new(IpVersion::V4, Visibility::All, Visibility::All); - assert_eq!( - combined - .traffic - .iter() - .find(|entry| entry.scope == all_v4) - .unwrap() - .metrics - .flows, - 2 - ); - assert_eq!(combined.coverage.state(), CoverageState::Complete); - - let partial = logical_source_bucket("uoregon_all", 0, 2, &[&cc]).unwrap(); - assert_eq!(partial.coverage.state(), CoverageState::Partial); - assert_eq!(partial.coverage.observed_units(), 1); - - let unknown = logical_source_bucket("uoregon_all", 0, 2, &[]).unwrap(); - assert_eq!(unknown.coverage.state(), CoverageState::Unknown); - assert!(unknown.traffic.is_empty()); - assert_eq!( - combined - .addresses - .iter() - .find(|entry| { - entry.scope == all_v4 && entry.address_side == AddressSide::Source - }) - .unwrap() - .addresses - .len(), - 1 - ); - assert_eq!( - combined - .addresses - .iter() - .find(|entry| { - entry.scope == all_v4 && entry.address_side == AddressSide::Destination - }) - .unwrap() - .addresses - .len(), - 2 - ); - } - - #[test] - fn strict_coverage_checks_only_the_finite_native_request() { - let temporary = tempdir().unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let capture_root = temporary.path().join("captures"); - fs::create_dir_all(capture_root.join("r1")).unwrap(); - let inside = parse_date_start("2025-01-01", "UTC").unwrap(); - let outside = parse_date_start("2025-01-03", "UTC").unwrap(); - for (source_id, bucket_start) in [("r1", inside), ("r1", outside), ("unrequested", inside)] - { - connection - .execute( - "INSERT INTO bucket_coverage ( - source_id, granularity, bucket_start, bucket_end, - coverage_state, observed_units, expected_units, rejected_units - ) VALUES (?1, '5m', ?2, ?3, 'unknown', 0, 1, 0)", - params![source_id, bucket_start, bucket_start + FIVE_MINUTES], - ) - .unwrap(); - } - let pipeline = ResolvedPipeline { - database_path: temporary.path().join("netflow.sqlite"), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: capture_root, - source_ids: vec!["r1".into()], - sources: Vec::new(), - start_date: "2025-01-01".into(), - end_date: Some("2025-01-01".into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: true, - }; - - assert_eq!( - count_incomplete_requested_coverage(&connection, &pipeline).unwrap(), - 288 - ); - } - - #[test] - fn open_ended_native_strict_coverage_includes_the_discovered_latest_day() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let day = root.join("edge/2025/06/01"); - fs::create_dir_all(&day).unwrap(); - fs::write(day.join("nfcapd.202506010000"), b"capture").unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let start = parse_date_start("2025-06-01", "UTC").unwrap(); - connection - .execute( - "INSERT INTO bucket_coverage ( - source_id, granularity, bucket_start, bucket_end, - coverage_state, observed_units, expected_units, rejected_units - ) VALUES ('edge', '5m', ?1, ?2, 'complete', 1, 1, 0)", - params![start, start + FIVE_MINUTES], - ) - .unwrap(); - let pipeline = ResolvedPipeline { - database_path: temporary.path().join("unused.sqlite"), - control_paths: Vec::new(), - timezone: "UTC".into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root, - source_ids: vec!["edge".into()], - sources: Vec::new(), - start_date: "2025-06-01".into(), - end_date: None, - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: true, - }; - - assert_eq!( - count_incomplete_requested_coverage(&connection, &pipeline).unwrap(), - 287 - ); - } - - #[test] - fn persisted_sibling_validation_is_cached_per_source_day_but_rejects_foreign_rows() { - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let bucket = |bucket_start| { - StatisticalBucket::dense(BucketKey::new( - "r1", - Granularity::FiveMinutes, - bucket_start, - bucket_start + FIVE_MINUTES, - )) - .finish_owned() - }; - - let mut aggregates = AggregateBuckets::default(); - for index in 0..288_i64 { - let child = bucket(index * FIVE_MINUTES); - aggregates - .reject_persisted_siblings(&connection, &child, "UTC") - .unwrap(); - aggregates.include(&child, "UTC").unwrap(); - } - assert_eq!(aggregates.persisted_sibling_queries, 1); - - let foreign = bucket(FIVE_MINUTES); - write_buckets(&connection, std::slice::from_ref(&foreign), false).unwrap(); - let error = AggregateBuckets::default() - .reject_persisted_siblings(&connection, &bucket(0), "UTC") - .unwrap_err(); - assert!(error.to_string().contains("cannot reopen")); - } - - #[test] - fn strict_coverage_counts_missing_partial_and_dst_successor_rows() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - fs::create_dir_all(root.join("r1")).unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - - let pipeline_for = |date: &str, timezone: &str| ResolvedPipeline { - database_path: temporary.path().join("unused.sqlite"), - control_paths: Vec::new(), - timezone: timezone.into(), - run_maad: false, - nfdump: "nfdump".into(), - nfdump_revision: None, - selection: FlowSelection::default(), - inputs: vec![InputSpec::NfcapdTree { - root_path: root.clone(), - source_ids: vec!["r1".into()], - sources: Vec::new(), - start_date: date.into(), - end_date: Some(date.into()), - start_time: None, - end_time: None, - force: false, - }], - datasets: Vec::new(), - require_complete: true, - }; - let local_starts = |date: &str, timezone: &str| { - let start = parse_date_start(date, timezone).unwrap(); - let end = next_date_start(date, timezone).unwrap(); - let mut starts = Vec::new(); - let mut current = start; - while current < end { - starts.push(current); - current = next_local_five_minute_start(current, timezone).unwrap(); - } - starts - }; - let insert = |start: i64, state: &str, observed: i64, expected: i64| { - connection - .execute( - "INSERT INTO bucket_coverage ( - source_id, granularity, bucket_start, bucket_end, - coverage_state, observed_units, expected_units, rejected_units - ) VALUES ('r1', '5m', ?1, ?2, ?3, ?4, ?5, 0)", - params![start, start + FIVE_MINUTES, state, observed, expected], - ) - .unwrap(); - }; - - let normal = local_starts("2025-01-01", "UTC"); - assert_eq!(normal.len(), 288); - for start in &normal { - insert(*start, "complete", 1, 1); - } - assert_eq!( - count_incomplete_requested_coverage(&connection, &pipeline_for("2025-01-01", "UTC")) - .unwrap(), - 0 - ); - connection - .execute( - "DELETE FROM bucket_coverage WHERE source_id = 'r1' AND bucket_start = ?1", - params![normal[0]], - ) - .unwrap(); - assert_eq!( - count_incomplete_requested_coverage(&connection, &pipeline_for("2025-01-01", "UTC")) - .unwrap(), - 1 - ); - insert(normal[0], "partial", 1, 2); - assert_eq!( - count_incomplete_requested_coverage(&connection, &pipeline_for("2025-01-01", "UTC")) - .unwrap(), - 1 - ); - - connection - .execute("DELETE FROM bucket_coverage", []) - .unwrap(); - let spring = local_starts("2025-03-09", DEFAULT_TIMEZONE); - assert_eq!(spring.len(), 276); - for start in &spring { - insert(*start, "complete", 1, 1); - } - assert_eq!( - count_incomplete_requested_coverage( - &connection, - &pipeline_for("2025-03-09", DEFAULT_TIMEZONE) - ) - .unwrap(), - 0 - ); - - connection - .execute("DELETE FROM bucket_coverage", []) - .unwrap(); - let fall = local_starts("2025-11-02", DEFAULT_TIMEZONE); - assert_eq!( - fall.len(), - 288, - "preserve the current fall-back successor contract" - ); - for start in &fall { - insert(*start, "complete", 1, 1); - } - assert_eq!( - count_incomplete_requested_coverage( - &connection, - &pipeline_for("2025-11-02", DEFAULT_TIMEZONE) - ) - .unwrap(), - 0 - ); - connection - .execute( - "UPDATE bucket_coverage SET coverage_state = 'partial', observed_units = 1, expected_units = 2 - WHERE source_id = 'r1' AND bucket_start = ?1", - params![fall[0]], - ) - .unwrap(); - assert_eq!( - count_incomplete_requested_coverage( - &connection, - &pipeline_for("2025-11-02", DEFAULT_TIMEZONE) - ) - .unwrap(), - 1 - ); - } - - #[test] - fn coarse_rollups_keep_observed_zeroes_without_fabricating_unknown_metrics() { - let unknown = StatisticalBucket::new(BucketKey::new( - "r1", - Granularity::FiveMinutes, - 0, - FIVE_MINUTES, - )) - .with_coverage(BucketCoverage::new(1, 0, 0).unwrap()) - .finish(); - let mut unknown_aggregates = AggregateBuckets::default(); - unknown_aggregates.include(&unknown, "UTC").unwrap(); - let (unknown_rollups, _) = unknown_aggregates.finish(); - assert_eq!(unknown_rollups.len(), 3); - assert!( - unknown_rollups - .iter() - .all(|bucket| bucket.traffic.is_empty()) - ); - - let observed_zero = StatisticalBucket::dense(BucketKey::new( - "r1", - Granularity::FiveMinutes, - 0, - FIVE_MINUTES, - )) - .with_coverage(BucketCoverage::complete_unit()) - .finish(); - let mut observed_aggregates = AggregateBuckets::default(); - observed_aggregates.include(&observed_zero, "UTC").unwrap(); - let (observed_rollups, _) = observed_aggregates.finish(); - assert_eq!(observed_rollups.len(), 3); - assert!(observed_rollups.iter().all(|bucket| { - !bucket.traffic.is_empty() - && bucket.traffic.iter().all(|entry| entry.metrics.flows == 0) - })); - } - - #[test] - fn pipeline_persists_in_process_maad_identity() { - let temporary = tempdir().unwrap(); - let database = temporary.path().join("pipeline.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "UTC", - "run_maad": true, - "inputs": [] - })) - .unwrap(), - ) - .unwrap(); - - run(PipelineRequest::config(&config)).unwrap(); - - let connection = Connection::open(database).unwrap(); - let config_json: Value = connection - .query_row( - "SELECT config_json FROM pipeline_product WHERE singleton = 1", - [], - |row| row.get::<_, String>(0), - ) - .unwrap() - .parse() - .unwrap(); - assert_eq!( - config_json["maad"], - json!({ - "enabled": true, - "backend": "in-process", - "contract_version": 2, - "config": { - "q_min": -0.5, - "q_max": 3.5, - "q_step": 0.125, - "min_prefix_length": 8, - "max_prefix_length": 24, - "full_threshold": 0.05 - } - }) - ); - } - - #[test] - fn config_run_publishes_csv_gaps_as_coverage_only_buckets() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let input = temporary.path().join("flows.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": { - "time_end": "time", - "src_ip": "src", - "dst_ip": "dst", - "protocol": "protocol", - "packets": "packets", - "bytes": "bytes" - }, - "source_id": {"value": "r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &input, - "time,src,dst,protocol,packets,bytes\n\ - 2025-01-15 00:00:00,192.0.2.1,198.51.100.1,6,1,10\n\ - 2025-01-15 00:25:00,192.0.2.2,198.51.100.2,17,2,20\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "UTC", - "run_maad": false, - "inputs": [{ - "input_kind": "csv", - "path": input, - "mapping_path": mapping - }] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(&config)).unwrap(); - - assert_eq!(report.five_minute_buckets, 6); - assert_eq!(report.complete_five_minute_buckets, 2); - assert_eq!(report.partial_five_minute_buckets, 0); - assert_eq!(report.unknown_five_minute_buckets, 4); - assert_eq!(report.rollup_buckets, 3); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m' AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state = 'unknown'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 4 - ); - assert_eq!( - connection - .query_row( - "SELECT flows FROM traffic_stats WHERE granularity = '30m' AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT coverage_state FROM bucket_coverage - WHERE granularity = '30m'", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "partial" - ); - - // Simulate a legacy/in-progress product without planner statistics. The strict run still - // returns its coverage error, but first leaves that inspectable product optimized. - connection.execute("DELETE FROM sqlite_stat1", []).unwrap(); - - let mut strict = PipelineRequest::config(&config); - strict.require_complete = true; - assert!(matches!( - run(strict), - Err(PipelineError::IncompleteCoverage(4)) - )); - assert!( - connection - .query_row("SELECT COUNT(*) FROM sqlite_stat1", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap() - > 0 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity IN ('1h', '1d')", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 20 - ); - } - - /// Run a one-dataset pipeline over optional CSV rows and read back the stored start date. - fn stored_default_start_date(dataset: Value, csv_rows: &str) -> String { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let input = temporary.path().join("flows.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": { - "time_end": "time", - "src_ip": "src", - "dst_ip": "dst", - "protocol": "protocol", - "packets": "packets", - "bytes": "bytes" - }, - "source_id": {"value": "r1"} - })) - .unwrap(), - ) - .unwrap(); - let inputs = if csv_rows.is_empty() { - json!([]) - } else { - fs::write( - &input, - format!("time,src,dst,protocol,packets,bytes\n{csv_rows}"), - ) - .unwrap(); - json!([{"input_kind": "csv", "path": input, "mapping_path": mapping}]) - }; - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "America/Los_Angeles", - "run_maad": false, - "inputs": inputs, - "datasets": [dataset] - })) - .unwrap(), - ) - .unwrap(); - - run(PipelineRequest::config(&config)).unwrap(); - - Connection::open(&database) - .unwrap() - .query_row( - "SELECT default_start_date FROM datasets WHERE id = 'example'", - [], - |row| row.get::<_, String>(0), - ) - .unwrap() - } - - #[test] - fn unset_start_date_becomes_the_earliest_ingested_local_day() { - // 2025-01-15 00:00 UTC is still 2025-01-14 in the pipeline timezone. - assert_eq!( - stored_default_start_date( - json!({"dataset_id": "example", "root_path": "/captures"}), - "2025-01-15 00:00:00,192.0.2.1,198.51.100.1,6,1,10\n", - ), - "2025-01-14" - ); - } - - #[test] - fn configured_start_date_survives_ingestion() { - assert_eq!( - stored_default_start_date( - json!({ - "dataset_id": "example", - "root_path": "/captures", - "default_start_date": "2024-12-25" - }), - "2025-01-15 00:00:00,192.0.2.1,198.51.100.1,6,1,10\n", - ), - "2024-12-25" - ); - } - - #[test] - fn unset_start_date_falls_back_when_nothing_is_ingested() { - assert_eq!( - stored_default_start_date( - json!({"dataset_id": "example", "root_path": "/captures"}), - "" - ), - crate::storage::FALLBACK_DEFAULT_START_DATE - ); - } - - #[test] - fn csv_tree_files_share_rollups_within_one_transaction() { - let temporary = tempdir().unwrap(); - let inputs = temporary.path().join("inputs"); - let mapping = temporary.path().join("mapping.json"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::create_dir(&inputs).unwrap(); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header":true, - "timestamp_format":"datetime", - "timestamp_timezone":"UTC", - "columns":{"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id":{"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - inputs.join("2025-01-a.csv"), - "time,src,dst\n\ - 2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n\ - 2025-01-15 11:55:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - fs::write( - inputs.join("2025-01-b.csv"), - "time,src,dst\n\ - 2025-01-15 12:00:00,192.0.2.3,198.51.100.3\n\ - 2025-01-15 23:55:00,192.0.2.4,198.51.100.4\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "run_maad":false, - "inputs":[{ - "input_kind":"csv_tree", - "root_path":inputs, - "mapping_path":mapping - }] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.input_scans, 2); - assert_eq!(report.five_minute_buckets, 288); - assert_eq!(report.rollup_buckets, 73); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(DISTINCT granularity || ':' || bucket_start) FROM traffic_stats", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 13 - ); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM bucket_coverage", [], |row| row - .get::<_, i64>(0),) - .unwrap(), - 361 - ); - } - - #[test] - fn explicit_csv_files_share_rollups_within_one_transaction() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("a.csv"); - let second = temporary.path().join("b.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header":true, - "timestamp_format":"datetime", - "timestamp_timezone":"UTC", - "columns":{"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id":{"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &first, - "time,src,dst\n\ - 2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n\ - 2025-01-15 00:10:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - fs::write( - &second, - "time,src,dst\n\ - 2025-01-15 00:15:00,192.0.2.3,198.51.100.3\n\ - 2025-01-15 00:25:00,192.0.2.4,198.51.100.4\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "run_maad":false, - "inputs":[ - {"input_kind":"csv", "path":first, "mapping_path":mapping}, - {"input_kind":"csv", "path":second, "mapping_path":mapping} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.input_scans, 2); - assert_eq!(report.five_minute_buckets, 6); - assert_eq!(report.rollup_buckets, 3); - } - - #[test] - fn overlapping_csv_batch_merges_metrics_and_coverage() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("first.csv"); - let second = temporary.path().join("second.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": {"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id": {"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - for path in [&first, &second] { - fs::write( - path, - "time,src,dst\n2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n", - ) - .unwrap(); - } - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone":"UTC", - "run_maad":false, - "inputs":[ - {"input_kind":"csv", "path":first, "mapping_path":mapping}, - {"input_kind":"csv", "path":second, "mapping_path":mapping} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(&config)).unwrap(); - assert_eq!(report.five_minute_buckets, 1); - assert_eq!(report.complete_five_minute_buckets, 1); - assert_eq!(report.partial_five_minute_buckets, 0); - assert_eq!(report.unknown_five_minute_buckets, 0); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT flows FROM traffic_stats - WHERE granularity = '5m' AND source_id = 'r1' - AND bucket_start = 1736899200 AND ip_version = 4 - AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT expected_units, observed_units, rejected_units - FROM bucket_coverage - WHERE granularity = '5m' AND source_id = 'r1' - AND bucket_start = 1736899200", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }, - ) - .unwrap(), - (1, 1, 0) - ); - } +fn next_date_start(raw: &str, timezone: &str) -> Result { + let date: Date = raw + .parse() + .map_err(|error: jiff::Error| PipelineError::Time(error.to_string()))?; + Ok(date + .tomorrow() + .and_then(|date| date.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second()) +} - #[test] - fn csv_files_fill_unknown_buckets_across_global_source_envelope() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("first.csv"); - let second = temporary.path().join("second.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": {"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id": {"value": "r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &first, - "time,src,dst\n2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n", - ) - .unwrap(); - fs::write( - &second, - "time,src,dst\n2025-01-15 00:25:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "UTC", - "run_maad": false, - "inputs": [ - {"input_kind": "csv", "path": first, "mapping_path": mapping}, - {"input_kind": "csv", "path": second, "mapping_path": mapping} - ] - })) - .unwrap(), - ) - .unwrap(); +fn parse_local_datetime(raw: &str, timezone: &str) -> Result { + let normalized = if raw.len() == 16 { + format!("{raw}:00") + } else { + raw.to_owned() + }; + let datetime = normalized + .parse::() + .map_err(|error| PipelineError::Time(error.to_string()))?; + Ok(datetime + .in_tz(timezone) + .map_err(|error| PipelineError::Time(error.to_string()))? + .timestamp() + .as_second()) +} - let report = run(PipelineRequest::config(&config)).unwrap(); - assert_eq!(report.five_minute_buckets, 6); - assert_eq!(report.complete_five_minute_buckets, 2); - assert_eq!(report.partial_five_minute_buckets, 0); - assert_eq!(report.unknown_five_minute_buckets, 4); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'r1' AND granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 6 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'r1' AND granularity = '5m' - AND coverage_state = 'unknown'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 4 - ); +fn validate_window( + selected_start: i64, + selected_end: i64, + start: i64, + end: i64, + timezone: &str, +) -> Result<(), PipelineError> { + if start < selected_start { + return Err(PipelineError::InvalidConfig( + "start_time must be on or after the selected start_date".into(), + )); + } + if end > selected_end { + return Err(PipelineError::InvalidConfig( + "end_time must be on or before the selected end_date window".into(), + )); } + if start >= end { + return Err(PipelineError::InvalidConfig( + "input time window must be non-empty".into(), + )); + } + for (label, value) in [("start_time", start), ("end_time", end)] { + if aggregate_bounds(value, Granularity::OneDay, timezone)?.0 != value { + return Err(PipelineError::InvalidConfig(format!( + "{label} must align to a local-day boundary so aggregate rows stay complete" + ))); + } + } + Ok(()) +} - #[test] - fn fall_back_hours_have_distinct_hour_rollups() { - let first = "2024-11-03T01:15:00-07:00[America/Los_Angeles]" - .parse::() - .unwrap() - .timestamp() - .as_second(); - let second = "2024-11-03T01:15:00-08:00[America/Los_Angeles]" - .parse::() - .unwrap() - .timestamp() - .as_second(); +fn expected_nfcapd_path( + root: &Path, + member: &str, + bucket_start: i64, + timezone: &str, +) -> Result { + let timestamp = Timestamp::from_second(bucket_start) + .and_then(|timestamp| timestamp.in_tz(timezone)) + .map_err(|error| PipelineError::Time(error.to_string()))?; + Ok(root + .join(member) + .join(timestamp.strftime("%Y").to_string()) + .join(timestamp.strftime("%m").to_string()) + .join(timestamp.strftime("%d").to_string()) + .join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M")))) +} - let first_bounds = - aggregate_bounds(first, Granularity::OneHour, "America/Los_Angeles").unwrap(); - let second_bounds = - aggregate_bounds(second, Granularity::OneHour, "America/Los_Angeles").unwrap(); +#[cfg(test)] +mod tests { + use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + }; - assert_eq!(first_bounds.0, first - 15 * 60); - assert_eq!(second_bounds.0, second - 15 * 60); - assert_ne!(first_bounds, second_bounds); - } + use rusqlite::Connection; + use serde_json::{Value, json}; + use tempfile::tempdir; - #[test] - fn tree_windows_must_cover_complete_selected_local_days() { - let selected_start = parse_date_start("2025-02-11", "America/Los_Angeles").unwrap(); - let selected_end = next_date_start("2025-02-11", "America/Los_Angeles").unwrap(); - - validate_window( - selected_start, - selected_end, - selected_start, - selected_end, - "America/Los_Angeles", - ) - .unwrap(); - assert!( - validate_window( - selected_start, - selected_end, - selected_start + FIVE_MINUTES, - selected_end, - "America/Los_Angeles", - ) - .unwrap_err() - .to_string() - .contains("local-day boundary") - ); - assert!( - validate_window( - selected_start, - selected_end, - selected_start - 86_400, - selected_end, - "America/Los_Angeles", - ) - .unwrap_err() - .to_string() - .contains("on or after") - ); - } + use super::*; - #[test] - fn later_partial_scan_cannot_reopen_persisted_rollups() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("first.csv"); - let second = temporary.path().join("second.csv"); - let database = temporary.path().join("netflow.sqlite"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header":true, - "timestamp_format":"datetime", - "timestamp_timezone":"UTC", - "columns":{"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id":{"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); + fn write_fake_nfdump(executable: &Path, invocation_log: &Path) { + let stream_path = executable.with_extension("stream"); + let empty_stream_path = executable.with_extension("empty-stream"); + let mut stream = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); + let record = 16; + stream[record + 32..record + 40].copy_from_slice(&20_u64.to_le_bytes()); + stream[record + 40..record + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + stream[record + 48..record + 56].copy_from_slice(&3_u64.to_le_bytes()); + stream[record + 64..record + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + stream[record + 69] = 0b010; + fs::write(&stream_path, stream).unwrap(); fs::write( - &first, - "time,src,dst\n2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n", + &empty_stream_path, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], ) .unwrap(); fs::write( - &second, - "time,src,dst\n2025-01-15 00:05:00,192.0.2.2,198.51.100.2\n", + executable, + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + empty_stream_path.display(), + invocation_log.display(), + stream_path.display(), + ), ) .unwrap(); - for (index, input) in [&first, &second].into_iter().enumerate() { - let config = temporary.path().join(format!("pipeline-{index}.json")); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "run_maad":false, - "inputs":[{"input_kind":"csv", "path":input, "mapping_path":mapping}] - })) - .unwrap(), - ) - .unwrap(); - if index == 0 { - run(PipelineRequest::config(config)).unwrap(); - } else { - let error = run(PipelineRequest::config(config)).unwrap_err(); - assert!(error.to_string().contains("cannot reopen")); - } - } - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 1 - ); + fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); } - #[test] - fn unchanged_file_revision_reuses_the_persisted_digest() { - let temporary = tempdir().unwrap(); - let path = temporary.path().join("nfcapd.202501010000"); - fs::write(&path, "original").unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let snapshot = FileSnapshot::capture(&path).unwrap(); - let locator = path.to_string_lossy().into_owned(); - let original = InputRevision::create("nfcapd", &locator, "digest", "decoder").unwrap(); - upsert_input_bucket( - &connection, - &InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: locator.clone(), - scan_locator: locator.clone(), - source_id: "r1".into(), - bucket_start: 0, - bucket_end: FIVE_MINUTES, - revision: original.clone(), - file_snapshot: Some(snapshot), - }, - false, - ) - .unwrap(); - mark_input_bucket_status( - &connection, - InputKind::Nfcapd, - &locator, - "r1", - 0, - InputStatus::Processed, - &original, - None, - ) - .unwrap(); - - let (cached, _) = prepare_file_revision_with( - &connection, - &path, - InputKind::Nfcapd, - "decoder".into(), - || panic!("unchanged input must not be rehashed"), - ) - .unwrap(); - assert_eq!(cached, original); - - fs::write(&path, "replacement with a different size").unwrap(); - let rehashed = std::cell::Cell::new(false); - let (changed, _) = prepare_file_revision_with( - &connection, - &path, - InputKind::Nfcapd, - "decoder".into(), - || { - rehashed.set(true); - capture_file_revision(&path) - }, - ) - .unwrap(); - assert!(rehashed.get()); - assert_ne!(changed.content_fingerprint, original.content_fingerprint); + fn write_nfcapd_day(root: &Path, member: &str, date: &str) { + let mut bucket_start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); + let end = next_date_start(date, DEFAULT_TIMEZONE).unwrap(); + while bucket_start < end { + let timestamp = Timestamp::from_second(bucket_start) + .unwrap() + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + let path = root + .join(member) + .join(timestamp.strftime("%Y").to_string()) + .join(timestamp.strftime("%m").to_string()) + .join(timestamp.strftime("%d").to_string()) + .join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"capture").unwrap(); + bucket_start = next_local_five_minute_start(bucket_start, DEFAULT_TIMEZONE).unwrap(); + } } - #[test] - fn force_nfcapd_revision_resolution_rehashes_matching_snapshots() { - let temporary = tempdir().unwrap(); - let path = temporary.path().join("nfcapd.202501010000"); - fs::write(&path, "actual bytes").unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let snapshot = FileSnapshot::capture(&path).unwrap(); - let locator = path.to_string_lossy().into_owned(); - let stale = InputRevision::create("nfcapd", &locator, "stale", "decoder").unwrap(); - upsert_input_bucket( - &connection, - &InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: locator.clone(), - scan_locator: locator.clone(), - source_id: "r1".into(), - bucket_start: 0, - bucket_end: FIVE_MINUTES, - revision: stale, - file_snapshot: Some(snapshot), - }, - false, - ) - .unwrap(); - mark_input_bucket_status( - &connection, - InputKind::Nfcapd, - &locator, - "r1", - 0, - InputStatus::Processed, - &InputRevision::create("nfcapd", &locator, "stale", "decoder").unwrap(), - None, - ) - .unwrap(); - - let sources = [DatasetSource { - source_id: "r1".into(), - members: vec!["r1".into()], - }]; - let paths = BTreeMap::from([(("r1".into(), 0), path.clone())]); - let bounds = BTreeMap::from([("r1".into(), (0, 0))]); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(1) - .build() - .unwrap(); - let normal_context = NfcapdRevisionContext { - connection: &connection, - sources: &sources, - by_member_and_start: &paths, - member_bounds: &bounds, - extend_gaps_to_window: false, + fn coordinated_request( + registry: PathBuf, + executable: &Path, + start_date: &str, + end_date: &str, + ) -> PipelineRequest { + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some(start_date.into()), + end_date: Some(end_date.into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), force: false, - decoder_fingerprint: "decoder".into(), - capture_snapshots: &BTreeMap::new(), - revision_pool: &pool, - }; - let cached = resolve_nfcapd_batch_revisions(&normal_context, &[0]) - .unwrap() - .remove(&path) - .unwrap(); - assert_eq!(cached.revision.content_fingerprint, "stale"); - - let forced_context = NfcapdRevisionContext { - force: true, - ..normal_context - }; - let forced = resolve_nfcapd_batch_revisions(&forced_context, &[0]) - .unwrap() - .remove(&path) - .unwrap(); - let (actual_digest, _) = capture_file_revision(&path).unwrap(); - assert_eq!(forced.revision.content_fingerprint, actual_digest); + run_maad: false, + require_complete: false, + } } - #[cfg(unix)] - #[test] - fn nfcapd_tree_commits_completed_days_before_a_later_day_fails() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let first = root.join("r1/2025/01/01/nfcapd.202501010000"); - let second = root.join("r1/2025/01/02/nfcapd.202501020000"); - fs::create_dir_all(first.parent().unwrap()).unwrap(); - fs::create_dir_all(second.parent().unwrap()).unwrap(); - fs::write(&first, "first").unwrap(); - fs::write(&second, "second").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, "case \"$*\" in *20250102*) exit 9;; esac"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "source_ids":["r1"], - "start_date":"2025-01-01", - "end_date":"2025-01-02" - }] - })) - .unwrap(), - ) - .unwrap(); - - let error = run(PipelineRequest::config(config)).unwrap_err(); - let message = error.to_string(); - assert!(message.contains("nfcapd decode failed"), "{message}"); - assert!(message.contains("member \"r1\""), "{message}"); - assert!( - message.contains(&second.to_string_lossy().to_string()), - "{message}" - ); - let connection = Connection::open(database).unwrap(); - let starts = connection - .prepare("SELECT bucket_start FROM processed_inputs ORDER BY bucket_start") - .unwrap() - .query_map([], |row| row.get::<_, i64>(0)) - .unwrap() - .collect::>>() - .unwrap(); - assert_eq!(starts, [1_735_689_600]); + fn daily_selection(prefix: &str) -> FlowSelection { + selection_from_value(&json!({ + "kind": "daily_active_sources", + "ip_prefix": prefix, + })) + .unwrap() } - #[cfg(unix)] - #[test] - fn explicit_nfcapd_files_share_one_transaction() { - let temporary = tempdir().unwrap(); - let first = temporary.path().join("nfcapd.202501010000"); - let second = temporary.path().join("nfcapd.202501010005"); - fs::write(&first, "first").unwrap(); - fs::write(&second, "second").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[ - {"input_kind":"nfcapd", "path":first, "source_id":"r1"}, - {"input_kind":"nfcapd", "path":second, "source_id":"r1"} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); + fn resolved_pipeline( + root: &Path, + database: PathBuf, + member: &str, + timezone: &str, + ) -> ResolvedPipeline { + ResolvedPipeline { + database_path: database, + timezone: timezone.into(), + run_maad: false, + nfdump: PathBuf::from("/bin/true"), + nfdump_revision: None, + selection: daily_selection("192.0.0.0/16"), + inputs: vec![InputSpec::NfcapdTree { + root_path: root.to_owned(), + source_ids: vec![member.into()], + sources: Vec::new(), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + force: false, + }], + datasets: Vec::new(), + require_complete: false, + } + } - assert_eq!(report.five_minute_buckets, 2); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 2 - ); + fn incompatibility(pipelines: Vec) -> String { + match validate_compatible_pipelines(pipelines) { + Ok(_) => panic!("pipelines unexpectedly compatible"), + Err(error) => error.to_string(), + } } - #[cfg(unix)] #[test] - fn fall_back_tree_uses_wall_clock_filenames_without_repeating_one_am() { + fn coordinated_compatibility_rejects_duplicate_ids_roots_layouts_timezones_and_outputs() { let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let capture = root.join("r1/2025/11/02/nfcapd.202511020100"); - fs::create_dir_all(capture.parent().unwrap()).unwrap(); - fs::write(&capture, "capture").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"America/Los_Angeles", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "source_ids":["r1"], - "start_date":"2025-11-02", - "end_date":"2025-11-02" - }] - })) - .unwrap(), + let first_root = temporary.path().join("first-root"); + let second_root = temporary.path().join("second-root"); + for path in [ + first_root.join("edge"), + first_root.join("other"), + second_root.join("edge"), + ] { + fs::create_dir_all(path).unwrap(); + } + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + + let duplicate = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(temporary.path().join("unused.json")), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: "/bin/true".into(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["same".into(), "same".into()], ) - .unwrap(); + .unwrap_err(); + assert!(duplicate.to_string().contains("cannot repeat dataset")); - let report = run(PipelineRequest::config(config)).unwrap(); + let roots = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&second_root, second_db.clone(), "edge", DEFAULT_TIMEZONE), + ]); + assert!(roots.contains("same nfcapd root"), "{roots}"); - assert_eq!(report.five_minute_buckets, 288); - assert_eq!(report.rollup_buckets, 73); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(DISTINCT bucket_start) FROM processed_inputs", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "missing inputs are evidence, not synthetic processed revisions" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state = 'unknown'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 287 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 10, - "unknown buckets must not fabricate zero-valued metric rows" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '1d'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 10 - ); + let layouts = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&first_root, second_db.clone(), "other", DEFAULT_TIMEZONE), + ]); + assert!(layouts.contains("same logical source layout"), "{layouts}"); + + let timezones = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&first_root, second_db.clone(), "edge", "UTC"), + ]); + assert!(timezones.contains("same timezone"), "{timezones}"); + + let outputs = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&first_root, first_db, "edge", DEFAULT_TIMEZONE), + ]); + assert!(outputs.contains("must be distinct"), "{outputs}"); } - #[cfg(unix)] #[test] - fn newly_arrived_member_repairs_five_minute_coverage_without_erasing_disappeared_inputs() { + fn auto_discovered_nfcapd_root_rejects_output_that_would_create_a_member_directory() { let temporary = tempdir().unwrap(); let root = temporary.path().join("captures"); - let first = root.join("r1/2025/01/01/nfcapd.202501010000"); - let second = root.join("r2/2025/01/01/nfcapd.202501010000"); - fs::create_dir_all(first.parent().unwrap()).unwrap(); - fs::create_dir_all(second.parent().unwrap()).unwrap(); - fs::write(&first, "first").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); + fs::create_dir_all(root.join("edge")).unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, &temporary.path().join("invocations")); + let database = root.join("future-member/netflow.sqlite"); let config = temporary.path().join("pipeline.json"); fs::write( &config, serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "sources":[{"source_id":"both", "members":["r1", "r2"]}], - "start_date":"2025-01-01", - "end_date":"2025-01-01" + "database_path": database, + "timezone": DEFAULT_TIMEZONE, + "nfdump": executable, + "inputs": [{ + "input_kind": "nfcapd_tree", + "root_path": root, + "start_date": "2025-06-01", + "end_date": "2025-06-01" }] })) .unwrap(), ) .unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT coverage_state FROM bucket_coverage - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "partial" - ); - drop(connection); - - fs::write(&second, "second").unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT coverage_state FROM bucket_coverage - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "complete" - ); - assert_eq!( - connection - .query_row( - "SELECT flows FROM traffic_stats - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600 AND ip_version = 4 - AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats - WHERE source_id = 'both' AND granularity <> '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "repair invalidates coarse derived rows that cannot be patched exactly" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'both' AND granularity <> '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 73, - "capture coverage remains available when derived metrics are invalidated" - ); - drop(connection); - - fs::remove_file(&first).unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT observed_units FROM bucket_coverage - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2, - "a disappearing file must not erase prior observations" + let error = run(PipelineRequest::config(&config)).unwrap_err(); + assert!( + error + .to_string() + .contains("overlaps the nfcapd capture tree") ); + assert!(!database.exists()); } - #[cfg(unix)] #[test] - fn implicit_tree_end_models_only_each_members_observed_bounds() { + fn dataset_mode_applies_its_persisted_selection() { let temporary = tempdir().unwrap(); let root = temporary.path().join("captures"); - let first = root.join("r1/2025/01/01/nfcapd.202501010000"); - let second = root.join("r2/2025/01/02/nfcapd.202501020000"); - fs::create_dir_all(first.parent().unwrap()).unwrap(); - fs::create_dir_all(second.parent().unwrap()).unwrap(); - fs::write(&first, "first").unwrap(); - fs::write(&second, "second").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); + fs::create_dir_all(root.join("edge")).unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, &temporary.path().join("invocations")); + let registry = temporary.path().join("datasets.json"); + let database = temporary.path().join("active.sqlite"); fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "source_ids":["r1", "r2"], - "start_date":"2025-01-01" - }] - })) + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "72.5.0.0/16" + } + }])) .unwrap(), ) .unwrap(); - let report = run(PipelineRequest::config(config)).unwrap(); + let resolved = resolve_request(&PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: true, + require_complete: false, + }) + .unwrap(); - assert_eq!(report.five_minute_buckets, 2); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 2 - ); + assert!(resolved.selection.selects_daily_active_sources()); + assert_eq!(resolved.database_path, database); } - #[cfg(unix)] #[test] - fn completed_nfcapd_input_is_skipped_before_running_decoder_again() { + fn coordinated_products_share_decode_and_resume_by_whole_day() { let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202504151200"); - let decoder = temporary.path().join("fake-nfdump"); - let calls = temporary.path().join("calls"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write(&capture, "fixture").unwrap(); - write_fake_nfdump(&decoder, &format!("echo called >> '{}'", calls.display())); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root, "edge", "2025-06-01"); + write_nfcapd_day(&root, "edge", "2025-06-02"); + let executable = temporary.path().join("fake-nfdump"); + let invocation_log = temporary.path().join("invocations"); + write_fake_nfdump(&executable, &invocation_log); + let registry = temporary.path().join("datasets.json"); + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"America/Los_Angeles", - "nfdump":decoder, - "run_maad":false, - "inputs":[{"input_kind":"nfcapd", "path":capture, "source_id":"r1"}] - })) + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + } + }, + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16" + } + } + ])) .unwrap(), ) .unwrap(); + let request = coordinated_request(registry, &executable, "2025-06-01", "2025-06-02"); - run(PipelineRequest::config(&config)).unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - - assert_eq!(fs::read_to_string(calls).unwrap().lines().count(), 1); - } + let initial = run_many(request.clone(), vec!["first".into(), "second".into()]).unwrap(); + assert_eq!(initial.five_minute_buckets, 1_152); + let initial_invocations = fs::read_to_string(&invocation_log).unwrap().lines().count(); + assert_eq!(initial_invocations, 578); - #[cfg(unix)] - #[test] - fn nfdump_revision_conflicts_before_a_second_binary_can_mix_product_rows() { - let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202504151200"); - let first_decoder = temporary.path().join("nfdump-a"); - let second_decoder = temporary.path().join("nfdump-b"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write(&capture, "fixture").unwrap(); - write_fake_nfdump(&first_decoder, ""); - write_fake_nfdump(&second_decoder, ""); - let write_config = |decoder: &Path| { - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "America/Los_Angeles", - "nfdump": decoder, - "run_maad": false, - "inputs": [{"input_kind": "nfcapd", "path": capture, "source_id": "r1"}] - })) - .unwrap(), + let first = Connection::open(&first_db).unwrap(); + let second = Connection::open(&second_db).unwrap(); + let first_max = first + .query_row( + "SELECT MAX(flows) FROM traffic_stats WHERE granularity = '5m'", + [], + |row| row.get::<_, Option>(0), ) + .unwrap() .unwrap(); - }; - - write_config(&first_decoder); - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(&database).unwrap(); - let before_inputs: i64 = connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get(0) - }) + let second_max = second + .query_row( + "SELECT MAX(flows) FROM traffic_stats WHERE granularity = '5m'", + [], + |row| row.get::<_, Option>(0), + ) + .unwrap() .unwrap(); - let before_traffic: i64 = connection - .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row.get(0)) + assert!(first_max > 0); + assert_eq!(second_max, 0); + let first_rows = first + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| { + row.get::<_, i64>(0) + }) .unwrap(); - drop(connection); - - write_config(&second_decoder); - let error = run(PipelineRequest::config(&config)).unwrap_err(); - assert!( - error - .to_string() - .contains("Pipeline product identity mismatch") - ); + drop(first); + drop(second); - let connection = Connection::open(database).unwrap(); + let no_op = run_many(request.clone(), vec!["second".into(), "first".into()]).unwrap(); + assert_eq!(no_op.five_minute_buckets, 0); assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - before_inputs + fs::read_to_string(&invocation_log).unwrap().lines().count(), + initial_invocations, ); + + let day_start = parse_date_start("2025-06-02", DEFAULT_TIMEZONE).unwrap(); + let day_end = next_date_start("2025-06-02", DEFAULT_TIMEZONE).unwrap(); + let second = Connection::open(&second_db).unwrap(); + second.execute_batch("BEGIN IMMEDIATE").unwrap(); + delete_stats_time_range(&second, &["edge".to_owned()], day_start, day_end).unwrap(); + second.execute_batch("COMMIT").unwrap(); assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row - .get::<_, i64>(0)) + second + .query_row("SELECT COUNT(*) FROM daily_product_completion", [], |row| { + row.get::<_, i64>(0) + },) .unwrap(), - before_traffic + 1, ); - } - - #[cfg(unix)] - #[test] - fn nfdump_replacement_after_decode_rolls_back_the_native_transaction() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202504151200"); - let decoder = temporary.path().join("nfdump"); - let replacement = temporary.path().join("nfdump-replacement"); - let stream = temporary.path().join("stream.bin"); - let empty_stream = temporary.path().join("empty.stream"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write(&capture, "fixture").unwrap(); - fs::write(&stream, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); - fs::write( - &empty_stream, - [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], - ) - .unwrap(); - fs::write( - &replacement, - format!("#!/bin/sh\ncat '{}'\n", stream.display()), - ) - .unwrap(); - fs::set_permissions(&replacement, fs::Permissions::from_mode(0o755)).unwrap(); - fs::write( - &decoder, - format!( - "#!/bin/sh\nif [ \"$1\" = \"-R\" ]; then cat '{}'; exit 0; fi\ncat '{}'\ncp '{}' \"$0\"\n", - empty_stream.display(), - stream.display(), - replacement.display() - ), - ) - .unwrap(); - fs::set_permissions(&decoder, fs::Permissions::from_mode(0o755)).unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "America/Los_Angeles", - "nfdump": decoder, - "run_maad": false, - "inputs": [{"input_kind": "nfcapd", "path": capture, "source_id": "r1"}] - })) - .unwrap(), - ) - .unwrap(); + drop(second); - let error = run(PipelineRequest::config(&config)).unwrap_err(); - assert!( - error.to_string().contains("nfdump executable changed"), - "{error}" - ); - let connection = Connection::open(database).unwrap(); + let resumed = run_many(request, vec!["first".into(), "second".into()]).unwrap(); + assert_eq!(resumed.five_minute_buckets, 288); assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 0 + fs::read_to_string(&invocation_log).unwrap().lines().count(), + initial_invocations + 289, ); assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| row - .get::<_, i64>(0)) + Connection::open(&first_db) + .unwrap() + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| { + row.get::<_, i64>(0) + }) .unwrap(), - 0 + first_rows, ); + for database in [&first_db, &second_db] { + assert_eq!( + Connection::open(database) + .unwrap() + .query_row("SELECT COUNT(*) FROM daily_product_completion", [], |row| { + row.get::<_, i64>(0) + },) + .unwrap(), + 2, + ); + } } - #[cfg(unix)] #[test] - fn incompatible_nfdump_probe_fails_before_output_setup() { - use std::os::unix::fs::PermissionsExt; - - let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202504151200"); - let decoder = temporary.path().join("incompatible-nfdump"); - let output_directory = temporary.path().join("outputs"); - let database = output_directory.join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write(&capture, "fixture").unwrap(); - fs::write(&decoder, "#!/bin/sh\nexit 0\n").unwrap(); - fs::set_permissions(&decoder, fs::Permissions::from_mode(0o755)).unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "America/Los_Angeles", - "nfdump": decoder, - "run_maad": false, - "inputs": [{"input_kind": "nfcapd", "path": capture, "source_id": "r1"}] - })) - .unwrap(), - ) - .unwrap(); - - let error = run(PipelineRequest::config(&config)).unwrap_err(); - assert!(error.to_string().contains("compatibility probe"), "{error}"); - assert!(!output_directory.exists()); - assert!(!database_operation_lock_path(&database).unwrap().exists()); + fn local_day_iteration_handles_both_dst_transitions() { + for (date, expected) in [("2025-03-09", 276), ("2025-11-02", 288)] { + let start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); + let end = next_date_start(date, DEFAULT_TIMEZONE).unwrap(); + let mut bucket_start = start; + let mut count = 0; + while bucket_start < end { + count += 1; + bucket_start = + next_local_five_minute_start(bucket_start, DEFAULT_TIMEZONE).unwrap(); + } + assert_eq!(count, expected, "{date}"); + assert_eq!(bucket_start, end, "{date}"); + } } } diff --git a/tools/netflow-db/src/provenance.rs b/tools/netflow-db/src/provenance.rs index 867d12d..76c93a7 100644 --- a/tools/netflow-db/src/provenance.rs +++ b/tools/netflow-db/src/provenance.rs @@ -160,7 +160,6 @@ impl FileSnapshot { } } -#[cfg(unix)] fn snapshot_from_metadata( path: &Path, metadata: &fs::Metadata, @@ -178,32 +177,6 @@ fn snapshot_from_metadata( }) } -#[cfg(not(unix))] -fn snapshot_from_metadata( - path: &Path, - metadata: &fs::Metadata, -) -> Result { - use std::time::UNIX_EPOCH; - - let modified = metadata - .modified() - .map_err(|source| ProvenanceError::Io { - context: format!("failed to read input timestamp: {}", path.display()), - source, - })? - .duration_since(UNIX_EPOCH) - .map_err(|_| ProvenanceError::TimestampOverflow(path.to_path_buf()))?; - let mtime_ns = i64::try_from(modified.as_nanos()) - .map_err(|_| ProvenanceError::TimestampOverflow(path.to_path_buf()))?; - Ok(FileSnapshot { - device: 0, - inode: 0, - size: metadata.len(), - mtime_ns, - ctime_ns: mtime_ns, - }) -} - fn timestamp_ns(seconds: i64, nanoseconds: i64, path: &Path) -> Result { seconds .checked_mul(1_000_000_000) @@ -404,12 +377,6 @@ pub fn capture_csv_input_revision( capture_input_revision(path, "csv", csv_decoder_fingerprint(config)?) } -pub fn capture_nfcapd_input_revision( - path: impl AsRef, -) -> Result<(InputRevision, FileSnapshot), ProvenanceError> { - capture_input_revision(path, "nfcapd", nfcapd_decoder_fingerprint()?) -} - fn capture_input_revision( path: impl AsRef, input_kind: &str, diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index 1fb161c..fd13ca3 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -59,7 +59,9 @@ pub struct WriteBucketsProfile { pub(crate) port_count_insert_elapsed: Duration, pub(crate) maad_elapsed: Duration, pub(crate) address_structure_insert_elapsed: Duration, + #[cfg(test)] pub(crate) write_calls: u64, + #[cfg(test)] pub(crate) bucket_keys: u64, pub(crate) traffic_rows: u64, pub(crate) protocol_rows: u64, @@ -72,29 +74,7 @@ pub struct WriteBucketsProfile { } impl WriteBucketsProfile { - pub(crate) fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.delete_elapsed += profile.delete_elapsed; - self.canonical_rows_elapsed += profile.canonical_rows_elapsed; - self.scalar_rows_elapsed += profile.scalar_rows_elapsed; - self.traffic_insert_elapsed += profile.traffic_insert_elapsed; - self.protocol_insert_elapsed += profile.protocol_insert_elapsed; - self.address_count_insert_elapsed += profile.address_count_insert_elapsed; - self.port_count_insert_elapsed += profile.port_count_insert_elapsed; - self.maad_elapsed += profile.maad_elapsed; - self.address_structure_insert_elapsed += profile.address_structure_insert_elapsed; - self.write_calls += profile.write_calls; - self.bucket_keys += profile.bucket_keys; - self.traffic_rows += profile.traffic_rows; - self.protocol_rows += profile.protocol_rows; - self.address_count_rows += profile.address_count_rows; - self.port_count_rows += profile.port_count_rows; - self.maad_address_sets += profile.maad_address_sets; - self.maad_addresses += profile.maad_addresses; - self.address_structure_rows += profile.address_structure_rows; - self.address_structure_json_bytes += profile.address_structure_json_bytes; - } - + #[cfg(test)] pub(crate) fn other_elapsed(&self) -> Duration { self.total_elapsed.saturating_sub( self.delete_elapsed @@ -172,7 +152,9 @@ pub(crate) fn write_buckets_profiled( ) -> Result { let total_started = Instant::now(); let mut profile = WriteBucketsProfile { + #[cfg(test)] write_calls: 1, + #[cfg(test)] bucket_keys: count(buckets.len()), ..WriteBucketsProfile::default() }; diff --git a/tools/netflow-db/src/registry.rs b/tools/netflow-db/src/registry.rs index 0c89f12..c5672d6 100644 --- a/tools/netflow-db/src/registry.rs +++ b/tools/netflow-db/src/registry.rs @@ -26,12 +26,14 @@ pub enum RegistryError { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DatasetSource { pub source_id: String, pub members: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Dataset { pub dataset_id: String, #[serde(default)] @@ -379,4 +381,31 @@ mod tests { root.path().join("data/uoregon-active-0-220/netflow.sqlite") ); } + + #[test] + fn registry_rejects_unknown_dataset_and_source_fields() { + for (registry_json, unknown_field) in [ + ( + r#"[{"dataset_id":"sample","root_path":"/captures","selecton":null}]"#, + "selecton", + ), + ( + r#"[{"dataset_id":"sample","root_path":"/captures","sources":[{"source_id":"r1","member":["r1"]}]}]"#, + "member", + ), + ] { + let root = tempdir().unwrap(); + let list = root.path().join("datasets.json"); + fs::write(&list, registry_json).unwrap(); + + let error = DatasetRegistry::load(&list, root.path()).unwrap_err(); + + assert!( + error + .to_string() + .contains(&format!("unknown field `{unknown_field}`")), + "unexpected error: {error}" + ); + } + } } diff --git a/tools/netflow-db/src/storage.rs b/tools/netflow-db/src/storage.rs index f52fc96..cc80a1c 100644 --- a/tools/netflow-db/src/storage.rs +++ b/tools/netflow-db/src/storage.rs @@ -1,17 +1,17 @@ //! Concrete SQLite persistence and atomic database publication. use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeSet, fs::{self, File, OpenOptions}, io::{Read, Seek, SeekFrom, Write}, - path::{Component, Path, PathBuf}, + path::{Path, PathBuf}, time::Duration, }; use fs2::FileExt; use rusqlite::{ Connection, OpenFlags, OptionalExtension, Transaction, TransactionBehavior, backup::Backup, - params, params_from_iter, types::ToSql, + params, }; use serde::Serialize; #[cfg(unix)] @@ -35,18 +35,6 @@ pub const STATS_TABLE_NAMES: [&str; 6] = [ "bucket_coverage", ]; const STATS_GRANULARITIES: [&str; 4] = ["5m", "30m", "1h", "1d"]; -const DAILY_PRODUCT_COMPLETION_BUCKET_SECONDS: i64 = 300; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DailyProductCompletionState { - /// The day has no completion marker or mutation tombstone and may use legacy recovery. - Missing, - /// The day has a matching completion marker with no post-certification mutation. - Clean, - /// A canonical row changed after the completion marker was written. - Dirty, -} - #[derive(Debug, Error)] pub enum StorageError { #[error("SQLite operation failed: {0}")] @@ -87,7 +75,7 @@ impl DatabaseOperationLock { database_path: impl AsRef, operation: impl Into, ) -> Result { - let database_path = absolute_path(database_path.as_ref())?; + let database_path = canonical_path(database_path.as_ref())?; let operation = operation.into(); let path = database_operation_lock_path(&database_path)?; if let Some(parent) = path.parent() { @@ -138,17 +126,6 @@ fn validate_lock_file(file: &File, path: &Path) -> Result<(), StorageError> { path.display() ))); } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - - if metadata.nlink() != 1 { - return Err(StorageError::InvalidInput(format!( - "database operation lock must not be hard-linked: {}", - path.display() - ))); - } - } Ok(()) } @@ -159,7 +136,7 @@ impl Drop for DatabaseOperationLock { } pub fn database_operation_lock_path(path: impl AsRef) -> Result { - let database = absolute_path(path.as_ref())?; + let database = canonical_path(path.as_ref())?; let name = database.file_name().ok_or_else(|| { StorageError::InvalidInput(format!( "database path has no file name: {}", @@ -169,12 +146,30 @@ pub fn database_operation_lock_path(path: impl AsRef) -> Result) -> Result { - absolute_path(path.as_ref()) + let path = path.as_ref(); + let expanded = if path.is_absolute() { + path.to_owned() + } else { + std::env::current_dir()?.join(path) + }; + match fs::canonicalize(&expanded) { + Ok(path) => Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = expanded.parent().ok_or_else(|| { + StorageError::InvalidInput(format!( + "path has no parent directory: {}", + expanded.display() + )) + })?; + let file_name = expanded.file_name().ok_or_else(|| { + StorageError::InvalidInput(format!("path has no file name: {}", expanded.display())) + })?; + Ok(canonical_path(parent)?.join(file_name)) + } + Err(error) => Err(error.into()), + } } pub fn connect_pipeline_writer(path: impl AsRef) -> Result { @@ -200,7 +195,7 @@ pub fn connect_local_writer(path: impl AsRef) -> Result) -> Result { let connection = Connection::open_with_flags( - absolute_path(path.as_ref())?, + canonical_path(path.as_ref())?, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_millis(BUSY_TIMEOUT_MS))?; @@ -245,125 +240,6 @@ pub fn init_schema(connection: &Connection) -> Result<(), StorageError> { Ok(()) } -fn absolute_path(path: &Path) -> Result { - let expanded = if path.starts_with("~") { - let home = std::env::var_os("HOME").ok_or_else(|| { - StorageError::InvalidInput(format!( - "cannot expand path without a home: {}", - path.display() - )) - })?; - PathBuf::from(home).join(path.strip_prefix("~").expect("prefix checked")) - } else if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir()?.join(path) - }; - - // Resolve components in operating-system order. In particular, `link/..` must apply `..` - // to the link target, rather than to the directory containing the link. Resolving the longest - // existing prefix and normalizing the rest lexically gets that case wrong. This resolver also - // expands dangling symlinks, which is needed before output alias checks can safely derive - // SQLite sidecar and operation-lock paths. - let mut pending = expanded - .components() - .map(|component| match component { - Component::Prefix(prefix) => OwnedPathComponent::Prefix(prefix.as_os_str().to_owned()), - Component::RootDir => OwnedPathComponent::Root, - Component::CurDir => OwnedPathComponent::CurDir, - Component::ParentDir => OwnedPathComponent::ParentDir, - Component::Normal(name) => OwnedPathComponent::Normal(name.to_owned()), - }) - .collect::>(); - let mut resolved = PathBuf::new(); - let mut symlink_count = 0_u8; - - while let Some(component) = pending.pop_front() { - match component { - OwnedPathComponent::Prefix(prefix) => resolved.push(prefix), - OwnedPathComponent::Root => resolved.push(Path::new(std::path::MAIN_SEPARATOR_STR)), - OwnedPathComponent::CurDir => {} - OwnedPathComponent::ParentDir => { - // `PathBuf::pop` already keeps an absolute path at its root. - resolved.pop(); - } - OwnedPathComponent::Normal(name) => { - let candidate = resolved.join(&name); - match fs::symlink_metadata(&candidate) { - Ok(metadata) if metadata.file_type().is_symlink() => { - symlink_count = symlink_count.checked_add(1).ok_or_else(|| { - StorageError::InvalidInput(format!( - "too many symlink components while resolving {}", - path.display() - )) - })?; - let target = fs::read_link(&candidate)?; - let parent = candidate.parent().ok_or_else(|| { - StorageError::InvalidInput(format!( - "cannot resolve symlink component in {}", - path.display() - )) - })?; - if target.is_absolute() { - resolved.clear(); - let target_components = target - .components() - .map(OwnedPathComponent::from) - .collect::>(); - for component in target_components.into_iter().rev() { - pending.push_front(component); - } - } else { - resolved = parent.to_path_buf(); - let target_components = target - .components() - .map(OwnedPathComponent::from) - .collect::>(); - for component in target_components.into_iter().rev() { - pending.push_front(component); - } - } - } - Ok(_) => resolved.push(name), - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory - ) => - { - // A missing suffix is allowed for output databases. Preserve it as an - // absolute path while still resolving any symlink components before it. - resolved.push(name); - } - Err(error) => return Err(error.into()), - } - } - } - } - Ok(resolved) -} - -#[derive(Debug)] -enum OwnedPathComponent { - Prefix(std::ffi::OsString), - Root, - CurDir, - ParentDir, - Normal(std::ffi::OsString), -} - -impl From> for OwnedPathComponent { - fn from(component: Component<'_>) -> Self { - match component { - Component::Prefix(prefix) => Self::Prefix(prefix.as_os_str().to_owned()), - Component::RootDir => Self::Root, - Component::CurDir => Self::CurDir, - Component::ParentDir => Self::ParentDir, - Component::Normal(name) => Self::Normal(name.to_owned()), - } - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum InputKind { Nfcapd, @@ -443,59 +319,6 @@ pub struct InputEvidenceRow { pub revision_fingerprint: Option, } -/// A processed native capture row used to rebuild bounded resume caches. -/// -/// The row carries both logical-bucket provenance and the file identity used by the content -/// digest cache. Callers should keep these rows scoped to one local day. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ProcessedNfcapdInput { - pub source_id: String, - pub bucket_start: i64, - pub input_locator: String, - pub content_fingerprint: String, - pub revision_fingerprint: String, - pub file_snapshot: Option, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct ResumeQueryCounters { - pub input_evidence: usize, - pub processed_nfcapd: usize, - pub content_fingerprint: usize, -} - -#[cfg(test)] -thread_local! { - static RESUME_QUERY_COUNTERS: std::cell::Cell = - const { std::cell::Cell::new(ResumeQueryCounters { - input_evidence: 0, - processed_nfcapd: 0, - content_fingerprint: 0, - }) }; -} - -#[cfg(test)] -fn count_resume_query(field: impl FnOnce(&mut ResumeQueryCounters)) { - RESUME_QUERY_COUNTERS.with(|counters| { - let mut value = counters.get(); - field(&mut value); - counters.set(value); - }); -} - -#[cfg(not(test))] -fn count_resume_query(_field: impl FnOnce(&mut ResumeQueryCounters)) {} - -#[cfg(test)] -pub(crate) fn reset_resume_query_counters() { - RESUME_QUERY_COUNTERS.with(|counters| counters.set(ResumeQueryCounters::default())); -} - -#[cfg(test)] -pub(crate) fn resume_query_counters() -> ResumeQueryCounters { - RESUME_QUERY_COUNTERS.with(std::cell::Cell::get) -} - impl InputEvidenceRow { #[must_use] pub fn new( @@ -629,7 +452,6 @@ pub fn query_input_evidence( source_id: &str, bucket_start: i64, ) -> Result, StorageError> { - count_resume_query(|counters| counters.input_evidence += 1); let mut statement = connection.prepare( " SELECT source_id, unit_id, bucket_start, bucket_end, input_locator, @@ -677,162 +499,6 @@ pub fn query_input_evidence( .collect() } -fn source_range_query( - source_ids: &[String], - start: i64, - end: i64, -) -> (String, Vec>) { - let placeholders = (1..=source_ids.len()) - .map(|index| format!("?{index}")) - .collect::>() - .join(", "); - let start_parameter = source_ids.len() + 1; - let end_parameter = source_ids.len() + 2; - let values = source_ids - .iter() - .map(|source_id| Box::new(source_id.clone()) as Box) - .chain([ - Box::new(start) as Box, - Box::new(end) as Box, - ]) - .collect(); - ( - format!( - "source_id IN ({placeholders}) AND bucket_start >= ?{start_parameter} AND bucket_start < ?{end_parameter}" - ), - values, - ) -} - -/// Load all native input evidence for one output's local day in one indexed range query. -pub(crate) fn query_input_evidence_range( - connection: &Connection, - source_ids: &[String], - start: i64, - end: i64, -) -> Result, StorageError> { - if source_ids.is_empty() || start >= end { - return Ok(Vec::new()); - } - count_resume_query(|counters| counters.input_evidence += 1); - let (predicate, values) = source_range_query(source_ids, start, end); - let mut statement = connection.prepare(&format!( - "SELECT source_id, unit_id, bucket_start, bucket_end, input_locator, - evidence_state, revision_fingerprint - FROM input_evidence - WHERE {predicate} - ORDER BY source_id, bucket_start, unit_id" - ))?; - let rows = statement - .query_map( - params_from_iter(values.iter().map(|value| value.as_ref())), - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, String>(4)?, - row.get::<_, String>(5)?, - row.get::<_, Option>(6)?, - )) - }, - )? - .collect::>>()?; - rows.into_iter() - .map( - |( - source_id, - unit_id, - bucket_start, - bucket_end, - input_locator, - evidence_state, - revision_fingerprint, - )| { - Ok(InputEvidenceRow { - source_id, - unit_id, - bucket_start, - bucket_end, - input_locator, - evidence_state: input_evidence_state(&evidence_state)?, - revision_fingerprint, - }) - }, - ) - .collect() -} - -fn file_snapshot_from_columns( - device: Option, - inode: Option, - size: Option, - mtime_ns: Option, - ctime_ns: Option, -) -> Option { - let (Some(device), Some(inode), Some(size), Some(mtime_ns), Some(ctime_ns)) = - (device, inode, size, mtime_ns, ctime_ns) - else { - return None; - }; - (size >= 0).then_some(FileSnapshot { - device: u64::from_ne_bytes(device.to_ne_bytes()), - inode: u64::from_ne_bytes(inode.to_ne_bytes()), - size: u64::try_from(size).ok()?, - mtime_ns, - ctime_ns, - }) -} - -/// Load processed nfcapd provenance and reusable file identities for one output's local day. -/// -/// The query keeps the existing `(source_id, bucket_start)` and source/bucket indexes in use. -/// Callers can derive both logical-bucket revision sets and locator/snapshot digest lookups from -/// the returned rows without issuing one query per bucket or capture. -pub(crate) fn query_processed_nfcapd_range( - connection: &Connection, - source_ids: &[String], - start: i64, - end: i64, -) -> Result, StorageError> { - if source_ids.is_empty() || start >= end { - return Ok(Vec::new()); - } - count_resume_query(|counters| counters.processed_nfcapd += 1); - let (predicate, values) = source_range_query(source_ids, start, end); - let mut statement = connection.prepare(&format!( - "SELECT source_id, bucket_start, input_locator, - content_fingerprint, revision_fingerprint, - file_device, file_inode, file_size, file_mtime_ns, file_ctime_ns - FROM processed_inputs - WHERE input_kind = 'nfcapd' AND status = 'processed' AND {predicate} - ORDER BY source_id, bucket_start, input_locator" - ))?; - statement - .query_map( - params_from_iter(values.iter().map(|value| value.as_ref())), - |row| { - Ok(ProcessedNfcapdInput { - source_id: row.get(0)?, - bucket_start: row.get(1)?, - input_locator: row.get(2)?, - content_fingerprint: row.get(3)?, - revision_fingerprint: row.get(4)?, - file_snapshot: file_snapshot_from_columns( - row.get(5)?, - row.get(6)?, - row.get(7)?, - row.get(8)?, - row.get(9)?, - ), - }) - }, - )? - .collect::>>() - .map_err(StorageError::from) -} - pub fn init_processed_inputs_table(connection: &Connection) -> Result<(), StorageError> { connection.execute_batch( " @@ -1202,7 +868,6 @@ pub fn cached_content_fingerprint( input_locator: &str, file_snapshot: &FileSnapshot, ) -> Result, StorageError> { - count_resume_query(|counters| counters.content_fingerprint += 1); let table = if input_kind == InputKind::Csv { "processed_input_scans" } else { @@ -1292,7 +957,6 @@ pub fn nfcapd_logical_bucket_processed( bucket_start: i64, revisions: &[InputRevision], ) -> Result { - count_resume_query(|counters| counters.processed_nfcapd += 1); if revisions.is_empty() { return Ok(false); } @@ -1855,11 +1519,7 @@ pub fn init_stats_tables(connection: &Connection) -> Result<(), StorageError> { Ok(()) } -/// Initialize the transactionally maintained completion marker for native daily-active products. -/// -/// The marker is deliberately separate from the canonical product tables. Every canonical table -/// has row-level invalidation triggers so direct SQL edits, as well as normal pipeline writes, -/// cannot leave a stale marker looking complete. +/// Initialize the completion marker for native daily-active products. pub fn init_daily_product_completion_table(connection: &Connection) -> Result<(), StorageError> { connection.execute_batch( " @@ -1874,269 +1534,8 @@ pub fn init_daily_product_completion_table(connection: &Connection) -> Result<() ) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS idx_daily_product_completion_source_range ON daily_product_completion(source_id, day_start, day_end); - CREATE TABLE IF NOT EXISTS daily_product_completion_dirty ( - source_id TEXT NOT NULL, - day_start INTEGER NOT NULL, - day_end INTEGER NOT NULL CHECK (day_end > day_start), - dirtied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (source_id, day_start) - ) WITHOUT ROWID; - CREATE INDEX IF NOT EXISTS idx_daily_product_completion_dirty_source_range - ON daily_product_completion_dirty(source_id, day_start, day_end); - CREATE TABLE IF NOT EXISTS daily_product_completion_bucket_guard ( - source_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - day_start INTEGER NOT NULL, - day_end INTEGER NOT NULL CHECK (day_end > day_start), - PRIMARY KEY (source_id, bucket_start) - ) WITHOUT ROWID; - CREATE INDEX IF NOT EXISTS idx_daily_product_completion_bucket_guard_day - ON daily_product_completion_bucket_guard(source_id, day_start, day_end); ", )?; - - // Databases created before the exact-bucket guard existed may already have completion - // markers. Only expand days whose indexed guard count is incomplete. A complete legacy day - // is therefore absent from the recursive CTE entirely, while a partially migrated day still - // gets every missing ownership row. The recursive depth is bounded by one local day. - seed_daily_product_completion_bucket_guards(connection)?; - - for table in STATS_TABLE_NAMES { - connection.execute_batch(&format!( - " - DROP TRIGGER IF EXISTS daily_product_completion_{table}_insert; - DROP TRIGGER IF EXISTS daily_product_completion_{table}_insert_fallback; - DROP TRIGGER IF EXISTS daily_product_completion_{table}_update; - DROP TRIGGER IF EXISTS daily_product_completion_{table}_update_old_fallback; - DROP TRIGGER IF EXISTS daily_product_completion_{table}_update_new_fallback; - DROP TRIGGER IF EXISTS daily_product_completion_{table}_delete; - DROP TRIGGER IF EXISTS daily_product_completion_{table}_delete_fallback; - - CREATE TRIGGER daily_product_completion_{table}_insert - AFTER INSERT ON {table} - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT guard.source_id, guard.day_start, guard.day_end - FROM daily_product_completion_bucket_guard AS guard - JOIN daily_product_completion AS completion - ON completion.source_id = guard.source_id - AND completion.day_start = guard.day_start - AND completion.day_end = guard.day_end - WHERE guard.source_id = NEW.source_id - AND guard.bucket_start = NEW.bucket_start; - END; - - CREATE TRIGGER daily_product_completion_{table}_insert_fallback - AFTER INSERT ON {table} - WHEN NOT EXISTS ( - SELECT 1 - FROM daily_product_completion_bucket_guard - WHERE source_id = NEW.source_id - AND bucket_start = NEW.bucket_start - ) - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT source_id, day_start, day_end - FROM daily_product_completion - WHERE source_id = NEW.source_id - AND day_start <= NEW.bucket_start - AND day_end > NEW.bucket_start; - END; - - CREATE TRIGGER daily_product_completion_{table}_update - AFTER UPDATE ON {table} - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT guard.source_id, guard.day_start, guard.day_end - FROM daily_product_completion_bucket_guard AS guard - JOIN daily_product_completion AS completion - ON completion.source_id = guard.source_id - AND completion.day_start = guard.day_start - AND completion.day_end = guard.day_end - WHERE guard.source_id = OLD.source_id - AND guard.bucket_start = OLD.bucket_start; - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT guard.source_id, guard.day_start, guard.day_end - FROM daily_product_completion_bucket_guard AS guard - JOIN daily_product_completion AS completion - ON completion.source_id = guard.source_id - AND completion.day_start = guard.day_start - AND completion.day_end = guard.day_end - WHERE guard.source_id = NEW.source_id - AND guard.bucket_start = NEW.bucket_start; - END; - - CREATE TRIGGER daily_product_completion_{table}_update_old_fallback - AFTER UPDATE ON {table} - WHEN NOT EXISTS ( - SELECT 1 - FROM daily_product_completion_bucket_guard - WHERE source_id = OLD.source_id - AND bucket_start = OLD.bucket_start - ) - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT source_id, day_start, day_end - FROM daily_product_completion - WHERE source_id = OLD.source_id - AND day_start <= OLD.bucket_start - AND day_end > OLD.bucket_start; - END; - - CREATE TRIGGER daily_product_completion_{table}_update_new_fallback - AFTER UPDATE ON {table} - WHEN NOT EXISTS ( - SELECT 1 - FROM daily_product_completion_bucket_guard - WHERE source_id = NEW.source_id - AND bucket_start = NEW.bucket_start - ) - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT source_id, day_start, day_end - FROM daily_product_completion - WHERE source_id = NEW.source_id - AND day_start <= NEW.bucket_start - AND day_end > NEW.bucket_start; - END; - - CREATE TRIGGER daily_product_completion_{table}_delete - AFTER DELETE ON {table} - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT guard.source_id, guard.day_start, guard.day_end - FROM daily_product_completion_bucket_guard AS guard - JOIN daily_product_completion AS completion - ON completion.source_id = guard.source_id - AND completion.day_start = guard.day_start - AND completion.day_end = guard.day_end - WHERE guard.source_id = OLD.source_id - AND guard.bucket_start = OLD.bucket_start; - END; - - CREATE TRIGGER daily_product_completion_{table}_delete_fallback - AFTER DELETE ON {table} - WHEN NOT EXISTS ( - SELECT 1 - FROM daily_product_completion_bucket_guard - WHERE source_id = OLD.source_id - AND bucket_start = OLD.bucket_start - ) - BEGIN - INSERT OR IGNORE INTO daily_product_completion_dirty (source_id, day_start, day_end) - SELECT source_id, day_start, day_end - FROM daily_product_completion - WHERE source_id = OLD.source_id - AND day_start <= OLD.bucket_start - AND day_end > OLD.bucket_start; - END; - " - ))?; - } - Ok(()) -} - -/// Backfill exact five-minute guard ownership for legacy completion markers. -/// -/// The returned count is useful to callers that need to profile initialization. In particular, -/// a repeat initializer should return zero after all completion days have their expected guards. -fn seed_daily_product_completion_bucket_guards( - connection: &Connection, -) -> Result { - connection.execute( - "WITH RECURSIVE completion_days(source_id, day_start, day_end) AS ( - SELECT completion.source_id, completion.day_start, completion.day_end - FROM daily_product_completion AS completion - WHERE ( - SELECT COUNT(*) - FROM daily_product_completion_bucket_guard AS guard - WHERE guard.source_id = completion.source_id - AND guard.day_start = completion.day_start - AND guard.day_end = completion.day_end - ) < (completion.day_end - completion.day_start) / ?1 - ), - completion_buckets(source_id, day_start, day_end, bucket_start) AS ( - SELECT source_id, day_start, day_end, day_start - FROM completion_days - UNION ALL - SELECT source_id, day_start, day_end, - bucket_start + ?1 - FROM completion_buckets - WHERE bucket_start + ?1 < day_end - ) - INSERT OR IGNORE INTO daily_product_completion_bucket_guard ( - source_id, bucket_start, day_start, day_end - ) - SELECT source_id, bucket_start, day_start, day_end - FROM completion_buckets", - [DAILY_PRODUCT_COMPLETION_BUCKET_SECONDS], - )?; - Ok(usize::try_from(connection.changes()).unwrap_or(usize::MAX)) -} - -/// Provision exact five-minute bucket ownership before publishing canonical rows for one day. -/// -/// Rollups begin on the same five-minute grid, so the ownership rows cover every expected -/// canonical bucket start in the day while keeping trigger invalidation point-lookups bounded. -pub fn provision_daily_product_completion_bucket_guards( - connection: &Connection, - source_ids: &[String], - day_start: i64, - day_end: i64, -) -> Result<(), StorageError> { - if day_start >= day_end { - return Err(StorageError::InvalidInput( - "daily product completion guard requires a non-empty day range".into(), - )); - } - let mut statement = connection.prepare_cached( - "INSERT INTO daily_product_completion_bucket_guard ( - source_id, bucket_start, day_start, day_end - ) VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(source_id, bucket_start) DO UPDATE SET - day_start = excluded.day_start, - day_end = excluded.day_end", - )?; - let mut bucket_start = day_start; - while bucket_start < day_end { - let next_bucket_start = bucket_start - .checked_add(DAILY_PRODUCT_COMPLETION_BUCKET_SECONDS) - .ok_or_else(|| { - StorageError::InvalidInput( - "daily product completion guard exceeds SQLite INTEGER range".into(), - ) - })?; - for source_id in source_ids { - statement.execute(params![source_id, bucket_start, day_start, day_end])?; - } - bucket_start = next_bucket_start; - } - Ok(()) -} - -/// Add one exact ownership row for a canonical bucket that is being published outside a full-day -/// nfcapd transaction, such as a staged CSV bucket. -pub fn ensure_daily_product_completion_bucket_guard( - connection: &Connection, - source_id: &str, - bucket_start: i64, - day_start: i64, - day_end: i64, -) -> Result<(), StorageError> { - if day_start >= day_end || bucket_start < day_start || bucket_start >= day_end { - return Err(StorageError::InvalidInput( - "daily product completion guard bucket must be inside a non-empty day range".into(), - )); - } - connection.execute( - "INSERT INTO daily_product_completion_bucket_guard ( - source_id, bucket_start, day_start, day_end - ) VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(source_id, bucket_start) DO UPDATE SET - day_start = excluded.day_start, - day_end = excluded.day_end", - params![source_id, bucket_start, day_start, day_end], - )?; Ok(()) } @@ -2172,10 +1571,6 @@ pub fn daily_product_completion_matches( SELECT 1 FROM daily_product_completion WHERE source_id = ?1 AND day_start = ?2 AND day_end = ?3 AND product_fingerprint = ?4 AND run_maad = ?5 - AND NOT EXISTS( - SELECT 1 FROM daily_product_completion_dirty - WHERE source_id = ?1 AND day_start = ?2 - ) )", params![ source_id, @@ -2188,55 +1583,6 @@ pub fn daily_product_completion_matches( )? != 0) } -/// Return whether a source/day is clean, dirty, or missing a completion marker. -/// -/// A dirty tombstone is checked independently of the marker so an interrupted or manually -/// altered cleanup cannot turn a previously certified day into a legacy-looking day. -pub fn daily_product_completion_state( - connection: &Connection, - source_id: &str, - day_start: i64, - day_end: i64, - product_fingerprint: &str, - run_maad: bool, -) -> Result { - if day_start >= day_end { - return Err(StorageError::InvalidInput( - "daily product completion requires a non-empty day range".into(), - )); - } - let state = connection.query_row( - "SELECT CASE - WHEN EXISTS( - SELECT 1 FROM daily_product_completion_dirty - WHERE source_id = ?1 AND day_start = ?2 - ) THEN 'dirty' - WHEN EXISTS( - SELECT 1 FROM daily_product_completion - WHERE source_id = ?1 AND day_start = ?2 AND day_end = ?3 - AND product_fingerprint = ?4 AND run_maad = ?5 - ) THEN 'clean' - ELSE 'missing' - END", - params![ - source_id, - day_start, - day_end, - product_fingerprint, - i64::from(run_maad) - ], - |row| row.get::<_, String>(0), - )?; - match state.as_str() { - "clean" => Ok(DailyProductCompletionState::Clean), - "dirty" => Ok(DailyProductCompletionState::Dirty), - "missing" => Ok(DailyProductCompletionState::Missing), - _ => Err(StorageError::InvalidInput(format!( - "invalid daily product completion state: {state:?}" - ))), - } -} - /// Publish or refresh one source/day marker. Callers must invoke this inside the same transaction /// that publishes the day's canonical rows, rollups, and evidence/provenance. pub fn upsert_daily_product_completion( @@ -2252,26 +1598,6 @@ pub fn upsert_daily_product_completion( "daily product completion requires a non-empty day range".into(), )); } - // Legacy marker backfills may not pass through the day publisher. Keep their exact ownership - // map coherent before the marker becomes visible to trigger invalidation, while avoiding a - // second full-day write when the publisher already provisioned the map in this transaction. - let guard_exists = connection.query_row( - "SELECT EXISTS( - SELECT 1 FROM daily_product_completion_bucket_guard - WHERE source_id = ?1 AND bucket_start = ?2 - AND day_start = ?2 AND day_end = ?3 - )", - params![source_id, day_start, day_end], - |row| row.get::<_, i64>(0), - )? != 0; - if !guard_exists { - provision_daily_product_completion_bucket_guards( - connection, - &[source_id.to_owned()], - day_start, - day_end, - )?; - } connection.execute( "INSERT INTO daily_product_completion ( source_id, day_start, day_end, product_fingerprint, run_maad @@ -2289,11 +1615,6 @@ pub fn upsert_daily_product_completion( i64::from(run_maad), ], )?; - connection.execute( - "DELETE FROM daily_product_completion_dirty - WHERE source_id = ?1 AND day_start = ?2", - params![source_id, day_start], - )?; Ok(()) } @@ -2315,16 +1636,6 @@ pub fn delete_daily_product_completion( WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2", params![source_id, start, end], )?; - connection.execute( - "DELETE FROM daily_product_completion_dirty - WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2", - params![source_id, start, end], - )?; - connection.execute( - "DELETE FROM daily_product_completion_bucket_guard - WHERE source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3", - params![source_id, start, end], - )?; } Ok(()) } @@ -3200,7 +2511,7 @@ pub fn promote_database( backup_existing_path: Option<&Path>, ) -> Result<(), StorageError> { let (candidate_path, target_path) = resolved_backup_paths(candidate_path, target_path)?; - let backup_existing_path = backup_existing_path.map(absolute_path).transpose()?; + let backup_existing_path = backup_existing_path.map(canonical_path).transpose()?; let mut database_paths = vec![candidate_path.as_path(), target_path.as_path()]; database_paths.extend(backup_existing_path.as_deref()); validate_database_path_separation(&database_paths)?; @@ -3224,8 +2535,8 @@ fn resolved_backup_paths( source_path: impl AsRef, target_path: impl AsRef, ) -> Result<(PathBuf, PathBuf), StorageError> { - let source_path = absolute_path(source_path.as_ref())?; - let target_path = absolute_path(target_path.as_ref())?; + let source_path = canonical_path(source_path.as_ref())?; + let target_path = canonical_path(target_path.as_ref())?; validate_database_path_separation(&[source_path.as_path(), target_path.as_path()])?; if !source_path.is_file() { return Err(StorageError::DatabaseNotFound(source_path)); @@ -3233,15 +2544,9 @@ fn resolved_backup_paths( Ok((source_path, target_path)) } -/// Reject database paths whose files, SQLite sidecars, or operation locks alias one another. -/// -/// Callers that are about to create directories, locks, or SQLite databases should invoke this -/// before their first mutation. Existing path aliases are compared by both resolved path and, -/// on Unix, device/inode identity so hard links cannot bypass the lexical checks. +/// Reject overlapping database files, SQLite sidecars, and operation locks. pub(crate) fn validate_database_path_separation(paths: &[&Path]) -> Result<(), StorageError> { let mut claimed = Vec::<(PathBuf, PathBuf)>::new(); - #[cfg(unix)] - let mut claimed_identities = BTreeMap::new(); for path in paths { for related in database_related_paths(path)? { if let Some((owner_related, owner)) = claimed.iter().find(|(owner_related, _)| { @@ -3258,40 +2563,20 @@ pub(crate) fn validate_database_path_separation(paths: &[&Path]) -> Result<(), S ))); } claimed.push((related.clone(), (*path).to_owned())); - #[cfg(unix)] - if let Some(identity) = existing_path_identity(&related)? - && let Some(owner) = claimed_identities.insert(identity, (*path).to_owned()) - { - return Err(StorageError::InvalidInput(format!( - "database paths and their SQLite sidecar/operation-lock paths must be distinct: {} aliases {} through device/inode {:?}", - owner.display(), - path.display(), - identity - ))); - } } } Ok(()) } -#[cfg(unix)] -fn existing_path_identity(path: &Path) -> Result, StorageError> { - use std::os::unix::fs::MetadataExt; - - match fs::metadata(path) { - Ok(metadata) => Ok(Some((metadata.dev(), metadata.ino()))), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(error) => Err(error.into()), - } -} - -fn database_related_paths(path: &Path) -> Result, StorageError> { - // Resolve the database itself before deriving related names. For a dangling `alias.sqlite` - // symlink, deriving `alias.sqlite-wal` would miss the real database's sidecar path. - let database = absolute_path(path)?; +pub(crate) fn database_related_paths(path: &Path) -> Result, StorageError> { + // Resolve the database before deriving the names SQLite and the operation lock use beside it. + let database = canonical_path(path)?; let mut paths = vec![database.clone(), database_operation_lock_path(&database)?]; paths.extend(["-journal", "-wal", "-shm"].map(|suffix| sidecar_path(&database, suffix))); - paths.into_iter().map(|path| absolute_path(&path)).collect() + paths + .into_iter() + .map(|path| canonical_path(&path)) + .collect() } fn acquire_database_operation_locks<'a>( @@ -3300,7 +2585,7 @@ fn acquire_database_operation_locks<'a>( ) -> Result, StorageError> { let paths = paths .into_iter() - .map(absolute_path) + .map(canonical_path) .collect::, _>>()?; paths .into_iter() @@ -3361,8 +2646,8 @@ pub fn atomic_replace_sqlite( source_path: impl AsRef, target_path: impl AsRef, ) -> Result<(), StorageError> { - let source_path = absolute_path(source_path.as_ref())?; - let target_path = absolute_path(target_path.as_ref())?; + let source_path = canonical_path(source_path.as_ref())?; + let target_path = canonical_path(target_path.as_ref())?; validate_database_path_separation(&[source_path.as_path(), target_path.as_path()])?; let mut displaced = Vec::new(); for suffix in ["-journal", "-wal", "-shm"] { @@ -3475,22 +2760,6 @@ mod tests { DatabaseOperationLock::acquire(&path, "backup").unwrap(); } - #[cfg(unix)] - #[test] - fn operation_lock_rejects_a_hard_link_to_the_database_before_truncating() { - let directory = tempdir().unwrap(); - let database = directory.path().join("netflow.sqlite"); - fs::write(&database, b"valuable database bytes").unwrap(); - let lock = database_operation_lock_path(&database).unwrap(); - fs::hard_link(&database, &lock).unwrap(); - let original = fs::read(&database).unwrap(); - - let error = DatabaseOperationLock::acquire(&database, "pipeline build").unwrap_err(); - - assert!(error.to_string().contains("must not be hard-linked")); - assert_eq!(fs::read(database).unwrap(), original); - } - #[test] fn product_and_source_layout_bind_once_to_empty_database() { let connection = Connection::open_in_memory().unwrap(); @@ -3670,539 +2939,35 @@ mod tests { } #[test] - fn daily_product_completion_markers_are_invalidated_by_every_canonical_family() { + fn daily_product_completion_matches_the_product_and_maad_configuration() { let connection = Connection::open_in_memory().unwrap(); init_schema(&connection).unwrap(); - let identity = ProductIdentity::create( - &json!({"version": 1}), - &json!({"kind": "daily_active_sources"}), - &json!({"run_maad": false}), - ) - .unwrap(); - bind_product_identity(&connection, &identity, &STATS_TABLE_NAMES).unwrap(); - - let marker = || { - upsert_daily_product_completion( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(); - assert!( - daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() - ); - }; - - let mut traffic = TrafficStatsRow::example(); - traffic.dimensions.source_id = "r1".into(); - insert_traffic_stats_rows(&connection, &[traffic.clone()]).unwrap(); - marker(); - connection.execute_batch("BEGIN IMMEDIATE").unwrap(); - connection - .execute( - "UPDATE traffic_stats SET flows = flows + 1 - WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", - [], - ) - .unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Dirty - ); - connection.execute_batch("ROLLBACK").unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Clean, - "rolling back a canonical mutation must roll back its dirty tombstone" - ); - connection - .execute( - "UPDATE traffic_stats SET flows = flows + 1 - WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", - [], - ) - .unwrap(); - assert!( - !daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() - ); - - marker(); - connection - .execute( - "DELETE FROM traffic_stats - WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", - [], - ) - .unwrap(); - assert!( - !daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() - ); + upsert_daily_product_completion(&connection, "r1", 0, 86_400, "product", false).unwrap(); - let mut protocol = ProtocolStatsRow::example(); - protocol.dimensions.source_id = "r1".into(); - insert_protocol_stats_rows(&connection, &[protocol]).unwrap(); - marker(); - let mut address_count = AddressCountStatsRow::example(); - address_count.dimensions.source_id = "r1".into(); - insert_address_count_stats_rows(&connection, &[address_count]).unwrap(); assert!( - !daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() + daily_product_completion_matches(&connection, "r1", 0, 86_400, "product", false,) + .unwrap() ); - - marker(); - let mut port_count = PortCountStatsRow::example(); - port_count.dimensions.source_id = "r1".into(); - insert_port_count_stats_rows(&connection, &[port_count]).unwrap(); assert!( !daily_product_completion_matches( &connection, "r1", 0, 86_400, - &identity.fingerprint, + "other-product", false, ) .unwrap() ); - - marker(); - let mut address_structure = AddressStructureStatsRow::example(); - address_structure.dimensions.source_id = "r1".into(); - insert_address_structure_stats_rows(&connection, &[address_structure]).unwrap(); assert!( - !daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() - ); - - marker(); - insert_bucket_coverage_rows( - &connection, - &[BucketCoverageRow::new( - "r1", - "5m", - 300, - 600, - BucketCoverage::complete_unit(), - )], - ) - .unwrap(); - assert!( - !daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() - ); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Dirty - ); - - upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) - .unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Clean - ); - - let mut extra = TrafficStatsRow::example(); - extra.dimensions.source_id = "r1".into(); - extra.dimensions.bucket_start = 300; - extra.dimensions.bucket_end = 600; - insert_traffic_stats_rows(&connection, &[extra]).unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Dirty - ); - - upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) - .unwrap(); - connection - .execute( - "DELETE FROM traffic_stats - WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 300", - [], - ) - .unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Dirty - ); - - upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) - .unwrap(); - connection - .execute( - "DELETE FROM daily_product_completion - WHERE source_id = 'r1' AND day_start = 0", - [], - ) - .unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Missing, - "a deliberately removed marker without a dirty tombstone is legacy evidence" + !daily_product_completion_matches(&connection, "r1", 0, 86_400, "product", true,) + .unwrap() ); - upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) - .unwrap(); delete_daily_product_completion(&connection, &["r1".into()], 0, 86_400).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'r1' AND day_start = 0", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "day deletion must clear dirty evidence with the completion marker" - ); - } - - #[test] - fn canonical_mutation_preserves_completion_marker_as_dirty_evidence() { - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let identity = ProductIdentity::create( - &json!({"version": 1}), - &json!({"kind": "daily_active_sources"}), - &json!({"run_maad": false}), - ) - .unwrap(); - bind_product_identity(&connection, &identity, &STATS_TABLE_NAMES).unwrap(); - - let mut traffic = TrafficStatsRow::example(); - traffic.dimensions.source_id = "r1".into(); - insert_traffic_stats_rows(&connection, &[traffic]).unwrap(); - upsert_daily_product_completion(&connection, "r1", 0, 86_400, &identity.fingerprint, false) - .unwrap(); - - connection - .execute( - "UPDATE traffic_stats SET flows = flows + 1 - WHERE source_id = 'r1' AND granularity = '5m' AND bucket_start = 0", - [], - ) - .unwrap(); - - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion - WHERE source_id = 'r1' AND day_start = 0", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "a mutation must retain the completion marker as evidence of prior certification" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_dirty - WHERE source_id = 'r1' AND day_start = 0", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 1, - "a mutation must leave a dirty tombstone" - ); assert!( - !daily_product_completion_matches( - &connection, - "r1", - 0, - 86_400, - &identity.fingerprint, - false, - ) - .unwrap() - ); - } - - #[test] - fn daily_product_completion_guards_keep_normal_writes_point_lookups_and_fallback_off_grid() { - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let identity = ProductIdentity::create( - &json!({"version": 1}), - &json!({"kind": "daily_active_sources"}), - &json!({"run_maad": false}), - ) - .unwrap(); - bind_product_identity(&connection, &identity, &STATS_TABLE_NAMES).unwrap(); - - // Seed a cold-build-shaped history without calling the marker helper for every day. The - // idempotent schema initializer must backfill its exact ownership rows in one pass. - for source_id in ["r1", "r2"] { - for day in 0..394_i64 { - let day_start = day * 86_400; - connection - .execute( - "INSERT INTO daily_product_completion ( - source_id, day_start, day_end, product_fingerprint, run_maad - ) VALUES (?1, ?2, ?3, ?4, 0)", - params![ - source_id, - day_start, - day_start + 86_400, - identity.fingerprint - ], - ) - .unwrap(); - } - } - init_daily_product_completion_table(&connection).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_bucket_guard", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 * 394 * 288 - ); - - // A repeat initializer must not replay the legacy INSERT for any completed day. The - // SQLite total-change counter is sampled around the actual initializer, so trigger DDL - // and planner work cannot hide a second guard write. - let existing_guard = connection - .query_row( - "SELECT day_end FROM daily_product_completion_bucket_guard - WHERE source_id = 'r1' AND bucket_start = 0", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(); - let changes_before = connection.total_changes(); - init_daily_product_completion_table(&connection).unwrap(); - let changes_after = connection.total_changes(); - assert_eq!(changes_after, changes_before); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_bucket_guard", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 * 394 * 288 - ); - assert_eq!( - connection - .query_row( - "SELECT day_end FROM daily_product_completion_bucket_guard - WHERE source_id = 'r1' AND bucket_start = 0", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - existing_guard - ); - - // A partially migrated legacy day remains eligible for the bounded recursive backfill. - connection - .execute( - "INSERT INTO daily_product_completion ( - source_id, day_start, day_end, product_fingerprint, run_maad - ) VALUES ('legacy-partial', ?1, ?2, ?3, 0)", - params![394 * 86_400, 395 * 86_400, identity.fingerprint], - ) - .unwrap(); - connection - .execute( - "INSERT INTO daily_product_completion_bucket_guard ( - source_id, bucket_start, day_start, day_end - ) VALUES ('legacy-partial', ?1, ?2, ?3)", - params![394 * 86_400, 394 * 86_400, 395 * 86_400], - ) - .unwrap(); - init_daily_product_completion_table(&connection).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM daily_product_completion_bucket_guard - WHERE source_id = 'legacy-partial'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288 - ); - - let exact_plan = connection - .prepare( - "EXPLAIN QUERY PLAN - SELECT guard.source_id, guard.day_start, guard.day_end - FROM daily_product_completion_bucket_guard AS guard - JOIN daily_product_completion AS completion - ON completion.source_id = guard.source_id - AND completion.day_start = guard.day_start - AND completion.day_end = guard.day_end - WHERE guard.source_id = ?1 AND guard.bucket_start = ?2", - ) - .unwrap() - .query_map(params!["r1", 393 * 86_400 + 300], |row| { - row.get::<_, String>(3) - }) - .unwrap() - .collect::>>() - .unwrap() - .join("\n"); - assert!(exact_plan.contains("SEARCH guard USING PRIMARY KEY")); - assert!(exact_plan.contains("SEARCH completion USING COVERING INDEX")); - assert!(!exact_plan.contains("SCAN daily_product_completion")); - - let mut normal = TrafficStatsRow::example(); - normal.dimensions.source_id = "r1".into(); - normal.dimensions.bucket_start = 393 * 86_400 + 300; - normal.dimensions.bucket_end = normal.dimensions.bucket_start + 300; - insert_traffic_stats_rows(&connection, &[normal]).unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 393 * 86_400, - 394 * 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Dirty - ); - - upsert_daily_product_completion( - &connection, - "r1", - 393 * 86_400, - 394 * 86_400, - &identity.fingerprint, - false, - ) - .unwrap(); - let mut off_grid = TrafficStatsRow::example(); - off_grid.dimensions.source_id = "r1".into(); - off_grid.dimensions.bucket_start = 393 * 86_400 + 301; - off_grid.dimensions.bucket_end = off_grid.dimensions.bucket_start + 300; - insert_traffic_stats_rows(&connection, &[off_grid]).unwrap(); - assert_eq!( - daily_product_completion_state( - &connection, - "r1", - 393 * 86_400, - 394 * 86_400, - &identity.fingerprint, - false, - ) - .unwrap(), - DailyProductCompletionState::Dirty + !daily_product_completion_matches(&connection, "r1", 0, 86_400, "product", false,) + .unwrap() ); } @@ -4833,154 +3598,6 @@ mod tests { } } - #[test] - fn maintenance_rejects_database_sidecar_and_lock_aliases_before_mutation() { - let directory = tempdir().unwrap(); - let target = directory.path().join("target.sqlite"); - for source in [ - sidecar_path(&target, "-wal"), - sidecar_path(&target, "-shm"), - sidecar_path(&target, "-journal"), - database_operation_lock_path(&target).unwrap(), - ] { - drop(Connection::open(&source).unwrap()); - let original = fs::read(&source).unwrap(); - - let error = backup_database(&source, &target).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(&source).unwrap(), original); - fs::remove_file(source).unwrap(); - } - } - - #[cfg(unix)] - #[test] - fn maintenance_resolves_symlinks_before_checking_related_paths() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let target = directory.path().join("target.sqlite"); - let lock = database_operation_lock_path(&target).unwrap(); - drop(Connection::open(&lock).unwrap()); - let candidate = directory.path().join("candidate.sqlite"); - symlink(&lock, &candidate).unwrap(); - let original = fs::read(&lock).unwrap(); - - let error = backup_database(&candidate, &target).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(lock).unwrap(), original); - } - - #[cfg(unix)] - #[test] - fn maintenance_rejects_a_derived_lock_symlink_to_the_source() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let source = directory.path().join("source.sqlite"); - let target = directory.path().join("target.sqlite"); - drop(Connection::open(&source).unwrap()); - let lock = database_operation_lock_path(&target).unwrap(); - symlink(&source, &lock).unwrap(); - let original = fs::read(&source).unwrap(); - - let error = backup_database(&source, &target).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(source).unwrap(), original); - } - - #[cfg(unix)] - #[test] - fn canonical_path_applies_parent_components_after_symlink_targets() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let real = directory.path().join("real"); - fs::create_dir_all(real.join("child")).unwrap(); - let alias = directory.path().join("alias"); - symlink(real.join("child"), &alias).unwrap(); - - assert_eq!(canonical_path(alias.join("..").as_path()).unwrap(), real); - } - - #[cfg(unix)] - #[test] - fn separation_resolves_dangling_database_sidecar_and_lock_aliases() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let database = directory.path().join("target.sqlite"); - let database_alias = directory.path().join("database-alias.sqlite"); - symlink(&database, &database_alias).unwrap(); - assert!( - validate_database_path_separation(&[database.as_path(), database_alias.as_path()]) - .is_err() - ); - - let sidecar = sidecar_path(&database, "-wal"); - let sidecar_alias = directory.path().join("sidecar-alias.sqlite"); - symlink(&sidecar, &sidecar_alias).unwrap(); - assert!( - validate_database_path_separation(&[database.as_path(), sidecar_alias.as_path()]) - .is_err() - ); - - let lock = database_operation_lock_path(&database).unwrap(); - let lock_alias = directory.path().join("lock-alias.sqlite"); - symlink(&lock, &lock_alias).unwrap(); - assert!( - validate_database_path_separation(&[database.as_path(), lock_alias.as_path()]).is_err() - ); - } - - #[test] - fn separation_rejects_database_ancestor_and_descendant_paths() { - let directory = tempdir().unwrap(); - let database = directory.path().join("first.sqlite"); - let nested_database = database.join("second.sqlite"); - assert!( - validate_database_path_separation(&[database.as_path(), nested_database.as_path()]) - .unwrap_err() - .to_string() - .contains("must be distinct") - ); - - let sidecar = sidecar_path(&database, "-wal"); - let nested_lock = sidecar.join("operation.lock"); - assert!( - validate_database_path_separation(&[database.as_path(), nested_lock.as_path()]) - .unwrap_err() - .to_string() - .contains("must be distinct") - ); - assert!(!database.exists()); - assert!(!nested_database.exists()); - assert!(!nested_lock.exists()); - } - - #[cfg(unix)] - #[test] - fn separation_rejects_ancestor_paths_through_dangling_symlinks() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let real_database = directory.path().join("real.sqlite"); - let alias = directory.path().join("alias.sqlite"); - symlink(&real_database, &alias).unwrap(); - let nested = real_database.join("second.sqlite"); - - let error = - validate_database_path_separation(&[alias.as_path(), nested.as_path()]).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert!(alias.is_symlink()); - assert!(!real_database.exists()); - assert!(!nested.exists()); - } - #[test] fn failed_transaction_rolls_back_all_persistence_changes() { let mut connection = Connection::open_in_memory().unwrap(); diff --git a/tools/netflow-db/tests/pipeline_cli_help.rs b/tools/netflow-db/tests/pipeline_cli_help.rs index 0da0c38..9a68b38 100644 --- a/tools/netflow-db/tests/pipeline_cli_help.rs +++ b/tools/netflow-db/tests/pipeline_cli_help.rs @@ -2,40 +2,10 @@ use std::{fs, process::Command}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; -#[cfg(unix)] -use std::os::unix::fs::symlink; use rusqlite::Connection; use tempfile::tempdir; -#[test] -fn pipeline_help_explains_repeated_dataset_mode() { - let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) - .args(["pipeline", "--help"]) - .output() - .unwrap(); - - assert!( - output.status.success(), - "stdout={}\nstderr={}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - let help = String::from_utf8(output.stdout).unwrap(); - assert!( - help.contains("Repeat --dataset for two or more values"), - "{help}" - ); - assert!( - help.contains("coordinated fixed daily-active subset run"), - "{help}" - ); - assert!( - help.contains("one value keeps the normal single-dataset path"), - "{help}" - ); -} - #[test] fn pipeline_repeated_dataset_uses_isolated_registry_and_outputs() { let temporary = tempdir().unwrap(); @@ -127,82 +97,6 @@ fn pipeline_repeated_dataset_uses_isolated_registry_and_outputs() { } } -#[cfg(unix)] -#[test] -fn pipeline_rejects_an_inaccessible_path_candidate_before_output_setup() { - let temporary = tempdir().unwrap(); - let first_path = temporary.path().join("first-path"); - let second_path = temporary.path().join("second-path"); - fs::create_dir_all(&first_path).unwrap(); - fs::create_dir_all(&second_path).unwrap(); - - // The owner has no execute permission, while another class does. Checking mode bits alone - // incorrectly treats this as the first PATH candidate for the current process. - let inaccessible = first_path.join("nfdump"); - fs::write(&inaccessible, b"inaccessible candidate").unwrap(); - fs::set_permissions(&inaccessible, fs::Permissions::from_mode(0o001)).unwrap(); - - let database = temporary.path().join("pipeline.sqlite"); - let sidecar = database.with_file_name("pipeline.sqlite-wal"); - let sentinel = b"do not overwrite this executable-related sentinel"; - fs::write(&sidecar, sentinel).unwrap(); - fs::set_permissions(&sidecar, fs::Permissions::from_mode(0o755)).unwrap(); - symlink(&sidecar, second_path.join("nfdump")).unwrap(); - - let capture_root = temporary.path().join("captures"); - fs::create_dir_all(capture_root.join("edge")).unwrap(); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&serde_json::json!({ - "database_path": database, - "timezone": "UTC", - "run_maad": false, - "nfdump": "nfdump", - "inputs": [{ - "input_kind": "nfcapd_tree", - "root_path": capture_root, - "source_ids": ["edge"], - "start_date": "2025-01-01", - "end_date": "2025-01-02" - }] - })) - .unwrap(), - ) - .unwrap(); - let path = std::env::join_paths([&first_path, &second_path]).unwrap(); - - let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) - .args([ - "pipeline", - "--config", - config.to_str().unwrap(), - "--no-maad", - ]) - .env("PATH", path) - .output() - .unwrap(); - - assert!(!output.status.success(), "pipeline unexpectedly succeeded"); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("aliases pipeline control path"), - "stderr={stderr}" - ); - assert_eq!(fs::read(&sidecar).unwrap(), sentinel); - assert!( - !database.exists(), - "output database was created after rejection" - ); - assert!( - !temporary - .path() - .join(".pipeline.sqlite.operation.lock") - .exists(), - "output operation lock was created after rejection" - ); -} - #[test] fn csv_pipeline_does_not_require_nfdump_from_path() { let temporary = tempdir().unwrap();