diff --git a/.gitignore b/.gitignore index ecdb0ba..6868f9b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,8 @@ scripts/local/ugr16-csv.pipeline-v2.json **/playwright-mcp data/** .codex + +# Local native helper artifacts +/tools/netflow-db/maad_fast +/tools/netflow-db/nfdump_reducer +/tools/netflow-db/nfdump_reducer.build-id diff --git a/Cargo.lock b/Cargo.lock index dbea0c4..56fbd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,6 +206,7 @@ dependencies = [ "ipnet", "jiff", "libc", + "nix", "parquet", "rayon", "regex", diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 0ae77d2..ac47884 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -26,6 +26,49 @@ Coverage is observed before selection. Thus, selected-out buckets remain as dens Native nfcapd input pushes the IP prefix condition into the nfdump filter. Visibility conditions apply before statistics accumulate. +`daily_active_sources` is a separate, fixed selection policy for the UOregon `/16` candidate +products. It accepts exactly one IPv4 `/16` and exactly one `nfcapd_tree` input. For each complete +local calendar day, it makes two bounded passes over every unique physical member: + +1. Select anonymized IPv4 source traffic in the `/16` that uses TCP or UDP and source port 1024 or + greater. Sum flows, packets, and bytes by exact source address across physical members. +2. Mark sources active at the inclusive thresholds of 3 flows, 20 packets, and 2,000 bytes. Publish + only the same qualifying-flow population from active sources into the existing five-minute and + rollup contracts. + +There is no destination-port filter and no TCP-flag or SYN filter. Overlapping logical sources do +not double-count activity because the first pass deduplicates their physical members. A day with +any missing physical capture is not published. Activity resets at local midnight, including DST +days. + +The normalized product identity records the entire fixed policy. It is not compatible with an old +prefix-only database. A late or changed input can alter the active set for every five-minute bucket +in a day, so repair requires a whole-day `--force` rebuild inside the existing day transaction. + +## Coordinated subset runs + +Repeat `--dataset` for two or more registry entries to build coordinated subset products. Each entry +supplies its own root and source configuration. Entries do not point to a parent dataset or a +`source_dataset`. + +Multi mode supports only `daily_active_sources`. The selected entries must resolve to the same +nfcapd root and logical source layout, and one whole local-day window. The command rejects `--config`, +`--database-path`, partial time bounds, and ambiguous selection overrides. It takes each output path +from the corresponding registry entry. + +The run coordinates one daily eligibility scan and one publication scan across the subsets. These +remain two physical phases because qualification needs the complete local day before any bucket can +publish. Local-day completeness is shared, so an incomplete required day blocks publication for every +subset. + +Each subset keeps its own immutable product database, product identity, transactions, resume state, +and MAAD configuration. Active sets are still resolved independently. Overlapping subsets may both +receive the same qualifying flow. + +Outputs commit sequentially rather than as one cross-database transaction. If the process stops +between product commits, sibling databases can differ by at most the local day that was in flight. +The next run sees the missing completion marker and rebuilds that day for each unfinished product. + ## Input identity Each input records an exact revision. The revision contains a SHA-256 content identity and a canonical decoder fingerprint. @@ -81,6 +124,16 @@ The fork's stdout is the private `atlantis-flow-stream-v1` binary contract. The The pipeline stops when the fork executable is absent or incompatible. +Request resolution stores the executable's canonical path, one SHA-256 content identity, and a +cheap device/inode/size/timestamp snapshot. The binary identity is part of the product config and +the native input decoder fingerprint, so replacing the executable requires a fresh product and +cannot mix revisions in a resumed database. The snapshot is rechecked around activity and decode +scans and immediately before native publication commits; a change rolls that transaction back. + +Before creating an output directory, lock, or database, the pipeline runs one bounded probe against +an isolated empty `-R` directory. The probe requires the exact empty Atlantis stream (header, +terminator, and EOF), including for incomplete-day requests. + To update the fork, rebase its `atlantis-binary-v1` branch onto a reviewed upstream nfdump tag and run the fork's serial test suite. Then advance this repository's submodule pointer and run `./vendor/scripts/compile-nfdump.sh`. Treat a protocol or normalization change as a versioned wire-contract change. Update the Rust decoder and the provenance revision in the same change. ## Analysis-window exports diff --git a/docs/user/datasets.md b/docs/user/datasets.md index 2761ed2..200c9e2 100644 --- a/docs/user/datasets.md +++ b/docs/user/datasets.md @@ -61,6 +61,41 @@ A logical source combines the captures from more than one collector directory. E } ``` +## Define coordinated subsets + +Give each subset its own registry entry. Repeat those dataset IDs in one pipeline command. Each entry +defines its own logical sources and `daily_active_sources` selection. + +```json +[ + { + "dataset_id": "campus-a", + "root_path": "/data/netflow/campus", + "source_ids": ["router-a"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16" + }, + "db_path": "data/campus-a/netflow.sqlite" + }, + { + "dataset_id": "campus-b", + "root_path": "/data/netflow/campus", + "source_ids": ["router-a"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "0.221.0.0/16" + }, + "db_path": "data/campus-b/netflow.sqlite" + } +] +``` + +The entries share a capture root and the same logical source layout. Their active sets remain independent, +so one flow may publish to both products when the selections overlap. Do not add a parent or +`source_dataset` relation. The multi-dataset command infers each subset from the selected entry and +uses each entry's `db_path`. + ## Required fields | Field | Purpose | @@ -80,8 +115,11 @@ A logical source combines the captures from more than one collector directory. E | `source_ids` | None | Simple source names for datasets without member directories. | | `discovery_mode` | `static` | `live` marks a dataset that continues to receive new captures. `static` marks a complete dataset. | | `sort_order` | `0` | The dataset order in the dashboard. Lower values sort first. | +| `selection` | All flows | A normalized flow-selection object applied automatically by dataset-mode pipeline runs. | Set `db_path` only for a database that must stay separate, such as a [flow selection](setup-pipeline.md#select-flows) product. +Persist `selection` with a dedicated `db_path` when the dataset is itself a selected product. Command-line +selection flags may only override it when `--database-path` names a different output product. Each pipeline run calculates `default_start_date` again. A run that adds earlier days moves the date back. Set the field to hold the dashboard at one date. diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 56151c3..291b230 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -64,6 +64,44 @@ Dataset mode calculates MAAD statistics by default. MAAD statistics describe the If a command fails, read [Troubleshooting](troubleshooting.md). +## Process coordinated subsets + +Repeat `--dataset` for two or more registry entries that select subsets of one nfcapd tree. +Native runs must name the pinned ATLANTIS nfdump fork explicitly: + +```bash +./scripts/netflow-db.sh pipeline \ + --nfdump target/nfdump/libexec/nfdump \ + --dataset campus-a \ + --dataset campus-b \ + --start-date \ + --end-date +``` + +Coordinated mode accepts only registry-backed `daily_active_sources` products. It rejects a run +unless all selected entries have compatible inputs and execution settings: + +- Dataset IDs and output database paths must be unique. +- Every entry must use the same canonical nfcapd root, logical source layout, and timezone. +- Every entry must use the same whole local-day window, force setting, MAAD setting, coverage + setting, and nfdump executable revision. +- Each entry supplies its own `daily_active_sources` prefix and `db_path`. +- Output databases, locks, and sidecars must not overlap one another or the capture tree. + +Do not combine repeated `--dataset` with `--config`, `--database-path`, `--start-time`, `--end-time`, +or command-line selection flags. Configure selection and output paths in `datasets.json`. The +command has no parent dataset or `source_dataset` relation. + +The pipeline discovers the capture plan once, scans each day once, and fans the decoded flow stream +out to the selected products. A missing required capture leaves that day unpublished for every +product. Each product still has its own identity, active-source set, transaction, and completion +marker, so overlapping prefixes may contain the same qualifying flow. + +After a successful run, repeating the exact command is a no-op. The report says +`Published five-minute buckets: 0`, and the pipeline does not rewrite completed days. If a previous +run stopped between product commits, the next run rebuilds only the unfinished day for the affected +product. + ## Select flows Selection conditions use AND logic. The IP prefix can match the source endpoint or the destination endpoint. @@ -79,13 +117,40 @@ Selection conditions use AND logic. The IP prefix can match the source endpoint ``` A selected population is a different database product. Thus, selection options require an explicit `--database-path`. +Dataset registry entries may instead persist a `selection` beside their dedicated `db_path`; dataset +mode applies that selection automatically. Available selection options are: - `--ip-prefix` +- `--daily-active-sources` - `--src-visibility literal|anonymized` - `--dst-visibility literal|anonymized` +`--daily-active-sources` applies the fixed active-user definition used to choose the UOregon +candidate subnets. It requires an IPv4 `/16` and cannot be combined with the visibility flags: + +```bash +./scripts/netflow-db.sh pipeline \ + --dataset example \ + --start-date \ + --end-date \ + --database-path data/example-active/netflow.sqlite \ + --ip-prefix 0.220.0.0/16 \ + --daily-active-sources +``` + +For each complete local day, the pipeline sums qualifying traffic by exact source address across +each unique physical capture member. A source is active when it has at least 3 flows, 20 packets, +and 2,000 bytes that day. Qualifying traffic is IPv4 TCP or UDP from an anonymized source in the +target `/16`, with source port at least 1024. Destination ports and TCP flags are unrestricted. +Only that qualifying traffic from active sources is published. + +This mode supports exactly one `nfcapd_tree` input and whole local days. A day missing any expected +physical capture is skipped rather than published as zero. If input evidence changes after a day +was published, rebuild the whole day with `--force`; a single five-minute repair is not safe because +it can change the active-source set for every bucket in that day. + ## Use a pipeline configuration Configuration mode supports CSV input, nfcapd input, and mixed input. Explicit `csv` and `nfcapd` inputs and `csv_tree` and `nfcapd_tree` discovery inputs go in the top-level `inputs` list. @@ -109,6 +174,27 @@ Put flow selection in the top-level `selection` object: } ``` +The equivalent active-source selection is deliberately a named policy rather than configurable +thresholds: + +```json +{ + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16" + }, + "inputs": [ + { + "input_kind": "nfcapd_tree", + "root_path": "/path/to/captures", + "source_ids": ["gateway-a", "gateway-b"], + "start_date": "2025-06-01", + "end_date": "2026-06-29" + } + ] +} +``` + On the native path, nfcapd input needs the fork path: set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`, or pass `--nfdump` when the configuration does not set it. ## Common options diff --git a/tools/netflow-db/Cargo.toml b/tools/netflow-db/Cargo.toml index b01215f..54cd946 100644 --- a/tools/netflow-db/Cargo.toml +++ b/tools/netflow-db/Cargo.toml @@ -26,6 +26,7 @@ fs2 = "0.4" ipnet = { version = "2", features = ["serde"] } jiff = "0.2" libc = "0.2" +nix = { version = "0.27", default-features = false, features = ["fs"] } parquet = { version = "59.2", default-features = false, features = ["arrow", "zstd"] } rayon = "1" regex = "1" diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index 7b64ab8..27d7d49 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -3,7 +3,6 @@ use std::{ collections::{BTreeMap, BTreeSet, HashSet}, net::IpAddr, - time::{Duration, Instant}, }; use fixedbitset::FixedBitSet; @@ -36,10 +35,18 @@ pub enum DomainError { UnknownSelectionKeys(String), #[error("selection version must be 1")] InvalidSelectionVersion, - #[error("selection kind must be 'all' or 'flows'")] + #[error("selection kind must be 'all', 'flows', or 'daily_active_sources'")] InvalidSelectionKind, #[error("selection kind 'all' cannot define flow criteria")] AllSelectionHasCriteria, + #[error("daily_active_sources selection requires one IPv4 /16 ip_prefix")] + DailyActiveSourcesRequireIpv4Prefix, + #[error( + "daily_active_sources selection fixes src_visibility to 'anonymized' and leaves dst_visibility unrestricted" + )] + InvalidDailyActiveSourceVisibility, + #[error("daily_active_sources criteria do not match the finalized active-source definition")] + InvalidDailyActiveSourceCriteria, #[error("Invalid selection ip_prefix: {0}")] InvalidIpPrefix(String), #[error("selection {0} must be 'literal' or 'anonymized'")] @@ -265,9 +272,21 @@ impl FlowObservation { } } -/// A validated predicate shared by all input adapters. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum FlowSelectionKind { + #[default] + Flows, + DailyActiveSources, +} + +pub(crate) const DAILY_ACTIVE_MIN_FLOWS: i64 = 3; +pub(crate) const DAILY_ACTIVE_MIN_PACKETS: i64 = 20; +pub(crate) const DAILY_ACTIVE_MIN_BYTES: i64 = 2_000; + +/// A validated selection shared by input adapters and pipeline product identity. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct FlowSelection { + kind: FlowSelectionKind, ip_prefix: Option, src_visibility: Option, dst_visibility: Option, @@ -286,7 +305,12 @@ impl FlowSelection { .filter(|key| { !matches!( key.as_str(), - "version" | "kind" | "ip_prefix" | "src_visibility" | "dst_visibility" + "version" + | "kind" + | "ip_prefix" + | "src_visibility" + | "dst_visibility" + | "criteria" ) }) .cloned() @@ -302,11 +326,13 @@ impl FlowSelection { Some(Value::String(kind)) => Some(kind.as_str()), Some(_) => return Err(DomainError::InvalidSelectionKind), }; - if !matches!(kind, None | Some("all" | "flows")) { - return Err(DomainError::InvalidSelectionKind); - } + let selection_kind = match kind { + None | Some("all" | "flows") => FlowSelectionKind::Flows, + Some("daily_active_sources") => FlowSelectionKind::DailyActiveSources, + Some(_) => return Err(DomainError::InvalidSelectionKind), + }; if kind == Some("all") - && ["ip_prefix", "src_visibility", "dst_visibility"] + && ["ip_prefix", "src_visibility", "dst_visibility", "criteria"] .iter() .any(|key| !is_empty_value(object.get(*key))) { @@ -322,16 +348,47 @@ impl FlowSelection { .map(|network| network.trunc()) }) .transpose()?; - Ok(Self { + let mut selection = Self { + kind: selection_kind, ip_prefix, src_visibility: parse_visibility(object, "src_visibility")?, dst_visibility: parse_visibility(object, "dst_visibility")?, - }) + }; + if selection.kind == FlowSelectionKind::DailyActiveSources { + if !matches!(selection.ip_prefix, Some(IpNet::V4(prefix)) if prefix.prefix_len() == 16) + { + return Err(DomainError::DailyActiveSourcesRequireIpv4Prefix); + } + if !matches!( + selection.src_visibility, + None | Some(ExactVisibility::Anonymized) + ) || selection.dst_visibility.is_some() + { + return Err(DomainError::InvalidDailyActiveSourceVisibility); + } + if object.get("criteria").is_some_and(|criteria| { + !criteria.is_null() && criteria != &daily_active_source_criteria() + }) { + return Err(DomainError::InvalidDailyActiveSourceCriteria); + } + selection.src_visibility = Some(ExactVisibility::Anonymized); + } else if !is_empty_value(object.get("criteria")) { + return Err(DomainError::InvalidDailyActiveSourceCriteria); + } + Ok(selection) } #[must_use] pub const fn is_unrestricted(&self) -> bool { - self.ip_prefix.is_none() && self.src_visibility.is_none() && self.dst_visibility.is_none() + matches!(self.kind, FlowSelectionKind::Flows) + && self.ip_prefix.is_none() + && self.src_visibility.is_none() + && self.dst_visibility.is_none() + } + + #[must_use] + pub const fn selects_daily_active_sources(&self) -> bool { + matches!(self.kind, FlowSelectionKind::DailyActiveSources) } #[must_use] @@ -350,11 +407,16 @@ impl FlowSelection { } #[must_use] - pub fn matches(&self, observation: &FlowObservation) -> bool { - let prefix_matches = self.ip_prefix.as_ref().is_none_or(|prefix| { - prefix.contains(&observation.src_ip) || prefix.contains(&observation.dst_ip) - }); - prefix_matches && self.allows_src_tos(observation.src_tos) + pub fn matches_qualifying_flow(&self, observation: &FlowObservation) -> bool { + let (source, destination) = exact_visibility_pair_from_tos(observation.src_tos); + self.matches_flow_fields( + observation.src_ip, + observation.dst_ip, + observation.protocol, + observation.src_port, + source, + destination, + ) } #[must_use] @@ -368,10 +430,56 @@ impl FlowSelection { } #[must_use] - pub fn nfdump_prefix_filter(&self) -> Option { - self.ip_prefix - .as_ref() - .map(|prefix| format!("net {prefix}")) + pub fn nfdump_filter(&self) -> Option { + self.ip_prefix.as_ref().map(|prefix| { + if self.selects_daily_active_sources() { + // nfdump filters the original record before -o atlantis expands its + // tunnel extension into a synthetic row; retain both branches and + // let the Rust decoder authoritatively match emitted rows. + format!( + "(src net {prefix} and ipv4 and (proto tcp or proto udp) and src port > 1023) or (src tun net {prefix} and (tun proto tcp or tun proto udp) and src port > 1023)" + ) + } else { + format!("net {prefix}") + } + }) + } + + #[must_use] + pub(crate) fn matches_flow_fields( + &self, + src_ip: IpAddr, + dst_ip: IpAddr, + protocol: u8, + src_port: Option, + source_visibility: ExactVisibility, + destination_visibility: ExactVisibility, + ) -> bool { + let prefix_matches = self.ip_prefix.as_ref().is_none_or(|prefix| { + if self.selects_daily_active_sources() { + prefix.contains(&src_ip) + } else { + prefix.contains(&src_ip) || prefix.contains(&dst_ip) + } + }); + let visibility_matches = self + .src_visibility + .is_none_or(|required| required == source_visibility) + && self + .dst_visibility + .is_none_or(|required| required == destination_visibility); + let activity_candidate_matches = !self.selects_daily_active_sources() + || (matches!(src_ip, IpAddr::V4(_)) + && matches!(protocol, 6 | 17) + && src_port.is_some_and(|port| port >= 1_024)); + prefix_matches && visibility_matches && activity_candidate_matches + } + + #[must_use] + pub const fn daily_activity_threshold_met(flows: i64, packets: i64, bytes: i64) -> bool { + flows >= DAILY_ACTIVE_MIN_FLOWS + && packets >= DAILY_ACTIVE_MIN_PACKETS + && bytes >= DAILY_ACTIVE_MIN_BYTES } #[must_use] @@ -379,6 +487,16 @@ impl FlowSelection { if self.is_unrestricted() { return json!({"version": 1, "kind": "all"}); } + if self.selects_daily_active_sources() { + return json!({ + "version": 1, + "kind": "daily_active_sources", + "ip_prefix": self.ip_prefix.map(|prefix| prefix.to_string()), + "src_visibility": self.src_visibility.map(ExactVisibility::as_str), + "dst_visibility": self.dst_visibility.map(ExactVisibility::as_str), + "criteria": daily_active_source_criteria(), + }); + } json!({ "version": 1, "kind": "flows", @@ -389,6 +507,23 @@ impl FlowSelection { } } +fn daily_active_source_criteria() -> Value { + json!({ + "address_side": "source", + "ip_version": 4, + "protocols": [6, 17], + "minimum_source_port": 1024, + "destination_ports": "all", + "tcp_flags": "all", + "activity_window": "local_day", + "minimum_flows": DAILY_ACTIVE_MIN_FLOWS, + "minimum_packets": DAILY_ACTIVE_MIN_PACKETS, + "minimum_bytes": DAILY_ACTIVE_MIN_BYTES, + "union_across_physical_sources": true, + "requires_complete_physical_day": true, + }) +} + fn optional_non_empty_string<'a>( object: &'a Map, key: &'static str, @@ -516,6 +651,11 @@ impl AddressSet { self.0.iter() } + #[must_use] + pub fn contains(&self, address: &IpAddr) -> bool { + self.0.contains(address) + } + #[must_use] pub fn len(&self) -> usize { self.0.len() @@ -966,38 +1106,6 @@ pub struct StatisticalBucket { five_minute_starts: BTreeSet, } -/// Aggregate timings for merging canonical children into one rollup builder. -#[derive(Clone, Debug, Default)] -pub(crate) struct StatisticalBucketIncludeProfile { - pub(crate) total_elapsed: Duration, - pub(crate) traffic_elapsed: Duration, - pub(crate) protocols_elapsed: Duration, - pub(crate) addresses_elapsed: Duration, - pub(crate) ports_elapsed: Duration, - pub(crate) coverage_elapsed: Duration, -} - -impl StatisticalBucketIncludeProfile { - pub(crate) fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.traffic_elapsed += profile.traffic_elapsed; - self.protocols_elapsed += profile.protocols_elapsed; - self.addresses_elapsed += profile.addresses_elapsed; - self.ports_elapsed += profile.ports_elapsed; - self.coverage_elapsed += profile.coverage_elapsed; - } - - pub(crate) fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.traffic_elapsed - + self.protocols_elapsed - + self.addresses_elapsed - + self.ports_elapsed - + self.coverage_elapsed, - ) - } -} - impl StatisticalBucket { #[must_use] pub fn new(key: BucketKey) -> Self { @@ -1057,15 +1165,6 @@ impl StatisticalBucket { } pub fn include(&mut self, child: &CanonicalBucket) -> Result<(), DomainError> { - self.include_profiled(child).map(|_| ()) - } - - pub(crate) fn include_profiled( - &mut self, - child: &CanonicalBucket, - ) -> Result { - let total_started = Instant::now(); - let traffic_started = Instant::now(); let mut updates = Vec::with_capacity(child.traffic.len()); for entry in &child.traffic { let mut metrics = self.traffic.get(&entry.scope).cloned().unwrap_or_default(); @@ -1075,44 +1174,28 @@ impl StatisticalBucket { for (scope, metrics) in updates { self.traffic.insert(scope, metrics); } - let traffic_elapsed = traffic_started.elapsed(); - let protocols_started = Instant::now(); for entry in &child.protocols { self.protocols .entry(entry.scope) .or_default() .extend(entry.protocols.iter().cloned()); } - let protocols_elapsed = protocols_started.elapsed(); - let addresses_started = Instant::now(); for entry in &child.addresses { self.addresses .entry((entry.scope, entry.address_side)) .or_default() .extend(entry.addresses.iter().copied()); } - let addresses_elapsed = addresses_started.elapsed(); - let ports_started = Instant::now(); for entry in &child.ports { self.ports .entry((entry.scope, entry.port_side)) .or_insert_with(empty_ports) .union_with(&entry.ports); } - let ports_elapsed = ports_started.elapsed(); - let coverage_started = Instant::now(); self.five_minute_starts .extend(child.five_minute_starts.iter().copied()); self.coverage.include(child.coverage)?; - let coverage_elapsed = coverage_started.elapsed(); - Ok(StatisticalBucketIncludeProfile { - total_elapsed: total_started.elapsed(), - traffic_elapsed, - protocols_elapsed, - addresses_elapsed, - ports_elapsed, - coverage_elapsed, - }) + Ok(()) } /// Whether every five-minute child in this bucket's interval was included. @@ -1165,6 +1248,54 @@ impl StatisticalBucket { } } + /// Consume this builder into its canonical representation without cloning + /// the large aggregate collections. + #[must_use] + pub fn finish_owned(self) -> CanonicalBucket { + let Self { + key, + coverage, + traffic, + protocols, + addresses, + ports, + five_minute_starts, + } = self; + + CanonicalBucket { + key, + coverage, + traffic: traffic + .into_iter() + .map(|(scope, metrics)| ScopedTraffic { scope, metrics }) + .collect(), + protocols: protocols + .into_iter() + .map(|(scope, protocols)| ScopedProtocols { + scope, + protocols: protocols.into_iter().collect(), + }) + .collect(), + addresses: addresses + .into_iter() + .map(|((scope, address_side), addresses)| ScopedAddresses { + scope, + address_side, + addresses, + }) + .collect(), + ports: ports + .into_iter() + .map(|((scope, port_side), ports)| ScopedPorts { + scope, + port_side, + ports, + }) + .collect(), + five_minute_starts, + } + } + fn add_observation(&mut self, observation: FlowObservation) -> Result<(), DomainError> { if IpVersion::of(observation.src_ip) != IpVersion::of(observation.dst_ip) { return Err(DomainError::MixedIpVersions); @@ -1341,11 +1472,11 @@ mod tests { let wrong_visibility = FlowObservation::new(matching.src_ip, matching.dst_ip, 6, 1, 100, 0).unwrap(); - assert!(selection.matches(&matching)); - assert!(!selection.matches(&wrong_visibility)); + assert!(selection.matches_qualifying_flow(&matching)); + assert!(!selection.matches_qualifying_flow(&wrong_visibility)); assert_eq!(selection.src_visibility(), Some(ExactVisibility::Literal)); assert_eq!( - selection.nfdump_prefix_filter().as_deref(), + selection.nfdump_filter().as_deref(), Some("net 192.0.2.0/24") ); assert_eq!( @@ -1360,6 +1491,140 @@ mod tests { ); } + #[test] + fn daily_active_source_selection_preserves_the_finalized_definition() { + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.99.1/16" + }))) + .unwrap(); + let matching = FlowObservation::new( + address([0, 220, 1, 2]), + address([198, 51, 100, 2]), + 6, + 20, + 2_000, + 2, + ) + .unwrap() + .with_ports(Some(55_000), Some(443)); + let low_source_port = matching.clone().with_ports(Some(443), Some(55_000)); + let boundary_source_port = matching.clone().with_ports(Some(1_024), Some(65_535)); + let icmp = FlowObservation::new( + address([0, 220, 1, 2]), + address([198, 51, 100, 2]), + 1, + 20, + 2_000, + 2, + ) + .unwrap() + .with_ports(None, None); + let literal_source = FlowObservation::new( + address([0, 220, 1, 2]), + address([198, 51, 100, 2]), + 17, + 20, + 2_000, + 0, + ) + .unwrap() + .with_ports(Some(55_000), Some(53)); + let destination_only = FlowObservation::new( + address([198, 51, 100, 2]), + address([0, 220, 1, 2]), + 6, + 20, + 2_000, + 1, + ) + .unwrap() + .with_ports(Some(55_000), Some(443)); + + assert!(selection.matches_qualifying_flow(&matching)); + assert!(selection.matches_qualifying_flow(&boundary_source_port)); + assert!(!selection.matches_qualifying_flow(&low_source_port)); + assert!(!selection.matches_qualifying_flow(&icmp)); + assert!(!selection.matches_qualifying_flow(&literal_source)); + assert!(!selection.matches_qualifying_flow(&destination_only)); + assert!(selection.selects_daily_active_sources()); + assert_eq!( + selection.src_visibility(), + Some(ExactVisibility::Anonymized) + ); + assert_eq!( + selection.nfdump_filter().as_deref(), + Some( + "(src net 0.220.0.0/16 and ipv4 and (proto tcp or proto udp) and src port > 1023) or (src tun net 0.220.0.0/16 and (tun proto tcp or tun proto udp) and src port > 1023)" + ) + ); + assert!(FlowSelection::daily_activity_threshold_met(3, 20, 2_000)); + assert!(!FlowSelection::daily_activity_threshold_met(2, 20, 2_000)); + assert!(!FlowSelection::daily_activity_threshold_met(3, 19, 2_000)); + assert!(!FlowSelection::daily_activity_threshold_met(3, 20, 1_999)); + + let normalized = json!({ + "version": 1, + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16", + "src_visibility": "anonymized", + "dst_visibility": null, + "criteria": { + "address_side": "source", + "ip_version": 4, + "protocols": [6, 17], + "minimum_source_port": 1024, + "destination_ports": "all", + "tcp_flags": "all", + "activity_window": "local_day", + "minimum_flows": 3, + "minimum_packets": 20, + "minimum_bytes": 2000, + "union_across_physical_sources": true, + "requires_complete_physical_day": true + } + }); + assert_eq!(selection.normalized_payload(), normalized); + assert_eq!( + FlowSelection::from_payload(Some(&normalized)).unwrap(), + selection + ); + } + + #[test] + fn daily_active_source_selection_rejects_semantic_drift() { + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "2001:db8::/32" + }))), + Err(DomainError::DailyActiveSourcesRequireIpv4Prefix) + ); + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/24" + }))), + Err(DomainError::DailyActiveSourcesRequireIpv4Prefix) + ); + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16", + "dst_visibility": "literal" + }))), + Err(DomainError::InvalidDailyActiveSourceVisibility) + ); + assert_eq!( + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "0.220.0.0/16", + "criteria": {"minimum_flows": 4} + }))), + Err(DomainError::InvalidDailyActiveSourceCriteria) + ); + } + #[test] fn selection_rejects_unknown_keys_and_invalid_all_criteria() { assert_eq!( @@ -1422,6 +1687,40 @@ mod tests { ); } + #[test] + fn consuming_finalizer_matches_borrowing_finalizer_for_dense_and_sparse_buckets() { + let dense = StatisticalBucket::dense(key(Granularity::FiveMinutes, 0, 300)); + let dense_expected = dense.finish(); + assert_eq!(dense_expected, dense.finish_owned()); + + let mut sparse = StatisticalBucket::new(key(Granularity::ThirtyMinutes, 0, 1_800)); + sparse + .add( + observation([192, 0, 2, 1], [198, 51, 100, 1], 6, 2) + .with_ports(Some(53), Some(1_024)), + ) + .unwrap(); + sparse + .add(GroupedTrafficFact { + ip_version: IpVersion::V4, + protocol: 17, + src_tos: 0, + flows: 2, + packets: 4, + bytes: 40, + }) + .unwrap(); + sparse + .add(ScopedAddressesFact::new( + Scope::new(IpVersion::V4, Visibility::Literal, Visibility::Anonymized), + AddressSide::Destination, + [address([203, 0, 113, 1]), address([203, 0, 113, 2])], + )) + .unwrap(); + let sparse_expected = sparse.finish(); + assert_eq!(sparse_expected, sparse.finish_owned()); + } + #[test] fn bucket_coverage_is_explicit_and_additive_across_rollups() { let partial = StatisticalBucket::new(key(Granularity::FiveMinutes, 0, 300)) diff --git a/tools/netflow-db/src/ingest.rs b/tools/netflow-db/src/ingest.rs index 306963a..7eca752 100644 --- a/tools/netflow-db/src/ingest.rs +++ b/tools/netflow-db/src/ingest.rs @@ -1,13 +1,14 @@ //! Streaming adapters that turn external CSV inputs into canonical five-minute buckets. use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashMap}, ffi::OsString, fs, io::{BufReader, Read, Seek, SeekFrom}, + net::IpAddr, path::{Path, PathBuf}, process::{Command, Stdio}, - sync::mpsc, + sync::{Arc, mpsc}, thread, time::{Duration, Instant}, }; @@ -24,8 +25,8 @@ use crate::{ config::{CsvSourceConfig, InputOrder}, coverage::BucketCoverage, domain::{ - BucketKey, CanonicalBucket, DomainError, FlowObservation, FlowSelection, Granularity, - StatisticalBucket, + AddressSet, BucketKey, CanonicalBucket, DomainError, FlowObservation, FlowSelection, + Granularity, StatisticalBucket, }, nfdump, normalize::{NormalizeError, field_indexes, normalize_csv_values}, @@ -33,6 +34,8 @@ use crate::{ const BUCKET_SECONDS: i64 = 300; const NFDUMP_TIMEOUT: Duration = Duration::from_secs(300); +const NFDUMP_DAY_TIMEOUT: Duration = Duration::from_secs(3_600); +const NFDUMP_PROBE_TIMEOUT: Duration = Duration::from_secs(10); const MAX_DIAGNOSTIC_BYTES: usize = 64 * 1024; const TIMESTAMP_KEYS: [&str; 3] = ["time_received", "time_end", "time_start"]; @@ -113,6 +116,9 @@ pub struct NfcapdInputSpec { pub bucket_start: i64, } +/// One coordinated daily selection and its resolved source set. +pub type NfcapdSelectionAndActiveSources = (FlowSelection, Arc); + /// Discover configured CSV inputs under one flat directory. pub fn discover_csv_inputs( root: impl AsRef, @@ -388,7 +394,7 @@ fn scan_csv_reader( match normalize_csv_values(&values, config, &indexes) { Ok(row) => { state.mark_valid(&row.source_id, row.bucket_start)?; - if selection.matches(&row.observation) { + if selection.matches_qualifying_flow(&row.observation) { state.accept(row)?; } } @@ -1131,12 +1137,180 @@ pub fn build_nfdump_command( "-o".into(), nfdump::OUTPUT_MODE.into(), ]; - if let Some(filter) = selection.nfdump_prefix_filter() { + if let Some(filter) = selection.nfdump_filter() { command.push(filter.into()); } command } +/// Build one nfcapd command for several daily active-source selections. +pub fn build_nfdump_command_for_selections( + path: impl AsRef, + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result, IngestError> { + let mut command = vec![ + executable.as_ref().to_owned(), + "-r".into(), + path.as_ref().as_os_str().to_owned(), + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + command.push(daily_active_union_filter(selections)?.into()); + Ok(command) +} + +fn daily_active_union_filter(selections: &[FlowSelection]) -> Result { + if selections.is_empty() { + return Err(IngestError::InvalidInput( + "daily active-source selections are empty".into(), + )); + } + let mut prefixes = selections + .iter() + .map(|selection| { + if !selection.selects_daily_active_sources() { + return Err(IngestError::InvalidInput( + "nfdump subset decoding requires daily_active_sources selections".into(), + )); + } + selection + .ip_prefix() + .map(ToString::to_string) + .ok_or_else(|| { + IngestError::InvalidInput( + "daily_active_sources selection is missing an ip_prefix".into(), + ) + }) + }) + .collect::, _>>()?; + prefixes.sort_unstable(); + prefixes.dedup(); + let source_filter = |qualifier: &str| { + if prefixes.len() == 1 { + format!("{qualifier} net {}", prefixes[0]) + } else { + format!( + "({})", + prefixes + .iter() + .map(|prefix| format!("{qualifier} net {prefix}")) + .collect::>() + .join(" or ") + ) + } + }; + let outer_source_filter = source_filter("src"); + let tunnel_source_filter = source_filter("src tun"); + Ok(format!( + "({outer_source_filter} and ipv4 and (proto tcp or proto udp) and src port > 1023) or ({tunnel_source_filter} and (tun proto tcp or tun proto udp) and src port > 1023)" + )) +} + +/// Create a private nfdump input directory containing exactly the requested captures. +/// +/// `nfdump -R first:last` selects every alphabetically intervening file. A manifest +/// directory keeps the pinned nfdump range reader while making the input membership +/// explicit and bounded to the paths discovered by the pipeline. +fn prepare_nfcapd_manifest(paths: &[PathBuf]) -> Result<(tempfile::TempDir, PathBuf), IngestError> { + let context = paths + .first() + .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; + if paths.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(IngestError::InvalidInput( + "nfcapd day range paths must be strictly chronological".into(), + )); + } + let manifest = tempfile::Builder::new() + .prefix("atlantis-nfcapd-") + .tempdir() + .map_err(|source| IngestError::Io { + path: context.clone(), + source, + })?; + for (index, path) in paths.iter().enumerate() { + let target = if path.is_absolute() { + path.clone() + } else { + std::env::current_dir() + .map_err(|source| IngestError::Io { + path: path.clone(), + source, + })? + .join(path) + }; + let link = manifest.path().join(format!("nfcapd.{index:020}")); + link_nfcapd_manifest_entry(&target, &link).map_err(|source| IngestError::Io { + path: path.clone(), + source, + })?; + } + let manifest_path = manifest.path().to_owned(); + Ok((manifest, manifest_path)) +} + +fn link_nfcapd_manifest_entry(target: &Path, link: &Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(target, link) +} + +fn build_nfdump_manifest_command_for_selections( + manifest: &Path, + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result, IngestError> { + let mut command = vec![ + executable.as_ref().to_owned(), + "-R".into(), + manifest.as_os_str().to_owned(), + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + command.push(daily_active_union_filter(selections)?.into()); + Ok(command) +} + +/// Prove that an executable implements the private Atlantis output contract before any pipeline +/// output is created. An empty `-R` directory makes the probe independent of capture contents, +/// while the normal streaming decoder still enforces the header, terminator, and EOF contract. +pub(crate) fn probe_nfdump_compatibility( + executable: impl AsRef, +) -> Result<(), IngestError> { + let manifest = tempfile::tempdir().map_err(|source| IngestError::Io { + path: PathBuf::from(""), + source, + })?; + let command = vec![ + executable.as_ref().to_owned(), + "-R".into(), + manifest.path().as_os_str().to_owned(), + "-q".into(), + "-o".into(), + nfdump::OUTPUT_MODE.into(), + ]; + let key = BucketKey::new("", Granularity::FiveMinutes, 0, BUCKET_SECONDS); + let bucket = run_nfdump( + command, + manifest.path(), + NFDUMP_PROBE_TIMEOUT, + move |stdout| nfdump::reduce_to_bucket(stdout, key, &FlowSelection::default()), + ) + .map_err(|error| { + IngestError::InvalidInput(format!( + "nfdump compatibility probe for {:?} failed: {error}", + executable.as_ref() + )) + })?; + if bucket.traffic.iter().any(|scope| scope.metrics.flows != 0) { + return Err(IngestError::InvalidInput(format!( + "nfdump compatibility probe for {:?} emitted a non-empty Atlantis stream", + executable.as_ref() + ))); + } + Ok(()) +} + /// Decode one canonical nfcapd file into its dense five-minute bucket. pub fn read_nfcapd_bucket( path: impl AsRef, @@ -1172,6 +1346,103 @@ fn read_nfcapd_bucket_with_timeout( bucket_start + BUCKET_SECONDS, ); let command = build_nfdump_command(path, selection, executable.as_ref()); + let selection = selection.clone(); + run_nfdump(command, path, timeout, move |stdout| { + nfdump::reduce_to_bucket(stdout, key, &selection) + }) +} + +pub fn read_nfcapd_bucket_with_active_sources( + path: impl AsRef, + source_id: &str, + selection: &FlowSelection, + active_sources: Arc, + executable: impl AsRef, + timezone: &str, +) -> Result { + let path = path.as_ref(); + let bucket_start = parse_nfcapd_bucket_start(path, timezone)?; + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + BUCKET_SECONDS, + ); + let command = build_nfdump_command(path, selection, executable.as_ref()); + let selection = selection.clone(); + run_nfdump(command, path, NFDUMP_TIMEOUT, move |stdout| { + nfdump::reduce_to_bucket_with_active_sources( + stdout, + key, + &selection, + active_sources.as_ref(), + ) + }) +} + +pub fn read_nfcapd_buckets_with_active_sources( + path: impl AsRef, + source_id: &str, + selections_and_active_sources: &[NfcapdSelectionAndActiveSources], + executable: impl AsRef, + timezone: &str, +) -> Result, IngestError> { + if selections_and_active_sources.is_empty() { + return Err(IngestError::InvalidInput( + "daily active-source selection pairs are empty".into(), + )); + } + let path = path.as_ref(); + let bucket_start = parse_nfcapd_bucket_start(path, timezone)?; + let key = BucketKey::new( + source_id, + Granularity::FiveMinutes, + bucket_start, + bucket_start + BUCKET_SECONDS, + ); + let command = build_nfdump_command_for_selections( + path, + &selections_and_active_sources + .iter() + .map(|(selection, _)| selection.clone()) + .collect::>(), + executable.as_ref(), + )?; + let selections_and_active_sources = selections_and_active_sources.to_vec(); + run_nfdump(command, path, NFDUMP_TIMEOUT, move |stdout| { + nfdump::reduce_to_buckets_with_active_sources(stdout, key, &selections_and_active_sources) + }) +} + +pub(crate) fn read_nfcapd_daily_source_activities( + paths: &[PathBuf], + selections: &[FlowSelection], + executable: impl AsRef, +) -> Result>, IngestError> { + let context = paths + .first() + .ok_or_else(|| IngestError::InvalidInput("nfcapd day range is empty".into()))?; + let (_manifest, manifest_path) = prepare_nfcapd_manifest(paths)?; + let command = + build_nfdump_manifest_command_for_selections(&manifest_path, selections, executable)?; + let selections = selections.to_vec(); + run_nfdump(command, context, NFDUMP_DAY_TIMEOUT, move |stdout| { + nfdump::reduce_to_daily_source_activities(stdout, &selections) + }) +} + +fn run_nfdump( + command: Vec, + path: &Path, + timeout: Duration, + decode: F, +) -> Result +where + T: Send + 'static, + F: FnOnce(BufReader) -> Result + + Send + + 'static, +{ let executable_name = command[0].clone(); let stderr_file = tempfile::tempfile().map_err(|source| IngestError::Io { path: path.to_owned(), @@ -1195,10 +1466,9 @@ fn read_nfcapd_bucket_with_timeout( .stdout .take() .ok_or_else(|| IngestError::InvalidInput("nfdump stdout was not captured".into()))?; - let selection = selection.clone(); let (sender, receiver) = mpsc::sync_channel(1); let reader = thread::spawn(move || { - let result = nfdump::reduce_to_bucket(BufReader::new(stdout), key, &selection); + let result = decode(BufReader::new(stdout)); let _ = sender.send(result); }); let deadline = Instant::now() + timeout; @@ -1283,7 +1553,7 @@ fn read_tail(mut file: fs::File, limit: usize) -> std::io::Result { #[cfg(test)] mod tests { - use std::{collections::BTreeMap, fs, io::Write}; + use std::{collections::BTreeMap, fs, io::Write, sync::Arc}; use flate2::{Compression, write::GzEncoder}; use serde_json::json; @@ -1613,6 +1883,421 @@ mod tests { ); } + #[test] + fn multi_daily_activity_command_unions_prefixes_and_keeps_fixed_filter() { + let selections = [ + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16", + }))) + .unwrap(), + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(), + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(), + ]; + let command = + build_nfdump_command_for_selections("capture", &selections, "nfdump").unwrap(); + + assert_eq!( + command.last().unwrap().to_string_lossy(), + "((src net 192.0.0.0/16 or src net 198.51.0.0/16) and ipv4 and (proto tcp or proto udp) and src port > 1023) or ((src tun net 192.0.0.0/16 or src tun net 198.51.0.0/16) and (tun proto tcp or tun proto udp) and src port > 1023)" + ); + } + + #[test] + fn multi_nfdump_commands_reject_empty_and_non_daily_selection_sets() { + assert!(matches!( + build_nfdump_command_for_selections("capture", &[], "nfdump"), + Err(IngestError::InvalidInput(message)) if message.contains("empty") + )); + assert!(matches!( + build_nfdump_command_for_selections("capture", &[FlowSelection::default()], "nfdump"), + Err(IngestError::InvalidInput(message)) if message.contains("daily_active_sources") + )); + } + + #[cfg(unix)] + #[test] + fn multi_bucket_reader_decodes_one_process_into_one_bucket_per_pair() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let invocation_log = directory.path().join("invocations.log"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 32..16 + 40].copy_from_slice(&20_u64.to_le_bytes()); + binary[16 + 40..16 + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + binary[16 + 48..16 + 56].copy_from_slice(&3_u64.to_le_bytes()); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + invocation_log.display(), + stream.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(); + let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); + let pairs = [ + ( + selection.clone(), + Arc::new([source].into_iter().collect::()), + ), + ( + selection, + Arc::new([source].into_iter().collect::()), + ), + ]; + + let buckets = read_nfcapd_buckets_with_active_sources( + &capture, + "edge-a", + &pairs, + &executable, + "America/Los_Angeles", + ) + .unwrap(); + + assert_eq!(buckets.len(), 2); + for bucket in buckets { + assert_eq!( + bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics + .flows, + 3 + ); + } + assert_eq!( + fs::read_to_string(invocation_log).unwrap().lines().count(), + 1 + ); + } + + #[cfg(unix)] + #[test] + fn single_bucket_reader_accepts_shared_active_source_set() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!("#!/bin/sh\ncat '{}'\n", stream.display()), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(); + let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); + let active_sources = Arc::new([source].into_iter().collect::()); + + let bucket = read_nfcapd_bucket_with_active_sources( + &capture, + "edge-a", + &selection, + active_sources, + &executable, + "America/Los_Angeles", + ) + .unwrap(); + + assert_eq!( + bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics + .flows, + 3 + ); + } + + #[cfg(unix)] + #[test] + fn multi_daily_activity_reader_decodes_one_range_into_distinct_maps() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let invocation_log = directory.path().join("invocations.log"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 32..16 + 40].copy_from_slice(&20_u64.to_le_bytes()); + binary[16 + 40..16 + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + binary[16 + 48..16 + 56].copy_from_slice(&3_u64.to_le_bytes()); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + invocation_log.display(), + stream.display() + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selections = [ + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(), + FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16", + }))) + .unwrap(), + ]; + + let activities = read_nfcapd_daily_source_activities( + std::slice::from_ref(&capture), + &selections, + &executable, + ) + .unwrap(); + + assert_eq!(activities.len(), 2); + assert_eq!(activities[0].len(), 1); + assert!(activities[1].is_empty()); + assert_eq!( + fs::read_to_string(invocation_log).unwrap().lines().count(), + 1 + ); + } + + #[cfg(unix)] + #[test] + fn daily_activity_reader_uses_only_the_discovered_capture_paths() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let manifest_members = directory.path().join("manifest-members.log"); + let manifest_path = directory.path().join("manifest-path.log"); + let mut binary = ONE_V4_BINARY_STREAM.to_vec(); + binary[16 + 64..16 + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + binary[16 + 69] = 0b010; + fs::write(&stream, binary).unwrap(); + let first = directory.path().join("nfcapd.202506010000"); + let second = directory.path().join("nfcapd.202506010010"); + fs::write(&first, "selected first").unwrap(); + fs::write(&second, "selected second").unwrap(); + // These files are alphabetically between the selected captures. A physical + // -R first:last range would read them even though they were not snapshotted. + fs::write( + directory.path().join("nfcapd.202506010005"), + "untracked valid capture", + ) + .unwrap(); + fs::write( + directory.path().join("nfcapd.202506010007.backup"), + "untracked backup", + ) + .unwrap(); + fs::write( + directory.path().join("nfcapd.202506010008.aria2"), + "untracked sidecar", + ) + .unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\n\ + set -eu\n\ + manifest=\"\"\n\ + while [ \"$#\" -gt 0 ]; do\n\ + if [ \"$1\" = \"-R\" ]; then manifest=\"$2\"; shift 2; else shift; fi\n\ + done\n\ + test -n \"$manifest\"\n\ + test -d \"$manifest\"\n\ + printf '%s\\n' \"$manifest\" > '{}'\n\ + for member in \"$manifest\"/*; do\n\ + test -L \"$member\"\n\ + readlink \"$member\" >> '{}'\n\ + done\n\ + cat '{}'\n", + manifest_path.display(), + manifest_members.display(), + stream.display(), + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16", + }))) + .unwrap(); + let mut activities = read_nfcapd_daily_source_activities( + &[first.clone(), second.clone()], + std::slice::from_ref(&selection), + &executable, + ) + .unwrap(); + let activity = activities.pop().unwrap(); + + let source = IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)); + assert_eq!(activity[&source].flows, 3); + assert_eq!( + fs::read_to_string(manifest_members).unwrap(), + format!("{}\n{}\n", first.display(), second.display()) + ); + let manifest_directory = fs::read_to_string(manifest_path).unwrap(); + assert!(!Path::new(manifest_directory.trim()).exists()); + } + + #[cfg(unix)] + #[test] + fn daily_activity_reader_keeps_a_qualifying_synthetic_tunnel_flow_once() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempdir().unwrap(); + let executable = directory.path().join("fake-nfdump"); + let stream = directory.path().join("stream.bin"); + let invocation_log = directory.path().join("invocation.log"); + + let mut synthetic: [u8; 72] = ONE_V4_BINARY_STREAM[16..88].try_into().unwrap(); + synthetic[..16].fill(0); + synthetic[..4].copy_from_slice(&[10, 0, 0, 1]); + synthetic[16..32].fill(0); + synthetic[16..20].copy_from_slice(&[10, 0, 0, 2]); + synthetic[32..40].copy_from_slice(&210_u64.to_le_bytes()); + synthetic[40..48].copy_from_slice(&4_467_904_u64.to_le_bytes()); + synthetic[48..56].copy_from_slice(&1_u64.to_le_bytes()); + synthetic[64..66].copy_from_slice(&22_222_u16.to_le_bytes()); + synthetic[66..68].copy_from_slice(&80_u16.to_le_bytes()); + synthetic[68] = 6; + synthetic[69] = 0b010; + synthetic[70..72].fill(0); + + let mut outer = synthetic; + outer[..16].fill(0); + outer[..4].copy_from_slice(&[72, 138, 170, 101]); + outer[16..32].fill(0); + outer[16..20].copy_from_slice(&[42, 16, 32, 6]); + outer[48..56].copy_from_slice(&7_u64.to_le_bytes()); + outer[70..72].copy_from_slice(&[40, 255]); + + let mut binary = Vec::new(); + binary.extend_from_slice(&ONE_V4_BINARY_STREAM[..12]); + binary.extend_from_slice(&2_u32.to_le_bytes()); + binary.extend_from_slice(&synthetic); + binary.extend_from_slice(&outer); + binary.extend_from_slice(&[0, 0, 0, 0]); + fs::write(&stream, binary).unwrap(); + fs::write( + &executable, + format!( + "#!/bin/sh\nset -eu\nprintf '%s\\n' \"$*\" > '{}'\ncat '{}'\n", + invocation_log.display(), + stream.display(), + ), + ) + .unwrap(); + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + + let capture = directory.path().join("nfcapd.202504151200"); + fs::write(&capture, "fixture").unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "kind": "daily_active_sources", + "ip_prefix": "10.0.0.0/16", + }))) + .unwrap(); + + let mut activities = read_nfcapd_daily_source_activities( + std::slice::from_ref(&capture), + std::slice::from_ref(&selection), + &executable, + ) + .unwrap(); + let activity = activities.pop().unwrap(); + + let source = IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 1)); + assert_eq!(activity.len(), 1); + assert_eq!( + activity[&source], + nfdump::SourceActivity { + flows: 1, + packets: 20, + bytes: 2_000, + } + ); + + let active_sources = Arc::new([source].into_iter().collect::()); + let bucket = read_nfcapd_bucket_with_active_sources( + &capture, + "edge-a", + &selection, + active_sources, + &executable, + "America/Los_Angeles", + ) + .unwrap(); + let metrics = &bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics; + assert_eq!(metrics.flows, 1); + assert_eq!(metrics.packets, 210); + assert_eq!(metrics.bytes, 4_467_904); + + let invocation = fs::read_to_string(invocation_log).unwrap(); + assert!(invocation.contains("src net 10.0.0.0/16")); + assert!(invocation.contains("src tun net 10.0.0.0/16")); + assert!(invocation.contains("tun proto tcp or tun proto udp")); + } + #[cfg(unix)] #[test] fn nfdump_decoder_builds_the_canonical_bucket() { diff --git a/tools/netflow-db/src/main.rs b/tools/netflow-db/src/main.rs index e047d5b..9525fe5 100644 --- a/tools/netflow-db/src/main.rs +++ b/tools/netflow-db/src/main.rs @@ -61,8 +61,9 @@ enum Command { struct PipelineArgs { #[arg(long, conflicts_with = "dataset")] config: Option, + /// Dataset ID. Repeat --dataset for two or more values to start one coordinated fixed daily-active subset run; one value keeps the normal single-dataset path. #[arg(long, requires = "start_date", conflicts_with = "config")] - dataset: Option, + dataset: Vec, #[arg(long)] start_date: Option, #[arg(long)] @@ -77,6 +78,13 @@ struct PipelineArgs { datasets: Option, #[arg(long)] ip_prefix: Option, + /// Select qualifying flows from sources active over each complete local day. + #[arg( + long, + requires = "ip_prefix", + conflicts_with_all = ["src_visibility", "dst_visibility"] + )] + daily_active_sources: bool, #[arg(long, value_enum)] src_visibility: Option, #[arg(long, value_enum)] @@ -328,14 +336,24 @@ fn main() -> Result<()> { } fn run_pipeline(args: PipelineArgs) -> Result<()> { - let selection = serde_json::json!({ - "ip_prefix": args.ip_prefix, - "src_visibility": args.src_visibility.map(VisibilityArg::as_str), - "dst_visibility": args.dst_visibility.map(VisibilityArg::as_str), - }); - let report = netflow_db::pipeline::run(netflow_db::pipeline::PipelineRequest { + let mut selection = serde_json::Map::new(); + if args.daily_active_sources { + selection.insert("kind".into(), serde_json::json!("daily_active_sources")); + } + selection.insert("ip_prefix".into(), serde_json::json!(args.ip_prefix)); + selection.insert( + "src_visibility".into(), + serde_json::json!(args.src_visibility.map(VisibilityArg::as_str)), + ); + selection.insert( + "dst_visibility".into(), + serde_json::json!(args.dst_visibility.map(VisibilityArg::as_str)), + ); + let selection = serde_json::Value::Object(selection); + let dataset_ids = args.dataset; + let request = netflow_db::pipeline::PipelineRequest { config_path: args.config, - dataset_id: args.dataset, + dataset_id: (dataset_ids.len() == 1).then(|| dataset_ids[0].clone()), datasets_path: args.datasets, start_date: args.start_date, end_date: args.end_date, @@ -347,13 +365,25 @@ fn run_pipeline(args: PipelineArgs) -> Result<()> { force: args.force, run_maad: !args.no_maad, require_complete: args.require_complete, - })?; + }; + let report = if dataset_ids.len() > 1 { + netflow_db::pipeline::run_many(request, dataset_ids)? + } else { + netflow_db::pipeline::run(request)? + }; println!( "Five-minute coverage: {} complete, {} partial, {} unknown", report.complete_five_minute_buckets, report.partial_five_minute_buckets, report.unknown_five_minute_buckets ); + println!( + "Published five-minute buckets: {}", + report.five_minute_buckets + ); + if report.skipped_inputs != 0 { + println!("Skipped inputs: {}", report.skipped_inputs); + } Ok(()) } diff --git a/tools/netflow-db/src/nfdump.rs b/tools/netflow-db/src/nfdump.rs index 2fcbcaa..4a8a211 100644 --- a/tools/netflow-db/src/nfdump.rs +++ b/tools/netflow-db/src/nfdump.rs @@ -1,16 +1,18 @@ //! Private decoder for the Atlantis Flow Stream emitted by the nfdump fork. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::fmt; use std::io::{self, Read}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::sync::Arc; use fixedbitset::FixedBitSet; use crate::{ coverage::BucketCoverage, domain::{ - AddressSet, AddressSide, BucketKey, CanonicalBucket, ExactVisibility, FlowSelection, + AddressSet, AddressSide, BucketKey, CanonicalBucket, DAILY_ACTIVE_MIN_BYTES, + DAILY_ACTIVE_MIN_FLOWS, DAILY_ACTIVE_MIN_PACKETS, ExactVisibility, FlowSelection, Granularity, IpVersion, Scope, ScopedAddresses, ScopedPorts, ScopedProtocols, ScopedTraffic, TrafficMetrics, Visibility, }, @@ -130,6 +132,9 @@ enum ErrorReason { NonzeroIcmpDestinationPort(u16), InvalidTtlOrder { minimum: u8, maximum: u8 }, AggregateOverflow, + DailyActivityRequiresSelection, + MissingDailyActiveSources, + DailyActivityRequiresDailyActiveSourceSelection, } impl fmt::Display for ErrorReason { @@ -178,10 +183,42 @@ impl fmt::Display for ErrorReason { ) } Self::AggregateOverflow => formatter.write_str("exceeds signed 64-bit aggregate range"), + Self::DailyActivityRequiresSelection => { + formatter.write_str("daily source activity requires at least one selection") + } + Self::MissingDailyActiveSources => formatter + .write_str("daily active-source selection was not resolved for this local day"), + Self::DailyActivityRequiresDailyActiveSourceSelection => formatter + .write_str("daily source activity requires a daily_active_sources selection"), } } } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct SourceActivity { + pub flows: i64, + pub packets: i64, + pub bytes: i64, +} + +impl SourceActivity { + /// Qualification only needs threshold-capped counters, keeping addition overflow-free. + pub fn include(&mut self, other: Self) { + self.flows = self + .flows + .saturating_add(other.flows) + .min(DAILY_ACTIVE_MIN_FLOWS); + self.packets = self + .packets + .saturating_add(other.packets) + .min(DAILY_ACTIVE_MIN_PACKETS); + self.bytes = self + .bytes + .saturating_add(other.bytes) + .min(DAILY_ACTIVE_MIN_BYTES); + } +} + #[derive(Debug)] pub(crate) struct NfdumpError { phase: Phase, @@ -356,7 +393,7 @@ pub(crate) fn reduce_to_bucket( key: BucketKey, selection: &FlowSelection, ) -> Result { - match reduce_stream(&mut input, selection) { + match reduce_stream(&mut input, selection, None) { Ok(scopes) => Ok(finish_bucket(scopes, key)), Err(error) => { drain_to_eof(&mut input); @@ -365,10 +402,190 @@ pub(crate) fn reduce_to_bucket( } } +pub(crate) fn reduce_to_bucket_with_active_sources( + mut input: R, + key: BucketKey, + selection: &FlowSelection, + active_sources: &AddressSet, +) -> Result { + match reduce_stream(&mut input, selection, Some(active_sources)) { + Ok(scopes) => Ok(finish_bucket(scopes, key)), + Err(error) => { + drain_to_eof(&mut input); + Err(error) + } + } +} + +pub(crate) fn reduce_to_buckets_with_active_sources( + mut input: R, + key: BucketKey, + selections_and_active_sources: &[(FlowSelection, Arc)], +) -> Result, NfdumpError> { + match reduce_stream_for_active_sources(&mut input, selections_and_active_sources) { + Ok(scopes) => Ok(scopes + .into_iter() + .map(|scopes| finish_bucket(scopes, key.clone())) + .collect()), + Err(error) => { + drain_to_eof(&mut input); + Err(error) + } + } +} + +pub(crate) fn reduce_to_daily_source_activities( + mut input: R, + selections: &[FlowSelection], +) -> Result>, NfdumpError> { + validate_daily_active_selections(selections)?; + let mut activities = (0..selections.len()) + .map(|_| HashMap::::new()) + .collect::>(); + let result = visit_stream(&mut input, |flow, _, _| { + for (selection, activity) in selections.iter().zip(&mut activities) { + if !matches_selection(flow, selection) { + continue; + } + let entry = activity.entry(flow.source_address).or_default(); + entry.include(SourceActivity { + flows: flow.flow_count, + packets: flow.packets, + bytes: flow.bytes, + }); + } + Ok(()) + }); + match result { + Ok(()) => Ok(activities), + Err(error) => { + drain_to_eof(&mut input); + Err(error) + } + } +} + +fn reduce_stream_for_active_sources( + input: &mut impl Read, + selections_and_active_sources: &[(FlowSelection, Arc)], +) -> Result, NfdumpError> { + validate_daily_active_selection_pairs(selections_and_active_sources)?; + let mut scopes = (0..selections_and_active_sources.len()) + .map(|_| std::array::from_fn(|_| ScopeAccumulator::default())) + .collect::>(); + visit_stream(input, |flow, block_index, record_ordinal| { + for ((selection, active_sources), scopes) in + selections_and_active_sources.iter().zip(&mut scopes) + { + if !matches_selection(flow, selection) || !active_sources.contains(&flow.source_address) + { + continue; + } + add_flow_to_scopes(scopes, flow, block_index, record_ordinal)?; + } + Ok(()) + })?; + Ok(scopes) +} + +fn validate_daily_active_selections(selections: &[FlowSelection]) -> Result<(), NfdumpError> { + if selections.is_empty() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresSelection, + )); + } + if selections + .iter() + .any(|selection| !selection.selects_daily_active_sources()) + { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection, + )); + } + Ok(()) +} + +fn validate_daily_active_selection_pairs( + selections_and_active_sources: &[(FlowSelection, Arc)], +) -> Result<(), NfdumpError> { + if selections_and_active_sources.is_empty() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresSelection, + )); + } + if selections_and_active_sources + .iter() + .any(|(selection, _)| !selection.selects_daily_active_sources()) + { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection, + )); + } + Ok(()) +} + fn reduce_stream( input: &mut R, selection: &FlowSelection, + active_sources: Option<&AddressSet>, ) -> Result<[ScopeAccumulator; 10], NfdumpError> { + if selection.selects_daily_active_sources() && active_sources.is_none() { + return Err(NfdumpError::new( + Phase::Aggregate, + Field::SourceAddress, + ErrorReason::MissingDailyActiveSources, + )); + } + let mut scopes = std::array::from_fn(|_| ScopeAccumulator::default()); + visit_stream(input, |flow, block_index, record_ordinal| { + if !matches_selection(flow, selection) + || active_sources.is_some_and(|sources| !sources.contains(&flow.source_address)) + { + return Ok(()); + } + add_flow_to_scopes(&mut scopes, flow, block_index, record_ordinal) + })?; + Ok(scopes) +} + +fn add_flow_to_scopes( + scopes: &mut [ScopeAccumulator; 10], + flow: &Flow, + block_index: u64, + record_ordinal: u64, +) -> Result<(), NfdumpError> { + let family_base = if flow.ip_version == IpVersion::V4 { + 0 + } else { + 5 + }; + let exact_index = + family_base + exact_scope_index(flow.source_anonymized, flow.destination_anonymized); + for index in [family_base, exact_index] { + scopes[index].validate_add(flow).map_err(|field| { + NfdumpError::new(Phase::Aggregate, field, ErrorReason::AggregateOverflow) + .at_block(block_index) + .at_record(record_ordinal) + })?; + } + for index in [family_base, exact_index] { + scopes[index].add(flow); + } + Ok(()) +} + +fn visit_stream(input: &mut R, mut visit: F) -> Result<(), NfdumpError> +where + F: FnMut(&Flow, u64, u64) -> Result<(), NfdumpError>, +{ let mut header = [0_u8; 12]; let read = read_fully(input, &mut header).map_err(|failure| { NfdumpError::new( @@ -411,7 +628,6 @@ fn reduce_stream( )); } - let mut scopes = std::array::from_fn(|_| ScopeAccumulator::default()); let mut payload = [0_u8; MAX_BLOCK_BYTES]; let mut block_index = 1_u64; let mut record_ordinal = 0_u64; @@ -447,7 +663,7 @@ fn reduce_stream( let count = u32::from_le_bytes(count_bytes); if count == 0 { ensure_eof(input, block_index)?; - return Ok(scopes); + return Ok(()); } if count > MAX_BLOCK_RECORDS as u32 { return Err(NfdumpError::new( @@ -484,33 +700,8 @@ fn reduce_stream( for record in payload[..payload_len].chunks_exact(RECORD_LEN) { record_ordinal += 1; let validated = validate_record(record, block_index, record_ordinal)?; - if !matches_visibility(&validated, selection) { - continue; - } let flow = validated.into_flow(); - if selection.ip_prefix().is_some_and(|prefix| { - !prefix.contains(&flow.source_address) - && !prefix.contains(&flow.destination_address) - }) { - continue; - } - let family_base = if flow.ip_version == IpVersion::V4 { - 0 - } else { - 5 - }; - let exact_index = family_base - + exact_scope_index(flow.source_anonymized, flow.destination_anonymized); - for index in [family_base, exact_index] { - scopes[index].validate_add(&flow).map_err(|field| { - NfdumpError::new(Phase::Aggregate, field, ErrorReason::AggregateOverflow) - .at_block(block_index) - .at_record(record_ordinal) - })?; - } - for index in [family_base, exact_index] { - scopes[index].add(&flow); - } + visit(&flow, block_index, record_ordinal)?; } block_index += 1; } @@ -710,12 +901,23 @@ where i64::try_from(value).map_err(|_| error(field, ErrorReason::NumericOverflow(value))) } -fn matches_visibility(record: &ValidatedRecord<'_>, selection: &FlowSelection) -> bool { - selection.src_visibility().is_none_or(|required| { - matches!(required, ExactVisibility::Anonymized) == record.source_anonymized - }) && selection.dst_visibility().is_none_or(|required| { - matches!(required, ExactVisibility::Anonymized) == record.destination_anonymized - }) +fn matches_selection(flow: &Flow, selection: &FlowSelection) -> bool { + selection.matches_flow_fields( + flow.source_address, + flow.destination_address, + flow.protocol, + Some(flow.source_port), + if flow.source_anonymized { + ExactVisibility::Anonymized + } else { + ExactVisibility::Literal + }, + if flow.destination_anonymized { + ExactVisibility::Anonymized + } else { + ExactVisibility::Literal + }, + ) } const fn exact_scope_index(source_anonymized: bool, destination_anonymized: bool) -> usize { @@ -924,6 +1126,7 @@ fn traffic_metrics(metrics: [i64; METRIC_COUNT]) -> TrafficMetrics { mod tests { use std::io::{self, Cursor, Read}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + use std::sync::Arc; use super::*; use crate::domain::{Granularity, Scope}; @@ -943,6 +1146,31 @@ mod tests { .expect("the exported fixture contains one fixed record") } + fn daily_selection(prefix: &str) -> FlowSelection { + FlowSelection::from_payload(Some(&serde_json::json!({ + "kind": "daily_active_sources", + "ip_prefix": prefix, + }))) + .unwrap() + } + + fn daily_record( + source: [u8; 4], + tag: u8, + flows: u64, + packets: u64, + bytes: u64, + ) -> [u8; RECORD_LEN] { + let mut record = base_record(); + record[..4].copy_from_slice(&source); + record[32..40].copy_from_slice(&packets.to_le_bytes()); + record[40..48].copy_from_slice(&bytes.to_le_bytes()); + record[48..56].copy_from_slice(&flows.to_le_bytes()); + record[64..66].copy_from_slice(&55_000_u16.to_le_bytes()); + record[69] = tag; + record + } + fn stream(records: &[[u8; RECORD_LEN]]) -> Vec { let mut bytes = vec![65, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0]; bytes.extend_from_slice(&(records.len() as u32).to_le_bytes()); @@ -1109,6 +1337,175 @@ mod tests { assert_eq!(bucket.protocols[0].protocols, ["47", "6"]); } + #[test] + fn daily_activity_resolves_exact_sources_before_bucket_reduction() { + let selection = FlowSelection::from_payload(Some(&serde_json::json!({ + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + }))) + .unwrap(); + let mut first = base_record(); + first[32..40].copy_from_slice(&5_u64.to_le_bytes()); + first[40..48].copy_from_slice(&500_u64.to_le_bytes()); + first[48..56].copy_from_slice(&1_u64.to_le_bytes()); + first[64..66].copy_from_slice(&55_000_u16.to_le_bytes()); + first[66..68].copy_from_slice(&443_u16.to_le_bytes()); + first[69] = 0b010; + let mut second = first; + second[32..40].copy_from_slice(&15_u64.to_le_bytes()); + second[40..48].copy_from_slice(&1_500_u64.to_le_bytes()); + second[48..56].copy_from_slice(&2_u64.to_le_bytes()); + let mut inactive = first; + inactive[0..4].copy_from_slice(&[192, 0, 3, 2]); + inactive[32..40].copy_from_slice(&19_u64.to_le_bytes()); + inactive[40..48].copy_from_slice(&1_999_u64.to_le_bytes()); + inactive[48..56].copy_from_slice(&2_u64.to_le_bytes()); + + let mut activities = reduce_to_daily_source_activities( + Cursor::new(stream(&[first, second, inactive])), + std::slice::from_ref(&selection), + ) + .unwrap(); + let activity = activities.pop().unwrap(); + assert_eq!( + activity[&IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))], + SourceActivity { + flows: 3, + packets: 20, + bytes: 2_000, + } + ); + let mut active = AddressSet::default(); + active.extend(activity.into_iter().filter_map(|(address, metrics)| { + FlowSelection::daily_activity_threshold_met( + metrics.flows, + metrics.packets, + metrics.bytes, + ) + .then_some(address) + })); + let mut low_port = first; + low_port[64..66].copy_from_slice(&1_023_u16.to_le_bytes()); + + let bucket = reduce_to_bucket_with_active_sources( + Cursor::new(stream(&[first, inactive, low_port])), + key(), + &selection, + &active, + ) + .unwrap(); + + assert_eq!(active.len(), 1); + assert_eq!(bucket.traffic[0].metrics.flows, 1); + assert_eq!(bucket.traffic[0].metrics.packets, 5); + assert_eq!(bucket.traffic[0].metrics.bytes, 500); + assert!(reduce_to_bucket(Cursor::new(stream(&[first])), key(), &selection).is_err()); + } + + #[test] + fn daily_activity_maps_are_reduced_independently_with_overlapping_prefixes() { + let source_a = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + let source_b = IpAddr::V4(Ipv4Addr::new(198, 51, 2, 1)); + let records = [ + daily_record([192, 0, 2, 1], 0b010, 3, 20, 2_000), + daily_record([198, 51, 2, 1], 0b010, 4, 21, 2_100), + daily_record([192, 0, 2, 1], 0, 1, 1, 1), + ]; + let selections = [ + daily_selection("192.0.0.0/16"), + daily_selection("198.51.0.0/16"), + daily_selection("192.0.0.0/16"), + ]; + + let activities = + reduce_to_daily_source_activities(Cursor::new(stream(&records)), &selections).unwrap(); + + assert_eq!(activities.len(), 3); + assert_eq!(activities[0][&source_a].flows, 3); + assert_eq!(activities[0][&source_a].packets, 20); + assert!(!activities[0].contains_key(&source_b)); + assert_eq!(activities[1][&source_b].flows, 3); + assert!(!activities[1].contains_key(&source_a)); + assert_eq!(activities[2], activities[0]); + } + + #[test] + fn active_source_buckets_fan_out_per_pair_and_allow_overlap() { + let source_a = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + let source_b = IpAddr::V4(Ipv4Addr::new(198, 51, 2, 1)); + let selection_a = daily_selection("192.0.0.0/16"); + let selection_b = daily_selection("198.51.0.0/16"); + let pairs = [ + ( + selection_a.clone(), + Arc::new([source_a].into_iter().collect::()), + ), + ( + selection_a.clone(), + Arc::new([source_b].into_iter().collect::()), + ), + ( + selection_a, + Arc::new([source_a, source_b].into_iter().collect::()), + ), + ( + selection_b, + Arc::new([source_b].into_iter().collect::()), + ), + ]; + let records = [ + daily_record([192, 0, 2, 1], 0b010, 3, 20, 2_000), + daily_record([198, 51, 2, 1], 0b010, 4, 21, 2_100), + ]; + + let buckets = + reduce_to_buckets_with_active_sources(Cursor::new(stream(&records)), key(), &pairs) + .unwrap(); + + assert_eq!(buckets.len(), 4); + let all_v4 = |bucket: &CanonicalBucket| { + bucket + .traffic + .iter() + .find(|entry| { + entry.scope == Scope::new(IpVersion::V4, Visibility::All, Visibility::All) + }) + .unwrap() + .metrics + .flows + }; + assert_eq!(buckets.iter().map(all_v4).collect::>(), [3, 0, 3, 4]); + assert_eq!( + buckets[0].addresses[1].addresses, + [source_a].into_iter().collect::() + ); + assert_eq!( + buckets[3].addresses[1].addresses, + [source_b].into_iter().collect::() + ); + } + + #[test] + fn multi_subset_decoding_rejects_empty_and_non_daily_inputs() { + let empty = + reduce_to_daily_source_activities(Cursor::new(Vec::::new()), &[]).unwrap_err(); + assert!(matches!( + empty.reason, + ErrorReason::DailyActivityRequiresSelection + )); + + let non_daily = reduce_to_buckets_with_active_sources( + Cursor::new(Vec::::new()), + key(), + &[(FlowSelection::default(), Arc::new(AddressSet::default()))], + ) + .unwrap_err(); + assert!(matches!( + non_daily.reason, + ErrorReason::DailyActivityRequiresDailyActiveSourceSelection + )); + } + #[test] fn malformed_excluded_record_is_rejected_and_input_is_drained() { let mut record = base_record(); diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index 0f8b89f..5f805e0 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -2,10 +2,11 @@ use std::{ borrow::Cow, - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashMap}, fs, + net::IpAddr, path::{Path, PathBuf}, - time::{Duration, Instant}, + sync::Arc, }; use jiff::{RoundMode, Timestamp, ToSpan, Unit, ZonedRound, civil::Date}; @@ -19,36 +20,71 @@ use crate::{ config::{ConfigError, CsvSourceConfig}, coverage::BucketCoverage, domain::{ - BucketKey, CanonicalBucket, DomainError, FlowSelection, Granularity, StatisticalBucket, - StatisticalBucketIncludeProfile, + AddressSet, BucketKey, CanonicalBucket, DomainError, FlowSelection, Granularity, + StatisticalBucket, }, ingest::{self, IngestError, ProducerError}, nfdump, provenance::{ - ExpectedAbsence, FileSnapshot, InputRevision, ProvenanceError, capture_file_revision, - csv_decoder_fingerprint, nfcapd_decoder_fingerprint, revision_for_locator, - verify_file_snapshot, + ExecutableRevision, ExpectedAbsence, FileSnapshot, InputRevision, ProvenanceError, + capture_file_revision, csv_decoder_fingerprint, nfcapd_decoder_fingerprint, + revision_for_locator, verify_file_snapshot, }, - publish::{PublishError, WriteBucketsProfile, write_buckets, write_buckets_profiled}, + publish::{PublishError, write_buckets}, registry::{Dataset, DatasetRegistry, DatasetSource, RegistryError, is_safe_path_component}, storage::{ - BucketCoverageRow, DatabaseOperationLock, DatasetMetadata, InputBucket, InputEvidenceRow, - InputEvidenceState, InputKind, InputStatus, ProductIdentity, SourceDefinition, - StatsBucketKey, StorageError, bind_nfcapd_source_layout, bind_product_identity, - cached_content_fingerprint, complete_input_scan, connect_pipeline_writer, - delete_stats_bucket_keys, earliest_traffic_bucket_start, init_schema, - input_scan_fully_processed, insert_bucket_coverage_rows, mark_input_bucket_status, - nfcapd_logical_bucket_processed, optimize_all_query_planner_statistics, - query_bucket_coverage, query_input_evidence, replace_input_evidence, - set_dataset_default_start_date, upsert_dataset_metadata, upsert_input_bucket, + DatabaseOperationLock, DatasetMetadata, InputBucket, InputEvidenceRow, InputEvidenceState, + InputKind, InputStatus, ProductIdentity, SourceDefinition, StorageError, + bind_nfcapd_source_layout, bind_product_identity, cached_content_fingerprint, + canonical_path, complete_input_scan, connect_pipeline_writer, current_product_fingerprint, + daily_product_completion_matches, database_related_paths, delete_stats_time_range, + earliest_traffic_bucket_start, init_schema, input_scan_fully_processed, + mark_input_bucket_status, nfcapd_logical_bucket_processed, + optimize_all_query_planner_statistics, query_input_evidence, replace_input_evidence, + set_dataset_default_start_date, upsert_daily_product_completion, upsert_dataset_metadata, + upsert_input_bucket, validate_database_path_separation, }, }; const FIVE_MINUTES: i64 = 300; const NFCAPD_DECODE_BATCH_SIZE: usize = 12; const NFCAPD_REVISION_HASH_MAX_WORKERS: usize = NFCAPD_DECODE_BATCH_SIZE * 2; +const MAX_MISSING_DAY_WARNING_DETAILS: usize = 8; const DEFAULT_TIMEZONE: &str = "America/Los_Angeles"; +fn build_revision_hash_pool() -> Result { + let revision_hash_workers = std::thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .min(NFCAPD_REVISION_HASH_MAX_WORKERS); + rayon::ThreadPoolBuilder::new() + .num_threads(revision_hash_workers) + .thread_name(|index| format!("nfcapd-revision-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build revision hash pool: {error}")) + }) +} + +fn build_nfcapd_decode_pool() -> Result { + rayon::ThreadPoolBuilder::new() + .num_threads(NFCAPD_DECODE_BATCH_SIZE) + .thread_name(|index| format!("nfcapd-decode-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build nfcapd decode pool: {error}")) + }) +} + +fn build_nfcapd_activity_pool() -> Result { + rayon::ThreadPoolBuilder::new() + .num_threads(NFCAPD_DECODE_BATCH_SIZE) + .thread_name(|index| format!("nfcapd-activity-{index}")) + .build() + .map_err(|error| { + PipelineError::InvalidConfig(format!("failed to build nfcapd activity pool: {error}")) + }) +} + #[derive(Clone, Debug)] pub struct PipelineRequest { pub config_path: Option, @@ -190,13 +226,142 @@ struct ResolvedPipeline { database_path: PathBuf, timezone: String, run_maad: bool, - nfdump: String, + nfdump: PathBuf, + nfdump_revision: Option, selection: FlowSelection, inputs: Vec, datasets: Vec, require_complete: bool, } +fn nfdump_control_path(value: &str) -> Option { + let path = Path::new(value); + (path.is_absolute() + || path + .parent() + .is_some_and(|parent| !parent.as_os_str().is_empty()) + || value.contains(['/', '\\'])) + .then(|| path.to_owned()) +} + +fn has_effective_execute_access(path: &Path) -> bool { + nix::unistd::faccessat( + None, + path, + nix::unistd::AccessFlags::X_OK, + nix::fcntl::AtFlags::AT_EACCESS, + ) + .is_ok() +} + +/// Resolve the executable that a nfdump command will select. +/// +/// Resolve explicit paths before output setup as well as bare names. The canonical path is stored +/// in the resolved pipeline so later command invocations use the same executable that preflight +/// checked, and output alias checks see the actual control file. +fn resolved_nfdump_control_path(value: &str) -> Result { + if let Some(path) = nfdump_control_path(value) { + let resolved = canonical_path(&path)?; + let metadata = fs::metadata(&resolved).map_err(|error| { + PipelineError::InvalidConfig(format!( + "cannot resolve explicit nfdump executable {value:?} at {}: {error}", + path.display() + )) + })?; + if !metadata.is_file() { + return Err(PipelineError::InvalidConfig(format!( + "explicit nfdump executable {value:?} at {} is not a regular file", + path.display() + ))); + } + if !has_effective_execute_access(&resolved) { + return Err(PipelineError::InvalidConfig(format!( + "explicit nfdump executable {value:?} at {} is not executable by this process", + path.display() + ))); + } + return Ok(resolved); + } + if value.is_empty() { + return Err(PipelineError::InvalidConfig( + "nfdump executable name is empty".into(), + )); + } + + let path_variable = std::env::var_os("PATH").ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "cannot resolve bare nfdump executable {value:?}: PATH is not set" + )) + })?; + for directory in std::env::split_paths(&path_variable) { + // An empty PATH component means the current working directory for process lookup. + let directory = if directory.as_os_str().is_empty() { + std::env::current_dir()? + } else { + directory + }; + let candidate = directory.join(value); + let Ok(metadata) = fs::metadata(&candidate) else { + continue; + }; + if !metadata.is_file() { + continue; + } + if !has_effective_execute_access(&candidate) { + continue; + } + return Ok(canonical_path(candidate)?); + } + + Err(PipelineError::InvalidConfig(format!( + "cannot resolve bare nfdump executable {value:?} through PATH" + ))) +} + +/// Capture and validate the exact native decoder before output setup. The digest is paid once; +/// every later boundary uses only the stored file snapshot. +fn resolve_nfdump_revision(value: &str) -> Result<(PathBuf, ExecutableRevision), PipelineError> { + let path = resolved_nfdump_control_path(value)?; + let revision = ExecutableRevision::capture(&path)?; + Ok((path, revision)) +} + +fn verify_nfdump_revision(pipeline: &ResolvedPipeline) -> Result<(), PipelineError> { + if let Some(revision) = &pipeline.nfdump_revision { + verify_nfdump_revision_snapshot(revision)?; + } + Ok(()) +} + +fn verify_nfdump_revision_snapshot(revision: &ExecutableRevision) -> Result<(), PipelineError> { + verify_file_snapshot(Path::new(&revision.locator), &revision.snapshot).map_err(|error| { + PipelineError::InvalidConfig(format!( + "nfdump executable changed during pipeline execution at {}: {error}", + revision.locator + )) + }) +} + +fn nfdump_decoder_fingerprint_for_pipeline( + pipeline: &ResolvedPipeline, +) -> Result { + if let Some(revision) = &pipeline.nfdump_revision { + return Ok(revision.decoder_fingerprint.clone()); + } + // Manually assembled test pipelines can exercise native helpers without going through + // request resolution. Production native requests always carry a revision. + Ok(nfcapd_decoder_fingerprint()?) +} + +fn inputs_require_nfdump(inputs: &[InputSpec]) -> bool { + inputs.iter().any(|input| { + matches!( + input, + InputSpec::Nfcapd { .. } | InputSpec::NfcapdTree { .. } + ) + }) +} + fn default_timezone() -> String { DEFAULT_TIMEZONE.into() } @@ -208,6 +373,417 @@ pub fn run( execute(pipeline) } +/// Run several registry datasets as coordinated daily-active-source products. +/// +/// The datasets share the same frozen discovery plan, day loop, and nfdump work used by [`run`], +/// while each output retains its own product identity, transaction, and completion markers. +pub fn run_many( + request: impl std::borrow::Borrow, + dataset_ids: Vec, +) -> Result { + let request = request.borrow(); + if dataset_ids.len() < 2 { + return Err(PipelineError::InvalidConfig( + "coordinated pipeline mode requires at least two --dataset values".into(), + )); + } + if request.config_path.is_some() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode cannot combine --config with repeated --dataset".into(), + )); + } + if request.database_path.is_some() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode cannot override --database-path".into(), + )); + } + if selection_override_requested(&request.selection) { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode cannot override registry selections from the CLI".into(), + )); + } + if request.start_time.is_some() || request.end_time.is_some() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode requires a whole-day date window; --start-time and --end-time are unsupported".into(), + )); + } + + let mut seen = BTreeSet::new(); + if let Some(duplicate) = dataset_ids.iter().find(|id| !seen.insert(id.as_str())) { + return Err(PipelineError::InvalidConfig(format!( + "coordinated dataset mode cannot repeat dataset {duplicate:?}" + ))); + } + + let repository_root = std::env::current_dir()?; + let registry_path = request + .datasets_path + .clone() + .unwrap_or_else(|| DatasetRegistry::default_path(&repository_root)); + let registry = load_dataset_registry(®istry_path, &repository_root)?; + let shared_nfdump = resolve_nfdump_revision(&request.nfdump)?; + let mut pipelines = Vec::with_capacity(dataset_ids.len()); + for dataset_id in &dataset_ids { + let mut single = request.clone(); + single.dataset_id = Some(dataset_id.clone()); + pipelines.push(resolve_dataset_request( + &single, + ®istry, + Some((&shared_nfdump.0, &shared_nfdump.1)), + )?); + } + execute_many(validate_compatible_pipelines(pipelines)?) +} + +fn selection_override_requested(value: &Value) -> bool { + value + .as_object() + .is_some_and(|object| object.values().any(|entry| !entry.is_null())) +} + +struct CompatiblePlan { + pipelines: Vec, + tree: FrozenNfcapdTreeLayout, +} + +fn validate_compatible_pipelines( + pipelines: Vec, +) -> Result { + let Some(first) = pipelines.first() else { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode requires at least two datasets".into(), + )); + }; + if !first.selection.selects_daily_active_sources() { + return Err(PipelineError::InvalidConfig( + "coordinated dataset mode requires every registry selection to be daily_active_sources" + .into(), + )); + } + let first_input = only_nfcapd_tree(first)?; + let first_config = nfcapd_tree_config(first_input)?; + let output_paths = pipelines + .iter() + .map(|pipeline| pipeline.database_path.as_path()) + .collect::>(); + validate_database_path_separation(&output_paths)?; + let tree = freeze_nfcapd_tree( + first_input, + &first.selection, + &first.timezone, + &output_paths, + )?; + for pipeline in pipelines.iter().skip(1) { + if !pipeline.selection.selects_daily_active_sources() { + return Err(PipelineError::InvalidConfig(format!( + "dataset {:?} does not use a daily_active_sources selection", + pipeline + .datasets + .first() + .map(|dataset| dataset.dataset_id.as_str()) + .unwrap_or("") + ))); + } + let input = only_nfcapd_tree(pipeline)?; + let config = nfcapd_tree_config(input)?; + if tree.root_path != fs::canonicalize(config.root_path)? { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfcapd root".into(), + )); + } + if first_config.start_date != config.start_date + || first_config.end_date != config.end_date + || first_config.start_time != config.start_time + || first_config.end_time != config.end_time + || first_config.force != config.force + { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same whole-day window and force settings".into(), + )); + } + if first.timezone != pipeline.timezone { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same timezone".into(), + )); + } + if first.run_maad != pipeline.run_maad { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same MAAD setting".into(), + )); + } + if first.nfdump != pipeline.nfdump { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfdump executable/configuration".into(), + )); + } + let same_executable_revision = match (&first.nfdump_revision, &pipeline.nfdump_revision) { + (Some(left), Some(right)) => { + left.locator == right.locator + && left.content_fingerprint == right.content_fingerprint + && left.decoder_fingerprint == right.decoder_fingerprint + } + (None, None) => true, + _ => false, + }; + if !same_executable_revision { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same nfdump executable revision".into(), + )); + } + if first.require_complete != pipeline.require_complete { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same coverage settings".into(), + )); + } + if tree.sources != canonical_logical_sources(input)? { + return Err(PipelineError::InvalidConfig( + "coordinated datasets must use the same logical source layout and membership" + .into(), + )); + } + } + if first.nfdump_revision.is_some() { + ingest::probe_nfdump_compatibility(&first.nfdump)?; + } + Ok(CompatiblePlan { pipelines, tree }) +} + +fn only_nfcapd_tree(pipeline: &ResolvedPipeline) -> Result<&InputSpec, PipelineError> { + if pipeline.inputs.len() != 1 { + return Err(PipelineError::InvalidConfig( + "coordinated datasets require exactly one nfcapd_tree input".into(), + )); + } + match pipeline.inputs.first() { + Some(input @ InputSpec::NfcapdTree { .. }) => Ok(input), + _ => Err(PipelineError::InvalidConfig( + "coordinated datasets require an nfcapd_tree input".into(), + )), + } +} + +struct NfcapdTreeConfig<'a> { + root_path: &'a Path, + start_date: &'a str, + end_date: Option<&'a str>, + start_time: Option<&'a str>, + end_time: Option<&'a str>, + force: bool, +} + +fn nfcapd_tree_config(input: &InputSpec) -> Result, PipelineError> { + let InputSpec::NfcapdTree { + root_path, + start_date, + end_date, + start_time, + end_time, + force, + .. + } = input + else { + return Err(PipelineError::InvalidConfig( + "coordinated datasets require an nfcapd_tree input".into(), + )); + }; + Ok(NfcapdTreeConfig { + root_path, + start_date, + end_date: end_date.as_deref(), + start_time: start_time.as_deref(), + end_time: end_time.as_deref(), + force: *force, + }) +} + +fn canonical_logical_sources(input: &InputSpec) -> Result, PipelineError> { + let InputSpec::NfcapdTree { + root_path, + source_ids, + sources, + .. + } = input + else { + return Err(PipelineError::InvalidConfig( + "coordinated datasets require an nfcapd_tree input".into(), + )); + }; + let mut sources = normalize_sources(root_path, source_ids, sources)?; + for source in &mut sources { + source.members.sort_unstable(); + } + Ok(sources) +} + +fn paths_overlap(left: &Path, right: &Path) -> bool { + left.starts_with(right) || right.starts_with(left) +} + +fn validate_output_capture_separation( + output_paths: &[&Path], + capture_root: &Path, + member_ids: &[String], +) -> Result<(), PipelineError> { + let capture_root = fs::canonicalize(capture_root)?; + let mut capture_paths = vec![capture_root.clone()]; + for member in member_ids { + capture_paths.push(fs::canonicalize(capture_root.join(member))?); + } + for path in output_paths { + for output in database_related_paths(path)? { + if capture_paths + .iter() + .any(|capture| paths_overlap(&output, capture)) + { + return Err(PipelineError::InvalidConfig(format!( + "output database {} overlaps the nfcapd capture tree {}", + path.display(), + capture_root.display() + ))); + } + } + } + Ok(()) +} + +fn validate_daily_active_source_layout( + sources: &[DatasetSource], + physical_ids: &[String], +) -> Result<(), PipelineError> { + if sources.is_empty() || physical_ids.is_empty() { + return Err(PipelineError::InvalidConfig( + "daily_active_sources requires at least one logical and physical source".into(), + )); + } + Ok(()) +} + +#[derive(Clone, Debug)] +struct FrozenNfcapdTreeLayout { + root_path: PathBuf, + sources: Vec, + physical_ids: Vec, + by_member_and_start: BTreeMap<(String, i64), PathBuf>, + member_bounds: BTreeMap, + start: i64, + end: i64, + extend_gaps_to_window: bool, + force: bool, +} + +#[derive(Clone, Debug, Default)] +struct SingleOutputPlan { + trees: BTreeMap, + dataset_sources: BTreeMap>, +} + +fn freeze_nfcapd_tree( + input: &InputSpec, + selection: &FlowSelection, + timezone: &str, + output_paths: &[&Path], +) -> Result { + let InputSpec::NfcapdTree { + root_path, + source_ids, + sources, + start_date, + end_date, + start_time, + end_time, + force, + } = input + else { + return Err(PipelineError::InvalidConfig( + "expected an nfcapd_tree input".into(), + )); + }; + if selection.selects_daily_active_sources() && (start_time.is_some() || end_time.is_some()) { + return Err(PipelineError::InvalidConfig( + "daily_active_sources selection requires whole local calendar days; start_time and end_time are unsupported".into(), + )); + } + + let root_path = fs::canonicalize(root_path)?; + let sources = normalize_sources(&root_path, source_ids, sources)?; + let physical_ids = sources + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + if selection.selects_daily_active_sources() { + validate_daily_active_source_layout(&sources, &physical_ids)?; + } + validate_output_capture_separation(output_paths, &root_path, &physical_ids)?; + + let discovered = ingest::discover_nfcapd_source_paths(&root_path, &physical_ids, timezone)?; + let mut by_member_and_start = BTreeMap::new(); + let mut member_bounds = BTreeMap::new(); + for input in discovered { + member_bounds + .entry(input.source_id.clone()) + .and_modify(|(first, last): &mut (i64, i64)| { + *first = (*first).min(input.bucket_start); + *last = (*last).max(input.bucket_start); + }) + .or_insert((input.bucket_start, input.bucket_start)); + by_member_and_start.insert((input.source_id, input.bucket_start), input.path); + } + let window = resolve_nfcapd_tree_window( + start_date, + end_date.as_deref(), + start_time.as_deref(), + end_time.as_deref(), + by_member_and_start.keys().map(|(_, start)| *start), + timezone, + )?; + + Ok(FrozenNfcapdTreeLayout { + root_path, + sources, + physical_ids, + by_member_and_start, + member_bounds, + start: window.start, + end: window.end, + extend_gaps_to_window: end_date.is_some(), + force: *force, + }) +} + +fn plan_single_output(pipeline: &ResolvedPipeline) -> Result { + let output_path = pipeline.database_path.as_path(); + let output_paths = std::slice::from_ref(&output_path); + let mut trees = BTreeMap::new(); + for (input_index, input) in pipeline.inputs.iter().enumerate() { + if matches!(input, InputSpec::NfcapdTree { .. }) { + trees.insert( + input_index, + freeze_nfcapd_tree(input, &pipeline.selection, &pipeline.timezone, output_paths)?, + ); + } + } + if pipeline.nfdump_revision.is_some() { + ingest::probe_nfdump_compatibility(&pipeline.nfdump)?; + } + + let mut dataset_sources = BTreeMap::new(); + for dataset in &pipeline.datasets { + let dataset_root = fs::canonicalize(&dataset.root_path)?; + let sources = trees + .values() + .find(|tree| tree.root_path == dataset_root) + .map(|tree| tree.sources.clone()) + .unwrap_or(dataset.logical_sources()?); + dataset_sources.insert(dataset.dataset_id.clone(), sources); + } + Ok(SingleOutputPlan { + trees, + dataset_sources, + }) +} + fn resolve_request(request: &PipelineRequest) -> Result { match (&request.config_path, &request.dataset_id) { (Some(_), Some(_)) => return Err(PipelineError::ConflictingModes), @@ -226,6 +802,7 @@ fn resolve_request(request: &PipelineRequest) -> Result Result Result DatasetRegistry::load(path, &repository_root)?, - None => DatasetRegistry::load_default(&repository_root)?, - }; + let registry_path = request + .datasets_path + .clone() + .unwrap_or_else(|| DatasetRegistry::default_path(&repository_root)); + let registry = load_dataset_registry(®istry_path, &repository_root)?; + resolve_dataset_request(request, ®istry, None) +} + +fn load_dataset_registry( + registry_path: &Path, + repository_root: &Path, +) -> Result { + Ok(DatasetRegistry::load(registry_path, repository_root)?) +} + +fn resolve_dataset_request( + request: &PipelineRequest, + registry: &DatasetRegistry, + shared_nfdump: Option<(&Path, &ExecutableRevision)>, +) -> Result { let dataset_id = request .dataset_id .as_deref() @@ -268,12 +870,25 @@ fn resolve_request(request: &PipelineRequest) -> Result (path.to_owned(), Some(revision.clone())), + None => { + let (path, revision) = resolve_nfdump_revision(&request.nfdump)?; + (path, Some(revision)) + } + }; Ok(ResolvedPipeline { database_path: request .database_path @@ -281,7 +896,8 @@ fn resolve_request(request: &PipelineRequest) -> Result Result { FlowSelection::from_payload((!value.is_null()).then_some(value)) } +fn validate_selection_inputs( + selection: &FlowSelection, + inputs: &[InputSpec], +) -> Result<(), PipelineError> { + if selection.selects_daily_active_sources() + && (inputs.len() != 1 || !matches!(inputs.first(), Some(InputSpec::NfcapdTree { .. }))) + { + return Err(PipelineError::InvalidConfig( + "daily_active_sources selection requires exactly one nfcapd_tree input".into(), + )); + } + Ok(()) +} + fn execute(pipeline: ResolvedPipeline) -> Result { + let plan = plan_single_output(&pipeline)?; if let Some(parent) = pipeline.database_path.parent() { fs::create_dir_all(parent)?; } let _lock = DatabaseOperationLock::acquire(&pipeline.database_path, "pipeline build")?; let connection = connect_pipeline_writer(&pipeline.database_path)?; init_schema(&connection)?; - initialize_metadata(&connection, &pipeline)?; + initialize_metadata_with_plan(&connection, &pipeline, &plan)?; let mut report = PipelineReport::default(); let mut csv_inputs = pipeline @@ -329,7 +960,7 @@ fn execute(pipeline: ResolvedPipeline) -> Result .filter(|input| matches!(input, InputSpec::Nfcapd { .. })) .cloned() .collect::>(); - for input in &pipeline.inputs { + for (input_index, input) in pipeline.inputs.iter().enumerate() { match input { InputSpec::Csv { .. } | InputSpec::Nfcapd { .. } => {} InputSpec::CsvTree { @@ -343,28 +974,19 @@ fn execute(pipeline: ResolvedPipeline) -> Result &mapping, )?); } - InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - start_date, - end_date, - start_time, - end_time, - force, - } => process_nfcapd_tree( - &connection, - root_path, - source_ids, - sources, - start_date, - end_date.as_deref(), - start_time.as_deref(), - end_time.as_deref(), - *force, - &pipeline, - &mut report, - )?, + InputSpec::NfcapdTree { .. } => { + let mut sinks = [ProductSink { + pipeline: &pipeline, + connection: &connection, + report: &mut report, + }]; + process_nfcapd_tree( + plan.trees + .get(&input_index) + .expect("every nfcapd_tree input has a frozen layout"), + &mut sinks, + )?; + } } } merge_report( @@ -383,7 +1005,8 @@ fn execute(pipeline: ResolvedPipeline) -> Result tracing::warn!(%error, "could not refresh SQLite planner statistics"); } if pipeline.require_complete { - let incomplete = count_incomplete_requested_coverage(&connection, &pipeline)?; + let incomplete = + count_incomplete_requested_coverage_with_plan(&connection, &pipeline, &plan)?; if incomplete != 0 { return Err(PipelineError::IncompleteCoverage(incomplete)); } @@ -391,10 +1014,91 @@ fn execute(pipeline: ResolvedPipeline) -> Result Ok(report) } -/// Give every dataset without a configured `default_start_date` the earliest ingested local day. -/// -/// This runs after ingestion so that newly ingested earlier days move the stored date back. Until -/// the database holds traffic, the row keeps the fallback that [`upsert_dataset_metadata`] wrote. +struct CoordinatedOutput { + pipeline: ResolvedPipeline, + connection: Connection, + report: PipelineReport, + _lock: DatabaseOperationLock, +} + +struct ProductSink<'a> { + pipeline: &'a ResolvedPipeline, + connection: &'a Connection, + report: &'a mut PipelineReport, +} + +fn execute_many(plan: CompatiblePlan) -> Result { + let CompatiblePlan { pipelines, tree } = plan; + let mut dataset_sources = BTreeMap::new(); + for pipeline in &pipelines { + let dataset = pipeline.datasets.first().ok_or_else(|| { + PipelineError::InvalidConfig( + "coordinated datasets require registry-backed dataset metadata".into(), + ) + })?; + dataset_sources.insert(dataset.dataset_id.clone(), tree.sources.clone()); + } + + let mut outputs = Vec::with_capacity(pipelines.len()); + for pipeline in pipelines { + if let Some(parent) = pipeline.database_path.parent() { + fs::create_dir_all(parent)?; + } + let lock = + DatabaseOperationLock::acquire(&pipeline.database_path, "coordinated pipeline build")?; + let connection = connect_pipeline_writer(&pipeline.database_path)?; + init_schema(&connection)?; + with_transaction(&connection, || { + initialize_coordinated_metadata_in_transaction( + &connection, + &pipeline, + &tree.sources, + &dataset_sources, + ) + })?; + outputs.push(CoordinatedOutput { + pipeline, + connection, + report: PipelineReport::default(), + _lock: lock, + }); + } + + { + let mut sinks = outputs + .iter_mut() + .map(|output| ProductSink { + pipeline: &output.pipeline, + connection: &output.connection, + report: &mut output.report, + }) + .collect::>(); + process_nfcapd_tree(&tree, &mut sinks)?; + } + + let mut report = PipelineReport::default(); + for output in &mut outputs { + infer_default_start_dates(&output.connection, &output.pipeline)?; + populate_coverage_summary(&output.connection, &mut output.report)?; + if let Err(error) = optimize_all_query_planner_statistics(&output.connection) { + tracing::warn!(%error, "could not refresh SQLite planner statistics"); + } + if output.pipeline.require_complete { + let incomplete = count_incomplete_coverage_for_layout( + &output.connection, + &tree.sources, + tree.start, + tree.end, + &output.pipeline.timezone, + )?; + if incomplete != 0 { + return Err(PipelineError::IncompleteCoverage(incomplete)); + } + } + merge_report(&mut report, std::mem::take(&mut output.report)); + } + Ok(report) +} fn infer_default_start_dates( connection: &Connection, pipeline: &ResolvedPipeline, @@ -436,14 +1140,14 @@ fn with_transaction( connection .execute_batch("BEGIN IMMEDIATE") .map_err(StorageError::from)?; - let result = operation(); + let result = operation().and_then(|value| { + connection + .execute_batch("COMMIT") + .map_err(StorageError::from)?; + Ok(value) + }); match result { - Ok(value) => { - connection - .execute_batch("COMMIT") - .map_err(StorageError::from)?; - Ok(value) - } + Ok(value) => Ok(value), Err(error) => { let _ = connection.execute_batch("ROLLBACK"); Err(error) @@ -451,25 +1155,15 @@ fn with_transaction( } } -fn initialize_metadata( +fn initialize_metadata_with_plan( connection: &Connection, pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, ) -> Result<(), PipelineError> { - let layouts = pipeline - .inputs - .iter() - .filter_map(|input| match input { - InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - .. - } => Some(normalize_sources(root_path, source_ids, sources)), - _ => None, - }) - .collect::, _>>()? - .into_iter() - .flatten() + let layouts = plan + .trees + .values() + .flat_map(|tree| tree.sources.iter().cloned()) .collect::>(); let mut source_ids = BTreeSet::new(); if let Some(duplicate) = layouts @@ -482,21 +1176,67 @@ fn initialize_metadata( ))); } with_transaction(connection, || { - bind_identity(connection, pipeline)?; - for dataset in &pipeline.datasets { - upsert_dataset(connection, dataset)?; - } - if !layouts.is_empty() { - let layout = layouts - .iter() - .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) - .collect::>(); - bind_nfcapd_source_layout(connection, &layout)?; - } - Ok(()) + initialize_metadata_in_transaction_with_layouts( + connection, + pipeline, + &layouts, + &plan.dataset_sources, + ) }) } +fn initialize_metadata_in_transaction_with_layouts( + connection: &Connection, + pipeline: &ResolvedPipeline, + layouts: &[DatasetSource], + dataset_sources: &BTreeMap>, +) -> Result<(), PipelineError> { + bind_identity(connection, pipeline)?; + for dataset in &pipeline.datasets { + let sources = dataset_sources.get(&dataset.dataset_id).ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "single-output plan has no frozen source layout for dataset {:?}", + dataset.dataset_id + )) + })?; + upsert_dataset_with_sources(connection, dataset, sources)?; + } + if !layouts.is_empty() { + let layout = layouts + .iter() + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) + .collect::>(); + bind_nfcapd_source_layout(connection, &layout)?; + } + Ok(()) +} + +fn initialize_coordinated_metadata_in_transaction( + connection: &Connection, + pipeline: &ResolvedPipeline, + layout: &[DatasetSource], + dataset_layouts: &BTreeMap>, +) -> Result<(), PipelineError> { + bind_identity(connection, pipeline)?; + for dataset in &pipeline.datasets { + let sources = dataset_layouts.get(&dataset.dataset_id).ok_or_else(|| { + PipelineError::InvalidConfig(format!( + "coordinated plan has no frozen source layout for dataset {:?}", + dataset.dataset_id + )) + })?; + upsert_dataset_with_sources(connection, dataset, sources)?; + } + if !layout.is_empty() { + let layout = layout + .iter() + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) + .collect::>(); + bind_nfcapd_source_layout(connection, &layout)?; + } + Ok(()) +} + fn process_atomic( connection: &Connection, pipeline: &ResolvedPipeline, @@ -506,7 +1246,8 @@ fn process_atomic( let mut report = PipelineReport::default(); with_transaction(connection, || { operation(&mut aggregates, &mut report)?; - publish_rollups(connection, aggregates, pipeline, &mut report) + publish_rollups(connection, aggregates, pipeline, &mut report)?; + verify_nfdump_revision(pipeline) })?; Ok(report) } @@ -561,56 +1302,79 @@ struct CoverageScope { end: i64, } -/// A finite native request can be checked independently of incomplete data -/// already stored outside that request. CSV and literal-input configurations -/// have no separately declared time window, so their configured product is -/// the strict scope. -fn requested_coverage_scopes( +#[derive(Clone, Debug, PartialEq, Eq)] +struct CoverageRange { + source_id: String, + start: i64, + end: i64, +} + +fn merged_requested_coverage_ranges(scopes: Vec) -> Vec { + let mut ranges = scopes + .into_iter() + .flat_map(|scope| { + scope + .source_ids + .into_iter() + .map(move |source_id| CoverageRange { + source_id, + start: scope.start, + end: scope.end, + }) + }) + .collect::>(); + ranges.sort_unstable_by(|left, right| { + (&left.source_id, left.start, left.end).cmp(&(&right.source_id, right.start, right.end)) + }); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(previous) = merged.last_mut() + && previous.source_id == range.source_id + && range.start <= previous.end + { + previous.end = previous.end.max(range.end); + } else { + merged.push(range); + } + } + merged +} + +/// A finite native request is checked against the same frozen window and source layout that was +/// used for publication. CSV and literal-input configurations have no separately declared window, +/// so their configured product remains the strict scope. +fn requested_coverage_scopes_with_plan( pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, ) -> Result>, PipelineError> { let mut scopes = Vec::new(); - for input in &pipeline.inputs { - let InputSpec::NfcapdTree { - root_path, - source_ids, - sources, - start_date, - end_date, - start_time, - end_time, - .. - } = input - else { + for (input_index, input) in pipeline.inputs.iter().enumerate() { + if !matches!(input, InputSpec::NfcapdTree { .. }) { return Ok(None); - }; - let selected_start = parse_date_start(start_date, &pipeline.timezone)?; - let start = match start_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_start, - }; - let end = match (end_time, end_date) { - (Some(value), _) => parse_local_datetime(value, &pipeline.timezone)?, - (None, Some(value)) => next_date_start(value, &pipeline.timezone)?, - (None, None) => return Ok(None), - }; - let source_ids = normalize_sources(root_path, source_ids, sources)? - .into_iter() - .map(|source| source.source_id) - .collect(); + } + let tree = plan + .trees + .get(&input_index) + .expect("every nfcapd_tree input has a frozen layout"); scopes.push(CoverageScope { - source_ids, - start, - end, + source_ids: tree + .sources + .iter() + .map(|source| source.source_id.clone()) + .collect(), + start: tree.start, + end: tree.end, }); } Ok(Some(scopes)) } -fn count_incomplete_requested_coverage( +fn count_incomplete_requested_coverage_with_plan( connection: &Connection, pipeline: &ResolvedPipeline, + plan: &SingleOutputPlan, ) -> Result { - let Some(scopes) = requested_coverage_scopes(pipeline)? else { + let Some(scopes) = requested_coverage_scopes_with_plan(pipeline, plan)? else { return connection .query_row( "SELECT COUNT(*) FROM bucket_coverage @@ -622,52 +1386,77 @@ fn count_incomplete_requested_coverage( .map_err(PipelineError::from); }; - connection - .execute_batch( - "CREATE TEMP TABLE IF NOT EXISTS requested_coverage_scope ( - source_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - bucket_end INTEGER NOT NULL, - PRIMARY KEY (source_id, bucket_start, bucket_end) - ); - DELETE FROM requested_coverage_scope;", - ) - .map_err(StorageError::from)?; - for scope in scopes { - for source_id in scope.source_ids { - connection - .execute( - "INSERT OR IGNORE INTO requested_coverage_scope ( - source_id, bucket_start, bucket_end - ) VALUES (?1, ?2, ?3)", - params![source_id, scope.start, scope.end], - ) - .map_err(StorageError::from)?; + count_incomplete_coverage_ranges( + connection, + merged_requested_coverage_ranges(scopes), + &pipeline.timezone, + ) +} + +fn count_incomplete_coverage_for_layout( + connection: &Connection, + sources: &[DatasetSource], + start: i64, + end: i64, + timezone: &str, +) -> Result { + let source_ids = sources + .iter() + .map(|source| source.source_id.clone()) + .collect::>(); + let ranges = merged_requested_coverage_ranges(vec![CoverageScope { + source_ids, + start, + end, + }]); + count_incomplete_coverage_ranges(connection, ranges, timezone) +} + +fn count_incomplete_coverage_ranges( + connection: &Connection, + ranges: Vec, + timezone: &str, +) -> Result { + let mut incomplete = 0_i64; + for range in ranges { + let complete = connection + .prepare( + "SELECT bucket_start + FROM bucket_coverage + WHERE source_id = ?1 + AND granularity = '5m' + AND bucket_start >= ?2 + AND bucket_start < ?3 + AND coverage_state = 'complete' + ORDER BY bucket_start", + ) + .map_err(StorageError::from)? + .query_map(params![&range.source_id, range.start, range.end], |row| { + row.get::<_, i64>(0) + }) + .map_err(StorageError::from)? + .collect::>>() + .map_err(StorageError::from)?; + let mut bucket_start = range.start; + while bucket_start < range.end { + if !complete.contains(&bucket_start) { + incomplete = incomplete.checked_add(1).ok_or_else(|| { + PipelineError::InvalidConfig( + "requested coverage count exceeds SQLite INTEGER range".into(), + ) + })?; + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; } } - connection - .query_row( - "SELECT COUNT(*) - FROM bucket_coverage AS coverage - WHERE coverage.granularity = '5m' - AND coverage.coverage_state <> 'complete' - AND EXISTS ( - SELECT 1 FROM requested_coverage_scope AS scope - WHERE scope.source_id = coverage.source_id - AND coverage.bucket_start >= scope.bucket_start - AND coverage.bucket_start < scope.bucket_end - )", - [], - |row| row.get(0), - ) - .map_err(StorageError::from) - .map_err(PipelineError::from) + Ok(incomplete) } fn bind_identity( connection: &Connection, pipeline: &ResolvedPipeline, ) -> Result<(), PipelineError> { + verify_nfdump_revision(pipeline)?; let maad_config = serde_json::to_value(crate::maad::MaadConfig::default())?; let schema = json!({ "version": 3, @@ -680,13 +1469,22 @@ fn bind_identity( {"name":"bucket_coverage","version":1} ] }); + let nfdump_executable = pipeline.nfdump_revision.as_ref().map(|revision| { + json!({ + "locator": revision.locator, + "content_fingerprint": revision.content_fingerprint, + }) + }); let result_config = json!({ - "version": 3, + "version": 4, "timezone": pipeline.timezone, "nfcapd_decoder": { "protocol_version": nfdump::CONTRACT_VERSION, "input_contract": nfdump::INPUT_CONTRACT, - "output_contract": nfdump::OUTPUT_CONTRACT + "output_contract": nfdump::OUTPUT_CONTRACT, + "contract_id": nfcapd_decoder_fingerprint()?, + "decoder_fingerprint": pipeline.nfdump_revision.as_ref().map(|revision| revision.decoder_fingerprint.clone()), + "executable": nfdump_executable, }, "maad": { "enabled": pipeline.run_maad, @@ -704,11 +1502,14 @@ fn bind_identity( Ok(()) } -fn upsert_dataset(connection: &Connection, dataset: &Dataset) -> Result<(), PipelineError> { - let sources = dataset - .logical_sources()? - .into_iter() - .map(|source| SourceDefinition::new(source.source_id, source.members)) +fn upsert_dataset_with_sources( + connection: &Connection, + dataset: &Dataset, + logical_sources: &[DatasetSource], +) -> Result<(), PipelineError> { + let sources = logical_sources + .iter() + .map(|source| SourceDefinition::new(&source.source_id, source.members.clone())) .collect::>(); let mut metadata = DatasetMetadata::new(&dataset.dataset_id); metadata.label = dataset.label.clone(); @@ -1065,7 +1866,7 @@ fn merged_csv_bucket( } let coverage = BucketCoverage::new(1, u64::from(any_observed), u64::from(any_rejected)) .map_err(DomainError::from)?; - let bucket = builder.with_coverage(coverage).finish(); + let bucket = builder.with_coverage(coverage).finish_owned(); let evidence_state = if any_rejected { InputEvidenceState::Rejected } else if any_observed { @@ -1144,454 +1945,475 @@ fn reject_cross_kind_overlap( Ok(()) } -#[allow(clippy::too_many_arguments)] +fn nfcapd_day_is_complete( + sink: &ProductSink<'_>, + sources: &[DatasetSource], + start: i64, + end: i64, +) -> Result { + let Some(product_fingerprint) = current_product_fingerprint(sink.connection)? else { + return Ok(false); + }; + if sources.is_empty() { + return Ok(false); + } + for source in sources { + if !daily_product_completion_matches( + sink.connection, + &source.source_id, + start, + end, + &product_fingerprint, + sink.pipeline.run_maad, + )? { + return Ok(false); + } + } + Ok(true) +} + +fn rollback_sink_transactions(sinks: &[ProductSink<'_>], transactions: &[bool]) { + for (sink, active) in sinks.iter().zip(transactions) { + if *active { + let _ = sink.connection.execute_batch("ROLLBACK"); + } + } +} + fn process_nfcapd_tree( - connection: &Connection, - root: &Path, - source_ids: &[String], - configured_sources: &[DatasetSource], - start_date: &str, - end_date: Option<&str>, - start_time: Option<&str>, - end_time: Option<&str>, - force: bool, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, + tree: &FrozenNfcapdTreeLayout, + sinks: &mut [ProductSink<'_>], ) -> Result<(), PipelineError> { - let sources = normalize_sources(root, source_ids, configured_sources)?; - let physical_ids = sources + let Some(first) = sinks.first() else { + return Ok(()); + }; + let timezone = first.pipeline.timezone.clone(); + let daily_active = first.pipeline.selection.selects_daily_active_sources(); + let source_ids = tree + .sources .iter() - .flat_map(|source| source.members.iter().cloned()) - .collect::>() - .into_iter() + .map(|source| source.source_id.clone()) .collect::>(); - let discovery_started = Instant::now(); - let discovered = ingest::discover_nfcapd_source_paths(root, &physical_ids, &pipeline.timezone)?; - tracing::info!( - target: "netflow_db::profile", - phase = "discovery", - elapsed_seconds = discovery_started.elapsed().as_secs_f64(), - physical_sources = physical_ids.len(), - discovered_inputs = discovered.len(), - ); - let mut by_member_and_start = BTreeMap::new(); - let mut member_bounds = BTreeMap::new(); - for input in discovered { - member_bounds - .entry(input.source_id.clone()) - .and_modify(|(first, last): &mut (i64, i64)| { - *first = (*first).min(input.bucket_start); - *last = (*last).max(input.bucket_start); + let mut day_start = tree.start; + + while day_start < tree.end { + let day_end = aggregate_bounds(day_start, Granularity::OneDay, &timezone)? + .1 + .min(tree.end); + let mut pending = Vec::new(); + for (index, sink) in sinks.iter().enumerate() { + if tree.force || !nfcapd_day_is_complete(sink, &tree.sources, day_start, day_end)? { + pending.push(index); + } + } + if pending.is_empty() { + day_start = day_end; + continue; + } + + let missing = daily_active + .then(|| { + missing_physical_day_inputs( + &tree.physical_ids, + &tree.by_member_and_start, + day_start, + day_end, + &timezone, + ) }) - .or_insert((input.bucket_start, input.bucket_start)); - by_member_and_start.insert((input.source_id, input.bucket_start), input.path); - } - let selected_start = parse_date_start(start_date, &pipeline.timezone)?; - let discovered_end = by_member_and_start - .keys() - .map(|(_, bucket_start)| *bucket_start) - .max() - .map(|start| aggregate_bounds(start, Granularity::OneDay, &pipeline.timezone)) - .transpose()? - .map(|(_, end)| end) - .unwrap_or(selected_start); - let selected_end = match end_date { - Some(date) => next_date_start(date, &pipeline.timezone)?, - None => discovered_end, - }; - let start = match start_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_start, - }; - let end = match end_time { - Some(value) => parse_local_datetime(value, &pipeline.timezone)?, - None => selected_end, - }; - validate_window(selected_start, selected_end, start, end, &pipeline.timezone)?; - - let mut day_start = start; - while day_start < end { - let day_end = aggregate_bounds(day_start, Granularity::OneDay, &pipeline.timezone)?.1; - let mut owned_keys = BTreeSet::new(); - let mut bucket_start = day_start; - while bucket_start < day_end { - for source in &sources { - if force - && source_has_candidate( + .transpose()? + .unwrap_or_default(); + + let mut transactions = vec![false; sinks.len()]; + for &index in &pending { + if let Err(error) = sinks[index].connection.execute_batch("BEGIN IMMEDIATE") { + rollback_sink_transactions(sinks, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = true; + if let Err(error) = + delete_stats_time_range(sinks[index].connection, &source_ids, day_start, day_end) + { + rollback_sink_transactions(sinks, &transactions); + return Err(PipelineError::Storage(error)); + } + } + + if daily_active && !missing.is_empty() { + let details = missing_day_warning_details(&tree.root_path, &missing, &timezone)?; + tracing::warn!( + day_start, + day_end, + missing_inputs = missing.len(), + missing_details = %details, + "skipping incomplete physical day for daily_active_sources selection" + ); + sinks[pending[0]].report.skipped_inputs += missing.len(); + } else { + let mut owned_keys = BTreeSet::new(); + let mut bucket_start = day_start; + while bucket_start < day_end { + for source in &tree.sources { + if source_has_candidate( source, bucket_start, - &by_member_and_start, - &member_bounds, - end_date.is_some(), - ) - { - owned_keys.insert((source.source_id.clone(), bucket_start)); + &tree.by_member_and_start, + &tree.member_bounds, + tree.extend_gaps_to_window, + ) { + owned_keys.insert((source.source_id.clone(), bucket_start)); + } + } + bucket_start = next_local_five_minute_start(bucket_start, &timezone)?; + } + let mut aggregates = (0..sinks.len()) + .map(|index| { + pending + .contains(&index) + .then(|| AggregateBuckets::with_owned_keys(owned_keys.clone())) + }) + .collect::>(); + let result = + process_nfcapd_tree_day(tree, day_start, day_end, sinks, &pending, &mut aggregates); + if let Err(error) = result { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + for &index in &pending { + let aggregate = aggregates[index] + .take() + .expect("pending output has aggregate state"); + if let Err(error) = publish_rollups( + sinks[index].connection, + aggregate, + sinks[index].pipeline, + sinks[index].report, + ) { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + if let Err(error) = mark_nfcapd_day_complete( + sinks[index].connection, + &tree.sources, + day_start, + day_end, + sinks[index].pipeline.run_maad, + ) { + rollback_sink_transactions(sinks, &transactions); + return Err(error); } } - bucket_start = next_local_five_minute_start(bucket_start, &pipeline.timezone)?; } - let transaction_started = Instant::now(); - let (day_report, day_profile) = with_transaction(connection, || { - let mut aggregates = AggregateBuckets::with_owned_keys(owned_keys); - let mut day_report = PipelineReport::default(); - let mut day_profile = process_nfcapd_tree_day( - connection, - root, - &sources, - &by_member_and_start, - &member_bounds, - day_start, - day_end, - end_date.is_some(), - force, - pipeline, - &mut aggregates, - &mut day_report, - )?; - day_profile.final_rollups = - publish_rollups_profiled(connection, aggregates, pipeline, &mut day_report)?; - Ok((day_report, day_profile)) - })?; - day_profile.log(day_start, day_end, transaction_started.elapsed()); - merge_report(report, day_report); + + if let Err(error) = verify_nfdump_revision(sinks[pending[0]].pipeline) { + rollback_sink_transactions(sinks, &transactions); + return Err(error); + } + + for &index in &pending { + if let Err(error) = sinks[index].connection.execute_batch("COMMIT") { + rollback_sink_transactions(sinks, &transactions); + return Err(PipelineError::Storage(StorageError::from(error))); + } + transactions[index] = false; + } day_start = day_end; } Ok(()) } -fn source_has_candidate( - source: &DatasetSource, - bucket_start: i64, +fn missing_physical_day_inputs( + physical_ids: &[String], paths: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, - extend_gaps_to_window: bool, -) -> bool { - let has_file = source - .members - .iter() - .any(|member| paths.contains_key(&(member.clone(), bucket_start))); - has_file - || extend_gaps_to_window - || source.members.iter().any(|member| { - member_bounds - .get(member) - .is_some_and(|(first, last)| *first <= bucket_start && bucket_start <= *last) + start: i64, + end: i64, + timezone: &str, +) -> Result, PipelineError> { + let mut missing = Vec::new(); + let mut bucket_start = start; + while bucket_start < end { + for member in physical_ids { + if !paths.contains_key(&(member.clone(), bucket_start)) { + missing.push((member.clone(), bucket_start)); + } + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(missing) +} + +fn missing_day_warning_details( + root: &Path, + missing: &[(String, i64)], + timezone: &str, +) -> Result { + let mut details = missing + .iter() + .take(MAX_MISSING_DAY_WARNING_DETAILS) + .map(|(member, bucket_start)| { + expected_nfcapd_path(root, member, *bucket_start, timezone).map(|expected_path| { + format!( + "member={member} timestamp={bucket_start} expected_path={}", + expected_path.display() + ) + }) }) + .collect::, _>>()?; + let omitted = missing.len().saturating_sub(details.len()); + if omitted != 0 { + details.push(format!("… {omitted} more missing inputs")); + } + Ok(details.join("; ")) } -#[allow(clippy::too_many_arguments)] -fn process_nfcapd_tree_day( +/// Publish completion markers in the same transaction as the day's product rows. +fn mark_nfcapd_day_complete( connection: &Connection, - root: &Path, sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, start: i64, end: i64, + run_maad: bool, +) -> Result<(), PipelineError> { + let product_fingerprint = current_product_fingerprint(connection)?.ok_or_else(|| { + PipelineError::InvalidConfig( + "cannot publish a daily product completion marker before product identity binding" + .into(), + ) + })?; + for source in sources { + upsert_daily_product_completion( + connection, + &source.source_id, + start, + end, + &product_fingerprint, + run_maad, + )?; + } + Ok(()) +} + +fn source_has_candidate( + source: &DatasetSource, + bucket_start: i64, + paths: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - aggregates: &mut AggregateBuckets, - report: &mut PipelineReport, -) -> Result { - let day_started = Instant::now(); - let mut prepare_elapsed = Duration::ZERO; - let mut decode_elapsed = Duration::ZERO; - let mut publish_elapsed = Duration::ZERO; - let mut publish_profile = NfcapdDayPublishProfile::default(); - let revision_hash_workers = std::thread::available_parallelism() - .map_or(1, std::num::NonZeroUsize::get) - .min(NFCAPD_REVISION_HASH_MAX_WORKERS); - let revision_pool = rayon::ThreadPoolBuilder::new() - .num_threads(revision_hash_workers) - .thread_name(|index| format!("nfcapd-revision-{index}")) - .build() - .map_err(|error| { - PipelineError::InvalidConfig(format!("failed to build revision hash pool: {error}")) - })?; - let revision_context = NfcapdRevisionContext { - connection, - sources, - by_member_and_start, - member_bounds, - extend_gaps_to_window, - force, - revision_pool: &revision_pool, - }; - let mut bucket_start = start; - while bucket_start < end { - let prepare_started = Instant::now(); - let mut batch_starts = Vec::with_capacity(NFCAPD_DECODE_BATCH_SIZE); - while bucket_start < end && batch_starts.len() < NFCAPD_DECODE_BATCH_SIZE { - batch_starts.push(bucket_start); - bucket_start = next_local_five_minute_start(bucket_start, &pipeline.timezone)?; - } - let revisions = resolve_nfcapd_batch_revisions(&revision_context, &batch_starts)?; - let mut batch = Vec::with_capacity(batch_starts.len()); - for bucket_start in batch_starts { - batch.push(prepare_nfcapd_tree_timestamp( - connection, - root, - sources, - by_member_and_start, - member_bounds, - bucket_start, - extend_gaps_to_window, - force, - pipeline, - report, - &revisions, - )?); - } - prepare_elapsed += prepare_started.elapsed(); +) -> bool { + let has_file = source + .members + .iter() + .any(|member| paths.contains_key(&(member.clone(), bucket_start))); + has_file + || extend_gaps_to_window + || source.members.iter().any(|member| { + member_bounds + .get(member) + .is_some_and(|(first, last)| *first <= bucket_start && bucket_start <= *last) + }) +} - let decode_started = Instant::now(); - let needed = batch +/// Group local five-minute starts so each decode batch has at most twelve physical requests when +/// possible. A timestamp with more than twelve members is kept as one batch and drained in +/// physical-request chunks by the decode caller. +fn nfcapd_batch_starts( + start: i64, + end: i64, + timezone: &str, + sources: &[DatasetSource], + paths: &BTreeMap<(String, i64), PathBuf>, + member_bounds: &BTreeMap, + extend_gaps_to_window: bool, +) -> Result, PipelineError> { + let mut starts = Vec::with_capacity(NFCAPD_DECODE_BATCH_SIZE); + let mut physical_requests = BTreeSet::new(); + let mut next = start; + while next < end { + let timestamp_requests = sources .iter() - .flat_map(|timestamp| { - timestamp.jobs.iter().flat_map(|job| { - job.present.iter().map(|(member, path)| { - let snapshot = timestamp - .revision_cache - .get(member) - .and_then(|owner| owner.snapshot.clone()) - .expect("present member has a snapshot"); - ( - (member.clone(), timestamp.bucket_start), - (path.clone(), snapshot), - ) - }) - }) + .filter(|source| { + source_has_candidate(source, next, paths, member_bounds, extend_gaps_to_window) }) - .collect::>(); - let mut decoded_cache = needed - .par_iter() - .map(|((member, bucket_start), (path, snapshot))| { - let bucket = ingest::read_nfcapd_bucket( - path, - member, - &pipeline.selection, - &pipeline.nfdump, - &pipeline.timezone, - )?; - verify_file_snapshot(path, snapshot)?; - Ok::<_, PipelineError>(((member.clone(), *bucket_start), bucket)) + .flat_map(|source| { + source.members.iter().filter_map(|member| { + paths + .contains_key(&(member.clone(), next)) + .then_some((member.clone(), next)) + }) }) - .collect::, _>>()?; - decode_elapsed += decode_started.elapsed(); - - let publish_started = Instant::now(); - for timestamp in batch { - for job in timestamp.jobs { - let member_buckets = job - .present - .iter() - .map(|(member, _)| { - decoded_cache - .get(&(member.clone(), timestamp.bucket_start)) - .expect("requested physical member was decoded") - }) - .collect::>(); - let logical_started = Instant::now(); - let logical = logical_source_bucket( - &job.source_id, - timestamp.bucket_start, - job.expected_units, - &member_buckets, - )?; - publish_profile.logical_source_elapsed += logical_started.elapsed(); - let sibling_started = Instant::now(); - if !job.is_repair { - aggregates.reject_persisted_siblings( - connection, - &logical, - &pipeline.timezone, - )?; - } - publish_profile.persisted_sibling_elapsed += sibling_started.elapsed(); - let bucket_profile = publish_nfcapd_bucket_profiled( - connection, - &logical, - &job.owners, - &job.absences, - &job.evidence, - true, - force, - pipeline.run_maad, - )?; - publish_profile.bucket_publish.include(bucket_profile); - let flushed = if job.is_repair { - refresh_rollups_after_five_minute_repair( - connection, - &logical, - &pipeline.timezone, - )?; - 0 - } else { - let aggregate_profile = - aggregates.include_profiled(&logical, &pipeline.timezone)?; - publish_profile.aggregate_include.include(aggregate_profile); - let flush_started = Instant::now(); - let (flushed, rollup_write) = - aggregates.flush_complete_profiled(connection, pipeline.run_maad)?; - publish_profile.completed_rollup_flush_elapsed += flush_started.elapsed(); - publish_profile.completed_rollup_write.include(rollup_write); - publish_profile.completed_rollup_flushes += 1; - if flushed > 0 { - publish_profile.nonempty_rollup_flushes += 1; - } - flushed - }; - publish_profile.logical_buckets += 1; - report.rollup_buckets += flushed; - report.five_minute_buckets += 1; - } - decoded_cache.retain(|(_, start), _| *start != timestamp.bucket_start); + .collect::>(); + let would_exceed_physical_limit = !starts.is_empty() + && physical_requests.len() + timestamp_requests.len() > NFCAPD_DECODE_BATCH_SIZE; + if would_exceed_physical_limit || starts.len() == NFCAPD_DECODE_BATCH_SIZE { + break; } - publish_elapsed += publish_started.elapsed(); + starts.push(next); + physical_requests.extend(timestamp_requests); + next = next_local_five_minute_start(next, timezone)?; } - publish_profile.day_elapsed = day_started.elapsed(); - publish_profile.prepare_elapsed = prepare_elapsed; - publish_profile.decode_elapsed = decode_elapsed; - publish_profile.batch_publish_elapsed = publish_elapsed; - tracing::info!( - target: "netflow_db::profile", - phase = "nfcapd_tree_day", - day_start = start, - day_end = end, - elapsed_seconds = publish_profile.day_elapsed.as_secs_f64(), - prepare_seconds = prepare_elapsed.as_secs_f64(), - decode_seconds = decode_elapsed.as_secs_f64(), - publish_seconds = publish_elapsed.as_secs_f64(), - ); - Ok(publish_profile) + Ok(starts) } -struct NfcapdRevisionProbe { - path: PathBuf, - observed: FileSnapshot, - cached_content_fingerprint: Option, +fn nfcapd_day_activity_paths( + paths: &BTreeMap<(String, i64), PathBuf>, + member: &str, + start: i64, + end: i64, + timezone: &str, +) -> Result, PipelineError> { + let mut result = Vec::new(); + let mut bucket_start = start; + while bucket_start < end { + if let Some(path) = paths.get(&(member.to_owned(), bucket_start)) { + result.push(path.clone()); + } + bucket_start = next_local_five_minute_start(bucket_start, timezone)?; + } + Ok(result) } -struct NfcapdRevisionContext<'a> { - connection: &'a Connection, - sources: &'a [DatasetSource], - by_member_and_start: &'a BTreeMap<(String, i64), PathBuf>, - member_bounds: &'a BTreeMap, - extend_gaps_to_window: bool, - force: bool, - revision_pool: &'a rayon::ThreadPool, +fn nfcapd_decode_error( + member: &str, + bucket_start: i64, + path: &Path, + error: impl std::fmt::Display, +) -> PipelineError { + PipelineError::InvalidConfig(format!( + "nfcapd decode failed for member {member:?}, bucket {bucket_start}, path {}: {error}", + path.display() + )) } -/// Resolve the physical files needed by a decode batch before making any job decisions. -/// SQLite access stays on the pipeline thread; only exact hashes run in parallel. -fn resolve_nfcapd_batch_revisions( - context: &NfcapdRevisionContext<'_>, - batch_starts: &[i64], -) -> Result, PipelineError> { - let mut paths = BTreeSet::new(); - for &bucket_start in batch_starts { - for source in context.sources { - if !source_has_candidate( - source, - bucket_start, - context.by_member_and_start, - context.member_bounds, - context.extend_gaps_to_window, - ) { - continue; +fn resolve_daily_active_sources( + tree: &FrozenNfcapdTreeLayout, + start: i64, + end: i64, + timezone: &str, + selections: &[FlowSelection], + executable: &Path, +) -> Result>, PipelineError> { + let pool = build_nfcapd_activity_pool()?; + let requests = tree + .physical_ids + .iter() + .map(|member| { + nfcapd_day_activity_paths(&tree.by_member_and_start, member, start, end, timezone) + .map(|paths| (member.clone(), paths)) + }) + .collect::, _>>()?; + let results = pool.install(|| { + requests + .par_iter() + .map(|(member, paths)| { + ingest::read_nfcapd_daily_source_activities(paths, selections, executable) + .map_err(|error| { + PipelineError::InvalidConfig(format!( + "daily activity scan failed for member {member:?}, day {start}..{end}: {error}" + )) + }) + }) + .collect::, _>>() + })?; + + let mut combined = (0..selections.len()) + .map(|_| HashMap::::new()) + .collect::>(); + for member_results in results { + for (selection_index, activity) in member_results.into_iter().enumerate() { + for (address, metrics) in activity { + combined[selection_index] + .entry(address) + .or_default() + .include(metrics); } - paths.extend(source.members.iter().filter_map(|member| { - context - .by_member_and_start - .get(&(member.clone(), bucket_start)) - .cloned() - })); } } - - let decoder_fingerprint = nfcapd_decoder_fingerprint()?; - let probes = paths + Ok(combined .into_iter() - .map(|path| { - let locator = path.to_string_lossy().into_owned(); - let observed = FileSnapshot::capture(&path)?; - let cached_fingerprint = if context.force { - None - } else { - cached_content_fingerprint( - context.connection, - InputKind::Nfcapd, - &locator, - &observed, - )? - }; - Ok::<_, PipelineError>(NfcapdRevisionProbe { - path, - observed, - cached_content_fingerprint: cached_fingerprint, - }) + .map(|activity| { + Arc::new( + activity + .into_iter() + .filter_map(|(address, metrics)| { + FlowSelection::daily_activity_threshold_met( + metrics.flows, + metrics.packets, + metrics.bytes, + ) + .then_some(address) + }) + .collect(), + ) }) - .collect::, _>>()?; + .collect()) +} - let resolved = context.revision_pool.install(|| { - probes +fn resolve_nfcapd_batch_revisions( + tree: &FrozenNfcapdTreeLayout, + batch_starts: &[i64], + decoder_fingerprint: &str, + pool: &rayon::ThreadPool, +) -> Result, PipelineError> { + let paths = batch_starts + .iter() + .flat_map(|bucket_start| { + tree.sources.iter().flat_map(move |source| { + source.members.iter().filter_map(move |member| { + tree.by_member_and_start + .get(&(member.clone(), *bucket_start)) + .cloned() + }) + }) + }) + .collect::>(); + pool.install(|| { + paths .par_iter() - .map(|probe| { - let captured = match &probe.cached_content_fingerprint { - Some(content_fingerprint) => { - Ok((content_fingerprint.clone(), probe.observed.clone())) - } - None => capture_file_revision(&probe.path), - }; - captured - .map_err(PipelineError::from) - .and_then(|(content_fingerprint, snapshot)| { - let revision = InputRevision::create( - "nfcapd", - probe.path.to_string_lossy().into_owned(), - content_fingerprint, - &decoder_fingerprint, - )?; - Ok(PreparedRevision { - revision, - snapshot: Some(snapshot), - }) - }) + .map(|path| { + let (content_fingerprint, snapshot) = capture_file_revision(path)?; + let revision = InputRevision::create( + "nfcapd", + path.to_string_lossy().into_owned(), + content_fingerprint, + decoder_fingerprint, + )?; + Ok(( + path.clone(), + PreparedRevision { + revision, + snapshot: Some(snapshot), + }, + )) }) - .collect::>() - }); + .collect::, PipelineError>>() + }) +} - probes - .into_iter() - .zip(resolved) - .map(|(probe, result)| result.map(|revision| (probe.path, revision))) - .collect::, _>>() +fn verify_prepared_revision_snapshots( + revisions: &BTreeMap, +) -> Result<(), PipelineError> { + for (path, prepared) in revisions { + if let Some(snapshot) = &prepared.snapshot { + verify_file_snapshot(path, snapshot)?; + } + } + Ok(()) } -#[allow(clippy::too_many_arguments)] fn prepare_nfcapd_tree_timestamp( - connection: &Connection, - root: &Path, - sources: &[DatasetSource], - by_member_and_start: &BTreeMap<(String, i64), PathBuf>, - member_bounds: &BTreeMap, + tree: &FrozenNfcapdTreeLayout, bucket_start: i64, - extend_gaps_to_window: bool, - force: bool, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, + timezone: &str, revisions: &BTreeMap, ) -> Result { - let mut revision_cache: BTreeMap = BTreeMap::new(); let mut jobs = Vec::new(); - for source in sources { + for source in &tree.sources { if !source_has_candidate( source, bucket_start, - by_member_and_start, - member_bounds, - extend_gaps_to_window, + &tree.by_member_and_start, + &tree.member_bounds, + tree.extend_gaps_to_window, ) { continue; } @@ -1599,26 +2421,20 @@ fn prepare_nfcapd_tree_timestamp( .members .iter() .filter_map(|member| { - by_member_and_start + tree.by_member_and_start .get(&(member.clone(), bucket_start)) .map(|path| (member.clone(), path.clone())) }) .collect::>(); - let mut owners = Vec::new(); - for (member, path) in &present { - let owner = match revision_cache.get(member) { - Some(owner) => owner.clone(), - None => { - let owner = revisions - .get(path) - .cloned() - .expect("present member has a resolved revision"); - revision_cache.insert(member.clone(), owner.clone()); - owner - } - }; - owners.push(owner); - } + let owners = present + .iter() + .map(|(_, path)| { + revisions + .get(path) + .cloned() + .expect("present capture has a prepared revision") + }) + .collect::>(); let mut absences = Vec::new(); let mut evidence = Vec::with_capacity(source.members.len()); for ((member, path), owner) in present.iter().zip(&owners) { @@ -1633,9 +2449,9 @@ fn prepare_nfcapd_tree_timestamp( )); } for member in &source.members { - if !present.iter().any(|(present, _)| present == member) { + if present.iter().all(|(present, _)| present != member) { let expected = - expected_nfcapd_path(root, member, bucket_start, &pipeline.timezone)?; + expected_nfcapd_path(&tree.root_path, member, bucket_start, timezone)?; absences.push(ExpectedAbsence::capture(&expected)?); evidence.push(InputEvidenceRow::new( &source.source_id, @@ -1649,41 +2465,6 @@ fn prepare_nfcapd_tree_timestamp( } } evidence.sort_unstable_by(|left, right| left.unit_id.cmp(&right.unit_id)); - let previous_evidence = query_input_evidence(connection, &source.source_id, bucket_start)?; - let observed_input_disappeared = previous_evidence.iter().any(|previous| { - previous.evidence_state == InputEvidenceState::Observed - && evidence.iter().any(|current| { - current.unit_id == previous.unit_id - && current.evidence_state == InputEvidenceState::Missing - }) - }); - if observed_input_disappeared { - tracing::warn!( - source_id = source.source_id, - bucket_start, - "preserving prior bucket because an observed input is now missing" - ); - report.skipped_inputs += 1; - continue; - } - let is_repair = !force && !previous_evidence.is_empty() && previous_evidence != evidence; - let revisions = owners - .iter() - .map(|owner| owner.revision.clone()) - .collect::>(); - if !force - && previous_evidence == evidence - && (revisions.is_empty() - || nfcapd_logical_bucket_processed( - connection, - &source.source_id, - bucket_start, - &revisions, - )?) - { - report.skipped_inputs += 1; - continue; - } jobs.push(PreparedTreeJob { source_id: source.source_id.clone(), expected_units: source.members.len(), @@ -1691,17 +2472,170 @@ fn prepare_nfcapd_tree_timestamp( owners, absences, evidence, - is_repair, }); } - Ok(PreparedTreeTimestamp { - bucket_start, - revision_cache, - jobs, - }) + Ok(PreparedTreeTimestamp { bucket_start, jobs }) } -#[allow(clippy::too_many_arguments)] +fn process_nfcapd_tree_day( + tree: &FrozenNfcapdTreeLayout, + start: i64, + end: i64, + sinks: &mut [ProductSink<'_>], + pending: &[usize], + aggregates: &mut [Option], +) -> Result<(), PipelineError> { + let first_pipeline = sinks[pending[0]].pipeline; + let timezone = first_pipeline.timezone.clone(); + let executable = first_pipeline.nfdump.clone(); + let daily_active = first_pipeline.selection.selects_daily_active_sources(); + let selections = pending + .iter() + .map(|index| sinks[*index].pipeline.selection.clone()) + .collect::>(); + let decoder_fingerprint = nfdump_decoder_fingerprint_for_pipeline(first_pipeline)?; + let revision_pool = build_revision_hash_pool()?; + let mut revision_starts = Vec::new(); + let mut revision_start = start; + while revision_start < end { + revision_starts.push(revision_start); + revision_start = next_local_five_minute_start(revision_start, &timezone)?; + } + let revisions = resolve_nfcapd_batch_revisions( + tree, + &revision_starts, + &decoder_fingerprint, + &revision_pool, + )?; + verify_nfdump_revision(first_pipeline)?; + let active_sources = if daily_active { + Some(resolve_daily_active_sources( + tree, + start, + end, + &timezone, + &selections, + &executable, + )?) + } else { + None + }; + verify_nfdump_revision(first_pipeline)?; + verify_prepared_revision_snapshots(&revisions)?; + let active_pairs = active_sources.as_ref().map(|active| { + selections + .iter() + .cloned() + .zip(active.iter().cloned()) + .collect::>() + }); + let decode_pool = build_nfcapd_decode_pool()?; + let mut next = start; + + while next < end { + let batch_starts = nfcapd_batch_starts( + next, + end, + &timezone, + &tree.sources, + &tree.by_member_and_start, + &tree.member_bounds, + tree.extend_gaps_to_window, + )?; + next = batch_starts + .last() + .copied() + .map(|last| next_local_five_minute_start(last, &timezone)) + .transpose()? + .expect("non-empty nfcapd batch"); + let prepared = batch_starts + .iter() + .map(|start| prepare_nfcapd_tree_timestamp(tree, *start, &timezone, &revisions)) + .collect::, _>>()?; + let requests = prepared + .iter() + .flat_map(|timestamp| { + timestamp.jobs.iter().flat_map(move |job| { + job.present.iter().map(move |(member, path)| { + ((member.clone(), timestamp.bucket_start), path.clone()) + }) + }) + }) + .collect::>() + .into_iter() + .collect::>(); + verify_nfdump_revision(first_pipeline)?; + let decoded = decode_pool.install(|| { + requests + .par_iter() + .map(|((member, bucket_start), path)| { + let buckets = match &active_pairs { + Some(pairs) => ingest::read_nfcapd_buckets_with_active_sources( + path, + member, + pairs, + &executable, + &timezone, + )?, + None => vec![ingest::read_nfcapd_bucket( + path, + member, + &selections[0], + &executable, + &timezone, + )?], + }; + Ok::<_, PipelineError>(((member.clone(), *bucket_start), buckets)) + }) + .collect::, _>>() + })?; + verify_nfdump_revision(first_pipeline)?; + + for (selection_index, sink_index) in pending.iter().copied().enumerate() { + let aggregate = aggregates[sink_index] + .as_mut() + .expect("pending output has aggregate state"); + for timestamp in &prepared { + for job in ×tamp.jobs { + let member_buckets = job + .present + .iter() + .map(|(member, _)| { + &decoded[&(member.clone(), timestamp.bucket_start)][selection_index] + }) + .collect::>(); + let logical = logical_source_bucket( + &job.source_id, + timestamp.bucket_start, + job.expected_units, + &member_buckets, + )?; + aggregate.reject_persisted_siblings( + sinks[sink_index].connection, + &logical, + &timezone, + )?; + publish_nfcapd_bucket( + sinks[sink_index].connection, + &logical, + &job.owners, + &job.absences, + &job.evidence, + true, + sinks[sink_index].pipeline.run_maad, + )?; + aggregate.include(&logical, &timezone)?; + sinks[sink_index].report.rollup_buckets += aggregate.flush_complete( + sinks[sink_index].connection, + sinks[sink_index].pipeline.run_maad, + )?; + sinks[sink_index].report.five_minute_buckets += 1; + } + } + } + } + Ok(()) +} enum PreparedExplicitNfcapdKind { File(PreparedRevision), Gap { expected_path: Option }, @@ -1749,7 +2683,7 @@ fn process_explicit_nfcapd_inputs( connection, path, InputKind::Nfcapd, - nfcapd_decoder_fingerprint()?, + nfdump_decoder_fingerprint_for_pipeline(pipeline)?, )?; PreparedExplicitNfcapdKind::File(PreparedRevision { revision, @@ -1822,18 +2756,16 @@ fn process_nfcapd( report.skipped_inputs += 1; return Ok(()); } + verify_nfdump_revision(pipeline)?; let bucket = ingest::read_nfcapd_bucket( path, source_id, &pipeline.selection, &pipeline.nfdump, &pipeline.timezone, - )?; - let snapshot = owner - .snapshot - .as_ref() - .expect("explicit file input has a snapshot"); - verify_file_snapshot(path, snapshot)?; + ) + .map_err(|error| nfcapd_decode_error(source_id, bucket_start, path, error))?; + verify_nfdump_revision(pipeline)?; aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; publish_nfcapd_bucket( connection, @@ -1850,7 +2782,6 @@ fn process_nfcapd( Some(owner.revision.fingerprint.clone()), )], false, - false, pipeline.run_maad, )?; aggregates.include(&bucket, &pipeline.timezone)?; @@ -1896,7 +2827,7 @@ fn process_nfcapd_gap( bucket_start + FIVE_MINUTES, )) .with_coverage(BucketCoverage::new(1, 0, 0).map_err(DomainError::from)?) - .finish(); + .finish_owned(); aggregates.reject_persisted_siblings(connection, &bucket, &pipeline.timezone)?; publish_nfcapd_bucket( connection, @@ -1905,7 +2836,6 @@ fn process_nfcapd_gap( &[absence], &evidence, false, - false, pipeline.run_maad, )?; aggregates.include(&bucket, &pipeline.timezone)?; @@ -1976,6 +2906,25 @@ fn normalize_sources( } } } + + let member_ids = normalized + .iter() + .flat_map(|source| source.members.iter().cloned()) + .collect::>(); + let mut member_paths = BTreeMap::::new(); + for member in member_ids { + let member_path = root.join(&member); + let canonical_member_path = canonical_path(&member_path)?; + if let Some(previous_member) = member_paths.get(&canonical_member_path) { + return Err(PipelineError::InvalidConfig(format!( + "nfcapd_tree member IDs {:?} and {:?} resolve to the same directory {}", + previous_member, + member, + canonical_member_path.display() + ))); + } + member_paths.insert(canonical_member_path, member); + } Ok(normalized) } @@ -2006,7 +2955,7 @@ fn merge_source_bucket( 0, ) .map_err(DomainError::from)?; - Ok(builder.with_coverage(coverage).finish()) + Ok(builder.with_coverage(coverage).finish_owned()) } fn logical_source_bucket<'a>( @@ -2035,285 +2984,6 @@ fn logical_source_bucket<'a>( )?)) } -#[derive(Debug, Default)] -struct NfcapdBucketPublishProfile { - total_elapsed: Duration, - preflight_elapsed: Duration, - overlap_elapsed: Duration, - force_delete_elapsed: Duration, - owner_upsert_elapsed: Duration, - write: WriteBucketsProfile, - owner_status_elapsed: Duration, - postflight_elapsed: Duration, - owners: u64, - absences: u64, -} - -impl NfcapdBucketPublishProfile { - fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.preflight_elapsed += profile.preflight_elapsed; - self.overlap_elapsed += profile.overlap_elapsed; - self.force_delete_elapsed += profile.force_delete_elapsed; - self.owner_upsert_elapsed += profile.owner_upsert_elapsed; - self.write.include(profile.write); - self.owner_status_elapsed += profile.owner_status_elapsed; - self.postflight_elapsed += profile.postflight_elapsed; - self.owners += profile.owners; - self.absences += profile.absences; - } - - fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.preflight_elapsed - + self.overlap_elapsed - + self.force_delete_elapsed - + self.owner_upsert_elapsed - + self.write.total_elapsed - + self.owner_status_elapsed - + self.postflight_elapsed, - ) - } -} - -#[derive(Debug, Default)] -struct FinalRollupProfile { - total_elapsed: Duration, - finish_elapsed: Duration, - delete_elapsed: Duration, - write: WriteBucketsProfile, - incomplete_keys: u64, - rollup_buckets: u64, -} - -impl FinalRollupProfile { - fn other_elapsed(&self) -> Duration { - self.total_elapsed - .saturating_sub(self.finish_elapsed + self.delete_elapsed + self.write.total_elapsed) - } -} - -#[derive(Debug, Default)] -struct AggregateGranularityProfile { - total_elapsed: Duration, - bounds_elapsed: Duration, - builder_elapsed: Duration, - bucket: StatisticalBucketIncludeProfile, -} - -impl AggregateGranularityProfile { - fn include( - &mut self, - total_elapsed: Duration, - bounds_elapsed: Duration, - builder_elapsed: Duration, - bucket: StatisticalBucketIncludeProfile, - ) { - self.total_elapsed += total_elapsed; - self.bounds_elapsed += bounds_elapsed; - self.builder_elapsed += builder_elapsed; - self.bucket.include(bucket); - } - - fn other_elapsed(&self) -> Duration { - self.total_elapsed - .saturating_sub(self.bounds_elapsed + self.builder_elapsed + self.bucket.total_elapsed) - } -} - -#[derive(Debug, Default)] -struct AggregateIncludeProfile { - total_elapsed: Duration, - thirty_minutes: AggregateGranularityProfile, - one_hour: AggregateGranularityProfile, - one_day: AggregateGranularityProfile, -} - -impl AggregateIncludeProfile { - fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.thirty_minutes.include( - profile.thirty_minutes.total_elapsed, - profile.thirty_minutes.bounds_elapsed, - profile.thirty_minutes.builder_elapsed, - profile.thirty_minutes.bucket, - ); - self.one_hour.include( - profile.one_hour.total_elapsed, - profile.one_hour.bounds_elapsed, - profile.one_hour.builder_elapsed, - profile.one_hour.bucket, - ); - self.one_day.include( - profile.one_day.total_elapsed, - profile.one_day.bounds_elapsed, - profile.one_day.builder_elapsed, - profile.one_day.bucket, - ); - } - - fn granularity_mut(&mut self, granularity: Granularity) -> &mut AggregateGranularityProfile { - match granularity { - Granularity::ThirtyMinutes => &mut self.thirty_minutes, - Granularity::OneHour => &mut self.one_hour, - Granularity::OneDay => &mut self.one_day, - Granularity::FiveMinutes => unreachable!("five-minute buckets are not rollups"), - } - } - - fn other_elapsed(&self) -> Duration { - self.total_elapsed.saturating_sub( - self.thirty_minutes.total_elapsed - + self.one_hour.total_elapsed - + self.one_day.total_elapsed, - ) - } -} - -#[derive(Debug, Default)] -struct NfcapdDayPublishProfile { - day_elapsed: Duration, - prepare_elapsed: Duration, - decode_elapsed: Duration, - batch_publish_elapsed: Duration, - logical_source_elapsed: Duration, - persisted_sibling_elapsed: Duration, - bucket_publish: NfcapdBucketPublishProfile, - aggregate_include: AggregateIncludeProfile, - completed_rollup_flush_elapsed: Duration, - completed_rollup_write: WriteBucketsProfile, - final_rollups: FinalRollupProfile, - logical_buckets: u64, - completed_rollup_flushes: u64, - nonempty_rollup_flushes: u64, -} - -impl NfcapdDayPublishProfile { - fn log(&self, day_start: i64, day_end: i64, transaction_elapsed: Duration) { - let mut rollup_write = self.completed_rollup_write.clone(); - rollup_write.include(self.final_rollups.write.clone()); - let publish_other = self.batch_publish_elapsed.saturating_sub( - self.logical_source_elapsed - + self.persisted_sibling_elapsed - + self.bucket_publish.total_elapsed - + self.aggregate_include.total_elapsed - + self.completed_rollup_flush_elapsed, - ); - let transaction_other = - transaction_elapsed.saturating_sub(self.day_elapsed + self.final_rollups.total_elapsed); - let completed_rollup_housekeeping = self - .completed_rollup_flush_elapsed - .saturating_sub(self.completed_rollup_write.total_elapsed); - tracing::info!( - target: "netflow_db::profile", - phase = "nfcapd_tree_day_publish_detail", - day_start, - day_end, - transaction_seconds = transaction_elapsed.as_secs_f64(), - transaction_other_seconds = transaction_other.as_secs_f64(), - day_seconds = self.day_elapsed.as_secs_f64(), - prepare_seconds = self.prepare_elapsed.as_secs_f64(), - decode_seconds = self.decode_elapsed.as_secs_f64(), - batch_publish_seconds = self.batch_publish_elapsed.as_secs_f64(), - publish_other_seconds = publish_other.as_secs_f64(), - logical_source_seconds = self.logical_source_elapsed.as_secs_f64(), - persisted_sibling_seconds = self.persisted_sibling_elapsed.as_secs_f64(), - bucket_publish_seconds = self.bucket_publish.total_elapsed.as_secs_f64(), - bucket_preflight_seconds = self.bucket_publish.preflight_elapsed.as_secs_f64(), - bucket_overlap_seconds = self.bucket_publish.overlap_elapsed.as_secs_f64(), - bucket_force_delete_seconds = self.bucket_publish.force_delete_elapsed.as_secs_f64(), - owner_upsert_seconds = self.bucket_publish.owner_upsert_elapsed.as_secs_f64(), - owner_status_seconds = self.bucket_publish.owner_status_elapsed.as_secs_f64(), - bucket_postflight_seconds = self.bucket_publish.postflight_elapsed.as_secs_f64(), - bucket_other_seconds = self.bucket_publish.other_elapsed().as_secs_f64(), - aggregate_include_seconds = self.aggregate_include.total_elapsed.as_secs_f64(), - aggregate_include_other_seconds = self.aggregate_include.other_elapsed().as_secs_f64(), - aggregate_30m_seconds = self.aggregate_include.thirty_minutes.total_elapsed.as_secs_f64(), - aggregate_30m_bounds_seconds = self.aggregate_include.thirty_minutes.bounds_elapsed.as_secs_f64(), - aggregate_30m_builder_seconds = self.aggregate_include.thirty_minutes.builder_elapsed.as_secs_f64(), - aggregate_30m_traffic_seconds = self.aggregate_include.thirty_minutes.bucket.traffic_elapsed.as_secs_f64(), - aggregate_30m_protocols_seconds = self.aggregate_include.thirty_minutes.bucket.protocols_elapsed.as_secs_f64(), - aggregate_30m_addresses_seconds = self.aggregate_include.thirty_minutes.bucket.addresses_elapsed.as_secs_f64(), - aggregate_30m_ports_seconds = self.aggregate_include.thirty_minutes.bucket.ports_elapsed.as_secs_f64(), - aggregate_30m_coverage_seconds = self.aggregate_include.thirty_minutes.bucket.coverage_elapsed.as_secs_f64(), - aggregate_30m_bucket_other_seconds = self.aggregate_include.thirty_minutes.bucket.other_elapsed().as_secs_f64(), - aggregate_30m_other_seconds = self.aggregate_include.thirty_minutes.other_elapsed().as_secs_f64(), - aggregate_1h_seconds = self.aggregate_include.one_hour.total_elapsed.as_secs_f64(), - aggregate_1h_bounds_seconds = self.aggregate_include.one_hour.bounds_elapsed.as_secs_f64(), - aggregate_1h_builder_seconds = self.aggregate_include.one_hour.builder_elapsed.as_secs_f64(), - aggregate_1h_traffic_seconds = self.aggregate_include.one_hour.bucket.traffic_elapsed.as_secs_f64(), - aggregate_1h_protocols_seconds = self.aggregate_include.one_hour.bucket.protocols_elapsed.as_secs_f64(), - aggregate_1h_addresses_seconds = self.aggregate_include.one_hour.bucket.addresses_elapsed.as_secs_f64(), - aggregate_1h_ports_seconds = self.aggregate_include.one_hour.bucket.ports_elapsed.as_secs_f64(), - aggregate_1h_coverage_seconds = self.aggregate_include.one_hour.bucket.coverage_elapsed.as_secs_f64(), - aggregate_1h_bucket_other_seconds = self.aggregate_include.one_hour.bucket.other_elapsed().as_secs_f64(), - aggregate_1h_other_seconds = self.aggregate_include.one_hour.other_elapsed().as_secs_f64(), - aggregate_1d_seconds = self.aggregate_include.one_day.total_elapsed.as_secs_f64(), - aggregate_1d_bounds_seconds = self.aggregate_include.one_day.bounds_elapsed.as_secs_f64(), - aggregate_1d_builder_seconds = self.aggregate_include.one_day.builder_elapsed.as_secs_f64(), - aggregate_1d_traffic_seconds = self.aggregate_include.one_day.bucket.traffic_elapsed.as_secs_f64(), - aggregate_1d_protocols_seconds = self.aggregate_include.one_day.bucket.protocols_elapsed.as_secs_f64(), - aggregate_1d_addresses_seconds = self.aggregate_include.one_day.bucket.addresses_elapsed.as_secs_f64(), - aggregate_1d_ports_seconds = self.aggregate_include.one_day.bucket.ports_elapsed.as_secs_f64(), - aggregate_1d_coverage_seconds = self.aggregate_include.one_day.bucket.coverage_elapsed.as_secs_f64(), - aggregate_1d_bucket_other_seconds = self.aggregate_include.one_day.bucket.other_elapsed().as_secs_f64(), - aggregate_1d_other_seconds = self.aggregate_include.one_day.other_elapsed().as_secs_f64(), - completed_rollup_flush_seconds = self.completed_rollup_flush_elapsed.as_secs_f64(), - completed_rollup_housekeeping_seconds = completed_rollup_housekeeping.as_secs_f64(), - final_rollup_seconds = self.final_rollups.total_elapsed.as_secs_f64(), - final_rollup_finish_seconds = self.final_rollups.finish_elapsed.as_secs_f64(), - final_rollup_delete_seconds = self.final_rollups.delete_elapsed.as_secs_f64(), - final_rollup_other_seconds = self.final_rollups.other_elapsed().as_secs_f64(), - five_minute_write_seconds = self.bucket_publish.write.total_elapsed.as_secs_f64(), - five_minute_delete_seconds = self.bucket_publish.write.delete_elapsed.as_secs_f64(), - five_minute_canonical_rows_seconds = self.bucket_publish.write.canonical_rows_elapsed.as_secs_f64(), - five_minute_scalar_rows_seconds = self.bucket_publish.write.scalar_rows_elapsed.as_secs_f64(), - five_minute_scalar_insert_seconds = scalar_insert_elapsed(&self.bucket_publish.write).as_secs_f64(), - five_minute_maad_seconds = self.bucket_publish.write.maad_elapsed.as_secs_f64(), - five_minute_address_structure_insert_seconds = self.bucket_publish.write.address_structure_insert_elapsed.as_secs_f64(), - five_minute_write_other_seconds = self.bucket_publish.write.other_elapsed().as_secs_f64(), - rollup_write_seconds = rollup_write.total_elapsed.as_secs_f64(), - rollup_delete_seconds = rollup_write.delete_elapsed.as_secs_f64(), - rollup_canonical_rows_seconds = rollup_write.canonical_rows_elapsed.as_secs_f64(), - rollup_scalar_rows_seconds = rollup_write.scalar_rows_elapsed.as_secs_f64(), - rollup_scalar_insert_seconds = scalar_insert_elapsed(&rollup_write).as_secs_f64(), - rollup_maad_seconds = rollup_write.maad_elapsed.as_secs_f64(), - rollup_address_structure_insert_seconds = rollup_write.address_structure_insert_elapsed.as_secs_f64(), - rollup_write_other_seconds = rollup_write.other_elapsed().as_secs_f64(), - logical_buckets = self.logical_buckets, - owners = self.bucket_publish.owners, - absences = self.bucket_publish.absences, - completed_rollup_flushes = self.completed_rollup_flushes, - nonempty_rollup_flushes = self.nonempty_rollup_flushes, - final_incomplete_keys = self.final_rollups.incomplete_keys, - final_rollup_buckets = self.final_rollups.rollup_buckets, - five_minute_write_calls = self.bucket_publish.write.write_calls, - rollup_write_calls = rollup_write.write_calls, - five_minute_bucket_keys = self.bucket_publish.write.bucket_keys, - rollup_bucket_keys = rollup_write.bucket_keys, - traffic_rows = self.bucket_publish.write.traffic_rows + rollup_write.traffic_rows, - protocol_rows = self.bucket_publish.write.protocol_rows + rollup_write.protocol_rows, - address_count_rows = self.bucket_publish.write.address_count_rows + rollup_write.address_count_rows, - port_count_rows = self.bucket_publish.write.port_count_rows + rollup_write.port_count_rows, - address_structure_rows = self.bucket_publish.write.address_structure_rows + rollup_write.address_structure_rows, - maad_address_sets = self.bucket_publish.write.maad_address_sets + rollup_write.maad_address_sets, - maad_addresses = self.bucket_publish.write.maad_addresses + rollup_write.maad_addresses, - address_structure_json_bytes = self.bucket_publish.write.address_structure_json_bytes + rollup_write.address_structure_json_bytes, - ); - } -} - -fn scalar_insert_elapsed(profile: &WriteBucketsProfile) -> Duration { - profile.traffic_insert_elapsed - + profile.protocol_insert_elapsed - + profile.address_count_insert_elapsed - + profile.port_count_insert_elapsed -} - -fn profile_count(value: usize) -> u64 { - u64::try_from(value).unwrap_or(u64::MAX) -} - #[derive(Clone, Debug)] struct PreparedRevision { revision: InputRevision, @@ -2327,12 +2997,10 @@ struct PreparedTreeJob { owners: Vec, absences: Vec, evidence: Vec, - is_repair: bool, } struct PreparedTreeTimestamp { bucket_start: i64, - revision_cache: BTreeMap, jobs: Vec, } @@ -2343,72 +3011,32 @@ fn publish_nfcapd_bucket( owners: &[PreparedRevision], absences: &[ExpectedAbsence], evidence: &[InputEvidenceRow], - allow_coverage_repair: bool, - force: bool, + replace_existing: bool, run_maad: bool, ) -> Result<(), PipelineError> { - publish_nfcapd_bucket_profiled( - connection, - bucket, - owners, - absences, - evidence, - allow_coverage_repair, - force, - run_maad, - ) - .map(|_| ()) -} - -#[allow(clippy::too_many_arguments)] -fn publish_nfcapd_bucket_profiled( - connection: &Connection, - bucket: &CanonicalBucket, - owners: &[PreparedRevision], - absences: &[ExpectedAbsence], - evidence: &[InputEvidenceRow], - allow_coverage_repair: bool, - force: bool, - run_maad: bool, -) -> Result { - let total_started = Instant::now(); - let mut profile = NfcapdBucketPublishProfile { - owners: profile_count(owners.len()), - absences: profile_count(absences.len()), - ..NfcapdBucketPublishProfile::default() - }; - let preflight_started = Instant::now(); + for prepared in owners { + if let Some(snapshot) = &prepared.snapshot { + verify_file_snapshot(Path::new(&prepared.revision.locator), snapshot)?; + } + } for absence in absences { absence.verify()?; } - for owner in owners { - if let Some(snapshot) = &owner.snapshot { - verify_file_snapshot(&owner.revision.locator, snapshot)?; - } - } - profile.preflight_elapsed += preflight_started.elapsed(); - let overlap_started = Instant::now(); - reject_overlapping_bucket( - connection, - bucket, - InputKind::Nfcapd, - "", - force || allow_coverage_repair, - )?; - profile.overlap_elapsed += overlap_started.elapsed(); - if force { - let force_delete_started = Instant::now(); - connection.execute( - "DELETE FROM processed_inputs WHERE input_kind = 'nfcapd' AND source_id = ?1 AND bucket_start = ?2", - params![bucket.key.source_id, bucket.key.bucket_start], - ).map_err(StorageError::from)?; - profile.force_delete_elapsed += force_delete_started.elapsed(); + reject_overlapping_bucket(connection, bucket, InputKind::Nfcapd, "", replace_existing)?; + if replace_existing { + connection + .execute( + "DELETE FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = ?1 AND bucket_start = ?2", + params![bucket.key.source_id, bucket.key.bucket_start], + ) + .map_err(StorageError::from)?; } - let publication = (|| -> Result<(), PipelineError> { - let owner_upsert_started = Instant::now(); - for prepared in owners { - let revision = &prepared.revision; - let owner = InputBucket { + for prepared in owners { + let revision = &prepared.revision; + upsert_input_bucket( + connection, + &InputBucket { input_kind: InputKind::Nfcapd, input_locator: revision.locator.clone(), scan_locator: revision.locator.clone(), @@ -2417,42 +3045,31 @@ fn publish_nfcapd_bucket_profiled( bucket_end: bucket.key.bucket_end, revision: revision.clone(), file_snapshot: prepared.snapshot.clone(), - }; - upsert_input_bucket(connection, &owner, force)?; - } - profile.owner_upsert_elapsed += owner_upsert_started.elapsed(); - profile.write = write_buckets_profiled(connection, std::slice::from_ref(bucket), run_maad)?; - replace_input_evidence( + }, + replace_existing, + )?; + } + write_buckets(connection, std::slice::from_ref(bucket), run_maad)?; + replace_input_evidence( + connection, + &bucket.key.source_id, + bucket.key.bucket_start, + evidence, + )?; + for prepared in owners { + let revision = &prepared.revision; + mark_input_bucket_status( connection, + InputKind::Nfcapd, + &revision.locator, &bucket.key.source_id, bucket.key.bucket_start, - evidence, + InputStatus::Processed, + revision, + None, )?; - let owner_status_started = Instant::now(); - for prepared in owners { - let revision = &prepared.revision; - mark_input_bucket_status( - connection, - InputKind::Nfcapd, - &revision.locator, - &bucket.key.source_id, - bucket.key.bucket_start, - InputStatus::Processed, - revision, - None, - )?; - } - profile.owner_status_elapsed += owner_status_started.elapsed(); - let postflight_started = Instant::now(); - for absence in absences { - absence.verify()?; - } - profile.postflight_elapsed += postflight_started.elapsed(); - Ok(()) - })(); - publication?; - profile.total_elapsed = total_started.elapsed(); - Ok(profile) + } + Ok(()) } fn reject_overlapping_bucket( @@ -2501,6 +3118,7 @@ struct AggregateBuckets { published_through: BTreeMap, owned_keys: BTreeSet<(String, i64)>, current_run_keys: BTreeSet<(String, i64)>, + persisted_sibling_validations: BTreeSet<(String, i64, bool)>, } impl AggregateBuckets { @@ -2512,7 +3130,7 @@ impl AggregateBuckets { } fn reject_persisted_siblings( - &self, + &mut self, connection: &Connection, child: &CanonicalBucket, timezone: &str, @@ -2521,7 +3139,7 @@ impl AggregateBuckets { } fn reject_persisted_csv_siblings( - &self, + &mut self, connection: &Connection, child: &CanonicalBucket, timezone: &str, @@ -2530,7 +3148,7 @@ impl AggregateBuckets { } fn reject_persisted_siblings_inner( - &self, + &mut self, connection: &Connection, child: &CanonicalBucket, timezone: &str, @@ -2538,6 +3156,14 @@ impl AggregateBuckets { ) -> Result<(), PipelineError> { let (day_start, day_end) = aggregate_bounds(child.key.bucket_start, Granularity::OneDay, timezone)?; + let validation_key = ( + child.key.source_id.clone(), + day_start, + allow_staged_csv_keys, + ); + if self.persisted_sibling_validations.contains(&validation_key) { + return Ok(()); + } let mut statement = connection .prepare( "SELECT DISTINCT bucket_start FROM traffic_stats @@ -2583,20 +3209,14 @@ impl AggregateBuckets { child.key.source_id, child.key.bucket_start ))); } + // This validation is intentionally local to one aggregate transaction/output. Persisted + // siblings cannot change except through keys owned by this run, which are already excluded + // above, so later children in the same source/day can reuse the successful result. + self.persisted_sibling_validations.insert(validation_key); Ok(()) } fn include(&mut self, child: &CanonicalBucket, timezone: &str) -> Result<(), PipelineError> { - self.include_profiled(child, timezone).map(|_| ()) - } - - fn include_profiled( - &mut self, - child: &CanonicalBucket, - timezone: &str, - ) -> Result { - let total_started = Instant::now(); - let mut profile = AggregateIncludeProfile::default(); if self .published_through .get(&child.key.source_id) @@ -2614,30 +3234,18 @@ impl AggregateBuckets { Granularity::OneHour, Granularity::OneDay, ] { - let granularity_started = Instant::now(); - let bounds_started = Instant::now(); let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; - let bounds_elapsed = bounds_started.elapsed(); let key = (child.key.source_id.clone(), granularity, start, end); - let builder_started = Instant::now(); let builder = self.builders.entry(key.clone()).or_insert_with(|| { StatisticalBucket::new(BucketKey::new(&key.0, key.1, key.2, key.3)) }); - let builder_elapsed = builder_started.elapsed(); - let bucket = builder.include_profiled(child)?; - profile.granularity_mut(granularity).include( - granularity_started.elapsed(), - bounds_elapsed, - builder_elapsed, - bucket, - ); + builder.include(child)?; } self.published_through .insert(child.key.source_id.clone(), child.key.bucket_start); self.current_run_keys .insert((child.key.source_id.clone(), child.key.bucket_start)); - profile.total_elapsed = total_started.elapsed(); - Ok(profile) + Ok(()) } fn flush_complete( @@ -2645,15 +3253,6 @@ impl AggregateBuckets { connection: &Connection, run_maad: bool, ) -> Result { - self.flush_complete_profiled(connection, run_maad) - .map(|(count, _)| count) - } - - fn flush_complete_profiled( - &mut self, - connection: &Connection, - run_maad: bool, - ) -> Result<(usize, WriteBucketsProfile), PipelineError> { let complete_keys = self .builders .iter() @@ -2663,21 +3262,18 @@ impl AggregateBuckets { let buckets = complete_keys .into_iter() .filter_map(|key| self.builders.remove(&key)) - .map(|builder| builder.finish()) + .map(StatisticalBucket::finish_owned) .collect::>(); let count = buckets.len(); - let profile = write_buckets_profiled(connection, &buckets, run_maad)?; - Ok((count, profile)) + write_buckets(connection, &buckets, run_maad)?; + Ok(count) } - fn finish(self) -> (Vec, Vec) { - ( - self.builders - .into_values() - .map(|builder| builder.finish()) - .collect(), - Vec::new(), - ) + fn finish(self) -> Vec { + self.builders + .into_values() + .map(StatisticalBucket::finish_owned) + .collect() } } @@ -2687,108 +3283,11 @@ fn publish_rollups( pipeline: &ResolvedPipeline, report: &mut PipelineReport, ) -> Result<(), PipelineError> { - publish_rollups_profiled(connection, aggregates, pipeline, report).map(|_| ()) -} - -fn publish_rollups_profiled( - connection: &Connection, - aggregates: AggregateBuckets, - pipeline: &ResolvedPipeline, - report: &mut PipelineReport, -) -> Result { - let total_started = Instant::now(); - let finish_started = Instant::now(); - let (rollups, incomplete) = aggregates.finish(); - let finish_elapsed = finish_started.elapsed(); - let delete_started = Instant::now(); - delete_stats_bucket_keys(connection, &incomplete)?; - let delete_elapsed = delete_started.elapsed(); - let write = write_buckets_profiled(connection, &rollups, pipeline.run_maad)?; + let rollups = aggregates.finish(); + write_buckets(connection, &rollups, pipeline.run_maad)?; report.rollup_buckets += rollups.len(); - Ok(FinalRollupProfile { - total_elapsed: total_started.elapsed(), - finish_elapsed, - delete_elapsed, - write, - incomplete_keys: profile_count(incomplete.len()), - rollup_buckets: profile_count(rollups.len()), - }) -} - -/// A repaired five-minute bucket is exact, but persisted coarse unique-count -/// and MAAD rows cannot be patched from scalar results. Keep additive capture -/// coverage current and remove only the affected derived metric rows. -fn refresh_rollups_after_five_minute_repair( - connection: &Connection, - child: &CanonicalBucket, - timezone: &str, -) -> Result<(), PipelineError> { - const DERIVED_TABLES: [&str; 5] = [ - "traffic_stats", - "protocol_stats", - "address_count_stats", - "port_count_stats", - "address_structure_stats", - ]; - - for granularity in [ - Granularity::ThirtyMinutes, - Granularity::OneHour, - Granularity::OneDay, - ] { - let (start, end) = aggregate_bounds(child.key.bucket_start, granularity, timezone)?; - for table in DERIVED_TABLES { - connection - .execute( - &format!( - "DELETE FROM {table} - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3" - ), - params![child.key.source_id, granularity.as_str(), start], - ) - .map_err(StorageError::from)?; - } - - let children = query_bucket_coverage( - connection, - &child.key.source_id, - Granularity::FiveMinutes.as_str(), - start, - end, - )?; - let expected_children = - usize::try_from((end - start).div_euclid(FIVE_MINUTES)).unwrap_or(usize::MAX); - if children.len() != expected_children { - connection - .execute( - "DELETE FROM bucket_coverage - WHERE source_id = ?1 AND granularity = ?2 AND bucket_start = ?3", - params![child.key.source_id, granularity.as_str(), start], - ) - .map_err(StorageError::from)?; - continue; - } - - let mut coverage = BucketCoverage::empty(); - for row in children { - coverage - .include(row.coverage()?) - .map_err(DomainError::from)?; - } - insert_bucket_coverage_rows( - connection, - &[BucketCoverageRow::new( - &child.key.source_id, - granularity.as_str(), - start, - end, - coverage, - )], - )?; - } Ok(()) } - fn aggregate_bounds( bucket_start: i64, granularity: Granularity, @@ -2865,6 +3364,46 @@ fn next_local_five_minute_start(bucket_start: i64, timezone: &str) -> Result, + start_time: Option<&str>, + end_time: Option<&str>, + discovered_bucket_starts: impl IntoIterator, + timezone: &str, +) -> Result { + let selected_start = parse_date_start(start_date, timezone)?; + let explicit_end = end_date + .map(|date| next_date_start(date, timezone)) + .transpose()?; + let explicit_start_time = start_time + .map(|value| parse_local_datetime(value, timezone)) + .transpose()?; + let explicit_end_time = end_time + .map(|value| parse_local_datetime(value, timezone)) + .transpose()?; + let discovered_end = discovered_bucket_starts + .into_iter() + .max() + .map(|start| aggregate_bounds(start, Granularity::OneDay, timezone)) + .transpose()? + .map(|(_, end)| end) + .unwrap_or(selected_start); + let selected_end = explicit_end.unwrap_or(discovered_end); + let start = explicit_start_time.unwrap_or(selected_start); + let end = explicit_end_time.unwrap_or(selected_end); + validate_window(selected_start, selected_end, start, end, timezone)?; + Ok(NfcapdTreeWindow { start, end }) +} + fn parse_date_start(raw: &str, timezone: &str) -> Result { let date: Date = raw .parse() @@ -2957,1403 +3496,412 @@ fn expected_nfcapd_path( mod tests { use std::{ fs, - net::{IpAddr, Ipv4Addr}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, }; use rusqlite::Connection; - use serde_json::json; + use serde_json::{Value, json}; use tempfile::tempdir; use super::*; - use crate::{ - coverage::CoverageState, - domain::{AddressSide, FlowObservation, IpVersion, Scope, Visibility}, - }; - - #[cfg(unix)] - fn write_fake_nfdump(executable: &Path, setup: &str) { - use std::os::unix::fs::PermissionsExt; - let stream = executable.with_extension("stream"); - fs::write(&stream, crate::nfdump::ONE_V4_TEST_STREAM).unwrap(); + fn write_fake_nfdump(executable: &Path, invocation_log: &Path) { + let stream_path = executable.with_extension("stream"); + let empty_stream_path = executable.with_extension("empty-stream"); + let mut stream = crate::nfdump::ONE_V4_TEST_STREAM.to_vec(); + let record = 16; + stream[record + 32..record + 40].copy_from_slice(&20_u64.to_le_bytes()); + stream[record + 40..record + 48].copy_from_slice(&2_000_u64.to_le_bytes()); + stream[record + 48..record + 56].copy_from_slice(&3_u64.to_le_bytes()); + stream[record + 64..record + 66].copy_from_slice(&55_000_u16.to_le_bytes()); + stream[record + 69] = 0b010; + fs::write(&stream_path, stream).unwrap(); + fs::write( + &empty_stream_path, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); fs::write( executable, - format!("#!/bin/sh\n{setup}\ncat '{}'\n", stream.display()), + format!( + "#!/bin/sh\nif [ \"$1\" = \"-R\" ] && [ -z \"$(find \"$2\" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then\ncat '{}'\nexit 0\nfi\nprintf 'x\\n' >> '{}'\ncat '{}'\n", + empty_stream_path.display(), + invocation_log.display(), + stream_path.display(), + ), ) .unwrap(); fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); } - #[test] - fn logical_sources_borrow_singletons_and_merge_overlapping_members() { - let build = |source_id: &str, destination: [u8; 4]| { - let mut bucket = StatisticalBucket::dense(BucketKey::new( - source_id, - Granularity::FiveMinutes, - 0, - FIVE_MINUTES, - )); - bucket - .add( - FlowObservation::new( - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), - IpAddr::V4(Ipv4Addr::from(destination)), - 6, - 2, - 128, - 0, - ) - .unwrap(), - ) - .unwrap(); - bucket.finish() - }; - let cc = build("cc_ir1_gw", [198, 51, 100, 1]); - let oh = build("oh_ir1_gw", [198, 51, 100, 2]); - - let singleton = logical_source_bucket("cc_ir1_gw", 0, 1, &[&cc]).unwrap(); - assert!(matches!(singleton, Cow::Borrowed(_))); - - let combined = logical_source_bucket("uoregon_all", 0, 2, &[&cc, &oh]).unwrap(); - assert!(matches!(combined, Cow::Owned(_))); - let all_v4 = Scope::new(IpVersion::V4, Visibility::All, Visibility::All); - assert_eq!( - combined - .traffic - .iter() - .find(|entry| entry.scope == all_v4) + fn write_nfcapd_day(root: &Path, member: &str, date: &str) { + let mut bucket_start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); + let end = next_date_start(date, DEFAULT_TIMEZONE).unwrap(); + while bucket_start < end { + let timestamp = Timestamp::from_second(bucket_start) .unwrap() - .metrics - .flows, - 2 - ); - assert_eq!(combined.coverage.state(), CoverageState::Complete); + .in_tz(DEFAULT_TIMEZONE) + .unwrap(); + let path = root + .join(member) + .join(timestamp.strftime("%Y").to_string()) + .join(timestamp.strftime("%m").to_string()) + .join(timestamp.strftime("%d").to_string()) + .join(format!("nfcapd.{}", timestamp.strftime("%Y%m%d%H%M"))); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"capture").unwrap(); + bucket_start = next_local_five_minute_start(bucket_start, DEFAULT_TIMEZONE).unwrap(); + } + } - let partial = logical_source_bucket("uoregon_all", 0, 2, &[&cc]).unwrap(); - assert_eq!(partial.coverage.state(), CoverageState::Partial); - assert_eq!(partial.coverage.observed_units(), 1); + fn coordinated_request( + registry: PathBuf, + executable: &Path, + start_date: &str, + end_date: &str, + ) -> PipelineRequest { + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(registry), + start_date: Some(start_date.into()), + end_date: Some(end_date.into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: false, + require_complete: false, + } + } - let unknown = logical_source_bucket("uoregon_all", 0, 2, &[]).unwrap(); - assert_eq!(unknown.coverage.state(), CoverageState::Unknown); - assert!(unknown.traffic.is_empty()); - assert_eq!( - combined - .addresses - .iter() - .find(|entry| { - entry.scope == all_v4 && entry.address_side == AddressSide::Source - }) - .unwrap() - .addresses - .len(), - 1 - ); - assert_eq!( - combined - .addresses - .iter() - .find(|entry| { - entry.scope == all_v4 && entry.address_side == AddressSide::Destination - }) - .unwrap() - .addresses - .len(), - 2 - ); + fn daily_selection(prefix: &str) -> FlowSelection { + selection_from_value(&json!({ + "kind": "daily_active_sources", + "ip_prefix": prefix, + })) + .unwrap() } - #[test] - fn strict_coverage_checks_only_the_finite_native_request() { - let temporary = tempdir().unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let capture_root = temporary.path().join("captures"); - fs::create_dir_all(capture_root.join("r1")).unwrap(); - let inside = parse_date_start("2025-01-01", "UTC").unwrap(); - let outside = parse_date_start("2025-01-03", "UTC").unwrap(); - for (source_id, bucket_start) in [("r1", inside), ("r1", outside), ("unrequested", inside)] - { - connection - .execute( - "INSERT INTO bucket_coverage ( - source_id, granularity, bucket_start, bucket_end, - coverage_state, observed_units, expected_units, rejected_units - ) VALUES (?1, '5m', ?2, ?3, 'unknown', 0, 1, 0)", - params![source_id, bucket_start, bucket_start + FIVE_MINUTES], - ) - .unwrap(); - } - let pipeline = ResolvedPipeline { - database_path: temporary.path().join("netflow.sqlite"), - timezone: "UTC".into(), + fn resolved_pipeline( + root: &Path, + database: PathBuf, + member: &str, + timezone: &str, + ) -> ResolvedPipeline { + ResolvedPipeline { + database_path: database, + timezone: timezone.into(), run_maad: false, - nfdump: "nfdump".into(), - selection: FlowSelection::default(), + nfdump: PathBuf::from("/bin/true"), + nfdump_revision: None, + selection: daily_selection("192.0.0.0/16"), inputs: vec![InputSpec::NfcapdTree { - root_path: capture_root, - source_ids: vec!["r1".into()], + root_path: root.to_owned(), + source_ids: vec![member.into()], sources: Vec::new(), - start_date: "2025-01-01".into(), - end_date: Some("2025-01-01".into()), + start_date: "2025-06-01".into(), + end_date: Some("2025-06-01".into()), start_time: None, end_time: None, force: false, }], datasets: Vec::new(), - require_complete: true, - }; + require_complete: false, + } + } - assert_eq!( - count_incomplete_requested_coverage(&connection, &pipeline).unwrap(), - 1 - ); + fn incompatibility(pipelines: Vec) -> String { + match validate_compatible_pipelines(pipelines) { + Ok(_) => panic!("pipelines unexpectedly compatible"), + Err(error) => error.to_string(), + } } #[test] - fn coarse_rollups_keep_observed_zeroes_without_fabricating_unknown_metrics() { - let unknown = StatisticalBucket::new(BucketKey::new( - "r1", - Granularity::FiveMinutes, - 0, - FIVE_MINUTES, - )) - .with_coverage(BucketCoverage::new(1, 0, 0).unwrap()) - .finish(); - let mut unknown_aggregates = AggregateBuckets::default(); - unknown_aggregates.include(&unknown, "UTC").unwrap(); - let (unknown_rollups, _) = unknown_aggregates.finish(); - assert_eq!(unknown_rollups.len(), 3); - assert!( - unknown_rollups - .iter() - .all(|bucket| bucket.traffic.is_empty()) - ); - - let observed_zero = StatisticalBucket::dense(BucketKey::new( - "r1", - Granularity::FiveMinutes, - 0, - FIVE_MINUTES, - )) - .with_coverage(BucketCoverage::complete_unit()) - .finish(); - let mut observed_aggregates = AggregateBuckets::default(); - observed_aggregates.include(&observed_zero, "UTC").unwrap(); - let (observed_rollups, _) = observed_aggregates.finish(); - assert_eq!(observed_rollups.len(), 3); - assert!(observed_rollups.iter().all(|bucket| { - !bucket.traffic.is_empty() - && bucket.traffic.iter().all(|entry| entry.metrics.flows == 0) - })); + fn coordinated_compatibility_rejects_duplicate_ids_roots_layouts_timezones_and_outputs() { + let temporary = tempdir().unwrap(); + let first_root = temporary.path().join("first-root"); + let second_root = temporary.path().join("second-root"); + for path in [ + first_root.join("edge"), + first_root.join("other"), + second_root.join("edge"), + ] { + fs::create_dir_all(path).unwrap(); + } + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); + + let duplicate = run_many( + PipelineRequest { + config_path: None, + dataset_id: None, + datasets_path: Some(temporary.path().join("unused.json")), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: "/bin/true".into(), + force: false, + run_maad: false, + require_complete: false, + }, + vec!["same".into(), "same".into()], + ) + .unwrap_err(); + assert!(duplicate.to_string().contains("cannot repeat dataset")); + + let roots = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&second_root, second_db.clone(), "edge", DEFAULT_TIMEZONE), + ]); + assert!(roots.contains("same nfcapd root"), "{roots}"); + + let layouts = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&first_root, second_db.clone(), "other", DEFAULT_TIMEZONE), + ]); + assert!(layouts.contains("same logical source layout"), "{layouts}"); + + let timezones = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&first_root, second_db.clone(), "edge", "UTC"), + ]); + assert!(timezones.contains("same timezone"), "{timezones}"); + + let outputs = incompatibility(vec![ + resolved_pipeline(&first_root, first_db.clone(), "edge", DEFAULT_TIMEZONE), + resolved_pipeline(&first_root, first_db, "edge", DEFAULT_TIMEZONE), + ]); + assert!(outputs.contains("must be distinct"), "{outputs}"); } #[test] - fn pipeline_persists_in_process_maad_identity() { + fn auto_discovered_nfcapd_root_rejects_output_that_would_create_a_member_directory() { let temporary = tempdir().unwrap(); - let database = temporary.path().join("pipeline.sqlite"); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, &temporary.path().join("invocations")); + let database = root.join("future-member/netflow.sqlite"); let config = temporary.path().join("pipeline.json"); fs::write( &config, serde_json::to_vec(&json!({ "database_path": database, - "timezone": "UTC", - "run_maad": true, - "inputs": [] + "timezone": DEFAULT_TIMEZONE, + "nfdump": executable, + "inputs": [{ + "input_kind": "nfcapd_tree", + "root_path": root, + "start_date": "2025-06-01", + "end_date": "2025-06-01" + }] })) .unwrap(), ) .unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - - let connection = Connection::open(database).unwrap(); - let config_json: Value = connection - .query_row( - "SELECT config_json FROM pipeline_product WHERE singleton = 1", - [], - |row| row.get::<_, String>(0), - ) - .unwrap() - .parse() - .unwrap(); - assert_eq!( - config_json["maad"], - json!({ - "enabled": true, - "backend": "in-process", - "contract_version": 2, - "config": { - "q_min": -0.5, - "q_max": 3.5, - "q_step": 0.125, - "min_prefix_length": 8, - "max_prefix_length": 24, - "full_threshold": 0.05 - } - }) - ); - } + let error = run(PipelineRequest::config(&config)).unwrap_err(); + assert!( + error + .to_string() + .contains("overlaps the nfcapd capture tree") + ); + assert!(!database.exists()); + } #[test] - fn config_run_publishes_csv_gaps_as_coverage_only_buckets() { + fn dataset_mode_applies_its_persisted_selection() { let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let input = temporary.path().join("flows.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": { - "time_end": "time", - "src_ip": "src", - "dst_ip": "dst", - "protocol": "protocol", - "packets": "packets", - "bytes": "bytes" - }, - "source_id": {"value": "r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &input, - "time,src,dst,protocol,packets,bytes\n\ - 2025-01-15 00:00:00,192.0.2.1,198.51.100.1,6,1,10\n\ - 2025-01-15 00:25:00,192.0.2.2,198.51.100.2,17,2,20\n", - ) - .unwrap(); + let root = temporary.path().join("captures"); + fs::create_dir_all(root.join("edge")).unwrap(); + let executable = temporary.path().join("fake-nfdump"); + write_fake_nfdump(&executable, &temporary.path().join("invocations")); + let registry = temporary.path().join("datasets.json"); + let database = temporary.path().join("active.sqlite"); fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "UTC", - "run_maad": false, - "inputs": [{ - "input_kind": "csv", - "path": input, - "mapping_path": mapping - }] - })) + ®istry, + serde_json::to_vec(&json!([{ + "dataset_id": "active", + "root_path": root, + "db_path": database, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "72.5.0.0/16" + } + }])) .unwrap(), ) .unwrap(); - let report = run(PipelineRequest::config(&config)).unwrap(); - - assert_eq!(report.five_minute_buckets, 6); - assert_eq!(report.complete_five_minute_buckets, 2); - assert_eq!(report.partial_five_minute_buckets, 0); - assert_eq!(report.unknown_five_minute_buckets, 4); - assert_eq!(report.rollup_buckets, 3); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m' AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state = 'unknown'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 4 - ); - assert_eq!( - connection - .query_row( - "SELECT flows FROM traffic_stats WHERE granularity = '30m' AND ip_version = 4 AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT coverage_state FROM bucket_coverage - WHERE granularity = '30m'", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "partial" - ); - - // Simulate a legacy/in-progress product without planner statistics. The strict run still - // returns its coverage error, but first leaves that inspectable product optimized. - connection.execute("DELETE FROM sqlite_stat1", []).unwrap(); + let resolved = resolve_request(&PipelineRequest { + config_path: None, + dataset_id: Some("active".into()), + datasets_path: Some(registry), + start_date: Some("2025-06-01".into()), + end_date: Some("2025-06-01".into()), + start_time: None, + end_time: None, + database_path: None, + selection: Value::Null, + nfdump: executable.to_string_lossy().into_owned(), + force: false, + run_maad: true, + require_complete: false, + }) + .unwrap(); - let mut strict = PipelineRequest::config(&config); - strict.require_complete = true; - assert!(matches!( - run(strict), - Err(PipelineError::IncompleteCoverage(4)) - )); - assert!( - connection - .query_row("SELECT COUNT(*) FROM sqlite_stat1", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap() - > 0 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity IN ('1h', '1d')", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 20 - ); + assert!(resolved.selection.selects_daily_active_sources()); + assert_eq!(resolved.database_path, database); } - /// Run a one-dataset pipeline over optional CSV rows and read back the stored start date. - fn stored_default_start_date(dataset: Value, csv_rows: &str) -> String { + #[test] + fn coordinated_products_share_decode_and_resume_by_whole_day() { let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let input = temporary.path().join("flows.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); + let root = temporary.path().join("captures"); + write_nfcapd_day(&root, "edge", "2025-06-01"); + write_nfcapd_day(&root, "edge", "2025-06-02"); + let executable = temporary.path().join("fake-nfdump"); + let invocation_log = temporary.path().join("invocations"); + write_fake_nfdump(&executable, &invocation_log); + let registry = temporary.path().join("datasets.json"); + let first_db = temporary.path().join("first.sqlite"); + let second_db = temporary.path().join("second.sqlite"); fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": { - "time_end": "time", - "src_ip": "src", - "dst_ip": "dst", - "protocol": "protocol", - "packets": "packets", - "bytes": "bytes" + ®istry, + serde_json::to_vec(&json!([ + { + "dataset_id": "first", + "root_path": root, + "db_path": first_db, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "192.0.0.0/16" + } }, - "source_id": {"value": "r1"} - })) - .unwrap(), - ) - .unwrap(); - let inputs = if csv_rows.is_empty() { - json!([]) - } else { - fs::write( - &input, - format!("time,src,dst,protocol,packets,bytes\n{csv_rows}"), - ) - .unwrap(); - json!([{"input_kind": "csv", "path": input, "mapping_path": mapping}]) - }; - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "America/Los_Angeles", - "run_maad": false, - "inputs": inputs, - "datasets": [dataset] - })) + { + "dataset_id": "second", + "root_path": root, + "db_path": second_db, + "source_ids": ["edge"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "198.51.0.0/16" + } + } + ])) .unwrap(), ) .unwrap(); + let request = coordinated_request(registry, &executable, "2025-06-01", "2025-06-02"); - run(PipelineRequest::config(&config)).unwrap(); + let initial = run_many(request.clone(), vec!["first".into(), "second".into()]).unwrap(); + assert_eq!(initial.five_minute_buckets, 1_152); + let initial_invocations = fs::read_to_string(&invocation_log).unwrap().lines().count(); + assert_eq!(initial_invocations, 578); - Connection::open(&database) - .unwrap() + let first = Connection::open(&first_db).unwrap(); + let second = Connection::open(&second_db).unwrap(); + let first_max = first .query_row( - "SELECT default_start_date FROM datasets WHERE id = 'example'", + "SELECT MAX(flows) FROM traffic_stats WHERE granularity = '5m'", [], - |row| row.get::<_, String>(0), + |row| row.get::<_, Option>(0), ) .unwrap() - } - - #[test] - fn unset_start_date_becomes_the_earliest_ingested_local_day() { - // 2025-01-15 00:00 UTC is still 2025-01-14 in the pipeline timezone. - assert_eq!( - stored_default_start_date( - json!({"dataset_id": "example", "root_path": "/captures"}), - "2025-01-15 00:00:00,192.0.2.1,198.51.100.1,6,1,10\n", - ), - "2025-01-14" - ); - } - - #[test] - fn configured_start_date_survives_ingestion() { - assert_eq!( - stored_default_start_date( - json!({ - "dataset_id": "example", - "root_path": "/captures", - "default_start_date": "2024-12-25" - }), - "2025-01-15 00:00:00,192.0.2.1,198.51.100.1,6,1,10\n", - ), - "2024-12-25" - ); - } - - #[test] - fn unset_start_date_falls_back_when_nothing_is_ingested() { - assert_eq!( - stored_default_start_date( - json!({"dataset_id": "example", "root_path": "/captures"}), - "" - ), - crate::storage::FALLBACK_DEFAULT_START_DATE - ); - } - - #[test] - fn csv_tree_files_share_rollups_within_one_transaction() { - let temporary = tempdir().unwrap(); - let inputs = temporary.path().join("inputs"); - let mapping = temporary.path().join("mapping.json"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::create_dir(&inputs).unwrap(); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header":true, - "timestamp_format":"datetime", - "timestamp_timezone":"UTC", - "columns":{"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id":{"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - inputs.join("2025-01-a.csv"), - "time,src,dst\n\ - 2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n\ - 2025-01-15 11:55:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - fs::write( - inputs.join("2025-01-b.csv"), - "time,src,dst\n\ - 2025-01-15 12:00:00,192.0.2.3,198.51.100.3\n\ - 2025-01-15 23:55:00,192.0.2.4,198.51.100.4\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "run_maad":false, - "inputs":[{ - "input_kind":"csv_tree", - "root_path":inputs, - "mapping_path":mapping - }] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.input_scans, 2); - assert_eq!(report.five_minute_buckets, 288); - assert_eq!(report.rollup_buckets, 73); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(DISTINCT granularity || ':' || bucket_start) FROM traffic_stats", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 13 - ); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM bucket_coverage", [], |row| row - .get::<_, i64>(0),) - .unwrap(), - 361 - ); - } - - #[test] - fn explicit_csv_files_share_rollups_within_one_transaction() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("a.csv"); - let second = temporary.path().join("b.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header":true, - "timestamp_format":"datetime", - "timestamp_timezone":"UTC", - "columns":{"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id":{"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &first, - "time,src,dst\n\ - 2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n\ - 2025-01-15 00:10:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - fs::write( - &second, - "time,src,dst\n\ - 2025-01-15 00:15:00,192.0.2.3,198.51.100.3\n\ - 2025-01-15 00:25:00,192.0.2.4,198.51.100.4\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "run_maad":false, - "inputs":[ - {"input_kind":"csv", "path":first, "mapping_path":mapping}, - {"input_kind":"csv", "path":second, "mapping_path":mapping} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.input_scans, 2); - assert_eq!(report.five_minute_buckets, 6); - assert_eq!(report.rollup_buckets, 3); - } - - #[test] - fn overlapping_csv_batch_merges_metrics_and_coverage() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("first.csv"); - let second = temporary.path().join("second.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": {"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id": {"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - for path in [&first, &second] { - fs::write( - path, - "time,src,dst\n2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n", - ) .unwrap(); - } - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone":"UTC", - "run_maad":false, - "inputs":[ - {"input_kind":"csv", "path":first, "mapping_path":mapping}, - {"input_kind":"csv", "path":second, "mapping_path":mapping} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(&config)).unwrap(); - assert_eq!(report.five_minute_buckets, 1); - assert_eq!(report.complete_five_minute_buckets, 1); - assert_eq!(report.partial_five_minute_buckets, 0); - assert_eq!(report.unknown_five_minute_buckets, 0); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT flows FROM traffic_stats - WHERE granularity = '5m' AND source_id = 'r1' - AND bucket_start = 1736899200 AND ip_version = 4 - AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); - assert_eq!( - connection - .query_row( - "SELECT expected_units, observed_units, rejected_units - FROM bucket_coverage - WHERE granularity = '5m' AND source_id = 'r1' - AND bucket_start = 1736899200", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }, - ) - .unwrap(), - (1, 1, 0) - ); - } - - #[test] - fn csv_files_fill_unknown_buckets_across_global_source_envelope() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("first.csv"); - let second = temporary.path().join("second.csv"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header": true, - "timestamp_format": "datetime", - "timestamp_timezone": "UTC", - "columns": {"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id": {"value": "r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &first, - "time,src,dst\n2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n", - ) - .unwrap(); - fs::write( - &second, - "time,src,dst\n2025-01-15 00:25:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path": database, - "timezone": "UTC", - "run_maad": false, - "inputs": [ - {"input_kind": "csv", "path": first, "mapping_path": mapping}, - {"input_kind": "csv", "path": second, "mapping_path": mapping} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(&config)).unwrap(); - assert_eq!(report.five_minute_buckets, 6); - assert_eq!(report.complete_five_minute_buckets, 2); - assert_eq!(report.partial_five_minute_buckets, 0); - assert_eq!(report.unknown_five_minute_buckets, 4); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'r1' AND granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 6 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'r1' AND granularity = '5m' - AND coverage_state = 'unknown'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 4 - ); - } - - #[test] - fn fall_back_hours_have_distinct_hour_rollups() { - let first = "2024-11-03T01:15:00-07:00[America/Los_Angeles]" - .parse::() - .unwrap() - .timestamp() - .as_second(); - let second = "2024-11-03T01:15:00-08:00[America/Los_Angeles]" - .parse::() - .unwrap() - .timestamp() - .as_second(); - - let first_bounds = - aggregate_bounds(first, Granularity::OneHour, "America/Los_Angeles").unwrap(); - let second_bounds = - aggregate_bounds(second, Granularity::OneHour, "America/Los_Angeles").unwrap(); - - assert_eq!(first_bounds.0, first - 15 * 60); - assert_eq!(second_bounds.0, second - 15 * 60); - assert_ne!(first_bounds, second_bounds); - } - - #[test] - fn tree_windows_must_cover_complete_selected_local_days() { - let selected_start = parse_date_start("2025-02-11", "America/Los_Angeles").unwrap(); - let selected_end = next_date_start("2025-02-11", "America/Los_Angeles").unwrap(); - - validate_window( - selected_start, - selected_end, - selected_start, - selected_end, - "America/Los_Angeles", - ) - .unwrap(); - assert!( - validate_window( - selected_start, - selected_end, - selected_start + FIVE_MINUTES, - selected_end, - "America/Los_Angeles", - ) - .unwrap_err() - .to_string() - .contains("local-day boundary") - ); - assert!( - validate_window( - selected_start, - selected_end, - selected_start - 86_400, - selected_end, - "America/Los_Angeles", - ) - .unwrap_err() - .to_string() - .contains("on or after") - ); - } - - #[test] - fn later_partial_scan_cannot_reopen_persisted_rollups() { - let temporary = tempdir().unwrap(); - let mapping = temporary.path().join("mapping.json"); - let first = temporary.path().join("first.csv"); - let second = temporary.path().join("second.csv"); - let database = temporary.path().join("netflow.sqlite"); - fs::write( - &mapping, - serde_json::to_vec(&json!({ - "has_header":true, - "timestamp_format":"datetime", - "timestamp_timezone":"UTC", - "columns":{"time_end":"time", "src_ip":"src", "dst_ip":"dst"}, - "source_id":{"value":"r1"} - })) - .unwrap(), - ) - .unwrap(); - fs::write( - &first, - "time,src,dst\n2025-01-15 00:00:00,192.0.2.1,198.51.100.1\n", - ) - .unwrap(); - fs::write( - &second, - "time,src,dst\n2025-01-15 00:05:00,192.0.2.2,198.51.100.2\n", - ) - .unwrap(); - for (index, input) in [&first, &second].into_iter().enumerate() { - let config = temporary.path().join(format!("pipeline-{index}.json")); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "run_maad":false, - "inputs":[{"input_kind":"csv", "path":input, "mapping_path":mapping}] - })) - .unwrap(), + let second_max = second + .query_row( + "SELECT MAX(flows) FROM traffic_stats WHERE granularity = '5m'", + [], + |row| row.get::<_, Option>(0), ) - .unwrap(); - if index == 0 { - run(PipelineRequest::config(config)).unwrap(); - } else { - let error = run(PipelineRequest::config(config)).unwrap_err(); - assert!(error.to_string().contains("cannot reopen")); - } - } - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 1 - ); - } - - #[test] - fn unchanged_file_revision_reuses_the_persisted_digest() { - let temporary = tempdir().unwrap(); - let path = temporary.path().join("nfcapd.202501010000"); - fs::write(&path, "original").unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let snapshot = FileSnapshot::capture(&path).unwrap(); - let locator = path.to_string_lossy().into_owned(); - let original = InputRevision::create("nfcapd", &locator, "digest", "decoder").unwrap(); - upsert_input_bucket( - &connection, - &InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: locator.clone(), - scan_locator: locator.clone(), - source_id: "r1".into(), - bucket_start: 0, - bucket_end: FIVE_MINUTES, - revision: original.clone(), - file_snapshot: Some(snapshot), - }, - false, - ) - .unwrap(); - mark_input_bucket_status( - &connection, - InputKind::Nfcapd, - &locator, - "r1", - 0, - InputStatus::Processed, - &original, - None, - ) - .unwrap(); - - let (cached, _) = prepare_file_revision_with( - &connection, - &path, - InputKind::Nfcapd, - "decoder".into(), - || panic!("unchanged input must not be rehashed"), - ) - .unwrap(); - assert_eq!(cached, original); - - fs::write(&path, "replacement with a different size").unwrap(); - let rehashed = std::cell::Cell::new(false); - let (changed, _) = prepare_file_revision_with( - &connection, - &path, - InputKind::Nfcapd, - "decoder".into(), - || { - rehashed.set(true); - capture_file_revision(&path) - }, - ) - .unwrap(); - assert!(rehashed.get()); - assert_ne!(changed.content_fingerprint, original.content_fingerprint); - } - - #[test] - fn force_nfcapd_revision_resolution_rehashes_matching_snapshots() { - let temporary = tempdir().unwrap(); - let path = temporary.path().join("nfcapd.202501010000"); - fs::write(&path, "actual bytes").unwrap(); - let connection = Connection::open_in_memory().unwrap(); - init_schema(&connection).unwrap(); - let snapshot = FileSnapshot::capture(&path).unwrap(); - let locator = path.to_string_lossy().into_owned(); - let stale = InputRevision::create("nfcapd", &locator, "stale", "decoder").unwrap(); - upsert_input_bucket( - &connection, - &InputBucket { - input_kind: InputKind::Nfcapd, - input_locator: locator.clone(), - scan_locator: locator.clone(), - source_id: "r1".into(), - bucket_start: 0, - bucket_end: FIVE_MINUTES, - revision: stale, - file_snapshot: Some(snapshot), - }, - false, - ) - .unwrap(); - mark_input_bucket_status( - &connection, - InputKind::Nfcapd, - &locator, - "r1", - 0, - InputStatus::Processed, - &InputRevision::create("nfcapd", &locator, "stale", "decoder").unwrap(), - None, - ) - .unwrap(); - - let sources = [DatasetSource { - source_id: "r1".into(), - members: vec!["r1".into()], - }]; - let paths = BTreeMap::from([(("r1".into(), 0), path.clone())]); - let bounds = BTreeMap::from([("r1".into(), (0, 0))]); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(1) - .build() - .unwrap(); - let normal_context = NfcapdRevisionContext { - connection: &connection, - sources: &sources, - by_member_and_start: &paths, - member_bounds: &bounds, - extend_gaps_to_window: false, - force: false, - revision_pool: &pool, - }; - let cached = resolve_nfcapd_batch_revisions(&normal_context, &[0]) - .unwrap() - .remove(&path) - .unwrap(); - assert_eq!(cached.revision.content_fingerprint, "stale"); - - let forced_context = NfcapdRevisionContext { - force: true, - ..normal_context - }; - let forced = resolve_nfcapd_batch_revisions(&forced_context, &[0]) .unwrap() - .remove(&path) .unwrap(); - let (actual_digest, _) = capture_file_revision(&path).unwrap(); - assert_eq!(forced.revision.content_fingerprint, actual_digest); - } - - #[cfg(unix)] - #[test] - fn nfcapd_tree_commits_completed_days_before_a_later_day_fails() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let first = root.join("r1/2025/01/01/nfcapd.202501010000"); - let second = root.join("r1/2025/01/02/nfcapd.202501020000"); - fs::create_dir_all(first.parent().unwrap()).unwrap(); - fs::create_dir_all(second.parent().unwrap()).unwrap(); - fs::write(&first, "first").unwrap(); - fs::write(&second, "second").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, "case \"$*\" in *20250102*) exit 9;; esac"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "source_ids":["r1"], - "start_date":"2025-01-01", - "end_date":"2025-01-02" - }] - })) - .unwrap(), - ) - .unwrap(); - - assert!(run(PipelineRequest::config(config)).is_err()); - let connection = Connection::open(database).unwrap(); - let starts = connection - .prepare("SELECT bucket_start FROM processed_inputs ORDER BY bucket_start") - .unwrap() - .query_map([], |row| row.get::<_, i64>(0)) - .unwrap() - .collect::>>() + assert!(first_max > 0); + assert_eq!(second_max, 0); + let first_rows = first + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| { + row.get::<_, i64>(0) + }) .unwrap(); - assert_eq!(starts, [1_735_689_600]); - } + drop(first); + drop(second); - #[cfg(unix)] - #[test] - fn explicit_nfcapd_files_share_one_transaction() { - let temporary = tempdir().unwrap(); - let first = temporary.path().join("nfcapd.202501010000"); - let second = temporary.path().join("nfcapd.202501010005"); - fs::write(&first, "first").unwrap(); - fs::write(&second, "second").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[ - {"input_kind":"nfcapd", "path":first, "source_id":"r1"}, - {"input_kind":"nfcapd", "path":second, "source_id":"r1"} - ] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.five_minute_buckets, 2); - let connection = Connection::open(database).unwrap(); + let no_op = run_many(request.clone(), vec!["second".into(), "first".into()]).unwrap(); + assert_eq!(no_op.five_minute_buckets, 0); assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { - row.get::<_, i64>(0) - }) - .unwrap(), - 2 + fs::read_to_string(&invocation_log).unwrap().lines().count(), + initial_invocations, ); - } - #[cfg(unix)] - #[test] - fn fall_back_tree_uses_wall_clock_filenames_without_repeating_one_am() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let capture = root.join("r1/2025/11/02/nfcapd.202511020100"); - fs::create_dir_all(capture.parent().unwrap()).unwrap(); - fs::write(&capture, "capture").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"America/Los_Angeles", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "source_ids":["r1"], - "start_date":"2025-11-02", - "end_date":"2025-11-02" - }] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.five_minute_buckets, 288); - assert_eq!(report.rollup_buckets, 73); - let connection = Connection::open(database).unwrap(); + let day_start = parse_date_start("2025-06-02", DEFAULT_TIMEZONE).unwrap(); + let day_end = next_date_start("2025-06-02", DEFAULT_TIMEZONE).unwrap(); + let second = Connection::open(&second_db).unwrap(); + second.execute_batch("BEGIN IMMEDIATE").unwrap(); + delete_stats_time_range(&second, &["edge".to_owned()], day_start, day_end).unwrap(); + second.execute_batch("COMMIT").unwrap(); assert_eq!( - connection - .query_row( - "SELECT COUNT(DISTINCT bucket_start) FROM processed_inputs", - [], - |row| row.get::<_, i64>(0), - ) + second + .query_row("SELECT COUNT(*) FROM daily_product_completion", [], |row| { + row.get::<_, i64>(0) + },) .unwrap(), 1, - "missing inputs are evidence, not synthetic processed revisions" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 288 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE granularity = '5m' AND coverage_state = 'unknown'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 287 - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 10, - "unknown buckets must not fabricate zero-valued metric rows" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats WHERE granularity = '1d'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 10 - ); - } - - #[cfg(unix)] - #[test] - fn newly_arrived_member_repairs_five_minute_coverage_without_erasing_disappeared_inputs() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let first = root.join("r1/2025/01/01/nfcapd.202501010000"); - let second = root.join("r2/2025/01/01/nfcapd.202501010000"); - fs::create_dir_all(first.parent().unwrap()).unwrap(); - fs::create_dir_all(second.parent().unwrap()).unwrap(); - fs::write(&first, "first").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "sources":[{"source_id":"both", "members":["r1", "r2"]}], - "start_date":"2025-01-01", - "end_date":"2025-01-01" - }] - })) - .unwrap(), - ) - .unwrap(); - - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT coverage_state FROM bucket_coverage - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "partial" ); - drop(connection); + drop(second); - fs::write(&second, "second").unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(&database).unwrap(); - assert_eq!( - connection - .query_row( - "SELECT coverage_state FROM bucket_coverage - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, String>(0), - ) - .unwrap(), - "complete" - ); - assert_eq!( - connection - .query_row( - "SELECT flows FROM traffic_stats - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600 AND ip_version = 4 - AND src_visibility = 'all' AND dst_visibility = 'all'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2 - ); + let resumed = run_many(request, vec!["first".into(), "second".into()]).unwrap(); + assert_eq!(resumed.five_minute_buckets, 288); assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM traffic_stats - WHERE source_id = 'both' AND granularity <> '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 0, - "repair invalidates coarse derived rows that cannot be patched exactly" - ); - assert_eq!( - connection - .query_row( - "SELECT COUNT(*) FROM bucket_coverage - WHERE source_id = 'both' AND granularity <> '5m'", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 73, - "capture coverage remains available when derived metrics are invalidated" + fs::read_to_string(&invocation_log).unwrap().lines().count(), + initial_invocations + 289, ); - drop(connection); - - fs::remove_file(&first).unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - let connection = Connection::open(database).unwrap(); assert_eq!( - connection - .query_row( - "SELECT observed_units FROM bucket_coverage - WHERE source_id = 'both' AND granularity = '5m' - AND bucket_start = 1735689600", - [], - |row| row.get::<_, i64>(0), - ) - .unwrap(), - 2, - "a disappearing file must not erase prior observations" - ); - } - - #[cfg(unix)] - #[test] - fn implicit_tree_end_models_only_each_members_observed_bounds() { - let temporary = tempdir().unwrap(); - let root = temporary.path().join("captures"); - let first = root.join("r1/2025/01/01/nfcapd.202501010000"); - let second = root.join("r2/2025/01/02/nfcapd.202501020000"); - fs::create_dir_all(first.parent().unwrap()).unwrap(); - fs::create_dir_all(second.parent().unwrap()).unwrap(); - fs::write(&first, "first").unwrap(); - fs::write(&second, "second").unwrap(); - let decoder = temporary.path().join("fake-nfdump"); - write_fake_nfdump(&decoder, ""); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"UTC", - "nfdump":decoder, - "run_maad":false, - "inputs":[{ - "input_kind":"nfcapd_tree", - "root_path":root, - "source_ids":["r1", "r2"], - "start_date":"2025-01-01" - }] - })) - .unwrap(), - ) - .unwrap(); - - let report = run(PipelineRequest::config(config)).unwrap(); - - assert_eq!(report.five_minute_buckets, 2); - let connection = Connection::open(database).unwrap(); - assert_eq!( - connection - .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + Connection::open(&first_db) + .unwrap() + .query_row("SELECT COUNT(*) FROM traffic_stats", [], |row| { row.get::<_, i64>(0) }) .unwrap(), - 2 + first_rows, ); + for database in [&first_db, &second_db] { + assert_eq!( + Connection::open(database) + .unwrap() + .query_row("SELECT COUNT(*) FROM daily_product_completion", [], |row| { + row.get::<_, i64>(0) + },) + .unwrap(), + 2, + ); + } } - #[cfg(unix)] #[test] - fn completed_nfcapd_input_is_skipped_before_running_decoder_again() { - let temporary = tempdir().unwrap(); - let capture = temporary.path().join("nfcapd.202504151200"); - let decoder = temporary.path().join("fake-nfdump"); - let calls = temporary.path().join("calls"); - let database = temporary.path().join("netflow.sqlite"); - let config = temporary.path().join("pipeline.json"); - fs::write(&capture, "fixture").unwrap(); - write_fake_nfdump(&decoder, &format!("echo called >> '{}'", calls.display())); - fs::write( - &config, - serde_json::to_vec(&json!({ - "database_path":database, - "timezone":"America/Los_Angeles", - "nfdump":decoder, - "run_maad":false, - "inputs":[{"input_kind":"nfcapd", "path":capture, "source_id":"r1"}] - })) - .unwrap(), - ) - .unwrap(); - - run(PipelineRequest::config(&config)).unwrap(); - run(PipelineRequest::config(&config)).unwrap(); - - assert_eq!(fs::read_to_string(calls).unwrap().lines().count(), 1); + fn local_day_iteration_handles_both_dst_transitions() { + for (date, expected) in [("2025-03-09", 276), ("2025-11-02", 288)] { + let start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); + let end = next_date_start(date, DEFAULT_TIMEZONE).unwrap(); + let mut bucket_start = start; + let mut count = 0; + while bucket_start < end { + count += 1; + bucket_start = + next_local_five_minute_start(bucket_start, DEFAULT_TIMEZONE).unwrap(); + } + assert_eq!(count, expected, "{date}"); + assert_eq!(bucket_start, end, "{date}"); + } } } diff --git a/tools/netflow-db/src/provenance.rs b/tools/netflow-db/src/provenance.rs index 880a845..76c93a7 100644 --- a/tools/netflow-db/src/provenance.rs +++ b/tools/netflow-db/src/provenance.rs @@ -160,7 +160,6 @@ impl FileSnapshot { } } -#[cfg(unix)] fn snapshot_from_metadata( path: &Path, metadata: &fs::Metadata, @@ -178,32 +177,6 @@ fn snapshot_from_metadata( }) } -#[cfg(not(unix))] -fn snapshot_from_metadata( - path: &Path, - metadata: &fs::Metadata, -) -> Result { - use std::time::UNIX_EPOCH; - - let modified = metadata - .modified() - .map_err(|source| ProvenanceError::Io { - context: format!("failed to read input timestamp: {}", path.display()), - source, - })? - .duration_since(UNIX_EPOCH) - .map_err(|_| ProvenanceError::TimestampOverflow(path.to_path_buf()))?; - let mtime_ns = i64::try_from(modified.as_nanos()) - .map_err(|_| ProvenanceError::TimestampOverflow(path.to_path_buf()))?; - Ok(FileSnapshot { - device: 0, - inode: 0, - size: metadata.len(), - mtime_ns, - ctime_ns: mtime_ns, - }) -} - fn timestamp_ns(seconds: i64, nanoseconds: i64, path: &Path) -> Result { seconds .checked_mul(1_000_000_000) @@ -355,6 +328,48 @@ pub fn nfcapd_decoder_fingerprint() -> Result { })) } +/// The executable identity is part of the native decoder identity. The contract fingerprint +/// describes the stream decoder in this crate; the executable fingerprint binds that contract to +/// the exact nfdump implementation that produced the stream. +pub fn nfcapd_decoder_fingerprint_for_executable( + executable_locator: &str, + executable_content_fingerprint: &str, +) -> Result { + fingerprint(&json!({ + "version": 1, + "contract_id": nfcapd_decoder_fingerprint()?, + "executable": { + "locator": executable_locator, + "content_fingerprint": executable_content_fingerprint, + }, + })) +} + +/// One read-only snapshot of the executable used by native decoding. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExecutableRevision { + pub locator: String, + pub content_fingerprint: String, + pub snapshot: FileSnapshot, + pub decoder_fingerprint: String, +} + +impl ExecutableRevision { + pub fn capture(path: impl AsRef) -> Result { + let path = path.as_ref(); + let locator = path.to_string_lossy().into_owned(); + let (content_fingerprint, snapshot) = capture_file_revision(path)?; + let decoder_fingerprint = + nfcapd_decoder_fingerprint_for_executable(&locator, &content_fingerprint)?; + Ok(Self { + locator, + content_fingerprint, + snapshot, + decoder_fingerprint, + }) + } +} + pub fn capture_csv_input_revision( path: impl AsRef, config: &CsvSourceConfig, @@ -362,12 +377,6 @@ pub fn capture_csv_input_revision( capture_input_revision(path, "csv", csv_decoder_fingerprint(config)?) } -pub fn capture_nfcapd_input_revision( - path: impl AsRef, -) -> Result<(InputRevision, FileSnapshot), ProvenanceError> { - capture_input_revision(path, "nfcapd", nfcapd_decoder_fingerprint()?) -} - fn capture_input_revision( path: impl AsRef, input_kind: &str, diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index c42ba5a..fd13ca3 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -3,6 +3,7 @@ use std::{ collections::BTreeMap, net::IpAddr, + sync::OnceLock, time::{Duration, Instant}, }; @@ -33,10 +34,15 @@ pub enum PublishError { Storage(#[from] StorageError), #[error("unable to serialize MAAD rows: {0}")] Json(#[from] serde_json::Error), + #[error("unable to build MAAD worker pool: {0}")] + MaadPool(String), #[error("aggregate bucket lacks complete five-minute coverage: {0:?}")] IncompleteCoverage(BucketKey), } +const MAAD_WORKERS: usize = 2; +static MAAD_POOL: OnceLock> = OnceLock::new(); + /// Aggregate timings and work counts for one or more `write_buckets` calls. /// /// Timers wrap batch boundaries rather than individual rows so profiling remains @@ -53,7 +59,9 @@ pub struct WriteBucketsProfile { pub(crate) port_count_insert_elapsed: Duration, pub(crate) maad_elapsed: Duration, pub(crate) address_structure_insert_elapsed: Duration, + #[cfg(test)] pub(crate) write_calls: u64, + #[cfg(test)] pub(crate) bucket_keys: u64, pub(crate) traffic_rows: u64, pub(crate) protocol_rows: u64, @@ -66,29 +74,7 @@ pub struct WriteBucketsProfile { } impl WriteBucketsProfile { - pub(crate) fn include(&mut self, profile: Self) { - self.total_elapsed += profile.total_elapsed; - self.delete_elapsed += profile.delete_elapsed; - self.canonical_rows_elapsed += profile.canonical_rows_elapsed; - self.scalar_rows_elapsed += profile.scalar_rows_elapsed; - self.traffic_insert_elapsed += profile.traffic_insert_elapsed; - self.protocol_insert_elapsed += profile.protocol_insert_elapsed; - self.address_count_insert_elapsed += profile.address_count_insert_elapsed; - self.port_count_insert_elapsed += profile.port_count_insert_elapsed; - self.maad_elapsed += profile.maad_elapsed; - self.address_structure_insert_elapsed += profile.address_structure_insert_elapsed; - self.write_calls += profile.write_calls; - self.bucket_keys += profile.bucket_keys; - self.traffic_rows += profile.traffic_rows; - self.protocol_rows += profile.protocol_rows; - self.address_count_rows += profile.address_count_rows; - self.port_count_rows += profile.port_count_rows; - self.maad_address_sets += profile.maad_address_sets; - self.maad_addresses += profile.maad_addresses; - self.address_structure_rows += profile.address_structure_rows; - self.address_structure_json_bytes += profile.address_structure_json_bytes; - } - + #[cfg(test)] pub(crate) fn other_elapsed(&self) -> Duration { self.total_elapsed.saturating_sub( self.delete_elapsed @@ -166,7 +152,9 @@ pub(crate) fn write_buckets_profiled( ) -> Result { let total_started = Instant::now(); let mut profile = WriteBucketsProfile { + #[cfg(test)] write_calls: 1, + #[cfg(test)] bucket_keys: count(buckets.len()), ..WriteBucketsProfile::default() }; @@ -334,46 +322,72 @@ fn insert_rows( fn maad_rows( address_sets: &[AddressSetRow<'_>], ) -> Result, PublishError> { - Ok(address_sets - .par_iter() + // Filter before entering the pool so the indexed collection below keeps + // canonical input order while still allowing independent scopes to run in + // parallel. + let address_sets = address_sets + .iter() .filter(|addresses| addresses.scope.ip_version == IpVersion::V4) - .map(|addresses| { - let result = maad::compute(addresses.addresses.iter().filter_map( - |address| match address { - IpAddr::V4(address) => Some(*address), - IpAddr::V6(_) => None, - }, - )); - let metadata_json = serde_json::to_string(&result.metadata)?; - let dimensions = dimensions(&addresses.key, addresses.scope); - Ok::<_, serde_json::Error>([ - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "structure".into(), - values_json: serde_json::to_string(&result.structure)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "spectrum".into(), - values_json: serde_json::to_string(&result.spectrum)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions, - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "dimension".into(), - values_json: serde_json::to_string(&result.dimensions)?, - metadata_json, - }, - ]) - }) - .collect::, _>>()? - .into_iter() - .flatten() - .collect()) + .collect::>(); + if address_sets.is_empty() { + return Ok(Vec::new()); + } + + let pool = maad_pool()?; + let rows = + pool.install(|| { + address_sets + .par_iter() + .map(|addresses| { + let result = + maad::compute(addresses.addresses.iter().filter_map( + |address| match address { + IpAddr::V4(address) => Some(*address), + IpAddr::V6(_) => None, + }, + )); + let metadata_json = serde_json::to_string(&result.metadata)?; + let dimensions = dimensions(&addresses.key, addresses.scope); + Ok::<_, serde_json::Error>([ + AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + structure_kind: "structure".into(), + values_json: serde_json::to_string(&result.structure)?, + metadata_json: metadata_json.clone(), + }, + AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + structure_kind: "spectrum".into(), + values_json: serde_json::to_string(&result.spectrum)?, + metadata_json: metadata_json.clone(), + }, + AddressStructureStatsRow { + dimensions, + address_side: addresses.address_side.as_str().to_owned(), + structure_kind: "dimension".into(), + values_json: serde_json::to_string(&result.dimensions)?, + metadata_json, + }, + ]) + }) + .collect::, _>>() + })?; + Ok(rows.into_iter().flatten().collect()) +} + +fn maad_pool() -> Result<&'static rayon::ThreadPool, PublishError> { + match MAAD_POOL.get_or_init(|| { + rayon::ThreadPoolBuilder::new() + .num_threads(MAAD_WORKERS) + .thread_name(|index| format!("maad-{index}")) + .build() + .map_err(|error| error.to_string()) + }) { + Ok(pool) => Ok(pool), + Err(error) => Err(PublishError::MaadPool(error.clone())), + } } fn dimensions(key: &BucketKey, scope: crate::domain::Scope) -> StatsDimensions { @@ -401,7 +415,10 @@ mod tests { use super::*; use crate::{ coverage::{BucketCoverage, CoverageState}, - domain::{AddressSide, FlowObservation, IpVersion, Scope, ScopedAddressesFact, Visibility}, + domain::{ + AddressSet, AddressSide, FlowObservation, IpVersion, Scope, ScopedAddressesFact, + Visibility, + }, storage::init_stats_tables, }; @@ -507,6 +524,63 @@ mod tests { assert_eq!(product_rows(&forward), product_rows(&reverse)); } + #[test] + fn maad_rows_preserve_scope_order_and_bytes() { + let key = BucketKey::new("r1", Granularity::FiveMinutes, 0, 300); + let first_addresses = AddressSet::from_iter([ + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 3)), + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 4)), + ]); + let second_addresses = AddressSet::from_iter([ + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 3)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 4)), + ]); + let rows = [ + AddressSetRow { + key: key.clone(), + scope: Scope::new(IpVersion::V4, Visibility::All, Visibility::All), + address_side: AddressSide::Source, + addresses: &first_addresses, + }, + AddressSetRow { + key, + scope: Scope::new(IpVersion::V4, Visibility::Literal, Visibility::All), + address_side: AddressSide::Destination, + addresses: &second_addresses, + }, + ]; + + let first = maad_rows(&rows).unwrap(); + let second = maad_rows(&rows).unwrap(); + + assert_eq!(first, second); + assert_eq!(first.len(), 6); + assert_eq!( + first + .chunks_exact(3) + .map(|rows| ( + rows[0].dimensions.src_visibility.as_str(), + rows[0].address_side.as_str(), + rows.iter() + .map(|row| row.structure_kind.as_str()) + .collect::>(), + )) + .collect::>(), + vec![ + ("all", "source", vec!["structure", "spectrum", "dimension"]), + ( + "literal", + "destination", + vec!["structure", "spectrum", "dimension"] + ), + ] + ); + } + #[test] fn rollups_keep_touched_edges_without_extending_the_input_envelope() { let raw = (0..6) diff --git a/tools/netflow-db/src/registry.rs b/tools/netflow-db/src/registry.rs index 9b00100..c5672d6 100644 --- a/tools/netflow-db/src/registry.rs +++ b/tools/netflow-db/src/registry.rs @@ -7,6 +7,7 @@ use std::{ }; use serde::{Deserialize, Serialize}; +use serde_json::Value; use thiserror::Error; #[derive(Debug, Error)] @@ -25,12 +26,14 @@ pub enum RegistryError { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DatasetSource { pub source_id: String, pub members: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Dataset { pub dataset_id: String, #[serde(default)] @@ -51,13 +54,19 @@ pub struct Dataset { pub source_ids: Vec, #[serde(default)] pub sources: Vec, + /// Optional product selection applied automatically by dataset-mode pipeline runs. + #[serde(default)] + pub selection: Value, } impl Dataset { pub fn validate(&mut self, repository_root: &Path) -> Result<(), RegistryError> { self.dataset_id = self.dataset_id.trim().to_owned(); - if self.dataset_id.is_empty() { - return Err(RegistryError::Invalid("dataset_id cannot be empty".into())); + if !is_safe_path_component(&self.dataset_id) { + return Err(RegistryError::Invalid(format!( + "dataset_id {:?} must be exactly one normal path component", + self.dataset_id + ))); } if self.label.trim().is_empty() { self.label = title(&self.dataset_id); @@ -167,10 +176,14 @@ impl DatasetRegistry { } pub fn load_default(repository_root: &Path) -> Result { - let configured = env::var_os("DATASETS_CONFIG_PATH") + Self::load(Self::default_path(repository_root), repository_root) + } + + /// Return the registry path selected by the environment, or the repository default. + pub fn default_path(repository_root: &Path) -> PathBuf { + env::var_os("DATASETS_CONFIG_PATH") .map(PathBuf::from) - .unwrap_or_else(|| repository_root.join("datasets.json")); - Self::load(configured, repository_root) + .unwrap_or_else(|| repository_root.join("datasets.json")) } pub fn get(&self, dataset_id: &str) -> Result<&Dataset, RegistryError> { @@ -318,4 +331,81 @@ mod tests { root.path().join("data/sample_data/netflow.sqlite") ); } + + #[test] + fn registry_rejects_dataset_ids_that_are_not_safe_path_components() { + for dataset_id in [ + "", + ".", + "..", + "../outside", + "/outside", + "nested/id", + r"nested\id", + ] { + let root = tempdir().unwrap(); + let list = root.path().join("datasets.json"); + fs::write( + &list, + serde_json::json!([{ + "dataset_id": dataset_id, + "root_path": "/captures" + }]) + .to_string(), + ) + .unwrap(); + + let error = DatasetRegistry::load(&list, root.path()).unwrap_err(); + assert!( + error.to_string().contains("dataset_id") + && error.to_string().contains("one normal path component"), + "dataset_id {dataset_id:?}: {error}" + ); + } + } + + #[test] + fn registry_accepts_hyphenated_dataset_ids() { + let root = tempdir().unwrap(); + let list = root.path().join("datasets.json"); + fs::write( + &list, + r#"[{"dataset_id":"uoregon-active-0-220","root_path":"/captures"}]"#, + ) + .unwrap(); + + let registry = DatasetRegistry::load(&list, root.path()).unwrap(); + + assert_eq!( + registry.get("uoregon-active-0-220").unwrap().db_path, + root.path().join("data/uoregon-active-0-220/netflow.sqlite") + ); + } + + #[test] + fn registry_rejects_unknown_dataset_and_source_fields() { + for (registry_json, unknown_field) in [ + ( + r#"[{"dataset_id":"sample","root_path":"/captures","selecton":null}]"#, + "selecton", + ), + ( + r#"[{"dataset_id":"sample","root_path":"/captures","sources":[{"source_id":"r1","member":["r1"]}]}]"#, + "member", + ), + ] { + let root = tempdir().unwrap(); + let list = root.path().join("datasets.json"); + fs::write(&list, registry_json).unwrap(); + + let error = DatasetRegistry::load(&list, root.path()).unwrap_err(); + + assert!( + error + .to_string() + .contains(&format!("unknown field `{unknown_field}`")), + "unexpected error: {error}" + ); + } + } } diff --git a/tools/netflow-db/src/storage.rs b/tools/netflow-db/src/storage.rs index d3fc8e9..cc80a1c 100644 --- a/tools/netflow-db/src/storage.rs +++ b/tools/netflow-db/src/storage.rs @@ -1,10 +1,10 @@ //! Concrete SQLite persistence and atomic database publication. use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeSet, fs::{self, File, OpenOptions}, io::{Read, Seek, SeekFrom, Write}, - path::{Component, Path, PathBuf}, + path::{Path, PathBuf}, time::Duration, }; @@ -34,7 +34,7 @@ pub const STATS_TABLE_NAMES: [&str; 6] = [ "address_structure_stats", "bucket_coverage", ]; - +const STATS_GRANULARITIES: [&str; 4] = ["5m", "30m", "1h", "1d"]; #[derive(Debug, Error)] pub enum StorageError { #[error("SQLite operation failed: {0}")] @@ -75,7 +75,7 @@ impl DatabaseOperationLock { database_path: impl AsRef, operation: impl Into, ) -> Result { - let database_path = absolute_path(database_path.as_ref())?; + let database_path = canonical_path(database_path.as_ref())?; let operation = operation.into(); let path = database_operation_lock_path(&database_path)?; if let Some(parent) = path.parent() { @@ -126,17 +126,6 @@ fn validate_lock_file(file: &File, path: &Path) -> Result<(), StorageError> { path.display() ))); } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - - if metadata.nlink() != 1 { - return Err(StorageError::InvalidInput(format!( - "database operation lock must not be hard-linked: {}", - path.display() - ))); - } - } Ok(()) } @@ -147,7 +136,7 @@ impl Drop for DatabaseOperationLock { } pub fn database_operation_lock_path(path: impl AsRef) -> Result { - let database = absolute_path(path.as_ref())?; + let database = canonical_path(path.as_ref())?; let name = database.file_name().ok_or_else(|| { StorageError::InvalidInput(format!( "database path has no file name: {}", @@ -157,6 +146,32 @@ pub fn database_operation_lock_path(path: impl AsRef) -> Result) -> Result { + let path = path.as_ref(); + let expanded = if path.is_absolute() { + path.to_owned() + } else { + std::env::current_dir()?.join(path) + }; + match fs::canonicalize(&expanded) { + Ok(path) => Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = expanded.parent().ok_or_else(|| { + StorageError::InvalidInput(format!( + "path has no parent directory: {}", + expanded.display() + )) + })?; + let file_name = expanded.file_name().ok_or_else(|| { + StorageError::InvalidInput(format!("path has no file name: {}", expanded.display())) + })?; + Ok(canonical_path(parent)?.join(file_name)) + } + Err(error) => Err(error.into()), + } +} + pub fn connect_pipeline_writer(path: impl AsRef) -> Result { connect_pipeline_writer_with_timeout(path, BUSY_TIMEOUT_MS) } @@ -180,7 +195,7 @@ pub fn connect_local_writer(path: impl AsRef) -> Result) -> Result { let connection = Connection::open_with_flags( - absolute_path(path.as_ref())?, + canonical_path(path.as_ref())?, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_millis(BUSY_TIMEOUT_MS))?; @@ -225,48 +240,6 @@ pub fn init_schema(connection: &Connection) -> Result<(), StorageError> { Ok(()) } -fn absolute_path(path: &Path) -> Result { - let expanded = if path.starts_with("~") { - let home = std::env::var_os("HOME").ok_or_else(|| { - StorageError::InvalidInput(format!( - "cannot expand path without a home: {}", - path.display() - )) - })?; - PathBuf::from(home).join(path.strip_prefix("~").expect("prefix checked")) - } else if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir()?.join(path) - }; - let mut normalized = PathBuf::new(); - for component in expanded.components() { - match component { - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - other => normalized.push(other.as_os_str()), - } - } - let mut existing = normalized.as_path(); - let mut suffix = Vec::new(); - while !existing.exists() { - let name = existing.file_name().ok_or_else(|| { - StorageError::InvalidInput(format!("cannot resolve path {}", path.display())) - })?; - suffix.push(name.to_owned()); - existing = existing.parent().ok_or_else(|| { - StorageError::InvalidInput(format!("cannot resolve path {}", path.display())) - })?; - } - let mut resolved = existing.canonicalize()?; - for component in suffix.into_iter().rev() { - resolved.push(component); - } - Ok(resolved) -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum InputKind { Nfcapd, @@ -1542,6 +1515,128 @@ pub fn init_stats_tables(connection: &Connection) -> Result<(), StorageError> { DROP INDEX IF EXISTS idx_port_count_stats_query; ", )?; + init_daily_product_completion_table(connection)?; + Ok(()) +} + +/// Initialize the completion marker for native daily-active products. +pub fn init_daily_product_completion_table(connection: &Connection) -> Result<(), StorageError> { + connection.execute_batch( + " + CREATE TABLE IF NOT EXISTS daily_product_completion ( + source_id TEXT NOT NULL, + day_start INTEGER NOT NULL, + day_end INTEGER NOT NULL CHECK (day_end > day_start), + product_fingerprint TEXT NOT NULL, + run_maad INTEGER NOT NULL CHECK (run_maad IN (0, 1)), + completed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source_id, day_start) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_daily_product_completion_source_range + ON daily_product_completion(source_id, day_start, day_end); + ", + )?; + Ok(()) +} + +/// Return the product identity currently bound to this database, if one exists. +pub fn current_product_fingerprint( + connection: &Connection, +) -> Result, StorageError> { + Ok(connection + .query_row( + "SELECT product_fingerprint FROM pipeline_product WHERE singleton = 1", + [], + |row| row.get(0), + ) + .optional()?) +} + +/// Test whether one source/day has a marker for the current product identity and MAAD setting. +pub fn daily_product_completion_matches( + connection: &Connection, + source_id: &str, + day_start: i64, + day_end: i64, + product_fingerprint: &str, + run_maad: bool, +) -> Result { + if day_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion requires a non-empty day range".into(), + )); + } + Ok(connection.query_row( + "SELECT EXISTS( + SELECT 1 FROM daily_product_completion + WHERE source_id = ?1 AND day_start = ?2 AND day_end = ?3 + AND product_fingerprint = ?4 AND run_maad = ?5 + )", + params![ + source_id, + day_start, + day_end, + product_fingerprint, + i64::from(run_maad) + ], + |row| row.get::<_, i64>(0), + )? != 0) +} + +/// Publish or refresh one source/day marker. Callers must invoke this inside the same transaction +/// that publishes the day's canonical rows, rollups, and evidence/provenance. +pub fn upsert_daily_product_completion( + connection: &Connection, + source_id: &str, + day_start: i64, + day_end: i64, + product_fingerprint: &str, + run_maad: bool, +) -> Result<(), StorageError> { + if day_start >= day_end { + return Err(StorageError::InvalidInput( + "daily product completion requires a non-empty day range".into(), + )); + } + connection.execute( + "INSERT INTO daily_product_completion ( + source_id, day_start, day_end, product_fingerprint, run_maad + ) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(source_id, day_start) DO UPDATE SET + day_end = excluded.day_end, + product_fingerprint = excluded.product_fingerprint, + run_maad = excluded.run_maad, + completed_at = CURRENT_TIMESTAMP", + params![ + source_id, + day_start, + day_end, + product_fingerprint, + i64::from(run_maad), + ], + )?; + Ok(()) +} + +/// Remove completion markers overlapping a deleted source/time range. +pub fn delete_daily_product_completion( + connection: &Connection, + source_ids: &[String], + start: i64, + end: i64, +) -> Result<(), StorageError> { + if start >= end { + return Err(StorageError::InvalidInput( + "daily product completion deletion requires a non-empty range".into(), + )); + } + for source_id in source_ids { + connection.execute( + "DELETE FROM daily_product_completion + WHERE source_id = ?1 AND day_start < ?3 AND day_end > ?2", + params![source_id, start, end], + )?; + } Ok(()) } @@ -2101,6 +2196,51 @@ pub fn delete_stats_bucket_keys( Ok(()) } +/// Remove every persisted product row and input-evidence row for a local time range. +/// +/// Callers use this inside their day transaction when a previously complete capture day must be +/// invalidated. Keeping input state with the stats deletion prevents a later resume from treating +/// stale revisions as proof that the deleted day is still published. +pub(crate) fn delete_stats_time_range( + connection: &Connection, + source_ids: &[String], + start: i64, + end: i64, +) -> Result<(), StorageError> { + if start >= end { + return Err(StorageError::InvalidInput( + "time-range deletion requires a non-empty range".into(), + )); + } + delete_daily_product_completion(connection, source_ids, start, end)?; + for table in STATS_TABLE_NAMES { + let mut statement = connection.prepare_cached(&format!( + "DELETE FROM {table} + WHERE source_id = ?1 AND granularity = ?2 + AND bucket_start >= ?3 AND bucket_start < ?4" + ))?; + for source_id in source_ids { + for granularity in STATS_GRANULARITIES { + statement.execute(params![source_id, granularity, start, end])?; + } + } + } + for source_id in source_ids { + connection.execute( + "DELETE FROM input_evidence + WHERE source_id = ?1 AND bucket_start >= ?2 AND bucket_start < ?3", + params![source_id, start, end], + )?; + connection.execute( + "DELETE FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = ?1 + AND bucket_start >= ?2 AND bucket_start < ?3", + params![source_id, start, end], + )?; + } + Ok(()) +} + #[cfg(test)] impl StatsDimensions { fn example() -> Self { @@ -2371,7 +2511,7 @@ pub fn promote_database( backup_existing_path: Option<&Path>, ) -> Result<(), StorageError> { let (candidate_path, target_path) = resolved_backup_paths(candidate_path, target_path)?; - let backup_existing_path = backup_existing_path.map(absolute_path).transpose()?; + let backup_existing_path = backup_existing_path.map(canonical_path).transpose()?; let mut database_paths = vec![candidate_path.as_path(), target_path.as_path()]; database_paths.extend(backup_existing_path.as_deref()); validate_database_path_separation(&database_paths)?; @@ -2395,8 +2535,8 @@ fn resolved_backup_paths( source_path: impl AsRef, target_path: impl AsRef, ) -> Result<(PathBuf, PathBuf), StorageError> { - let source_path = absolute_path(source_path.as_ref())?; - let target_path = absolute_path(target_path.as_ref())?; + let source_path = canonical_path(source_path.as_ref())?; + let target_path = canonical_path(target_path.as_ref())?; validate_database_path_separation(&[source_path.as_path(), target_path.as_path()])?; if !source_path.is_file() { return Err(StorageError::DatabaseNotFound(source_path)); @@ -2404,29 +2544,38 @@ fn resolved_backup_paths( Ok((source_path, target_path)) } -fn validate_database_path_separation(paths: &[&Path]) -> Result<(), StorageError> { - let mut claimed = BTreeMap::new(); +/// Reject overlapping database files, SQLite sidecars, and operation locks. +pub(crate) fn validate_database_path_separation(paths: &[&Path]) -> Result<(), StorageError> { + let mut claimed = Vec::<(PathBuf, PathBuf)>::new(); for path in paths { for related in database_related_paths(path)? { - if let Some(owner) = claimed.insert(related.clone(), (*path).to_owned()) { + if let Some((owner_related, owner)) = claimed.iter().find(|(owner_related, _)| { + owner_related == &related + || owner_related.starts_with(&related) + || related.starts_with(owner_related) + }) { return Err(StorageError::InvalidInput(format!( - "database paths and their SQLite sidecar/operation-lock paths must be distinct: {} aliases {} through {}", + "database paths and their SQLite sidecar/operation-lock paths must be distinct: {} and {} overlap through {} and {}", owner.display(), path.display(), + owner_related.display(), related.display() ))); } + claimed.push((related.clone(), (*path).to_owned())); } } Ok(()) } -fn database_related_paths(path: &Path) -> Result, StorageError> { - let mut paths = vec![path.to_owned(), database_operation_lock_path(path)?]; - paths.extend(["-journal", "-wal", "-shm"].map(|suffix| sidecar_path(path, suffix))); +pub(crate) fn database_related_paths(path: &Path) -> Result, StorageError> { + // Resolve the database before deriving the names SQLite and the operation lock use beside it. + let database = canonical_path(path)?; + let mut paths = vec![database.clone(), database_operation_lock_path(&database)?]; + paths.extend(["-journal", "-wal", "-shm"].map(|suffix| sidecar_path(&database, suffix))); paths .into_iter() - .map(|related| absolute_path(&related)) + .map(|path| canonical_path(&path)) .collect() } @@ -2436,7 +2585,7 @@ fn acquire_database_operation_locks<'a>( ) -> Result, StorageError> { let paths = paths .into_iter() - .map(absolute_path) + .map(canonical_path) .collect::, _>>()?; paths .into_iter() @@ -2497,8 +2646,8 @@ pub fn atomic_replace_sqlite( source_path: impl AsRef, target_path: impl AsRef, ) -> Result<(), StorageError> { - let source_path = absolute_path(source_path.as_ref())?; - let target_path = absolute_path(target_path.as_ref())?; + let source_path = canonical_path(source_path.as_ref())?; + let target_path = canonical_path(target_path.as_ref())?; validate_database_path_separation(&[source_path.as_path(), target_path.as_path()])?; let mut displaced = Vec::new(); for suffix in ["-journal", "-wal", "-shm"] { @@ -2611,22 +2760,6 @@ mod tests { DatabaseOperationLock::acquire(&path, "backup").unwrap(); } - #[cfg(unix)] - #[test] - fn operation_lock_rejects_a_hard_link_to_the_database_before_truncating() { - let directory = tempdir().unwrap(); - let database = directory.path().join("netflow.sqlite"); - fs::write(&database, b"valuable database bytes").unwrap(); - let lock = database_operation_lock_path(&database).unwrap(); - fs::hard_link(&database, &lock).unwrap(); - let original = fs::read(&database).unwrap(); - - let error = DatabaseOperationLock::acquire(&database, "pipeline build").unwrap_err(); - - assert!(error.to_string().contains("must not be hard-linked")); - assert_eq!(fs::read(database).unwrap(), original); - } - #[test] fn product_and_source_layout_bind_once_to_empty_database() { let connection = Connection::open_in_memory().unwrap(); @@ -2805,6 +2938,276 @@ mod tests { } } + #[test] + fn daily_product_completion_matches_the_product_and_maad_configuration() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + upsert_daily_product_completion(&connection, "r1", 0, 86_400, "product", false).unwrap(); + + assert!( + daily_product_completion_matches(&connection, "r1", 0, 86_400, "product", false,) + .unwrap() + ); + assert!( + !daily_product_completion_matches( + &connection, + "r1", + 0, + 86_400, + "other-product", + false, + ) + .unwrap() + ); + assert!( + !daily_product_completion_matches(&connection, "r1", 0, 86_400, "product", true,) + .unwrap() + ); + + delete_daily_product_completion(&connection, &["r1".into()], 0, 86_400).unwrap(); + assert!( + !daily_product_completion_matches(&connection, "r1", 0, 86_400, "product", false,) + .unwrap() + ); + } + + #[test] + fn time_range_deletion_is_granularity_bounded_and_preserves_other_rows() { + let connection = Connection::open_in_memory().unwrap(); + init_stats_tables(&connection).unwrap(); + init_input_evidence_table(&connection).unwrap(); + init_processed_inputs_table(&connection).unwrap(); + + for table in STATS_TABLE_NAMES { + let mut statement = connection + .prepare(&format!( + "EXPLAIN QUERY PLAN DELETE FROM {table} + WHERE source_id = ?1 AND granularity = ?2 + AND bucket_start >= ?3 AND bucket_start < ?4" + )) + .unwrap(); + let plan = statement + .query_map(params!["r1", "5m", 0_i64, 86_400_i64], |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + for clause in [ + "USING PRIMARY KEY", + "source_id=?", + "granularity=?", + "bucket_start>?", + "bucket_start Result<(), StorageError> { + let locator = format!( + "/captures/{}-{source_id}-{bucket_start}", + input_kind.as_str() + ); + let revision = + InputRevision::create(input_kind.as_str(), &locator, "content", "decoder")?; + upsert_input_bucket( + &connection, + &InputBucket { + input_kind, + input_locator: locator.clone(), + scan_locator: locator.clone(), + source_id: source_id.into(), + bucket_start, + bucket_end: bucket_start + 300, + revision: revision.clone(), + file_snapshot: None, + }, + false, + )?; + mark_input_bucket_status( + &connection, + input_kind, + &locator, + source_id, + bucket_start, + InputStatus::Processed, + &revision, + None, + ) + }; + + for &(source_id, bucket_start) in + &[("r1", 0_i64), ("r1", 86_400), ("r2", 0), ("r2", 86_400)] + { + insert_processed(source_id, bucket_start, InputKind::Nfcapd).unwrap(); + } + insert_processed("r1", 0, InputKind::Csv).unwrap(); + + upsert_daily_product_completion(&connection, "r1", 0, 86_400, "product", false).unwrap(); + upsert_daily_product_completion(&connection, "r2", 0, 86_400, "product", false).unwrap(); + + delete_stats_time_range(&connection, &["r1".into()], 0, 86_400).unwrap(); + + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'r1'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "deleting a day must remove its completion marker" + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM daily_product_completion + WHERE source_id = 'r2'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "deleting one source/day must preserve another source marker" + ); + + for table in STATS_TABLE_NAMES { + let count = connection + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + assert_eq!(count, 12, "{table}"); + assert_eq!( + connection + .query_row( + &format!( + "SELECT COUNT(*) FROM {table} + WHERE source_id = 'r1' AND bucket_start = 0" + ), + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0, + "the requested source and range should be deleted from {table}" + ); + } + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM input_evidence", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 3 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM input_evidence + WHERE source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM processed_inputs", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 4 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM processed_inputs + WHERE input_kind = 'nfcapd' AND source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!( + connection + .query_row( + "SELECT COUNT(*) FROM processed_inputs + WHERE input_kind = 'csv' AND source_id = 'r1' AND bucket_start = 0", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + } + #[test] fn stats_timeseries_indexes_apply_equality_filters_before_bucket_range() { let connection = Connection::open_in_memory().unwrap(); @@ -3195,65 +3598,6 @@ mod tests { } } - #[test] - fn maintenance_rejects_database_sidecar_and_lock_aliases_before_mutation() { - let directory = tempdir().unwrap(); - let target = directory.path().join("target.sqlite"); - for source in [ - sidecar_path(&target, "-wal"), - sidecar_path(&target, "-shm"), - sidecar_path(&target, "-journal"), - database_operation_lock_path(&target).unwrap(), - ] { - drop(Connection::open(&source).unwrap()); - let original = fs::read(&source).unwrap(); - - let error = backup_database(&source, &target).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(&source).unwrap(), original); - fs::remove_file(source).unwrap(); - } - } - - #[cfg(unix)] - #[test] - fn maintenance_resolves_symlinks_before_checking_related_paths() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let target = directory.path().join("target.sqlite"); - let lock = database_operation_lock_path(&target).unwrap(); - drop(Connection::open(&lock).unwrap()); - let candidate = directory.path().join("candidate.sqlite"); - symlink(&lock, &candidate).unwrap(); - let original = fs::read(&lock).unwrap(); - - let error = backup_database(&candidate, &target).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(lock).unwrap(), original); - } - - #[cfg(unix)] - #[test] - fn maintenance_rejects_a_derived_lock_symlink_to_the_source() { - use std::os::unix::fs::symlink; - - let directory = tempdir().unwrap(); - let source = directory.path().join("source.sqlite"); - let target = directory.path().join("target.sqlite"); - drop(Connection::open(&source).unwrap()); - let lock = database_operation_lock_path(&target).unwrap(); - symlink(&source, &lock).unwrap(); - let original = fs::read(&source).unwrap(); - - let error = backup_database(&source, &target).unwrap_err(); - - assert!(error.to_string().contains("must be distinct")); - assert_eq!(fs::read(source).unwrap(), original); - } - #[test] fn failed_transaction_rolls_back_all_persistence_changes() { let mut connection = Connection::open_in_memory().unwrap(); diff --git a/tools/netflow-db/tests/pipeline_cli_help.rs b/tools/netflow-db/tests/pipeline_cli_help.rs new file mode 100644 index 0000000..9a68b38 --- /dev/null +++ b/tools/netflow-db/tests/pipeline_cli_help.rs @@ -0,0 +1,211 @@ +use std::{fs, process::Command}; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +use rusqlite::Connection; +use tempfile::tempdir; + +#[test] +fn pipeline_repeated_dataset_uses_isolated_registry_and_outputs() { + let temporary = tempdir().unwrap(); + let capture_root = temporary.path().join("captures"); + fs::create_dir_all(capture_root.join("shared")).unwrap(); + let registry_path = temporary.path().join("registry.json"); + let first_database = temporary.path().join("first.sqlite"); + let second_database = temporary.path().join("second.sqlite"); + let nfdump = temporary.path().join("nfdump"); + let empty_stream = temporary.path().join("empty.stream"); + fs::write( + &empty_stream, + [65_u8, 84, 76, 78, 70, 76, 79, 87, 1, 0, 72, 0, 0, 0, 0, 0], + ) + .unwrap(); + fs::write( + &nfdump, + format!("#!/bin/sh\ncat '{}'\n", empty_stream.display()), + ) + .unwrap(); + #[cfg(unix)] + fs::set_permissions(&nfdump, fs::Permissions::from_mode(0o755)).unwrap(); + let registry = serde_json::json!({ + "datasets": [ + { + "dataset_id": "first", + "root_path": capture_root, + "db_path": first_database, + "source_ids": ["shared"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "10.0.0.0/16" + } + }, + { + "dataset_id": "second", + "root_path": capture_root, + "db_path": second_database, + "source_ids": ["shared"], + "selection": { + "kind": "daily_active_sources", + "ip_prefix": "10.0.0.0/16" + } + } + ] + }); + fs::write(®istry_path, serde_json::to_vec(®istry).unwrap()).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--dataset", + "first", + "--dataset", + "second", + "--start-date", + "2025-01-01", + "--end-date", + "2025-01-02", + "--datasets", + registry_path.to_str().unwrap(), + "--no-maad", + "--nfdump", + nfdump.to_str().unwrap(), + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + for database in [&first_database, &second_database] { + assert!( + database.is_file(), + "missing coordinated output {database:?}" + ); + let connection = Connection::open(database).unwrap(); + let selection: String = connection + .query_row( + "SELECT selection_json FROM pipeline_product WHERE singleton = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(selection.contains("daily_active_sources"), "{selection}"); + } +} + +#[test] +fn csv_pipeline_does_not_require_nfdump_from_path() { + let temporary = tempdir().unwrap(); + let empty_path = temporary.path().join("empty-path"); + fs::create_dir(&empty_path).unwrap(); + let csv = temporary.path().join("flows.csv"); + let mapping = temporary.path().join("mapping.json"); + let database = temporary.path().join("csv.sqlite"); + fs::write(&csv, "received,src,dst\n0,192.0.2.1,198.51.100.1\n").unwrap(); + fs::write( + &mapping, + serde_json::to_vec(&serde_json::json!({ + "timestamp_format": "unix", + "timestamp_timezone": "UTC", + "columns": { + "time_received": "received", + "src_ip": "src", + "dst_ip": "dst" + }, + "source_id": {"value": "edge"} + })) + .unwrap(), + ) + .unwrap(); + let config = temporary.path().join("csv-pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "inputs": [{ + "input_kind": "csv", + "path": csv, + "mapping_path": mapping + }] + })) + .unwrap(), + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("PATH", empty_path) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("Published five-minute buckets: 1\n"), + "stdout={stdout}" + ); + assert!(database.is_file()); +} + +#[test] +fn native_pipeline_requires_nfdump_before_output_setup() { + let temporary = tempdir().unwrap(); + let empty_path = temporary.path().join("empty-path"); + fs::create_dir(&empty_path).unwrap(); + let capture_root = temporary.path().join("captures"); + fs::create_dir_all(capture_root.join("edge")).unwrap(); + let database = temporary.path().join("native.sqlite"); + let config = temporary.path().join("native-pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "inputs": [{ + "input_kind": "nfcapd_tree", + "root_path": capture_root, + "source_ids": ["edge"], + "start_date": "2025-01-01", + "end_date": "2025-01-02" + }] + })) + .unwrap(), + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("PATH", empty_path) + .output() + .unwrap(); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("cannot resolve bare nfdump executable"), + "stderr={stderr}" + ); + assert!(!database.exists()); +}