From 0516065deff76b389dc769b8051553345b38d858 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Thu, 13 Aug 2026 15:09:40 -0700 Subject: [PATCH 01/34] Unify audit with capture denial routing Route wxc-exec --audit through captureDenials so native PSEC/V2 remains preferred and guarded WPR provides compatible analysis and ETL retention. Reuse canonical denials for adjusted policy generation and support retainEtl consistently across both capture providers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- README.md | 4 +- docs/learning-mode/capabilities.md | 18 +- src/Cargo.lock | 1 + .../common/src/appcontainer_runner.rs | 87 ++- .../common/src/base_container_runner.rs | 72 +- .../common/src/guarded_capture.rs | 39 +- src/core/mxc_engine/src/guarded_capture.rs | 25 +- src/core/mxc_engine/src/lib.rs | 2 + src/core/mxc_engine/src/run.rs | 20 +- src/core/wxc/Cargo.toml | 1 + src/core/wxc/src/audit.rs | 640 +++++++++--------- src/core/wxc/src/main.rs | 204 ++---- src/host/plm/src/analysis.rs | 10 +- src/host/plm/src/elevated.rs | 161 ++++- src/host/plm/src/stop.rs | 169 ++++- 15 files changed, 871 insertions(+), 582 deletions(-) diff --git a/README.md b/README.md index 02465dd58..19eae5a35 100644 --- a/README.md +++ b/README.md @@ -219,13 +219,13 @@ See [docs/diagnostics.md](docs/diagnostics.md) for full diagnostics reference. ### Audit Mode (Permissive Learning Mode) -`--audit` runs a Windows **ProcessContainer** policy in permissive mode — denied operations are logged but allowed to proceed — and starts a Permissive Learning Mode (PLM) ETW trace alongside the workload. `wxc-exec.exe` and the public `plm.exe` remain unelevated; PLM uses UAC only for fixed WPR start/stop controls, no service is added, and ETL/status handoff stays on a unique authenticated local named pipe while every caller-selected log/config/trace destination and ETL analysis stay under the caller token. It is rejected for Windows Sandbox, WSLC, IsolationSession, and every other containment backend because those paths do not honor the AppContainer learning-mode capability. It is the developer inner-loop flow for discovering the capabilities and paths a workload needs. See [src/host/plm/readme.md](src/host/plm/readme.md) for the full PLM tool reference, including standalone `plm.exe` invocation (e.g. re-processing an existing `.etl` with `plm stop --trace-file …`). +`--audit` is a compatibility wrapper over `processContainer.captureDenials` in allow mode with ETL retention forced on. It injects `permissiveLearningMode`, so denied operations are recorded but allowed to proceed. On hosts with the complete PSEC/V2 Learning Mode API set, the selected ProcessContainer runner uses native capture without launching PLM or prompting for elevation. Older or policy-incompatible tiers use the guarded-WPR fallback: `wxc-exec.exe` remains unelevated and starts a session-scoped UAC-elevated PLM guardian only for the privileged WPR lifecycle, communicating over an authenticated local named pipe. It is rejected for Windows Sandbox, WSLC, IsolationSession, and every other containment backend. ```bash wxc-exec.exe --audit policy.json ``` -Use `--audit-verbose` to also stream PLM lifecycle diagnostics to stderr. Successful runs always print the artifact directory containing `trace.etl`, `denials.json`, and any adjusted configuration. +Successful non-dry-run audits require capture metadata, canonical denials JSON, and a retained ETL. The CLI relocates the backend-selected paths to `denials.json` and `trace.etl` in the per-user audit directory, then generates a source-config snapshot and `Adjusted_*.json` from the canonical JSON without decoding the ETL again. Base64-only input keeps JSON and ETL but has no source config to snapshot or adjust. Truncated analysis keeps JSON, ETL, and the source snapshot but skips adjusted-config generation. Use `--audit-verbose` to print learned-policy details. > **Warning:** `--audit` injects `permissiveLearningMode` — AppContainer restrictions are **not** enforced for the duration of the run. Use only for policy authoring. It cannot be combined with `processContainer.captureDenials`; use `captureDenials.mode: "allow"` for permissive application-driven capture. `learningModeLogging` and `permissiveLearningMode` are reserved internal capability names and are rejected in `processContainer.capabilities`. See [docs/learning-mode/capabilities.md](docs/learning-mode/capabilities.md) for the three learning-mode flows. diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index be6e5c665..0f70b11cb 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -99,9 +99,15 @@ stays enforced: their process needs. `--audit` is rejected for every other Windows backend. It is also mutually exclusive with `captureDenials`; use `captureDenials.mode: "allow"` for permissive application-driven capture. - It triggers UAC, injects `permissiveLearningMode`, and drives a WPR/ETW - permissive-learning-mode trace for the run. This is typically a static - config the developer iterates on locally. + It is a compatibility wrapper over `captureDenials.mode: "allow"` with ETL + retention forced on, and injects `permissiveLearningMode`. The selected + ProcessContainer capture backend owns the trace lifecycle: complete PSEC/V2 + hosts use native capture without PLM or UAC, while legacy or incompatible + tiers use the session-scoped guarded-WPR fallback and elevate only its + fixed-operation guardian. The CLI consumes the returned JSON and ETL paths, + relocates them to `denials.json` and `trace.etl`, and generates the source + snapshot and `Adjusted_*.json` from canonical denials without decoding ETL + again. Truncated analysis skips the adjusted config. ``` wxc-exec --audit --config @@ -235,9 +241,9 @@ C# SDK exposes it through `RunResult.OutputMetadata` and By default, the intermediate ETW `.etl` trace is an internal, runner-managed file in a protected per-run temporary directory that MXC deletes after analysis. Set `captureDenials.retainEtl` to `true` to preserve the sealed trace -for diagnostics after a terminal wait when native PSEC/V2 capture is selected. -Guarded-WPR fallback rejects `retainEtl: true` with `backend_unavailable` -rather than returning its raw host-wide trace. Retention-enabled captures begin under +for diagnostics after a terminal wait. Both native PSEC/V2 capture and the +guarded-WPR fallback honor retention; guarded WPR transfers the sealed trace +back to the unelevated caller after process-scoped analysis. Retention-enabled captures begin under `%LOCALAPPDATA%\Microsoft\MXC\capture-denials\working` and move to a protected per-run directory under `capture-denials\retained` only after sealing succeeds. Abandoning or disposing a process without a terminal wait deletes the internal diff --git a/src/Cargo.lock b/src/Cargo.lock index 349838cc0..f0c9c9b62 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -3153,6 +3153,7 @@ dependencies = [ "appcontainer_common", "clap", "hyperlight_common", + "learning_mode_core", "mxc_build_common", "mxc_engine", "nanvix_binaries", diff --git a/src/backends/appcontainer/common/src/appcontainer_runner.rs b/src/backends/appcontainer/common/src/appcontainer_runner.rs index d2c435674..f447f23dd 100644 --- a/src/backends/appcontainer/common/src/appcontainer_runner.rs +++ b/src/backends/appcontainer/common/src/appcontainer_runner.rs @@ -1116,9 +1116,10 @@ impl AppContainerScriptRunner { // ordering means a guardian-start failure can terminate the // still-suspended child without ever leaving an active host-wide // WPR trace running. - let (capture_session, capture_output_path): ( + let (capture_session, capture_output_path, capture_etl_path): ( Option>, Option, + Option, ) = match ( self.guarded_capture_factory.as_ref(), request.policy.capture_denials.as_ref(), @@ -1159,7 +1160,10 @@ impl AppContainerScriptRunner { "guarded WPR captureDenials session started (output: {})", output_path.display() )); - (Some(session), Some(output_path)) + let etl_path = (capture_config.retain_etl + && factory.allows_trace_transfer()) + .then(|| output_path.with_extension("etl")); + (Some(session), Some(output_path), etl_path) } Err(e) => { // No active trace exists yet -- terminate the @@ -1179,7 +1183,7 @@ impl AppContainerScriptRunner { } } } - _ => (None, None), + _ => (None, None, None), }; let (stdout_read, stderr_read) = match capture_reads { @@ -1196,6 +1200,7 @@ impl AppContainerScriptRunner { pid: pi.dwProcessId, capture_session, capture_output_path, + capture_etl_path, stdin_write: captured_stdin_write, stdout_read, stderr_read, @@ -1250,6 +1255,8 @@ struct SpawnedChild { /// Resolved JSON denials deliverable path. `Some` iff `capture_session` /// is `Some`. capture_output_path: Option, + /// Caller-visible ETL destination when retention is requested. + capture_etl_path: Option, /// Parent's stdin write-end (Some only when spawned for streaming). stdin_write: Option, /// Parent's stdout/stderr read-ends (Some only in streaming mode). @@ -1500,6 +1507,10 @@ impl SandboxBackend for AppContainerScriptRunner { .capture_denials .as_ref() .is_some_and(|config| config.retain_etl) + && !self + .guarded_capture_factory + .as_ref() + .is_some_and(|factory| factory.allows_trace_transfer()) { return Err(ScriptResponse { failure_phase: FailurePhase::BackendUnavailable, @@ -1602,6 +1613,8 @@ struct AppContainerSandboxProcess { /// Resolved JSON denials deliverable path. `Some` iff `capture_session` /// is `Some`. capture_output_path: Option, + /// Caller-visible ETL destination when retention is requested. + capture_etl_path: Option, /// Exit code of the child, recorded by `wait` before teardown so the /// denials summary can carry it. `None` on the `Drop`/early-exit path. last_exit_code: Option, @@ -1658,12 +1671,13 @@ impl AppContainerSandboxProcess { teardown_result: None, capture_session: child.capture_session.take(), capture_output_path: child.capture_output_path.take(), + capture_etl_path: child.capture_etl_path.take(), last_exit_code: None, output_metadata: None, } } - fn run_teardown(&mut self) -> std::io::Result<()> { + fn run_teardown(&mut self, allow_trace_transfer: bool) -> std::io::Result<()> { if let Some(result) = &self.teardown_result { return result.clone().map_err(std::io::Error::other); } @@ -1680,18 +1694,30 @@ impl AppContainerSandboxProcess { // Stop and analyze the guarded WPR capture now that the child has // exited and been reaped (both `wait` and `Drop` kill + reap before - // calling this). `stop_analyzed` returns only the bounded, - // process-scoped `AnalysisResult` -- the raw host-wide ETL never - // crosses back into this process. Write it through the same shared - // `capture_output` plumbing the native BaseContainer path uses, so - // the two paths emit byte-identical JSON. + // calling this). All callers receive the bounded, process-scoped + // `AnalysisResult`; requests with `retainEtl` also receive the sealed + // trace. Write the analysis through the same shared `capture_output` + // plumbing the native BaseContainer path uses. let result: std::io::Result<()> = if let Some(mut session) = self.capture_session.take() { let output_path = self.capture_output_path.take(); + let etl_path = self + .capture_etl_path + .take() + .filter(|_| allow_trace_transfer); let exit_code = self.last_exit_code.unwrap_or(-1); - let capture_result = match session.stop_analyzed() { + let capture_result = match etl_path.as_deref() { + Some(etl_path) => session.stop_analyzed_with_trace(etl_path), + None => session.stop_analyzed(), + }; + let capture_result = match capture_result { Ok(analysis) => match output_path { Some(output_path) => { capture_output::write_denials_document(analysis, exit_code, &output_path) + .map(|mut metadata| { + metadata.etl_path = + etl_path.map(|path| path.to_string_lossy().into_owned()); + metadata + }) } None => Err(std::io::Error::other( "captureDenials internal output path was not initialized", @@ -1850,7 +1876,7 @@ impl SandboxProcess for AppContainerSandboxProcess { // Record the child's exit code so `run_teardown` can stamp it into the // denials summary. On a timeout / wait failure there is no exit code. self.last_exit_code = result.as_ref().ok().copied(); - let teardown_result = self.run_teardown(); + let teardown_result = self.run_teardown(true); capture_output::combine_process_and_teardown_results(result, teardown_result) } } @@ -1870,7 +1896,7 @@ impl Drop for AppContainerSandboxProcess { unsafe { let _ = WaitForSingleObject(self.process.get(), u32::MAX); } - if let Err(error) = self.run_teardown() { + if let Err(error) = self.run_teardown(false) { capture_output::write_stderr_line_best_effort(format_args!( "captureDenials teardown failed during drop: {error}" )); @@ -2175,6 +2201,15 @@ mod tests { fn stop_analyzed(&mut self) -> Result { Ok(AnalysisResult::complete(Vec::new())) } + + fn stop_analyzed_with_trace( + &mut self, + trace_destination: &std::path::Path, + ) -> Result { + std::fs::write(trace_destination, b"fake etl") + .map_err(|error| error.to_string())?; + Ok(AnalysisResult::complete(Vec::new())) + } } Ok(Box::new(FakeSession)) } @@ -2425,7 +2460,7 @@ mod tests { let error = runner .validate(&request) - .expect_err("guarded capture must not expose its host-wide ETL"); + .expect_err("the configured guarded provider cannot transfer ETL"); assert_eq!(error.failure_phase, FailurePhase::BackendUnavailable); assert_eq!( @@ -2433,4 +2468,30 @@ mod tests { crate::guarded_capture::RETAIN_ETL_UNSUPPORTED_MSG ); } + + #[test] + fn validate_runner_allows_guarded_etl_when_provider_supports_transfer() { + struct RetainingGuardedCaptureFactory; + impl GuardedCaptureFactory for RetainingGuardedCaptureFactory { + fn allows_trace_transfer(&self) -> bool { + true + } + + fn start(&self, owner_pid: u32) -> Result, String> { + FakeGuardedCaptureFactory.start(owner_pid) + } + } + + let runner = AppContainerScriptRunner::new() + .with_guarded_capture_factory(Arc::new(RetainingGuardedCaptureFactory)); + let mut request = ExecutionRequest::default(); + request.policy.capture_denials = Some(wxc_common::models::CaptureDenialsConfig { + retain_etl: true, + ..Default::default() + }); + + runner + .validate(&request) + .expect("a transfer-capable guarded provider should permit ETL retention"); + } } diff --git a/src/backends/appcontainer/common/src/base_container_runner.rs b/src/backends/appcontainer/common/src/base_container_runner.rs index 67f9f4d69..6afb6b6a0 100644 --- a/src/backends/appcontainer/common/src/base_container_runner.rs +++ b/src/backends/appcontainer/common/src/base_container_runner.rs @@ -2188,6 +2188,20 @@ impl BaseContainerRunner { } else { None }; + let guarded_capture_etl_path = (guarded_capture_session.is_some() + && capture_denials + .as_ref() + .is_some_and(|config| config.retain_etl) + && self + .guarded_capture_factory + .as_ref() + .is_some_and(|factory| factory.allows_trace_transfer())) + .then(|| { + capture_output_path + .as_ref() + .map(|path| path.with_extension("etl")) + }) + .flatten(); // The child was created suspended; now that it is in the job object (so // every descendant it spawns is captured), resume its main thread. @@ -2251,6 +2265,7 @@ impl BaseContainerRunner { proxy_coordinator: std::mem::take(&mut self.proxy_coordinator), capture_session, guarded_capture_session, + guarded_capture_etl_path, security_environment, managed_capture: managed_capture.take(), capture_output_path, @@ -2290,6 +2305,8 @@ struct BaseChild { /// Live guarded WPR session used when the legacy SBOX tier supplies /// containment and native PSEC/V2 capture is unavailable. guarded_capture_session: Option>, + /// Caller-visible guarded ETL destination when retention is requested. + guarded_capture_etl_path: Option, /// Non-capture PSEC environment retained until the child exits so policy /// enforcement outlives the process tree. security_environment: Option, @@ -2322,6 +2339,10 @@ impl SandboxBackend for BaseContainerRunner { .capture_denials .as_ref() .is_some_and(|config| config.retain_etl) + && !self + .guarded_capture_factory + .as_ref() + .is_some_and(|factory| factory.allows_trace_transfer()) { return Err(ScriptResponse { failure_phase: FailurePhase::BackendUnavailable, @@ -2466,6 +2487,8 @@ struct BaseContainerSandboxProcess { /// in `run_teardown` once the child has exited and been reaped. capture_session: Option>, guarded_capture_session: Option>, + /// Caller-visible guarded ETL destination when retention is requested. + guarded_capture_etl_path: Option, /// Non-capture PSEC environment, closed after the child exits and is reaped. security_environment: Option, /// Protected per-run ETL path and its cleanup guard. @@ -2515,6 +2538,7 @@ impl BaseContainerSandboxProcess { teardown_result: None, capture_session: child.capture_session.take(), guarded_capture_session: child.guarded_capture_session.take(), + guarded_capture_etl_path: child.guarded_capture_etl_path.take(), security_environment: child.security_environment.take(), managed_capture: child.managed_capture.take(), capture_output_path: child.capture_output_path.take(), @@ -2672,9 +2696,16 @@ impl BaseContainerSandboxProcess { let guarded_capture_result = if let Some(mut session) = self.guarded_capture_session.take() { let output_path = self.capture_output_path.take(); + let etl_path = self + .guarded_capture_etl_path + .take() + .filter(|_| allow_retention); let exit_code = self.last_exit_code.unwrap_or(-1); - let result = session - .stop_analyzed() + let stop_result = match etl_path.as_deref() { + Some(etl_path) => session.stop_analyzed_with_trace(etl_path), + None => session.stop_analyzed(), + }; + let result = stop_result .map_err(|error| { std::io::Error::other(format!( "captureDenials failed to stop and analyze guarded WPR: {error}" @@ -2684,7 +2715,11 @@ impl BaseContainerSandboxProcess { let output_path = output_path.ok_or_else(|| { std::io::Error::other("captureDenials output path was not initialized") })?; - write_denials_document(analysis, exit_code, &output_path) + write_denials_document(analysis, exit_code, &output_path).map(|mut metadata| { + metadata.etl_path = + etl_path.map(|path| path.to_string_lossy().into_owned()); + metadata + }) }); if let Ok(metadata) = &result { self.output_metadata = Some(SandboxOutputMetadata { @@ -4593,7 +4628,7 @@ mod tests { let error = runner .validate(&request) - .expect_err("guarded capture must not expose its host-wide ETL"); + .expect_err("the configured guarded provider cannot transfer ETL"); assert_eq!(error.failure_phase, FailurePhase::BackendUnavailable); assert_eq!( @@ -4601,4 +4636,33 @@ mod tests { crate::guarded_capture::RETAIN_ETL_UNSUPPORTED_MSG ); } + + #[test] + fn validate_runner_allows_guarded_etl_when_provider_supports_transfer() { + struct NeverStartedFactory; + impl GuardedCaptureFactory for NeverStartedFactory { + fn allows_trace_transfer(&self) -> bool { + true + } + + fn start(&self, _owner_pid: u32) -> Result, String> { + Err("not started during validation".to_string()) + } + } + + let runner = BaseContainerRunner { + psec_usable_override: Some(false), + guarded_capture_factory: Some(Arc::new(NeverStartedFactory)), + ..Default::default() + }; + let mut request = ExecutionRequest::default(); + request.policy.capture_denials = Some(wxc_common::models::CaptureDenialsConfig { + retain_etl: true, + ..Default::default() + }); + + runner + .validate(&request) + .expect("a transfer-capable guarded provider should permit ETL retention"); + } } diff --git a/src/backends/appcontainer/common/src/guarded_capture.rs b/src/backends/appcontainer/common/src/guarded_capture.rs index fd8d0a905..8b17f681d 100644 --- a/src/backends/appcontainer/common/src/guarded_capture.rs +++ b/src/backends/appcontainer/common/src/guarded_capture.rs @@ -25,19 +25,19 @@ use learning_mode_core::AnalysisResult; -/// Guarded WPR returns only bounded process-scoped analysis; its raw host-wide -/// ETL cannot be exposed through caller-visible output. +/// Error returned when an injected guarded-capture implementation cannot +/// transfer retained ETL requested by the caller. pub const RETAIN_ETL_UNSUPPORTED_MSG: &str = - "processContainer.captureDenials.retainEtl requires native PSEC/V2 capture; \ - guarded-WPR fallback cannot return the raw host-wide ETL"; + "processContainer.captureDenials.retainEtl is not supported by the configured \ + guarded-WPR capture provider"; /// A live guarded WPR capture session scoped to one sandboxed process tree. /// /// Implementations own the elevated PLM child connection. [`stop_analyzed`] /// asks the guardian to stop the host-wide WPR trace and decode it, returning -/// only the bounded, process-scoped [`AnalysisResult`] — the raw ETL never -/// crosses back into this process, satisfying the "raw host-wide ETL must -/// never cross into SDK output" requirement. +/// the bounded, process-scoped [`AnalysisResult`]. When the caller explicitly +/// requests `retainEtl`, implementations that support trace transfer may also +/// return the sealed ETL through [`stop_analyzed_with_trace`]. /// /// [`stop_analyzed`]: GuardedCaptureSession::stop_analyzed pub trait GuardedCaptureSession: Send { @@ -70,6 +70,13 @@ pub trait GuardedCaptureSession: Send { /// the stop/analyze round trip fails, or the guardian reports a decode /// error. fn stop_analyzed(&mut self) -> Result; + + /// Stops and analyzes the guarded capture, and transfers the sealed ETL to + /// `trace_destination` when the caller explicitly requests ETL retention. + fn stop_analyzed_with_trace( + &mut self, + trace_destination: &std::path::Path, + ) -> Result; } /// Starts a [`GuardedCaptureSession`] for the calling (unelevated) process. @@ -77,6 +84,12 @@ pub trait GuardedCaptureSession: Send { /// Implementations are constructed by a higher layer (`mxc_engine`) that can /// depend on `plm`; `appcontainer_common` only ever sees the trait object. pub trait GuardedCaptureFactory: Send + Sync { + /// Whether this factory can transfer the sealed ETL when `retainEtl` is + /// requested. Implementations that only support analysis keep the default. + fn allows_trace_transfer(&self) -> bool { + false + } + /// Starts a new guarded WPR capture session. /// /// `owner_pid` is the calling (unelevated) process's own OS process id — @@ -132,11 +145,23 @@ mod tests { fn stop_analyzed(&mut self) -> Result { Ok(self.analysis.clone()) } + + fn stop_analyzed_with_trace( + &mut self, + trace_destination: &std::path::Path, + ) -> Result { + std::fs::write(trace_destination, b"fake etl").map_err(|error| error.to_string())?; + Ok(self.analysis.clone()) + } } struct FakeFactory; impl GuardedCaptureFactory for FakeFactory { + fn allows_trace_transfer(&self) -> bool { + true + } + fn start(&self, owner_pid: u32) -> Result, String> { if owner_pid == 0 { return Err("owner pid must be non-zero".to_string()); diff --git a/src/core/mxc_engine/src/guarded_capture.rs b/src/core/mxc_engine/src/guarded_capture.rs index 2915425fd..11881c436 100644 --- a/src/core/mxc_engine/src/guarded_capture.rs +++ b/src/core/mxc_engine/src/guarded_capture.rs @@ -181,6 +181,15 @@ impl GuardedCaptureSession for PlmGuardedCaptureSession { .stop_analyzed() .map_err(|e| format!("guarded WPR stop/analyze failed: {e:#}")) } + + fn stop_analyzed_with_trace( + &mut self, + trace_destination: &std::path::Path, + ) -> Result { + self.session + .stop_analyzed_with_trace(trace_destination) + .map_err(|e| format!("guarded WPR stop/analyze/trace transfer failed: {e:#}")) + } } /// [`GuardedCaptureFactory`] that starts a guarded WPR capture session via the @@ -188,6 +197,10 @@ impl GuardedCaptureSession for PlmGuardedCaptureSession { pub struct PlmGuardedCaptureFactory; impl GuardedCaptureFactory for PlmGuardedCaptureFactory { + fn allows_trace_transfer(&self) -> bool { + true + } + fn start(&self, owner_pid: u32) -> Result, String> { let plm_path = plm_exe_path()?; start_with_plm_path(&plm_path, owner_pid) @@ -231,7 +244,7 @@ fn start_with_plm_path( /// than constructing a `PlmGuardedCaptureFactory` unconditionally at every call /// site) keeps `run.rs` / `dispatch.rs` from wiring a factory the request never /// needed. -pub fn factory_for_request( +pub(crate) fn factory_for_request( request: &wxc_common::models::ExecutionRequest, ) -> Option> { if request.policy.capture_denials.is_some() { @@ -273,6 +286,16 @@ mod tests { assert!(factory_for_request(&request).is_some()); } + #[test] + fn plm_factory_supports_requested_trace_transfer() { + let mut request = wxc_common::models::ExecutionRequest::default(); + request.policy.capture_denials = Some(Default::default()); + + assert!(factory_for_request(&request) + .unwrap() + .allows_trace_transfer()); + } + #[test] fn resolver_locates_the_current_native_module() { let module = module_containing_plm_resolver().expect("test module path should resolve"); diff --git a/src/core/mxc_engine/src/lib.rs b/src/core/mxc_engine/src/lib.rs index e5fe40fb0..401d8a347 100644 --- a/src/core/mxc_engine/src/lib.rs +++ b/src/core/mxc_engine/src/lib.rs @@ -50,6 +50,8 @@ pub use policy::{ WslcSection, }; pub use probe::{available_backends, AvailableBackend, BackendCapability}; +#[cfg(target_os = "windows")] +pub use run::resolve_runner_for_audit; #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] pub use run::{resolve_runner, run, ResolvedRunner}; pub use state_aware::{exec_state_aware_json, run_state_aware, run_state_aware_json}; diff --git a/src/core/mxc_engine/src/run.rs b/src/core/mxc_engine/src/run.rs index 51617e9aa..aa958ebe5 100644 --- a/src/core/mxc_engine/src/run.rs +++ b/src/core/mxc_engine/src/run.rs @@ -85,9 +85,23 @@ pub fn resolve_runner( request: &ExecutionRequest, logger: &mut Logger, ) -> Result { + #[cfg(target_os = "windows")] + { + resolve_runner_inner_windows(request, logger).map_err(Error::from) + } + #[cfg(not(target_os = "windows"))] resolve_runner_inner(request, logger).map_err(Error::from) } +/// Resolve a runner for the `wxc-exec --audit` compatibility workflow. +#[cfg(target_os = "windows")] +pub fn resolve_runner_for_audit( + request: &ExecutionRequest, + logger: &mut Logger, +) -> Result { + resolve_runner_inner_windows(request, logger).map_err(Error::from) +} + /// Resolve `request`'s backend and run it to completion. /// /// Convenience over [`resolve_runner`] for callers without external guard / @@ -107,7 +121,7 @@ pub fn run(request: &ExecutionRequest, logger: &mut Logger) -> Result Result { @@ -403,7 +417,7 @@ mod tests { for config in [None, Some(WindowsSandboxConfig::default())] { let request = windows_sandbox_request(config); let mut logger = Logger::new(Mode::Buffer); - resolve_runner_inner(&request, &mut logger).unwrap(); + resolve_runner_inner_windows(&request, &mut logger).unwrap(); assert!(logger.get_buffer().is_empty()); } } @@ -417,7 +431,7 @@ mod tests { let request = windows_sandbox_request(Some(config)); let mut logger = Logger::new(Mode::Buffer); - resolve_runner_inner(&request, &mut logger).unwrap(); + resolve_runner_inner_windows(&request, &mut logger).unwrap(); let warning = logger.get_buffer(); assert!(warning.contains("idleTimeoutMs")); diff --git a/src/core/wxc/Cargo.toml b/src/core/wxc/Cargo.toml index 2e3758809..346fb399f 100644 --- a/src/core/wxc/Cargo.toml +++ b/src/core/wxc/Cargo.toml @@ -26,6 +26,7 @@ wslc_common = { workspace = true, optional = true } # Shared PLM dep: coordination, guarded lifecycle, and restricted WPR control. # wxc-exec never bypasses PLM's public singleton. plm = { path = "../../host/plm" } +learning_mode_core.workspace = true tempfile.workspace = true [build-dependencies] diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index a85ecbe17..6d7f0b68d 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -1,378 +1,374 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Graceful-exit PLM audit-trace lifecycle for `wxc-exec --audit`. -//! -//! **Invariant: `wxc-exec.exe` runs unelevated.** Starting a WPR -//! kernel ETW session requires administrator, so `--audit` does NOT -//! self-elevate `wxc-exec`; instead it launches PLM's hidden, fixed-operation -//! START child through UAC and retains its authenticated control pipe. -//! -//! The child opens and validates the wxc-exec owner before starting WPR, stays -//! alive through the workload, and cancels on owner death or pipe break. -//! The retained child also performs the fixed stop and ETL transfer operation, -//! so the normal audit lifecycle needs only the original UAC prompt. The -//! singleton is released only after that child reports the trace stopped. -//! -//! The retained child exclusively owns the protected host-wide named-mutex -//! singleton (`Global\Mxc_Plm_Audit`) together with its WPR session. +//! `wxc-exec --audit` compatibility artifacts built on `captureDenials`. -use wxc_common::logger::Logger; +use std::path::{Path, PathBuf}; -const PLM_ANALYSIS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10 * 60); -const PLM_WAIT_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); -const MAX_PLM_OUTPUT_BYTES: usize = 1024 * 1024; -const OUTPUT_TRUNCATED_MARKER: &[u8] = b"\n[PLM output truncated]\n"; +use wxc_common::models::{ + CaptureDenialsConfig, CaptureDenialsMode, ExecutionRequest, ScriptResponse, +}; -/// Path to `plm.exe`, expected to sit next to `wxc-exec.exe` in the -/// same install directory. Returns `None` when the current exe path -/// can't be resolved. -pub fn plm_exe_path() -> Option { - std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(|d| d.join("plm.exe"))) +#[derive(Debug)] +pub struct AuditContext { + pub log_dir: PathBuf, + pub config_path: Option, } -pub struct CapturedAudit { - pub log_dir: std::path::PathBuf, - pub trace_path: std::path::PathBuf, +pub fn prepare_request( + request: &mut ExecutionRequest, + config_path: Option, +) -> Result { + prepare_request_in(request, config_path, plm::stop::default_log_dir()) } -pub fn capture_and_stop( - guard: &mut AuditTraceGuard, - logger: &mut Logger, -) -> Result { - use std::fmt::Write as _; - - let log_dir = plm::stop::default_log_dir(); +fn prepare_request_in( + request: &mut ExecutionRequest, + config_path: Option, + log_dir: PathBuf, +) -> Result { std::fs::create_dir_all(&log_dir) .map_err(|error| format!("failed to create audit log directory: {error}"))?; - let trace_path = log_dir.join("trace.etl"); - - let _ = writeln!( - logger, - "[audit] stopping trace into {}", - trace_path.display() - ); - guard.stop(&trace_path, logger).map_err(|error| { - let message = format!("failed to stop and transfer PLM trace: {error:#}"); - let _ = writeln!(logger, "[audit] {message}"); - message - })?; - Ok(CapturedAudit { + request.policy.capture_denials = Some(CaptureDenialsConfig { + mode: CaptureDenialsMode::Allow, + output_path: Some(log_dir.join("denials.json").to_string_lossy().into_owned()), + retain_etl: true, + }); + Ok(AuditContext { log_dir, - trace_path, + config_path, }) } -/// Run a normal public PLM command after wxc-exec has released the singleton. -pub fn run_plm_command(args: &[&std::ffi::OsStr], logger: &mut Logger, verbose: bool) -> bool { - use std::fmt::Write as _; - - let Some(plm) = plm_exe_path() else { - let _ = writeln!(logger, "[audit] could not resolve plm.exe path"); - return false; - }; - if !plm.exists() { - let _ = writeln!( - logger, - "[audit] plm.exe not found at {} - skipping", - plm.display() - ); - return false; +pub fn finalize( + response: &mut ScriptResponse, + context: &AuditContext, + exe_dir: &Path, + verbose: bool, +) -> Result<(), String> { + let metadata = response + .output_metadata + .as_mut() + .ok_or_else(|| "captureDenials returned no output metadata".to_string())?; + if let Some(error) = metadata.capture_denials_error.as_ref() { + return Err(format!( + "captureDenials finalization failed: {}; retained ETL: {}", + error.message, error.etl_path + )); } - - let mut summary = String::new(); - let _ = write!(summary, "[audit] running {}", plm.display()); - for a in args { - let _ = write!(summary, " {}", a.to_string_lossy()); + let capture = metadata + .capture_denials + .as_mut() + .ok_or_else(|| "captureDenials returned no successful output metadata".to_string())?; + let source_denials = PathBuf::from(&capture.output_path); + let source_etl = capture + .etl_path + .as_deref() + .map(PathBuf::from) + .ok_or_else(|| "captureDenials did not return the retained ETL path".to_string())?; + if !source_denials.is_file() { + return Err(format!( + "captureDenials output file is missing: {}", + source_denials.display() + )); } - let _ = writeln!(logger, "{summary}"); - if verbose { - eprintln!("{summary}"); + if !source_etl.is_file() { + return Err(format!( + "captureDenials retained ETL is missing: {}", + source_etl.display() + )); } - let output = std::process::Command::new(&plm) - .args(args) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .map_err(|error| format!("failed to spawn public plm.exe: {error}")) - .and_then(|child| wait_with_output_timeout(child, PLM_ANALYSIS_TIMEOUT)); - report_plm_output(output, logger, verbose) + let document: learning_mode_core::DenialsDocument = + serde_json::from_slice(&std::fs::read(&source_denials).map_err(|error| { + format!( + "failed to read captureDenials output {}: {error}", + source_denials.display() + ) + })?) + .map_err(|error| { + format!( + "captureDenials output {} is not valid canonical denials JSON: {error}", + source_denials.display() + ) + })?; + validate_metadata(capture, &document)?; + + let final_denials = context.log_dir.join("denials.json"); + let final_etl = context.log_dir.join("trace.etl"); + ensure_destination_available(&source_etl, &final_etl)?; + ensure_destination_available(&source_denials, &final_denials)?; + move_new_file(&source_etl, &final_etl)?; + move_new_file(&source_denials, &final_denials)?; + remove_empty_managed_directory(source_etl.parent(), &context.log_dir)?; + + plm::stop::postprocess_denials( + &document, + &plm::stop::PostProcessOptions { + log_dir: context.log_dir.clone(), + bin_path: None, + config_path: context.config_path.clone(), + trace_path: final_etl.clone(), + denials_path: final_denials.clone(), + verbose, + }, + exe_dir, + ) + .map_err(|error| format!("failed to generate audit compatibility artifacts: {error:#}"))?; + + capture.output_path = final_denials.to_string_lossy().into_owned(); + capture.etl_path = Some(final_etl.to_string_lossy().into_owned()); + Ok(()) } -fn wait_with_output_timeout( - mut child: std::process::Child, - timeout: std::time::Duration, -) -> Result { - let Some(stdout) = child.stdout.take() else { - let _ = child.kill(); - let _ = child.wait(); - return Err("public plm.exe stdout was not captured".to_string()); - }; - let Some(stderr) = child.stderr.take() else { - let _ = child.kill(); - let _ = child.wait(); - return Err("public plm.exe stderr was not captured".to_string()); - }; - let stdout_reader = match std::thread::Builder::new() - .name("plm-audit-stdout".to_string()) - .spawn(move || read_bounded_output(stdout, MAX_PLM_OUTPUT_BYTES)) +fn validate_metadata( + capture: &wxc_common::models::CaptureDenialsOutput, + document: &learning_mode_core::DenialsDocument, +) -> Result<(), String> { + if capture.kind != wxc_common::models::CaptureDenialsOutput::KIND + || capture.exit_code != document.summary.exit_code + || capture.total_denials != document.summary.total_denials + || capture.denied_resources_truncated != document.summary.denied_resources_truncated { - Ok(reader) => reader, - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(format!("failed to start plm stdout reader: {error}")); - } - }; - let stderr_reader = match std::thread::Builder::new() - .name("plm-audit-stderr".to_string()) - .spawn(move || read_bounded_output(stderr, MAX_PLM_OUTPUT_BYTES)) - { - Ok(reader) => reader, - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - let _ = stdout_reader.join(); - return Err(format!("failed to start plm stderr reader: {error}")); - } - }; - - let deadline = std::time::Instant::now() + timeout; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break status, - Ok(None) if std::time::Instant::now() < deadline => { - std::thread::sleep(PLM_WAIT_POLL_INTERVAL); - } - Ok(None) => { - child.kill().map_err(|error| { - format!( - "public plm.exe exceeded the {} second audit-analysis timeout, \ - but termination failed: {error}", - timeout.as_secs() - ) - })?; - child.wait().map_err(|error| { - format!( - "public plm.exe exceeded the {} second audit-analysis timeout, \ - was terminated, but could not be reaped: {error}", - timeout.as_secs() - ) - })?; - return Err(format!( - "public plm.exe exceeded the {} second audit-analysis timeout and was terminated", - timeout.as_secs() - )); - } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(format!("failed to poll public plm.exe: {error}")); - } - } - }; - let stdout = stdout_reader - .join() - .map_err(|_| "public plm.exe stdout reader panicked".to_string())? - .map_err(|error| format!("failed to read public plm.exe stdout: {error}"))?; - let stderr = stderr_reader - .join() - .map_err(|_| "public plm.exe stderr reader panicked".to_string())? - .map_err(|error| format!("failed to read public plm.exe stderr: {error}"))?; - Ok(std::process::Output { - status, - stdout, - stderr, - }) + return Err( + "captureDenials metadata does not match the canonical denials document".to_string(), + ); + } + Ok(()) } -fn read_bounded_output(mut reader: impl std::io::Read, limit: usize) -> std::io::Result> { - let mut output = Vec::with_capacity(limit.min(64 * 1024)); - let mut buffer = [0u8; 8192]; - let mut truncated = false; - loop { - let count = reader.read(&mut buffer)?; - if count == 0 { - break; - } - let remaining = limit.saturating_sub(output.len()); - let retained = remaining.min(count); - output.extend_from_slice(&buffer[..retained]); - truncated |= retained != count; +fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { + if source == destination { + return Ok(()); } - if truncated { - let marker = &OUTPUT_TRUNCATED_MARKER[..OUTPUT_TRUNCATED_MARKER.len().min(limit)]; - let marker_start = output.len().saturating_sub(marker.len()); - output.truncate(marker_start); - output.extend_from_slice(marker); + ensure_destination_available(source, destination)?; + match std::fs::rename(source, destination) { + Ok(()) => Ok(()), + Err(rename_error) => { + std::fs::copy(source, destination).map_err(|copy_error| { + format!( + "failed to move audit artifact {} to {}: {rename_error}; \ + copy fallback failed: {copy_error}", + source.display(), + destination.display() + ) + })?; + std::fs::remove_file(source).map_err(|remove_error| { + let _ = std::fs::remove_file(destination); + format!( + "copied audit artifact to {}, but failed to remove source {}: {remove_error}", + destination.display(), + source.display() + ) + }) + } } - Ok(output) } -fn report_plm_output( - output: Result, - logger: &mut Logger, - verbose: bool, -) -> bool { - use std::fmt::Write as _; - - match output { - Ok(run) if run.status.success() => { - if verbose { - replay_captured(logger, &run.stdout, &run.stderr); - } - true - } - Ok(run) => { - let code = run.status.code().unwrap_or(-1); - let _ = writeln!(logger, "[audit] plm exited with code {code}"); - replay_captured(logger, &run.stdout, &run.stderr); - if verbose { - eprintln!("[audit] plm exited with code {code}"); - } - false - } - Err(error) => { - let _ = writeln!(logger, "[audit] failed to launch plm: {error}"); - if verbose { - eprintln!("[audit] failed to launch plm: {error}"); - } - false - } +fn ensure_destination_available(source: &Path, destination: &Path) -> Result<(), String> { + if source != destination && destination.exists() { + return Err(format!( + "audit artifact destination already exists: {}", + destination.display() + )); } + Ok(()) } -/// Replay captured stdout/stderr bytes to the current process's own -/// streams. Used on failure (and in verbose mode on success) so the happy path -/// can stay silent while diagnostics still surface. -fn replay_captured(logger: &mut Logger, stdout: &[u8], stderr: &[u8]) { - use std::fmt::Write as _; - use std::io::Write as _; - if !stdout.is_empty() { - let _ = std::io::stdout().write_all(stdout); - let _ = write!(logger, "{}", String::from_utf8_lossy(stdout)); +fn remove_empty_managed_directory( + directory: Option<&Path>, + audit_dir: &Path, +) -> Result<(), String> { + let Some(directory) = directory else { + return Ok(()); + }; + let is_retained_capture_dir = directory + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name.eq_ignore_ascii_case("retained")); + if directory == audit_dir || !is_retained_capture_dir { + return Ok(()); } - if !stderr.is_empty() { - let _ = std::io::stderr().write_all(stderr); - let _ = write!(logger, "{}", String::from_utf8_lossy(stderr)); + match std::fs::remove_dir(directory) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::DirectoryNotEmpty + ) => + { + Ok(()) + } + Err(error) => Err(format!( + "failed to remove empty retained-ETL directory {}: {error}", + directory.display() + )), } } -/// Stack-owned guarded START session. -#[derive(Debug)] -pub struct AuditTraceGuard { - session: plm::elevated::GuardedSession, -} +#[cfg(test)] +mod tests { + use super::*; + use learning_mode_core::{DenialSummary, DenialsDocument}; + use wxc_common::models::{ + CaptureDenialsErrorOutput, CaptureDenialsOutput, SandboxOutputMetadata, + }; -impl AuditTraceGuard { - pub fn start(logger: &mut Logger, verbose: bool) -> Result { - use std::fmt::Write as _; + #[test] + fn prepare_injects_allow_capture_and_retention() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + let config_path = PathBuf::from(r"C:\policy.json"); + let mut request = ExecutionRequest::default(); - let plm = plm_exe_path().ok_or_else(|| { - let message = "could not resolve plm.exe path".to_string(); - let _ = writeln!(logger, "[audit] {message}"); - message - })?; - if !plm.exists() { - let message = format!("plm.exe not found at {}", plm.display()); - let _ = writeln!(logger, "[audit] {message}"); - return Err(message); - } + let context = + prepare_request_in(&mut request, Some(config_path.clone()), log_dir.clone()).unwrap(); + let capture = request.policy.capture_denials.as_ref().unwrap(); - let summary = format!("[audit] running {} start", plm.display()); - let _ = writeln!(logger, "{summary}"); - if verbose { - eprintln!("{summary}"); - } - let owner_pid = unsafe { windows::Win32::System::Threading::GetCurrentProcessId() }; - plm::elevated::start_guarded_session_with_executable(&plm, owner_pid) - .map(|session| Self { session }) - .map_err(|error| { - let message = format!("guarded plm start failed: {error:#}"); - let _ = writeln!(logger, "[audit] {message}"); - if verbose { - eprintln!("[audit] {message}"); - } - message - }) + assert_eq!(capture.mode, CaptureDenialsMode::Allow); + assert!(capture.retain_etl); + assert_eq!( + capture.output_path.as_deref(), + Some(log_dir.join("denials.json").to_string_lossy().as_ref()) + ); + assert_eq!(context.config_path, Some(config_path)); + assert!(context.log_dir.is_dir()); } - pub fn stop( - &mut self, - trace_destination: &std::path::Path, - logger: &mut Logger, - ) -> Result<(), String> { - use std::fmt::Write as _; + #[test] + fn finalize_relocates_outputs_and_updates_metadata() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + let retained_dir = directory.path().join("retained").join("run"); + std::fs::create_dir_all(&log_dir).unwrap(); + std::fs::create_dir_all(&retained_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_etl = retained_dir.join("capture.etl"); + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(23, 0, false)); + std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 23); + let context = AuditContext { + log_dir: log_dir.clone(), + config_path: None, + }; + + finalize(&mut response, &context, directory.path(), false).unwrap(); - self.session.stop(trace_destination).map_err(|error| { - let message = format!("guarded PLM stop failed: {error:#}"); - let _ = writeln!(logger, "[audit] {message}"); - message - }) + assert!(!source_denials.exists()); + assert!(!source_etl.exists()); + assert!(!retained_dir.exists()); + assert_eq!(std::fs::read(log_dir.join("trace.etl")).unwrap(), b"etl"); + let capture = response + .output_metadata + .as_ref() + .unwrap() + .capture_denials + .as_ref() + .unwrap(); + assert_eq!( + Path::new(&capture.output_path), + log_dir.join("denials.json") + ); + assert_eq!( + capture.etl_path.as_deref().map(Path::new), + Some(log_dir.join("trace.etl").as_path()) + ); } - pub fn cancel(&mut self, logger: &mut Logger) -> Result<(), String> { - use std::fmt::Write as _; + #[test] + fn finalize_rejects_missing_or_failed_capture_metadata() { + let directory = tempfile::tempdir().unwrap(); + let context = AuditContext { + log_dir: directory.path().join("audit"), + config_path: None, + }; + let mut missing = ScriptResponse::default(); + assert!(finalize(&mut missing, &context, directory.path(), false) + .unwrap_err() + .contains("no output metadata")); - self.session.cancel().map_err(|error| { - let message = format!("guarded PLM cancellation failed: {error:#}"); - let _ = writeln!(logger, "[audit] {message}"); - message - }) + let mut failed = ScriptResponse { + output_metadata: Some(Box::new(SandboxOutputMetadata { + capture_denials: None, + capture_denials_error: Some(CaptureDenialsErrorOutput { + message: "decode failed".to_string(), + etl_path: r"C:\retained\capture.etl".to_string(), + }), + })), + ..Default::default() + }; + let error = finalize(&mut failed, &context, directory.path(), false).unwrap_err(); + assert!(error.contains("decode failed")); + assert!(error.contains("capture.etl")); } -} - -#[cfg(test)] -mod tests { - use super::*; - use windows::Win32::Foundation::{CloseHandle, WAIT_OBJECT_0}; - use windows::Win32::System::Threading::{ - OpenProcess, WaitForSingleObject, PROCESS_SYNCHRONIZE, - }; #[test] - fn analysis_watchdog_terminates_timed_out_child() { - let child = std::process::Command::new("powershell.exe") - .args(["-NoProfile", "-Command", "Start-Sleep -Seconds 60"]) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .expect("spawn long-running child"); - let process = unsafe { OpenProcess(PROCESS_SYNCHRONIZE, false, child.id()) } - .expect("open child synchronization handle"); + fn finalize_rejects_malformed_json_without_moving_artifacts() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&log_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_etl = log_dir.join("denials.unique.etl"); + std::fs::write(&source_denials, b"{").unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 0); + let context = AuditContext { + log_dir, + config_path: None, + }; - let error = wait_with_output_timeout(child, std::time::Duration::from_millis(50)) - .expect_err("watchdog should time out"); - assert!(error.contains("timeout")); - assert_eq!( - unsafe { WaitForSingleObject(process, 1_000) }, - WAIT_OBJECT_0, - "timed-out child should be terminated and reaped" - ); - unsafe { - let _ = CloseHandle(process); - } + let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); + + assert!(error.contains("canonical denials JSON")); + assert!(source_denials.exists()); + assert!(source_etl.exists()); } #[test] - fn analysis_output_reader_caps_and_drains_child_output() { - let input_len = MAX_PLM_OUTPUT_BYTES + 4096; - let mut input = std::io::Cursor::new(vec![b'x'; input_len]); - let captured = - read_bounded_output(&mut input, MAX_PLM_OUTPUT_BYTES).expect("bounded output read"); - assert_eq!(input.position(), input_len as u64); - assert_eq!(captured.len(), MAX_PLM_OUTPUT_BYTES); - assert!(captured.ends_with(OUTPUT_TRUNCATED_MARKER)); + fn finalize_preflights_both_destinations_before_moving_artifacts() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&log_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_etl = log_dir.join("denials.unique.etl"); + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, false)); + std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + std::fs::write(log_dir.join("denials.json"), b"existing").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 0); + let context = AuditContext { + log_dir: log_dir.clone(), + config_path: None, + }; + + let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); + + assert!(error.contains("destination already exists")); + assert!(source_denials.exists()); + assert!(source_etl.exists()); + assert!(!log_dir.join("trace.etl").exists()); } - #[test] - fn analysis_output_reader_honors_limits_smaller_than_marker() { - let captured = - read_bounded_output(std::io::Cursor::new([b'x'; 32]), 8).expect("bounded output read"); - assert_eq!(captured.len(), 8); - assert_eq!(captured, OUTPUT_TRUNCATED_MARKER[..8]); + fn response_with_capture( + denials_path: &Path, + etl_path: &Path, + exit_code: i32, + ) -> ScriptResponse { + ScriptResponse { + exit_code, + output_metadata: Some(Box::new(SandboxOutputMetadata { + capture_denials: Some(CaptureDenialsOutput { + kind: CaptureDenialsOutput::KIND.to_string(), + output_path: denials_path.to_string_lossy().into_owned(), + exit_code, + total_denials: 0, + denied_resources_truncated: false, + etl_path: Some(etl_path.to_string_lossy().into_owned()), + }), + capture_denials_error: None, + })), + ..Default::default() + } } } diff --git a/src/core/wxc/src/main.rs b/src/core/wxc/src/main.rs index 3b4f5f098..a6bcc86c6 100644 --- a/src/core/wxc/src/main.rs +++ b/src/core/wxc/src/main.rs @@ -115,20 +115,14 @@ struct Cli { #[arg(long = "force-reclaim")] force_reclaim: bool, - /// Audit mode: drive the permissive-learning-mode (PLM) WPR/ETW trace - /// pipeline for a developer inner-loop run — inject `permissiveLearningMode` - /// and record every access check to a trace. Windows ProcessContainer only: - /// the PLM trace/capability path is not honored by Windows Sandbox, WSLC, - /// IsolationSession, or the other containment backends. + /// Audit mode: run captureDenials in permissive allow mode with ETL + /// retention, then generate developer policy-authoring artifacts. Windows + /// ProcessContainer only. #[cfg(target_os = "windows")] #[arg(long)] audit: bool, - /// Surface the PLM lifecycle diagnostics (spawn banner, plm.exe stderr - /// lines, non-zero-exit / spawn-failure reasons) on wxc-exec's stderr in - /// addition to the log buffer. Off by default so `--audit` doesn't pollute - /// the wrapped workload's stdout/stderr; opt in when debugging the audit - /// pipeline itself. + /// Print learned-policy post-processing details for `--audit`. #[cfg(target_os = "windows")] #[arg(long)] audit_verbose: bool, @@ -530,31 +524,6 @@ fn config_file_path(cli: &Cli) -> Option { .map(std::path::PathBuf::from) } -#[cfg(target_os = "windows")] -fn audit_stop_args( - config_path: Option<&std::path::Path>, - log_dir: &std::path::Path, - trace_path: &std::path::Path, - exit_code: i32, -) -> Vec { - let mut args = vec![ - std::ffi::OsString::from("stop"), - std::ffi::OsString::from("--log-dir"), - log_dir.as_os_str().to_owned(), - std::ffi::OsString::from("--trace-file"), - trace_path.as_os_str().to_owned(), - ]; - if let Some(config_path) = config_path { - args.push(std::ffi::OsString::from("--config-path")); - args.push(config_path.as_os_str().to_owned()); - } - args.push(std::ffi::OsString::from(format!("--exit-code={exit_code}"))); - args -} - -#[cfg(target_os = "windows")] -use audit::{capture_and_stop, run_plm_command, AuditTraceGuard}; - // --------------------------------------------------------------------------- // Graceful-exit DACL cleanup // --------------------------------------------------------------------------- @@ -1081,6 +1050,8 @@ fn main() { // deny-and-record before injecting it. Comparisons are case-insensitive // because Windows derives capability SIDs case-insensitively. #[cfg(target_os = "windows")] + let mut audit_context = None; + #[cfg(target_os = "windows")] if cli.audit { if let Err(message) = validate_audit_request(&request) { eprintln!("Error: {message}"); @@ -1092,6 +1063,19 @@ fn main() { process::exit(1); } let injected = apply_permissive_learning_mode(&mut request.policy.capabilities); + let context = match audit::prepare_request(&mut request, config_file_path(&cli)) { + Ok(context) => context, + Err(message) => { + eprintln!("Error: {message}"); + telemetry::emit_early_exit( + telemetry_active, + &request.containment, + telemetry::FailureReason::ConfigError, + ); + process::exit(1); + } + }; + audit_context = Some(context); logger.log("WARNING: --audit enabled - AppContainer restrictions will NOT be enforced\n"); if injected && cli.audit_verbose { eprintln!( @@ -1194,8 +1178,15 @@ fn main() { // it on signal as well as the normal-exit path below; it is `None` when no // DACL augmentation was required. Tier-selection warnings and the selected // tier are logged to `logger` by the engine. - let mut runner: Box = match mxc_engine::resolve_runner(&request, &mut logger) - { + #[cfg(target_os = "windows")] + let resolved_runner = if cli.audit { + mxc_engine::resolve_runner_for_audit(&request, &mut logger) + } else { + mxc_engine::resolve_runner(&request, &mut logger) + }; + #[cfg(not(target_os = "windows"))] + let resolved_runner = mxc_engine::resolve_runner(&request, &mut logger); + let mut runner: Box = match resolved_runner { Ok(resolved) => { if let Some(mgr) = resolved.dacl_manager { park_dacl_for_cleanup(mgr); @@ -1214,79 +1205,32 @@ fn main() { } }; - // --audit: start the PLM (permissive learning mode) WPR trace before - // the runner spawns the container so we capture access-denied events - // for the lifetime of the workload. The matching `plm stop` below - // tears the trace down and (when the policy came from a file) - // merges findings back into it. - // - #[cfg(target_os = "windows")] - let mut audit_guard: Option; #[cfg(target_os = "windows")] - let audit_config_file = if cli.audit { - match AuditTraceGuard::start(&mut logger, cli.audit_verbose) { - Ok(guard) => audit_guard = Some(guard), - Err(_) => { - eprintln!( - "error: plm start failed; refusing to run --audit without an \ - active trace. See logs for details." - ); - drop(take_parked_dacl()); - std::process::exit(1); - } - } - config_file_path(&cli) - } else { - audit_guard = None; - None - }; + let exe_dir = std::env::current_exe() + .ok() + .and_then(|path| path.parent().map(std::path::Path::to_path_buf)) + .unwrap_or_else(|| std::path::PathBuf::from(".")); let run_start = Instant::now(); - let response = runner.run(&request, &mut logger); + let mut response = runner.run(&request, &mut logger); let run_elapsed = run_start.elapsed(); let _ = writeln!(logger, "Runner completed in {}ms", run_elapsed.as_millis()); - // Tear down the PLM trace after the container exits, regardless of - // its exit code. Done before the runner is dropped so the trace - // tooling sees a fully-quiesced workload. #[cfg(target_os = "windows")] - if cli.audit { - let capture = audit_guard - .as_mut() - .ok_or_else(|| "audit trace guard is missing".to_string()) - .and_then(|guard| capture_and_stop(guard, &mut logger)); - match capture { - Ok(capture) => { - let stop_args = audit_stop_args( - audit_config_file.as_deref(), - &capture.log_dir, - &capture.trace_path, - response.exit_code, - ); - let borrowed: Vec<&std::ffi::OsStr> = stop_args - .iter() - .map(std::ffi::OsString::as_os_str) - .collect(); - if run_plm_command(&borrowed, &mut logger, cli.audit_verbose) { - eprintln!("[audit] artifacts written to {}", capture.log_dir.display()); - } else { - let _ = writeln!(logger, "[audit] PLM trace analysis failed"); - } - } - Err(error) => { - let _ = writeln!( - logger, - "[audit] {error}; guarded cleanup remains armed only if WPR stop did not complete" - ); - if let Some(guard) = audit_guard.as_mut() { - if let Err(cancel_error) = guard.cancel(&mut logger) { - let _ = writeln!( - logger, - "[audit] explicit guarded cleanup failed: {cancel_error}" - ); - eprintln!("error: audit trace cleanup failed: {cancel_error}"); - } - } + if let Some(context) = audit_context.as_ref() { + if !cli.dry_run { + if let Err(error) = audit::finalize(&mut response, context, &exe_dir, cli.audit_verbose) + { + let message = format!("audit finalization failed: {error}"); + let _ = writeln!(logger, "[audit] {message}"); + eprintln!("error: {message}"); + response.exit_code = -1; + response.error_message = match response.error_message.is_empty() { + true => message, + false => format!("{}; {message}", response.error_message), + }; + } else { + eprintln!("[audit] artifacts written to {}", context.log_dir.display()); } } } @@ -1301,12 +1245,6 @@ fn main() { drop(runner); drop(take_parked_dacl()); - // `process::exit` below skips destructors. Drop any still-armed guarded - // connection first so the elevated child cancels and releases its - // singleton before this process exits. - #[cfg(target_os = "windows")] - drop(audit_guard); - // Surface security warnings (e.g. permissiveLearningMode relaxing // deny-by-default under --audit). The logger only records these rather than // writing them itself, so that `mxc_engine` embedders don't get unannounced @@ -1450,54 +1388,6 @@ mod tests { } } - #[cfg(target_os = "windows")] - #[test] - fn audit_stop_args_include_workload_exit_code() { - let args = audit_stop_args( - Some(std::path::Path::new(r"C:\config.json")), - std::path::Path::new(r"C:\logs"), - std::path::Path::new(r"C:\logs\trace.etl"), - 23, - ); - assert_eq!( - args, - [ - "stop", - "--log-dir", - r"C:\logs", - "--trace-file", - r"C:\logs\trace.etl", - "--config-path", - r"C:\config.json", - "--exit-code=23" - ] - .map(std::ffi::OsString::from) - ); - } - - #[cfg(target_os = "windows")] - #[test] - fn audit_stop_args_accept_negative_workload_exit_code() { - let args = audit_stop_args( - None, - std::path::Path::new(r"C:\logs"), - std::path::Path::new(r"C:\logs\trace.etl"), - -1, - ); - assert_eq!( - args, - [ - "stop", - "--log-dir", - r"C:\logs", - "--trace-file", - r"C:\logs\trace.etl", - "--exit-code=-1" - ] - .map(std::ffi::OsString::from) - ); - } - #[test] fn state_aware_dispatch_errors_use_only_auxiliary_diagnostic_sinks() { let directory = tempfile::tempdir().unwrap(); diff --git a/src/host/plm/src/analysis.rs b/src/host/plm/src/analysis.rs index 876cff47d..17c70c907 100644 --- a/src/host/plm/src/analysis.rs +++ b/src/host/plm/src/analysis.rs @@ -24,8 +24,12 @@ pub fn analyze_trace(trace_file: &Path) -> Result { .with_context(|| format!("failed to analyze {}", trace_file.display())) } -/// Write canonical denials JSON atomically. -pub fn write_denials(output_path: &Path, analysis: &AnalysisResult, exit_code: i32) -> Result<()> { +/// Write canonical denials JSON atomically and return the document that was written. +pub fn write_denials( + output_path: &Path, + analysis: &AnalysisResult, + exit_code: i32, +) -> Result { let parent = output_path.parent().unwrap_or_else(|| Path::new(".")); std::fs::create_dir_all(parent) .with_context(|| format!("failed to create {}", parent.display()))?; @@ -46,7 +50,7 @@ pub fn write_denials(output_path: &Path, analysis: &AnalysisResult, exit_code: i temp.persist(output_path) .map_err(|error| error.error) .with_context(|| format!("failed to replace {}", output_path.display()))?; - Ok(()) + Ok(document) } /// Build the temporary legacy config-generator inputs from canonical denials. diff --git a/src/host/plm/src/elevated.rs b/src/host/plm/src/elevated.rs index a28f16fe2..7309bee92 100644 --- a/src/host/plm/src/elevated.rs +++ b/src/host/plm/src/elevated.rs @@ -91,6 +91,7 @@ const CONTROL_STOP: u8 = 1; const CONTROL_STOP_AND_ANALYZE: u8 = 2; const CONTROL_STOP_AND_DISCARD: u8 = 3; const CONTROL_ATTACH_JOB: u8 = 4; +const CONTROL_STOP_ANALYZE_AND_TRANSFER: u8 = 5; const TRACKER_STOP_MESSAGE: u32 = u32::MAX; static GUARDIAN_SINGLETON: AtomicIsize = AtomicIsize::new(0); @@ -100,6 +101,7 @@ enum GuardControl { StopAndAnalyze, StopAndDiscard, AttachJob, + StopAnalyzeAndTransfer, } fn attest_job_process(job: HANDLE, pid: u32) -> Result<(OwnedHandle, u64)> { @@ -142,6 +144,7 @@ fn parse_guard_control(value: u8) -> Result { CONTROL_STOP_AND_ANALYZE => Ok(GuardControl::StopAndAnalyze), CONTROL_STOP_AND_DISCARD => Ok(GuardControl::StopAndDiscard), CONTROL_ATTACH_JOB => Ok(GuardControl::AttachJob), + CONTROL_STOP_ANALYZE_AND_TRANSFER => Ok(GuardControl::StopAnalyzeAndTransfer), _ => anyhow::bail!("invalid guarded PLM control message {value}"), } } @@ -366,6 +369,9 @@ impl GuardedOwner { Ok(GuardControl::StopAndDiscard) => { return run_guarded_stop(pipe, self, StopDisposition::Discard); } + Ok(GuardControl::StopAnalyzeAndTransfer) => { + return run_guarded_stop(pipe, self, StopDisposition::AnalyzeAndTrace); + } Ok(GuardControl::AttachJob) => { let deadline = Instant::now() + ATTACH_HANDOFF_TIMEOUT; let handles = @@ -1548,6 +1554,45 @@ impl GuardedSession { result } } + + pub fn stop_analyzed_with_trace(&mut self, trace_destination: &Path) -> Result { + if self.disarmed { + anyhow::bail!("guarded PLM session is already stopped"); + } + let mut pipe = self + .pipe + .take() + .context("guarded PLM control connection is already closed")?; + send_control_unless_response_pending(&mut pipe, CONTROL_STOP_ANALYZE_AND_TRANSFER) + .context("failed to send guarded PLM analyzed trace-transfer STOP")?; + + let stopped = std::cell::Cell::new(false); + let deadline = Instant::now() + WAIT_TIMEOUT_DURATION; + let result = read_analysis_and_trace_response( + &mut pipe, + self.process.0, + trace_destination, + deadline, + || { + stopped.set(true); + Ok(()) + }, + ); + if stopped.get() { + self.disarmed = true; + drop(pipe); + let wait_result = wait_for_child_exit( + self.process.0, + deadline.saturating_duration_since(Instant::now()), + ); + let analysis = result?; + wait_result?; + Ok(analysis) + } else { + self.pipe = Some(pipe); + result + } + } } fn send_control_unless_response_pending(pipe: &mut std::fs::File, control: u8) -> Result<()> { @@ -1865,6 +1910,7 @@ fn start_owned_trace(owner: &mut GuardedOwner) -> Result<()> { enum StopDisposition { Trace, Analyze, + AnalyzeAndTrace, Discard, } @@ -1939,6 +1985,11 @@ fn run_guarded_stop_with_stopped( let membership = owner.finish_job_tracking()?; write_analysis_response(pipe, &scratch, &membership) } + StopDisposition::AnalyzeAndTrace => { + let membership = owner.finish_job_tracking()?; + write_analysis_response(pipe, &scratch, &membership)?; + write_trace_response(pipe, &scratch) + } StopDisposition::Trace => write_trace_response(pipe, &scratch), } } @@ -2184,37 +2235,7 @@ fn read_response( ResponseKind::Success => anyhow::bail!("elevated stop returned no ETL payload"), ResponseKind::Trace if operation == Operation::Stop && stopped => { let destination = trace_destination.context("missing unelevated trace destination")?; - let parent = destination.parent().unwrap_or_else(|| Path::new(".")); - std::fs::create_dir_all(parent).with_context(|| { - format!( - "failed to create trace output directory {}", - parent.display() - ) - })?; - let mut temporary = tempfile::NamedTempFile::new_in(parent).with_context(|| { - format!( - "failed to create unelevated trace temp file in {}", - parent.display() - ) - })?; - copy_exact_polling( - pipe, - temporary.as_file_mut(), - process, - header.payload_len, - deadline, - )?; - temporary - .as_file_mut() - .sync_all() - .context("failed to flush unelevated trace temp file")?; - temporary - .persist(destination) - .map_err(|error| error.error) - .with_context(|| { - format!("failed to persist trace output {}", destination.display()) - })?; - Ok(()) + receive_trace_payload(pipe, process, destination, header.payload_len, deadline) } ResponseKind::Trace if operation == Operation::Stop => { anyhow::bail!("elevated stop returned ETL before the WPR-stopped milestone") @@ -2235,6 +2256,78 @@ fn read_response( } } +fn read_analysis_and_trace_response( + pipe: &mut std::fs::File, + process: HANDLE, + trace_destination: &Path, + deadline: Instant, + on_stopped: impl FnOnce() -> Result<()>, +) -> Result { + let analysis = read_analysis_response(pipe, process, deadline, on_stopped)?; + let header = read_header_polling(pipe, process, deadline)?; + match header.kind { + ResponseKind::Trace => { + receive_trace_payload( + pipe, + process, + trace_destination, + header.payload_len, + deadline, + )?; + Ok(analysis) + } + ResponseKind::Error => { + let mut message = vec![0u8; header.payload_len as usize]; + read_exact_polling(pipe, &mut message, process, deadline)?; + anyhow::bail!( + "elevated guarded WPR trace transfer failed: {}", + String::from_utf8_lossy(&message) + ) + } + other => anyhow::bail!( + "elevated guarded WPR analyzed trace transfer returned unexpected {other:?} payload" + ), + } +} + +fn receive_trace_payload( + pipe: &mut std::fs::File, + process: HANDLE, + destination: &Path, + payload_len: u64, + deadline: Instant, +) -> Result<()> { + let parent = destination.parent().unwrap_or_else(|| Path::new(".")); + std::fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create trace output directory {}", + parent.display() + ) + })?; + let mut temporary = tempfile::NamedTempFile::new_in(parent).with_context(|| { + format!( + "failed to create unelevated trace temp file in {}", + parent.display() + ) + })?; + copy_exact_polling( + pipe, + temporary.as_file_mut(), + process, + payload_len, + deadline, + )?; + temporary + .as_file_mut() + .sync_all() + .context("failed to flush unelevated trace temp file")?; + temporary + .persist(destination) + .map_err(|error| error.error) + .with_context(|| format!("failed to persist trace output {}", destination.display()))?; + Ok(()) +} + fn read_analysis_response( pipe: &mut std::fs::File, process: HANDLE, @@ -2808,7 +2901,11 @@ mod tests { parse_guard_control(CONTROL_ATTACH_JOB).unwrap(), GuardControl::AttachJob ); - for invalid in [0, 5, u8::MAX] { + assert_eq!( + parse_guard_control(CONTROL_STOP_ANALYZE_AND_TRANSFER).unwrap(), + GuardControl::StopAnalyzeAndTransfer + ); + for invalid in [0, 6, u8::MAX] { assert!(parse_guard_control(invalid).is_err()); } } diff --git a/src/host/plm/src/stop.rs b/src/host/plm/src/stop.rs index e11cc7209..ab4cfefd0 100644 --- a/src/host/plm/src/stop.rs +++ b/src/host/plm/src/stop.rs @@ -7,6 +7,7 @@ use anyhow::{Context, Result}; use chrono::Local; +use learning_mode_core::DenialsDocument; use serde::Serialize; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -35,6 +36,17 @@ pub struct StopOptions { pub verbose: bool, } +/// Inputs for generating compatibility artifacts from canonical denials. +#[derive(Debug, Clone)] +pub struct PostProcessOptions { + pub log_dir: PathBuf, + pub bin_path: Option, + pub config_path: Option, + pub trace_path: PathBuf, + pub denials_path: PathBuf, + pub verbose: bool, +} + #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct StopResult { @@ -199,11 +211,6 @@ pub fn run(opts: StopOptions, exe_dir: &Path) -> Result { // in `config::update_from_access_events` can compare it against the // verbatim-prefixed paths ETW emits. The fallback chain is in // `resolve_bin_path`. - let (bin_path, warning) = resolve_bin_path(opts.bin_path.as_deref(), exe_dir); - if let Some(w) = warning { - eprintln!("[plm] warning: {w}"); - } - let trace_file = opts .trace_file .context("plm stop requires --trace-file from a retained guardian capture")?; @@ -225,19 +232,61 @@ pub fn run(opts: StopOptions, exe_dir: &Path) -> Result { let analysis = analyze_trace(&trace_file)?; write_detection_summary(&analysis); - write_denials(&denials_path, &analysis, opts.exit_code)?; + let document = write_denials(&denials_path, &analysis, opts.exit_code)?; + + let adjusted_config_path = postprocess_denials_with_paths( + &document, + &PostProcessOptions { + log_dir, + bin_path: opts.bin_path, + config_path: opts.config_path, + trace_path: trace_file.clone(), + denials_path: denials_path.clone(), + verbose: opts.verbose, + }, + exe_dir, + config_outputs, + )?; - let config_outputs = match config_outputs { - Some(paths) => paths, - None => { - return Ok(StopResult { - trace_path: trace_file, - denials_path, - adjusted_config_path: None, - }) - } + Ok(StopResult { + trace_path: trace_file, + denials_path, + adjusted_config_path, + }) +} + +/// Generates the source-config snapshot and adjusted config from canonical denials. +pub fn postprocess_denials( + document: &DenialsDocument, + opts: &PostProcessOptions, + exe_dir: &Path, +) -> Result> { + std::fs::create_dir_all(&opts.log_dir) + .with_context(|| format!("failed to create log dir {}", opts.log_dir.display()))?; + let config_outputs = prepare_config_output_paths( + opts.config_path.as_deref(), + &opts.log_dir, + &opts.trace_path, + &opts.denials_path, + )?; + postprocess_denials_with_paths(document, opts, exe_dir, config_outputs) +} + +fn postprocess_denials_with_paths( + document: &DenialsDocument, + opts: &PostProcessOptions, + exe_dir: &Path, + config_outputs: Option, +) -> Result> { + let Some(config_outputs) = config_outputs else { + return Ok(None); }; + let (bin_path, warning) = resolve_bin_path(opts.bin_path.as_deref(), exe_dir); + if let Some(warning) = warning { + eprintln!("[plm] warning: {warning}"); + } + // Load the source config before copying or mutating it. The trace and // canonical denials remain useful even if this compatibility-only // adjusted-config phase fails. @@ -255,34 +304,26 @@ pub fn run(opts: StopOptions, exe_dir: &Path) -> Result { .with_context(|| format!("failed to copy {}", config_outputs.source.display()))?; } - if analysis.denied_resources_truncated { + if document.summary.denied_resources_truncated { eprintln!( "[plm] warning: denial analysis was truncated; skipping adjusted-config \ generation because the learned policy would be incomplete" ); - return Ok(StopResult { - trace_path: trace_file, - denials_path, - adjusted_config_path: None, - }); + return Ok(None); } let current_directory = std::env::current_dir() .ok() .map(|path| path.to_string_lossy().into_owned()); let (valid_access_events, requested_capabilities) = - legacy_config_inputs(&analysis.denials, current_directory.as_deref()); + legacy_config_inputs(&document.denials, current_directory.as_deref()); write_requested_capabilities_summary(&requested_capabilities, opts.verbose); if valid_access_events.is_empty() && requested_capabilities.is_empty() { // Nothing mergeable -- skip producing an Adjusted_*.json (which // would be byte-identical to the input and confuse the harness's // diff-based pass/fail signal). - return Ok(StopResult { - trace_path: trace_file, - denials_path, - adjusted_config_path: None, - }); + return Ok(None); } // Edit the pre-loaded copy of the config in memory rather than @@ -323,11 +364,7 @@ pub fn run(opts: StopOptions, exe_dir: &Path) -> Result { save_adjusted_config(&config, &config_outputs.adjusted)?; write_added_paths_summary(&added, opts.verbose); - Ok(StopResult { - trace_path: trace_file, - denials_path, - adjusted_config_path: Some(config_outputs.adjusted), - }) + Ok(Some(config_outputs.adjusted)) } /// True iff `a` and `b` denote the same Windows target. @@ -423,6 +460,74 @@ fn normalize_win32_components(path: &str, trim_trailing_dots_and_spaces: bool) - #[cfg(test)] mod tests { use super::*; + use learning_mode_core::{AccessType, DenialSummary, DeniedResource, ResourceType}; + + fn postprocess_fixture( + document: &DenialsDocument, + ) -> (tempfile::TempDir, PathBuf, Option) { + let directory = tempfile::tempdir().unwrap(); + let source_dir = directory.path().join("source"); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&source_dir).unwrap(); + std::fs::create_dir_all(&log_dir).unwrap(); + let config_path = source_dir.join("policy.json"); + std::fs::write(&config_path, r#"{"processContainer":{"capabilities":[]}}"#).unwrap(); + let trace_path = log_dir.join("trace.etl"); + let denials_path = log_dir.join("denials.json"); + std::fs::write(&trace_path, b"etl").unwrap(); + std::fs::write(&denials_path, b"{}").unwrap(); + + let adjusted = postprocess_denials( + document, + &PostProcessOptions { + log_dir: log_dir.clone(), + bin_path: None, + config_path: Some(config_path), + trace_path, + denials_path, + verbose: false, + }, + directory.path(), + ) + .unwrap(); + (directory, log_dir, adjusted) + } + + #[test] + fn canonical_denials_generate_snapshot_and_adjusted_config() { + let document = DenialsDocument::new( + vec![DeniedResource { + resource: "internetClient".to_string(), + resource_type: ResourceType::Capability, + access_type: AccessType::Unknown, + pid: 42, + filetime: 1, + }], + DenialSummary::new(7, 1, false), + ); + + let (_directory, log_dir, adjusted) = postprocess_fixture(&document); + + assert!(log_dir.join("policy.json").is_file()); + let adjusted = adjusted.expect("capability denial should produce adjusted config"); + let config: serde_json::Value = + serde_json::from_slice(&std::fs::read(adjusted).unwrap()).unwrap(); + assert_eq!( + config["processContainer"]["capabilities"], + serde_json::json!(["internetClient"]) + ); + } + + #[test] + fn truncated_canonical_denials_snapshot_but_skip_adjusted_config() { + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, true)); + + let (_directory, log_dir, adjusted) = postprocess_fixture(&document); + + assert!(log_dir.join("policy.json").is_file()); + assert!(adjusted.is_none()); + assert!(!log_dir.join("Adjusted_policy.json").exists()); + } // ---- resolve_bin_path ----------------------------------------------- From 381650311f69f8c59f86368c6184fe99cd5e3f68 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Thu, 13 Aug 2026 15:11:47 -0700 Subject: [PATCH 02/34] Document unified audit architecture Update repository guidance for captureDenials-backed audit routing and guarded retainEtl parity. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- .github/copilot-instructions.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8925b4a3c..2a632beed 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -111,7 +111,7 @@ The Rust workspace (`src/`) implements multiple sandboxing backends behind the ` | Backend | Binary | Platform | Module | |---------|--------|----------|--------| | AppContainer | `wxc-exec.exe` | Windows | `backends/appcontainer/common/src/appcontainer_runner.rs` | -| BaseContainer (OS sandbox API) | `wxc-exec.exe` | Windows | `backends/appcontainer/common/src/base_container_runner.rs` — prefers `CreateProcessSecurityEnvironment` with PSEC whenever its runtime probe succeeds and the requested policy is compatible, independent of schema version. It temporarily falls back to `Experimental_CreateProcessInSandbox` with the SBOX FlatBuffer contract when PSEC is unavailable or policy-incompatible, then retains the AppContainer tier fallback. Proxy requests use legacy SBOX only on query-less hosts; capability-aware SBOX hosts fall back to AppContainer until MXC can author the model-2 AppContainer-peer contract. `captureDenials` prefers the complete compatible PSEC + V2 Learning Mode path; when that path cannot fully honor a request, MXC retains the highest compatible legacy tier and pairs it with guarded WPR using exact handle-attested process scope. | +| BaseContainer (OS sandbox API) | `wxc-exec.exe` | Windows | `backends/appcontainer/common/src/base_container_runner.rs` — prefers `CreateProcessSecurityEnvironment` with PSEC whenever its runtime probe succeeds and the requested policy is compatible, independent of schema version. It temporarily falls back to `Experimental_CreateProcessInSandbox` with the SBOX FlatBuffer contract when PSEC is unavailable or policy-incompatible, then retains the AppContainer tier fallback. Proxy requests use legacy SBOX only on query-less hosts; capability-aware SBOX hosts fall back to AppContainer until MXC can author the model-2 AppContainer-peer contract. `captureDenials` prefers the complete compatible PSEC + V2 Learning Mode path; when that path cannot fully honor a request, MXC retains the highest compatible legacy tier and pairs it with guarded WPR using exact handle-attested process scope. Both providers honor explicit `retainEtl` after a terminal wait; abandoned processes discard the trace. | | Windows Sandbox | `wxc-exec.exe` | Windows | `backends/windows_sandbox/lifecycle/src/` (live transient one-shot `WindowsSandboxRunner` + state-aware `StatefulSandboxBackend`). Experimental — requires `--experimental`. Supports both **one-shot** (a fresh, disposable VM per invocation with guaranteed teardown, via `ScriptRunner`) and **state-aware** (multi-invocation provision/start/exec/stop/deprovision, via `StatefulSandboxBackend`) modes. State-aware holds a single live VM across separate `wxc-exec` phase processes behind a persistent detached host-side daemon (`backends/windows_sandbox/daemon/`); the OS enforces a single running Windows Sandbox VM per host, so the daemon owns it and reclaims an orphaned VM on restart only via positive process-identity proof. The shared boot sequence (write per-launch nonce, launch VM, capture ownership proof, wait rendezvous, connect) lives in `backends/windows_sandbox/lifecycle/src/vm.rs::launch_managed_vm`; each mode plugs in its own `LaunchObserver` for the per-caller ownership / proof bookkeeping. Honors `readwritePaths`/`readonlyPaths`/`deniedPaths` (HOST paths) at provision via `.wsb` `` entries (mapped at the same absolute host path inside the guest; rejects `deniedPaths` equal-to or nested-within a mapped share since `.wsb` has no Deny primitive); filesystem policy is immutable post-provision. Network isolation is enforced unconditionally by the in-guest agent; `network`/`ui` are not honored. ID prefix `wsb` (strict `wsb:<8-hex>` grammar). Per-launch handshake: 32-byte `Nonce` + 1-byte `ChannelRole` tag on every TCP connection (boot + reconnect); the guest pairs accepted sockets by declared role, not by accept order. The guest agent binary `wxc-windows-sandbox-guest.exe` (`backends/windows_sandbox/guest/`) is injected into the VM. | | MicroVM (NanVix) | `wxc-exec.exe` | Windows | `backends/nanvix/runner/src/lib.rs` — feature-gated behind `microvm` | | Hyperlight | `wxc-exec.exe` | Windows | `backends/hyperlight/common/src/lib.rs` — Hyperlight + Unikraft micro-VM backend | @@ -206,14 +206,14 @@ The workspace is organized into six top-level directories under `src/`: - Each Windows containment backend lives in its own `backends/*/common` crate (e.g. `appcontainer_common`, `windows_sandbox_common`, `isolation_session_common`, `hyperlight_common`, `nanvix_runner`). Backend crates depend on `wxc_common`; there are no cross-edges between backend crates. Windows Sandbox additionally has `windows_sandbox_lifecycle`, which owns the one-shot and state-aware runners and depends on `windows_sandbox_common` for the wire protocol, plus separate daemon and guest binaries. - `learning_mode_core` is the cross-platform learning-mode denial model and output layer. It owns denial types, summaries, analyzer abstractions, plain-JSON document emission, and the serializable output-pointer type, and must not depend on any `backends/*` crate. - `learning_mode_windows` (`backends/learning_mode/windows`) is a Windows-only backend support crate for the AppInfo-brokered Learning Mode APIs in `processmodel.dll`. It runtime-resolves the Learning Mode trace and process security-environment exports, owns their typed handle/lifecycle wrappers, decodes sealed ETL traces through `learning_mode_core`, and depends on `wxc_common` plus `learning_mode_core`; runner integration consumes it from the AppContainer backend layer. The trace contract is `HRESULT Start` + retryable `HRESULT Stop` + infallible `Close`: `Stop` never consumes the trace handle, and every started trace must be closed exactly once (closing without stopping is the early-exit discard path). The process security-environment contract is `HRESULT Create` + infallible by-value `Close` and consumes a PSEC 1.0 FlatBuffer, not the legacy SBOX buffer; generated PSEC bindings live in `core/generated/process_security_environment_specification`. -- `plm` (`host/plm`) is the Windows-only legacy WPR Learning Mode helper. Public `plm.exe` is `asInvoker`: ETL analysis and every caller-selected file path stay under the caller token. It self-elevates only the hidden fixed WPR start operation; the retained elevated guardian accepts authenticated attach and stop/discard controls over unique local PID-checked named pipes, uses the compiled-in profile from protected fixed-volume ProgramData scratch, and returns bounded analysis or trace bytes to the unelevated parent. Successful authenticated stop/discard disarms the child before releasing the PLM singleton. Owner death, pipe break, or another uncertain control failure preserves the recovery marker and deliberately leaves WPR untouched for administrator recovery. -- `wxc`, `lxc`, and `mxc_darwin` are thin binary crates (`wxc-exec` / `lxc-exec` / `mxc-exec-mac`) that wire up CLI args (`clap`), load/validate config, handle maintenance modes (`--probe`, `--delete`, `--setup-*`, `--audit`), and **delegate all backend dispatch to `mxc_engine`**. They contain no `match request.containment` of their own. `wxc-exec` additionally owns the Windows Ctrl-C / DACL-cleanup / `--audit` PLM-trace / telemetry orchestration around the engine call. +- `plm` (`host/plm`) is the Windows-only legacy WPR Learning Mode helper. Public `plm.exe` is `asInvoker`: ETL analysis and every caller-selected file path stay under the caller token. It self-elevates only hidden fixed WPR operations; the retained elevated guardian accepts authenticated attach and stop/analyze/discard controls over unique local PID-checked named pipes, uses the compiled-in profile from protected fixed-volume ProgramData scratch, and returns bounded process-scoped analysis plus, when explicitly retained, the sealed trace bytes to the unelevated parent. Successful authenticated stop/discard disarms the child before releasing the PLM singleton. Owner death, pipe break, or another uncertain control failure preserves the recovery marker and deliberately leaves WPR untouched for administrator recovery. +- `wxc`, `lxc`, and `mxc_darwin` are thin binary crates (`wxc-exec` / `lxc-exec` / `mxc-exec-mac`) that wire up CLI args (`clap`), load/validate config, handle maintenance modes (`--probe`, `--delete`, `--setup-*`, `--audit`), and **delegate all backend dispatch to `mxc_engine`**. They contain no `match request.containment` of their own. `wxc-exec` additionally owns the Windows Ctrl-C / DACL-cleanup / telemetry orchestration around the engine call. Its `--audit` compatibility workflow synthesizes allow-mode `captureDenials` with ETL retention, then generates policy-authoring artifacts from the canonical denials document returned by the selected engine backend. - `mxc_engine` is the **single execution engine** — the one home for "given an `ExecutionRequest`, run it". It owns: run-to-completion backend selection (`run` / `resolve_runner`, covering **all** backends, incl. the Windows ProcessContainer BaseContainer/AppContainer BFS/DACL fallback tiers via `appcontainer_common::dispatcher::dispatch_with_fallback`, and every experimental backend, feature-gated); streaming (`spawn` → `Box`); state-aware lifecycle dispatch (`run_state_aware`, including Windows Sandbox and IsolationSession); host probing (`platform_support` / `PlatformSupport`); and config building (`build_request` / `build_request_with_containment`, `SandboxPolicy` + sections, `available_tools_policy`/`user_profile_policy`/`temporary_files_policy`). It depends on the backend crates (cfg-split: appcontainer/windows_sandbox lifecycle/isolation_session/wslc/nanvix on Windows, bubblewrap/lxc/nanvix on Linux, seatbelt on macOS) so it can't live in `wxc_common`. Both the executor binaries and `mxc-sdk` call into it. `ResolvedRunner` carries the boxed runner plus (Windows only) the optional `DaclManager` guard, so `wxc-exec` can park the guard for its signal handler. - `mxc-sdk` is the **public Rust SDK** — a thin facade over `mxc_engine`. Build a `SandboxRequest` with `build_request`, then either `run(request)` (run-to-completion; returns an `Output` with the `WaitOutcome`, captured `stdout`/`stderr`, warnings, and optional structured output metadata) or `spawn_sandbox(request)` (returns a `Sandbox` handle for live bidirectional stdio — `take_stdin`/`take_stdout`/`take_stderr`, `kill()`, `wait()` returning a `WaitOutcome` (`Exited(i32)` / `TimedOut`) as `io::Result`, `output_metadata()` after terminal completion, or `wait_with_output()`). It re-exports the engine's config-building surface (`build_request`, `build_request_with_containment` + `Containment`/`WslcSection`, `mxc_sdk::policy::{SandboxPolicy sections}`, discovery helpers) and `platform_support`; `mod sandbox` (wrapping the engine's `SandboxProcess` in `Sandbox`) is its only local module. No pty is ever allocated. Streaming supports Seatbelt (macOS), Bubblewrap (Linux), Windows ProcessContainer (AppContainer + BaseContainer), and WSLC (Windows, experimental — needs the crate's `wslc` feature plus `SandboxRequest::set_experimental(true)`; no stdin and `id() == 0`, since the WSLC SDK exposes neither); other backends return `ErrorCode::UnsupportedContainment`. - The lower-level execution surface lives in `wxc_common::sandbox_process`: the `SandboxBackend` trait (`validate` + `spawn(request, logger, StdioMode) -> Box` + a `diagnose_exit` hook) and the generic `Runner` adapter that bridges any `SandboxBackend` to the run-to-completion `ScriptRunner` (via `spawn(StdioMode::Inherit)` then `wait()`). `SandboxProcess::output_metadata()` carries backend-produced structured outputs after terminal teardown without writing to process-global stdio. `StdioMode::Pipes` hands the caller live stdin/stdout/stderr (what the `mxc-sdk` streaming path uses); `StdioMode::Inherit` lets the child inherit the host's stdio (what the executor binaries use, preserving the TTY under a pty). `SandboxBackend` is implemented for Seatbelt, Bubblewrap, Windows ProcessContainer, and WSLC (on `wslc_common::WSLContainerRunner` itself, which shares one container lifecycle — `start_container` — between its streaming `SandboxBackend` and run-to-completion `ScriptRunner` impls, differing only in where the WSLC SDK's output callbacks write). - `mxc_ffi` (`ffi/mxc_ffi`, `crate-type = ["cdylib", "staticlib", "lib"]`) is a flat, panic-safe **C ABI over `mxc-sdk`** for language bindings. `mxc_run(policyJson, command, out)` runs a sandbox to completion, filling a `#[repr(C)] MxcRunResult` (status + exit_code + timed_out + owned stdout/stderr/error/output-metadata C strings); every entry point is `catch_unwind`-wrapped so a panic becomes a status code, never an unwind. Its `build.rs` runs **csbindgen** to generate the C# P/Invoke (`sdk/dotnet/Microsoft.Mxc.Sdk/Native/NativeMethods.g.cs`), gated behind the crate's **`dotnetsdk`** feature (off by default, so the whole-workspace backend build matrix doesn't compile csbindgen). The generated file is **not committed** (gitignored); the C# csproj regenerates it at build time and `scripts/check-dotnet-bindings-codegen.js` runs the codegen in CI and asserts the expected entry points are produced. The C ABI is **not a stable external contract** (native + binding are co-versioned and generated together; see the crate docs). It exposes three surfaces: **run-to-completion** (`mxc_run`), **streaming** (`mxc_spawn` → opaque `MxcSandbox` handle; `mxc_stream_read`/`write`/`flush`, `mxc_sandbox_take_stdin`/`stdout`/`stderr`, `mxc_sandbox_id`/`try_wait`/`wait`/`kill`/`output_metadata_json`/`free`, in `src/streaming.rs`), and the **state-aware lifecycle** (`mxc_state_aware` for the envelope phases + `mxc_state_aware_exec` returning a live streaming handle, in `src/state_aware.rs`). All three `.rs` files are csbindgen inputs in `build.rs`; the `MXC_STATUS_*` space already reserves the state-aware phase codes. - `mxc_pty` is the shared pty bridge used by the LXC backend (`lxc_common::lxc_bindings::attach_run`) so the inner shell sees a real TTY and host stdio is streamed live. (Seatbelt and Bubblewrap no longer use it: they spawn directly and let the child inherit the host's stdio — a TTY when the executor binary runs under a pty — via `SandboxBackend::spawn(StdioMode::Inherit)`.) -- `learning_mode_core` is the **cross-platform learning-mode / captureDenials model + output emitter**: `DeniedResource` (+ `ResourceType`/`AccessType`), `DenialSummary`, the `DenialAnalyzer` decode trait, and `emit` — which writes the on-disk denials deliverable as a **single JSON document** `{ "denials": [...], "summary": {...} }` (`write_document` / `DenialsDocument`) and defines the serializable `DenialsOutputPointer`. It carries no OS-specific code (must not depend on any `backends/*` crate); the Windows ETL decoder implementing `DenialAnalyzer` lives in `backends/learning_mode/windows`. When `processContainer.captureDenials` is set, the BaseContainer runner seals a unique internal ETL temp, decodes it via that backend with bounded event/unique-denial processing, writes the JSON file (caller's `outputPath` with a unique per-run identifier stamped into the stem, e.g. `denials..json`, or a managed temp), deletes the ETL, and returns neutral `wxc_common` output metadata. `wxc-exec` serializes that metadata as the one-line stderr pointer at the CLI boundary; Rust/C#/FFI callers receive it programmatically. Each denial's `resource` field holds the file path or the AppContainer capability name; capability denials resolve their capability SID to a friendly name via `backends/learning_mode/windows`'s `capability_names` (well-known `S-1-15-3-…` SID → policy name; custom hashed SIDs fall back to the SID string). +- `learning_mode_core` is the **cross-platform learning-mode / captureDenials model + output emitter**: `DeniedResource` (+ `ResourceType`/`AccessType`), `DenialSummary`, the `DenialAnalyzer` decode trait, and `emit` — which writes the on-disk denials deliverable as a **single JSON document** `{ "denials": [...], "summary": {...} }` (`write_document` / `DenialsDocument`) and defines the serializable `DenialsOutputPointer`. It carries no OS-specific code (must not depend on any `backends/*` crate); the Windows ETL decoder implementing `DenialAnalyzer` lives in `backends/learning_mode/windows`. When `processContainer.captureDenials` is set, native PSEC/V2 seals and decodes a managed ETL locally, while guarded WPR returns an already process-scoped `AnalysisResult`; both routes write the same canonical JSON through shared output plumbing and return neutral `wxc_common` metadata. Explicit `retainEtl` preserves the sealed trace after a terminal wait on either provider; abandonment discards it. `wxc-exec` serializes the metadata as the one-line stderr pointer at the CLI boundary; Rust/C#/FFI callers receive it programmatically. Each denial's `resource` field holds the file path or the AppContainer capability name; capability denials resolve their capability SID to a friendly name via `backends/learning_mode/windows`'s `capability_names` (well-known `S-1-15-3-…` SID → policy name; custom hashed SIDs fall back to the SID string). - `mxc_build_common` is a build-time helper crate — all Windows binary crates use it in their `build.rs` to embed VersionInfo (ProductName, FileDescription, copyright, version+commit). When adding a new Windows binary crate, add `mxc_build_common` as a build-dependency and call `mxc_build_common::embed_version_info()` from `build.rs` - `nanvix_build_common` is a **build-only** helper crate (never linked into the runtime): it stages NanVix binaries next to the executable and resolves the `NANVIX_BIN` prefetch directory. The `nanvix_binaries`, `wxc`, and `lxc` build scripts consume it as a `[build-dependencies]` entry. Runtime constants it needs (binary/snapshot filenames) stay in `nanvix_common`. Keep build-only file-staging logic here, not in `nanvix_common` (which is a runtime dependency of `nanvix_runner`). - Platform-specific modules use `#[cfg(target_os = "windows")]` / `#[cfg(target_os = "linux")]` From 5e1090c640c3778eb1c214569820a2b1e0d7fd79 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Thu, 13 Aug 2026 15:50:37 -0700 Subject: [PATCH 03/34] Address audit capture review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- docs/learning-mode/capabilities.md | 28 +++++++------ .../common/src/appcontainer_runner.rs | 26 ++++++++++-- .../common/src/base_container_runner.rs | 16 +++++-- .../common/src/guarded_capture.rs | 38 +++++++++++++++++ src/core/wxc/src/audit.rs | 42 ++++++++++++++++++- 5 files changed, 129 insertions(+), 21 deletions(-) diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index 0f70b11cb..3709ebd93 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -243,20 +243,24 @@ file in a protected per-run temporary directory that MXC deletes after analysis. Set `captureDenials.retainEtl` to `true` to preserve the sealed trace for diagnostics after a terminal wait. Both native PSEC/V2 capture and the guarded-WPR fallback honor retention; guarded WPR transfers the sealed trace -back to the unelevated caller after process-scoped analysis. Retention-enabled captures begin under -`%LOCALAPPDATA%\Microsoft\MXC\capture-denials\working` and move to a protected -per-run directory under `capture-denials\retained` only after sealing succeeds. -Abandoning or disposing a process without a terminal wait deletes the internal -trace because no caller can observe its structured path. When retention -succeeds, the structured pointer and in-process metadata include its absolute -`etlPath`: +back to the unelevated caller after process-scoped analysis. Native retention +begins under `%LOCALAPPDATA%\Microsoft\MXC\capture-denials\working` and moves +to a protected per-run directory under `capture-denials\retained` only after +sealing succeeds. Guarded WPR writes the retained ETL beside its unique +denials JSON output, using the same file stem with an `.etl` extension. +Abandoning or disposing a process without a terminal wait deletes or discards +the internal trace because no caller can observe its structured path. When +retention succeeds, the structured pointer and in-process metadata include its +absolute `etlPath`: ```json {"type":"captureDenials","outputPath":"C:\\logs\\denials.4321_0123456789abcdef0123456789abcdef.json","exitCode":0,"totalDenials":2,"deniedResourcesTruncated":false,"etlPath":"C:\\Users\\runneradmin\\AppData\\Local\\Microsoft\\MXC\\capture-denials\\retained\\4321_0123456789abcdef0123456789abcdef\\capture.etl"} ``` -If analysis fails while retention is enabled, MXC preserves the ETL, includes -its path in the returned error, and exposes `captureDenialsError` through -in-process output metadata. ETL traces can contain sensitive resource paths -and identifiers; callers that retain them are responsible for deleting the ETL -and its now-empty per-run parent directory when they are no longer needed. +If native post-seal analysis fails while retention is enabled, MXC preserves +the ETL and exposes its path through `captureDenialsError`. Guarded WPR +transfers the ETL only after process-scoped analysis succeeds; if a later JSON +output step fails, the same error metadata identifies the transferred trace. +ETL traces can contain sensitive resource paths and identifiers; callers that +retain them are responsible for deleting the ETL and, for native managed +retention, its now-empty per-run parent directory when no longer needed. diff --git a/src/backends/appcontainer/common/src/appcontainer_runner.rs b/src/backends/appcontainer/common/src/appcontainer_runner.rs index f447f23dd..28541154d 100644 --- a/src/backends/appcontainer/common/src/appcontainer_runner.rs +++ b/src/backends/appcontainer/common/src/appcontainer_runner.rs @@ -32,7 +32,9 @@ use windows::Win32::System::Threading::{ use windows_core::{PCWSTR, PWSTR}; use crate::capture_output; -use crate::guarded_capture::{GuardedCaptureFactory, GuardedCaptureSession}; +use crate::guarded_capture::{ + transferred_trace_error_metadata, GuardedCaptureFactory, GuardedCaptureSession, +}; use crate::job_object::UiJobObject; use crate::process_mitigation; use wxc_common::error::WxcError; @@ -1709,13 +1711,15 @@ impl AppContainerSandboxProcess { Some(etl_path) => session.stop_analyzed_with_trace(etl_path), None => session.stop_analyzed(), }; + let etl_was_transferred = etl_path.is_some() && capture_result.is_ok(); let capture_result = match capture_result { Ok(analysis) => match output_path { Some(output_path) => { capture_output::write_denials_document(analysis, exit_code, &output_path) .map(|mut metadata| { - metadata.etl_path = - etl_path.map(|path| path.to_string_lossy().into_owned()); + metadata.etl_path = etl_path + .as_ref() + .map(|path| path.to_string_lossy().into_owned()); metadata }) } @@ -1735,7 +1739,14 @@ impl AppContainerSandboxProcess { }); Ok(()) } - Err(error) => Err(error), + Err(error) => { + self.output_metadata = transferred_trace_error_metadata( + &error, + etl_path.as_deref(), + etl_was_transferred, + ); + Err(error) + } } } else { Ok(()) @@ -2283,6 +2294,13 @@ mod tests { .push("stop_analyzed".to_string()); Ok(AnalysisResult::complete(Vec::new())) } + + fn stop_analyzed_with_trace( + &mut self, + _trace_destination: &std::path::Path, + ) -> Result { + unreachable!() + } } #[test] diff --git a/src/backends/appcontainer/common/src/base_container_runner.rs b/src/backends/appcontainer/common/src/base_container_runner.rs index 6afb6b6a0..37ae5d2fb 100644 --- a/src/backends/appcontainer/common/src/base_container_runner.rs +++ b/src/backends/appcontainer/common/src/base_container_runner.rs @@ -43,7 +43,9 @@ use crate::capture_output::{ remove_internal_capture_file, unique_denials_output_path, write_denials_document, write_stderr_line_best_effort, }; -use crate::guarded_capture::{GuardedCaptureFactory, GuardedCaptureSession}; +use crate::guarded_capture::{ + transferred_trace_error_metadata, GuardedCaptureFactory, GuardedCaptureSession, +}; use crate::job_object::UiJobObject; use crate::launch_diagnostics::{ diagnose_create_process_failure, diagnose_environment_not_supported, diagnose_process_exit, @@ -2705,6 +2707,7 @@ impl BaseContainerSandboxProcess { Some(etl_path) => session.stop_analyzed_with_trace(etl_path), None => session.stop_analyzed(), }; + let etl_was_transferred = etl_path.is_some() && stop_result.is_ok(); let result = stop_result .map_err(|error| { std::io::Error::other(format!( @@ -2716,8 +2719,9 @@ impl BaseContainerSandboxProcess { std::io::Error::other("captureDenials output path was not initialized") })?; write_denials_document(analysis, exit_code, &output_path).map(|mut metadata| { - metadata.etl_path = - etl_path.map(|path| path.to_string_lossy().into_owned()); + metadata.etl_path = etl_path + .as_ref() + .map(|path| path.to_string_lossy().into_owned()); metadata }) }); @@ -2726,6 +2730,12 @@ impl BaseContainerSandboxProcess { capture_denials: Some(metadata.clone()), capture_denials_error: None, }); + } else if let Err(error) = &result { + self.output_metadata = transferred_trace_error_metadata( + error, + etl_path.as_deref(), + etl_was_transferred, + ); } result.map(Some) } else { diff --git a/src/backends/appcontainer/common/src/guarded_capture.rs b/src/backends/appcontainer/common/src/guarded_capture.rs index 8b17f681d..7aa2a85a1 100644 --- a/src/backends/appcontainer/common/src/guarded_capture.rs +++ b/src/backends/appcontainer/common/src/guarded_capture.rs @@ -24,6 +24,7 @@ //! guarded capture silently. use learning_mode_core::AnalysisResult; +use wxc_common::models::{CaptureDenialsErrorOutput, SandboxOutputMetadata}; /// Error returned when an injected guarded-capture implementation cannot /// transfer retained ETL requested by the caller. @@ -113,6 +114,21 @@ pub(crate) fn release_after_termination_failure( session.discard() } +pub(crate) fn transferred_trace_error_metadata( + error: &std::io::Error, + etl_path: Option<&std::path::Path>, + etl_was_transferred: bool, +) -> Option { + let etl_path = etl_path.filter(|_| etl_was_transferred)?; + Some(SandboxOutputMetadata { + capture_denials: None, + capture_denials_error: Some(CaptureDenialsErrorOutput { + message: error.to_string(), + etl_path: etl_path.to_string_lossy().into_owned(), + }), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -226,6 +242,13 @@ mod tests { fn stop_analyzed(&mut self) -> Result { unreachable!() } + + fn stop_analyzed_with_trace( + &mut self, + _trace_destination: &std::path::Path, + ) -> Result { + unreachable!() + } } let (entered_tx, entered_rx) = std::sync::mpsc::channel(); @@ -250,4 +273,19 @@ mod tests { ); thread.join().unwrap(); } + + #[test] + fn transferred_trace_failure_metadata_requires_a_persisted_path() { + let error = std::io::Error::other("JSON write failed"); + let path = std::path::Path::new(r"C:\capture\denials.etl"); + + assert!( + transferred_trace_error_metadata(&error, Some(path), false).is_none(), + "an untransferred trace must not be advertised" + ); + let metadata = transferred_trace_error_metadata(&error, Some(path), true).unwrap(); + let failure = metadata.capture_denials_error.unwrap(); + assert_eq!(failure.message, "JSON write failed"); + assert_eq!(failure.etl_path, path.to_string_lossy()); + } } diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index 6d7f0b68d..f2d6dfb06 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -100,6 +100,8 @@ pub fn finalize( ensure_destination_available(&source_denials, &final_denials)?; move_new_file(&source_etl, &final_etl)?; move_new_file(&source_denials, &final_denials)?; + capture.output_path = final_denials.to_string_lossy().into_owned(); + capture.etl_path = Some(final_etl.to_string_lossy().into_owned()); remove_empty_managed_directory(source_etl.parent(), &context.log_dir)?; plm::stop::postprocess_denials( @@ -116,8 +118,6 @@ pub fn finalize( ) .map_err(|error| format!("failed to generate audit compatibility artifacts: {error:#}"))?; - capture.output_path = final_denials.to_string_lossy().into_owned(); - capture.etl_path = Some(final_etl.to_string_lossy().into_owned()); Ok(()) } @@ -276,6 +276,44 @@ mod tests { ); } + #[test] + fn finalize_updates_metadata_before_postprocessing_failure() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&log_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_etl = log_dir.join("denials.unique.etl"); + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, false)); + std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 0); + let context = AuditContext { + log_dir: log_dir.clone(), + config_path: Some(directory.path().join("missing-policy.json")), + }; + + let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); + + assert!(error.contains("audit compatibility artifacts")); + let capture = response + .output_metadata + .as_ref() + .unwrap() + .capture_denials + .as_ref() + .unwrap(); + assert_eq!( + Path::new(&capture.output_path), + log_dir.join("denials.json") + ); + assert_eq!( + capture.etl_path.as_deref().map(Path::new), + Some(log_dir.join("trace.etl").as_path()) + ); + assert!(log_dir.join("denials.json").is_file()); + assert!(log_dir.join("trace.etl").is_file()); + } + #[test] fn finalize_rejects_missing_or_failed_capture_metadata() { let directory = tempfile::tempdir().unwrap(); From 20a822f39fe8666088403722fe75304c137bb48e Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 10:34:59 -0700 Subject: [PATCH 04/34] Filter guarded WPR traces by process Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- .github/copilot-instructions.md | 6 +- docs/learning-mode/capabilities.md | 28 +- .../learning_mode/windows/src/etl_decode.rs | 117 +++++++- .../learning_mode/windows/src/etl_filter.rs | 279 ++++++++++++++++++ src/backends/learning_mode/windows/src/lib.rs | 4 + src/host/plm/src/elevated.rs | 36 ++- src/host/plm/src/secure_scratch.rs | 42 ++- 7 files changed, 476 insertions(+), 36 deletions(-) create mode 100644 src/backends/learning_mode/windows/src/etl_filter.rs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 2a632beed..4f6fc36bf 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -205,15 +205,15 @@ The workspace is organized into six top-level directories under `src/`: - `wxc_common` is the **cross-platform foundation**: config parsing, models, errors, logger, `ScriptRunner` / `StatefulSandboxBackend` traits, state-aware dispatch helpers, validators, ids, ui-policy, encoding. Plus a few thin Windows API helpers shared by host tools and backends (`process_util`, `string_util`, `filesystem_dacl`, `diagnostic`). It must not depend on any `backends/*` crate. - Each Windows containment backend lives in its own `backends/*/common` crate (e.g. `appcontainer_common`, `windows_sandbox_common`, `isolation_session_common`, `hyperlight_common`, `nanvix_runner`). Backend crates depend on `wxc_common`; there are no cross-edges between backend crates. Windows Sandbox additionally has `windows_sandbox_lifecycle`, which owns the one-shot and state-aware runners and depends on `windows_sandbox_common` for the wire protocol, plus separate daemon and guest binaries. - `learning_mode_core` is the cross-platform learning-mode denial model and output layer. It owns denial types, summaries, analyzer abstractions, plain-JSON document emission, and the serializable output-pointer type, and must not depend on any `backends/*` crate. -- `learning_mode_windows` (`backends/learning_mode/windows`) is a Windows-only backend support crate for the AppInfo-brokered Learning Mode APIs in `processmodel.dll`. It runtime-resolves the Learning Mode trace and process security-environment exports, owns their typed handle/lifecycle wrappers, decodes sealed ETL traces through `learning_mode_core`, and depends on `wxc_common` plus `learning_mode_core`; runner integration consumes it from the AppContainer backend layer. The trace contract is `HRESULT Start` + retryable `HRESULT Stop` + infallible `Close`: `Stop` never consumes the trace handle, and every started trace must be closed exactly once (closing without stopping is the early-exit discard path). The process security-environment contract is `HRESULT Create` + infallible by-value `Close` and consumes a PSEC 1.0 FlatBuffer, not the legacy SBOX buffer; generated PSEC bindings live in `core/generated/process_security_environment_specification`. -- `plm` (`host/plm`) is the Windows-only legacy WPR Learning Mode helper. Public `plm.exe` is `asInvoker`: ETL analysis and every caller-selected file path stay under the caller token. It self-elevates only hidden fixed WPR operations; the retained elevated guardian accepts authenticated attach and stop/analyze/discard controls over unique local PID-checked named pipes, uses the compiled-in profile from protected fixed-volume ProgramData scratch, and returns bounded process-scoped analysis plus, when explicitly retained, the sealed trace bytes to the unelevated parent. Successful authenticated stop/discard disarms the child before releasing the PLM singleton. Owner death, pipe break, or another uncertain control failure preserves the recovery marker and deliberately leaves WPR untouched for administrator recovery. +- `learning_mode_windows` (`backends/learning_mode/windows`) is a Windows-only backend support crate for the AppInfo-brokered Learning Mode APIs in `processmodel.dll`. It runtime-resolves the Learning Mode trace and process security-environment exports, owns their typed handle/lifecycle wrappers, decodes sealed ETL traces through `learning_mode_core`, and process-scopes guarded WPR retention with Windows Trace Relogger using exact job-attested PID/creation/exit `FILETIME` ranges. It depends on `wxc_common` plus `learning_mode_core`; runner integration consumes it from the AppContainer backend layer. The trace contract is `HRESULT Start` + retryable `HRESULT Stop` + infallible `Close`: `Stop` never consumes the trace handle, and every started trace must be closed exactly once (closing without stopping is the early-exit discard path). The process security-environment contract is `HRESULT Create` + infallible by-value `Close` and consumes a PSEC 1.0 FlatBuffer, not the legacy SBOX buffer; generated PSEC bindings live in `core/generated/process_security_environment_specification`. +- `plm` (`host/plm`) is the Windows-only legacy WPR Learning Mode helper. Public `plm.exe` is `asInvoker`: ETL analysis and every caller-selected file path stay under the caller token. It self-elevates only hidden fixed WPR operations; the retained elevated guardian accepts authenticated attach and stop/analyze/discard controls over unique local PID-checked named pipes and uses the compiled-in profile from protected fixed-volume ProgramData scratch. WPR's host-wide source ETL never crosses the privilege boundary: after terminal job tracking, the guardian relogs a separate ETL containing only supported Learning Mode events from exact handle-attested process generations, analyzes that filtered ETL, and returns its bytes only when explicitly retained. Relogging failure transfers no trace. Successful authenticated stop/discard disarms the child before releasing the PLM singleton. Owner death, pipe break, or another uncertain control failure preserves the recovery marker and deliberately leaves WPR untouched for administrator recovery. - `wxc`, `lxc`, and `mxc_darwin` are thin binary crates (`wxc-exec` / `lxc-exec` / `mxc-exec-mac`) that wire up CLI args (`clap`), load/validate config, handle maintenance modes (`--probe`, `--delete`, `--setup-*`, `--audit`), and **delegate all backend dispatch to `mxc_engine`**. They contain no `match request.containment` of their own. `wxc-exec` additionally owns the Windows Ctrl-C / DACL-cleanup / telemetry orchestration around the engine call. Its `--audit` compatibility workflow synthesizes allow-mode `captureDenials` with ETL retention, then generates policy-authoring artifacts from the canonical denials document returned by the selected engine backend. - `mxc_engine` is the **single execution engine** — the one home for "given an `ExecutionRequest`, run it". It owns: run-to-completion backend selection (`run` / `resolve_runner`, covering **all** backends, incl. the Windows ProcessContainer BaseContainer/AppContainer BFS/DACL fallback tiers via `appcontainer_common::dispatcher::dispatch_with_fallback`, and every experimental backend, feature-gated); streaming (`spawn` → `Box`); state-aware lifecycle dispatch (`run_state_aware`, including Windows Sandbox and IsolationSession); host probing (`platform_support` / `PlatformSupport`); and config building (`build_request` / `build_request_with_containment`, `SandboxPolicy` + sections, `available_tools_policy`/`user_profile_policy`/`temporary_files_policy`). It depends on the backend crates (cfg-split: appcontainer/windows_sandbox lifecycle/isolation_session/wslc/nanvix on Windows, bubblewrap/lxc/nanvix on Linux, seatbelt on macOS) so it can't live in `wxc_common`. Both the executor binaries and `mxc-sdk` call into it. `ResolvedRunner` carries the boxed runner plus (Windows only) the optional `DaclManager` guard, so `wxc-exec` can park the guard for its signal handler. - `mxc-sdk` is the **public Rust SDK** — a thin facade over `mxc_engine`. Build a `SandboxRequest` with `build_request`, then either `run(request)` (run-to-completion; returns an `Output` with the `WaitOutcome`, captured `stdout`/`stderr`, warnings, and optional structured output metadata) or `spawn_sandbox(request)` (returns a `Sandbox` handle for live bidirectional stdio — `take_stdin`/`take_stdout`/`take_stderr`, `kill()`, `wait()` returning a `WaitOutcome` (`Exited(i32)` / `TimedOut`) as `io::Result`, `output_metadata()` after terminal completion, or `wait_with_output()`). It re-exports the engine's config-building surface (`build_request`, `build_request_with_containment` + `Containment`/`WslcSection`, `mxc_sdk::policy::{SandboxPolicy sections}`, discovery helpers) and `platform_support`; `mod sandbox` (wrapping the engine's `SandboxProcess` in `Sandbox`) is its only local module. No pty is ever allocated. Streaming supports Seatbelt (macOS), Bubblewrap (Linux), Windows ProcessContainer (AppContainer + BaseContainer), and WSLC (Windows, experimental — needs the crate's `wslc` feature plus `SandboxRequest::set_experimental(true)`; no stdin and `id() == 0`, since the WSLC SDK exposes neither); other backends return `ErrorCode::UnsupportedContainment`. - The lower-level execution surface lives in `wxc_common::sandbox_process`: the `SandboxBackend` trait (`validate` + `spawn(request, logger, StdioMode) -> Box` + a `diagnose_exit` hook) and the generic `Runner` adapter that bridges any `SandboxBackend` to the run-to-completion `ScriptRunner` (via `spawn(StdioMode::Inherit)` then `wait()`). `SandboxProcess::output_metadata()` carries backend-produced structured outputs after terminal teardown without writing to process-global stdio. `StdioMode::Pipes` hands the caller live stdin/stdout/stderr (what the `mxc-sdk` streaming path uses); `StdioMode::Inherit` lets the child inherit the host's stdio (what the executor binaries use, preserving the TTY under a pty). `SandboxBackend` is implemented for Seatbelt, Bubblewrap, Windows ProcessContainer, and WSLC (on `wslc_common::WSLContainerRunner` itself, which shares one container lifecycle — `start_container` — between its streaming `SandboxBackend` and run-to-completion `ScriptRunner` impls, differing only in where the WSLC SDK's output callbacks write). - `mxc_ffi` (`ffi/mxc_ffi`, `crate-type = ["cdylib", "staticlib", "lib"]`) is a flat, panic-safe **C ABI over `mxc-sdk`** for language bindings. `mxc_run(policyJson, command, out)` runs a sandbox to completion, filling a `#[repr(C)] MxcRunResult` (status + exit_code + timed_out + owned stdout/stderr/error/output-metadata C strings); every entry point is `catch_unwind`-wrapped so a panic becomes a status code, never an unwind. Its `build.rs` runs **csbindgen** to generate the C# P/Invoke (`sdk/dotnet/Microsoft.Mxc.Sdk/Native/NativeMethods.g.cs`), gated behind the crate's **`dotnetsdk`** feature (off by default, so the whole-workspace backend build matrix doesn't compile csbindgen). The generated file is **not committed** (gitignored); the C# csproj regenerates it at build time and `scripts/check-dotnet-bindings-codegen.js` runs the codegen in CI and asserts the expected entry points are produced. The C ABI is **not a stable external contract** (native + binding are co-versioned and generated together; see the crate docs). It exposes three surfaces: **run-to-completion** (`mxc_run`), **streaming** (`mxc_spawn` → opaque `MxcSandbox` handle; `mxc_stream_read`/`write`/`flush`, `mxc_sandbox_take_stdin`/`stdout`/`stderr`, `mxc_sandbox_id`/`try_wait`/`wait`/`kill`/`output_metadata_json`/`free`, in `src/streaming.rs`), and the **state-aware lifecycle** (`mxc_state_aware` for the envelope phases + `mxc_state_aware_exec` returning a live streaming handle, in `src/state_aware.rs`). All three `.rs` files are csbindgen inputs in `build.rs`; the `MXC_STATUS_*` space already reserves the state-aware phase codes. - `mxc_pty` is the shared pty bridge used by the LXC backend (`lxc_common::lxc_bindings::attach_run`) so the inner shell sees a real TTY and host stdio is streamed live. (Seatbelt and Bubblewrap no longer use it: they spawn directly and let the child inherit the host's stdio — a TTY when the executor binary runs under a pty — via `SandboxBackend::spawn(StdioMode::Inherit)`.) -- `learning_mode_core` is the **cross-platform learning-mode / captureDenials model + output emitter**: `DeniedResource` (+ `ResourceType`/`AccessType`), `DenialSummary`, the `DenialAnalyzer` decode trait, and `emit` — which writes the on-disk denials deliverable as a **single JSON document** `{ "denials": [...], "summary": {...} }` (`write_document` / `DenialsDocument`) and defines the serializable `DenialsOutputPointer`. It carries no OS-specific code (must not depend on any `backends/*` crate); the Windows ETL decoder implementing `DenialAnalyzer` lives in `backends/learning_mode/windows`. When `processContainer.captureDenials` is set, native PSEC/V2 seals and decodes a managed ETL locally, while guarded WPR returns an already process-scoped `AnalysisResult`; both routes write the same canonical JSON through shared output plumbing and return neutral `wxc_common` metadata. Explicit `retainEtl` preserves the sealed trace after a terminal wait on either provider; abandonment discards it. `wxc-exec` serializes the metadata as the one-line stderr pointer at the CLI boundary; Rust/C#/FFI callers receive it programmatically. Each denial's `resource` field holds the file path or the AppContainer capability name; capability denials resolve their capability SID to a friendly name via `backends/learning_mode/windows`'s `capability_names` (well-known `S-1-15-3-…` SID → policy name; custom hashed SIDs fall back to the SID string). +- `learning_mode_core` is the **cross-platform learning-mode / captureDenials model + output emitter**: `DeniedResource` (+ `ResourceType`/`AccessType`), `DenialSummary`, the `DenialAnalyzer` decode trait, and `emit` — which writes the on-disk denials deliverable as a **single JSON document** `{ "denials": [...], "summary": {...} }` (`write_document` / `DenialsDocument`) and defines the serializable `DenialsOutputPointer`. It carries no OS-specific code (must not depend on any `backends/*` crate); the Windows ETL decoder implementing `DenialAnalyzer` lives in `backends/learning_mode/windows`. When `processContainer.captureDenials` is set, native PSEC/V2 seals and decodes a managed ETL locally, while guarded WPR relogs its host-wide source into a process-scoped ETL before analysis; both routes write the same canonical JSON through shared output plumbing and return neutral `wxc_common` metadata. Explicit `retainEtl` preserves the native sealed trace or the guarded process-scoped relogged trace after a terminal wait; abandonment discards it. `wxc-exec` serializes the metadata as the one-line stderr pointer at the CLI boundary; Rust/C#/FFI callers receive it programmatically. Each denial's `resource` field holds the file path or the AppContainer capability name; capability denials resolve their capability SID to a friendly name via `backends/learning_mode/windows`'s `capability_names` (well-known `S-1-15-3-…` SID → policy name; custom hashed SIDs fall back to the SID string). - `mxc_build_common` is a build-time helper crate — all Windows binary crates use it in their `build.rs` to embed VersionInfo (ProductName, FileDescription, copyright, version+commit). When adding a new Windows binary crate, add `mxc_build_common` as a build-dependency and call `mxc_build_common::embed_version_info()` from `build.rs` - `nanvix_build_common` is a **build-only** helper crate (never linked into the runtime): it stages NanVix binaries next to the executable and resolves the `NANVIX_BIN` prefetch directory. The `nanvix_binaries`, `wxc`, and `lxc` build scripts consume it as a `[build-dependencies]` entry. Runtime constants it needs (binary/snapshot filenames) stay in `nanvix_common`. Keep build-only file-staging logic here, not in `nanvix_common` (which is a runtime dependency of `nanvix_runner`). - Platform-specific modules use `#[cfg(target_os = "windows")]` / `#[cfg(target_os = "linux")]` diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index 3709ebd93..f34160f48 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -242,12 +242,23 @@ By default, the intermediate ETW `.etl` trace is an internal, runner-managed file in a protected per-run temporary directory that MXC deletes after analysis. Set `captureDenials.retainEtl` to `true` to preserve the sealed trace for diagnostics after a terminal wait. Both native PSEC/V2 capture and the -guarded-WPR fallback honor retention; guarded WPR transfers the sealed trace -back to the unelevated caller after process-scoped analysis. Native retention -begins under `%LOCALAPPDATA%\Microsoft\MXC\capture-denials\working` and moves -to a protected per-run directory under `capture-denials\retained` only after -sealing succeeds. Guarded WPR writes the retained ETL beside its unique -denials JSON output, using the same file stem with an `.etl` extension. +guarded-WPR fallback honor retention. Native retention begins under +`%LOCALAPPDATA%\Microsoft\MXC\capture-denials\working` and moves to a protected +per-run directory under `capture-denials\retained` only after sealing +succeeds. + +WPR's source ETL is host-wide, so the elevated guarded-WPR helper never +transfers that file across the privilege boundary. After the sandbox process +tree terminates, the helper uses the retained, job-attested process handles and +their exact PID/creation/exit `FILETIME` ranges to relog a second ETL. The +retained ETL contains only supported Learning Mode events whose event header +falls inside one of those attested process generations. Guarded analysis and +retention both consume that same filtered ETL; filtering failure transfers no +trace. The host-wide source remains in protected elevated scratch and is +deleted with that scratch. The unelevated caller writes the filtered retained +ETL beside its unique denials JSON output, using the same file stem with an +`.etl` extension. + Abandoning or disposing a process without a terminal wait deletes or discards the internal trace because no caller can observe its structured path. When retention succeeds, the structured pointer and in-process metadata include its @@ -259,8 +270,9 @@ absolute `etlPath`: If native post-seal analysis fails while retention is enabled, MXC preserves the ETL and exposes its path through `captureDenialsError`. Guarded WPR -transfers the ETL only after process-scoped analysis succeeds; if a later JSON -output step fails, the same error metadata identifies the transferred trace. +transfers the filtered ETL only after process-scoped analysis succeeds; if a +later JSON output step fails, the same error metadata identifies the +transferred trace. ETL traces can contain sensitive resource paths and identifiers; callers that retain them are responsible for deleting the ETL and, for native managed retention, its now-empty per-run parent directory when no longer needed. diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index bf3715357..1ea867581 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -58,6 +58,7 @@ struct CollectedEvent { enum CollectionMode { Analyze, Raw, + SelectForRelogging, } type RawEventVisitor<'a> = dyn FnMut(&DecodedEventParts) -> std::io::Result<()> + 'a; @@ -69,12 +70,17 @@ struct LifetimeRange { } #[derive(Debug, Default)] -struct ProcessLifetimeIndex { +pub(crate) struct ProcessLifetimeIndex { ranges_by_pid: HashMap>, } +pub(crate) struct RelogSelection { + pub(crate) selected_event_indices: Vec, + pub(crate) total_event_count: usize, +} + impl ProcessLifetimeIndex { - fn new(process_lifetimes: &[ProcessLifetime]) -> Self { + pub(crate) fn new(process_lifetimes: &[ProcessLifetime]) -> Self { let mut ranges_by_pid = HashMap::>::with_capacity(process_lifetimes.len()); for lifetime in process_lifetimes { @@ -105,7 +111,7 @@ impl ProcessLifetimeIndex { Self { ranges_by_pid } } - fn contains(&self, pid: u32, filetime: u64) -> bool { + pub(crate) fn contains(&self, pid: u32, filetime: u64) -> bool { let Some(ranges) = self.ranges_by_pid.get(&pid) else { return false; }; @@ -123,6 +129,8 @@ struct Accumulator<'visitor> { seen: HashSet<(String, learning_mode_core::AccessType)>, truncated: bool, raw_visitor: Option<&'visitor mut RawEventVisitor<'visitor>>, + relog_selected_event_indices: Vec, + relog_event_count: usize, raw_event_count: usize, processed_event_count: usize, processing_limit_reached: bool, @@ -141,6 +149,8 @@ impl<'visitor> Accumulator<'visitor> { seen: HashSet::new(), truncated: false, raw_visitor: None, + relog_selected_event_indices: Vec::new(), + relog_event_count: 0, raw_event_count: 0, processed_event_count: 0, processing_limit_reached: false, @@ -166,6 +176,28 @@ impl<'visitor> Accumulator<'visitor> { seen: HashSet::new(), truncated: false, raw_visitor: Some(visitor), + relog_selected_event_indices: Vec::new(), + relog_event_count: 0, + raw_event_count: 0, + processed_event_count: 0, + processing_limit_reached: false, + stop_requested: false, + decode_error: None, + panic_payload: None, + schema_cache: tdh_decode::EventSchemaCache::default(), + } + } + + fn select_for_relogging(process_lifetimes: &[ProcessLifetime]) -> Self { + Self { + mode: CollectionMode::SelectForRelogging, + process_lifetimes: Some(ProcessLifetimeIndex::new(process_lifetimes)), + denials: Vec::new(), + seen: HashSet::new(), + truncated: false, + raw_visitor: None, + relog_selected_event_indices: Vec::new(), + relog_event_count: 0, raw_event_count: 0, processed_event_count: 0, processing_limit_reached: false, @@ -395,6 +427,25 @@ pub fn visit_raw_events( Ok(accumulator.raw_event_count) } +/// Builds a bounded decision vector for supported Learning Mode events in +/// source order. `ProcessTrace` normalizes each timestamp to FILETIME before +/// the exact process-generation test, so Trace Relogger can replay these +/// decisions without comparing its raw trace-clock timestamps. +pub(crate) fn select_learning_mode_events_for_relogging( + source_path: &Path, + process_lifetimes: &[ProcessLifetime], +) -> Result { + let mut accumulator = Accumulator::select_for_relogging(process_lifetimes); + process_trace_file(source_path, &mut accumulator)?; + if let Some(error) = accumulator.decode_error { + return Err(AnalyzeError::Decode(error)); + } + Ok(RelogSelection { + selected_event_indices: accumulator.relog_selected_event_indices, + total_event_count: accumulator.relog_event_count, + }) +} + /// De-duplicates raw denials by `(user-visible resource, accessType)`, /// normalising case-insensitive Windows file/registry identifiers while /// preserving first-seen display spelling and order. @@ -543,6 +594,34 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let provider = header.ProviderId; let event_id = header.EventDescriptor.Id; + if matches!(acc.mode, CollectionMode::SelectForRelogging) { + if !is_learning_mode_event(provider, event_id) { + return; + } + let event_index = acc.relog_event_count; + let Some(next_event_count) = acc.relog_event_count.checked_add(1) else { + acc.decode_error = + Some("trace contained too many Learning Mode events to index".to_string()); + acc.stop_requested = true; + return; + }; + acc.relog_event_count = next_event_count; + let Some(filetime) = normalized_filetime(header.TimeStamp, acc) else { + return; + }; + if acc.in_analysis_scope(header.ProcessId, filetime) { + if acc.relog_selected_event_indices.len() >= MAX_PROCESSED_EVENTS { + acc.decode_error = Some(format!( + "trace exceeded the {MAX_PROCESSED_EVENTS}-event process-scoped relogging limit" + )); + acc.stop_requested = true; + return; + } + acc.relog_selected_event_indices.push(event_index); + } + return; + } + // Establish scope BEFORE charging the event against the shared processing // budget. Provider, event id, PID and timestamp all live in the event // header, so the scope test needs no (comparatively expensive) TDH decode. @@ -651,6 +730,38 @@ mod tests { assert!(!index.contains(7, 10)); } + #[test] + fn relog_selection_tracks_supported_event_ordinals_and_exact_lifetimes() { + let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + + let mut visit = |provider, event_id, pid, filetime| { + // SAFETY: this zeroed record is used only by selection mode, which + // reads the initialized POD header fields below and never decodes + // payload pointers. + let mut record: EVENT_RECORD = unsafe { core::mem::zeroed() }; + record.EventHeader.ProviderId = provider; + record.EventHeader.EventDescriptor.Id = event_id; + record.EventHeader.ProcessId = pid; + record.EventHeader.TimeStamp = filetime; + // SAFETY: `record` remains live for this synchronous call. + unsafe { process_event_record(&mut record, &mut accumulator) }; + }; + + visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 14, 42, 100); + visit(windows::core::GUID::from_u128(1), 14, 42, 150); + visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 27, 42, 201); + visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 28, 42, 200); + visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 14, 43, 150); + + assert_eq!(accumulator.relog_event_count, 4); + assert_eq!(accumulator.relog_selected_event_indices, [0, 2]); + assert!(accumulator.decode_error.is_none()); + } + #[test] fn raw_visitor_panic_is_captured_inside_callback_state() { let mut visitor = diff --git a/src/backends/learning_mode/windows/src/etl_filter.rs b/src/backends/learning_mode/windows/src/etl_filter.rs new file mode 100644 index 000000000..c964bc204 --- /dev/null +++ b/src/backends/learning_mode/windows/src/etl_filter.rs @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Process-scoped ETL rewriting for guarded WPR captures. + +use std::os::windows::ffi::OsStrExt; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use learning_mode_core::{AnalyzeError, ProcessLifetime}; +use windows::core::{implement, Ref, BSTR}; +use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, +}; +use windows::Win32::System::Diagnostics::Etw::{ + CLSID_TraceRelogger, ITraceEvent, ITraceEventCallback, ITraceEventCallback_Impl, ITraceRelogger, +}; + +use crate::etl_decode::select_learning_mode_events_for_relogging; +use crate::extractors::is_learning_mode_event; +use crate::process_lifetime::{attested_process_lifetimes, JobMembershipSnapshot}; + +const RPC_E_CHANGED_MODE: u32 = 0x8001_0106; + +struct ComApartment { + owns_init: bool, +} + +impl ComApartment { + fn new() -> Result { + // SAFETY: the matching CoUninitialize runs on this thread when this + // call acquires an initialization reference. + let result = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + if result.is_ok() { + Ok(Self { owns_init: true }) + } else if result.0 as u32 == RPC_E_CHANGED_MODE { + Ok(Self { owns_init: false }) + } else { + Err(decode_error("CoInitializeEx", result)) + } + } +} + +impl Drop for ComApartment { + fn drop(&mut self) { + if self.owns_init { + // SAFETY: balances the successful CoInitializeEx call on this + // same thread. + unsafe { CoUninitialize() }; + } + } +} + +#[implement(ITraceEventCallback)] +struct ProcessScopedTraceFilter { + selected_event_indices: Vec, + event_cursor: Arc, + selected_event_cursor: Arc, +} + +impl ITraceEventCallback_Impl for ProcessScopedTraceFilter_Impl { + fn OnBeginProcessTrace( + &self, + header_event: Ref, + relogger: Ref, + ) -> windows::core::Result<()> { + let header_event = header_event.ok()?; + let relogger = relogger.ok()?; + // SAFETY: both interfaces are valid for this synchronous callback. + // The trace header is required for a standalone output ETL. + unsafe { relogger.Inject(header_event)? }; + Ok(()) + } + + fn OnFinalizeProcessTrace(&self, _relogger: Ref) -> windows::core::Result<()> { + Ok(()) + } + + fn OnEvent( + &self, + event: Ref, + relogger: Ref, + ) -> windows::core::Result<()> { + let event = event.ok()?; + let relogger = relogger.ok()?; + // SAFETY: Trace Relogger owns the event and keeps its EVENT_RECORD + // valid for the duration of this callback. + let record = unsafe { event.GetEventRecord()? }; + let Some(record) = (unsafe { record.as_ref() }) else { + return Err(windows::core::Error::from_hresult( + windows::Win32::Foundation::E_POINTER, + )); + }; + let header = &record.EventHeader; + if !is_learning_mode_event(header.ProviderId, header.EventDescriptor.Id) { + return Ok(()); + } + let event_index = self.event_cursor.fetch_add(1, Ordering::Relaxed); + let selected_index = self.selected_event_cursor.load(Ordering::Relaxed); + if self.selected_event_indices.get(selected_index) == Some(&event_index) { + // SAFETY: both interfaces are valid for this synchronous callback, + // and Inject clones the event into the output trace. + unsafe { relogger.Inject(event)? }; + self.selected_event_cursor.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } +} + +/// Rewrites a host-wide guarded WPR capture into a process-scoped Learning +/// Mode ETL. The destination is removed if relogging does not complete. +pub fn filter_trace_for_job_membership( + source: &Path, + destination: &Path, + membership: &JobMembershipSnapshot, +) -> Result<(), AnalyzeError> { + let process_lifetimes = attested_process_lifetimes(membership)?; + filter_trace_for_process_lifetimes(source, destination, &process_lifetimes) +} + +fn filter_trace_for_process_lifetimes( + source: &Path, + destination: &Path, + process_lifetimes: &[ProcessLifetime], +) -> Result<(), AnalyzeError> { + if source == destination { + return Err(AnalyzeError::Decode( + "filtered ETL destination must differ from its source".to_string(), + )); + } + if destination + .try_exists() + .map_err(|error| AnalyzeError::Open { + path: destination.display().to_string(), + source: error, + })? + { + return Err(AnalyzeError::Decode(format!( + "filtered ETL destination '{}' already exists", + destination.display() + ))); + } + + let result = relog_trace(source, destination, process_lifetimes); + if result.is_err() { + let _ = std::fs::remove_file(destination); + } + result +} + +fn relog_trace( + source: &Path, + destination: &Path, + process_lifetimes: &[ProcessLifetime], +) -> Result<(), AnalyzeError> { + let selection = select_learning_mode_events_for_relogging(source, process_lifetimes)?; + let expected_event_count = selection.total_event_count; + let expected_selected_event_count = selection.selected_event_indices.len(); + let _apartment = ComApartment::new()?; + // SAFETY: COM is initialized on this thread, the in-proc class ID is + // fixed, and the returned interface remains apartment-local. + let relogger: ITraceRelogger = unsafe { + CoCreateInstance(&CLSID_TraceRelogger, None, CLSCTX_INPROC_SERVER) + .map_err(|error| windows_error("CoCreateInstance(CLSID_TraceRelogger)", error))? + }; + + let source = path_bstr(source); + let destination_bstr = path_bstr(destination); + // SAFETY: the BSTRs remain valid for each synchronous COM call. + unsafe { + relogger + .AddLogfileTraceStream(&source, std::ptr::null()) + .map_err(|error| windows_error("ITraceRelogger::AddLogfileTraceStream", error))?; + relogger + .SetOutputFilename(&destination_bstr) + .map_err(|error| windows_error("ITraceRelogger::SetOutputFilename", error))?; + } + + let event_cursor = Arc::new(AtomicUsize::new(0)); + let selected_event_cursor = Arc::new(AtomicUsize::new(0)); + let callback: ITraceEventCallback = ProcessScopedTraceFilter { + selected_event_indices: selection.selected_event_indices, + event_cursor: Arc::clone(&event_cursor), + selected_event_cursor: Arc::clone(&selected_event_cursor), + } + .into(); + // SAFETY: callback and relogger stay alive until ProcessTrace returns. + unsafe { + relogger + .RegisterCallback(&callback) + .map_err(|error| windows_error("ITraceRelogger::RegisterCallback", error))?; + relogger + .ProcessTrace() + .map_err(|error| windows_error("ITraceRelogger::ProcessTrace", error))?; + } + let consumed_event_count = event_cursor.load(Ordering::Relaxed); + if consumed_event_count != expected_event_count { + return Err(AnalyzeError::Decode(format!( + "Trace Relogger observed {consumed_event_count} supported Learning Mode events, but ProcessTrace selected {expected_event_count}" + ))); + } + let consumed_selected_event_count = selected_event_cursor.load(Ordering::Relaxed); + if consumed_selected_event_count != expected_selected_event_count { + return Err(AnalyzeError::Decode(format!( + "Trace Relogger injected {consumed_selected_event_count} process-scoped Learning Mode events, but ProcessTrace selected {expected_selected_event_count}" + ))); + } + + let metadata = std::fs::metadata(destination).map_err(|source| AnalyzeError::Open { + path: destination.display().to_string(), + source, + })?; + if !metadata.is_file() || metadata.len() == 0 { + return Err(AnalyzeError::Decode(format!( + "Trace Relogger did not produce a regular non-empty ETL at '{}'", + destination.display() + ))); + } + Ok(()) +} + +fn path_bstr(path: &Path) -> BSTR { + BSTR::from_wide(&path.as_os_str().encode_wide().collect::>()) +} + +fn decode_error(operation: &str, result: windows::core::HRESULT) -> AnalyzeError { + AnalyzeError::Decode(format!( + "{operation} failed while filtering guarded WPR trace (HRESULT = 0x{:08X})", + result.0 as u32 + )) +} + +fn windows_error(operation: &str, error: windows::core::Error) -> AnalyzeError { + AnalyzeError::Decode(format!( + "{operation} failed while filtering guarded WPR trace: {error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + const START: u64 = 100; + const END: u64 = 200; + + #[test] + fn refuses_to_overwrite_existing_destination() { + let unique = format!( + "mxc-etl-filter-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let directory = std::env::temp_dir().join(unique); + std::fs::create_dir(&directory).unwrap(); + let source = directory.join("source.etl"); + let destination = directory.join("filtered.etl"); + std::fs::write(&destination, b"sentinel").unwrap(); + + let error = filter_trace_for_process_lifetimes( + &source, + &destination, + &[ProcessLifetime { + pid: 42, + start_filetime: START, + end_filetime: END, + }], + ) + .unwrap_err(); + + assert!(error.to_string().contains("already exists")); + assert_eq!(std::fs::read(&destination).unwrap(), b"sentinel"); + std::fs::remove_file(destination).unwrap(); + std::fs::remove_dir(directory).unwrap(); + } +} diff --git a/src/backends/learning_mode/windows/src/lib.rs b/src/backends/learning_mode/windows/src/lib.rs index 7302a71cd..da180d2b0 100644 --- a/src/backends/learning_mode/windows/src/lib.rs +++ b/src/backends/learning_mode/windows/src/lib.rs @@ -44,6 +44,8 @@ mod capability_names; #[cfg(target_os = "windows")] mod etl_decode; #[cfg(target_os = "windows")] +mod etl_filter; +#[cfg(target_os = "windows")] mod extractors; #[cfg(target_os = "windows")] mod path_norm; @@ -57,6 +59,8 @@ mod ui; #[cfg(target_os = "windows")] pub use etl_decode::{visit_raw_events, EtlDenialAnalyzer}; #[cfg(target_os = "windows")] +pub use etl_filter::filter_trace_for_job_membership; +#[cfg(target_os = "windows")] pub use extractors::DecodedEventParts; #[cfg(target_os = "windows")] pub use ffi::{is_learning_mode_api_available, LearningModeApi, LearningModeTraceHandle}; diff --git a/src/host/plm/src/elevated.rs b/src/host/plm/src/elevated.rs index 7309bee92..88a1212ac 100644 --- a/src/host/plm/src/elevated.rs +++ b/src/host/plm/src/elevated.rs @@ -28,7 +28,8 @@ use std::time::{Duration, Instant}; use learning_mode_core::{AnalysisResult, ProcessLifetime}; use learning_mode_windows::{ - EtlDenialAnalyzer, JobMembershipSnapshot, JobProcessMembership, MAX_JOB_PROCESS_LIFETIMES, + filter_trace_for_job_membership, EtlDenialAnalyzer, JobMembershipSnapshot, + JobProcessMembership, MAX_JOB_PROCESS_LIFETIMES, }; use windows::core::{BOOL, PCWSTR}; use windows::Win32::Foundation::{ @@ -1983,14 +1984,35 @@ fn run_guarded_stop_with_stopped( .context("failed to return elevated PLM discard success"), StopDisposition::Analyze => { let membership = owner.finish_job_tracking()?; - write_analysis_response(pipe, &scratch, &membership) + filter_trace_for_job_membership( + scratch.trace_path(), + scratch.filtered_trace_path(), + &membership, + ) + .context("failed to create process-scoped guarded WPR trace")?; + write_analysis_response(pipe, scratch.filtered_trace_path(), &membership) } StopDisposition::AnalyzeAndTrace => { let membership = owner.finish_job_tracking()?; - write_analysis_response(pipe, &scratch, &membership)?; + filter_trace_for_job_membership( + scratch.trace_path(), + scratch.filtered_trace_path(), + &membership, + ) + .context("failed to create process-scoped guarded WPR trace")?; + write_analysis_response(pipe, scratch.filtered_trace_path(), &membership)?; + write_trace_response(pipe, &scratch) + } + StopDisposition::Trace => { + let membership = owner.finish_job_tracking()?; + filter_trace_for_job_membership( + scratch.trace_path(), + scratch.filtered_trace_path(), + &membership, + ) + .context("failed to create process-scoped guarded WPR trace")?; write_trace_response(pipe, &scratch) } - StopDisposition::Trace => write_trace_response(pipe, &scratch), } } @@ -2053,7 +2075,7 @@ fn run_start() -> Result<()> { } fn write_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Result<()> { - let (mut trace_file, len) = scratch.open_trace()?; + let (mut trace_file, len) = scratch.open_filtered_trace()?; if len > MAX_TRACE_BYTES { anyhow::bail!( "captured ETL is {len} bytes, exceeding the {} byte transfer limit", @@ -2068,11 +2090,11 @@ fn write_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Re fn write_analysis_response( pipe: &mut std::fs::File, - scratch: &SecureScratch, + trace_path: &Path, membership: &JobMembershipSnapshot, ) -> Result<()> { let analysis = EtlDenialAnalyzer - .analyze_for_job_membership(scratch.trace_path(), membership) + .analyze_for_job_membership(trace_path, membership) .context("failed to decode guarded WPR trace for the sandbox process tree")?; let payload = serde_json::to_vec(&analysis).context("failed to serialize guarded WPR analysis")?; diff --git a/src/host/plm/src/secure_scratch.rs b/src/host/plm/src/secure_scratch.rs index 25d760e9a..123271c2b 100644 --- a/src/host/plm/src/secure_scratch.rs +++ b/src/host/plm/src/secure_scratch.rs @@ -40,6 +40,7 @@ use windows::Win32::UI::Shell::{FOLDERID_ProgramData, SHGetKnownFolderPath, KF_F const DIRECTORY_PREFIX: &str = "mxc-plm-elevated-"; const PROFILE_FILE: &str = "embedded.wprp"; const TRACE_FILE: &str = "trace.etl"; +const FILTERED_TRACE_FILE: &str = "filtered.etl"; const RECOVERY_STATE_PARENT: &str = "Microsoft"; const RECOVERY_STATE_COMPONENTS: [&str; 2] = ["MXC", "PLM"]; const RECOVERY_MARKER_FILE: &str = "active.marker"; @@ -54,6 +55,7 @@ pub struct SecureScratch { directory: PathBuf, profile: PathBuf, trace: PathBuf, + filtered_trace: PathBuf, _ancestor_handles: Vec, directory_handle: Option, trace_opened: AtomicBool, @@ -66,6 +68,7 @@ impl fmt::Debug for SecureScratch { .field("directory", &self.directory) .field("profile", &self.profile) .field("trace", &self.trace) + .field("filtered_trace", &self.filtered_trace) .finish_non_exhaustive() } } @@ -131,6 +134,7 @@ impl SecureScratch { return Ok(Self { profile: directory.join(PROFILE_FILE), trace: directory.join(TRACE_FILE), + filtered_trace: directory.join(FILTERED_TRACE_FILE), directory, _ancestor_handles: ancestor_handles, directory_handle: Some(directory_handle), @@ -164,6 +168,11 @@ impl SecureScratch { &self.trace } + /// Returns the protected destination for the process-scoped ETL. + pub fn filtered_trace_path(&self) -> &Path { + &self.filtered_trace + } + /// Creates, flushes, verifies, and seals the embedded WPR profile. pub fn write_and_seal_profile(&self, contents: &[u8]) -> Result { let security = OwnedSecurityDescriptor::from_sddl(PROTECTED_SDDL)?; @@ -226,8 +235,12 @@ impl SecureScratch { }) } - /// Opens the WPR-created ETL once, hardens it, and returns its exact size. - pub fn open_trace(&self) -> Result<(std::fs::File, u64)> { + /// Opens the process-scoped ETL once, hardens it, and returns its exact size. + pub fn open_filtered_trace(&self) -> Result<(std::fs::File, u64)> { + self.open_trace_at(&self.filtered_trace, "filtered PLM trace") + } + + fn open_trace_at(&self, path: &Path, description: &str) -> Result<(std::fs::File, u64)> { if self .trace_opened .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -236,27 +249,23 @@ impl SecureScratch { bail!("secure PLM trace has already been opened"); } - let result = self.open_trace_inner(); + let result = self.open_trace_inner(path, description); if result.is_err() { self.trace_opened.store(false, Ordering::Release); } result } - fn open_trace_inner(&self) -> Result<(std::fs::File, u64)> { + fn open_trace_inner(&self, path: &Path, description: &str) -> Result<(std::fs::File, u64)> { let access = FILE_GENERIC_READ.0 | READ_CONTROL.0 | WRITE_DAC.0 | WRITE_OWNER.0 | FILE_READ_ATTRIBUTES.0; - let handle = open_handle( - &self.trace, - access, - PROTECTOR_SHARE, - FILE_FLAG_OPEN_REPARSE_POINT, - ) - .with_context(|| format!("failed to open secure PLM trace {}", self.trace.display()))?; - verify_file_kind(handle.0, false).context("PLM trace has an unsafe file type")?; + let handle = open_handle(path, access, PROTECTOR_SHARE, FILE_FLAG_OPEN_REPARSE_POINT) + .with_context(|| format!("failed to open secure {description} {}", path.display()))?; + verify_file_kind(handle.0, false) + .with_context(|| format!("{description} has an unsafe file type"))?; let security = OwnedSecurityDescriptor::from_sddl(PROTECTED_SDDL)?; // SAFETY: `handle` is valid and the parsed descriptor remains alive @@ -268,12 +277,14 @@ impl SecureScratch { security.as_ptr(), ) } - .context("failed to harden PLM trace security")?; + .with_context(|| format!("failed to harden {description} security"))?; let mut size = 0i64; // SAFETY: `handle` is valid and `size` is a correctly sized out-param. - unsafe { GetFileSizeEx(handle.0, &mut size) }.context("failed to read PLM trace size")?; - let size = u64::try_from(size).context("PLM trace reported a negative size")?; + unsafe { GetFileSizeEx(handle.0, &mut size) } + .with_context(|| format!("failed to read {description} size"))?; + let size = u64::try_from(size) + .with_context(|| format!("{description} reported a negative size"))?; // Transfer ownership from the RAII handle to File. let raw = handle.into_raw(); @@ -286,6 +297,7 @@ impl SecureScratch { fn cleanup(&mut self) { delete_known_leaf(&self.profile); delete_known_leaf(&self.trace); + delete_known_leaf(&self.filtered_trace); self.directory_handle.take(); let wide = wide_path(&self.directory); // SAFETY: `wide` is a NUL-terminated path to the one known directory. From ca6d8d5cd9e8f60a6130c5d92f6f17c59ee2f79f Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 10:40:27 -0700 Subject: [PATCH 05/34] Document signed PLM developer loop Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- src/host/plm/readme.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/host/plm/readme.md b/src/host/plm/readme.md index 587f4eeda..403c71efc 100644 --- a/src/host/plm/readme.md +++ b/src/host/plm/readme.md @@ -195,6 +195,14 @@ consumer's local Cargo output directory. A locally built or user-writable adjacent `plm.exe` is intentionally rejected, which means native PSEC capture may remain available but the guarded-WPR legacy fallback is unavailable. +For an isolated developer validation loop, a locally built `plm.exe` can be +Authenticode-signed with a short-lived development code-signing certificate +whose signer Organization is `Microsoft Corporation`. Trust that certificate +only on the validation machine, deploy the signed binary in an +administrator-protected directory, and remove the certificate after testing. +This exercises the production trust gate without adding an unsigned-build +bypass; production packages still require the normal Microsoft-signed binary. + ### Bounded discard-confirmation If discarding a guarded session fails, MXC confirms that the elevated guardian From 565534c5708859c4949a6ef69ca3b7f53007b807 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 10:45:18 -0700 Subject: [PATCH 06/34] Document local PLM signing workflow Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- src/host/plm/readme.md | 62 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/host/plm/readme.md b/src/host/plm/readme.md index 403c71efc..feb13dc60 100644 --- a/src/host/plm/readme.md +++ b/src/host/plm/readme.md @@ -203,6 +203,68 @@ administrator-protected directory, and remove the certificate after testing. This exercises the production trust gate without adding an unsigned-build bypass; production packages still require the normal Microsoft-signed binary. +`signtool.exe` is a public tool included with the Windows SDK; it is not +Microsoft-internal. The following PowerShell example creates a one-day, +non-exportable local certificate, signs a development build, and exports only +its public certificate: + +```powershell +$cert = New-SelfSignedCertificate ` + -Type CodeSigningCert ` + -Subject 'CN=MXC Local Validation TEST ONLY,O=Microsoft Corporation' ` + -CertStoreLocation 'Cert:\CurrentUser\My' ` + -KeyExportPolicy NonExportable ` + -NotAfter (Get-Date).AddDays(1) ` + -HashAlgorithm SHA256 + +$signTool = Get-ChildItem ` + "${env:ProgramFiles(x86)}\Windows Kits\10\bin" ` + -Filter signtool.exe -Recurse | + Where-Object FullName -Like '*\x64\signtool.exe' | + Sort-Object FullName -Descending | + Select-Object -First 1 + +& $signTool.FullName sign ` + /sha1 $cert.Thumbprint /s My /fd SHA256 .\plm.exe + +Export-Certificate -Cert $cert -FilePath .\mxc-dev-validation.cer +$cert.Thumbprint | Set-Content .\mxc-dev-validation.thumbprint +``` + +On an isolated validation machine, an administrator can trust the public +certificate locally and verify the signed binary: + +```powershell +$certificate = Import-Certificate ` + -FilePath .\mxc-dev-validation.cer ` + -CertStoreLocation 'Cert:\LocalMachine\Root' +Import-Certificate ` + -FilePath .\mxc-dev-validation.cer ` + -CertStoreLocation 'Cert:\LocalMachine\TrustedPublisher' + +Get-AuthenticodeSignature .\plm.exe +``` + +Deploy `plm.exe` in an administrator-protected directory such as +`C:\Program Files\MXC-Dev\`; the directory-integrity gate still rejects a +signed binary from a user-writable location. After validation, remove the +certificate from the validation machine and remove the private-key certificate +from the development machine: + +```powershell +$thumbprint = Get-Content .\mxc-dev-validation.thumbprint + +# Run elevated on the validation machine. +Remove-Item "Cert:\LocalMachine\Root\$thumbprint" +Remove-Item "Cert:\LocalMachine\TrustedPublisher\$thumbprint" + +# Run on the development machine. +Remove-Item "Cert:\CurrentUser\My\$thumbprint" +``` + +The certificate and signed binary are test artifacts: do not publish, +redistribute, or use them outside the isolated validation environment. + ### Bounded discard-confirmation If discarding a guarded session fails, MXC confirms that the elevated guardian From b9b15a7bb65ec511d75dcb4b1e0a15cf196869ec Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 10:47:35 -0700 Subject: [PATCH 07/34] Link Windows SDK signing tools Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- src/host/plm/readme.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/host/plm/readme.md b/src/host/plm/readme.md index feb13dc60..2a1c4ac40 100644 --- a/src/host/plm/readme.md +++ b/src/host/plm/readme.md @@ -203,8 +203,10 @@ administrator-protected directory, and remove the certificate after testing. This exercises the production trust gate without adding an unsigned-build bypass; production packages still require the normal Microsoft-signed binary. -`signtool.exe` is a public tool included with the Windows SDK; it is not -Microsoft-internal. The following PowerShell example creates a one-day, +`signtool.exe` is included with the Windows SDK. If it is not available on the +development machine, install the SDK from the +[official Windows SDK downloads](https://learn.microsoft.com/windows/apps/windows-sdk/downloads) +before running the example. The following PowerShell creates a one-day, non-exportable local certificate, signs a development build, and exports only its public certificate: From 1dd75d1d4f63a9c59141ecca5f071317297158bf Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 11:04:32 -0700 Subject: [PATCH 08/34] Address retained trace review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- docs/learning-mode/capabilities.md | 12 +- .../common/src/appcontainer_runner.rs | 12 +- .../common/src/base_container_runner.rs | 111 ++++++++++-------- .../appcontainer/common/src/capture_output.rs | 104 ++++++++++++++-- src/host/plm/readme.md | 2 +- src/host/plm/src/elevated.rs | 33 ++++-- src/host/plm/src/secure_scratch.rs | 5 + 7 files changed, 197 insertions(+), 82 deletions(-) diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index f34160f48..0ff90fc86 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -248,16 +248,18 @@ per-run directory under `capture-denials\retained` only after sealing succeeds. WPR's source ETL is host-wide, so the elevated guarded-WPR helper never -transfers that file across the privilege boundary. After the sandbox process -tree terminates, the helper uses the retained, job-attested process handles and -their exact PID/creation/exit `FILETIME` ranges to relog a second ETL. The +transfers that file across the privilege boundary for `captureDenials` or +`--audit`. After the sandbox process tree terminates, the helper uses the +retained, job-attested process handles and their exact PID/creation/exit +`FILETIME` ranges to relog a second ETL. The retained ETL contains only supported Learning Mode events whose event header falls inside one of those attested process generations. Guarded analysis and retention both consume that same filtered ETL; filtering failure transfers no trace. The host-wide source remains in protected elevated scratch and is deleted with that scratch. The unelevated caller writes the filtered retained -ETL beside its unique denials JSON output, using the same file stem with an -`.etl` extension. +ETL beside its unique denials JSON output. Both paths contain the same run +identifier and remain distinct even when the configured output path has no +extension or already ends in `.etl`. Abandoning or disposing a process without a terminal wait deletes or discards the internal trace because no caller can observe its structured path. When diff --git a/src/backends/appcontainer/common/src/appcontainer_runner.rs b/src/backends/appcontainer/common/src/appcontainer_runner.rs index 28541154d..4eb793f12 100644 --- a/src/backends/appcontainer/common/src/appcontainer_runner.rs +++ b/src/backends/appcontainer/common/src/appcontainer_runner.rs @@ -1127,10 +1127,12 @@ impl AppContainerScriptRunner { request.policy.capture_denials.as_ref(), ) { (Some(factory), Some(capture_config)) => { - let output_path = match capture_output::unique_denials_output_path( + let retain_etl = capture_config.retain_etl && factory.allows_trace_transfer(); + let output_paths = match capture_output::unique_denials_output_paths( capture_config.output_path.as_deref(), + retain_etl, ) { - Ok(path) => path, + Ok(paths) => paths, Err(e) => { job.terminate_and_wait(u32::MAX) .map_err(|terminate_error| { @@ -1145,6 +1147,7 @@ impl AppContainerScriptRunner { ))); } }; + let output_path = output_paths.denials; match factory.start(std::process::id()) { Ok(session) => { let session = attach_guarded_capture_or_cleanup( @@ -1162,10 +1165,7 @@ impl AppContainerScriptRunner { "guarded WPR captureDenials session started (output: {})", output_path.display() )); - let etl_path = (capture_config.retain_etl - && factory.allows_trace_transfer()) - .then(|| output_path.with_extension("etl")); - (Some(session), Some(output_path), etl_path) + (Some(session), Some(output_path), output_paths.etl) } Err(e) => { // No active trace exists yet -- terminate the diff --git a/src/backends/appcontainer/common/src/base_container_runner.rs b/src/backends/appcontainer/common/src/base_container_runner.rs index 37ae5d2fb..25e6aebd3 100644 --- a/src/backends/appcontainer/common/src/base_container_runner.rs +++ b/src/backends/appcontainer/common/src/base_container_runner.rs @@ -40,7 +40,7 @@ use windows_core::{PCWSTR, PWSTR}; use crate::capture_output::{ combine_capture_and_cleanup_results, combine_process_and_teardown_results, - remove_internal_capture_file, unique_denials_output_path, write_denials_document, + remove_internal_capture_file, unique_denials_output_paths, write_denials_document, write_stderr_line_best_effort, }; use crate::guarded_capture::{ @@ -1453,11 +1453,23 @@ impl BaseContainerRunner { } else { None }; - let capture_output_path = capture_denials + let capture_output_paths = capture_denials .as_ref() - .map(|config| unique_denials_output_path(config.output_path.as_deref())) + .map(|config| { + let retain_guarded_etl = use_guarded_capture + && config.retain_etl + && self + .guarded_capture_factory + .as_ref() + .is_some_and(|factory| factory.allows_trace_transfer()); + unique_denials_output_paths(config.output_path.as_deref(), retain_guarded_etl) + }) .transpose() .map_err(|error| ScriptResponse::error(&error))?; + let (capture_output_path, guarded_capture_etl_path) = match capture_output_paths { + Some(paths) => (Some(paths.denials), paths.etl), + None => (None, None), + }; let _ = writeln!(logger, "{EMOJI_SECTION} SECTION: Load API"); @@ -2190,21 +2202,6 @@ impl BaseContainerRunner { } else { None }; - let guarded_capture_etl_path = (guarded_capture_session.is_some() - && capture_denials - .as_ref() - .is_some_and(|config| config.retain_etl) - && self - .guarded_capture_factory - .as_ref() - .is_some_and(|factory| factory.allows_trace_transfer())) - .then(|| { - capture_output_path - .as_ref() - .map(|path| path.with_extension("etl")) - }) - .flatten(); - // The child was created suspended; now that it is in the job object (so // every descendant it spawns is captured), resume its main thread. // SAFETY: `pi.hThread` is the just-created, still-owned main-thread @@ -2321,6 +2318,20 @@ struct BaseChild { retain_capture_etl: bool, } +fn validate_guarded_etl_retention( + use_process_security_environment: bool, + retain_etl: bool, + allows_trace_transfer: bool, +) -> Result<(), ScriptResponse> { + if !use_process_security_environment && retain_etl && !allows_trace_transfer { + return Err(ScriptResponse { + failure_phase: FailurePhase::BackendUnavailable, + ..ScriptResponse::error(crate::guarded_capture::RETAIN_ETL_UNSUPPORTED_MSG) + }); + } + Ok(()) +} + impl SandboxBackend for BaseContainerRunner { fn validate(&self, request: &ExecutionRequest) -> Result<(), ScriptResponse> { let capture_denials = request.policy.capture_denials.is_some(); @@ -2335,22 +2346,17 @@ impl SandboxBackend for BaseContainerRunner { return Ok(()); } let use_process_security_environment = self.uses_process_security_environment(request); - if !use_process_security_environment - && request + validate_guarded_etl_retention( + use_process_security_environment, + request .policy .capture_denials .as_ref() - .is_some_and(|config| config.retain_etl) - && !self - .guarded_capture_factory + .is_some_and(|config| config.retain_etl), + self.guarded_capture_factory .as_ref() - .is_some_and(|factory| factory.allows_trace_transfer()) - { - return Err(ScriptResponse { - failure_phase: FailurePhase::BackendUnavailable, - ..ScriptResponse::error(crate::guarded_capture::RETAIN_ETL_UNSUPPORTED_MSG) - }); - } + .is_some_and(|factory| factory.allows_trace_transfer()), + )?; if capture_denials && !use_process_security_environment && self.guarded_capture_factory.is_none() @@ -4619,26 +4625,26 @@ mod tests { #[test] fn validate_runner_rejects_etl_retention_when_guarded_fallback_is_selected() { - // Hold the shared env lock and force native capture unavailable, so a - // concurrent capability-guarded test cannot leak - // `MXC_FORCE_NATIVE_CAPTURE_USABLE=1` and flip this runner onto the - // native (non-guarded) path — which would skip the retain-ETL rejection - // under test. `psec_usable_override` alone is insufficient because the - // env override takes precedence in `native_capture_eligible`. - let _capture_guard = crate::test_env::CaptureCapabilityGuard::set(false, false); - let runner = BaseContainerRunner { - psec_usable_override: Some(false), - ..Default::default() - }; + let runner = BaseContainerRunner::default(); let mut request = ExecutionRequest::default(); request.policy.capture_denials = Some(wxc_common::models::CaptureDenialsConfig { retain_etl: true, ..Default::default() }); - let error = runner - .validate(&request) - .expect_err("the configured guarded provider cannot transfer ETL"); + let error = validate_guarded_etl_retention( + false, + request + .policy + .capture_denials + .as_ref() + .is_some_and(|config| config.retain_etl), + runner + .guarded_capture_factory + .as_ref() + .is_some_and(|factory| factory.allows_trace_transfer()), + ) + .expect_err("the configured guarded provider cannot transfer ETL"); assert_eq!(error.failure_phase, FailurePhase::BackendUnavailable); assert_eq!( @@ -4661,7 +4667,6 @@ mod tests { } let runner = BaseContainerRunner { - psec_usable_override: Some(false), guarded_capture_factory: Some(Arc::new(NeverStartedFactory)), ..Default::default() }; @@ -4671,8 +4676,18 @@ mod tests { ..Default::default() }); - runner - .validate(&request) - .expect("a transfer-capable guarded provider should permit ETL retention"); + validate_guarded_etl_retention( + false, + request + .policy + .capture_denials + .as_ref() + .is_some_and(|config| config.retain_etl), + runner + .guarded_capture_factory + .as_ref() + .is_some_and(|factory| factory.allows_trace_transfer()), + ) + .expect("a transfer-capable guarded provider should permit ETL retention"); } } diff --git a/src/backends/appcontainer/common/src/capture_output.rs b/src/backends/appcontainer/common/src/capture_output.rs index f81f3e2b0..b135998bc 100644 --- a/src/backends/appcontainer/common/src/capture_output.rs +++ b/src/backends/appcontainer/common/src/capture_output.rs @@ -24,6 +24,12 @@ use learning_mode_core::{ }; use wxc_common::models::CaptureDenialsOutput; +#[derive(Debug, Eq, PartialEq)] +pub struct DenialsOutputPaths { + pub denials: PathBuf, + pub etl: Option, +} + /// Writes a bounded [`AnalysisResult`] to `output_path` as the JSON denials /// document and returns the caller-facing [`CaptureDenialsOutput`] summary. /// @@ -122,16 +128,61 @@ pub fn insert_run_id_into_stem(path: &Path, run_id: &str) -> PathBuf { } } -/// Resolves the JSON denials deliverable path for a run: the caller's -/// configured path with a per-run identifier stamped into the stem, or a -/// managed per-run temp file when unset. -pub fn unique_denials_output_path(configured_path: Option<&str>) -> Result { +/// Resolves the paired denials and optional retained-ETL paths for one run. +/// +/// Both names contain the same run identifier. The retained path is always +/// distinct, including when the configured denials path has no extension or +/// already ends in `.etl`. +pub fn unique_denials_output_paths( + configured_path: Option<&str>, + retain_etl: bool, +) -> Result { let suffix = random_capture_suffix()?; let run_id = format!("{}_{suffix}", std::process::id()); - Ok(match configured_path { - Some(path) => insert_run_id_into_stem(Path::new(path), &run_id), + Ok(denials_output_paths_for_run( + configured_path, + &run_id, + retain_etl, + )) +} + +fn denials_output_paths_for_run( + configured_path: Option<&str>, + run_id: &str, + retain_etl: bool, +) -> DenialsOutputPaths { + let denials = match configured_path { + Some(path) => insert_run_id_into_stem(Path::new(path), run_id), None => std::env::temp_dir().join(format!("mxc_denials_{run_id}.json")), - }) + }; + let etl = retain_etl.then(|| match configured_path { + Some(path) => retained_etl_path(Path::new(path), run_id), + None => std::env::temp_dir().join(format!("mxc_denials_{run_id}.etl")), + }); + DenialsOutputPaths { denials, etl } +} + +fn retained_etl_path(configured_path: &Path, run_id: &str) -> PathBuf { + let file_name = configured_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("denials"); + let extension = configured_path.extension().and_then(|ext| ext.to_str()); + let stem = configured_path + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or(file_name); + let retained_name = match extension { + Some(ext) if ext.eq_ignore_ascii_case("etl") => { + format!("{stem}.{run_id}.trace.etl") + } + Some(_) => format!("{stem}.{run_id}.etl"), + None => format!("{file_name}.{run_id}.etl"), + }; + match configured_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.join(retained_name), + _ => PathBuf::from(retained_name), + } } /// A short random hex suffix used to keep per-run temp/output paths from @@ -329,8 +380,12 @@ mod tests { #[test] fn managed_denials_paths_are_unique_per_run() { - let first = unique_denials_output_path(None).expect("first path"); - let second = unique_denials_output_path(None).expect("second path"); + let first = unique_denials_output_paths(None, false) + .expect("first path") + .denials; + let second = unique_denials_output_paths(None, false) + .expect("second path") + .denials; assert_ne!(first, second); assert_eq!(first.parent(), Some(std::env::temp_dir().as_path())); @@ -338,6 +393,37 @@ mod tests { assert_eq!(first.extension().and_then(|ext| ext.to_str()), Some("json")); } + #[test] + fn paired_paths_preserve_run_id_for_extensionless_output() { + let paths = denials_output_paths_for_run(Some(r"C:\app\denials"), "77_abcd", true); + + assert_eq!(paths.denials, PathBuf::from(r"C:\app\denials.77_abcd")); + assert_eq!( + paths.etl, + Some(PathBuf::from(r"C:\app\denials.77_abcd.etl")) + ); + } + + #[test] + fn paired_paths_are_distinct_for_etl_output() { + let paths = denials_output_paths_for_run(Some(r"C:\app\denials.ETL"), "77_abcd", true); + + assert_eq!(paths.denials, PathBuf::from(r"C:\app\denials.77_abcd.ETL")); + assert_eq!( + paths.etl, + Some(PathBuf::from(r"C:\app\denials.77_abcd.trace.etl")) + ); + assert_ne!( + paths.denials.to_string_lossy().to_ascii_lowercase(), + paths + .etl + .as_ref() + .expect("retained path") + .to_string_lossy() + .to_ascii_lowercase() + ); + } + #[test] fn missing_internal_capture_file_is_already_clean() { let directory = tempfile::tempdir().expect("temp directory"); diff --git a/src/host/plm/readme.md b/src/host/plm/readme.md index 2a1c4ac40..a492dd99c 100644 --- a/src/host/plm/readme.md +++ b/src/host/plm/readme.md @@ -98,7 +98,7 @@ plm.exe extract-caps --hex-bytes [--verbose-logging] ### `plm log` -Interactive iteration mode: press Enter to start a trace, run the workload, press Enter again to stop. It then synthesizes a blank config, runs the filesystem merge, and prints the resulting config as a "diff against a blank config" preview. +Interactive iteration mode: press Enter to start a host-wide trace, run the operator-selected workload, then press Enter again to stop. Because this standalone flow has no sandbox job to define a process scope, it returns the host-wide trace to the same unelevated caller that initiated the session. It then synthesizes a blank config, runs the filesystem merge, and prints the resulting config as a "diff against a blank config" preview. Automated `--audit` and `captureDenials` flows instead attach an authenticated sandbox job and return only the process-scoped filtered trace. It also has no public `--wprp` or destination override flags. diff --git a/src/host/plm/src/elevated.rs b/src/host/plm/src/elevated.rs index 88a1212ac..000fa9eee 100644 --- a/src/host/plm/src/elevated.rs +++ b/src/host/plm/src/elevated.rs @@ -362,7 +362,7 @@ impl GuardedOwner { match pipe.read(&mut control) { Ok(1) => match parse_guard_control(control[0]) { Ok(GuardControl::Stop) => { - return run_guarded_stop(pipe, self, StopDisposition::Trace); + return run_guarded_stop(pipe, self, StopDisposition::InteractiveTrace); } Ok(GuardControl::StopAndAnalyze) => { return run_guarded_stop(pipe, self, StopDisposition::Analyze); @@ -1909,7 +1909,7 @@ fn start_owned_trace(owner: &mut GuardedOwner) -> Result<()> { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum StopDisposition { - Trace, + InteractiveTrace, Analyze, AnalyzeAndTrace, Discard, @@ -2003,16 +2003,10 @@ fn run_guarded_stop_with_stopped( write_analysis_response(pipe, scratch.filtered_trace_path(), &membership)?; write_trace_response(pipe, &scratch) } - StopDisposition::Trace => { - let membership = owner.finish_job_tracking()?; - filter_trace_for_job_membership( - scratch.trace_path(), - scratch.filtered_trace_path(), - &membership, - ) - .context("failed to create process-scoped guarded WPR trace")?; - write_trace_response(pipe, &scratch) - } + // `CONTROL_STOP` is reserved for standalone `plm log`, which has no + // sandbox job to attest because the operator chooses the workload + // interactively. Automated capture uses the scoped dispositions above. + StopDisposition::InteractiveTrace => write_source_trace_response(pipe, &scratch), } } @@ -2076,6 +2070,19 @@ fn run_start() -> Result<()> { fn write_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Result<()> { let (mut trace_file, len) = scratch.open_filtered_trace()?; + write_trace_file_response(pipe, &mut trace_file, len) +} + +fn write_source_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Result<()> { + let (mut trace_file, len) = scratch.open_source_trace()?; + write_trace_file_response(pipe, &mut trace_file, len) +} + +fn write_trace_file_response( + pipe: &mut std::fs::File, + trace_file: &mut std::fs::File, + len: u64, +) -> Result<()> { if len > MAX_TRACE_BYTES { anyhow::bail!( "captured ETL is {len} bytes, exceeding the {} byte transfer limit", @@ -2084,7 +2091,7 @@ fn write_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Re } write_header(pipe, ResponseKind::Trace, len)?; - copy_exact_len(&mut trace_file, pipe, len)?; + copy_exact_len(trace_file, pipe, len)?; pipe.flush().context("failed to flush elevated PLM trace") } diff --git a/src/host/plm/src/secure_scratch.rs b/src/host/plm/src/secure_scratch.rs index 123271c2b..4cc07ad1b 100644 --- a/src/host/plm/src/secure_scratch.rs +++ b/src/host/plm/src/secure_scratch.rs @@ -240,6 +240,11 @@ impl SecureScratch { self.open_trace_at(&self.filtered_trace, "filtered PLM trace") } + /// Opens the host-wide source ETL for the explicit interactive log flow. + pub fn open_source_trace(&self) -> Result<(std::fs::File, u64)> { + self.open_trace_at(&self.trace, "interactive PLM trace") + } + fn open_trace_at(&self, path: &Path, description: &str) -> Result<(std::fs::File, u64)> { if self .trace_opened From c6a73536cb7b0352fb213c895568ebad77d31546 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 11:10:49 -0700 Subject: [PATCH 09/34] Keep interactive traces elevated Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- src/host/plm/readme.md | 2 +- src/host/plm/src/elevated.rs | 58 ++++++++++++++++-------------- src/host/plm/src/log.rs | 18 ++-------- src/host/plm/src/secure_scratch.rs | 5 --- 4 files changed, 35 insertions(+), 48 deletions(-) diff --git a/src/host/plm/readme.md b/src/host/plm/readme.md index a492dd99c..ae4937f15 100644 --- a/src/host/plm/readme.md +++ b/src/host/plm/readme.md @@ -98,7 +98,7 @@ plm.exe extract-caps --hex-bytes [--verbose-logging] ### `plm log` -Interactive iteration mode: press Enter to start a host-wide trace, run the operator-selected workload, then press Enter again to stop. Because this standalone flow has no sandbox job to define a process scope, it returns the host-wide trace to the same unelevated caller that initiated the session. It then synthesizes a blank config, runs the filesystem merge, and prints the resulting config as a "diff against a blank config" preview. Automated `--audit` and `captureDenials` flows instead attach an authenticated sandbox job and return only the process-scoped filtered trace. +Interactive iteration mode: press Enter to start a host-wide trace, run the operator-selected workload, then press Enter again to stop. Because this standalone flow has no sandbox job to define a process scope, the elevated guardian analyzes the source trace in protected scratch and returns only the bounded canonical findings; the host-wide ETL never crosses the privilege boundary. It then synthesizes a blank config, runs the filesystem merge, and prints the resulting config as a "diff against a blank config" preview. Automated `--audit` and `captureDenials` flows instead attach an authenticated sandbox job and analyze or retain only the process-scoped filtered trace. It also has no public `--wprp` or destination override flags. diff --git a/src/host/plm/src/elevated.rs b/src/host/plm/src/elevated.rs index 000fa9eee..93f9275eb 100644 --- a/src/host/plm/src/elevated.rs +++ b/src/host/plm/src/elevated.rs @@ -26,7 +26,7 @@ use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use learning_mode_core::{AnalysisResult, ProcessLifetime}; +use learning_mode_core::{AnalysisResult, DenialAnalyzer, ProcessLifetime}; use learning_mode_windows::{ filter_trace_for_job_membership, EtlDenialAnalyzer, JobMembershipSnapshot, JobProcessMembership, MAX_JOB_PROCESS_LIFETIMES, @@ -362,7 +362,7 @@ impl GuardedOwner { match pipe.read(&mut control) { Ok(1) => match parse_guard_control(control[0]) { Ok(GuardControl::Stop) => { - return run_guarded_stop(pipe, self, StopDisposition::InteractiveTrace); + return run_guarded_stop(pipe, self, StopDisposition::InteractiveAnalyze); } Ok(GuardControl::StopAndAnalyze) => { return run_guarded_stop(pipe, self, StopDisposition::Analyze); @@ -1445,7 +1445,7 @@ impl GuardedSession { Ok(()) } - pub fn stop(&mut self, trace_destination: &Path) -> Result<()> { + fn stop_interactive_analyzed(&mut self) -> Result { if self.disarmed { anyhow::bail!("guarded PLM session is already stopped"); } @@ -1458,17 +1458,10 @@ impl GuardedSession { let stopped = std::cell::Cell::new(false); let deadline = Instant::now() + WAIT_TIMEOUT_DURATION; - let result = read_response( - &mut pipe, - self.process.0, - Operation::Stop, - Some(trace_destination), - deadline, - || { - stopped.set(true); - Ok(()) - }, - ); + let result = read_analysis_response(&mut pipe, self.process.0, deadline, || { + stopped.set(true); + Ok(()) + }); if stopped.get() { self.disarmed = true; drop(pipe); @@ -1476,8 +1469,9 @@ impl GuardedSession { self.process.0, deadline.saturating_duration_since(Instant::now()), ); - result?; - wait_result + let analysis = result?; + wait_result?; + Ok(analysis) } else { self.pipe = Some(pipe); result @@ -1682,13 +1676,13 @@ pub fn invoke_guarded_start(owner_pid: u32) -> Result<()> { }) } -pub fn stop_current_guarded_start(trace_destination: &Path) -> Result<()> { +pub fn stop_current_guarded_start() -> Result { CURRENT_GUARDED_SESSION.with(|slot| { let mut session = slot .borrow_mut() .take() .context("no guarded PLM session is active on this thread")?; - let result = session.stop(trace_destination); + let result = session.stop_interactive_analyzed(); if result.is_err() && !session.disarmed { *slot.borrow_mut() = Some(session); } @@ -1909,7 +1903,7 @@ fn start_owned_trace(owner: &mut GuardedOwner) -> Result<()> { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum StopDisposition { - InteractiveTrace, + InteractiveAnalyze, Analyze, AnalyzeAndTrace, Discard, @@ -2005,8 +1999,11 @@ fn run_guarded_stop_with_stopped( } // `CONTROL_STOP` is reserved for standalone `plm log`, which has no // sandbox job to attest because the operator chooses the workload - // interactively. Automated capture uses the scoped dispositions above. - StopDisposition::InteractiveTrace => write_source_trace_response(pipe, &scratch), + // interactively. Decode in the elevated child and return only the + // bounded analysis; automated capture uses the scoped dispositions. + StopDisposition::InteractiveAnalyze => { + write_interactive_analysis_response(pipe, scratch.trace_path()) + } } } @@ -2073,11 +2070,6 @@ fn write_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Re write_trace_file_response(pipe, &mut trace_file, len) } -fn write_source_trace_response(pipe: &mut std::fs::File, scratch: &SecureScratch) -> Result<()> { - let (mut trace_file, len) = scratch.open_source_trace()?; - write_trace_file_response(pipe, &mut trace_file, len) -} - fn write_trace_file_response( pipe: &mut std::fs::File, trace_file: &mut std::fs::File, @@ -2103,6 +2095,20 @@ fn write_analysis_response( let analysis = EtlDenialAnalyzer .analyze_for_job_membership(trace_path, membership) .context("failed to decode guarded WPR trace for the sandbox process tree")?; + write_serialized_analysis_response(pipe, &analysis) +} + +fn write_interactive_analysis_response(pipe: &mut std::fs::File, trace_path: &Path) -> Result<()> { + let analysis = EtlDenialAnalyzer + .analyze(trace_path) + .context("failed to decode interactive guarded WPR trace")?; + write_serialized_analysis_response(pipe, &analysis) +} + +fn write_serialized_analysis_response( + pipe: &mut std::fs::File, + analysis: &AnalysisResult, +) -> Result<()> { let payload = serde_json::to_vec(&analysis).context("failed to serialize guarded WPR analysis")?; let len = payload.len() as u64; diff --git a/src/host/plm/src/log.rs b/src/host/plm/src/log.rs index 6f2565b2a..b818d97d0 100644 --- a/src/host/plm/src/log.rs +++ b/src/host/plm/src/log.rs @@ -9,13 +9,11 @@ //! 4. Stops the trace into a temp .etl and reports where it landed. use anyhow::{Context, Result}; -use chrono::Local; use learning_mode_core::AnalysisResult; use serde_json::{json, Value}; use std::io::{self, BufRead, Write}; -use std::path::PathBuf; -use crate::analysis::{analyze_trace, legacy_config_inputs, write_detection_summary}; +use crate::analysis::{legacy_config_inputs, write_detection_summary}; use crate::config::{ deny_file_set, initialize_filesystem, update_from_access_events, write_added_paths_summary, }; @@ -49,22 +47,10 @@ pub fn run(owner_pid: u32, verbose: bool, on_trace_started: impl FnOnce()) -> Re prompt_enter("Press Enter to stop logging...")?; - // Per-run trace file in temp; PID + sub-second component prevents - // parallel `plm log` invocations from colliding on the same .etl. - let stamp = Local::now().format("%Y-%m-%d_%H%M%S%.3f").to_string(); - let trace_file: PathBuf = std::env::temp_dir().join(format!("plm_log_{stamp}.etl")); - elevated::stop_current_guarded_start(&trace_file)?; - if verbose { println!("Beginning event parsing, this may take several minutes"); } - - let analysis = analyze_trace(&trace_file); - - // Clean up the temp .etl regardless of analysis outcome. - let _ = std::fs::remove_file(&trace_file); - - let analysis = analysis?; + let analysis = elevated::stop_current_guarded_start()?; write_detection_summary(&analysis); if !can_generate_policy_preview(&analysis) { eprintln!( diff --git a/src/host/plm/src/secure_scratch.rs b/src/host/plm/src/secure_scratch.rs index 4cc07ad1b..123271c2b 100644 --- a/src/host/plm/src/secure_scratch.rs +++ b/src/host/plm/src/secure_scratch.rs @@ -240,11 +240,6 @@ impl SecureScratch { self.open_trace_at(&self.filtered_trace, "filtered PLM trace") } - /// Opens the host-wide source ETL for the explicit interactive log flow. - pub fn open_source_trace(&self) -> Result<(std::fs::File, u64)> { - self.open_trace_at(&self.trace, "interactive PLM trace") - } - fn open_trace_at(&self, path: &Path, description: &str) -> Result<(std::fs::File, u64)> { if self .trace_opened From abf121902bc9e473f51b22c331f843e09a1a2952 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Fri, 14 Aug 2026 14:41:55 -0700 Subject: [PATCH 10/34] Add Data Loop denial diagnostics Emit a bounded, username-redacted Data Loop JSON sibling containing canonical denial occurrences and classified decoder outcomes. Keep paired output transactional across native, guarded, and audit capture paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4eff3ee2-323c-494d-99b9-f7b54e495216 --- docs/learning-mode/capabilities.md | 108 +- .../appcontainer/common/src/capture_output.rs | 261 ++++- .../windows/src/capability_dacl.rs | 7 + .../learning_mode/windows/src/etl_decode.rs | 953 ++++++++++++++++-- .../learning_mode/windows/src/extractors.rs | 803 ++++++++++++++- .../learning_mode/windows/src/tdh_decode.rs | 288 +++++- src/core/learning_mode_core/src/analyze.rs | 124 +++ src/core/learning_mode_core/src/data_loop.rs | 532 ++++++++++ src/core/learning_mode_core/src/lib.rs | 17 +- src/core/learning_mode_core/src/model.rs | 4 +- src/core/wxc/src/audit.rs | 156 ++- src/host/plm/src/analysis.rs | 1 + src/host/plm/src/log.rs | 1 + 13 files changed, 3032 insertions(+), 223 deletions(-) create mode 100644 src/core/learning_mode_core/src/data_loop.rs diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index 0ff90fc86..0365fe55b 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -105,9 +105,10 @@ stays enforced: hosts use native capture without PLM or UAC, while legacy or incompatible tiers use the session-scoped guarded-WPR fallback and elevate only its fixed-operation guardian. The CLI consumes the returned JSON and ETL paths, - relocates them to `denials.json` and `trace.etl`, and generates the source - snapshot and `Adjusted_*.json` from canonical denials without decoding ETL - again. Truncated analysis skips the adjusted config. + relocates the canonical output, its Data Loop sibling, and the trace to + `denials.json`, `denials.data-loop.json`, and `trace.etl`, and generates the + source snapshot and `Adjusted_*.json` from canonical denials without decoding + ETL again. Truncated analysis skips the adjusted config. ``` wxc-exec --audit --config @@ -171,8 +172,8 @@ ungranted access is handled while it is recorded: ### Output file the caller consumes After the sandboxed workload exits, MXC decodes the captured denials and writes -them to a **single JSON file** — the deliverable a host application reads to -regenerate its sandbox policy: +the canonical JSON deliverable a host application reads to regenerate its +sandbox policy: ```json { @@ -203,8 +204,9 @@ regenerate its sandbox policy: - `denials` is already de-duplicated per `(resource, accessType)`, so `summary.totalDenials` equals `denials.length`. - Analysis retains at most 10,000 unique denials and processes at most - 1,000,000 ETW events. Reaching either bound stops further analysis and sets - `summary.deniedResourcesTruncated` to `true`. + 1,000,000 ETW events. Reaching the unique-denial bound stops adding canonical + entries but continues bounded aggregate accounting; reaching either bound + sets `summary.deniedResourcesTruncated` to `true`. - `resource` is the user-visible identifier for the denied resource, interpreted by `resourceType`: a canonical `C:\…` path for `file`, the AppContainer **capability name** (e.g. `internetClient`) for `capability`, @@ -219,6 +221,94 @@ regenerate its sandbox policy: - `filetime` is a decimal string containing the Windows `FILETIME` value, so JavaScript consumers retain all 64 bits without numeric precision loss. +### Data Loop event signatures + +Every successful decode also writes a deterministic sibling file: +`denials..json` produces `denials..data-loop.json`. This Data +Loop artifact is a bounded, username-redacted superset containing canonical +denial occurrences plus outcomes omitted from canonical denials: + +```json +{ + "version": 1, + "signatures": [ + { + "signature": { + "provider": "kernelGeneral", + "providerGuid": "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}", + "eventId": 14, + "reason": "canonicalDenial", + "pid": 4321, + "accessType": "read", + "resourceType": "file", + "properties": [ + ["PackageSid", "S-1-15-3-1"], + ["resource", "C:\\Users\\\\data.txt"] + ] + }, + "count": 37 + } + ], + "summary": { + "totalOccurrences": 37, + "overflowOccurrences": 0, + "canonicalOverflowOccurrences": 0, + "aggregateGroupsTruncated": false, + "processedEventsTruncated": false, + "canonicalDenialLimitReached": false + } +} +``` + +Signatures are keyed by symbolic provider category, provider GUID, +provider-scoped event ID, closed exclusion reason, PID, and sorted sanitized +properties. SIDs, capability names, GUIDs, PIDs/process identifiers, and +resource values are retained. Standalone user/account names and username +components inside paths or resource names are replaced with +``; the rest of each value remains available for diagnostics. +Exact header timestamps and timestamp-like properties are omitted so otherwise +identical events deduplicate, and free-form decoder errors are never serialized. + +Every valid denial candidate is classified as `canonicalDenial`, including its +first canonical occurrence, later duplicates, and candidates observed after +the canonical unique-denial bound. Those occurrences deduplicate under the +same signature and increment its count. `accessType` and `resourceType` are +included when denial extraction determined them; diagnostic outcomes without +those classifications omit the fields. +Unknown event IDs from known Learning Mode providers are classified as +`unsupportedEventSchema`; the real ETL path retains their provider GUID and +PID without attempting an unsupported TDH payload decode. + +Per-event TDH failures use closed diagnostic reasons: +`eventPayloadMalformed` means the payload conflicts with its declared schema, +`decoderLimitReached` means a nesting/element/work safety bound stopped +decoding, and `unsupportedPropertyEncoding` means the decoder cannot consume +that property shape. When TDH exposes it, the schema-declared name is retained +as the bounded `EventName` signature property. Free-form decoder errors are +never serialized. Failure to obtain the event schema remains a fatal analysis +error rather than being represented as a Data Loop signature. + +Data Loop retains at most 4,096 distinct signatures, 24 sorted properties per +signature, 256 characters per property value, and 16 MiB of compact serialized +signature data. Canonical-denial signatures take priority over diagnostic +signatures at these bounds. Additional keys are collapsed into +`overflowOccurrences`; `canonicalOverflowOccurrences` reports the subset of +canonical occurrences that could not be represented even after diagnostic +groups were evicted, and `aggregateGroupsTruncated` is set. +Before returning analysis, the decoder also fits the complete canonical plus +Data Loop result within the guarded 64 MiB transport limit by moving additional +Data Loop groups into overflow; canonical denials are never discarded. +`processedEventsTruncated` indicates that the 1,000,000-event bound prevented +complete accounting. Counts are candidate-level when extraction identifies +individual denial candidates and one event-level outcome when decoding cannot +determine candidate cardinality. + +The canonical and Data Loop files fail together: MXC stages both and reports +capture failure unless both final artifacts are committed. The Data Loop path +is intentionally absent from stderr pointers and Rust, Node, C#, and FFI output +metadata; callers derive it from the canonical path using the naming rule +above. + **Locating the file.** Set `captureDenials.outputPath` to name the file explicitly (its parent directory must already exist). MXC inserts a unique per-run identifier (process id plus random suffix) into the file stem @@ -232,8 +322,8 @@ so CLI callers can locate the deliverable without scanning the filesystem: {"type":"captureDenials","outputPath":"C:\\logs\\denials.4321_0123456789abcdef0123456789abcdef.json","exitCode":0,"totalDenials":2,"deniedResourcesTruncated":false} ``` -The pointer echoes the file's `summary`; the authoritative record is the file -itself. In-process Rust callers receive the same information through +The pointer echoes the canonical file's `summary`; the authoritative policy +record is that file. In-process Rust callers receive the same information through `Output::output_metadata` or `Sandbox::output_metadata()` after waiting. The C# SDK exposes it through `RunResult.OutputMetadata` and `MxcSandboxProcess.OutputMetadata`. diff --git a/src/backends/appcontainer/common/src/capture_output.rs b/src/backends/appcontainer/common/src/capture_output.rs index b135998bc..c26330d71 100644 --- a/src/backends/appcontainer/common/src/capture_output.rs +++ b/src/backends/appcontainer/common/src/capture_output.rs @@ -7,9 +7,9 @@ //! decoding its own sealed ETL) and the guarded-WPR legacy-tier fallback //! (`appcontainer_runner`, consuming an already-decoded [`AnalysisResult`] //! handed back by the elevated PLM guardian) must emit byte-for-byte the same -//! [`DenialsDocument`] JSON shape, [`CaptureDenialsOutput`] summary, and -//! resolved output-path convention. Centralizing that here is what guarantees -//! the two paths can't drift. +//! [`DenialsDocument`] JSON shape, username-redacted Data Loop sibling, +//! [`CaptureDenialsOutput`] summary, and resolved output-path convention. +//! Centralizing that here is what guarantees the two paths can't drift. //! //! Deliberately free of any Windows API or [`ScriptResponse`] coupling so it //! can be shared by both runner modules without either owning the other's @@ -20,7 +20,8 @@ use std::io::Write; use std::path::{Path, PathBuf}; use learning_mode_core::{ - write_document, AnalysisResult, DenialSummary, DenialsDocument, DenialsOutputPointer, + data_loop_sibling_path, write_data_loop_document, write_document, AnalysisResult, + DataLoopDocument, DenialSummary, DenialsDocument, DenialsOutputPointer, }; use wxc_common::models::CaptureDenialsOutput; @@ -30,8 +31,9 @@ pub struct DenialsOutputPaths { pub etl: Option, } -/// Writes a bounded [`AnalysisResult`] to `output_path` as the JSON denials -/// document and returns the caller-facing [`CaptureDenialsOutput`] summary. +/// Writes a bounded [`AnalysisResult`] as the canonical denials document and +/// deterministic Data Loop sibling, then returns caller-facing metadata for +/// the canonical document. /// /// Never overwrites an existing file: a run whose output path collides with a /// leftover file from a previous run fails loudly rather than clobbering it. @@ -40,14 +42,21 @@ pub fn write_denials_document( exit_code: i32, output_path: &Path, ) -> std::io::Result { + let data_loop_path = data_loop_output_path(output_path)?; let summary = DenialSummary::new( exit_code, analysis.denials.len(), analysis.denied_resources_truncated, ); let document = DenialsDocument::new(analysis.denials, summary); + let data_loop_document = DataLoopDocument::new(&analysis.data_loop); - write_denials_output_file(output_path, |writer| write_document(writer, &document))?; + write_paired_output_files( + output_path, + &data_loop_path, + |writer| write_document(writer, &document), + |writer| write_data_loop_document(writer, &data_loop_document), + )?; let pointer = DenialsOutputPointer::new(output_path.to_string_lossy(), &document.summary); Ok(CaptureDenialsOutput { @@ -60,6 +69,156 @@ pub fn write_denials_document( }) } +/// Derives the deterministic Data Loop sibling path for a denials output. +pub fn data_loop_output_path(output_path: &Path) -> std::io::Result { + data_loop_sibling_path(output_path) + .map_err(|error| std::io::Error::other(format!("captureDenials {error}"))) +} + +fn write_paired_output_files( + denials_path: &Path, + data_loop_path: &Path, + write_denials: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, + write_data_loop: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, +) -> std::io::Result<()> { + ensure_output_absent(denials_path, "denials")?; + ensure_output_absent(data_loop_path, "Data Loop")?; + + let denials_temp = stage_output_file(denials_path, "denials", write_denials)?; + let data_loop_temp = match stage_output_file(data_loop_path, "Data Loop", write_data_loop) { + Ok(path) => path, + Err(error) => { + return Err(combine_error_with_cleanup( + error, + remove_if_present(&denials_temp), + &denials_temp, + )) + } + }; + + if let Err(error) = std::fs::rename(&data_loop_temp, data_loop_path) { + let error = std::io::Error::other(format!( + "captureDenials failed to promote Data Loop output file {}: {error}", + data_loop_path.display() + )); + let error = + combine_error_with_cleanup(error, remove_if_present(&data_loop_temp), &data_loop_temp); + return Err(combine_error_with_cleanup( + error, + remove_if_present(&denials_temp), + &denials_temp, + )); + } + + if let Err(error) = std::fs::rename(&denials_temp, denials_path) { + let error = std::io::Error::other(format!( + "captureDenials failed to promote denials output file {}: {error}", + denials_path.display() + )); + let error = + combine_error_with_cleanup(error, remove_if_present(data_loop_path), data_loop_path); + return Err(combine_error_with_cleanup( + error, + remove_if_present(&denials_temp), + &denials_temp, + )); + } + + Ok(()) +} + +fn ensure_output_absent(path: &Path, kind: &str) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "captureDenials {kind} output file already exists: {}", + path.display() + ), + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(std::io::Error::other(format!( + "captureDenials failed to inspect {kind} output file {}: {error}", + path.display() + ))), + } +} + +fn stage_output_file( + final_path: &Path, + kind: &str, + write: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, +) -> std::io::Result { + let temp_path = temporary_sibling_path(final_path)?; + let file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp_path) + .map_err(|error| { + std::io::Error::other(format!( + "captureDenials failed to create temporary {kind} output file {}: {error}", + temp_path.display() + )) + })?; + let mut writer = std::io::BufWriter::new(file); + if let Err(error) = write(&mut writer) + .and_then(|()| writer.flush()) + .and_then(|()| writer.get_ref().sync_all()) + { + let error = std::io::Error::other(format!( + "captureDenials failed to write temporary {kind} output file {}: {error}", + temp_path.display() + )); + drop(writer); + return Err(combine_error_with_cleanup( + error, + remove_if_present(&temp_path), + &temp_path, + )); + } + Ok(temp_path) +} + +fn temporary_sibling_path(final_path: &Path) -> std::io::Result { + let file_name = final_path + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + std::io::Error::other(format!( + "captureDenials output path has no usable file name: {}", + final_path.display() + )) + })?; + let suffix = random_capture_suffix().map_err(std::io::Error::other)?; + let temp_name = format!(".{file_name}.tmp.{}.{suffix}", std::process::id()); + Ok(match final_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.join(temp_name), + _ => PathBuf::from(temp_name), + }) +} + +fn remove_if_present(path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn combine_error_with_cleanup( + error: std::io::Error, + cleanup: std::io::Result<()>, + cleanup_path: &Path, +) -> std::io::Error { + match cleanup { + Ok(()) => error, + Err(cleanup_error) => std::io::Error::other(format!( + "{error}; additionally failed to remove {}: {cleanup_error}", + cleanup_path.display() + )), + } +} + /// Creates `output_path` (failing if it already exists) and writes through /// `write`, cleaning up a partial file if `write` fails. pub fn write_denials_output_file( @@ -286,7 +445,10 @@ pub fn write_stderr_line_best_effort(message: std::fmt::Arguments<'_>) { #[cfg(test)] mod tests { use super::*; - use learning_mode_core::{AccessType, DeniedResource, ResourceType}; + use learning_mode_core::{ + AccessType, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, DeniedResource, + ResourceType, + }; #[test] fn write_denials_document_writes_summary_and_document() { @@ -308,8 +470,13 @@ mod tests { assert_eq!(metadata.total_denials, 1); assert!(!metadata.denied_resources_truncated); let document: DenialsDocument = - serde_json::from_slice(&std::fs::read(output_path).unwrap()).unwrap(); + serde_json::from_slice(&std::fs::read(&output_path).unwrap()).unwrap(); assert_eq!(document.denials.len(), 1); + let data_loop: DataLoopDocument = serde_json::from_slice( + &std::fs::read(data_loop_output_path(&output_path).unwrap()).unwrap(), + ) + .unwrap(); + assert_eq!(data_loop.version, DataLoopDocument::VERSION); } #[test] @@ -323,6 +490,82 @@ mod tests { assert_eq!(metadata.total_denials, 0); assert!(output_path.exists()); + assert!(data_loop_output_path(&output_path).unwrap().exists()); + } + + #[test] + fn write_denials_document_writes_data_loop_aggregates() { + let directory = tempfile::tempdir().expect("temp directory"); + let output_path = directory.path().join("denials.json"); + let mut analysis = AnalysisResult::complete(Vec::new()); + analysis.data_loop.record(DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}".to_string(), + event_id: 14, + reason: DataLoopExclusionReason::CanonicalDenial, + pid: 42, + access_type: Some(learning_mode_core::AccessType::Read), + resource_type: Some(learning_mode_core::ResourceType::File), + properties: vec![("Sid".to_string(), "S-1-15-3-1".to_string())], + }); + + write_denials_document(analysis, 0, &output_path).expect("write should succeed"); + + let data_loop: DataLoopDocument = serde_json::from_slice( + &std::fs::read(data_loop_output_path(&output_path).unwrap()).unwrap(), + ) + .unwrap(); + assert_eq!(data_loop.signatures.len(), 1); + assert_eq!(data_loop.signatures[0].count, 1); + assert_eq!(data_loop.signatures[0].signature.pid, 42); + } + + #[test] + fn data_loop_path_is_a_deterministic_json_sibling() { + assert_eq!( + data_loop_output_path(Path::new(r"C:\out\denials.123.json")).unwrap(), + PathBuf::from(r"C:\out\denials.123.data-loop.json") + ); + assert_eq!( + data_loop_output_path(Path::new("denials")).unwrap(), + PathBuf::from("denials.data-loop.json") + ); + } + + #[test] + fn data_loop_collision_leaves_canonical_output_absent() { + let directory = tempfile::tempdir().expect("temp directory"); + let output_path = directory.path().join("denials.json"); + let data_loop_path = data_loop_output_path(&output_path).unwrap(); + std::fs::write(&data_loop_path, b"existing").expect("seed Data Loop output"); + + write_denials_document(AnalysisResult::complete(Vec::new()), 0, &output_path) + .expect_err("collision should fail"); + + assert!(!output_path.exists()); + assert_eq!(std::fs::read(data_loop_path).unwrap(), b"existing"); + } + + #[test] + fn second_staged_write_failure_leaves_no_final_output() { + let directory = tempfile::tempdir().expect("temp directory"); + let output_path = directory.path().join("denials.json"); + let data_loop_path = data_loop_output_path(&output_path).unwrap(); + + let error = write_paired_output_files( + &output_path, + &data_loop_path, + |writer| writer.write_all(b"denials"), + |writer| { + writer.write_all(b"partial")?; + Err(std::io::Error::other("simulated Data Loop failure")) + }, + ) + .expect_err("paired write should fail"); + + assert!(error.to_string().contains("simulated Data Loop failure")); + assert!(!output_path.exists()); + assert!(!data_loop_path.exists()); } #[test] diff --git a/src/backends/learning_mode/windows/src/capability_dacl.rs b/src/backends/learning_mode/windows/src/capability_dacl.rs index 059f656d9..0fb9c90b2 100644 --- a/src/backends/learning_mode/windows/src/capability_dacl.rs +++ b/src/backends/learning_mode/windows/src/capability_dacl.rs @@ -252,6 +252,11 @@ pub(crate) fn extract_denials( if !is_supported_event { return Vec::new(); } + // `is_supported_event` above only matches the two known Learning Mode + // providers, so this always resolves. + let Some(provider) = crate::extractors::provider_category(parts.provider) else { + return Vec::new(); + }; let names = extract_names(parts, CAPABILITY_INDEX.get_or_init(build_capability_index)); names @@ -263,6 +268,8 @@ pub(crate) fn extract_denials( access_type: AccessType::Unknown, filetime, event_id: parts.event_id, + provider, + data_loop_properties: crate::extractors::sanitize_properties(&parts.props), }) .collect() } diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 1ea867581..7c5b335f6 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -29,7 +29,9 @@ use std::os::windows::ffi::OsStrExt; use std::path::Path; use learning_mode_core::{ - AnalysisResult, AnalyzeError, DenialAnalyzer, DeniedResource, ProcessLifetime, + AnalysisResult, AnalyzeError, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, + DataLoopSummary, DenialAnalyzer, DeniedResource, ProcessLifetime, + MAX_DATA_LOOP_SIGNATURE_BYTES, }; use windows::core::PWSTR; use windows::Win32::System::Diagnostics::Etw::{ @@ -138,6 +140,8 @@ struct Accumulator<'visitor> { decode_error: Option, panic_payload: Option>, schema_cache: tdh_decode::EventSchemaCache, + data_loop: DataLoopSummary, + data_loop_signature_bytes: usize, } impl<'visitor> Accumulator<'visitor> { @@ -158,6 +162,8 @@ impl<'visitor> Accumulator<'visitor> { decode_error: None, panic_payload: None, schema_cache: tdh_decode::EventSchemaCache::default(), + data_loop: DataLoopSummary::default(), + data_loop_signature_bytes: 0, } } @@ -185,6 +191,8 @@ impl<'visitor> Accumulator<'visitor> { decode_error: None, panic_payload: None, schema_cache: tdh_decode::EventSchemaCache::default(), + data_loop: DataLoopSummary::default(), + data_loop_signature_bytes: 0, } } @@ -205,38 +213,43 @@ impl<'visitor> Accumulator<'visitor> { decode_error: None, panic_payload: None, schema_cache: tdh_decode::EventSchemaCache::default(), + data_loop: DataLoopSummary::default(), + data_loop_signature_bytes: 0, } } - /// Whether an event for `pid` at `filetime` falls within the attested - /// sandbox process lifetimes. Legacy full-trace analysis (no lifetime - /// index) treats every event as in scope. - fn in_analysis_scope(&self, pid: u32, filetime: u64) -> bool { - self.process_lifetimes - .as_ref() - .is_none_or(|lifetimes| lifetimes.contains(pid, filetime)) - } - fn add_raw_denial(&mut self, raw: RawDenial) { - if self - .process_lifetimes - .as_ref() - .is_some_and(|lifetimes| !lifetimes.contains(raw.pid, raw.filetime)) - { + if !self.event_in_scope(raw.pid, raw.filetime) { return; } let resource = if raw.resource_type == learning_mode_core::ResourceType::File { match path_norm::to_user_visible(&raw.object_name) { Some(resource) if path_norm::is_user_visible_absolute(&resource) => resource, - Some(_) => return, + Some(candidate) => { + self.record_raw_denial_outcome( + &raw, + &candidate, + DataLoopExclusionReason::UnusableResourcePath, + ); + return; + } None if path_norm::is_user_visible_absolute(&raw.object_name) => { raw.object_name.clone() } - None => return, + None => { + let candidate = raw.object_name.clone(); + self.record_raw_denial_outcome( + &raw, + &candidate, + DataLoopExclusionReason::UnusableResourcePath, + ); + return; + } } } else { path_norm::to_user_visible(&raw.object_name).unwrap_or_else(|| raw.object_name.clone()) }; + self.record_raw_denial_outcome(&raw, &resource, DataLoopExclusionReason::CanonicalDenial); let dedup_resource = match raw.resource_type { learning_mode_core::ResourceType::File | learning_mode_core::ResourceType::Other => { resource.to_ascii_lowercase() @@ -249,9 +262,13 @@ impl<'visitor> Accumulator<'visitor> { { return; } + // The canonical unique-denial bound is reached: keep reading (up to + // the processed-event bound) so every remaining outcome is still + // aggregated into the Data Loop summary, rather than stopping the + // trace early. if self.denials.len() >= MAX_UNIQUE_DENIALS { self.truncated = true; - self.stop_requested = true; + self.data_loop.mark_canonical_denial_limit_reached(); return; } self.seen.insert((dedup_resource, raw.access_type)); @@ -264,10 +281,102 @@ impl<'visitor> Accumulator<'visitor> { }); } + /// Records an outcome for an already-built [`RawDenial`]. Retains the + /// resolved resource/capability identity (redacting only its + /// user-profile path segment, if any) plus the resource/access type — + /// never the exact `filetime`. + fn record_raw_denial_outcome( + &mut self, + raw: &RawDenial, + resource: &str, + reason: DataLoopExclusionReason, + ) { + let mut properties = raw + .data_loop_properties + .iter() + .cloned() + .collect::>(); + let has_object_name = properties + .keys() + .any(|name| name.eq_ignore_ascii_case("ObjectName")); + if !matches!( + raw.resource_type, + learning_mode_core::ResourceType::File | learning_mode_core::ResourceType::Other + ) || !has_object_name + { + properties.insert( + "resource".to_string(), + crate::extractors::redact_username_in_path(resource), + ); + } + let properties = + crate::extractors::bound_properties(properties.into_iter().collect::>()); + self.record_outcome( + raw.provider, + raw.event_id, + reason, + raw.pid, + (Some(raw.access_type), Some(raw.resource_type)), + properties, + ); + } + + /// Records one excluded outcome as a deduplicated Data Loop signature: + /// symbolic provider, provider GUID, event ID, reason, PID, and the + /// already-sanitized/bounded property list all identify the group; + /// repeats of the same signature only increment its `count`. + fn record_exclusion( + &mut self, + provider: DataLoopProvider, + event_id: u16, + reason: DataLoopExclusionReason, + pid: u32, + properties: Vec<(String, String)>, + ) { + self.record_outcome(provider, event_id, reason, pid, (None, None), properties); + } + + fn record_outcome( + &mut self, + provider: DataLoopProvider, + event_id: u16, + reason: DataLoopExclusionReason, + pid: u32, + classification: ( + Option, + Option, + ), + properties: Vec<(String, String)>, + ) { + let (access_type, resource_type) = classification; + let signature = DataLoopSignature { + provider, + provider_guid: crate::extractors::provider_guid_string(provider), + event_id, + reason, + pid, + access_type, + resource_type, + properties, + }; + self.data_loop.record_with_byte_budget( + signature, + &mut self.data_loop_signature_bytes, + MAX_DATA_LOOP_SIGNATURE_BYTES, + ); + } + + fn event_in_scope(&self, pid: u32, filetime: u64) -> bool { + self.process_lifetimes + .as_ref() + .is_none_or(|lifetimes| lifetimes.contains(pid, filetime)) + } + fn begin_event(&mut self) -> bool { if self.processed_event_count >= MAX_PROCESSED_EVENTS { self.processing_limit_reached = true; self.truncated = true; + self.data_loop.mark_processed_events_truncated(); self.stop_requested = true; return false; } @@ -288,16 +397,62 @@ impl<'visitor> Accumulator<'visitor> { } } + /// Handles a TDH decode failure for one event. + /// + /// Schema-level failures (the event's manifest itself could not be + /// resolved) are fatal in every mode: they indicate the trace/schema + /// state is unreliable beyond this single event. Per-event decode + /// failures are fatal only for the raw diagnostic visitor (which needs + /// every event to succeed); in [`CollectionMode::Analyze`] they are + /// aggregated into the Data Loop summary for a known provider instead of + /// silently dropped. fn record_event_decode_error( &mut self, provider: windows::core::GUID, event_id: u16, + pid: u32, error: tdh_decode::DecodeError, ) { - if (matches!(self.mode, CollectionMode::Raw) || error.is_schema_error()) - && self.decode_error.is_none() - { - self.decode_error = Some(format!("provider {:?} event {event_id}: {error}", provider)); + let fatal = matches!(self.mode, CollectionMode::Raw) || error.is_schema_error(); + if fatal { + if self.decode_error.is_none() { + self.decode_error = + Some(format!("provider {:?} event {event_id}: {error}", provider)); + } + return; + } + if let Some(category) = crate::extractors::provider_category(provider) { + let reason = match error.event_kind() { + Some(tdh_decode::EventDecodeKind::PayloadMalformed) => { + DataLoopExclusionReason::EventPayloadMalformed + } + Some(tdh_decode::EventDecodeKind::DecoderLimitReached) => { + DataLoopExclusionReason::DecoderLimitReached + } + Some(tdh_decode::EventDecodeKind::UnsupportedPropertyEncoding) => { + DataLoopExclusionReason::UnsupportedPropertyEncoding + } + None => return, + }; + // Retain only the bounded schema-declared name. The free-form + // decoder message can include property values and is never emitted. + let properties = error + .event_name() + .map(|name| vec![("EventName".to_string(), name.to_string())]) + .map(crate::extractors::bound_properties) + .unwrap_or_default(); + let classification = match event_id { + crate::extractors::LEARNING_MODE_VIOLATION_EVENT_ID => ( + Some(learning_mode_core::AccessType::Unknown), + Some(learning_mode_core::ResourceType::Ui), + ), + crate::extractors::CAPABILITY_DENIAL_EVENT_ID => ( + Some(learning_mode_core::AccessType::Unknown), + Some(learning_mode_core::ResourceType::Capability), + ), + _ => (None, None), + }; + self.record_outcome(category, event_id, reason, pid, classification, properties); } } @@ -305,10 +460,22 @@ impl<'visitor> Accumulator<'visitor> { if let Some(error) = self.decode_error { return Err(AnalyzeError::Decode(error)); } - Ok(AnalysisResult { + let mut result = AnalysisResult { denials: self.denials, denied_resources_truncated: self.truncated, - }) + data_loop: self.data_loop, + }; + match result.fit_data_loop_within_serialized_bytes( + crate::guarded_wpr_protocol::MAX_ANALYSIS_BYTES as usize, + ) { + Ok(true) => Ok(result), + Ok(false) => Err(AnalyzeError::Decode( + "canonical Learning Mode analysis exceeds the guarded transport limit".to_string(), + )), + Err(error) => Err(AnalyzeError::Decode(format!( + "failed to size Learning Mode analysis: {error}" + ))), + } } } @@ -361,9 +528,10 @@ impl DenialAnalyzer for EtlDenialAnalyzer { } } -/// Runs the pure decode composition over already-collected events: -/// route each event through [`extract_denial`], then normalise + -/// de-duplicate into public [`DeniedResource`]s. Split out from +/// Runs the pure decode composition over already-collected events, using the +/// same event-classification path ([`handle_decoded_event`]) as the real ETW +/// callback: provider/vocabulary gating, extraction, capability-DACL +/// fallback, and Data Loop aggregation. Split out from /// [`EtlDenialAnalyzer::analyze`] so it can be tested with hand-built events /// that mirror real traces, without a live ETW/TDH read (which needs the /// provider manifests registered on the machine). @@ -377,26 +545,15 @@ fn resources_from_events_for_process_lifetimes( events: &[CollectedEvent], process_lifetimes: Option<&[ProcessLifetime]>, ) -> AnalysisResult { - let mut raws = Vec::new(); - for event in events { - if let Some(raw) = extract_denial(&event.parts, event.pid, event.filetime) { - raws.push(raw); - } - raws.extend(crate::capability_dacl::extract_denials( - &event.parts, - event.pid, - event.filetime, - )); - } let mut accumulator = match process_lifetimes { Some(lifetimes) => Accumulator::analyze_for_process_lifetimes(lifetimes), None => Accumulator::analyze(), }; - for raw in raws { - accumulator.add_raw_denial(raw); - if accumulator.stop_requested { + for event in events { + if !accumulator.begin_event() { break; } + handle_decoded_event(&event.parts, event.pid, event.filetime, &mut accumulator); } accumulator .into_analysis() @@ -454,9 +611,6 @@ fn dedup_to_resources>(raws: I) -> AnalysisRes let mut accumulator = Accumulator::analyze(); for raw in raws { accumulator.add_raw_denial(raw); - if accumulator.stop_requested { - break; - } } accumulator .into_analysis() @@ -593,7 +747,6 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let header = unsafe { (*event_record).EventHeader }; let provider = header.ProviderId; let event_id = header.EventDescriptor.Id; - if matches!(acc.mode, CollectionMode::SelectForRelogging) { if !is_learning_mode_event(provider, event_id) { return; @@ -609,7 +762,7 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let Some(filetime) = normalized_filetime(header.TimeStamp, acc) else { return; }; - if acc.in_analysis_scope(header.ProcessId, filetime) { + if acc.event_in_scope(header.ProcessId, filetime) { if acc.relog_selected_event_indices.len() >= MAX_PROCESSED_EVENTS { acc.decode_error = Some(format!( "trace exceeded the {MAX_PROCESSED_EVENTS}-event process-scoped relogging limit" @@ -622,51 +775,123 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu return; } - // Establish scope BEFORE charging the event against the shared processing - // budget. Provider, event id, PID and timestamp all live in the event - // header, so the scope test needs no (comparatively expensive) TDH decode. - // Out-of-scope host events — a foreign provider, or a PID/time outside the - // attested sandbox process lifetimes — must not consume the budget; - // otherwise a noisy host could exhaust the limit before a single in-scope - // sandbox denial is ever decoded, truncating the analysis of an innocent - // sandbox. + let mut analyze_filetime = None; + if matches!(acc.mode, CollectionMode::Analyze) { - if !is_learning_mode_event(provider, event_id) { + let Some(category) = crate::extractors::provider_category(provider) else { + // Unrelated provider: unrelated host traffic, ignored entirely + // (not aggregated as an excluded Learning Mode outcome). return; - } + }; let Some(filetime) = normalized_filetime(header.TimeStamp, acc) else { return; }; - if !acc.in_analysis_scope(header.ProcessId, filetime) { - return; - } - if !acc.begin_event() { + analyze_filetime = Some(filetime); + if !is_learning_mode_event(provider, event_id) { + if !acc.event_in_scope(header.ProcessId, filetime) || !acc.begin_event() { + return; + } + // A known provider, but outside its supported event vocabulary: + // aggregate (as a signature with no decoded properties) without + // paying for a TDH decode. + acc.record_exclusion( + category, + event_id, + DataLoopExclusionReason::UnsupportedEventSchema, + header.ProcessId, + Vec::new(), + ); return; } - match unsafe { tdh_decode::decode_event_parts(event_record, &mut acc.schema_cache) } { - Ok(parts) => { - if let Some(raw) = extract_denial(&parts, header.ProcessId, filetime) { - acc.add_raw_denial(raw); + } + + match unsafe { tdh_decode::decode_event_parts(event_record, &mut acc.schema_cache) } { + Ok(parts) => match acc.mode { + CollectionMode::Analyze => { + let filetime = analyze_filetime.expect("analyze mode has normalized FILETIME"); + handle_decoded_event(&parts, header.ProcessId, filetime, acc); + } + CollectionMode::Raw => { + if acc.begin_event() { + acc.visit_raw_event(&parts); } - for raw in - crate::capability_dacl::extract_denials(&parts, header.ProcessId, filetime) + } + CollectionMode::SelectForRelogging => unreachable!("relogging returns before decode"), + }, + Err(error) => { + if matches!(acc.mode, CollectionMode::Analyze) { + if !acc.event_in_scope( + header.ProcessId, + analyze_filetime.expect("analyze mode has normalized FILETIME"), + ) || !acc.begin_event() { - acc.add_raw_denial(raw); + return; } + } else if !acc.begin_event() { + return; } - Err(error) => acc.record_event_decode_error(provider, event_id, error), + acc.record_event_decode_error(provider, event_id, header.ProcessId, error); } - return; } +} - // Raw diagnostic mode has no provider/lifetime scoping, so every decoded - // event legitimately counts against the budget. - if !acc.begin_event() { +/// Extracts denials from one decoded, in-vocabulary event and feeds them +/// (or their closed exclusion reason) into `acc`. +/// +/// Re-checks the provider/event vocabulary so it stays a single source of +/// truth for both the real ETW path (which gates before decoding, above) +/// and pure-composition tests that hand this already-"decoded" fixtures. +fn handle_decoded_event( + parts: &DecodedEventParts, + header_pid: u32, + filetime: u64, + acc: &mut Accumulator<'_>, +) { + let pid = crate::extractors::effective_event_pid(parts, header_pid); + if !acc.event_in_scope(pid, filetime) || !acc.begin_event() { return; } - match unsafe { tdh_decode::decode_event_parts(event_record, &mut acc.schema_cache) } { - Ok(parts) => acc.visit_raw_event(&parts), - Err(error) => acc.record_event_decode_error(provider, event_id, error), + let Some(category) = crate::extractors::provider_category(parts.provider) else { + return; + }; + if !is_learning_mode_event(parts.provider, parts.event_id) { + acc.record_exclusion( + category, + parts.event_id, + DataLoopExclusionReason::UnsupportedEventSchema, + pid, + crate::extractors::sanitize_properties(&parts.props), + ); + return; + } + + let primary = extract_denial(parts, pid, filetime); + // Some permissive capability checks leave the primary access-check event + // unidentified (`ObjectType`/`ObjectName` both empty); the identifier may + // still be recoverable from the event's DACL payload. Each recovered + // candidate is fed independently, and the primary `UnresolvedCapability` + // outcome is only counted if none of them panned out. + let capability_candidates = crate::capability_dacl::extract_denials(parts, pid, filetime); + for raw in capability_candidates.iter().cloned() { + acc.add_raw_denial(raw); + } + + match primary { + Ok(raw) => acc.add_raw_denial(raw), + Err(reason) => { + let recovered_by_dacl = reason == DataLoopExclusionReason::UnresolvedCapability + && !capability_candidates.is_empty(); + if !recovered_by_dacl { + acc.record_outcome( + category, + parts.event_id, + reason, + pid, + crate::extractors::data_loop_classification(parts), + crate::extractors::sanitize_properties(&parts.props), + ); + } + } } } @@ -794,7 +1019,12 @@ mod tests { accumulator.record_event_decode_error( windows::core::GUID::from_u128(1), 14, - tdh_decode::DecodeError::Event("malformed property".to_string()), + 1, + tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::PayloadMalformed, + "malformed property".to_string(), + None, + ), ); assert!(accumulator.into_analysis().is_ok()); @@ -807,7 +1037,12 @@ mod tests { accumulator.record_event_decode_error( windows::core::GUID::from_u128(1), 14, - tdh_decode::DecodeError::Event("malformed property".to_string()), + 1, + tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::PayloadMalformed, + "malformed property".to_string(), + None, + ), ); let error = accumulator.decode_error.as_deref().unwrap(); @@ -821,6 +1056,7 @@ mod tests { accumulator.record_event_decode_error( windows::core::GUID::from_u128(1), 14, + 1, tdh_decode::DecodeError::Schema("manifest unavailable".to_string()), ); @@ -836,6 +1072,8 @@ mod tests { access_type: access, filetime: 1, event_id: 4907, + provider: learning_mode_core::DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + data_loop_properties: Vec::new(), } } @@ -920,7 +1158,25 @@ mod tests { } #[test] - fn unique_denial_bound_requests_trace_stop() { + fn unique_denial_bound_marks_truncated_without_stopping() { + fn exclusion_count( + summary: &learning_mode_core::DataLoopSummary, + provider: learning_mode_core::DataLoopProvider, + event_id: u16, + reason: DataLoopExclusionReason, + ) -> u64 { + summary + .signatures + .iter() + .filter(|group| { + group.signature.provider == provider + && group.signature.event_id == event_id + && group.signature.reason == reason + }) + .map(|group| group.count) + .sum() + } + let mut accumulator = Accumulator::analyze(); accumulator.denials = (0..MAX_UNIQUE_DENIALS) .map(|index| DeniedResource { @@ -931,6 +1187,9 @@ mod tests { filetime: 1, }) .collect(); + accumulator.seen = (0..MAX_UNIQUE_DENIALS) + .map(|index| (format!(r"c:\data\{index}.txt"), AccessType::Read)) + .collect(); accumulator.add_raw_denial(raw( r"C:\data\overflow.txt", @@ -938,8 +1197,42 @@ mod tests { ResourceType::File, )); - assert!(accumulator.stop_requested); + assert!( + !accumulator.stop_requested, + "hitting the unique-denial cap must not halt the trace early" + ); assert!(accumulator.truncated); + assert!(accumulator.data_loop.canonical_denial_limit_reached); + assert_eq!( + exclusion_count( + &accumulator.data_loop, + learning_mode_core::DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + 4907, + DataLoopExclusionReason::CanonicalDenial + ), + 1 + ); + + // Processing continues past the cap: a further overflow candidate is + // still aggregated (not silently discarded), and a duplicate of an + // already-canonical denial retains the same canonical classification. + accumulator.add_raw_denial(raw( + r"C:\data\overflow-2.txt", + AccessType::Read, + ResourceType::File, + )); + accumulator.add_raw_denial(raw(r"C:\data\0.txt", AccessType::Read, ResourceType::File)); + + assert!(!accumulator.stop_requested); + assert_eq!( + exclusion_count( + &accumulator.data_loop, + learning_mode_core::DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + 4907, + DataLoopExclusionReason::CanonicalDenial + ), + 3 + ); } #[test] @@ -970,12 +1263,12 @@ mod tests { start_filetime: 100, end_filetime: 200, }]); - assert!(scoped.in_analysis_scope(7, 150)); - assert!(!scoped.in_analysis_scope(7, 250), "outside the time range"); - assert!(!scoped.in_analysis_scope(9, 150), "unrelated PID"); + assert!(scoped.event_in_scope(7, 150)); + assert!(!scoped.event_in_scope(7, 250), "outside the time range"); + assert!(!scoped.event_in_scope(9, 150), "unrelated PID"); let legacy = Accumulator::analyze(); - assert!(legacy.in_analysis_scope(9, 150), "no lifetime filter"); + assert!(legacy.event_in_scope(9, 150), "no lifetime filter"); } #[test] @@ -1182,7 +1475,38 @@ mod tests { kernel_event(28, 0, 2, &[("ProcessId", "0x10"), ("Denied", "true")]), kernel_event(28, 0, 3, &[("ProcessId", "0x20"), ("Denied", "true")]), ]; - assert!(resources_from_events(&events).denials.is_empty()); + let out = resources_from_events(&events); + assert!(out.denials.is_empty()); + + // Every dropped candidate is still aggregated by closed reason, never + // by raw payload: one unresolved access-check event (14) plus two + // unresolved capability denials (28). The two event-28 signatures stay + // distinct because their retained ProcessId properties differ. + let groups = &out.data_loop.signatures; + assert_eq!(groups.len(), 3); + let event_14_group = groups + .iter() + .find(|group| group.signature.event_id == 14) + .expect("event 14 exclusion group present"); + assert_eq!( + event_14_group.signature.provider, + DataLoopProvider::KernelGeneral + ); + assert_eq!( + event_14_group.signature.reason, + DataLoopExclusionReason::UnresolvedCapability + ); + assert_eq!(event_14_group.count, 1); + let event_28_groups = groups + .iter() + .filter(|group| group.signature.event_id == 28) + .collect::>(); + assert_eq!(event_28_groups.len(), 2); + assert!(event_28_groups.iter().all(|group| { + group.signature.provider == DataLoopProvider::KernelGeneral + && group.signature.reason == DataLoopExclusionReason::UnresolvedCapability + && group.count == 1 + })); } #[test] @@ -1198,6 +1522,8 @@ mod tests { ("AccessMask", "0x1"), ], ), + kernel_event(9999, 43, 150, &[("Marker", "\"host\"")]), + kernel_event(9999, 42, 151, &[("Marker", "\"owned\"")]), kernel_event( 14, 42, @@ -1239,6 +1565,56 @@ mod tests { assert_eq!(analysis.denials.len(), 1); assert_eq!(analysis.denials[0].resource, r"C:\owned.txt"); + assert_eq!(analysis.data_loop.signatures.len(), 2); + assert!(analysis + .data_loop + .signatures + .iter() + .all(|group| group.signature.pid == 42)); + let unsupported = analysis + .data_loop + .signatures + .iter() + .find(|group| group.signature.reason == DataLoopExclusionReason::UnsupportedEventSchema) + .expect("owned unknown event retained"); + assert_eq!(property(&unsupported.signature, "Marker"), "owned"); + } + + #[test] + fn brokered_capability_uses_payload_pid_for_process_scope() { + let events = vec![kernel_event( + 28, + 900, + 150, + &[ + ("Denied", "\"true\""), + ("ProcessId", "0x2a"), + ("CapabilityName", "\"internetClient\""), + ], + )]; + let lifetimes = [ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]; + + let analysis = resources_from_events_for_process_lifetimes(&events, Some(&lifetimes)); + + assert_eq!(analysis.denials.len(), 1); + assert_eq!(analysis.denials[0].pid, 42); + assert_eq!(analysis.denials[0].resource, "internetClient"); + let group = analysis + .data_loop + .signatures + .iter() + .find(|group| group.signature.reason == DataLoopExclusionReason::CanonicalDenial) + .expect("canonical capability event retained"); + assert_eq!(group.signature.pid, 42); + assert_eq!(group.signature.access_type, Some(AccessType::Unknown)); + assert_eq!( + group.signature.resource_type, + Some(ResourceType::Capability) + ); } #[test] @@ -1257,10 +1633,13 @@ mod tests { let analysis = resources_from_events_for_process_lifetimes(&events, Some(&[])); assert!(analysis.denials.is_empty()); + assert!(analysis.data_loop.is_empty()); } /// Non-actionable object types and not-denied capability records are - /// dropped by the pipeline; unknown event IDs are ignored. + /// dropped by the pipeline as closed extraction reasons; an unknown event + /// ID from a known provider is classified `UnsupportedEventSchema` + /// without ever reaching TDH-decoded extraction logic. #[test] fn non_actionable_events_are_dropped() { let events = vec![ @@ -1268,7 +1647,25 @@ mod tests { kernel_event(28, 0, 2, &[("ProcessId", "0x10"), ("Denied", "false")]), kernel_event(9999, 1, 3, &[("Foo", "\"bar\"")]), ]; - assert!(resources_from_events(&events).denials.is_empty()); + let out = resources_from_events(&events); + assert!(out.denials.is_empty()); + + let reason_for = |event_id: u16| { + out.data_loop + .signatures + .iter() + .find(|group| group.signature.event_id == event_id) + .map(|group| group.signature.reason) + }; + assert_eq!( + reason_for(14), + Some(DataLoopExclusionReason::UnsupportedObjectType) + ); + assert_eq!(reason_for(28), Some(DataLoopExclusionReason::NotActionable)); + assert_eq!( + reason_for(9999), + Some(DataLoopExclusionReason::UnsupportedEventSchema) + ); } #[test] @@ -1292,6 +1689,388 @@ mod tests { ), ]; - assert!(resources_from_events(&events).denials.is_empty()); + let out = resources_from_events(&events); + assert!(out.denials.is_empty()); + + // Each event ID is valid for the *other* known provider, so both are + // classified `UnsupportedEventSchema` for their own provider rather + // than silently ignored or misrouted. + assert_eq!(out.data_loop.signatures.len(), 2); + assert!( + out.data_loop + .signatures + .iter() + .all(|group| group.signature.reason + == DataLoopExclusionReason::UnsupportedEventSchema) + ); + assert!(out + .data_loop + .signatures + .iter() + .any(|group| group.signature.provider + == DataLoopProvider::PrivacyAuditingPermissiveLearningMode + && group.signature.event_id == 28)); + assert!(out + .data_loop + .signatures + .iter() + .any( + |group| group.signature.provider == DataLoopProvider::KernelGeneral + && group.signature.event_id == 4907 + )); + } + + #[test] + fn unrelated_provider_is_ignored_without_accounting() { + // A provider outside the known Learning Mode vocabulary must not + // contribute any Data Loop accounting at all, even though its event + // ID happens to collide with a known access-check ID. + let events = vec![event_with_provider( + windows::core::GUID::from_u128(0xdead_beef), + 14, + 1, + 1, + &[ + ("ObjectType", "\"File\""), + ("ObjectName", "\"C:\\unrelated.txt\""), + ("AccessMask", "0x1"), + ], + )]; + + let out = resources_from_events(&events); + assert!(out.denials.is_empty()); + assert!( + out.data_loop.is_empty(), + "unrelated providers are ignored, not aggregated" + ); + } + + #[test] + fn canonical_candidates_and_duplicates_share_one_data_loop_signature() { + let make_event = |sequence_no: u64| { + kernel_event( + 14, + 7, + sequence_no, + &[ + ("ObjectType", "\"File\""), + ("ObjectName", "\"D:\\profiles\\jsmith\\dup.txt\""), + ("AccessMask", "0x1"), + ("UserName", "\"jsmith\""), + ], + ) + }; + let events = vec![make_event(1), make_event(2), make_event(3)]; + + let out = resources_from_events(&events); + assert_eq!(out.denials.len(), 1, "duplicates collapse to one denial"); + + let canonical_group = out + .data_loop + .signatures + .iter() + .find(|group| group.signature.reason == DataLoopExclusionReason::CanonicalDenial) + .expect("canonical outcome recorded"); + assert_eq!( + canonical_group.signature.provider, + DataLoopProvider::KernelGeneral + ); + assert_eq!(canonical_group.signature.event_id, 14); + assert_eq!( + canonical_group.count, 3, + "all three occurrences are retained" + ); + assert_eq!( + canonical_group.signature.access_type, + Some(AccessType::Read) + ); + assert_eq!( + canonical_group.signature.resource_type, + Some(ResourceType::File) + ); + assert_eq!( + property(&canonical_group.signature, "ObjectName"), + r"D:\profiles\\dup.txt" + ); + assert_eq!( + property(&canonical_group.signature, "UserName"), + "" + ); + } + + #[test] + fn processing_continues_past_the_unique_denial_cap_through_the_full_pipeline() { + let overflow_candidates = 5usize; + let events: Vec = (0..MAX_UNIQUE_DENIALS + overflow_candidates) + .map(|index| { + kernel_event( + 14, + 1, + index as u64, + &[ + ("ObjectType", "\"File\""), + ("ObjectName", &format!("\"C:\\data\\{index}.txt\"")), + ("AccessMask", "0x1"), + ], + ) + }) + .collect(); + + let out = resources_from_events(&events); + + assert_eq!(out.denials.len(), MAX_UNIQUE_DENIALS); + assert!(out.denied_resources_truncated); + assert!(out.data_loop.canonical_denial_limit_reached); + assert!(!out.data_loop.processed_events_truncated); + + // If the trace had stopped as soon as the cap was reached, only the + // The Data Loop accounts for every denial candidate, including those + // observed after the canonical unique-denial cap. + assert_eq!( + out.data_loop.total_occurrences, + (MAX_UNIQUE_DENIALS + overflow_candidates) as u64 + ); + } + + #[test] + fn capability_denial_recovered_from_dacl_records_no_exclusion() { + // Same shape as `allow_shape_recovers_capability_from_dacl`, but + // asserting the Data Loop side: a successfully DACL-recovered + // capability candidate must not also surface an `UnresolvedCapability` + // exclusion for the same event. + let dacl = "hex:000000000000000001000000010200000000000F0300000001000000"; + let events = vec![permissive_event( + 14, + 5900, + 21, + &[ + ("Mode", "\"Permissive\""), + ("ObjectType", "\"\""), + ("ObjectName", "\"\""), + ("AccessMask", "0x1"), + ("Dacl", dacl), + ], + )]; + + let out = resources_from_events(&events); + assert_eq!(out.denials.len(), 1); + assert_eq!(out.denials[0].resource, "internetClient"); + assert_eq!(out.data_loop.signatures.len(), 1); + let signature = &out.data_loop.signatures[0].signature; + assert_eq!(signature.reason, DataLoopExclusionReason::CanonicalDenial); + assert_eq!(signature.access_type, Some(AccessType::Unknown)); + assert_eq!(signature.resource_type, Some(ResourceType::Capability)); + } + + // ---- Data Loop signature retention (identifiers + redaction + dedup) -- + + fn find_signature( + summary: &learning_mode_core::DataLoopSummary, + event_id: u16, + ) -> &learning_mode_core::DataLoopAggregate { + summary + .signatures + .iter() + .find(|group| group.signature.event_id == event_id) + .unwrap_or_else(|| panic!("expected a Data Loop signature for event {event_id}")) + } + + fn property<'a>(signature: &'a learning_mode_core::DataLoopSignature, name: &str) -> &'a str { + signature + .properties + .iter() + .find(|(key, _)| key == name) + .unwrap_or_else(|| panic!("expected property {name:?} in signature {signature:?}")) + .1 + .as_str() + } + + #[test] + fn data_loop_byte_budget_preserves_guarded_protocol_headroom() { + let mut accumulator = Accumulator::analyze(); + let properties = (0..crate::extractors::MAX_SIGNATURE_PROPERTIES) + .map(|index| { + ( + format!("Property{index:02}"), + "x".repeat(crate::extractors::MAX_SIGNATURE_VALUE_LEN), + ) + }) + .collect::>(); + for pid in 0..learning_mode_core::MAX_DATA_LOOP_GROUPS as u32 { + accumulator.record_exclusion( + DataLoopProvider::KernelGeneral, + 14, + DataLoopExclusionReason::CanonicalDenial, + pid, + properties.clone(), + ); + } + + assert!(accumulator.data_loop.aggregate_groups_truncated); + assert!(accumulator.data_loop.overflow_occurrences > 0); + assert!(accumulator.data_loop_signature_bytes <= MAX_DATA_LOOP_SIGNATURE_BYTES); + } + + #[test] + fn unknown_event_id_signature_redacts_only_the_username_segment_of_a_path() { + let events = vec![kernel_event( + 9999, + 7, + 1, + &[("ObjectName", "\"C:\\Users\\jsmith\\secret.txt\"")], + )]; + + let out = resources_from_events(&events); + + let group = find_signature(&out.data_loop, 9999); + assert_eq!( + group.signature.reason, + DataLoopExclusionReason::UnsupportedEventSchema + ); + assert_eq!( + property(&group.signature, "ObjectName"), + r"C:\Users\\secret.txt", + "the rest of the path must survive redaction unchanged" + ); + } + + #[test] + fn unresolved_capability_signature_retains_sid_pid_and_provider_guid() { + let events = vec![kernel_event( + 28, + 42, + 1, + &[ + ("ProcessId", "0x2A"), + ("Denied", "true"), + ("CandidateSid", "\"S-1-15-3-1\""), + ], + )]; + + let out = resources_from_events(&events); + + let group = find_signature(&out.data_loop, 28); + assert_eq!( + group.signature.reason, + DataLoopExclusionReason::UnresolvedCapability + ); + assert_eq!(group.signature.provider, DataLoopProvider::KernelGeneral); + assert_eq!( + group.signature.provider_guid, + crate::extractors::provider_guid_string(DataLoopProvider::KernelGeneral) + ); + assert_eq!(group.signature.pid, 42, "process identifier retained"); + assert_eq!( + property(&group.signature, "CandidateSid"), + "S-1-15-3-1", + "capability SIDs are retained identifiers, never redacted" + ); + } + + #[test] + fn signatures_with_identical_properties_dedupe_across_differing_timestamps() { + // Same event id/pid/properties, only `filetime` differs: the + // signature must exclude the exact timestamp so both collapse into + // one group with an incremented count, rather than two singletons. + let events = vec![ + kernel_event(9999, 7, 10, &[("Foo", "\"bar\"")]), + kernel_event(9999, 7, 20_000_000, &[("Foo", "\"bar\"")]), + ]; + + let out = resources_from_events(&events); + + assert_eq!( + out.data_loop.signatures.len(), + 1, + "differing only by timestamp must dedupe to a single signature" + ); + assert_eq!(find_signature(&out.data_loop, 9999).count, 2); + } + + #[test] + fn timestamp_like_properties_are_excluded_from_the_signature() { + let events = vec![kernel_event( + 9999, + 7, + 1, + &[ + ("Foo", "\"bar\""), + ("LastWriteTime", "\"132847890123456789\""), + ], + )]; + + let out = resources_from_events(&events); + + let group = find_signature(&out.data_loop, 9999); + assert!( + group + .signature + .properties + .iter() + .all(|(name, _)| !name.to_ascii_lowercase().contains("time")), + "timestamp-like properties must never reach the signature: {:?}", + group.signature.properties + ); + } + + #[test] + fn event_decode_failures_use_closed_reasons_and_schema_names() { + let mut accumulator = Accumulator::analyze(); + accumulator.record_event_decode_error( + crate::extractors::KERNEL_GENERAL_PROVIDER, + 14, + 9, + tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::PayloadMalformed, + "property payload is malformed".to_string(), + Some("AccessCheck".to_string()), + ), + ); + accumulator.record_event_decode_error( + crate::extractors::KERNEL_GENERAL_PROVIDER, + 27, + 10, + tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::DecoderLimitReached, + "property decode work exceeds limit 100000".to_string(), + Some("LearningModeViolation".to_string()), + ), + ); + accumulator.record_event_decode_error( + crate::extractors::KERNEL_GENERAL_PROVIDER, + 28, + 11, + tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::UnsupportedPropertyEncoding, + "property has unsupported variable length".to_string(), + Some("CapabilityDenial".to_string()), + ), + ); + let out = accumulator + .into_analysis() + .expect("per-event decode failures are non-fatal in Analyze mode"); + + let malformed = find_signature(&out.data_loop, 14); + assert_eq!( + malformed.signature.reason, + DataLoopExclusionReason::EventPayloadMalformed + ); + assert_eq!(malformed.signature.pid, 9); + assert_eq!(property(&malformed.signature, "EventName"), "AccessCheck"); + assert_eq!( + find_signature(&out.data_loop, 27).signature.reason, + DataLoopExclusionReason::DecoderLimitReached + ); + assert_eq!( + find_signature(&out.data_loop, 28).signature.reason, + DataLoopExclusionReason::UnsupportedPropertyEncoding + ); + assert!(out.data_loop.signatures.iter().all(|aggregate| { + aggregate + .signature + .properties + .iter() + .all(|(name, _)| name == "EventName") + })); } } diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index e61da0c23..42edbaa6d 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -20,7 +20,10 @@ //! //! The learning-mode ETL carries a set of event IDs that map onto the //! resource types we surface. This list grows as more denial sources are -//! decoded; unknown event IDs are discarded. The IDs handled today: +//! decoded; event IDs outside this vocabulary are excluded rather than +//! extracted, and (for the known providers below) that exclusion is +//! aggregated into [`learning_mode_core::DataLoopSummary`] rather than +//! silently dropped. The IDs handled today: //! //! - **14 / 4907 — access check** — the primary denial event //! (`ObjectType` / `ObjectName` / `AccessMask`). `ObjectType` selects the @@ -44,7 +47,7 @@ //! hashed capabilities fall back to the SID string). Records without a //! decoded identifier are omitted. -use learning_mode_core::{AccessType, ResourceType}; +use learning_mode_core::{AccessType, DataLoopExclusionReason, DataLoopProvider, ResourceType}; use windows::core::GUID; /// Microsoft-Windows-Kernel-General provider. @@ -103,23 +106,84 @@ pub struct RawDenial { pub filetime: u64, /// Originating ETW event ID (kept for diagnostics). pub event_id: u16, + /// Symbolic category of the originating provider, for Data Loop + /// aggregation. Never a raw provider GUID. + pub provider: DataLoopProvider, + /// Bounded username-redacted properties retained for Data Loop signatures. + pub data_loop_properties: Vec<(String, String)>, } /// Routes a decoded event to the matching extractor by its event ID. /// -/// Returns `None` for events that are not learning-mode denials or that -/// carry an object type we don't surface. -pub fn extract_denial(parts: &DecodedEventParts, pid: u32, filetime: u64) -> Option { +/// Returns `Err` with a closed [`DataLoopExclusionReason`] for events that +/// are not learning-mode denials, that carry an object type we don't +/// surface, or that otherwise fail extraction. Callers aggregate the +/// returned reason into [`learning_mode_core::DataLoopSummary`] rather than +/// discarding it, except when a fallback extractor (e.g. +/// [`crate::capability_dacl`]) recovers an equivalent denial from the same +/// event. +pub fn extract_denial( + parts: &DecodedEventParts, + pid: u32, + filetime: u64, +) -> Result { + // Callers on the real trace path gate on `is_learning_mode_event` before + // decoding via TDH at all, so this branch only matters for direct/test + // callers that skip that gate. + let provider = + provider_category(parts.provider).ok_or(DataLoopExclusionReason::UnsupportedEventSchema)?; if !is_learning_mode_event(parts.provider, parts.event_id) { - return None; + return Err(DataLoopExclusionReason::UnsupportedEventSchema); } + match parts.event_id { ACCESS_CHECK_EVENT_ID | PRIVACY_ACCESS_CHECK_EVENT_ID => { - build_denial_from_access_check(parts, pid, filetime) + build_denial_from_access_check(parts, pid, filetime, provider) } - LEARNING_MODE_VIOLATION_EVENT_ID => build_denial_from_learning_mode(parts, pid, filetime), - CAPABILITY_DENIAL_EVENT_ID => build_denial_from_capability(parts, pid, filetime), - _ => None, + LEARNING_MODE_VIOLATION_EVENT_ID => { + build_denial_from_learning_mode(parts, pid, filetime, provider) + } + CAPABILITY_DENIAL_EVENT_ID => build_denial_from_capability(parts, pid, filetime, provider), + _ => Err(DataLoopExclusionReason::UnsupportedEventSchema), + } +} + +/// Returns typed denial classifications that can be determined independently +/// of whether the event contains every property required for canonical output. +pub(crate) fn data_loop_classification( + parts: &DecodedEventParts, +) -> (Option, Option) { + match parts.event_id { + ACCESS_CHECK_EVENT_ID | PRIVACY_ACCESS_CHECK_EVENT_ID => { + let Some(object_type) = find_prop(&parts.props, "ObjectType") else { + return (None, None); + }; + match object_type.trim_matches('"') { + "File" => ( + Some( + find_prop(&parts.props, "AccessMask") + .and_then(|value| parse_u32(value)) + .map(|mask| access_type_from_mask(mask, false)) + .unwrap_or(AccessType::Unknown), + ), + Some(ResourceType::File), + ), + "Key" => ( + Some( + find_prop(&parts.props, "AccessMask") + .and_then(|value| parse_u32(value)) + .map(|mask| access_type_from_mask(mask, true)) + .unwrap_or(AccessType::Unknown), + ), + Some(ResourceType::Other), + ), + "" => (Some(AccessType::Unknown), Some(ResourceType::Capability)), + _ => (None, None), + } + } + LEARNING_MODE_VIOLATION_EVENT_ID => (Some(AccessType::Unknown), Some(ResourceType::Ui)), + CAPABILITY_DENIAL_EVENT_ID => (Some(AccessType::Unknown), Some(ResourceType::Capability)), + _ => (None, None), } } @@ -142,7 +206,266 @@ pub(crate) fn is_learning_mode_event(provider: GUID, event_id: u16) -> bool { } } -/// Builds a [`RawDenial`] from an access-check (event 14 / 4907) payload. +pub(crate) fn effective_event_pid(parts: &DecodedEventParts, header_pid: u32) -> u32 { + if parts.event_id == CAPABILITY_DENIAL_EVENT_ID { + find_prop(&parts.props, "ProcessId") + .and_then(|value| parse_u32(value)) + .unwrap_or(header_pid) + } else { + header_pid + } +} + +/// Maps a raw ETW provider GUID to its symbolic Data Loop category. +/// +/// Returns `None` for providers outside the Learning Mode vocabulary; those +/// events are ignored entirely (not aggregated), since they are unrelated +/// host traffic rather than an excluded Learning Mode outcome. +pub(crate) fn provider_category(provider: GUID) -> Option { + if provider == KERNEL_GENERAL_PROVIDER { + Some(DataLoopProvider::KernelGeneral) + } else if provider == PRIVACY_LEARNING_MODE_PROVIDER { + Some(DataLoopProvider::PrivacyAuditingPermissiveLearningMode) + } else { + None + } +} + +/// Renders a symbolic Data Loop provider category back to its raw ETW +/// provider GUID, in canonical braced string form, for retention in a Data +/// Loop signature. Provider GUIDs are stable component identifiers, not +/// personal data, so unlike account/user values they are never redacted. +pub(crate) fn provider_guid_string(provider: DataLoopProvider) -> String { + match provider { + DataLoopProvider::KernelGeneral => format_guid(KERNEL_GENERAL_PROVIDER), + DataLoopProvider::PrivacyAuditingPermissiveLearningMode => { + format_guid(PRIVACY_LEARNING_MODE_PROVIDER) + } + } +} + +fn format_guid(guid: GUID) -> String { + format!( + "{{{:08X}-{:04X}-{:04X}-{:02X}{:02X}-{:02X}{:02X}{:02X}{:02X}{:02X}{:02X}}}", + guid.data1, + guid.data2, + guid.data3, + guid.data4[0], + guid.data4[1], + guid.data4[2], + guid.data4[3], + guid.data4[4], + guid.data4[5], + guid.data4[6], + guid.data4[7], + ) +} + +// ---- Data Loop signature sanitization ------------------------------------- +// +// A Data Loop signature retains identifiers useful for triage (SIDs, +// capability names, PIDs, provider/object GUIDs) but must never leak a +// human account/user name, an exact timestamp, or a free-form decoder error +// message. These bounds apply uniformly to every decoded property, in a +// fixed order: drop timestamp-like properties outright, redact +// account/user-identity content, *then* bound the surviving property count +// and value length — sanitizing always happens before truncation so a +// redaction is never chopped in half by a length cap. + +/// Fixed replacement for any redacted username/account-name value. +pub(crate) const REDACTED_USER: &str = ""; +/// Maximum number of `(name, value)` properties retained in one Data Loop +/// signature. Bounds pathological/huge TDH property lists; sanitization +/// (never raw decoding) determines which survive, via deterministic +/// (sorted-by-name) truncation. +pub(crate) const MAX_SIGNATURE_PROPERTIES: usize = 24; +/// Maximum retained length (in `char`s) of one sanitized property value. +pub(crate) const MAX_SIGNATURE_VALUE_LEN: usize = 256; + +/// Returns whether `name` looks like it carries a timestamp, so it is +/// dropped from a Data Loop signature entirely (never redacted or +/// truncated) — exact timestamps must not prevent otherwise-identical +/// events from deduplicating. +fn is_timestamp_like_property(name: &str) -> bool { + let lower = name.to_ascii_lowercase().replace(['_', '-'], ""); + matches!( + lower.as_str(), + "time" + | "timestamp" + | "filetime" + | "systemtime" + | "eventtime" + | "timecreated" + | "creationtime" + | "createtime" + | "starttime" + | "endtime" + | "exittime" + | "lastwritetime" + ) || lower.ends_with("timestamp") + || lower.ends_with("filetime") +} + +/// Returns whether `name` is itself a standalone user/account-identity +/// property (as opposed to a path that merely *contains* a username), so +/// its value is replaced outright with [`REDACTED_USER`] regardless of +/// content. +fn is_identity_property(name: &str) -> bool { + let lower = name.to_ascii_lowercase().replace(['_', '-'], ""); + if lower.contains("sid") + || lower.contains("guid") + || lower.contains("identifier") + || lower.ends_with("id") + { + return false; + } + lower == "user" + || lower == "account" + || lower == "owner" + || lower.contains("username") + || lower.contains("accountname") + || lower.contains("ownername") + || lower.contains("principalname") + || lower.ends_with("account") + || lower == "upn" + || lower.ends_with("upn") +} + +/// Redacts the user-profile segment of a Windows path, preserving +/// everything else. Matches a `\Users\` or legacy `\Documents and +/// Settings\` segment (case-insensitively, on an exact path-segment +/// boundary so e.g. `usersdata` never matches) and replaces the segment +/// immediately following it with [`REDACTED_USER`]. +/// +/// Values that aren't path-shaped (capability names, SIDs, GUIDs, small +/// enums, ...) never contain a matching segment, so they pass through +/// unchanged — this is intentionally a single sanitizer applied uniformly +/// rather than a per-property-name allowlist. +pub(crate) fn redact_username_in_path(value: &str) -> String { + const USER_PROFILE_MARKERS: [&str; 2] = ["users", "documents and settings"]; + if !value.contains(['\\', '/']) { + return value.to_string(); + } + let mut redact_next = false; + let mut result = String::with_capacity(value.len()); + let mut segment_start = 0; + for (index, separator) in value + .char_indices() + .filter(|(_, character)| matches!(character, '\\' | '/')) + { + let segment = &value[segment_start..index]; + if redact_next && !segment.is_empty() { + result.push_str(REDACTED_USER); + redact_next = false; + } else { + result.push_str(segment); + redact_next = USER_PROFILE_MARKERS + .iter() + .any(|marker| segment.eq_ignore_ascii_case(marker)); + } + result.push(separator); + segment_start = index + separator.len_utf8(); + } + let segment = &value[segment_start..]; + if redact_next && !segment.is_empty() { + result.push_str(REDACTED_USER); + } else { + result.push_str(segment); + } + result +} + +/// Bounds an already-sanitized property list to [`MAX_SIGNATURE_PROPERTIES`] +/// entries and [`MAX_SIGNATURE_VALUE_LEN`] characters per value. +/// +/// Callers must sanitize (redact) *before* calling this: truncation is a +/// pure length cap and does not know how to avoid splitting a redaction +/// marker or a still-sensitive suffix. +pub(crate) fn bound_properties(mut properties: Vec<(String, String)>) -> Vec<(String, String)> { + properties.truncate(MAX_SIGNATURE_PROPERTIES); + for (_, value) in &mut properties { + if value.chars().count() > MAX_SIGNATURE_VALUE_LEN { + *value = value.chars().take(MAX_SIGNATURE_VALUE_LEN).collect(); + } + } + properties +} + +/// Produces the deterministic, sanitized, bounded property list for a Data +/// Loop signature from one decoded event's raw TDH properties. +/// +/// Never includes a free-form decoder error message (decode failures are +/// recorded with an empty property list by the caller, before this +/// function would ever run) and never includes a timestamp-like property, +/// so exact-timestamp differences between otherwise-identical events don't +/// prevent their signatures from deduplicating. SIDs, capability names, +/// PIDs carried as properties, and GUID-shaped values are retained +/// verbatim; only account/user-identity content is redacted. +pub(crate) fn sanitize_properties(props: &[(String, String)]) -> Vec<(String, String)> { + let usernames = props + .iter() + .filter(|(name, _)| is_identity_property(name)) + .flat_map(|(_, value)| username_match_candidates(value.trim_matches('"'))) + .collect::>(); + let mut sanitized = std::collections::BTreeMap::new(); + for (name, raw_value) in props { + if is_timestamp_like_property(name) { + continue; + } + let value = raw_value.trim_matches('"'); + let sanitized_value = if is_identity_property(name) { + REDACTED_USER.to_string() + } else { + redact_known_username_components(&redact_username_in_path(value), &usernames) + }; + sanitized.insert(name.clone(), sanitized_value); + } + + fn username_match_candidates(identity: &str) -> Vec { + let mut candidates = Vec::new(); + let account = identity + .rsplit_once('\\') + .map_or(identity, |(_, account)| account); + for candidate in [ + identity, + account, + account.split_once('@').map_or(account, |(name, _)| name), + ] { + if !candidate.is_empty() + && !candidates + .iter() + .any(|existing: &String| windows_paths_equal_ignore_case(existing, candidate)) + { + candidates.push(candidate.to_string()); + } + } + candidates + } + + fn redact_known_username_components(value: &str, usernames: &[String]) -> String { + value + .split_inclusive(['\\', '/']) + .map(|component| { + let (segment, separator) = if component.ends_with('\\') || component.ends_with('/') + { + component.split_at(component.len() - 1) + } else { + (component, "") + }; + if usernames + .iter() + .any(|username| windows_paths_equal_ignore_case(segment, username)) + { + format!("{REDACTED_USER}{separator}") + } else { + component.to_string() + } + }) + .collect() + } + bound_properties(sanitized.into_iter().collect()) +} + /// /// The `ObjectType` field selects the resource type: `File` and `Key` /// (registry) map to concrete resources, an **empty** `ObjectType` is a @@ -156,12 +479,27 @@ pub(crate) fn is_learning_mode_event(provider: GUID, event_id: u16) -> bool { /// the type falls back to [`AccessType::Unknown`] so a decode gap never /// drops the denial itself. Capability checks carry a mask that is not a /// read/write/execute verb, so their access type is left `Unknown`. +/// +/// # Errors +/// +/// Returns the closed [`DataLoopExclusionReason`] describing why no denial +/// could be built: a missing `ObjectType` ([`DataLoopExclusionReason::MissingObjectType`]), +/// an object type this model can't represent +/// ([`DataLoopExclusionReason::UnsupportedObjectType`]), a missing/empty +/// object name (registry/file: +/// [`DataLoopExclusionReason::MissingObjectName`]; capability: an +/// unidentified brokered check, +/// [`DataLoopExclusionReason::UnresolvedCapability`] — [`crate::capability_dacl`] +/// may still recover it from the event's DACL payload), or a self-access +/// check that isn't actionable ([`DataLoopExclusionReason::NotActionable`]). pub fn build_denial_from_access_check( parts: &DecodedEventParts, pid: u32, filetime: u64, -) -> Option { - let object_type = find_prop(&parts.props, "ObjectType")?; + provider: DataLoopProvider, +) -> Result { + let object_type = + find_prop(&parts.props, "ObjectType").ok_or(DataLoopExclusionReason::MissingObjectType)?; let object_type_str = object_type.trim_matches('"'); let resource_type = match object_type_str { @@ -169,19 +507,28 @@ pub fn build_denial_from_access_check( "Key" => ResourceType::Other, // A present-but-empty object type is a brokered-capability check. "" => ResourceType::Capability, - _ => return None, + _ => return Err(DataLoopExclusionReason::UnsupportedObjectType), }; let object_name = find_prop(&parts.props, "ObjectName") .map(|v| v.trim_matches('"').to_string()) - .filter(|name| !name.is_empty())?; + .filter(|name| !name.is_empty()); + let object_name = match (resource_type, object_name) { + // An unidentified brokered-capability check: the identifier may + // still be recoverable from the event's DACL payload. + (ResourceType::Capability, None) => { + return Err(DataLoopExclusionReason::UnresolvedCapability) + } + (_, None) => return Err(DataLoopExclusionReason::MissingObjectName), + (_, Some(name)) => name, + }; if resource_type == ResourceType::File { let app_path = find_prop(&parts.props, "AppPath") .or_else(|| find_prop(&parts.props, "ApplicationPath")) .map(|value| value.trim_matches('"')); if app_path.is_some_and(|app_path| is_self_access(&object_name, app_path)) { - return None; + return Err(DataLoopExclusionReason::NotActionable); } } @@ -201,13 +548,15 @@ pub fn build_denial_from_access_check( .unwrap_or(AccessType::Unknown) }; - Some(RawDenial { + Ok(RawDenial { pid, resource_type, object_name, access_type, filetime, event_id: parts.event_id, + provider, + data_loop_properties: sanitize_properties(&parts.props), }) } @@ -264,26 +613,40 @@ fn strip_dos_namespace_prefix(path: &str) -> &str { /// These represent UI-surface denials. `Category` identifies the class and /// `Detail` identifies the concrete UI operation; `ProcessName` is the caller /// and must not be emitted as the denied resource. +/// +/// # Errors +/// +/// Returns [`DataLoopExclusionReason::MissingObjectType`] when `Category` is +/// absent or unparseable, [`DataLoopExclusionReason::MissingObjectName`] +/// when the required `Detail` is absent or unparseable, and +/// [`DataLoopExclusionReason::NotActionable`] when the category/detail pair +/// describes no violation (`Category == 0`). pub fn build_denial_from_learning_mode( parts: &DecodedEventParts, pid: u32, filetime: u64, -) -> Option { - let category = find_prop(&parts.props, "Category").and_then(|value| parse_u32(value))?; + provider: DataLoopProvider, +) -> Result { + let category = find_prop(&parts.props, "Category") + .and_then(|value| parse_u32(value)) + .ok_or(DataLoopExclusionReason::MissingObjectType)?; let detail = match find_prop(&parts.props, "Detail") { - Some(value) => parse_u32(value)?, + Some(value) => parse_u32(value).ok_or(DataLoopExclusionReason::MissingObjectName)?, None if category == crate::ui::CONVERT_TO_GUI => 0, - None => return None, + None => return Err(DataLoopExclusionReason::MissingObjectName), }; - let object_name = crate::ui::resource_name(category, detail)?; + let object_name = + crate::ui::resource_name(category, detail).ok_or(DataLoopExclusionReason::NotActionable)?; - Some(RawDenial { + Ok(RawDenial { pid, resource_type: ResourceType::Ui, object_name, access_type: AccessType::Unknown, filetime, event_id: parts.event_id, + provider, + data_loop_properties: sanitize_properties(&parts.props), }) } @@ -295,19 +658,24 @@ pub fn build_denial_from_learning_mode( /// brokered checks) when present, else the header pid. The capability name /// comes from the `PackageSid` capability SID, resolved to its friendly /// policy name via [`crate::capability_names`] (custom hashed capabilities -/// fall back to the SID string). Records without a decoded identifier are -/// omitted. +/// fall back to the SID string). +/// +/// # Errors +/// +/// Returns [`DataLoopExclusionReason::NotActionable`] when `Denied` is +/// absent or not `true`, and [`DataLoopExclusionReason::UnresolvedCapability`] +/// when no usable capability identifier could be decoded. pub fn build_denial_from_capability( parts: &DecodedEventParts, pid: u32, filetime: u64, -) -> Option { + provider: DataLoopProvider, +) -> Result { // A partially decoded event must not become a policy recommendation. - if !find_prop(&parts.props, "Denied")? - .trim_matches('"') - .eq_ignore_ascii_case("true") - { - return None; + let denied = find_prop(&parts.props, "Denied") + .is_some_and(|value| value.trim_matches('"').eq_ignore_ascii_case("true")); + if !denied { + return Err(DataLoopExclusionReason::NotActionable); } let pid = find_prop(&parts.props, "ProcessId") @@ -332,15 +700,18 @@ pub fn build_denial_from_capability( && name != "" && name != "" && name != "" - })?; + }) + .ok_or(DataLoopExclusionReason::UnresolvedCapability)?; - Some(RawDenial { + Ok(RawDenial { pid, resource_type: ResourceType::Capability, object_name, access_type: AccessType::Unknown, filetime, event_id: parts.event_id, + provider, + data_loop_properties: sanitize_properties(&parts.props), }) } @@ -555,7 +926,10 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::NotActionable + ); } } @@ -581,7 +955,7 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_some()); + assert!(extract_denial(&p, 1, FIXED_FILETIME).is_ok()); } #[test] @@ -598,7 +972,7 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_some()); + assert!(extract_denial(&p, 1, FIXED_FILETIME).is_ok()); } #[test] @@ -620,7 +994,7 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_some()); + assert!(extract_denial(&p, 1, FIXED_FILETIME).is_ok()); } #[test] @@ -634,7 +1008,7 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_some()); + assert!(extract_denial(&p, 1, FIXED_FILETIME).is_ok()); } #[test] @@ -648,7 +1022,7 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_some()); + assert!(extract_denial(&p, 1, FIXED_FILETIME).is_ok()); } #[test] @@ -681,7 +1055,10 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 5900, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 5900, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::UnresolvedCapability + ); } #[test] @@ -695,7 +1072,10 @@ mod tests { ("AccessMask", "0x1"), ], ); - assert!(extract_denial(&p, 5900, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 5900, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::MissingObjectName + ); } } @@ -712,13 +1092,19 @@ mod tests { #[test] fn access_check_unknown_object_type_dropped() { let p = parts(14, &[("ObjectType", "\"Section\"")]); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::UnsupportedObjectType + ); } #[test] fn access_check_absent_object_type_dropped() { let p = parts(14, &[("ObjectName", "\"x\"")]); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::MissingObjectType + ); } // ---- event 27 UI ------------------------------------------------------ @@ -798,16 +1184,25 @@ mod tests { #[test] fn learning_mode_violation_rejects_invalid_required_numbers() { let invalid_category = parts(27, &[("Category", "not-a-number"), ("Detail", "4")]); - assert!(extract_denial(&invalid_category, 9999, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&invalid_category, 9999, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::MissingObjectType + ); let invalid_detail = parts(27, &[("Category", "2"), ("Detail", "not-a-number")]); - assert!(extract_denial(&invalid_detail, 9999, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&invalid_detail, 9999, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::MissingObjectName + ); } #[test] fn learning_mode_category_none_is_dropped() { let p = parts(27, &[("Category", "0"), ("Detail", "0")]); - assert!(extract_denial(&p, 9999, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 9999, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::NotActionable + ); } // ---- event 28 capability denial --------------------------------------- @@ -831,6 +1226,7 @@ mod tests { // pid comes from the payload ProcessId (0x1acc), not the header. assert_eq!(ev.pid, 0x1acc); assert_eq!(ev.object_name, "internetClient"); + assert_eq!(ev.provider, DataLoopProvider::KernelGeneral); } #[test] @@ -871,13 +1267,28 @@ mod tests { #[test] fn capability_denial_not_denied_is_dropped() { let p = parts(28, &[("ProcessId", "0x10"), ("Denied", "false")]); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::NotActionable + ); } #[test] fn capability_denial_without_denied_field_is_dropped() { let p = parts(28, &[("PackageSid", "S-1-15-3-1")]); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::NotActionable + ); + } + + #[test] + fn capability_denial_without_identifier_is_unresolved() { + let p = parts(28, &[("ProcessId", "0x10"), ("Denied", "true")]); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::UnresolvedCapability + ); } #[test] @@ -887,7 +1298,13 @@ mod tests { 27, &[("Category", "2"), ("Detail", "4")], ); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::UnsupportedEventSchema + ); + assert!( + provider_category(GUID::from_u128(0x12345678_1234_1234_1234_1234567890ab)).is_none() + ); } #[test] @@ -900,7 +1317,293 @@ mod tests { #[test] fn unrelated_event_ignored() { let p = parts(9999, &[("Foo", "\"bar\"")]); - assert!(extract_denial(&p, 1, FIXED_FILETIME).is_none()); + assert_eq!( + extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), + DataLoopExclusionReason::UnsupportedEventSchema + ); + } + + #[test] + fn known_provider_category_is_symbolic_and_pii_free() { + assert_eq!( + provider_category(KERNEL_GENERAL_PROVIDER), + Some(DataLoopProvider::KernelGeneral) + ); + assert_eq!( + provider_category(PRIVACY_LEARNING_MODE_PROVIDER), + Some(DataLoopProvider::PrivacyAuditingPermissiveLearningMode) + ); + assert_eq!(provider_category(GUID::from_u128(0)), None); + } + + #[test] + fn provider_guid_string_is_stable_and_round_trips_by_category() { + let kernel_guid = provider_guid_string(DataLoopProvider::KernelGeneral); + let privacy_guid = + provider_guid_string(DataLoopProvider::PrivacyAuditingPermissiveLearningMode); + assert_ne!(kernel_guid, privacy_guid); + assert_eq!(kernel_guid, format_guid(KERNEL_GENERAL_PROVIDER)); + assert_eq!(privacy_guid, format_guid(PRIVACY_LEARNING_MODE_PROVIDER)); + assert!(kernel_guid.starts_with('{') && kernel_guid.ends_with('}')); + } + + // ---- Data Loop signature sanitization ---------------------------------- + + #[test] + fn redact_username_in_path_replaces_only_the_profile_segment() { + assert_eq!( + redact_username_in_path(r"C:\Users\jsmith\Documents\secret.txt"), + format!(r"C:\Users\{REDACTED_USER}\Documents\secret.txt") + ); + assert_eq!( + redact_username_in_path(r"\Device\HarddiskVolume3\Users\jane.doe\AppData\x"), + format!(r"\Device\HarddiskVolume3\Users\{REDACTED_USER}\AppData\x") + ); + assert_eq!( + redact_username_in_path(r"\??\C:\Documents and Settings\bob\Desktop\x"), + format!(r"\??\C:\Documents and Settings\{REDACTED_USER}\Desktop\x") + ); + assert_eq!( + redact_username_in_path("C:/Users/alice/Documents/secret.txt"), + format!("C:/Users/{REDACTED_USER}/Documents/secret.txt") + ); + } + + #[test] + fn redact_username_in_path_is_case_insensitive_on_the_marker_only() { + assert_eq!( + redact_username_in_path(r"C:\USERS\Alice\file.txt"), + format!(r"C:\USERS\{REDACTED_USER}\file.txt") + ); + } + + #[test] + fn redact_username_in_path_leaves_unrelated_paths_untouched() { + // Registry hive segment "USER" (singular) must not be mistaken for + // the "Users" (plural) profile-directory marker. + let registry_path = r"\REGISTRY\USER\S-1-5-21-1-2-3-1000\Software\Test"; + assert_eq!(redact_username_in_path(registry_path), registry_path); + assert_eq!(redact_username_in_path("internetClient"), "internetClient"); + assert_eq!( + redact_username_in_path(r"C:\data\test\bin\"), + r"C:\data\test\bin\" + ); + } + + #[test] + fn sanitize_properties_drops_timestamp_like_properties() { + let props = vec![ + ("ObjectName".to_string(), "\"C:\\a.txt\"".to_string()), + ("Timestamp".to_string(), "132847890123456789".to_string()), + ( + "LastWriteTime".to_string(), + "132847890123456789".to_string(), + ), + ("RuntimePolicy".to_string(), "\"retain\"".to_string()), + ("Timeout".to_string(), "\"30\"".to_string()), + ]; + let out = sanitize_properties(&props); + assert!(out + .iter() + .all(|(name, _)| name != "Timestamp" && name != "LastWriteTime")); + assert!(out.iter().any(|(name, _)| name == "ObjectName")); + assert!(out.iter().any(|(name, _)| name == "RuntimePolicy")); + assert!(out.iter().any(|(name, _)| name == "Timeout")); + } + + #[test] + fn sanitize_properties_redacts_standalone_identity_properties() { + let props = vec![ + ("UserName".to_string(), "\"jsmith\"".to_string()), + ( + "AccountName".to_string(), + "\"DOMAIN\\\\jsmith\"".to_string(), + ), + ("PackageSid".to_string(), "\"S-1-15-3-1\"".to_string()), + ("UserSid".to_string(), "\"S-1-5-21-123\"".to_string()), + ("AccountId".to_string(), "\"42\"".to_string()), + ]; + let out = sanitize_properties(&props); + let value_for = |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); + assert_eq!(value_for("UserName"), Some(REDACTED_USER)); + assert_eq!(value_for("AccountName"), Some(REDACTED_USER)); + // SIDs are retained identifiers, never redacted. + assert_eq!(value_for("PackageSid"), Some("S-1-15-3-1")); + assert_eq!(value_for("UserSid"), Some("S-1-5-21-123")); + assert_eq!(value_for("AccountId"), Some("42")); + } + + #[test] + fn sanitize_properties_redacts_username_segment_inside_a_path_value() { + let props = vec![( + "ObjectName".to_string(), + "\"C:\\Users\\jsmith\\secret.txt\"".to_string(), + )]; + let out = sanitize_properties(&props); + assert_eq!( + out, + vec![( + "ObjectName".to_string(), + format!(r"C:\Users\{REDACTED_USER}\secret.txt") + )] + ); + } + + #[test] + fn sanitize_properties_redacts_known_username_in_arbitrary_resource_path() { + let props = vec![ + ("UserName".to_string(), "\"jsmith\"".to_string()), + ( + "ObjectName".to_string(), + "\"D:\\profiles\\jsmith\\secret.txt\"".to_string(), + ), + ]; + let out = sanitize_properties(&props); + let value_for = |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); + assert_eq!(value_for("UserName"), Some(REDACTED_USER)); + assert_eq!( + value_for("ObjectName"), + Some(r"D:\profiles\\secret.txt") + ); + } + + #[test] + fn sanitize_properties_redacts_qualified_username_in_arbitrary_resource_path() { + for (property_name, identity) in [ + ("UserName", r"CONTOSO\jsmith"), + ("UserPrincipalName", "jsmith@example.com"), + ] { + let props = vec![ + (property_name.to_string(), format!("\"{identity}\"")), + ( + "ObjectName".to_string(), + "\"D:\\profiles\\jsmith\\secret.txt\"".to_string(), + ), + ]; + let out = sanitize_properties(&props); + let value_for = |name: &str| { + out.iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value) + }; + assert_eq!( + value_for("ObjectName").map(String::as_str), + Some(r"D:\profiles\\secret.txt") + ); + assert_eq!( + value_for(property_name).map(String::as_str), + Some(REDACTED_USER) + ); + } + } + + #[test] + fn sanitize_properties_redacts_standard_account_identity_fields() { + for (property_name, identity) in [ + ("PrincipalName", "jsmith@example.com"), + ("TargetAccount", r"CONTOSO\jsmith"), + ("SubjectAccount", "jsmith"), + ] { + let props = vec![ + (property_name.to_string(), format!("\"{identity}\"")), + ( + "ObjectName".to_string(), + "\"D:\\profiles\\jsmith\\secret.txt\"".to_string(), + ), + ]; + let out = sanitize_properties(&props); + let value_for = |name: &str| { + out.iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value) + }; + assert_eq!( + value_for(property_name).map(String::as_str), + Some(REDACTED_USER) + ); + assert_eq!( + value_for("ObjectName").map(String::as_str), + Some(r"D:\profiles\\secret.txt") + ); + } + } + + #[test] + fn sanitize_properties_redacts_unicode_username_case_variants() { + let props = vec![ + ("UserName".to_string(), "\"JÖRG\"".to_string()), + ( + "ObjectName".to_string(), + "\"D:\\profiles\\jörg\\secret.txt\"".to_string(), + ), + ]; + let out = sanitize_properties(&props); + assert_eq!( + out.iter() + .find(|(name, _)| name == "ObjectName") + .map(|(_, value)| value.as_str()), + Some(r"D:\profiles\\secret.txt") + ); + } + + #[test] + fn sanitize_properties_is_deterministically_sorted_by_name() { + let props = vec![ + ("Zeta".to_string(), "\"z\"".to_string()), + ("Alpha".to_string(), "\"a\"".to_string()), + ]; + let out = sanitize_properties(&props); + assert_eq!( + out, + vec![ + ("Alpha".to_string(), "a".to_string()), + ("Zeta".to_string(), "z".to_string()), + ] + ); + } + + #[test] + fn sanitize_properties_never_carries_a_decoder_error_message() { + // Decode-failure signatures are built by the caller with an empty + // property list (there is no decoded payload to sanitize), so a + // free-form TDH error string can never reach `sanitize_properties` + // in the first place. + let props: Vec<(String, String)> = Vec::new(); + assert!(sanitize_properties(&props).is_empty()); + } + + #[test] + fn bound_properties_truncates_count_and_value_length_after_sanitizing() { + let long_value = format!( + r"C:\Users\{}\{}", + "jsmith", + "x".repeat(MAX_SIGNATURE_VALUE_LEN * 2) + ); + let mut props = vec![("ObjectName".to_string(), long_value)]; + for i in 0..(MAX_SIGNATURE_PROPERTIES + 10) { + // `bound_properties` truncates in the given order (callers are + // expected to have already sorted deterministically); appending + // these after `ObjectName` keeps it within the retained prefix. + props.push((format!("Extra{i:03}"), "v".to_string())); + } + let sanitized: Vec<(String, String)> = props + .into_iter() + .map(|(name, value)| (name, redact_username_in_path(&value))) + .collect(); + + let out = bound_properties(sanitized); + + assert_eq!(out.len(), MAX_SIGNATURE_PROPERTIES); + let object_name = out + .iter() + .find(|(name, _)| name == "ObjectName") + .expect("ObjectName retained (sorted first)"); + assert!( + object_name.1.contains(REDACTED_USER), + "redaction must survive truncation: {}", + object_name.1 + ); + assert!(object_name.1.chars().count() <= MAX_SIGNATURE_VALUE_LEN); } // ---- parse_u32 -------------------------------------------------------- diff --git a/src/backends/learning_mode/windows/src/tdh_decode.rs b/src/backends/learning_mode/windows/src/tdh_decode.rs index 87c759a8e..b7cfa48ff 100644 --- a/src/backends/learning_mode/windows/src/tdh_decode.rs +++ b/src/backends/learning_mode/windows/src/tdh_decode.rs @@ -91,19 +91,56 @@ pub(crate) struct EventSchemaCache { #[derive(Debug)] pub(crate) enum DecodeError { Schema(String), - Event(String), + Event { + kind: EventDecodeKind, + event_name: Option, + message: String, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EventDecodeKind { + PayloadMalformed, + DecoderLimitReached, + UnsupportedPropertyEncoding, } impl DecodeError { pub(crate) fn is_schema_error(&self) -> bool { matches!(self, Self::Schema(_)) } + + pub(crate) fn event_kind(&self) -> Option { + match self { + Self::Event { kind, .. } => Some(*kind), + Self::Schema(_) => None, + } + } + + pub(crate) fn event_name(&self) -> Option<&str> { + match self { + Self::Event { event_name, .. } => event_name.as_deref(), + Self::Schema(_) => None, + } + } + + pub(crate) fn event( + kind: EventDecodeKind, + message: String, + event_name: Option, + ) -> Self { + Self::Event { + kind, + event_name, + message, + } + } } impl std::fmt::Display for DecodeError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Schema(message) | Self::Event(message) => f.write_str(message), + Self::Schema(message) | Self::Event { message, .. } => f.write_str(message), } } } @@ -192,8 +229,25 @@ pub unsafe fn decode_event_parts( let header = event.EventHeader; let event_id = header.EventDescriptor.Id; let pointer_size = pointer_size_from_header_flags(header.Flags); - let props = decode_properties(buffer.as_bytes(), info, event_record, pointer_size) - .map_err(DecodeError::Event)?; + let event_name = schema_event_name(buffer.as_bytes(), info); + let props = decode_properties(buffer.as_bytes(), info, event_record, pointer_size).map_err( + |error| match error.kind { + PropertyDecodeErrorKind::Schema => DecodeError::Schema(error.message), + PropertyDecodeErrorKind::PayloadMalformed => { + DecodeError::event(EventDecodeKind::PayloadMalformed, error.message, event_name) + } + PropertyDecodeErrorKind::DecoderLimitReached => DecodeError::event( + EventDecodeKind::DecoderLimitReached, + error.message, + event_name, + ), + PropertyDecodeErrorKind::UnsupportedPropertyEncoding => DecodeError::event( + EventDecodeKind::UnsupportedPropertyEncoding, + error.message, + event_name, + ), + }, + )?; Ok(DecodedEventParts { provider: header.ProviderId, @@ -202,6 +256,13 @@ pub unsafe fn decode_event_parts( }) } +fn schema_event_name(info_buf: &[u8], info: &TRACE_EVENT_INFO) -> Option { + // SAFETY: `info` points into the TDH buffer and this union member is the + // event-name offset for the decoding sources used by these providers. + let event_name_offset = unsafe { info.Anonymous1.EventNameOffset }; + wide_str_at(info_buf, event_name_offset).or_else(|| wide_str_at(info_buf, info.TaskNameOffset)) +} + unsafe fn load_event_schema(event_record: *mut EVENT_RECORD) -> Result { let mut buf_size: u32 = 0; // First call: discover required buffer size. ERROR_INSUFFICIENT_BUFFER = 122. @@ -242,12 +303,56 @@ unsafe fn has_trace_logging_schema(event_record: &EVENT_RECORD) -> bool { .any(|item| u32::from(item.ExtType) == EVENT_HEADER_EXT_TYPE_EVENT_SCHEMA_TL) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PropertyDecodeErrorKind { + Schema, + PayloadMalformed, + DecoderLimitReached, + UnsupportedPropertyEncoding, +} + +#[derive(Debug)] +struct PropertyDecodeError { + kind: PropertyDecodeErrorKind, + message: String, +} + +impl PropertyDecodeError { + fn schema(message: String) -> Self { + Self { + kind: PropertyDecodeErrorKind::Schema, + message, + } + } + + fn payload(message: String) -> Self { + Self { + kind: PropertyDecodeErrorKind::PayloadMalformed, + message, + } + } + + fn limit(message: String) -> Self { + Self { + kind: PropertyDecodeErrorKind::DecoderLimitReached, + message, + } + } + + fn unsupported(message: String) -> Self { + Self { + kind: PropertyDecodeErrorKind::UnsupportedPropertyEncoding, + message, + } + } +} + fn decode_properties( info_buf: &[u8], info: &TRACE_EVENT_INFO, event_record: *mut EVENT_RECORD, pointer_size: usize, -) -> Result, String> { +) -> Result, PropertyDecodeError> { // SAFETY: caller passes a valid EVENT_RECORD; the field accesses // are reads of POD fields. let event = unsafe { &*event_record }; @@ -261,10 +366,11 @@ fn decode_properties( let property_count = info.PropertyCount as usize; let prop_count = info.TopLevelPropertyCount as usize; if prop_count > property_count { - return Err(format!( + return Err(PropertyDecodeError::schema(format!( "top-level property count {prop_count} exceeds property count {property_count}" - )); + ))); } + let mut results = Vec::with_capacity(prop_count); let mut numeric_values = vec![None; property_count]; let mut offset: usize = 0; @@ -308,15 +414,21 @@ fn decode_property( depth: usize, work_remaining: &mut usize, results: &mut Vec<(String, String)>, -) -> Result<(), String> { - *work_remaining = work_remaining - .checked_sub(1) - .ok_or_else(|| format!("property decode work exceeds limit {MAX_DECODE_WORK}"))?; +) -> Result<(), PropertyDecodeError> { + *work_remaining = work_remaining.checked_sub(1).ok_or_else(|| { + PropertyDecodeError::limit(format!( + "property decode work exceeds limit {MAX_DECODE_WORK}" + )) + })?; if depth > MAX_STRUCT_DEPTH { - return Err(format!("property nesting exceeds limit {MAX_STRUCT_DEPTH}")); + return Err(PropertyDecodeError::limit(format!( + "property nesting exceeds limit {MAX_STRUCT_DEPTH}" + ))); } if index >= context.info.PropertyCount as usize { - return Err(format!("property index {index} is out of range")); + return Err(PropertyDecodeError::schema(format!( + "property index {index} is out of range" + ))); } let prop_info = event_property_info(context.info, index); let prop_name = wide_str_at(context.info_buf, prop_info.NameOffset) @@ -324,25 +436,29 @@ fn decode_property( let flags = prop_info.Flags.0; let count = resolve_property_count(prop_info, flags, numeric_values, index)?; if count > MAX_PROPERTY_ELEMENTS { - return Err(format!( + return Err(PropertyDecodeError::limit(format!( "property '{prop_name}' count {count} exceeds limit {MAX_PROPERTY_ELEMENTS}" - )); + ))); } if flags & PROPERTY_STRUCT != 0 { let start_index = unsafe { prop_info.Anonymous1.structType.StructStartIndex } as usize; let member_count = unsafe { prop_info.Anonymous1.structType.NumOfStructMembers } as usize; - let end_index = start_index - .checked_add(member_count) - .ok_or_else(|| format!("property '{prop_name}' struct member range overflows"))?; + let end_index = start_index.checked_add(member_count).ok_or_else(|| { + PropertyDecodeError::schema(format!( + "property '{prop_name}' struct member range overflows" + )) + })?; if end_index > context.info.PropertyCount as usize { - return Err(format!( + return Err(PropertyDecodeError::schema(format!( "property '{prop_name}' struct member range {start_index}..{end_index} exceeds property count {}", context.info.PropertyCount - )); + ))); } if member_count == 0 { - return Err(format!("property '{prop_name}' has no struct members")); + return Err(PropertyDecodeError::schema(format!( + "property '{prop_name}' has no struct members" + ))); } results.push((prop_name, "".to_string())); for _ in 0..count { @@ -378,7 +494,9 @@ fn decode_property( for _ in 0..count { let remaining = context.user_data_len.saturating_sub(*offset); if remaining == 0 { - return Err(format!("property '{prop_name}' exceeds the event payload")); + return Err(PropertyDecodeError::payload(format!( + "property '{prop_name}' exceeds the event payload" + ))); } let data_ptr = if remaining > 0 { unsafe { context.user_data.add(*offset) } @@ -393,15 +511,17 @@ fn decode_property( context.pointer_size, ); if consumed == 0 && remaining > 0 { - return Err(format!( - "property '{prop_name}' has unsupported variable length" - )); + return Err(zero_consumption_error(&prop_name, in_type)); } *offset = offset .checked_add(consumed) .filter(|next| *next <= context.user_data_len) - .ok_or_else(|| format!("property '{prop_name}' exceeds the event payload"))?; + .ok_or_else(|| { + PropertyDecodeError::payload(format!( + "property '{prop_name}' exceeds the event payload" + )) + })?; if count == 1 { numeric_value = parse_numeric_metadata(&value); } @@ -423,7 +543,7 @@ fn resolve_property_count( flags: i32, numeric_values: &[Option], property_index: usize, -) -> Result { +) -> Result { if flags & PROPERTY_PARAM_COUNT != 0 { let count_index = unsafe { prop_info.Anonymous2.countPropertyIndex } as usize; resolved_metadata(numeric_values, count_index, "count", property_index) @@ -465,15 +585,56 @@ fn resolved_metadata( metadata_index: usize, kind: &str, property_index: usize, -) -> Result { - numeric_values - .get(metadata_index) - .and_then(|value| *value) - .ok_or_else(|| { - format!( - "property {property_index} references unresolved {kind} property {metadata_index}" - ) - }) +) -> Result { + let Some(value) = numeric_values.get(metadata_index) else { + return Err(PropertyDecodeError::schema(format!( + "property {property_index} references out-of-range {kind} property {metadata_index}" + ))); + }; + value.ok_or_else(|| { + PropertyDecodeError::schema(format!( + "property {property_index} references unresolved {kind} property {metadata_index}" + )) + }) +} + +fn is_known_property_type(in_type: u16) -> bool { + matches!( + in_type, + TDH_INTYPE_UNICODESTRING + | TDH_INTYPE_ANSISTRING + | TDH_INTYPE_INT8 + | TDH_INTYPE_UINT8 + | TDH_INTYPE_INT16 + | TDH_INTYPE_UINT16 + | TDH_INTYPE_INT32 + | TDH_INTYPE_UINT32 + | TDH_INTYPE_INT64 + | TDH_INTYPE_UINT64 + | TDH_INTYPE_FLOAT + | TDH_INTYPE_DOUBLE + | TDH_INTYPE_BOOLEAN + | TDH_INTYPE_BINARY + | TDH_INTYPE_GUID + | TDH_INTYPE_POINTER + | TDH_INTYPE_FILETIME + | TDH_INTYPE_SYSTEMTIME + | TDH_INTYPE_SID + | TDH_INTYPE_HEXINT32 + | TDH_INTYPE_HEXINT64 + | TDH_INTYPE_UNICODECHAR + | TDH_INTYPE_ANSICHAR + | TDH_INTYPE_SIZET + ) +} + +fn zero_consumption_error(property_name: &str, in_type: u16) -> PropertyDecodeError { + let message = format!("property '{property_name}' could not be decoded"); + if is_known_property_type(in_type) { + PropertyDecodeError::payload(message) + } else { + PropertyDecodeError::unsupported(message) + } } fn parse_numeric_metadata(value: &str) -> Option { @@ -663,12 +824,12 @@ fn format_property_value( fn format_sid(data: *const u8, available: usize) -> (String, usize) { let header = unsafe { std::slice::from_raw_parts(data, available.min(8)) }; if header.len() < 8 { - return ("".to_string(), available); + return ("".to_string(), 0); } let sub_authority_count = header[1] as usize; let length = 8usize.saturating_add(sub_authority_count.saturating_mul(4)); if length > available { - return ("".to_string(), available); + return ("".to_string(), 0); } let bytes = unsafe { std::slice::from_raw_parts(data, length) }; let authority = bytes[2..8] @@ -815,7 +976,7 @@ mod tests { fn decode_single_property( buffer: &TdhInfoBuffer, payload: &[u8], - ) -> Result, String> { + ) -> Result, PropertyDecodeError> { let info = unsafe { &*buffer.as_ptr() }; let context = PropertyDecodeContext { info_buf: buffer.as_bytes(), @@ -860,7 +1021,8 @@ mod tests { let error = decode_single_property(&buffer, &[1]).unwrap_err(); - assert!(error.contains("struct member range")); + assert_eq!(error.kind, PropertyDecodeErrorKind::Schema); + assert!(error.message.contains("struct member range")); } #[test] @@ -883,7 +1045,8 @@ mod tests { let error = decode_single_property(&buffer, &[1]).unwrap_err(); - assert!(error.contains("has no struct members")); + assert_eq!(error.kind, PropertyDecodeErrorKind::Schema); + assert!(error.message.contains("has no struct members")); } #[test] @@ -906,17 +1069,29 @@ mod tests { let error = decode_single_property(&buffer, &[1]).unwrap_err(); - assert!(error.contains("nesting exceeds limit")); + assert_eq!(error.kind, PropertyDecodeErrorKind::DecoderLimitReached); + assert!(error.message.contains("nesting exceeds limit")); } #[test] fn invalid_metadata_references_are_rejected() { - assert!(resolved_metadata(&[None], 3, "count", 0) - .unwrap_err() - .contains("unresolved count property 3")); - assert!(resolved_metadata(&[None], 2, "length", 0) - .unwrap_err() - .contains("unresolved length property 2")); + let count_error = resolved_metadata(&[None], 3, "count", 0).unwrap_err(); + assert_eq!(count_error.kind, PropertyDecodeErrorKind::Schema); + assert!(count_error + .message + .contains("out-of-range count property 3")); + + let length_error = resolved_metadata(&[None], 2, "length", 0).unwrap_err(); + assert_eq!(length_error.kind, PropertyDecodeErrorKind::Schema); + assert!(length_error + .message + .contains("out-of-range length property 2")); + + let unresolved_error = resolved_metadata(&[None], 0, "count", 1).unwrap_err(); + assert_eq!(unresolved_error.kind, PropertyDecodeErrorKind::Schema); + assert!(unresolved_error + .message + .contains("unresolved count property 0")); } #[test] @@ -960,7 +1135,8 @@ mod tests { ) .unwrap_err(); - assert!(error.contains("decode work exceeds limit")); + assert_eq!(error.kind, PropertyDecodeErrorKind::DecoderLimitReached); + assert!(error.message.contains("decode work exceeds limit")); } #[test] @@ -980,6 +1156,18 @@ mod tests { assert!(wide_str_at(&buf, 0).is_none()); } + #[test] + fn property_decode_errors_use_typed_categories() { + assert_eq!( + zero_consumption_error("Name", TDH_INTYPE_UNICODESTRING).kind, + PropertyDecodeErrorKind::PayloadMalformed + ); + assert_eq!( + zero_consumption_error("Value", u16::MAX).kind, + PropertyDecodeErrorKind::UnsupportedPropertyEncoding + ); + } + #[test] fn format_property_value_unicode_string_extracts_content() { let s = "hello"; @@ -1207,7 +1395,7 @@ mod tests { let bytes = [1u8, 3, 0, 0, 0, 0, 0, 15]; let (val, consumed) = format_property_value(TDH_INTYPE_SID, 0, bytes.as_ptr(), bytes.len()); assert_eq!(val, ""); - assert_eq!(consumed, bytes.len()); + assert_eq!(consumed, 0); } #[test] diff --git a/src/core/learning_mode_core/src/analyze.rs b/src/core/learning_mode_core/src/analyze.rs index 534f0f46f..889a1e36a 100644 --- a/src/core/learning_mode_core/src/analyze.rs +++ b/src/core/learning_mode_core/src/analyze.rs @@ -16,6 +16,7 @@ use std::path::Path; use serde::{Deserialize, Serialize}; use thiserror::Error; +use crate::data_loop::DataLoopSummary; use crate::model::DeniedResource; /// Result of decoding a capture source into bounded, de-duplicated denials. @@ -27,6 +28,9 @@ pub struct AnalysisResult { /// Whether additional unique denials were observed after the result bound /// was reached. pub denied_resources_truncated: bool, + /// Username-redacted signatures for outcomes omitted from canonical denials. + #[serde(default)] + pub data_loop: DataLoopSummary, } /// Inclusive process-lifetime window used to scope a host-wide capture. @@ -61,8 +65,62 @@ impl AnalysisResult { Self { denials, denied_resources_truncated: false, + data_loop: DataLoopSummary::default(), } } + + /// Trims Data Loop signatures so the complete compact JSON result fits. + /// + /// Returns `false` when the canonical denials and empty Data Loop envelope + /// alone exceed `max_bytes`; canonical denials are never discarded. + pub fn fit_data_loop_within_serialized_bytes( + &mut self, + max_bytes: usize, + ) -> Result { + if serde_json::to_vec(self)?.len() <= max_bytes { + return Ok(true); + } + + let original = std::mem::take(&mut self.data_loop.signatures); + let mut retained_bytes = 0usize; + const ENVELOPE_HEADROOM: usize = 4 * 1024; + let base_len = serde_json::to_vec(self)?.len(); + if base_len > max_bytes { + return Ok(false); + } + let signature_budget = max_bytes + .saturating_sub(base_len) + .saturating_sub(ENVELOPE_HEADROOM); + let mut original = original; + original.sort_by_key(|aggregate| !aggregate.signature.reason.is_canonical_denial()); + for aggregate in original { + let aggregate_len = serde_json::to_vec(&aggregate)?.len().saturating_add(1); + if retained_bytes.saturating_add(aggregate_len) <= signature_budget { + retained_bytes += aggregate_len; + self.data_loop.signatures.push(aggregate); + } else { + self.data_loop.move_to_overflow(aggregate); + } + } + self.data_loop + .signatures + .sort_by(|left, right| left.signature.cmp(&right.signature)); + + while serde_json::to_vec(self)?.len() > max_bytes { + let index = self + .data_loop + .signatures + .iter() + .rposition(|aggregate| !aggregate.signature.reason.is_canonical_denial()) + .or_else(|| self.data_loop.signatures.len().checked_sub(1)); + let Some(index) = index else { + return Ok(false); + }; + let aggregate = self.data_loop.signatures.remove(index); + self.data_loop.move_to_overflow(aggregate); + } + Ok(true) + } } /// Failure modes when analysing a capture source into denials. @@ -107,6 +165,9 @@ pub trait DenialAnalyzer { #[cfg(test)] mod tests { use super::*; + use crate::data_loop::{ + DataLoopAggregate, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, + }; use crate::model::{AccessType, ResourceType}; /// A trivial analyzer returning a fixed set, proving the trait is @@ -132,6 +193,14 @@ mod tests { let got = analyzer.analyze(Path::new("ignored.etl")).unwrap(); assert_eq!(got.denials, denials); assert!(!got.denied_resources_truncated); + assert!(got.data_loop.is_empty()); + } + + #[test] + fn legacy_guarded_payload_defaults_data_loop_summary() { + let payload = br#"{"denials":[],"deniedResourcesTruncated":false}"#; + let result: AnalysisResult = serde_json::from_slice(payload).unwrap(); + assert!(result.data_loop.is_empty()); } #[test] @@ -160,4 +229,59 @@ mod tests { assert!(!lifetime.contains(42, 201)); assert!(!lifetime.contains(43, 150)); } + + #[test] + fn serialized_size_limit_moves_data_loop_groups_to_overflow() { + let signature = |pid| DataLoopAggregate { + signature: DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "provider".to_string(), + event_id: 14, + reason: DataLoopExclusionReason::CanonicalDenial, + pid, + access_type: Some(AccessType::Read), + resource_type: Some(ResourceType::File), + properties: vec![("ObjectName".to_string(), "x".repeat(4_096))], + }, + count: 3, + }; + let mut result = AnalysisResult { + denials: vec![DeniedResource { + resource: "canonical".repeat(512), + resource_type: ResourceType::File, + access_type: AccessType::Read, + pid: 1, + filetime: 2, + }], + denied_resources_truncated: false, + data_loop: DataLoopSummary { + signatures: vec![signature(1), signature(2)], + total_occurrences: 6, + ..Default::default() + }, + }; + let original_len = serde_json::to_vec(&result).unwrap().len(); + + assert!(result + .fit_data_loop_within_serialized_bytes(original_len - 1) + .unwrap()); + assert!(serde_json::to_vec(&result).unwrap().len() < original_len); + assert!(result.data_loop.aggregate_groups_truncated); + assert_eq!(result.data_loop.total_occurrences, 6); + assert!(result.data_loop.overflow_occurrences > 0); + } + + #[test] + fn serialized_size_limit_never_discards_canonical_denials() { + let mut result = AnalysisResult::complete(vec![DeniedResource { + resource: "canonical".repeat(512), + resource_type: ResourceType::File, + access_type: AccessType::Read, + pid: 1, + filetime: 2, + }]); + + assert!(!result.fit_data_loop_within_serialized_bytes(1).unwrap()); + assert_eq!(result.denials.len(), 1); + } } diff --git a/src/core/learning_mode_core/src/data_loop.rs b/src/core/learning_mode_core/src/data_loop.rs new file mode 100644 index 000000000..374cbbd4e --- /dev/null +++ b/src/core/learning_mode_core/src/data_loop.rs @@ -0,0 +1,532 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Username-redacted, deduplicated diagnostics for Learning Mode events, +//! including every canonical denial occurrence. + +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// Maximum number of distinct aggregate groups retained in one analysis. +pub const MAX_DATA_LOOP_GROUPS: usize = 4_096; +/// Maximum compact-JSON bytes retained across distinct serialized signatures. +/// +/// This leaves substantial headroom under the guarded WPR analysis protocol's +/// 64 MiB frame limit for canonical denials and envelope overhead. +pub const MAX_DATA_LOOP_SIGNATURE_BYTES: usize = 16 * 1024 * 1024; + +/// Stable category for a known Learning Mode ETW provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DataLoopProvider { + /// Microsoft-Windows-Kernel-General. + KernelGeneral, + /// Microsoft-Windows-Privacy-Auditing-PermissiveLearningMode. + PrivacyAuditingPermissiveLearningMode, +} + +/// Closed reason why a decoder outcome was omitted from canonical denials. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DataLoopExclusionReason { + /// The event produced a valid canonical denial candidate. + CanonicalDenial, + /// The provider is known, but the event ID is not a supported denial schema. + UnsupportedEventSchema, + /// The event payload conflicted with its declared TDH schema. + EventPayloadMalformed, + /// A decoder safety bound prevented full payload processing. + DecoderLimitReached, + /// The payload uses a property encoding the decoder cannot consume. + UnsupportedPropertyEncoding, + /// A required object-type property was absent or empty. + MissingObjectType, + /// A required object/resource-name property was absent or empty. + MissingObjectName, + /// The event described an object category the canonical model cannot represent. + UnsupportedObjectType, + /// A resource value could not be converted to a safe user-visible resource. + UnusableResourcePath, + /// A capability event did not contain a usable capability denial. + UnresolvedCapability, + /// The event was valid but did not describe an actionable denial. + NotActionable, +} + +impl DataLoopExclusionReason { + /// Returns whether this outcome represents a valid denial candidate. + #[must_use] + pub fn is_canonical_denial(self) -> bool { + self == Self::CanonicalDenial + } +} + +/// One sanitized event signature used as a Data Loop deduplication key. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DataLoopSignature { + /// Symbolic provider category. + pub provider: DataLoopProvider, + /// Stable ETW provider GUID. + pub provider_guid: String, + /// Provider-scoped ETW schema identifier. + pub event_id: u16, + /// Closed exclusion category. + pub reason: DataLoopExclusionReason, + /// Process identifier from the event header. + pub pid: u32, + /// Classified access type when the event produced a denial candidate. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub access_type: Option, + /// Classified resource type when the event produced a denial candidate. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_type: Option, + /// Sorted, bounded, username-redacted event properties. + pub properties: Vec<(String, String)>, +} + +/// One deduplicated Data Loop signature and its occurrence count. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DataLoopAggregate { + /// Sanitized signature shared by all counted occurrences. + pub signature: DataLoopSignature, + /// Number of matching exclusion outcomes. + pub count: u64, +} + +/// Bounded aggregate state produced by one decoder pass. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DataLoopSummary { + /// Sorted sanitized signatures retained within [`MAX_DATA_LOOP_GROUPS`]. + pub signatures: Vec, + /// Total recorded event outcomes, including groups collapsed into overflow. + pub total_occurrences: u64, + /// Outcomes whose new aggregate key could not be retained at the group bound. + pub overflow_occurrences: u64, + /// Canonical-denial occurrences whose signatures could not be retained. + pub canonical_overflow_occurrences: u64, + /// Whether aggregate keys were omitted at the group bound. + pub aggregate_groups_truncated: bool, + /// Whether the ETL processed-event bound prevented complete accounting. + pub processed_events_truncated: bool, + /// Whether the canonical unique-denial bound was reached. + pub canonical_denial_limit_reached: bool, +} + +impl DataLoopSummary { + /// Records one event outcome while preserving deterministic group order. + pub fn record(&mut self, signature: DataLoopSignature) { + self.total_occurrences = self.total_occurrences.saturating_add(1); + match self + .signatures + .binary_search_by(|group| group.signature.cmp(&signature)) + { + Ok(index) => { + self.signatures[index].count = self.signatures[index].count.saturating_add(1); + } + Err(index) if self.signatures.len() < MAX_DATA_LOOP_GROUPS => { + self.signatures.insert( + index, + DataLoopAggregate { + signature, + count: 1, + }, + ); + } + Err(_) => { + if signature.reason.is_canonical_denial() && self.evict_one_noncanonical_group(None) + { + let index = self + .signatures + .binary_search_by(|group| group.signature.cmp(&signature)) + .unwrap_err(); + self.signatures.insert( + index, + DataLoopAggregate { + signature, + count: 1, + }, + ); + } else { + self.record_overflow(signature.reason.is_canonical_denial()); + self.total_occurrences = self.total_occurrences.saturating_sub(1); + } + } + } + } + + /// Records an outcome while bounding compact serialized signature bytes. + pub fn record_with_byte_budget( + &mut self, + signature: DataLoopSignature, + retained_bytes: &mut usize, + max_bytes: usize, + ) { + if let Ok(index) = self + .signatures + .binary_search_by(|group| group.signature.cmp(&signature)) + { + self.total_occurrences = self.total_occurrences.saturating_add(1); + self.signatures[index].count = self.signatures[index].count.saturating_add(1); + return; + } + + let serialized_len = Self::serialized_signature_len(&signature); + while self.signatures.len() >= MAX_DATA_LOOP_GROUPS + || retained_bytes.saturating_add(serialized_len) > max_bytes + { + if !signature.reason.is_canonical_denial() + || !self.evict_one_noncanonical_group(Some(retained_bytes)) + { + self.record_overflow(signature.reason.is_canonical_denial()); + return; + } + } + + let index = self + .signatures + .binary_search_by(|group| group.signature.cmp(&signature)) + .unwrap_err(); + self.signatures.insert( + index, + DataLoopAggregate { + signature, + count: 1, + }, + ); + self.total_occurrences = self.total_occurrences.saturating_add(1); + *retained_bytes = retained_bytes.saturating_add(serialized_len); + } + + /// Counts an outcome whose new signature could not be retained at a bound. + pub fn record_overflow(&mut self, canonical: bool) { + self.total_occurrences = self.total_occurrences.saturating_add(1); + self.overflow_occurrences = self.overflow_occurrences.saturating_add(1); + if canonical { + self.canonical_overflow_occurrences = + self.canonical_overflow_occurrences.saturating_add(1); + } + self.aggregate_groups_truncated = true; + } + + /// Moves a retained aggregate into overflow accounting. + pub(crate) fn move_to_overflow(&mut self, aggregate: DataLoopAggregate) { + self.overflow_occurrences = self.overflow_occurrences.saturating_add(aggregate.count); + if aggregate.signature.reason.is_canonical_denial() { + self.canonical_overflow_occurrences = self + .canonical_overflow_occurrences + .saturating_add(aggregate.count); + } + self.aggregate_groups_truncated = true; + } + + fn evict_one_noncanonical_group(&mut self, retained_bytes: Option<&mut usize>) -> bool { + let Some(index) = self + .signatures + .iter() + .rposition(|group| !group.signature.reason.is_canonical_denial()) + else { + return false; + }; + let aggregate = self.signatures.remove(index); + if let Some(retained_bytes) = retained_bytes { + *retained_bytes = + retained_bytes.saturating_sub(Self::serialized_signature_len(&aggregate.signature)); + } + self.move_to_overflow(aggregate); + true + } + + /// Marks accounting incomplete because the processed-event bound was reached. + pub fn mark_processed_events_truncated(&mut self) { + self.processed_events_truncated = true; + } + + /// Marks that otherwise-valid candidates exceeded the canonical result bound. + pub fn mark_canonical_denial_limit_reached(&mut self) { + self.canonical_denial_limit_reached = true; + } + + /// Returns whether this analysis observed no excluded outcomes or truncation. + #[must_use] + pub fn is_empty(&self) -> bool { + self.signatures.is_empty() + && self.total_occurrences == 0 + && self.overflow_occurrences == 0 + && self.canonical_overflow_occurrences == 0 + && !self.aggregate_groups_truncated + && !self.processed_events_truncated + && !self.canonical_denial_limit_reached + } + + fn serialized_signature_len(signature: &DataLoopSignature) -> usize { + serde_json::to_vec(&DataLoopAggregate { + signature: signature.clone(), + count: 1, + }) + .map_or(usize::MAX, |bytes| bytes.len()) + } +} + +/// Versioned on-disk Data Loop document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DataLoopDocument { + /// Data Loop schema version. + pub version: u32, + /// Sorted deduplicated sanitized signatures. + pub signatures: Vec, + /// Aggregate summary and bounds state. + pub summary: DataLoopDocumentSummary, +} + +/// Bounds and count summary for a [`DataLoopDocument`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DataLoopDocumentSummary { + /// Total recorded event outcomes, including overflow. + pub total_occurrences: u64, + /// Outcomes collapsed because the group bound was reached. + pub overflow_occurrences: u64, + /// Canonical-denial occurrences collapsed into overflow. + pub canonical_overflow_occurrences: u64, + /// Whether aggregate keys were omitted. + pub aggregate_groups_truncated: bool, + /// Whether the processed-event limit prevented complete accounting. + pub processed_events_truncated: bool, + /// Whether the canonical unique-denial limit was reached. + pub canonical_denial_limit_reached: bool, +} + +impl DataLoopDocument { + /// Current Data Loop document schema version. + pub const VERSION: u32 = 1; + + /// Builds an on-disk document from decoder aggregate state. + #[must_use] + pub fn new(summary: &DataLoopSummary) -> Self { + Self { + version: Self::VERSION, + signatures: summary.signatures.clone(), + summary: DataLoopDocumentSummary { + total_occurrences: summary.total_occurrences, + overflow_occurrences: summary.overflow_occurrences, + canonical_overflow_occurrences: summary.canonical_overflow_occurrences, + aggregate_groups_truncated: summary.aggregate_groups_truncated, + processed_events_truncated: summary.processed_events_truncated, + canonical_denial_limit_reached: summary.canonical_denial_limit_reached, + }, + } + } +} + +/// Derives the deterministic Data Loop sibling for a canonical denials path. +/// +/// # Errors +/// +/// Returns an error when `output_path` has no usable file name. +pub fn data_loop_sibling_path(output_path: &Path) -> io::Result { + let file_name = output_path + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + io::Error::other(format!( + "denials output path has no usable file name: {}", + output_path.display() + )) + })?; + let stem = output_path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or(file_name); + let sibling = format!("{stem}.data-loop.json"); + Ok(match output_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.join(sibling), + _ => PathBuf::from(sibling), + }) +} + +/// Writes a pretty-printed Data Loop JSON document and trailing newline. +/// +/// # Errors +/// +/// Returns serialization or underlying writer failures. +pub fn write_data_loop_document( + writer: &mut W, + document: &DataLoopDocument, +) -> io::Result<()> { + serde_json::to_writer_pretty(&mut *writer, document).map_err(io::Error::other)?; + writer.write_all(b"\n")?; + writer.flush() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aggregates_and_sorts_sanitized_signatures() { + let mut summary = DataLoopSummary::default(); + let signature = DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}".to_string(), + event_id: 14, + reason: DataLoopExclusionReason::MissingObjectName, + pid: 42, + access_type: None, + resource_type: None, + properties: vec![ + ( + "ObjectName".to_string(), + r"C:\Users\\x".to_string(), + ), + ("Sid".to_string(), "S-1-15-3-1".to_string()), + ], + }; + summary.record(signature.clone()); + summary.record(signature); + + assert_eq!(summary.total_occurrences, 2); + assert_eq!(summary.signatures.len(), 1); + assert_eq!(summary.signatures[0].count, 2); + + let mut bytes = Vec::new(); + write_data_loop_document(&mut bytes, &DataLoopDocument::new(&summary)).unwrap(); + let text = String::from_utf8(bytes).unwrap(); + assert!(text.contains("S-1-15-3-1")); + assert!(text.contains("")); + assert!(text.contains("providerGuid")); + assert!(text.contains("\"pid\": 42")); + assert!(!text.contains("accessType")); + assert!(!text.contains("resourceType")); + } + + #[test] + fn group_bound_collapses_new_keys_into_overflow() { + let mut summary = DataLoopSummary::default(); + for event_id in 0..MAX_DATA_LOOP_GROUPS as u16 { + summary.record(DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "kernel".to_string(), + event_id, + reason: DataLoopExclusionReason::UnsupportedEventSchema, + pid: 1, + access_type: None, + resource_type: None, + properties: Vec::new(), + }); + } + summary.record(DataLoopSignature { + provider: DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + provider_guid: "privacy".to_string(), + event_id: u16::MAX, + reason: DataLoopExclusionReason::UnsupportedEventSchema, + pid: 1, + access_type: None, + resource_type: None, + properties: Vec::new(), + }); + + assert_eq!(summary.signatures.len(), MAX_DATA_LOOP_GROUPS); + assert_eq!(summary.overflow_occurrences, 1); + assert!(summary.aggregate_groups_truncated); + } + + #[test] + fn canonical_signature_evicts_noncanonical_signature_at_group_bound() { + let mut summary = DataLoopSummary::default(); + for event_id in 0..MAX_DATA_LOOP_GROUPS as u16 { + summary.record(DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "kernel".to_string(), + event_id, + reason: DataLoopExclusionReason::UnsupportedEventSchema, + pid: 1, + access_type: None, + resource_type: None, + properties: Vec::new(), + }); + } + summary.record(DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "kernel".to_string(), + event_id: u16::MAX, + reason: DataLoopExclusionReason::CanonicalDenial, + pid: 1, + access_type: Some(crate::AccessType::Read), + resource_type: Some(crate::ResourceType::File), + properties: Vec::new(), + }); + + assert!(summary + .signatures + .iter() + .any(|group| group.signature.reason == DataLoopExclusionReason::CanonicalDenial)); + assert_eq!(summary.overflow_occurrences, 1); + assert_eq!(summary.canonical_overflow_occurrences, 0); + } + + #[test] + fn byte_budget_leaves_guarded_analysis_protocol_headroom() { + let mut summary = DataLoopSummary::default(); + let mut retained_bytes = 0; + let properties = (0..24) + .map(|index| (format!("Property{index:02}"), "x".repeat(256))) + .collect::>(); + for pid in 0..MAX_DATA_LOOP_GROUPS as u32 { + summary.record_with_byte_budget( + DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}".to_string(), + event_id: 14, + reason: DataLoopExclusionReason::CanonicalDenial, + pid, + access_type: Some(crate::AccessType::Read), + resource_type: Some(crate::ResourceType::File), + properties: properties.clone(), + }, + &mut retained_bytes, + MAX_DATA_LOOP_SIGNATURE_BYTES, + ); + } + + assert!(summary.aggregate_groups_truncated); + assert!(summary.overflow_occurrences > 0); + assert!(retained_bytes <= MAX_DATA_LOOP_SIGNATURE_BYTES); + let payload = serde_json::to_vec(&crate::AnalysisResult { + denials: Vec::new(), + denied_resources_truncated: false, + data_loop: summary, + }) + .unwrap(); + assert!(payload.len() < 64 * 1024 * 1024); + } + + #[test] + fn document_round_trips() { + let mut summary = DataLoopSummary::default(); + summary.mark_canonical_denial_limit_reached(); + summary.mark_processed_events_truncated(); + let document = DataLoopDocument::new(&summary); + let mut bytes = Vec::new(); + write_data_loop_document(&mut bytes, &document).unwrap(); + let parsed: DataLoopDocument = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(parsed, document); + } + + #[test] + fn sibling_path_replaces_the_canonical_extension() { + assert_eq!( + data_loop_sibling_path(Path::new(r"C:\out\denials.123.json")).unwrap(), + PathBuf::from(r"C:\out\denials.123.data-loop.json") + ); + assert_eq!( + data_loop_sibling_path(Path::new("denials")).unwrap(), + PathBuf::from("denials.data-loop.json") + ); + } +} diff --git a/src/core/learning_mode_core/src/lib.rs b/src/core/learning_mode_core/src/lib.rs index b6d79508b..46a3a03dc 100644 --- a/src/core/learning_mode_core/src/lib.rs +++ b/src/core/learning_mode_core/src/lib.rs @@ -12,14 +12,15 @@ //! [`DeniedResource`] records. The per-OS decoder implements //! [`DenialAnalyzer`]; this crate owns the trait and the model. //! 3. **Emit** — the records plus a terminating [`DenialSummary`] are -//! written to a single JSON output file that host applications read to -//! regenerate their sandbox policy. See [`emit`]. +//! written to the canonical JSON output that host applications read to +//! regenerate policy. A deterministic Data Loop sibling contains bounded, +//! username-redacted signatures for excluded decoder outcomes. //! //! This crate is the cross-platform hinge between stages 2 and 3: it //! defines the public [`DeniedResource`] model, the [`DenialSummary`] -//! terminator, the [`DenialsDocument`] output shape plus its [`emit`]ter, -//! and the [`DenialAnalyzer`] decode trait. It carries no OS-specific code -//! so the wire format never encodes a platform assumption. +//! terminator, the [`DenialsDocument`] and [`DataLoopDocument`] output shapes, +//! and the [`DenialAnalyzer`] decode trait. It carries no OS-specific code so +//! the wire format never encodes a platform assumption. //! //! ## Mode caveat //! @@ -32,11 +33,17 @@ #![deny(missing_docs)] pub mod analyze; +pub mod data_loop; pub mod emit; pub mod model; pub mod summary; pub use analyze::{AnalysisResult, AnalyzeError, DenialAnalyzer, ProcessLifetime}; +pub use data_loop::{ + data_loop_sibling_path, write_data_loop_document, DataLoopAggregate, DataLoopDocument, + DataLoopDocumentSummary, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, + DataLoopSummary, MAX_DATA_LOOP_GROUPS, MAX_DATA_LOOP_SIGNATURE_BYTES, +}; pub use emit::{write_document, DenialsDocument, DenialsOutputPointer}; pub use model::{AccessType, DedupKey, DeniedResource, ResourceType}; pub use summary::DenialSummary; diff --git a/src/core/learning_mode_core/src/model.rs b/src/core/learning_mode_core/src/model.rs index 26110b275..1354d6735 100644 --- a/src/core/learning_mode_core/src/model.rs +++ b/src/core/learning_mode_core/src/model.rs @@ -43,7 +43,7 @@ mod decimal_u64 { /// The variant set is deliberately closed and cross-platform. The /// Windows decoder classifies ETW events into these buckets; other /// backends map their native sources onto the same vocabulary. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum ResourceType { /// Filesystem path (file or directory). @@ -62,7 +62,7 @@ pub enum ResourceType { } /// The kind of access that was attempted and denied. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum AccessType { /// Read / query access. diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index f2d6dfb06..73044e63f 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -61,6 +61,13 @@ pub fn finalize( .as_mut() .ok_or_else(|| "captureDenials returned no successful output metadata".to_string())?; let source_denials = PathBuf::from(&capture.output_path); + let source_data_loop = + learning_mode_core::data_loop_sibling_path(&source_denials).map_err(|error| { + format!( + "failed to derive Data Loop path from {}: {error}", + source_denials.display() + ) + })?; let source_etl = capture .etl_path .as_deref() @@ -78,6 +85,12 @@ pub fn finalize( source_etl.display() )); } + if !source_data_loop.is_file() { + return Err(format!( + "captureDenials Data Loop output file is missing: {}", + source_data_loop.display() + )); + } let document: learning_mode_core::DenialsDocument = serde_json::from_slice(&std::fs::read(&source_denials).map_err(|error| { @@ -93,13 +106,31 @@ pub fn finalize( ) })?; validate_metadata(capture, &document)?; + let _: learning_mode_core::DataLoopDocument = + serde_json::from_slice(&std::fs::read(&source_data_loop).map_err(|error| { + format!( + "failed to read captureDenials Data Loop output {}: {error}", + source_data_loop.display() + ) + })?) + .map_err(|error| { + format!( + "captureDenials Data Loop output {} is not valid JSON: {error}", + source_data_loop.display() + ) + })?; let final_denials = context.log_dir.join("denials.json"); + let final_data_loop = context.log_dir.join("denials.data-loop.json"); let final_etl = context.log_dir.join("trace.etl"); ensure_destination_available(&source_etl, &final_etl)?; + ensure_destination_available(&source_data_loop, &final_data_loop)?; ensure_destination_available(&source_denials, &final_denials)?; - move_new_file(&source_etl, &final_etl)?; - move_new_file(&source_denials, &final_denials)?; + move_new_files(&[ + (&source_etl, &final_etl), + (&source_data_loop, &final_data_loop), + (&source_denials, &final_denials), + ])?; capture.output_path = final_denials.to_string_lossy().into_owned(); capture.etl_path = Some(final_etl.to_string_lossy().into_owned()); remove_empty_managed_directory(source_etl.parent(), &context.log_dir)?; @@ -145,14 +176,26 @@ fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { match std::fs::rename(source, destination) { Ok(()) => Ok(()), Err(rename_error) => { - std::fs::copy(source, destination).map_err(|copy_error| { - format!( - "failed to move audit artifact {} to {}: {rename_error}; \ - copy fallback failed: {copy_error}", - source.display(), - destination.display() - ) - })?; + if let Err(copy_error) = std::fs::copy(source, destination) { + let cleanup_error = std::fs::remove_file(destination) + .err() + .filter(|error| error.kind() != std::io::ErrorKind::NotFound); + return Err(match cleanup_error { + Some(cleanup_error) => format!( + "failed to move audit artifact {} to {}: {rename_error}; \ + copy fallback failed: {copy_error}; partial destination cleanup failed: \ + {cleanup_error}", + source.display(), + destination.display() + ), + None => format!( + "failed to move audit artifact {} to {}: {rename_error}; \ + copy fallback failed: {copy_error}", + source.display(), + destination.display() + ), + }); + } std::fs::remove_file(source).map_err(|remove_error| { let _ = std::fs::remove_file(destination); format!( @@ -165,6 +208,31 @@ fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { } } +fn move_new_files(files: &[(&Path, &Path)]) -> Result<(), String> { + let mut moved = Vec::new(); + for &(source, destination) in files { + if source == destination { + continue; + } + if let Err(error) = move_new_file(source, destination) { + let rollback_errors = moved + .into_iter() + .rev() + .filter_map(|(source, destination)| move_new_file(destination, source).err()) + .collect::>(); + if rollback_errors.is_empty() { + return Err(error); + } + return Err(format!( + "{error}; audit artifact rollback also failed: {}", + rollback_errors.join("; ") + )); + } + moved.push((source, destination)); + } + Ok(()) +} + fn ensure_destination_available(source: &Path, destination: &Path) -> Result<(), String> { if source != destination && destination.exists() { return Err(format!( @@ -209,7 +277,7 @@ fn remove_empty_managed_directory( #[cfg(test)] mod tests { use super::*; - use learning_mode_core::{DenialSummary, DenialsDocument}; + use learning_mode_core::{DataLoopDocument, DataLoopSummary, DenialSummary, DenialsDocument}; use wxc_common::models::{ CaptureDenialsErrorOutput, CaptureDenialsOutput, SandboxOutputMetadata, }; @@ -256,9 +324,13 @@ mod tests { finalize(&mut response, &context, directory.path(), false).unwrap(); assert!(!source_denials.exists()); + assert!(!learning_mode_core::data_loop_sibling_path(&source_denials) + .unwrap() + .exists()); assert!(!source_etl.exists()); assert!(!retained_dir.exists()); assert_eq!(std::fs::read(log_dir.join("trace.etl")).unwrap(), b"etl"); + assert!(log_dir.join("denials.data-loop.json").is_file()); let capture = response .output_metadata .as_ref() @@ -311,6 +383,7 @@ mod tests { Some(log_dir.join("trace.etl").as_path()) ); assert!(log_dir.join("denials.json").is_file()); + assert!(log_dir.join("denials.data-loop.json").is_file()); assert!(log_dir.join("trace.etl").is_file()); } @@ -360,6 +433,9 @@ mod tests { assert!(error.contains("canonical denials JSON")); assert!(source_denials.exists()); + assert!(learning_mode_core::data_loop_sibling_path(&source_denials) + .unwrap() + .exists()); assert!(source_etl.exists()); } @@ -384,15 +460,73 @@ mod tests { assert!(error.contains("destination already exists")); assert!(source_denials.exists()); + assert!(learning_mode_core::data_loop_sibling_path(&source_denials) + .unwrap() + .exists()); assert!(source_etl.exists()); assert!(!log_dir.join("trace.etl").exists()); } + #[test] + fn finalize_rejects_missing_data_loop_output() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&log_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_etl = log_dir.join("denials.unique.etl"); + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, false)); + std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 0); + std::fs::remove_file(learning_mode_core::data_loop_sibling_path(&source_denials).unwrap()) + .unwrap(); + let context = AuditContext { + log_dir, + config_path: None, + }; + + let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); + + assert!(error.contains("Data Loop output file is missing")); + assert!(source_denials.exists()); + assert!(source_etl.exists()); + } + + #[test] + fn artifact_move_failure_rolls_back_prior_moves() { + let directory = tempfile::tempdir().unwrap(); + let source_one = directory.path().join("source-one"); + let source_two = directory.path().join("source-two"); + let missing_source = directory.path().join("missing"); + let destination_one = directory.path().join("destination-one"); + let destination_two = directory.path().join("destination-two"); + let destination_three = directory.path().join("destination-three"); + std::fs::write(&source_one, b"one").unwrap(); + std::fs::write(&source_two, b"two").unwrap(); + + let error = move_new_files(&[ + (&source_one, &destination_one), + (&source_two, &destination_two), + (&missing_source, &destination_three), + ]) + .unwrap_err(); + + assert!(error.contains("failed to move audit artifact")); + assert_eq!(std::fs::read(&source_one).unwrap(), b"one"); + assert_eq!(std::fs::read(&source_two).unwrap(), b"two"); + assert!(!destination_one.exists()); + assert!(!destination_two.exists()); + assert!(!destination_three.exists()); + } + fn response_with_capture( denials_path: &Path, etl_path: &Path, exit_code: i32, ) -> ScriptResponse { + let data_loop_path = learning_mode_core::data_loop_sibling_path(denials_path).unwrap(); + let data_loop = DataLoopDocument::new(&DataLoopSummary::default()); + std::fs::write(data_loop_path, serde_json::to_vec(&data_loop).unwrap()).unwrap(); ScriptResponse { exit_code, output_metadata: Some(Box::new(SandboxOutputMetadata { diff --git a/src/host/plm/src/analysis.rs b/src/host/plm/src/analysis.rs index 17c70c907..c334b7228 100644 --- a/src/host/plm/src/analysis.rs +++ b/src/host/plm/src/analysis.rs @@ -292,6 +292,7 @@ mod tests { let analysis = AnalysisResult { denials: vec![denial(r"C:\read.txt", ResourceType::File, AccessType::Read)], denied_resources_truncated: true, + data_loop: Default::default(), }; write_denials(&path, &analysis, 7).unwrap(); diff --git a/src/host/plm/src/log.rs b/src/host/plm/src/log.rs index b818d97d0..5d4998fb4 100644 --- a/src/host/plm/src/log.rs +++ b/src/host/plm/src/log.rs @@ -98,6 +98,7 @@ mod tests { assert!(!can_generate_policy_preview(&AnalysisResult { denials: Vec::new(), denied_resources_truncated: true, + data_loop: Default::default(), })); } } From a2208623bfb4434dd9d31129f7c889251fc0ab1e Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 17:23:11 -0700 Subject: [PATCH 11/34] Preserve Learning Mode Data Loop exclusions Keep excluded namespace evidence uniquely identifiable, retain guarded-trace unknown schemas and brokered capability events, and emit canonical/Data Loop outputs as a transactional pair. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- docs/learning-mode/capabilities.md | 20 + src/Cargo.lock | 2 + .../appcontainer/common/src/capture_output.rs | 153 +------ src/backends/learning_mode/windows/Cargo.toml | 1 + .../learning_mode/windows/src/etl_decode.rs | 314 +++++++++++-- .../learning_mode/windows/src/etl_filter.rs | 4 +- .../learning_mode/windows/src/extractors.rs | 117 ++++- src/core/learning_mode_core/Cargo.toml | 1 + src/core/learning_mode_core/src/lib.rs | 2 + .../learning_mode_core/src/paired_output.rs | 419 ++++++++++++++++++ src/host/plm/src/analysis.rs | 56 ++- src/host/plm/src/stop.rs | 39 +- 12 files changed, 920 insertions(+), 208 deletions(-) create mode 100644 src/core/learning_mode_core/src/paired_output.rs diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index 0365fe55b..dbef5d63e 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -275,6 +275,26 @@ the canonical unique-denial bound. Those occurrences deduplicate under the same signature and increment its count. `accessType` and `resourceType` are included when denial extraction determined them; diagnostic outcomes without those classifications omit the fields. + +Candidates excluded from canonical policy output retain a closed diagnostic +reason and their sanitized event properties: + +- `unusableResourcePath` means a File access-check resource could not be + converted to a safe absolute DOS or UNC path. For example, + `\Device\MountPointManager` is useful Devices-namespace evidence, but it is + not a directly authorable filesystem grant. +- `unsupportedObjectType` means the event names a resource outside the + canonical policy model. Examples include `\BaseNamedObjects` as a Directory, + shared-cache Sections, SymbolicLinks, ALPC Ports such as + `ubpmtaskhostchannel`, and RPC Interface GUIDs. + +Property values longer than 256 characters retain bounded prefix and suffix +context plus a SHA-256 digest of the complete sanitized value. This keeps long +named-object resources individually identifiable when they share a prefix +without exceeding the per-property bound. Username redaction occurs before the +digest is computed, so neither the retained context nor the digest is derived +from the original username. + Unknown event IDs from known Learning Mode providers are classified as `unsupportedEventSchema`; the real ETL path retains their provider GUID and PID without attempting an unsupported TDH payload decode. diff --git a/src/Cargo.lock b/src/Cargo.lock index f0c9c9b62..782535943 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1275,6 +1275,7 @@ version = "0.7.0" dependencies = [ "serde", "serde_json", + "tempfile", "thiserror", ] @@ -1285,6 +1286,7 @@ dependencies = [ "flatbuffers", "learning_mode_core", "process_security_environment_spec", + "sha2 0.10.9", "thiserror", "windows", "windows-core", diff --git a/src/backends/appcontainer/common/src/capture_output.rs b/src/backends/appcontainer/common/src/capture_output.rs index c26330d71..71e7e6eb1 100644 --- a/src/backends/appcontainer/common/src/capture_output.rs +++ b/src/backends/appcontainer/common/src/capture_output.rs @@ -20,8 +20,9 @@ use std::io::Write; use std::path::{Path, PathBuf}; use learning_mode_core::{ - data_loop_sibling_path, write_data_loop_document, write_document, AnalysisResult, - DataLoopDocument, DenialSummary, DenialsDocument, DenialsOutputPointer, + data_loop_sibling_path, write_data_loop_document, write_document, write_paired_output_files, + AnalysisResult, DataLoopDocument, DenialSummary, DenialsDocument, DenialsOutputPointer, + ExistingOutputPolicy, }; use wxc_common::models::CaptureDenialsOutput; @@ -52,8 +53,10 @@ pub fn write_denials_document( let data_loop_document = DataLoopDocument::new(&analysis.data_loop); write_paired_output_files( + "captureDenials", output_path, &data_loop_path, + ExistingOutputPolicy::CreateNew, |writer| write_document(writer, &document), |writer| write_data_loop_document(writer, &data_loop_document), )?; @@ -75,150 +78,6 @@ pub fn data_loop_output_path(output_path: &Path) -> std::io::Result { .map_err(|error| std::io::Error::other(format!("captureDenials {error}"))) } -fn write_paired_output_files( - denials_path: &Path, - data_loop_path: &Path, - write_denials: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, - write_data_loop: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, -) -> std::io::Result<()> { - ensure_output_absent(denials_path, "denials")?; - ensure_output_absent(data_loop_path, "Data Loop")?; - - let denials_temp = stage_output_file(denials_path, "denials", write_denials)?; - let data_loop_temp = match stage_output_file(data_loop_path, "Data Loop", write_data_loop) { - Ok(path) => path, - Err(error) => { - return Err(combine_error_with_cleanup( - error, - remove_if_present(&denials_temp), - &denials_temp, - )) - } - }; - - if let Err(error) = std::fs::rename(&data_loop_temp, data_loop_path) { - let error = std::io::Error::other(format!( - "captureDenials failed to promote Data Loop output file {}: {error}", - data_loop_path.display() - )); - let error = - combine_error_with_cleanup(error, remove_if_present(&data_loop_temp), &data_loop_temp); - return Err(combine_error_with_cleanup( - error, - remove_if_present(&denials_temp), - &denials_temp, - )); - } - - if let Err(error) = std::fs::rename(&denials_temp, denials_path) { - let error = std::io::Error::other(format!( - "captureDenials failed to promote denials output file {}: {error}", - denials_path.display() - )); - let error = - combine_error_with_cleanup(error, remove_if_present(data_loop_path), data_loop_path); - return Err(combine_error_with_cleanup( - error, - remove_if_present(&denials_temp), - &denials_temp, - )); - } - - Ok(()) -} - -fn ensure_output_absent(path: &Path, kind: &str) -> std::io::Result<()> { - match std::fs::symlink_metadata(path) { - Ok(_) => Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - format!( - "captureDenials {kind} output file already exists: {}", - path.display() - ), - )), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(std::io::Error::other(format!( - "captureDenials failed to inspect {kind} output file {}: {error}", - path.display() - ))), - } -} - -fn stage_output_file( - final_path: &Path, - kind: &str, - write: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, -) -> std::io::Result { - let temp_path = temporary_sibling_path(final_path)?; - let file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&temp_path) - .map_err(|error| { - std::io::Error::other(format!( - "captureDenials failed to create temporary {kind} output file {}: {error}", - temp_path.display() - )) - })?; - let mut writer = std::io::BufWriter::new(file); - if let Err(error) = write(&mut writer) - .and_then(|()| writer.flush()) - .and_then(|()| writer.get_ref().sync_all()) - { - let error = std::io::Error::other(format!( - "captureDenials failed to write temporary {kind} output file {}: {error}", - temp_path.display() - )); - drop(writer); - return Err(combine_error_with_cleanup( - error, - remove_if_present(&temp_path), - &temp_path, - )); - } - Ok(temp_path) -} - -fn temporary_sibling_path(final_path: &Path) -> std::io::Result { - let file_name = final_path - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - std::io::Error::other(format!( - "captureDenials output path has no usable file name: {}", - final_path.display() - )) - })?; - let suffix = random_capture_suffix().map_err(std::io::Error::other)?; - let temp_name = format!(".{file_name}.tmp.{}.{suffix}", std::process::id()); - Ok(match final_path.parent() { - Some(parent) if !parent.as_os_str().is_empty() => parent.join(temp_name), - _ => PathBuf::from(temp_name), - }) -} - -fn remove_if_present(path: &Path) -> std::io::Result<()> { - match std::fs::remove_file(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error), - } -} - -fn combine_error_with_cleanup( - error: std::io::Error, - cleanup: std::io::Result<()>, - cleanup_path: &Path, -) -> std::io::Error { - match cleanup { - Ok(()) => error, - Err(cleanup_error) => std::io::Error::other(format!( - "{error}; additionally failed to remove {}: {cleanup_error}", - cleanup_path.display() - )), - } -} - /// Creates `output_path` (failing if it already exists) and writes through /// `write`, cleaning up a partial file if `write` fails. pub fn write_denials_output_file( @@ -553,8 +412,10 @@ mod tests { let data_loop_path = data_loop_output_path(&output_path).unwrap(); let error = write_paired_output_files( + "captureDenials", &output_path, &data_loop_path, + ExistingOutputPolicy::CreateNew, |writer| writer.write_all(b"denials"), |writer| { writer.write_all(b"partial")?; diff --git a/src/backends/learning_mode/windows/Cargo.toml b/src/backends/learning_mode/windows/Cargo.toml index 3b45e78b2..861bcdd1b 100644 --- a/src/backends/learning_mode/windows/Cargo.toml +++ b/src/backends/learning_mode/windows/Cargo.toml @@ -9,6 +9,7 @@ thiserror = { workspace = true } learning_mode_core = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] +sha2 = { workspace = true } wxc_common = { workspace = true } windows = { workspace = true } windows-core = { workspace = true } diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 7c5b335f6..950418e61 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -584,7 +584,7 @@ pub fn visit_raw_events( Ok(accumulator.raw_event_count) } -/// Builds a bounded decision vector for supported Learning Mode events in +/// Builds a bounded decision vector for known-provider Learning Mode events in /// source order. `ProcessTrace` normalizes each timestamp to FILETIME before /// the exact process-generation test, so Trace Relogger can replay these /// decisions without comparing its raw trace-clock timestamps. @@ -748,7 +748,7 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let provider = header.ProviderId; let event_id = header.EventDescriptor.Id; if matches!(acc.mode, CollectionMode::SelectForRelogging) { - if !is_learning_mode_event(provider, event_id) { + if crate::extractors::provider_category(provider).is_none() { return; } let event_index = acc.relog_event_count; @@ -762,15 +762,27 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let Some(filetime) = normalized_filetime(header.TimeStamp, acc) else { return; }; - if acc.event_in_scope(header.ProcessId, filetime) { - if acc.relog_selected_event_indices.len() >= MAX_PROCESSED_EVENTS { - acc.decode_error = Some(format!( - "trace exceeded the {MAX_PROCESSED_EVENTS}-event process-scoped relogging limit" - )); - acc.stop_requested = true; - return; + if event_id == crate::extractors::CAPABILITY_DENIAL_EVENT_ID + && is_learning_mode_event(provider, event_id) + { + match unsafe { tdh_decode::decode_event_parts(event_record, &mut acc.schema_cache) } { + Ok(parts) => select_decoded_event_for_relogging( + acc, + event_index, + &parts, + header.ProcessId, + filetime, + ), + Err(error) => { + acc.decode_error = Some(format!( + "failed to decode brokered capability event while scoping guarded trace: {error}" + )); + acc.stop_requested = true; + return; + } } - acc.relog_selected_event_indices.push(event_index); + } else { + select_event_for_relogging(acc, event_index, header.ProcessId, filetime); } return; } @@ -835,6 +847,41 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu } } +fn select_decoded_event_for_relogging( + acc: &mut Accumulator<'_>, + event_index: usize, + parts: &DecodedEventParts, + header_pid: u32, + filetime: u64, +) { + let Some(effective_pid) = crate::extractors::effective_event_pid(parts, header_pid) else { + acc.decode_error = + Some("brokered capability event has no valid payload ProcessId".to_string()); + acc.stop_requested = true; + return; + }; + select_event_for_relogging(acc, event_index, effective_pid, filetime); +} + +fn select_event_for_relogging( + acc: &mut Accumulator<'_>, + event_index: usize, + pid: u32, + filetime: u64, +) { + if !acc.event_in_scope(pid, filetime) { + return; + } + if acc.relog_selected_event_indices.len() >= MAX_PROCESSED_EVENTS { + acc.decode_error = Some(format!( + "trace exceeded the {MAX_PROCESSED_EVENTS}-event process-scoped relogging limit" + )); + acc.stop_requested = true; + return; + } + acc.relog_selected_event_indices.push(event_index); +} + /// Extracts denials from one decoded, in-vocabulary event and feeds them /// (or their closed exclusion reason) into `acc`. /// @@ -847,21 +894,35 @@ fn handle_decoded_event( filetime: u64, acc: &mut Accumulator<'_>, ) { - let pid = crate::extractors::effective_event_pid(parts, header_pid); - if !acc.event_in_scope(pid, filetime) || !acc.begin_event() { - return; - } let Some(category) = crate::extractors::provider_category(parts.provider) else { return; }; if !is_learning_mode_event(parts.provider, parts.event_id) { - acc.record_exclusion( - category, - parts.event_id, - DataLoopExclusionReason::UnsupportedEventSchema, - pid, - crate::extractors::sanitize_properties(&parts.props), - ); + if acc.event_in_scope(header_pid, filetime) && acc.begin_event() { + acc.record_exclusion( + category, + parts.event_id, + DataLoopExclusionReason::UnsupportedEventSchema, + header_pid, + crate::extractors::sanitize_properties(&parts.props), + ); + } + return; + } + let Some(pid) = crate::extractors::effective_event_pid(parts, header_pid) else { + if acc.process_lifetimes.is_none() && acc.begin_event() { + acc.record_outcome( + category, + parts.event_id, + DataLoopExclusionReason::EventPayloadMalformed, + header_pid, + crate::extractors::data_loop_classification(parts), + crate::extractors::sanitize_properties(&parts.props), + ); + } + return; + }; + if !acc.event_in_scope(pid, filetime) || !acc.begin_event() { return; } @@ -956,7 +1017,7 @@ mod tests { } #[test] - fn relog_selection_tracks_supported_event_ordinals_and_exact_lifetimes() { + fn relog_selection_tracks_known_provider_ordinals_and_exact_lifetimes() { let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { pid: 42, start_filetime: 100, @@ -964,9 +1025,8 @@ mod tests { }]); let mut visit = |provider, event_id, pid, filetime| { - // SAFETY: this zeroed record is used only by selection mode, which - // reads the initialized POD header fields below and never decodes - // payload pointers. + // SAFETY: these fixtures avoid the brokered capability event, so + // selection reads only the initialized POD header fields below. let mut record: EVENT_RECORD = unsafe { core::mem::zeroed() }; record.EventHeader.ProviderId = provider; record.EventHeader.EventDescriptor.Id = event_id; @@ -979,7 +1039,7 @@ mod tests { visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 14, 42, 100); visit(windows::core::GUID::from_u128(1), 14, 42, 150); visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 27, 42, 201); - visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 28, 42, 200); + visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 999, 42, 200); visit(crate::extractors::KERNEL_GENERAL_PROVIDER, 14, 43, 150); assert_eq!(accumulator.relog_event_count, 4); @@ -987,6 +1047,47 @@ mod tests { assert!(accumulator.decode_error.is_none()); } + #[test] + fn relog_selection_scopes_brokered_capability_events_by_payload_pid() { + let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + let parts = DecodedEventParts { + provider: crate::extractors::KERNEL_GENERAL_PROVIDER, + event_id: crate::extractors::CAPABILITY_DENIAL_EVENT_ID, + props: vec![("ProcessId".to_string(), "42".to_string())], + }; + + select_decoded_event_for_relogging(&mut accumulator, 0, &parts, 9000, 150); + + assert_eq!(accumulator.relog_selected_event_indices, [0]); + } + + #[test] + fn relog_selection_rejects_brokered_capability_without_payload_pid() { + let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + let parts = DecodedEventParts { + provider: crate::extractors::KERNEL_GENERAL_PROVIDER, + event_id: crate::extractors::CAPABILITY_DENIAL_EVENT_ID, + props: Vec::new(), + }; + + select_decoded_event_for_relogging(&mut accumulator, 0, &parts, 42, 150); + + assert!(accumulator.relog_selected_event_indices.is_empty()); + assert!(accumulator.stop_requested); + assert!(accumulator + .decode_error + .as_deref() + .is_some_and(|error| error.contains("payload ProcessId"))); + } + #[test] fn raw_visitor_panic_is_captured_inside_callback_state() { let mut visitor = @@ -1126,10 +1227,32 @@ mod tests { ), ]; - let out = dedup_to_resources(denials).denials; + let analysis = dedup_to_resources(denials); - assert_eq!(out.len(), 1); - assert_eq!(out[0].resource, r"\\server\share\file.txt"); + assert_eq!(analysis.denials.len(), 1); + assert_eq!(analysis.denials[0].resource, r"\\server\share\file.txt"); + let excluded = analysis + .data_loop + .signatures + .iter() + .filter(|group| group.signature.reason == DataLoopExclusionReason::UnusableResourcePath) + .collect::>(); + assert_eq!(excluded.len(), 4); + assert!(excluded.iter().all(|group| { + group.signature.access_type == Some(AccessType::Read) + && group.signature.resource_type == Some(ResourceType::File) + && group.count == 1 + })); + for expected in [ + r"\Device\UnknownVolume\file.txt", + r"C:relative.txt", + r"PIPE\name", + r"\\server\pipe\name", + ] { + assert!(excluded + .iter() + .any(|group| property(&group.signature, "resource") == expected)); + } } #[test] @@ -1668,6 +1791,139 @@ mod tests { ); } + #[test] + fn device_namespace_file_is_retained_as_unusable_resource_path() { + let events = vec![kernel_event( + 14, + 1996, + 134_309_021_955_593_414, + &[ + ("Mode", "\"Permissive\""), + ("ObjectType", "\"File\""), + ("ObjectName", "\"\\Device\\MountPointManager\""), + ("AccessMask", "0x100080"), + ], + )]; + + let out = resources_from_events(&events); + + assert!(out.denials.is_empty()); + assert_eq!(out.data_loop.signatures.len(), 1); + let group = &out.data_loop.signatures[0]; + assert_eq!(group.count, 1); + assert_eq!(group.signature.provider, DataLoopProvider::KernelGeneral); + assert_eq!( + group.signature.provider_guid, + "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}" + ); + assert_eq!(group.signature.event_id, 14); + assert_eq!( + group.signature.reason, + DataLoopExclusionReason::UnusableResourcePath + ); + assert_eq!(group.signature.pid, 1996); + assert_eq!(group.signature.access_type, Some(AccessType::Read)); + assert_eq!(group.signature.resource_type, Some(ResourceType::File)); + assert_eq!( + property(&group.signature, "ObjectName"), + r"\Device\MountPointManager" + ); + } + + #[test] + fn output_gap_object_types_are_individually_retained() { + let cases = [ + ("Directory", r"\BaseNamedObjects"), + ( + "Section", + r"\Sessions\1\AppContainerNamedObjects\S-1-15-2-1\cache.db", + ), + ("SymbolicLink", r"\??\FDC#GENERIC_FLOPPY_DRIVE"), + ( + "ALPC Port", + r"\Sessions\1\AppContainerNamedObjects\S-1-15-2-1\RPC Control\ubpmtaskhostchannel", + ), + ("RPC Interface", "f6beaff7-1e19-4fbb-9f8f-b89e2018337c"), + ]; + let events = cases + .iter() + .enumerate() + .map(|(index, (object_type, object_name))| CollectedEvent { + pid: 42, + filetime: index as u64 + 1, + parts: DecodedEventParts { + provider: crate::extractors::KERNEL_GENERAL_PROVIDER, + event_id: 14, + props: vec![ + ("Mode".to_string(), "\"Permissive\"".to_string()), + ("ObjectType".to_string(), format!("\"{object_type}\"")), + ("ObjectName".to_string(), format!("\"{object_name}\"")), + ("AccessMask".to_string(), "0x1".to_string()), + ], + }, + }) + .collect::>(); + + let out = resources_from_events(&events); + + assert!(out.denials.is_empty()); + assert_eq!(out.data_loop.signatures.len(), cases.len()); + for (object_type, object_name) in cases { + let group = out + .data_loop + .signatures + .iter() + .find(|group| property(&group.signature, "ObjectType") == object_type) + .unwrap_or_else(|| panic!("missing Data Loop signature for {object_type}")); + assert_eq!( + group.signature.reason, + DataLoopExclusionReason::UnsupportedObjectType + ); + assert_eq!(property(&group.signature, "ObjectName"), object_name); + assert_eq!(group.count, 1); + } + } + + #[test] + fn long_section_names_with_shared_prefix_remain_distinct() { + let prefix = format!( + r"\Sessions\1\AppContainerNamedObjects\S-1-15-2-{}\C:*ProgramData*Microsoft*Windows*Caches*", + "1-".repeat(100) + ); + let names = [ + format!("{prefix}{{6AF0698E-D558-4F6E-9B3C-3716689AF493}}.db"), + format!("{prefix}{{DDF571F2-BE98-426D-8288-1A9A39C3FDA2}}.db"), + format!("{prefix}cversions.2.ro"), + ]; + let events = names + .iter() + .enumerate() + .map(|(index, object_name)| CollectedEvent { + pid: 42, + filetime: index as u64 + 1, + parts: DecodedEventParts { + provider: crate::extractors::KERNEL_GENERAL_PROVIDER, + event_id: 14, + props: vec![ + ("ObjectType".to_string(), "\"Section\"".to_string()), + ("ObjectName".to_string(), format!("\"{object_name}\"")), + ("AccessMask".to_string(), "0x6".to_string()), + ], + }, + }) + .collect::>(); + + let out = resources_from_events(&events); + + assert!(out.denials.is_empty()); + assert_eq!(out.data_loop.signatures.len(), names.len()); + assert!(out.data_loop.signatures.iter().all(|group| { + group.signature.reason == DataLoopExclusionReason::UnsupportedObjectType + && group.count == 1 + && property(&group.signature, "ObjectName").contains(" bool { } } -pub(crate) fn effective_event_pid(parts: &DecodedEventParts, header_pid: u32) -> u32 { +pub(crate) fn effective_event_pid(parts: &DecodedEventParts, header_pid: u32) -> Option { if parts.event_id == CAPABILITY_DENIAL_EVENT_ID { - find_prop(&parts.props, "ProcessId") - .and_then(|value| parse_u32(value)) - .unwrap_or(header_pid) + find_prop(&parts.props, "ProcessId").and_then(|value| parse_u32(value)) } else { - header_pid + Some(header_pid) } } @@ -281,6 +280,9 @@ pub(crate) const REDACTED_USER: &str = ""; pub(crate) const MAX_SIGNATURE_PROPERTIES: usize = 24; /// Maximum retained length (in `char`s) of one sanitized property value. pub(crate) const MAX_SIGNATURE_VALUE_LEN: usize = 256; +const SHA256_HEX_LEN: usize = 64; +const BOUNDED_VALUE_MARKER_LEN: usize = "......".len(); +const _: () = assert!(MAX_SIGNATURE_VALUE_LEN > BOUNDED_VALUE_MARKER_LEN); /// Returns whether `name` looks like it carries a timestamp, so it is /// dropped from a Data Loop signature entirely (never redacted or @@ -378,19 +380,41 @@ pub(crate) fn redact_username_in_path(value: &str) -> String { /// Bounds an already-sanitized property list to [`MAX_SIGNATURE_PROPERTIES`] /// entries and [`MAX_SIGNATURE_VALUE_LEN`] characters per value. /// -/// Callers must sanitize (redact) *before* calling this: truncation is a -/// pure length cap and does not know how to avoid splitting a redaction -/// marker or a still-sensitive suffix. +/// Overlong values retain prefix and suffix context plus a digest of the full +/// sanitized value. The digest prevents distinct values with a long shared +/// prefix from collapsing into one Data Loop signature. +/// +/// Callers must sanitize (redact) *before* calling this so neither the retained +/// context nor the digest is derived from sensitive identity content. pub(crate) fn bound_properties(mut properties: Vec<(String, String)>) -> Vec<(String, String)> { properties.truncate(MAX_SIGNATURE_PROPERTIES); for (_, value) in &mut properties { if value.chars().count() > MAX_SIGNATURE_VALUE_LEN { - *value = value.chars().take(MAX_SIGNATURE_VALUE_LEN).collect(); + *value = bound_property_value(value); } } properties } +fn bound_property_value(value: &str) -> String { + let digest = Sha256::digest(value.as_bytes()); + let marker = format!("......"); + debug_assert_eq!(marker.len(), BOUNDED_VALUE_MARKER_LEN); + let context_len = MAX_SIGNATURE_VALUE_LEN - BOUNDED_VALUE_MARKER_LEN; + let prefix_len = context_len.div_ceil(2); + let suffix_len = context_len / 2; + let prefix = value.chars().take(prefix_len).collect::(); + let suffix = value + .chars() + .rev() + .take(suffix_len) + .collect::() + .chars() + .rev() + .collect::(); + format!("{prefix}{marker}{suffix}") +} + /// Produces the deterministic, sanitized, bounded property list for a Data /// Loop signature from one decoded event's raw TDH properties. /// @@ -1308,12 +1332,23 @@ mod tests { } #[test] - fn capability_denial_falls_back_to_header_pid() { + fn capability_denial_uses_supplied_effective_pid() { let p = parts(28, &[("Denied", "true"), ("PackageSid", "S-1-15-3-1")]); let ev = extract_denial(&p, 555, FIXED_FILETIME).unwrap(); assert_eq!(ev.pid, 555); } + #[test] + fn capability_event_requires_payload_pid_for_scoping() { + let missing = parts(28, &[("Denied", "true")]); + let malformed = parts(28, &[("ProcessId", "\"broker\"")]); + let valid = parts(28, &[("ProcessId", "42")]); + + assert_eq!(effective_event_pid(&missing, 555), None); + assert_eq!(effective_event_pid(&malformed, 555), None); + assert_eq!(effective_event_pid(&valid, 555), Some(42)); + } + #[test] fn unrelated_event_ignored() { let p = parts(9999, &[("Foo", "\"bar\"")]); @@ -1573,7 +1608,7 @@ mod tests { } #[test] - fn bound_properties_truncates_count_and_value_length_after_sanitizing() { + fn bound_properties_preserves_identity_within_count_and_value_limits() { let long_value = format!( r"C:\Users\{}\{}", "jsmith", @@ -1600,10 +1635,66 @@ mod tests { .expect("ObjectName retained (sorted first)"); assert!( object_name.1.contains(REDACTED_USER), - "redaction must survive truncation: {}", + "redaction must survive bounding: {}", object_name.1 ); - assert!(object_name.1.chars().count() <= MAX_SIGNATURE_VALUE_LEN); + assert!(object_name.1.contains(" std::io::Result { + let file_name = canonical_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + std::io::Error::other(format!( + "{operation} canonical output path has no usable file name: {}", + canonical_path.display() + )) + })?; + let lock_name = format!(".{file_name}.pair.lock"); + let lock_path = canonical_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(lock_name); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to open output-pair lock {}: {error}", + lock_path.display() + )) + })?; + file.try_lock().map_err(|error| { + std::io::Error::other(format!( + "{operation} output pair is being written by another process (lock {}): {error}", + lock_path.display() + )) + })?; + Ok(Self { file }) + } +} + +impl Drop for OutputPairLock { + fn drop(&mut self) { + let _ = std::fs::File::unlock(&self.file); + } +} + +/// Controls how an existing output pair is handled during promotion. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ExistingOutputPolicy { + /// Fail rather than overwrite either output. + CreateNew, + /// Atomically replace each individual output. + Replace, +} + +/// Stages, flushes, and promotes a canonical output and its Data Loop sibling. +/// +/// Both documents are serialized before either final path is changed. Replace +/// mode retains the previous files until both new files are promoted, allowing +/// a failed promotion to restore the complete previous pair. +pub fn write_paired_output_files( + operation: &str, + canonical_path: &Path, + data_loop_path: &Path, + policy: ExistingOutputPolicy, + write_canonical: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, + write_data_loop: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, +) -> std::io::Result<()> { + let _pair_lock = OutputPairLock::acquire(operation, canonical_path)?; + match policy { + ExistingOutputPolicy::CreateNew => { + ensure_output_absent(operation, canonical_path, "canonical")?; + ensure_output_absent(operation, data_loop_path, "Data Loop")?; + } + ExistingOutputPolicy::Replace => { + ensure_output_replaceable(operation, canonical_path, "canonical")?; + ensure_output_replaceable(operation, data_loop_path, "Data Loop")?; + } + } + + let canonical_temp = + stage_output_file(operation, canonical_path, "canonical", write_canonical)?; + let data_loop_temp = + stage_output_file(operation, data_loop_path, "Data Loop", write_data_loop)?; + + let canonical_backup = if policy == ExistingOutputPolicy::Replace { + backup_existing_output(operation, canonical_path, "canonical")? + } else { + None + }; + let data_loop_backup = if policy == ExistingOutputPolicy::Replace { + match backup_existing_output(operation, data_loop_path, "Data Loop") { + Ok(backup) => backup, + Err(error) => { + return Err(combine_error_with_cleanup( + error, + restore_output(canonical_path, canonical_backup.as_deref(), false), + canonical_path, + )) + } + } + } else { + None + }; + + if let Err(error) = promote_output_file(operation, data_loop_temp, data_loop_path, "Data Loop") + { + return Err(combine_error_with_cleanup( + error, + restore_output_pair( + canonical_path, + canonical_backup.as_deref(), + false, + data_loop_path, + data_loop_backup.as_deref(), + false, + ), + data_loop_path, + )); + } + if let Err(error) = promote_output_file(operation, canonical_temp, canonical_path, "canonical") + { + return Err(combine_error_with_cleanup( + error, + restore_output_pair( + canonical_path, + canonical_backup.as_deref(), + false, + data_loop_path, + data_loop_backup.as_deref(), + true, + ), + canonical_path, + )); + } + + remove_backups(canonical_backup.as_deref(), data_loop_backup.as_deref()) +} + +fn ensure_output_absent(operation: &str, path: &Path, kind: &str) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "{operation} {kind} output file already exists: {}", + path.display() + ), + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(std::io::Error::other(format!( + "{operation} failed to inspect {kind} output file {}: {error}", + path.display() + ))), + } +} + +fn ensure_output_replaceable(operation: &str, path: &Path, kind: &str) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_file() => Ok(()), + Ok(_) => Err(std::io::Error::other(format!( + "{operation} cannot replace non-file {kind} output path {}", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(std::io::Error::other(format!( + "{operation} failed to inspect {kind} output file {}: {error}", + path.display() + ))), + } +} + +fn stage_output_file( + operation: &str, + final_path: &Path, + kind: &str, + write: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, +) -> std::io::Result { + let parent = final_path.parent().unwrap_or_else(|| Path::new(".")); + let temp = tempfile::NamedTempFile::new_in(parent).map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to create temporary {kind} output file in {}: {error}", + parent.display() + )) + })?; + let file = temp.as_file().try_clone().map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to open temporary {kind} output file for {}: {error}", + final_path.display() + )) + })?; + let mut writer = std::io::BufWriter::new(file); + write(&mut writer) + .and_then(|()| writer.flush()) + .and_then(|()| writer.get_ref().sync_all()) + .map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to write temporary {kind} output file for {}: {error}", + final_path.display() + )) + })?; + Ok(temp) +} + +fn promote_output_file( + operation: &str, + temp: tempfile::NamedTempFile, + final_path: &Path, + kind: &str, +) -> std::io::Result<()> { + temp.persist_noclobber(final_path) + .map(|_| ()) + .map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to promote {kind} output file {}: {}", + final_path.display(), + error.error + )) + }) +} + +fn backup_existing_output( + operation: &str, + final_path: &Path, + kind: &str, +) -> std::io::Result> { + if !final_path.try_exists().map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to inspect {kind} output file {}: {error}", + final_path.display() + )) + })? { + return Ok(None); + } + + let parent = final_path.parent().unwrap_or_else(|| Path::new(".")); + let backup = tempfile::NamedTempFile::new_in(parent).map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to reserve backup for {kind} output file {}: {error}", + final_path.display() + )) + })?; + let backup_path = backup.path().to_path_buf(); + backup.close().map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to prepare backup for {kind} output file {}: {error}", + final_path.display() + )) + })?; + std::fs::rename(final_path, &backup_path).map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to back up {kind} output file {}: {error}", + final_path.display() + )) + })?; + Ok(Some(backup_path)) +} + +fn restore_output_pair( + canonical_path: &Path, + canonical_backup: Option<&Path>, + canonical_promoted: bool, + data_loop_path: &Path, + data_loop_backup: Option<&Path>, + data_loop_promoted: bool, +) -> std::io::Result<()> { + let data_loop_result = restore_output(data_loop_path, data_loop_backup, data_loop_promoted); + let canonical_result = restore_output(canonical_path, canonical_backup, canonical_promoted); + match (data_loop_result, canonical_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(first), Err(second)) => Err(std::io::Error::other(format!( + "{first}; additionally failed to restore {}: {second}", + canonical_path.display() + ))), + } +} + +fn restore_output( + final_path: &Path, + backup_path: Option<&Path>, + promoted: bool, +) -> std::io::Result<()> { + if promoted { + remove_if_present(final_path)?; + } + if let Some(backup_path) = backup_path { + std::fs::hard_link(backup_path, final_path).map_err(|error| { + std::io::Error::other(format!( + "failed to restore {} from retained backup {}: {error}", + final_path.display(), + backup_path.display() + )) + })?; + std::fs::remove_file(backup_path)?; + } + Ok(()) +} + +fn remove_backups( + canonical_backup: Option<&Path>, + data_loop_backup: Option<&Path>, +) -> std::io::Result<()> { + if let Some(path) = canonical_backup { + remove_if_present(path)?; + } + if let Some(path) = data_loop_backup { + remove_if_present(path)?; + } + Ok(()) +} + +fn remove_if_present(path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn combine_error_with_cleanup( + error: std::io::Error, + cleanup: std::io::Result<()>, + cleanup_path: &Path, +) -> std::io::Error { + match cleanup { + Ok(()) => error, + Err(cleanup_error) => std::io::Error::other(format!( + "{error}; additionally failed to remove {}: {cleanup_error}", + cleanup_path.display() + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replace_rejects_non_file_output_before_promoting_sibling() { + let directory = tempfile::tempdir().unwrap(); + let canonical_path = directory.path().join("denials.json"); + let data_loop_path = directory.path().join("denials.data-loop.json"); + std::fs::create_dir(&canonical_path).unwrap(); + + let error = write_paired_output_files( + "test", + &canonical_path, + &data_loop_path, + ExistingOutputPolicy::Replace, + |writer| writer.write_all(b"canonical"), + |writer| writer.write_all(b"data-loop"), + ) + .unwrap_err(); + + assert!(error + .to_string() + .contains("cannot replace non-file canonical")); + assert!(!data_loop_path.exists()); + } + + #[test] + fn failed_second_promotion_restores_previous_output() { + let directory = tempfile::tempdir().unwrap(); + let shared_path = directory.path().join("denials.json"); + std::fs::write(&shared_path, b"previous").unwrap(); + + let error = write_paired_output_files( + "test", + &shared_path, + &shared_path, + ExistingOutputPolicy::Replace, + |writer| writer.write_all(b"canonical"), + |writer| writer.write_all(b"data-loop"), + ) + .unwrap_err(); + + assert!(error.to_string().contains("failed to promote canonical")); + assert_eq!(std::fs::read(shared_path).unwrap(), b"previous"); + } + + #[test] + fn rollback_does_not_delete_unpromoted_collision() { + let directory = tempfile::tempdir().unwrap(); + let final_path = directory.path().join("denials.json"); + let backup_path = directory.path().join("backup.json"); + std::fs::write(&final_path, b"concurrent").unwrap(); + std::fs::write(&backup_path, b"previous").unwrap(); + + let error = restore_output(&final_path, Some(&backup_path), false).unwrap_err(); + + assert!(error.to_string().contains("failed to restore")); + assert_eq!(std::fs::read(final_path).unwrap(), b"concurrent"); + assert_eq!(std::fs::read(backup_path).unwrap(), b"previous"); + } + + #[test] + fn output_pair_lock_rejects_concurrent_writer() { + let directory = tempfile::tempdir().unwrap(); + let canonical_path = directory.path().join("denials.json"); + let first = OutputPairLock::acquire("test", &canonical_path).unwrap(); + + let error = OutputPairLock::acquire("test", &canonical_path).unwrap_err(); + + assert!(error.to_string().contains("another process")); + drop(first); + OutputPairLock::acquire("test", &canonical_path).unwrap(); + } +} diff --git a/src/host/plm/src/analysis.rs b/src/host/plm/src/analysis.rs index c334b7228..4efe83803 100644 --- a/src/host/plm/src/analysis.rs +++ b/src/host/plm/src/analysis.rs @@ -4,13 +4,13 @@ //! Canonical Learning Mode analysis and compatibility views for `plm.exe`. use std::collections::{HashMap, HashSet}; -use std::io::Write; use std::path::Path; use anyhow::{Context, Result}; use learning_mode_core::{ - write_document, AccessType, AnalysisResult, DenialAnalyzer, DenialSummary, DenialsDocument, - DeniedResource, ResourceType, + data_loop_sibling_path, write_data_loop_document, write_document, write_paired_output_files, + AccessType, AnalysisResult, DataLoopDocument, DenialAnalyzer, DenialSummary, DenialsDocument, + DeniedResource, ExistingOutputPolicy, ResourceType, }; use learning_mode_windows::EtlDenialAnalyzer; @@ -24,7 +24,8 @@ pub fn analyze_trace(trace_file: &Path) -> Result { .with_context(|| format!("failed to analyze {}", trace_file.display())) } -/// Write canonical denials JSON atomically and return the document that was written. +/// Write canonical denials JSON and its Data Loop sibling, then return the +/// canonical document. pub fn write_denials( output_path: &Path, analysis: &AnalysisResult, @@ -40,16 +41,17 @@ pub fn write_denials( analysis.denied_resources_truncated, ); let document = DenialsDocument::new(analysis.denials.clone(), summary); - let mut temp = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("failed to create denials temp file in {}", parent.display()))?; - write_document(&mut temp, &document) - .with_context(|| format!("failed to write {}", output_path.display()))?; - temp.flush() - .and_then(|_| temp.as_file().sync_all()) - .with_context(|| format!("failed to flush {}", output_path.display()))?; - temp.persist(output_path) - .map_err(|error| error.error) - .with_context(|| format!("failed to replace {}", output_path.display()))?; + let data_loop_document = DataLoopDocument::new(&analysis.data_loop); + let data_loop_path = data_loop_sibling_path(output_path) + .with_context(|| format!("failed to derive sibling for {}", output_path.display()))?; + write_paired_output_files( + "plm stop", + output_path, + &data_loop_path, + ExistingOutputPolicy::Replace, + |writer| write_document(writer, &document), + |writer| write_data_loop_document(writer, &data_loop_document), + )?; Ok(document) } @@ -297,9 +299,33 @@ mod tests { write_denials(&path, &analysis, 7).unwrap(); let document: DenialsDocument = - serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let data_loop_path = learning_mode_core::data_loop_sibling_path(&path).unwrap(); + let data_loop_document: learning_mode_core::DataLoopDocument = + serde_json::from_slice(&std::fs::read(data_loop_path).unwrap()).unwrap(); assert_eq!(document.denials, analysis.denials); assert_eq!(document.summary.exit_code, 7); assert!(document.summary.denied_resources_truncated); + assert_eq!(data_loop_document.summary.total_occurrences, 0); + } + + #[test] + fn canonical_promotion_failure_removes_data_loop_sibling() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("denials.json"); + std::fs::create_dir(&path).unwrap(); + let data_loop_path = learning_mode_core::data_loop_sibling_path(&path).unwrap(); + let analysis = AnalysisResult { + denials: Vec::new(), + denied_resources_truncated: false, + data_loop: Default::default(), + }; + + let error = write_denials(&path, &analysis, 0).unwrap_err(); + + assert!(error + .to_string() + .contains("cannot replace non-file canonical")); + assert!(!data_loop_path.exists()); } } diff --git a/src/host/plm/src/stop.rs b/src/host/plm/src/stop.rs index ab4cfefd0..392aedbcf 100644 --- a/src/host/plm/src/stop.rs +++ b/src/host/plm/src/stop.rs @@ -89,9 +89,13 @@ fn prepare_config_output_paths( trace_path: &Path, denials_path: &Path, ) -> Result> { - if same_config_target(trace_path, denials_path) { + let data_loop_path = learning_mode_core::data_loop_sibling_path(denials_path) + .context("failed to derive Data Loop output path")?; + if same_config_target(trace_path, denials_path) + || same_config_target(trace_path, &data_loop_path) + { anyhow::bail!( - "trace output {} would be overwritten by denials output {}", + "trace output {} would be overwritten by a denials output derived from {}", trace_path.display(), denials_path.display() ); @@ -112,7 +116,10 @@ fn prepare_config_output_paths( ("config snapshot", snapshot.as_path()), ("adjusted config", adjusted.as_path()), ] { - if same_config_target(path, trace_path) || same_config_target(path, denials_path) { + if same_config_target(path, trace_path) + || same_config_target(path, denials_path) + || same_config_target(path, &data_loop_path) + { anyhow::bail!( "{label} path {} collides with a capture output", path.display() @@ -659,6 +666,32 @@ mod tests { assert!(error.to_string().contains("would be overwritten")); } + #[test] + fn trace_input_cannot_collide_with_data_loop_output() { + let trace = Path::new(r"C:\captures\denials.data-loop.json"); + let error = prepare_config_output_paths( + None, + Path::new(r"C:\captures"), + trace, + Path::new(r"C:\captures\denials.json"), + ) + .unwrap_err(); + assert!(error.to_string().contains("would be overwritten")); + } + + #[test] + fn config_snapshot_cannot_collide_with_data_loop_output() { + let source = Path::new(r"C:\source\denials.data-loop.json"); + let error = prepare_config_output_paths( + Some(source), + Path::new(r"C:\captures"), + Path::new(r"C:\captures\trace.etl"), + Path::new(r"C:\captures\denials.json"), + ) + .unwrap_err(); + assert!(error.to_string().contains("config snapshot")); + } + #[test] fn trailing_dot_trace_alias_cannot_collide_with_denials_output() { let dir = std::env::temp_dir().join(format!("plm_alias_target_{}", std::process::id())); From 809f8d50f7184bab377227af547a24ea2383a394 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 18:17:31 -0700 Subject: [PATCH 12/34] Harden guarded decoding and paired rollback Use targeted broker PID decoding without aborting on unrelated malformed payloads, and make rollback ownership-aware, portable, and no-clobber. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- src/Cargo.lock | 1 + src/Cargo.toml | 1 + .../learning_mode/windows/src/etl_decode.rs | 119 +++++--- .../learning_mode/windows/src/extractors.rs | 25 +- .../learning_mode/windows/src/tdh_decode.rs | 160 ++++++++--- src/core/learning_mode_core/Cargo.toml | 1 + .../learning_mode_core/src/paired_output.rs | 262 +++++++++++++----- 7 files changed, 425 insertions(+), 144 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 782535943..2df0bbcd5 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1273,6 +1273,7 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" name = "learning_mode_core" version = "0.7.0" dependencies = [ + "same-file", "serde", "serde_json", "tempfile", diff --git a/src/Cargo.toml b/src/Cargo.toml index 85f0e8bea..950502a08 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -90,6 +90,7 @@ plm = { path = "host/plm" } quick-xml = "0.41" sandbox_spec = { path = "core/generated/base_container_specification" } seatbelt_common = { path = "backends/seatbelt/common" } +same-file = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_path_to_error = "0.1" diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 950418e61..87898aac5 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -765,22 +765,16 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu if event_id == crate::extractors::CAPABILITY_DENIAL_EVENT_ID && is_learning_mode_event(provider, event_id) { - match unsafe { tdh_decode::decode_event_parts(event_record, &mut acc.schema_cache) } { - Ok(parts) => select_decoded_event_for_relogging( - acc, - event_index, - &parts, - header.ProcessId, - filetime, - ), - Err(error) => { - acc.decode_error = Some(format!( - "failed to decode brokered capability event while scoping guarded trace: {error}" - )); - acc.stop_requested = true; - return; - } - } + let process_id = unsafe { + tdh_decode::decode_event_property(event_record, &mut acc.schema_cache, "ProcessId") + }; + select_capability_decode_result_for_relogging( + acc, + event_index, + process_id, + header.ProcessId, + filetime, + ); } else { select_event_for_relogging(acc, event_index, header.ProcessId, filetime); } @@ -847,17 +841,45 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu } } -fn select_decoded_event_for_relogging( +fn select_capability_decode_result_for_relogging( acc: &mut Accumulator<'_>, event_index: usize, - parts: &DecodedEventParts, + process_id: Result, tdh_decode::DecodeError>, header_pid: u32, filetime: u64, ) { - let Some(effective_pid) = crate::extractors::effective_event_pid(parts, header_pid) else { - acc.decode_error = - Some("brokered capability event has no valid payload ProcessId".to_string()); - acc.stop_requested = true; + match process_id { + Ok(process_id) => select_capability_event_for_relogging( + acc, + event_index, + process_id.as_deref(), + header_pid, + filetime, + ), + Err(error) if error.is_schema_error() => { + acc.decode_error = Some(format!( + "failed to decode brokered capability event while scoping guarded trace: {error}" + )); + acc.stop_requested = true; + } + Err(_) => { + select_capability_event_for_relogging(acc, event_index, None, header_pid, filetime); + } + } +} + +fn select_capability_event_for_relogging( + acc: &mut Accumulator<'_>, + event_index: usize, + process_id: Option<&str>, + header_pid: u32, + filetime: u64, +) { + let Some(effective_pid) = crate::extractors::effective_capability_event_pid(process_id) else { + // The event cannot be attributed to a workload without its brokered + // payload PID. Retain it only when the emitter itself is in scope so + // the analysis pass can classify the malformed payload. + select_event_for_relogging(acc, event_index, header_pid, filetime); return; }; select_event_for_relogging(acc, event_index, effective_pid, filetime); @@ -1054,38 +1076,55 @@ mod tests { start_filetime: 100, end_filetime: 200, }]); - let parts = DecodedEventParts { - provider: crate::extractors::KERNEL_GENERAL_PROVIDER, - event_id: crate::extractors::CAPABILITY_DENIAL_EVENT_ID, - props: vec![("ProcessId".to_string(), "42".to_string())], - }; + select_capability_event_for_relogging(&mut accumulator, 0, Some("42"), 9000, 150); - select_decoded_event_for_relogging(&mut accumulator, 0, &parts, 9000, 150); + assert_eq!(accumulator.relog_selected_event_indices, [0]); + } + + #[test] + fn relog_selection_skips_unscopable_brokered_capability_without_payload_pid() { + let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + select_capability_event_for_relogging(&mut accumulator, 0, None, 9000, 150); + + assert!(accumulator.relog_selected_event_indices.is_empty()); + assert!(!accumulator.stop_requested); + assert!(accumulator.decode_error.is_none()); + } + + #[test] + fn relog_selection_retains_malformed_capability_from_in_scope_emitter() { + let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + select_capability_event_for_relogging(&mut accumulator, 0, None, 42, 150); assert_eq!(accumulator.relog_selected_event_indices, [0]); } #[test] - fn relog_selection_rejects_brokered_capability_without_payload_pid() { + fn relog_selection_skips_payload_decode_failure_from_unrelated_emitter() { let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { pid: 42, start_filetime: 100, end_filetime: 200, }]); - let parts = DecodedEventParts { - provider: crate::extractors::KERNEL_GENERAL_PROVIDER, - event_id: crate::extractors::CAPABILITY_DENIAL_EVENT_ID, - props: Vec::new(), - }; + let error = tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::PayloadMalformed, + "truncated ProcessId".to_string(), + None, + ); - select_decoded_event_for_relogging(&mut accumulator, 0, &parts, 42, 150); + select_capability_decode_result_for_relogging(&mut accumulator, 0, Err(error), 9000, 150); assert!(accumulator.relog_selected_event_indices.is_empty()); - assert!(accumulator.stop_requested); - assert!(accumulator - .decode_error - .as_deref() - .is_some_and(|error| error.contains("payload ProcessId"))); + assert!(!accumulator.stop_requested); + assert!(accumulator.decode_error.is_none()); } #[test] diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index 99e07c3d2..6f8e04352 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -209,12 +209,18 @@ pub(crate) fn is_learning_mode_event(provider: GUID, event_id: u16) -> bool { pub(crate) fn effective_event_pid(parts: &DecodedEventParts, header_pid: u32) -> Option { if parts.event_id == CAPABILITY_DENIAL_EVENT_ID { - find_prop(&parts.props, "ProcessId").and_then(|value| parse_u32(value)) + effective_capability_event_pid( + find_prop(&parts.props, "ProcessId").map(std::string::String::as_str), + ) } else { Some(header_pid) } } +pub(crate) fn effective_capability_event_pid(process_id: Option<&str>) -> Option { + process_id.and_then(parse_u32) +} + /// Maps a raw ETW provider GUID to its symbolic Data Loop category. /// /// Returns `None` for providers outside the Learning Mode vocabulary; those @@ -389,14 +395,15 @@ pub(crate) fn redact_username_in_path(value: &str) -> String { pub(crate) fn bound_properties(mut properties: Vec<(String, String)>) -> Vec<(String, String)> { properties.truncate(MAX_SIGNATURE_PROPERTIES); for (_, value) in &mut properties { - if value.chars().count() > MAX_SIGNATURE_VALUE_LEN { - *value = bound_property_value(value); + let char_count = value.chars().count(); + if char_count > MAX_SIGNATURE_VALUE_LEN { + *value = bound_property_value(value, char_count); } } properties } -fn bound_property_value(value: &str) -> String { +fn bound_property_value(value: &str, char_count: usize) -> String { let digest = Sha256::digest(value.as_bytes()); let marker = format!("......"); debug_assert_eq!(marker.len(), BOUNDED_VALUE_MARKER_LEN); @@ -406,11 +413,7 @@ fn bound_property_value(value: &str) -> String { let prefix = value.chars().take(prefix_len).collect::(); let suffix = value .chars() - .rev() - .take(suffix_len) - .collect::() - .chars() - .rev() + .skip(char_count - suffix_len) .collect::(); format!("{prefix}{marker}{suffix}") } @@ -1691,6 +1694,10 @@ mod tests { let bounded_short = bound_properties(vec![("ObjectName".to_string(), short.clone())]); assert_eq!(bounded_short[0].1, short); + let exact = "x".repeat(MAX_SIGNATURE_VALUE_LEN); + let bounded_exact = bound_properties(vec![("ObjectName".to_string(), exact.clone())]); + assert_eq!(bounded_exact[0].1, exact); + let long = format!("{}尾", "é".repeat(MAX_SIGNATURE_VALUE_LEN * 2)); let bounded_long = bound_properties(vec![("ObjectName".to_string(), long)]); assert_eq!(bounded_long[0].1.chars().count(), MAX_SIGNATURE_VALUE_LEN); diff --git a/src/backends/learning_mode/windows/src/tdh_decode.rs b/src/backends/learning_mode/windows/src/tdh_decode.rs index b7cfa48ff..eadc3be17 100644 --- a/src/backends/learning_mode/windows/src/tdh_decode.rs +++ b/src/backends/learning_mode/windows/src/tdh_decode.rs @@ -201,17 +201,64 @@ pub unsafe fn decode_event_parts( schema_cache: &mut EventSchemaCache, ) -> Result { let event = unsafe { &*event_record }; - let key = EventSchemaKey::from_record(event); let mut uncached_schema = None; + let buffer = unsafe { event_schema(event_record, event, schema_cache, &mut uncached_schema) }?; + let info = unsafe { &*buffer.as_ptr() }; + + let header = event.EventHeader; + let event_id = header.EventDescriptor.Id; + let pointer_size = pointer_size_from_header_flags(header.Flags); + let event_name = schema_event_name(buffer.as_bytes(), info); + let props = decode_properties(buffer.as_bytes(), info, event_record, pointer_size) + .map_err(|error| map_property_decode_error(error, event_name))?; + + Ok(DecodedEventParts { + provider: header.ProviderId, + event_id, + props, + }) +} + +/// Decodes one named property without decoding properties that follow it. +/// +/// # Safety +/// `event_record` must satisfy the same requirements as [`decode_event_parts`]. +pub unsafe fn decode_event_property( + event_record: *mut EVENT_RECORD, + schema_cache: &mut EventSchemaCache, + property_name: &str, +) -> Result, DecodeError> { + let event = unsafe { &*event_record }; + let mut uncached_schema = None; + let buffer = unsafe { event_schema(event_record, event, schema_cache, &mut uncached_schema) }?; + let info = unsafe { &*buffer.as_ptr() }; + let event_name = schema_event_name(buffer.as_bytes(), info); + decode_named_property( + buffer.as_bytes(), + info, + event_record, + pointer_size_from_header_flags(event.EventHeader.Flags), + property_name, + ) + .map_err(|error| map_property_decode_error(error, event_name)) +} + +unsafe fn event_schema<'a>( + event_record: *mut EVENT_RECORD, + event: &EVENT_RECORD, + schema_cache: &'a mut EventSchemaCache, + uncached_schema: &'a mut Option, +) -> Result<&'a TdhInfoBuffer, DecodeError> { + let key = EventSchemaKey::from_record(event); let cacheable = !unsafe { has_trace_logging_schema(event) }; if !cacheable { - uncached_schema = Some(unsafe { load_event_schema(event_record) }?); + *uncached_schema = Some(unsafe { load_event_schema(event_record) }?); } else if !schema_cache.schemas.contains_key(&key) { let schema = unsafe { load_event_schema(event_record) }?; if schema_cache.schemas.len() < MAX_SCHEMA_CACHE_ENTRIES { schema_cache.schemas.insert(key, schema); } else { - uncached_schema = Some(schema); + *uncached_schema = Some(schema); } } let buffer = if cacheable { @@ -224,36 +271,29 @@ pub unsafe fn decode_event_parts( .as_ref() .ok_or_else(|| DecodeError::Schema("uncached event schema missing".to_string()))? }; - let info = unsafe { &*buffer.as_ptr() }; - - let header = event.EventHeader; - let event_id = header.EventDescriptor.Id; - let pointer_size = pointer_size_from_header_flags(header.Flags); - let event_name = schema_event_name(buffer.as_bytes(), info); - let props = decode_properties(buffer.as_bytes(), info, event_record, pointer_size).map_err( - |error| match error.kind { - PropertyDecodeErrorKind::Schema => DecodeError::Schema(error.message), - PropertyDecodeErrorKind::PayloadMalformed => { - DecodeError::event(EventDecodeKind::PayloadMalformed, error.message, event_name) - } - PropertyDecodeErrorKind::DecoderLimitReached => DecodeError::event( - EventDecodeKind::DecoderLimitReached, - error.message, - event_name, - ), - PropertyDecodeErrorKind::UnsupportedPropertyEncoding => DecodeError::event( - EventDecodeKind::UnsupportedPropertyEncoding, - error.message, - event_name, - ), - }, - )?; + Ok(buffer) +} - Ok(DecodedEventParts { - provider: header.ProviderId, - event_id, - props, - }) +fn map_property_decode_error( + error: PropertyDecodeError, + event_name: Option, +) -> DecodeError { + match error.kind { + PropertyDecodeErrorKind::Schema => DecodeError::Schema(error.message), + PropertyDecodeErrorKind::PayloadMalformed => { + DecodeError::event(EventDecodeKind::PayloadMalformed, error.message, event_name) + } + PropertyDecodeErrorKind::DecoderLimitReached => DecodeError::event( + EventDecodeKind::DecoderLimitReached, + error.message, + event_name, + ), + PropertyDecodeErrorKind::UnsupportedPropertyEncoding => DecodeError::event( + EventDecodeKind::UnsupportedPropertyEncoding, + error.message, + event_name, + ), + } } fn schema_event_name(info_buf: &[u8], info: &TRACE_EVENT_INFO) -> Option { @@ -398,6 +438,62 @@ fn decode_properties( Ok(results) } +fn decode_named_property( + info_buf: &[u8], + info: &TRACE_EVENT_INFO, + event_record: *mut EVENT_RECORD, + pointer_size: usize, + property_name: &str, +) -> Result, PropertyDecodeError> { + // SAFETY: caller passes a valid EVENT_RECORD; the field accesses + // are reads of POD fields. + let event = unsafe { &*event_record }; + let user_data = event.UserData as *const u8; + let user_data_len = event.UserDataLength as usize; + if user_data.is_null() || user_data_len == 0 { + return Ok(None); + } + + let property_count = info.PropertyCount as usize; + let prop_count = info.TopLevelPropertyCount as usize; + if prop_count > property_count { + return Err(PropertyDecodeError::schema(format!( + "top-level property count {prop_count} exceeds property count {property_count}" + ))); + } + + let mut numeric_values = vec![None; property_count]; + let mut offset = 0; + let mut work_remaining = MAX_DECODE_WORK; + let mut decoded = Vec::new(); + let context = PropertyDecodeContext { + info_buf, + info, + user_data, + user_data_len, + pointer_size, + }; + for index in 0..prop_count { + decode_property( + index, + &context, + &mut offset, + &mut numeric_values, + 0, + &mut work_remaining, + &mut decoded, + )?; + if let Some((_, value)) = decoded + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(property_name)) + { + return Ok(Some(value.clone())); + } + decoded.clear(); + } + Ok(None) +} + struct PropertyDecodeContext<'a> { info_buf: &'a [u8], info: &'a TRACE_EVENT_INFO, diff --git a/src/core/learning_mode_core/Cargo.toml b/src/core/learning_mode_core/Cargo.toml index fdf96809b..48cc4f7ed 100644 --- a/src/core/learning_mode_core/Cargo.toml +++ b/src/core/learning_mode_core/Cargo.toml @@ -7,5 +7,6 @@ license.workspace = true [dependencies] serde = { workspace = true } serde_json = { workspace = true } +same-file = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index be3bb0b7c..cc363fecf 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -55,6 +55,18 @@ impl Drop for OutputPairLock { } } +#[derive(Debug)] +struct PromotedOutput { + handle: same_file::Handle, +} + +impl PromotedOutput { + fn matches_path(&self, path: &Path) -> std::io::Result { + let path_handle = same_file::Handle::from_path(path)?; + Ok(self.handle == path_handle) + } +} + /// Controls how an existing output pair is handled during promotion. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum ExistingOutputPolicy { @@ -105,7 +117,7 @@ pub fn write_paired_output_files( Err(error) => { return Err(combine_error_with_cleanup( error, - restore_output(canonical_path, canonical_backup.as_deref(), false), + restore_output(canonical_path, canonical_backup.as_deref(), None), canonical_path, )) } @@ -114,37 +126,44 @@ pub fn write_paired_output_files( None }; - if let Err(error) = promote_output_file(operation, data_loop_temp, data_loop_path, "Data Loop") - { - return Err(combine_error_with_cleanup( - error, - restore_output_pair( - canonical_path, - canonical_backup.as_deref(), - false, - data_loop_path, - data_loop_backup.as_deref(), - false, - ), - data_loop_path, - )); - } - if let Err(error) = promote_output_file(operation, canonical_temp, canonical_path, "canonical") - { - return Err(combine_error_with_cleanup( - error, - restore_output_pair( - canonical_path, - canonical_backup.as_deref(), - false, - data_loop_path, - data_loop_backup.as_deref(), - true, - ), - canonical_path, - )); - } + let data_loop_promoted = + match promote_output_file(operation, data_loop_temp, data_loop_path, "Data Loop") { + Ok(promoted) => promoted, + Err(error) => { + return Err(combine_error_with_cleanup( + error, + restore_output_pair( + canonical_path, + canonical_backup.as_deref(), + None, + data_loop_path, + data_loop_backup.as_deref(), + None, + ), + data_loop_path, + )) + } + }; + let canonical_promoted = + match promote_output_file(operation, canonical_temp, canonical_path, "canonical") { + Ok(promoted) => promoted, + Err(error) => { + return Err(combine_error_with_cleanup( + error, + restore_output_pair( + canonical_path, + canonical_backup.as_deref(), + None, + data_loop_path, + data_loop_backup.as_deref(), + Some(data_loop_promoted), + ), + canonical_path, + )) + } + }; + drop((canonical_promoted, data_loop_promoted)); remove_backups(canonical_backup.as_deref(), data_loop_backup.as_deref()) } @@ -217,9 +236,21 @@ fn promote_output_file( temp: tempfile::NamedTempFile, final_path: &Path, kind: &str, -) -> std::io::Result<()> { +) -> std::io::Result { + let identity_file = temp.reopen().map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to identify temporary {kind} output file for {}: {error}", + final_path.display() + )) + })?; + let handle = same_file::Handle::from_file(identity_file).map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to identify temporary {kind} output file for {}: {error}", + final_path.display() + )) + })?; temp.persist_noclobber(final_path) - .map(|_| ()) + .map(|_| PromotedOutput { handle }) .map_err(|error| { std::io::Error::other(format!( "{operation} failed to promote {kind} output file {}: {}", @@ -243,36 +274,41 @@ fn backup_existing_output( return Ok(None); } - let parent = final_path.parent().unwrap_or_else(|| Path::new(".")); - let backup = tempfile::NamedTempFile::new_in(parent).map_err(|error| { + let backup_path = vacant_sibling_path(operation, final_path, kind)?; + std::fs::rename(final_path, &backup_path).map_err(|error| { std::io::Error::other(format!( - "{operation} failed to reserve backup for {kind} output file {}: {error}", + "{operation} failed to back up {kind} output file {}: {error}", final_path.display() )) })?; - let backup_path = backup.path().to_path_buf(); - backup.close().map_err(|error| { + Ok(Some(backup_path)) +} + +fn vacant_sibling_path(operation: &str, final_path: &Path, kind: &str) -> std::io::Result { + let parent = final_path.parent().unwrap_or_else(|| Path::new(".")); + let backup = tempfile::NamedTempFile::new_in(parent).map_err(|error| { std::io::Error::other(format!( - "{operation} failed to prepare backup for {kind} output file {}: {error}", + "{operation} failed to reserve temporary path for {kind} output file {}: {error}", final_path.display() )) })?; - std::fs::rename(final_path, &backup_path).map_err(|error| { + let backup_path = backup.path().to_path_buf(); + backup.close().map_err(|error| { std::io::Error::other(format!( - "{operation} failed to back up {kind} output file {}: {error}", + "{operation} failed to prepare temporary path for {kind} output file {}: {error}", final_path.display() )) })?; - Ok(Some(backup_path)) + Ok(backup_path) } fn restore_output_pair( canonical_path: &Path, canonical_backup: Option<&Path>, - canonical_promoted: bool, + canonical_promoted: Option, data_loop_path: &Path, data_loop_backup: Option<&Path>, - data_loop_promoted: bool, + data_loop_promoted: Option, ) -> std::io::Result<()> { let data_loop_result = restore_output(data_loop_path, data_loop_backup, data_loop_promoted); let canonical_result = restore_output(canonical_path, canonical_backup, canonical_promoted); @@ -289,35 +325,91 @@ fn restore_output_pair( fn restore_output( final_path: &Path, backup_path: Option<&Path>, - promoted: bool, + promoted: Option, ) -> std::io::Result<()> { - if promoted { - remove_if_present(final_path)?; + if let Some(promoted) = promoted { + remove_promoted_output_if_owned(final_path, promoted, backup_path)?; + } else if backup_path.is_some() && final_path.try_exists()? { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "cannot restore {} because another writer created it; backup retained at {}", + final_path.display(), + backup_path.map_or_else(|| "".into(), |path| path.display().to_string()) + ), + )); } if let Some(backup_path) = backup_path { - std::fs::hard_link(backup_path, final_path).map_err(|error| { - std::io::Error::other(format!( - "failed to restore {} from retained backup {}: {error}", - final_path.display(), - backup_path.display() - )) - })?; - std::fs::remove_file(backup_path)?; + restore_backup_noclobber(backup_path, final_path)?; } Ok(()) } +fn remove_promoted_output_if_owned( + final_path: &Path, + promoted: PromotedOutput, + backup_path: Option<&Path>, +) -> std::io::Result<()> { + let quarantine_path = vacant_sibling_path("paired output rollback", final_path, "promoted")?; + std::fs::rename(final_path, &quarantine_path)?; + let matches = promoted.matches_path(&quarantine_path); + drop(promoted); + match matches { + Ok(true) => remove_if_present(&quarantine_path), + Ok(false) => { + let restore_result = persist_existing_file_noclobber(&quarantine_path, final_path); + if let Err(restore_error) = restore_result { + return Err(std::io::Error::other(format!( + "cannot restore {} because another writer replaced the promoted output; \ + unowned file retained at {} and backup retained at {}: {restore_error}", + final_path.display(), + quarantine_path.display(), + backup_path.map_or_else(|| "".into(), |path| path.display().to_string()) + ))); + } + Err(std::io::Error::other(format!( + "cannot restore {} because another writer replaced the promoted output; backup retained at {}", + final_path.display(), + backup_path.map_or_else(|| "".into(), |path| path.display().to_string()) + ))) + } + Err(error) => { + let _ = persist_existing_file_noclobber(&quarantine_path, final_path); + Err(error) + } + } +} + +fn restore_backup_noclobber(backup_path: &Path, final_path: &Path) -> std::io::Result<()> { + persist_existing_file_noclobber(backup_path, final_path) +} + +fn persist_existing_file_noclobber(source_path: &Path, final_path: &Path) -> std::io::Result<()> { + let file = std::fs::OpenOptions::new().read(true).open(source_path)?; + let temp_path = tempfile::TempPath::try_from_path(source_path.to_path_buf())?; + let mut source = tempfile::NamedTempFile::from_parts(file, temp_path); + // On a failed no-clobber promotion the original file is the recovery + // artifact, so it must outlive the temporary wrapper. + source.disable_cleanup(true); + source + .persist_noclobber(final_path) + .map(|_| ()) + .map_err(|error| error.error) +} + fn remove_backups( canonical_backup: Option<&Path>, data_loop_backup: Option<&Path>, ) -> std::io::Result<()> { - if let Some(path) = canonical_backup { - remove_if_present(path)?; - } - if let Some(path) = data_loop_backup { - remove_if_present(path)?; + let canonical_result = canonical_backup.map_or(Ok(()), remove_if_present); + let data_loop_result = data_loop_backup.map_or(Ok(()), remove_if_present); + match (canonical_result, data_loop_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(first), Err(second)) => Err(std::io::Error::other(format!( + "{first}; additionally failed to remove Data Loop backup: {second}" + ))), } - Ok(()) } fn remove_if_present(path: &Path) -> std::io::Result<()> { @@ -397,13 +489,57 @@ mod tests { std::fs::write(&final_path, b"concurrent").unwrap(); std::fs::write(&backup_path, b"previous").unwrap(); - let error = restore_output(&final_path, Some(&backup_path), false).unwrap_err(); + let error = restore_output(&final_path, Some(&backup_path), None).unwrap_err(); + + assert!(error.to_string().contains("cannot restore")); + assert_eq!(std::fs::read(final_path).unwrap(), b"concurrent"); + assert_eq!(std::fs::read(backup_path).unwrap(), b"previous"); + } + + #[test] + fn rollback_does_not_delete_replacement_of_promoted_output() { + let directory = tempfile::tempdir().unwrap(); + let final_path = directory.path().join("denials.json"); + let displaced_path = directory.path().join("displaced.json"); + let backup_path = directory.path().join("backup.json"); + std::fs::write(&final_path, b"promoted").unwrap(); + let promoted = PromotedOutput { + handle: same_file::Handle::from_file(std::fs::File::open(&final_path).unwrap()) + .unwrap(), + }; + std::fs::rename(&final_path, &displaced_path).unwrap(); + std::fs::write(&final_path, b"concurrent").unwrap(); + std::fs::write(&backup_path, b"previous").unwrap(); + + let error = restore_output(&final_path, Some(&backup_path), Some(promoted)).unwrap_err(); - assert!(error.to_string().contains("failed to restore")); + assert!(error.to_string().contains("another writer replaced")); assert_eq!(std::fs::read(final_path).unwrap(), b"concurrent"); assert_eq!(std::fs::read(backup_path).unwrap(), b"previous"); } + #[test] + #[allow(clippy::permissions_set_readonly_false)] + fn rollback_restores_read_only_backup() { + let directory = tempfile::tempdir().unwrap(); + let backup_path = directory.path().join("backup.json"); + let final_path = directory.path().join("denials.json"); + std::fs::write(&backup_path, b"previous").unwrap(); + let mut permissions = std::fs::metadata(&backup_path).unwrap().permissions(); + permissions.set_readonly(true); + std::fs::set_permissions(&backup_path, permissions).unwrap(); + + persist_existing_file_noclobber(&backup_path, &final_path).unwrap(); + + assert_eq!(std::fs::read(&final_path).unwrap(), b"previous"); + #[cfg(windows)] + { + let mut permissions = std::fs::metadata(&final_path).unwrap().permissions(); + permissions.set_readonly(false); + std::fs::set_permissions(final_path, permissions).unwrap(); + } + } + #[test] fn output_pair_lock_rejects_concurrent_writer() { let directory = tempfile::tempdir().unwrap(); From b17117240cb7b9dcf10d46a4a4367e6a5083c9a7 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 18:37:36 -0700 Subject: [PATCH 13/34] Close Learning Mode review gaps Preserve malformed scoped capability events, retain uncached TDH schemas after the cache cap, and make rollback recovery no-clobber and deletion tolerant. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- .../learning_mode/windows/src/etl_decode.rs | 55 +++++++- .../learning_mode/windows/src/tdh_decode.rs | 123 ++++++++++++++++-- .../learning_mode_core/src/paired_output.rs | 27 +++- 3 files changed, 191 insertions(+), 14 deletions(-) diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 87898aac5..ae918c701 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -932,7 +932,7 @@ fn handle_decoded_event( return; } let Some(pid) = crate::extractors::effective_event_pid(parts, header_pid) else { - if acc.process_lifetimes.is_none() && acc.begin_event() { + if acc.event_in_scope(header_pid, filetime) && acc.begin_event() { acc.record_outcome( category, parts.event_id, @@ -1107,6 +1107,33 @@ mod tests { assert_eq!(accumulator.relog_selected_event_indices, [0]); } + #[test] + fn scoped_analysis_classifies_malformed_capability_from_in_scope_emitter() { + let mut accumulator = Accumulator::analyze_for_process_lifetimes(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + let parts = DecodedEventParts { + provider: crate::extractors::KERNEL_GENERAL_PROVIDER, + event_id: crate::extractors::CAPABILITY_DENIAL_EVENT_ID, + props: Vec::new(), + }; + + handle_decoded_event(&parts, 42, 150, &mut accumulator); + let analysis = accumulator.into_analysis().unwrap(); + + assert_eq!( + find_signature( + &analysis.data_loop, + crate::extractors::CAPABILITY_DENIAL_EVENT_ID + ) + .signature + .reason, + DataLoopExclusionReason::EventPayloadMalformed + ); + } + #[test] fn relog_selection_skips_payload_decode_failure_from_unrelated_emitter() { let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { @@ -1127,6 +1154,32 @@ mod tests { assert!(accumulator.decode_error.is_none()); } + #[test] + fn relog_selection_stops_on_capability_schema_failure() { + let mut accumulator = Accumulator::select_for_relogging(&[ProcessLifetime { + pid: 42, + start_filetime: 100, + end_filetime: 200, + }]); + + select_capability_decode_result_for_relogging( + &mut accumulator, + 0, + Err(tdh_decode::DecodeError::Schema( + "manifest unavailable".to_string(), + )), + 9000, + 150, + ); + + assert!(accumulator.relog_selected_event_indices.is_empty()); + assert!(accumulator.stop_requested); + assert!(accumulator + .decode_error + .as_deref() + .is_some_and(|error| error.contains("manifest unavailable"))); + } + #[test] fn raw_visitor_panic_is_captured_inside_callback_state() { let mut visitor = diff --git a/src/backends/learning_mode/windows/src/tdh_decode.rs b/src/backends/learning_mode/windows/src/tdh_decode.rs index eadc3be17..140102bfd 100644 --- a/src/backends/learning_mode/windows/src/tdh_decode.rs +++ b/src/backends/learning_mode/windows/src/tdh_decode.rs @@ -261,17 +261,21 @@ unsafe fn event_schema<'a>( *uncached_schema = Some(schema); } } - let buffer = if cacheable { - schema_cache - .schemas - .get(&key) - .ok_or_else(|| DecodeError::Schema("event schema cache lookup failed".to_string()))? - } else { - uncached_schema - .as_ref() - .ok_or_else(|| DecodeError::Schema("uncached event schema missing".to_string()))? - }; - Ok(buffer) + schema_buffer(schema_cache, &key, uncached_schema) +} + +fn schema_buffer<'a>( + schema_cache: &'a EventSchemaCache, + key: &EventSchemaKey, + uncached_schema: &'a Option, +) -> Result<&'a TdhInfoBuffer, DecodeError> { + if let Some(buffer) = uncached_schema { + return Ok(buffer); + } + schema_cache + .schemas + .get(key) + .ok_or_else(|| DecodeError::Schema("event schema cache lookup failed".to_string())) } fn map_property_decode_error( @@ -991,6 +995,47 @@ mod tests { .collect() } + fn uint32_property_buffer(names: &[&str]) -> TdhInfoBuffer { + assert!(!names.is_empty()); + let encoded_names = names + .iter() + .map(|name| utf16_bytes(name)) + .collect::>(); + let metadata_len = std::mem::size_of::() + + (names.len() - 1) * std::mem::size_of::(); + let mut offsets = Vec::with_capacity(names.len()); + let mut next_offset = metadata_len; + for name in &encoded_names { + offsets.push(next_offset); + next_offset += name.len(); + } + + let mut buffer = TdhInfoBuffer::new(next_offset); + for (name, offset) in encoded_names.iter().zip(&offsets) { + buffer.as_bytes_mut()[*offset..*offset + name.len()].copy_from_slice(name); + } + let info = unsafe { &mut *buffer.as_mut_ptr() }; + info.PropertyCount = names.len() as u32; + info.TopLevelPropertyCount = names.len() as u32; + let properties = + std::ptr::addr_of_mut!(info.EventPropertyInfoArray) as *mut EVENT_PROPERTY_INFO; + for (index, offset) in offsets.into_iter().enumerate() { + let property = unsafe { &mut *properties.add(index) }; + property.NameOffset = offset as u32; + property.Anonymous1.nonStructType.InType = TDH_INTYPE_UINT32; + property.Anonymous2.count = 1; + property.Anonymous3.length = 4; + } + buffer + } + + fn event_record_for_payload(payload: &[u8]) -> EVENT_RECORD { + let mut record: EVENT_RECORD = unsafe { core::mem::zeroed() }; + record.UserData = payload.as_ptr().cast_mut().cast(); + record.UserDataLength = payload.len() as u16; + record + } + #[test] fn tdh_info_buffer_is_aligned_and_initialized() { let mut buffer = TdhInfoBuffer::new(std::mem::size_of::() + 7); @@ -1004,6 +1049,18 @@ mod tests { assert!(buffer.as_bytes().iter().all(|byte| *byte == 0)); } + #[test] + fn uncached_schema_is_used_when_cache_does_not_contain_key() { + let cache = EventSchemaCache::default(); + let record: EVENT_RECORD = unsafe { core::mem::zeroed() }; + let key = EventSchemaKey::from_record(&record); + let uncached = Some(TdhInfoBuffer::new(std::mem::size_of::())); + + let selected = schema_buffer(&cache, &key, &uncached).unwrap(); + + assert!(std::ptr::eq(selected, uncached.as_ref().unwrap())); + } + #[test] fn decode_property_surfaces_struct_children() { let struct_name = utf16_bytes("AccessCheck"); @@ -1069,6 +1126,50 @@ mod tests { assert_eq!(offset, payload.len()); } + #[test] + fn decode_named_property_stops_after_requested_property() { + let buffer = uint32_property_buffer(&["Count", "ProcessId", "Trailing"]); + let payload = [7u32, 42, 99] + .into_iter() + .flat_map(u32::to_le_bytes) + .collect::>(); + let mut record = event_record_for_payload(&payload); + let info = unsafe { &*buffer.as_ptr() }; + + let process_id = + decode_named_property(buffer.as_bytes(), info, &mut record, 8, "processid").unwrap(); + + assert_eq!(process_id.as_deref(), Some("42")); + } + + #[test] + fn decode_named_property_reports_missing_and_truncated_properties() { + let buffer = uint32_property_buffer(&["Count", "ProcessId"]); + let payload = [7u32, 42] + .into_iter() + .flat_map(u32::to_le_bytes) + .collect::>(); + let mut record = event_record_for_payload(&payload); + let info = unsafe { &*buffer.as_ptr() }; + + assert_eq!( + decode_named_property(buffer.as_bytes(), info, &mut record, 8, "Missing").unwrap(), + None + ); + + let truncated_payload = 7u32.to_le_bytes(); + let mut truncated_record = event_record_for_payload(&truncated_payload); + let error = decode_named_property( + buffer.as_bytes(), + info, + &mut truncated_record, + 8, + "ProcessId", + ) + .unwrap_err(); + assert_eq!(error.kind, PropertyDecodeErrorKind::PayloadMalformed); + } + fn decode_single_property( buffer: &TdhInfoBuffer, payload: &[u8], diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index cc363fecf..b8213d75c 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -275,7 +275,7 @@ fn backup_existing_output( } let backup_path = vacant_sibling_path(operation, final_path, kind)?; - std::fs::rename(final_path, &backup_path).map_err(|error| { + persist_existing_file_noclobber(final_path, &backup_path).map_err(|error| { std::io::Error::other(format!( "{operation} failed to back up {kind} output file {}: {error}", final_path.display() @@ -351,7 +351,11 @@ fn remove_promoted_output_if_owned( backup_path: Option<&Path>, ) -> std::io::Result<()> { let quarantine_path = vacant_sibling_path("paired output rollback", final_path, "promoted")?; - std::fs::rename(final_path, &quarantine_path)?; + match persist_existing_file_noclobber(final_path, &quarantine_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } let matches = promoted.matches_path(&quarantine_path); drop(promoted); match matches { @@ -518,6 +522,25 @@ mod tests { assert_eq!(std::fs::read(backup_path).unwrap(), b"previous"); } + #[test] + fn rollback_restores_backup_when_promoted_output_was_deleted() { + let directory = tempfile::tempdir().unwrap(); + let final_path = directory.path().join("denials.json"); + let backup_path = directory.path().join("backup.json"); + std::fs::write(&final_path, b"promoted").unwrap(); + let promoted = PromotedOutput { + handle: same_file::Handle::from_file(std::fs::File::open(&final_path).unwrap()) + .unwrap(), + }; + std::fs::remove_file(&final_path).unwrap(); + std::fs::write(&backup_path, b"previous").unwrap(); + + restore_output(&final_path, Some(&backup_path), Some(promoted)).unwrap(); + + assert_eq!(std::fs::read(final_path).unwrap(), b"previous"); + assert!(!backup_path.exists()); + } + #[test] #[allow(clippy::permissions_set_readonly_false)] fn rollback_restores_read_only_backup() { From 732f1eac65784f3b5518fd60d6f7179f44a5a7b6 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 18:52:16 -0700 Subject: [PATCH 14/34] Verify promoted output contents during rollback Detect in-place overwrites before deleting a promoted output and directly exercise TDH short-circuit and cache-capacity behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- src/Cargo.lock | 1 + .../learning_mode/windows/src/tdh_decode.rs | 54 +++++++++++---- src/core/learning_mode_core/Cargo.toml | 1 + .../learning_mode_core/src/paired_output.rs | 65 +++++++++++++++---- 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 2df0bbcd5..944216ad2 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1276,6 +1276,7 @@ dependencies = [ "same-file", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "thiserror", ] diff --git a/src/backends/learning_mode/windows/src/tdh_decode.rs b/src/backends/learning_mode/windows/src/tdh_decode.rs index 140102bfd..68d46bfb7 100644 --- a/src/backends/learning_mode/windows/src/tdh_decode.rs +++ b/src/backends/learning_mode/windows/src/tdh_decode.rs @@ -255,15 +255,24 @@ unsafe fn event_schema<'a>( *uncached_schema = Some(unsafe { load_event_schema(event_record) }?); } else if !schema_cache.schemas.contains_key(&key) { let schema = unsafe { load_event_schema(event_record) }?; - if schema_cache.schemas.len() < MAX_SCHEMA_CACHE_ENTRIES { - schema_cache.schemas.insert(key, schema); - } else { - *uncached_schema = Some(schema); - } + cache_or_retain_schema(schema_cache, key, schema, uncached_schema); } schema_buffer(schema_cache, &key, uncached_schema) } +fn cache_or_retain_schema( + schema_cache: &mut EventSchemaCache, + key: EventSchemaKey, + schema: TdhInfoBuffer, + uncached_schema: &mut Option, +) { + if schema_cache.schemas.len() < MAX_SCHEMA_CACHE_ENTRIES { + schema_cache.schemas.insert(key, schema); + } else { + *uncached_schema = Some(schema); + } +} + fn schema_buffer<'a>( schema_cache: &'a EventSchemaCache, key: &EventSchemaKey, @@ -1050,15 +1059,34 @@ mod tests { } #[test] - fn uncached_schema_is_used_when_cache_does_not_contain_key() { - let cache = EventSchemaCache::default(); - let record: EVENT_RECORD = unsafe { core::mem::zeroed() }; - let key = EventSchemaKey::from_record(&record); - let uncached = Some(TdhInfoBuffer::new(std::mem::size_of::())); + fn schema_is_retained_uncached_at_cache_capacity() { + let mut cache = EventSchemaCache::default(); + for id in 0..MAX_SCHEMA_CACHE_ENTRIES { + let mut record: EVENT_RECORD = unsafe { core::mem::zeroed() }; + record.EventHeader.EventDescriptor.Id = id as u16; + let mut uncached = None; + cache_or_retain_schema( + &mut cache, + EventSchemaKey::from_record(&record), + TdhInfoBuffer::new(1), + &mut uncached, + ); + assert!(uncached.is_none()); + } - let selected = schema_buffer(&cache, &key, &uncached).unwrap(); + let mut overflow_record: EVENT_RECORD = unsafe { core::mem::zeroed() }; + overflow_record.EventHeader.EventDescriptor.Id = MAX_SCHEMA_CACHE_ENTRIES as u16; + let overflow_key = EventSchemaKey::from_record(&overflow_record); + let mut uncached = None; + cache_or_retain_schema( + &mut cache, + overflow_key, + TdhInfoBuffer::new(std::mem::size_of::()), + &mut uncached, + ); - assert!(std::ptr::eq(selected, uncached.as_ref().unwrap())); + assert_eq!(cache.schemas.len(), MAX_SCHEMA_CACHE_ENTRIES); + assert!(schema_buffer(&cache, &overflow_key, &uncached).is_ok()); } #[test] @@ -1129,7 +1157,7 @@ mod tests { #[test] fn decode_named_property_stops_after_requested_property() { let buffer = uint32_property_buffer(&["Count", "ProcessId", "Trailing"]); - let payload = [7u32, 42, 99] + let payload = [7u32, 42] .into_iter() .flat_map(u32::to_le_bytes) .collect::>(); diff --git a/src/core/learning_mode_core/Cargo.toml b/src/core/learning_mode_core/Cargo.toml index 48cc4f7ed..5ec4c520c 100644 --- a/src/core/learning_mode_core/Cargo.toml +++ b/src/core/learning_mode_core/Cargo.toml @@ -8,5 +8,6 @@ license.workspace = true serde = { workspace = true } serde_json = { workspace = true } same-file = { workspace = true } +sha2 = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index b8213d75c..4d8b1ba04 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -3,7 +3,8 @@ //! Transactional staging for the canonical and Data Loop output pair. -use std::io::Write; +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; #[derive(Debug)] @@ -58,15 +59,39 @@ impl Drop for OutputPairLock { #[derive(Debug)] struct PromotedOutput { handle: same_file::Handle, + digest: [u8; 32], } impl PromotedOutput { + fn from_file(mut file: std::fs::File) -> std::io::Result { + let handle = same_file::Handle::from_file(file.try_clone()?)?; + let digest = file_digest(&mut file)?; + Ok(Self { handle, digest }) + } + fn matches_path(&self, path: &Path) -> std::io::Result { - let path_handle = same_file::Handle::from_path(path)?; - Ok(self.handle == path_handle) + let mut file = std::fs::File::open(path)?; + let path_handle = same_file::Handle::from_file(file.try_clone()?)?; + if self.handle != path_handle { + return Ok(false); + } + Ok(self.digest == file_digest(&mut file)?) } } +fn file_digest(reader: &mut impl Read) -> std::io::Result<[u8; 32]> { + let mut digest = Sha256::new(); + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + Ok(digest.finalize().into()) +} + /// Controls how an existing output pair is handled during promotion. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum ExistingOutputPolicy { @@ -243,14 +268,14 @@ fn promote_output_file( final_path.display() )) })?; - let handle = same_file::Handle::from_file(identity_file).map_err(|error| { + let promoted = PromotedOutput::from_file(identity_file).map_err(|error| { std::io::Error::other(format!( "{operation} failed to identify temporary {kind} output file for {}: {error}", final_path.display() )) })?; temp.persist_noclobber(final_path) - .map(|_| PromotedOutput { handle }) + .map(|_| promoted) .map_err(|error| { std::io::Error::other(format!( "{operation} failed to promote {kind} output file {}: {}", @@ -507,10 +532,8 @@ mod tests { let displaced_path = directory.path().join("displaced.json"); let backup_path = directory.path().join("backup.json"); std::fs::write(&final_path, b"promoted").unwrap(); - let promoted = PromotedOutput { - handle: same_file::Handle::from_file(std::fs::File::open(&final_path).unwrap()) - .unwrap(), - }; + let promoted = + PromotedOutput::from_file(std::fs::File::open(&final_path).unwrap()).unwrap(); std::fs::rename(&final_path, &displaced_path).unwrap(); std::fs::write(&final_path, b"concurrent").unwrap(); std::fs::write(&backup_path, b"previous").unwrap(); @@ -522,16 +545,32 @@ mod tests { assert_eq!(std::fs::read(backup_path).unwrap(), b"previous"); } + #[test] + fn rollback_does_not_delete_in_place_overwrite_of_promoted_output() { + let directory = tempfile::tempdir().unwrap(); + let final_path = directory.path().join("denials.json"); + let backup_path = directory.path().join("backup.json"); + std::fs::write(&final_path, b"promoted").unwrap(); + let promoted = + PromotedOutput::from_file(std::fs::File::open(&final_path).unwrap()).unwrap(); + std::fs::write(&final_path, b"replaced").unwrap(); + std::fs::write(&backup_path, b"previous").unwrap(); + + let error = restore_output(&final_path, Some(&backup_path), Some(promoted)).unwrap_err(); + + assert!(error.to_string().contains("another writer replaced")); + assert_eq!(std::fs::read(final_path).unwrap(), b"replaced"); + assert_eq!(std::fs::read(backup_path).unwrap(), b"previous"); + } + #[test] fn rollback_restores_backup_when_promoted_output_was_deleted() { let directory = tempfile::tempdir().unwrap(); let final_path = directory.path().join("denials.json"); let backup_path = directory.path().join("backup.json"); std::fs::write(&final_path, b"promoted").unwrap(); - let promoted = PromotedOutput { - handle: same_file::Handle::from_file(std::fs::File::open(&final_path).unwrap()) - .unwrap(), - }; + let promoted = + PromotedOutput::from_file(std::fs::File::open(&final_path).unwrap()).unwrap(); std::fs::remove_file(&final_path).unwrap(); std::fs::write(&backup_path, b"previous").unwrap(); From 1bc642b121a23906860ca926174c2f618f9f4d80 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 18:59:46 -0700 Subject: [PATCH 15/34] Complete paired output rollback after cleanup errors Restore backups even when quarantined output cleanup fails, preserve both errors, and hash promoted files from a known offset. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- .../learning_mode_core/src/paired_output.rs | 58 +++++++++++++++---- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index 4d8b1ba04..e45957497 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -4,7 +4,7 @@ //! Transactional staging for the canonical and Data Loop output pair. use sha2::{Digest, Sha256}; -use std::io::{Read, Write}; +use std::io::{Read, Seek, Write}; use std::path::{Path, PathBuf}; #[derive(Debug)] @@ -79,7 +79,8 @@ impl PromotedOutput { } } -fn file_digest(reader: &mut impl Read) -> std::io::Result<[u8; 32]> { +fn file_digest(reader: &mut (impl Read + Seek)) -> std::io::Result<[u8; 32]> { + reader.rewind()?; let mut digest = Sha256::new(); let mut buffer = [0u8; 16 * 1024]; loop { @@ -352,8 +353,8 @@ fn restore_output( backup_path: Option<&Path>, promoted: Option, ) -> std::io::Result<()> { - if let Some(promoted) = promoted { - remove_promoted_output_if_owned(final_path, promoted, backup_path)?; + let cleanup_error = if let Some(promoted) = promoted { + remove_promoted_output_if_owned(final_path, promoted, backup_path)? } else if backup_path.is_some() && final_path.try_exists()? { return Err(std::io::Error::new( std::io::ErrorKind::AlreadyExists, @@ -363,28 +364,30 @@ fn restore_output( backup_path.map_or_else(|| "".into(), |path| path.display().to_string()) ), )); - } - if let Some(backup_path) = backup_path { - restore_backup_noclobber(backup_path, final_path)?; - } - Ok(()) + } else { + None + }; + let restore_result = backup_path.map_or(Ok(()), |backup_path| { + restore_backup_noclobber(backup_path, final_path) + }); + finish_restore(final_path, cleanup_error, restore_result) } fn remove_promoted_output_if_owned( final_path: &Path, promoted: PromotedOutput, backup_path: Option<&Path>, -) -> std::io::Result<()> { +) -> std::io::Result> { let quarantine_path = vacant_sibling_path("paired output rollback", final_path, "promoted")?; match persist_existing_file_noclobber(final_path, &quarantine_path) { Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(error), } let matches = promoted.matches_path(&quarantine_path); drop(promoted); match matches { - Ok(true) => remove_if_present(&quarantine_path), + Ok(true) => Ok(remove_if_present(&quarantine_path).err()), Ok(false) => { let restore_result = persist_existing_file_noclobber(&quarantine_path, final_path); if let Err(restore_error) = restore_result { @@ -409,6 +412,21 @@ fn remove_promoted_output_if_owned( } } +fn finish_restore( + final_path: &Path, + cleanup_error: Option, + restore_result: std::io::Result<()>, +) -> std::io::Result<()> { + match (cleanup_error, restore_result) { + (None, Ok(())) => Ok(()), + (None, Err(error)) | (Some(error), Ok(())) => Err(error), + (Some(cleanup_error), Err(restore_error)) => Err(std::io::Error::other(format!( + "{cleanup_error}; additionally failed to restore {}: {restore_error}", + final_path.display() + ))), + } +} + fn restore_backup_noclobber(backup_path: &Path, final_path: &Path) -> std::io::Result<()> { persist_existing_file_noclobber(backup_path, final_path) } @@ -580,6 +598,22 @@ mod tests { assert!(!backup_path.exists()); } + #[test] + fn rollback_reports_cleanup_failure_after_attempting_restore() { + let cleanup_error = std::io::Error::other("quarantine cleanup failed"); + let restore_error = std::io::Error::other("backup restore failed"); + + let error = finish_restore( + Path::new("denials.json"), + Some(cleanup_error), + Err(restore_error), + ) + .unwrap_err(); + + assert!(error.to_string().contains("quarantine cleanup failed")); + assert!(error.to_string().contains("backup restore failed")); + } + #[test] #[allow(clippy::permissions_set_readonly_false)] fn rollback_restores_read_only_backup() { From 5ab71790bcd3e1c10a2cfcdaa9ca3fba505e2969 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 19:06:16 -0700 Subject: [PATCH 16/34] Preserve workload PID on partial capability decode Recover the brokered payload ProcessId when full event decoding fails after that property, and drop only events that remain unscopable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- .../learning_mode/windows/src/etl_decode.rs | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index ae918c701..38f5efd38 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -826,13 +826,37 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu }, Err(error) => { if matches!(acc.mode, CollectionMode::Analyze) { - if !acc.event_in_scope( - header.ProcessId, - analyze_filetime.expect("analyze mode has normalized FILETIME"), - ) || !acc.begin_event() - { + if error.is_schema_error() { + acc.record_event_decode_error(provider, event_id, header.ProcessId, error); return; } + let filetime = analyze_filetime.expect("analyze mode has normalized FILETIME"); + let pid = if event_id == crate::extractors::CAPABILITY_DENIAL_EVENT_ID { + let process_id = unsafe { + tdh_decode::decode_event_property( + event_record, + &mut acc.schema_cache, + "ProcessId", + ) + } + .ok() + .flatten(); + decode_error_effective_pid( + process_id.as_deref(), + header.ProcessId, + acc.event_in_scope(header.ProcessId, filetime), + ) + } else { + Some(header.ProcessId) + }; + let Some(pid) = pid else { + return; + }; + if !acc.event_in_scope(pid, filetime) || !acc.begin_event() { + return; + } + acc.record_event_decode_error(provider, event_id, pid, error); + return; } else if !acc.begin_event() { return; } @@ -841,6 +865,15 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu } } +fn decode_error_effective_pid( + process_id: Option<&str>, + header_pid: u32, + header_in_scope: bool, +) -> Option { + crate::extractors::effective_capability_event_pid(process_id) + .or_else(|| header_in_scope.then_some(header_pid)) +} + fn select_capability_decode_result_for_relogging( acc: &mut Accumulator<'_>, event_index: usize, @@ -1180,6 +1213,16 @@ mod tests { .is_some_and(|error| error.contains("manifest unavailable"))); } + #[test] + fn capability_decode_error_prefers_payload_pid_and_fails_closed_without_scope() { + assert_eq!( + decode_error_effective_pid(Some("42"), 9000, false), + Some(42) + ); + assert_eq!(decode_error_effective_pid(None, 42, true), Some(42)); + assert_eq!(decode_error_effective_pid(None, 9000, false), None); + } + #[test] fn raw_visitor_panic_is_captured_inside_callback_state() { let mut visitor = From a261e4253dad72982ef1bf909c9d93ef354bfaa5 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 19:19:20 -0700 Subject: [PATCH 17/34] Preserve rollback diagnostics and PID fallback coverage Surface quarantine restoration failures with accurate rollback context and codify malformed capability PID fallback scoping. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- .../learning_mode/windows/src/etl_decode.rs | 8 ++ .../learning_mode_core/src/paired_output.rs | 76 ++++++++++++++----- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 38f5efd38..258f2066b 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -1219,6 +1219,14 @@ mod tests { decode_error_effective_pid(Some("42"), 9000, false), Some(42) ); + assert_eq!( + decode_error_effective_pid(Some("\"broker\""), 42, true), + Some(42) + ); + assert_eq!( + decode_error_effective_pid(Some("\"broker\""), 9000, false), + None + ); assert_eq!(decode_error_effective_pid(None, 42, true), Some(42)); assert_eq!(decode_error_effective_pid(None, 9000, false), None); } diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index e45957497..c8d8655b9 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -141,10 +141,9 @@ pub fn write_paired_output_files( match backup_existing_output(operation, data_loop_path, "Data Loop") { Ok(backup) => backup, Err(error) => { - return Err(combine_error_with_cleanup( + return Err(combine_error_with_rollback( error, restore_output(canonical_path, canonical_backup.as_deref(), None), - canonical_path, )) } } @@ -156,7 +155,7 @@ pub fn write_paired_output_files( match promote_output_file(operation, data_loop_temp, data_loop_path, "Data Loop") { Ok(promoted) => promoted, Err(error) => { - return Err(combine_error_with_cleanup( + return Err(combine_error_with_rollback( error, restore_output_pair( canonical_path, @@ -166,7 +165,6 @@ pub fn write_paired_output_files( data_loop_backup.as_deref(), None, ), - data_loop_path, )) } }; @@ -174,7 +172,7 @@ pub fn write_paired_output_files( match promote_output_file(operation, canonical_temp, canonical_path, "canonical") { Ok(promoted) => promoted, Err(error) => { - return Err(combine_error_with_cleanup( + return Err(combine_error_with_rollback( error, restore_output_pair( canonical_path, @@ -184,7 +182,6 @@ pub fn write_paired_output_files( data_loop_backup.as_deref(), Some(data_loop_promoted), ), - canonical_path, )) } }; @@ -405,10 +402,28 @@ fn remove_promoted_output_if_owned( backup_path.map_or_else(|| "".into(), |path| path.display().to_string()) ))) } - Err(error) => { - let _ = persist_existing_file_noclobber(&quarantine_path, final_path); - Err(error) - } + Err(error) => combine_error_with_restore( + error, + persist_existing_file_noclobber(&quarantine_path, final_path), + &quarantine_path, + final_path, + ), + } +} + +fn combine_error_with_restore( + error: std::io::Error, + restore: std::io::Result<()>, + source_path: &Path, + final_path: &Path, +) -> std::io::Result> { + match restore { + Ok(()) => Err(error), + Err(restore_error) => Err(std::io::Error::other(format!( + "{error}; additionally failed to restore {} to {}: {restore_error}", + source_path.display(), + final_path.display() + ))), } } @@ -467,16 +482,14 @@ fn remove_if_present(path: &Path) -> std::io::Result<()> { } } -fn combine_error_with_cleanup( +fn combine_error_with_rollback( error: std::io::Error, - cleanup: std::io::Result<()>, - cleanup_path: &Path, + rollback: std::io::Result<()>, ) -> std::io::Error { - match cleanup { + match rollback { Ok(()) => error, - Err(cleanup_error) => std::io::Error::other(format!( - "{error}; additionally failed to remove {}: {cleanup_error}", - cleanup_path.display() + Err(rollback_error) => std::io::Error::other(format!( + "{error}; additionally failed to roll back output pair: {rollback_error}" )), } } @@ -614,6 +627,35 @@ mod tests { assert!(error.to_string().contains("backup restore failed")); } + #[test] + fn rollback_reports_failed_quarantine_restore() { + let error = combine_error_with_restore( + std::io::Error::other("identity check failed"), + Err(std::io::Error::other("restore failed")), + Path::new("quarantine.tmp"), + Path::new("denials.json"), + ) + .unwrap_err(); + + assert!(error.to_string().contains("identity check failed")); + assert!(error.to_string().contains("quarantine.tmp")); + assert!(error.to_string().contains("restore failed")); + } + + #[test] + fn promotion_error_describes_rollback_failure() { + let error = combine_error_with_rollback( + std::io::Error::other("promotion failed"), + Err(std::io::Error::other("restore failed")), + ); + + assert!(error.to_string().contains("promotion failed")); + assert!(error + .to_string() + .contains("failed to roll back output pair")); + assert!(error.to_string().contains("restore failed")); + } + #[test] #[allow(clippy::permissions_set_readonly_false)] fn rollback_restores_read_only_backup() { From 33832ca0db56a60fd5ee71f4b0849962724efb54 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Mon, 17 Aug 2026 19:29:35 -0700 Subject: [PATCH 18/34] Add path context to paired rollback errors Identify backup, quarantine, and destination paths in cleanup and restoration failures while preserving strict error propagation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: acb632fc-7b75-4d0e-9be5-f11de51652bf --- .../learning_mode_core/src/paired_output.rs | 79 +++++++++++++++++-- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index c8d8655b9..3bfeb653e 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -379,12 +379,28 @@ fn remove_promoted_output_if_owned( match persist_existing_file_noclobber(final_path, &quarantine_path) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error), + Err(error) => { + return Err(std::io::Error::other(format!( + "failed to quarantine promoted output {} at {}: {error}", + final_path.display(), + quarantine_path.display() + ))) + } } - let matches = promoted.matches_path(&quarantine_path); + let matches = promoted.matches_path(&quarantine_path).map_err(|error| { + std::io::Error::other(format!( + "failed to verify ownership of quarantined output {}: {error}", + quarantine_path.display() + )) + }); drop(promoted); match matches { - Ok(true) => Ok(remove_if_present(&quarantine_path).err()), + Ok(true) => Ok(remove_if_present(&quarantine_path).err().map(|error| { + std::io::Error::other(format!( + "failed to remove quarantined promoted output {}: {error}", + quarantine_path.display() + )) + })), Ok(false) => { let restore_result = persist_existing_file_noclobber(&quarantine_path, final_path); if let Err(restore_error) = restore_result { @@ -443,7 +459,13 @@ fn finish_restore( } fn restore_backup_noclobber(backup_path: &Path, final_path: &Path) -> std::io::Result<()> { - persist_existing_file_noclobber(backup_path, final_path) + persist_existing_file_noclobber(backup_path, final_path).map_err(|error| { + std::io::Error::other(format!( + "failed to restore backup {} to {}: {error}", + backup_path.display(), + final_path.display() + )) + }) } fn persist_existing_file_noclobber(source_path: &Path, final_path: &Path) -> std::io::Result<()> { @@ -463,8 +485,22 @@ fn remove_backups( canonical_backup: Option<&Path>, data_loop_backup: Option<&Path>, ) -> std::io::Result<()> { - let canonical_result = canonical_backup.map_or(Ok(()), remove_if_present); - let data_loop_result = data_loop_backup.map_or(Ok(()), remove_if_present); + let canonical_result = canonical_backup.map_or(Ok(()), |path| { + remove_if_present(path).map_err(|error| { + std::io::Error::other(format!( + "failed to remove canonical backup {}: {error}", + path.display() + )) + }) + }); + let data_loop_result = data_loop_backup.map_or(Ok(()), |path| { + remove_if_present(path).map_err(|error| { + std::io::Error::other(format!( + "failed to remove Data Loop backup {}: {error}", + path.display() + )) + }) + }); match (canonical_result, data_loop_result) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), @@ -656,6 +692,37 @@ mod tests { assert!(error.to_string().contains("restore failed")); } + #[test] + fn backup_restore_error_names_source_and_destination() { + let directory = tempfile::tempdir().unwrap(); + let backup_path = directory.path().join("backup.json"); + let final_path = directory.path().join("denials.json"); + std::fs::write(&backup_path, b"previous").unwrap(); + std::fs::write(&final_path, b"concurrent").unwrap(); + + let error = restore_backup_noclobber(&backup_path, &final_path).unwrap_err(); + + assert!(error + .to_string() + .contains(&backup_path.display().to_string())); + assert!(error + .to_string() + .contains(&final_path.display().to_string())); + } + + #[test] + fn backup_cleanup_error_names_the_failed_path() { + let directory = tempfile::tempdir().unwrap(); + let canonical_backup = directory.path().join("canonical-backup"); + std::fs::create_dir(&canonical_backup).unwrap(); + + let error = remove_backups(Some(&canonical_backup), None).unwrap_err(); + + assert!(error + .to_string() + .contains(&canonical_backup.display().to_string())); + } + #[test] #[allow(clippy::permissions_set_readonly_false)] fn rollback_restores_read_only_backup() { From 469d125c59671332ca277eacf25a90d3e2763d3c Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 12:04:38 -0700 Subject: [PATCH 19/34] Redact complete file paths from Data Loop output Keep canonical denial resources actionable while replacing file paths in the diagnostic sibling JSON with . Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- docs/learning-mode/capabilities.md | 15 +- .../learning_mode/windows/src/etl_decode.rs | 54 ++--- .../learning_mode/windows/src/extractors.rs | 204 +++++++++--------- src/core/learning_mode_core/src/data_loop.rs | 11 +- src/core/learning_mode_core/src/lib.rs | 2 +- 5 files changed, 144 insertions(+), 142 deletions(-) diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index dbef5d63e..0d8220ff4 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -225,7 +225,7 @@ sandbox policy: Every successful decode also writes a deterministic sibling file: `denials..json` produces `denials..data-loop.json`. This Data -Loop artifact is a bounded, username-redacted superset containing canonical +Loop artifact is a bounded, sensitive-value-redacted superset containing canonical denial occurrences plus outcomes omitted from canonical denials: ```json @@ -243,7 +243,7 @@ denial occurrences plus outcomes omitted from canonical denials: "resourceType": "file", "properties": [ ["PackageSid", "S-1-15-3-1"], - ["resource", "C:\\Users\\\\data.txt"] + ["resource", ""] ] }, "count": 37 @@ -263,9 +263,9 @@ denial occurrences plus outcomes omitted from canonical denials: Signatures are keyed by symbolic provider category, provider GUID, provider-scoped event ID, closed exclusion reason, PID, and sorted sanitized properties. SIDs, capability names, GUIDs, PIDs/process identifiers, and -resource values are retained. Standalone user/account names and username -components inside paths or resource names are replaced with -``; the rest of each value remains available for diagnostics. +non-file resource values are retained. Complete file paths are replaced with +``; standalone user/account names remain replaced with +``. Exact header timestamps and timestamp-like properties are omitted so otherwise identical events deduplicate, and free-form decoder errors are never serialized. @@ -291,9 +291,8 @@ reason and their sanitized event properties: Property values longer than 256 characters retain bounded prefix and suffix context plus a SHA-256 digest of the complete sanitized value. This keeps long named-object resources individually identifiable when they share a prefix -without exceeding the per-property bound. Username redaction occurs before the -digest is computed, so neither the retained context nor the digest is derived -from the original username. +without exceeding the per-property bound. Redaction occurs before the digest is computed, so neither retained context nor +a digest is derived from a sensitive value. Unknown event IDs from known Learning Mode providers are classified as `unsupportedEventSchema`; the real ETL path retains their provider GUID and diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index b8510aacb..ec21a2c07 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -287,9 +287,9 @@ impl<'visitor> Accumulator<'visitor> { } /// Records an outcome for an already-built [`RawDenial`]. Retains the - /// resolved resource/capability identity (redacting only its - /// user-profile path segment, if any) plus the resource/access type — - /// never the exact `filetime`. + /// resolved resource/capability identity plus the resource/access type, + /// redacting complete file paths and never retaining the exact + /// `filetime`. fn record_raw_denial_outcome( &mut self, raw: &RawDenial, @@ -304,6 +304,16 @@ impl<'visitor> Accumulator<'visitor> { let has_object_name = properties .keys() .any(|name| name.eq_ignore_ascii_case("ObjectName")); + if raw.resource_type == learning_mode_core::ResourceType::File { + for (name, value) in &mut properties { + if name.eq_ignore_ascii_case("ObjectName") + || name.to_ascii_lowercase().contains("path") + || name.to_ascii_lowercase().ends_with("filename") + { + *value = crate::extractors::REDACTED_PATH.to_string(); + } + } + } if !matches!( raw.resource_type, learning_mode_core::ResourceType::File | learning_mode_core::ResourceType::Other @@ -311,7 +321,11 @@ impl<'visitor> Accumulator<'visitor> { { properties.insert( "resource".to_string(), - crate::extractors::redact_username_in_path(resource), + if raw.resource_type == learning_mode_core::ResourceType::File { + crate::extractors::REDACTED_PATH.to_string() + } else { + resource.to_string() + }, ); } let properties = @@ -1392,22 +1406,13 @@ mod tests { .iter() .filter(|group| group.signature.reason == DataLoopExclusionReason::UnusableResourcePath) .collect::>(); - assert_eq!(excluded.len(), 4); + assert_eq!(excluded.len(), 1); assert!(excluded.iter().all(|group| { group.signature.access_type == Some(AccessType::Read) && group.signature.resource_type == Some(ResourceType::File) - && group.count == 1 + && group.count == 4 })); - for expected in [ - r"\Device\UnknownVolume\file.txt", - r"C:relative.txt", - r"PIPE\name", - r"\\server\pipe\name", - ] { - assert!(excluded - .iter() - .any(|group| property(&group.signature, "resource") == expected)); - } + assert_eq!(property(&excluded[0].signature, "resource"), ""); } #[test] @@ -1979,10 +1984,7 @@ mod tests { assert_eq!(group.signature.pid, 1996); assert_eq!(group.signature.access_type, Some(AccessType::Read)); assert_eq!(group.signature.resource_type, Some(ResourceType::File)); - assert_eq!( - property(&group.signature, "ObjectName"), - r"\Device\MountPointManager" - ); + assert_eq!(property(&group.signature, "ObjectName"), ""); } #[test] @@ -2175,6 +2177,10 @@ mod tests { let out = resources_from_events(&events); assert_eq!(out.denials.len(), 1, "duplicates collapse to one denial"); + assert_eq!( + out.denials[0].resource, r"D:\profiles\jsmith\dup.txt", + "canonical output must retain the actionable path" + ); let canonical_group = out .data_loop @@ -2201,7 +2207,7 @@ mod tests { ); assert_eq!( property(&canonical_group.signature, "ObjectName"), - r"D:\profiles\\dup.txt" + "" ); assert_eq!( property(&canonical_group.signature, "UserName"), @@ -2323,7 +2329,7 @@ mod tests { } #[test] - fn unknown_event_id_signature_redacts_only_the_username_segment_of_a_path() { + fn unknown_event_id_signature_redacts_the_entire_file_path() { let events = vec![kernel_event( 9999, 7, @@ -2340,8 +2346,8 @@ mod tests { ); assert_eq!( property(&group.signature, "ObjectName"), - r"C:\Users\\secret.txt", - "the rest of the path must survive redaction unchanged" + "", + "no part of the path may survive redaction" ); } diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index 6f8e04352..f6d39938e 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -110,7 +110,7 @@ pub struct RawDenial { /// Symbolic category of the originating provider, for Data Loop /// aggregation. Never a raw provider GUID. pub provider: DataLoopProvider, - /// Bounded username-redacted properties retained for Data Loop signatures. + /// Bounded sensitive-value-redacted properties retained for Data Loop signatures. pub data_loop_properties: Vec<(String, String)>, } @@ -270,15 +270,17 @@ fn format_guid(guid: GUID) -> String { // // A Data Loop signature retains identifiers useful for triage (SIDs, // capability names, PIDs, provider/object GUIDs) but must never leak a -// human account/user name, an exact timestamp, or a free-form decoder error -// message. These bounds apply uniformly to every decoded property, in a -// fixed order: drop timestamp-like properties outright, redact -// account/user-identity content, *then* bound the surviving property count -// and value length — sanitizing always happens before truncation so a -// redaction is never chopped in half by a length cap. +// human account/user name, a file path, an exact timestamp, or a free-form +// decoder error message. These bounds apply uniformly to every decoded +// property, in a fixed order: drop timestamp-like properties outright, +// redact sensitive content, *then* bound the surviving property count and +// value length — sanitizing always happens before truncation so a redaction +// is never chopped in half by a length cap. /// Fixed replacement for any redacted username/account-name value. pub(crate) const REDACTED_USER: &str = ""; +/// Fixed replacement for a complete file path. +pub(crate) const REDACTED_PATH: &str = ""; /// Maximum number of `(name, value)` properties retained in one Data Loop /// signature. Bounds pathological/huge TDH property lists; sanitization /// (never raw decoding) determines which survive, via deterministic @@ -339,48 +341,32 @@ fn is_identity_property(name: &str) -> bool { || lower.ends_with("upn") } -/// Redacts the user-profile segment of a Windows path, preserving -/// everything else. Matches a `\Users\` or legacy `\Documents and -/// Settings\` segment (case-insensitively, on an exact path-segment -/// boundary so e.g. `usersdata` never matches) and replaces the segment -/// immediately following it with [`REDACTED_USER`]. -/// -/// Values that aren't path-shaped (capability names, SIDs, GUIDs, small -/// enums, ...) never contain a matching segment, so they pass through -/// unchanged — this is intentionally a single sanitizer applied uniformly -/// rather than a per-property-name allowlist. -pub(crate) fn redact_username_in_path(value: &str) -> String { - const USER_PROFILE_MARKERS: [&str; 2] = ["users", "documents and settings"]; - if !value.contains(['\\', '/']) { - return value.to_string(); - } - let mut redact_next = false; - let mut result = String::with_capacity(value.len()); - let mut segment_start = 0; - for (index, separator) in value - .char_indices() - .filter(|(_, character)| matches!(character, '\\' | '/')) +fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&str>) -> bool { + let normalized_name = name.to_ascii_lowercase().replace(['_', '-'], ""); + if normalized_name.contains("filepath") + || normalized_name == "path" + || normalized_name.ends_with("filename") { - let segment = &value[segment_start..index]; - if redact_next && !segment.is_empty() { - result.push_str(REDACTED_USER); - redact_next = false; - } else { - result.push_str(segment); - redact_next = USER_PROFILE_MARKERS - .iter() - .any(|marker| segment.eq_ignore_ascii_case(marker)); - } - result.push(separator); - segment_start = index + separator.len_utf8(); + return true; } - let segment = &value[segment_start..]; - if redact_next && !segment.is_empty() { - result.push_str(REDACTED_USER); - } else { - result.push_str(segment); + + if normalized_name != "objectname" && normalized_name != "resource" { + return false; + } + if let Some(object_type) = object_type { + return object_type.eq_ignore_ascii_case("File"); } - result + + let bytes = value.as_bytes(); + let drive_absolute = bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'\\' | b'/'); + drive_absolute + || value.starts_with(r"\\") + || value.starts_with(r"\??\") + || value.starts_with(r"\\?\") + || value.starts_with(r"\\.\") } /// Bounds an already-sanitized property list to [`MAX_SIGNATURE_PROPERTIES`] @@ -427,13 +413,18 @@ fn bound_property_value(value: &str, char_count: usize) -> String { /// so exact-timestamp differences between otherwise-identical events don't /// prevent their signatures from deduplicating. SIDs, capability names, /// PIDs carried as properties, and GUID-shaped values are retained -/// verbatim; only account/user-identity content is redacted. +/// verbatim; account/user-identity content and complete file paths are +/// redacted. pub(crate) fn sanitize_properties(props: &[(String, String)]) -> Vec<(String, String)> { let usernames = props .iter() .filter(|(name, _)| is_identity_property(name)) .flat_map(|(_, value)| username_match_candidates(value.trim_matches('"'))) .collect::>(); + let object_type = props + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("ObjectType")) + .map(|(_, value)| value.trim_matches('"')); let mut sanitized = std::collections::BTreeMap::new(); for (name, raw_value) in props { if is_timestamp_like_property(name) { @@ -442,8 +433,10 @@ pub(crate) fn sanitize_properties(props: &[(String, String)]) -> Vec<(String, St let value = raw_value.trim_matches('"'); let sanitized_value = if is_identity_property(name) { REDACTED_USER.to_string() + } else if looks_like_file_path_property(name, value, object_type) { + REDACTED_PATH.to_string() } else { - redact_known_username_components(&redact_username_in_path(value), &usernames) + redact_known_username_components(value, &usernames) }; sanitized.insert(name.clone(), sanitized_value); } @@ -1388,44 +1381,32 @@ mod tests { // ---- Data Loop signature sanitization ---------------------------------- #[test] - fn redact_username_in_path_replaces_only_the_profile_segment() { - assert_eq!( - redact_username_in_path(r"C:\Users\jsmith\Documents\secret.txt"), - format!(r"C:\Users\{REDACTED_USER}\Documents\secret.txt") - ); - assert_eq!( - redact_username_in_path(r"\Device\HarddiskVolume3\Users\jane.doe\AppData\x"), - format!(r"\Device\HarddiskVolume3\Users\{REDACTED_USER}\AppData\x") - ); - assert_eq!( - redact_username_in_path(r"\??\C:\Documents and Settings\bob\Desktop\x"), - format!(r"\??\C:\Documents and Settings\{REDACTED_USER}\Desktop\x") - ); - assert_eq!( - redact_username_in_path("C:/Users/alice/Documents/secret.txt"), - format!("C:/Users/{REDACTED_USER}/Documents/secret.txt") - ); - } - - #[test] - fn redact_username_in_path_is_case_insensitive_on_the_marker_only() { - assert_eq!( - redact_username_in_path(r"C:\USERS\Alice\file.txt"), - format!(r"C:\USERS\{REDACTED_USER}\file.txt") - ); - } - - #[test] - fn redact_username_in_path_leaves_unrelated_paths_untouched() { - // Registry hive segment "USER" (singular) must not be mistaken for - // the "Users" (plural) profile-directory marker. - let registry_path = r"\REGISTRY\USER\S-1-5-21-1-2-3-1000\Software\Test"; - assert_eq!(redact_username_in_path(registry_path), registry_path); - assert_eq!(redact_username_in_path("internetClient"), "internetClient"); - assert_eq!( - redact_username_in_path(r"C:\data\test\bin\"), - r"C:\data\test\bin\" - ); + fn file_path_detection_preserves_non_file_identifiers() { + assert!(looks_like_file_path_property( + "ObjectName", + r"C:\Users\jsmith\secret.txt", + None + )); + assert!(looks_like_file_path_property( + "FilePath", + r"\Device\HarddiskVolume3\secret.txt", + Some("Section") + )); + assert!(!looks_like_file_path_property( + "ObjectName", + r"\BaseNamedObjects\shared-cache", + Some("Section") + )); + assert!(!looks_like_file_path_property( + "ObjectName", + r"\REGISTRY\USER\S-1-5-21-1-2-3-1000\Software\Test", + Some("Key") + )); + assert!(!looks_like_file_path_property( + "resource", + "internetClient", + None + )); } #[test] @@ -1472,7 +1453,7 @@ mod tests { } #[test] - fn sanitize_properties_redacts_username_segment_inside_a_path_value() { + fn sanitize_properties_redacts_the_entire_file_path() { let props = vec![( "ObjectName".to_string(), "\"C:\\Users\\jsmith\\secret.txt\"".to_string(), @@ -1480,15 +1461,12 @@ mod tests { let out = sanitize_properties(&props); assert_eq!( out, - vec![( - "ObjectName".to_string(), - format!(r"C:\Users\{REDACTED_USER}\secret.txt") - )] + vec![("ObjectName".to_string(), REDACTED_PATH.to_string())] ); } #[test] - fn sanitize_properties_redacts_known_username_in_arbitrary_resource_path() { + fn sanitize_properties_redacts_arbitrary_file_path() { let props = vec![ ("UserName".to_string(), "\"jsmith\"".to_string()), ( @@ -1499,14 +1477,29 @@ mod tests { let out = sanitize_properties(&props); let value_for = |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); assert_eq!(value_for("UserName"), Some(REDACTED_USER)); + assert_eq!(value_for("ObjectName"), Some(REDACTED_PATH)); + } + + #[test] + fn sanitize_properties_still_redacts_username_in_non_file_identifier() { + let props = vec![ + ("ObjectType".to_string(), "\"Section\"".to_string()), + ("UserName".to_string(), "\"jsmith\"".to_string()), + ( + "ObjectName".to_string(), + "\"\\BaseNamedObjects\\jsmith\\cache\"".to_string(), + ), + ]; + let out = sanitize_properties(&props); + let value_for = |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); assert_eq!( value_for("ObjectName"), - Some(r"D:\profiles\\secret.txt") + Some(r"\BaseNamedObjects\\cache") ); } #[test] - fn sanitize_properties_redacts_qualified_username_in_arbitrary_resource_path() { + fn sanitize_properties_redacts_path_with_qualified_username() { for (property_name, identity) in [ ("UserName", r"CONTOSO\jsmith"), ("UserPrincipalName", "jsmith@example.com"), @@ -1526,7 +1519,7 @@ mod tests { }; assert_eq!( value_for("ObjectName").map(String::as_str), - Some(r"D:\profiles\\secret.txt") + Some(REDACTED_PATH) ); assert_eq!( value_for(property_name).map(String::as_str), @@ -1561,7 +1554,7 @@ mod tests { ); assert_eq!( value_for("ObjectName").map(String::as_str), - Some(r"D:\profiles\\secret.txt") + Some(REDACTED_PATH) ); } } @@ -1580,7 +1573,7 @@ mod tests { out.iter() .find(|(name, _)| name == "ObjectName") .map(|(_, value)| value.as_str()), - Some(r"D:\profiles\\secret.txt") + Some(REDACTED_PATH) ); } @@ -1611,7 +1604,7 @@ mod tests { } #[test] - fn bound_properties_preserves_identity_within_count_and_value_limits() { + fn bound_properties_preserves_redaction_within_count_and_value_limits() { let long_value = format!( r"C:\Users\{}\{}", "jsmith", @@ -1626,7 +1619,14 @@ mod tests { } let sanitized: Vec<(String, String)> = props .into_iter() - .map(|(name, value)| (name, redact_username_in_path(&value))) + .map(|(name, value)| { + let value = if looks_like_file_path_property(&name, &value, None) { + REDACTED_PATH.to_string() + } else { + value + }; + (name, value) + }) .collect(); let out = bound_properties(sanitized); @@ -1637,12 +1637,12 @@ mod tests { .find(|(name, _)| name == "ObjectName") .expect("ObjectName retained (sorted first)"); assert!( - object_name.1.contains(REDACTED_USER), + object_name.1.contains(REDACTED_PATH), "redaction must survive bounding: {}", object_name.1 ); - assert!(object_name.1.contains(", - /// Sorted, bounded, username-redacted event properties. + /// Sorted, bounded, sensitive-value-redacted event properties. pub properties: Vec<(String, String)>, } @@ -380,10 +380,7 @@ mod tests { access_type: None, resource_type: None, properties: vec![ - ( - "ObjectName".to_string(), - r"C:\Users\\x".to_string(), - ), + ("ObjectName".to_string(), "".to_string()), ("Sid".to_string(), "S-1-15-3-1".to_string()), ], }; @@ -398,7 +395,7 @@ mod tests { write_data_loop_document(&mut bytes, &DataLoopDocument::new(&summary)).unwrap(); let text = String::from_utf8(bytes).unwrap(); assert!(text.contains("S-1-15-3-1")); - assert!(text.contains("")); + assert!(text.contains("")); assert!(text.contains("providerGuid")); assert!(text.contains("\"pid\": 42")); assert!(!text.contains("accessType")); diff --git a/src/core/learning_mode_core/src/lib.rs b/src/core/learning_mode_core/src/lib.rs index ac3c99838..b1fbf9efc 100644 --- a/src/core/learning_mode_core/src/lib.rs +++ b/src/core/learning_mode_core/src/lib.rs @@ -14,7 +14,7 @@ //! 3. **Emit** — the records plus a terminating [`DenialSummary`] are //! written to the canonical JSON output that host applications read to //! regenerate policy. A deterministic Data Loop sibling contains bounded, -//! username-redacted signatures for excluded decoder outcomes. +//! sensitive-value-redacted signatures for excluded decoder outcomes. //! //! This crate is the cross-platform hinge between stages 2 and 3: it //! defines the public [`DeniedResource`] model, the [`DenialSummary`] From 8d11809fc76541b03409a7c2e577a8250622f883 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 12:08:06 -0700 Subject: [PATCH 20/34] Make audit copy fallback race-safe Acquire fallback destinations with create_new and clean up only files owned by the current relocation attempt. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- src/core/wxc/src/audit.rs | 124 +++++++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 35 deletions(-) diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index afab88a99..7cf0199ac 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -125,12 +125,14 @@ pub fn finalize( let final_etl = context.log_dir.join("trace.etl"); relocate_artifacts( capture, - &source_denials, - &final_denials, - &source_data_loop, - &final_data_loop, - &source_etl, - &final_etl, + &ArtifactRelocation { + source_denials: &source_denials, + final_denials: &final_denials, + source_data_loop: &source_data_loop, + final_data_loop: &final_data_loop, + source_etl: &source_etl, + final_etl: &final_etl, + }, move_new_file, )?; remove_empty_managed_directory(source_etl.parent(), &context.log_dir)?; @@ -180,33 +182,37 @@ fn validate_metadata( /// /// `move_file` is injected so tests can force a second-move failure /// deterministically; production passes [`move_new_file`]. +struct ArtifactRelocation<'a> { + source_denials: &'a Path, + final_denials: &'a Path, + source_data_loop: &'a Path, + final_data_loop: &'a Path, + source_etl: &'a Path, + final_etl: &'a Path, +} + fn relocate_artifacts( capture: &mut wxc_common::models::CaptureDenialsOutput, - source_denials: &Path, - final_denials: &Path, - source_data_loop: &Path, - final_data_loop: &Path, - source_etl: &Path, - final_etl: &Path, + paths: &ArtifactRelocation<'_>, mut move_file: impl FnMut(&Path, &Path) -> Result<(), String>, ) -> Result<(), String> { - ensure_destination_available(source_denials, final_denials)?; - ensure_destination_available(source_data_loop, final_data_loop)?; - ensure_destination_available(source_etl, final_etl)?; + ensure_destination_available(paths.source_denials, paths.final_denials)?; + ensure_destination_available(paths.source_data_loop, paths.final_data_loop)?; + ensure_destination_available(paths.source_etl, paths.final_etl)?; // Primary denials JSON first: publish its final path the instant it moves so // a later ETL failure can never leave `output_path` referencing a file that // has already been renamed away. - move_file(source_denials, final_denials)?; - capture.output_path = final_denials.to_string_lossy().into_owned(); + move_file(paths.source_denials, paths.final_denials)?; + capture.output_path = paths.final_denials.to_string_lossy().into_owned(); - move_file(source_data_loop, final_data_loop)?; + move_file(paths.source_data_loop, paths.final_data_loop)?; // Retained ETL second: on failure it stays at its source and `etl_path` // still points there (unchanged), so each artifact remains individually // truthful even though the relocation as a whole failed. - move_file(source_etl, final_etl)?; - capture.etl_path = Some(final_etl.to_string_lossy().into_owned()); + move_file(paths.source_etl, paths.final_etl)?; + capture.etl_path = Some(paths.final_etl.to_string_lossy().into_owned()); Ok(()) } @@ -218,23 +224,22 @@ fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { match std::fs::rename(source, destination) { Ok(()) => Ok(()), Err(rename_error) => { - if let Err(copy_error) = std::fs::copy(source, destination) { - let cleanup_error = std::fs::remove_file(destination) - .err() - .filter(|error| error.kind() != std::io::ErrorKind::NotFound); - return Err(match cleanup_error { + if let Err(copy_error) = copy_to_new_file(source, destination) { + return Err(match copy_error.cleanup_error { Some(cleanup_error) => format!( "failed to move audit artifact {} to {}: {rename_error}; \ - copy fallback failed: {copy_error}; partial destination cleanup failed: \ + copy fallback failed: {}; partial destination cleanup failed: \ {cleanup_error}", source.display(), - destination.display() + destination.display(), + copy_error.error ), None => format!( "failed to move audit artifact {} to {}: {rename_error}; \ - copy fallback failed: {copy_error}", + copy fallback failed: {}", source.display(), - destination.display() + destination.display(), + copy_error.error ), }); } @@ -250,6 +255,37 @@ fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { } } +struct CopyNewFileError { + error: std::io::Error, + cleanup_error: Option, +} + +fn copy_to_new_file(source: &Path, destination: &Path) -> Result<(), CopyNewFileError> { + let mut source_file = std::fs::File::open(source).map_err(|error| CopyNewFileError { + error, + cleanup_error: None, + })?; + let mut destination_file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(destination) + .map_err(|error| CopyNewFileError { + error, + cleanup_error: None, + })?; + if let Err(error) = std::io::copy(&mut source_file, &mut destination_file) { + drop(destination_file); + let cleanup_error = std::fs::remove_file(destination) + .err() + .filter(|cleanup_error| cleanup_error.kind() != std::io::ErrorKind::NotFound); + return Err(CopyNewFileError { + error, + cleanup_error, + }); + } + Ok(()) +} + fn ensure_destination_available(source: &Path, destination: &Path) -> Result<(), String> { if source != destination && destination.exists() { return Err(format!( @@ -485,6 +521,22 @@ mod tests { assert!(!log_dir.join("trace.etl").exists()); } + #[test] + fn copy_fallback_never_clobbers_an_existing_destination() { + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source.json"); + let destination = directory.path().join("destination.json"); + std::fs::write(&source, b"new").unwrap(); + std::fs::write(&destination, b"existing").unwrap(); + + let error = copy_to_new_file(&source, &destination).unwrap_err(); + + assert_eq!(error.error.kind(), std::io::ErrorKind::AlreadyExists); + assert!(error.cleanup_error.is_none()); + assert_eq!(std::fs::read(&source).unwrap(), b"new"); + assert_eq!(std::fs::read(&destination).unwrap(), b"existing"); + } + #[test] fn finalize_rejects_missing_data_loop_output() { let directory = tempfile::tempdir().unwrap(); @@ -536,12 +588,14 @@ mod tests { let mut calls = 0; let error = relocate_artifacts( &mut capture, - &source_denials, - &final_denials, - &source_data_loop, - &final_data_loop, - &source_etl, - &final_etl, + &ArtifactRelocation { + source_denials: &source_denials, + final_denials: &final_denials, + source_data_loop: &source_data_loop, + final_data_loop: &final_data_loop, + source_etl: &source_etl, + final_etl: &final_etl, + }, |source, destination| { calls += 1; if calls <= 2 { From 262cc6b24d4eccbed0faba98a57f5c4633129969 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 12:49:09 -0700 Subject: [PATCH 21/34] Redact path-suffixed Data Loop properties Treat AppPath, ApplicationPath, and other path-suffixed fields as complete file paths before diagnostic bounding or hashing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode/windows/src/extractors.rs | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index f6d39938e..e00ea273d 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -343,10 +343,7 @@ fn is_identity_property(name: &str) -> bool { fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&str>) -> bool { let normalized_name = name.to_ascii_lowercase().replace(['_', '-'], ""); - if normalized_name.contains("filepath") - || normalized_name == "path" - || normalized_name.ends_with("filename") - { + if normalized_name.ends_with("path") || normalized_name.ends_with("filename") { return true; } @@ -1454,14 +1451,28 @@ mod tests { #[test] fn sanitize_properties_redacts_the_entire_file_path() { - let props = vec![( - "ObjectName".to_string(), - "\"C:\\Users\\jsmith\\secret.txt\"".to_string(), - )]; + let props = vec![ + ( + "ObjectName".to_string(), + "\"C:\\Users\\jsmith\\secret.txt\"".to_string(), + ), + ( + "AppPath".to_string(), + "\"C:\\Users\\jsmith\\app.exe\"".to_string(), + ), + ( + "ApplicationPath".to_string(), + "\"D:\\Profiles\\alice\\tool.exe\"".to_string(), + ), + ]; let out = sanitize_properties(&props); assert_eq!( out, - vec![("ObjectName".to_string(), REDACTED_PATH.to_string())] + vec![ + ("AppPath".to_string(), REDACTED_PATH.to_string()), + ("ApplicationPath".to_string(), REDACTED_PATH.to_string()), + ("ObjectName".to_string(), REDACTED_PATH.to_string()), + ] ); } From 9ebe3a1129de437aa4d69474e8deb1db7d3b796b Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 13:02:02 -0700 Subject: [PATCH 22/34] Make audit artifact relocation transactional Relocate canonical and Data Loop JSON as an ownership-verified pair and stage ETL copy fallbacks through private no-clobber temporary files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- src/core/learning_mode_core/src/lib.rs | 4 +- .../learning_mode_core/src/paired_output.rs | 98 +++++++++++ src/core/wxc/src/audit.rs | 161 ++++++++++-------- 3 files changed, 190 insertions(+), 73 deletions(-) diff --git a/src/core/learning_mode_core/src/lib.rs b/src/core/learning_mode_core/src/lib.rs index b1fbf9efc..679f5eee8 100644 --- a/src/core/learning_mode_core/src/lib.rs +++ b/src/core/learning_mode_core/src/lib.rs @@ -47,5 +47,7 @@ pub use data_loop::{ }; pub use emit::{write_document, DenialsDocument, DenialsOutputPointer}; pub use model::{AccessType, DedupKey, DeniedResource, ResourceType}; -pub use paired_output::{write_paired_output_files, ExistingOutputPolicy}; +pub use paired_output::{ + relocate_paired_output_files, write_paired_output_files, ExistingOutputPolicy, +}; pub use summary::DenialSummary; diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index 3bfeb653e..9e451c7dd 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -190,6 +190,80 @@ pub fn write_paired_output_files( remove_backups(canonical_backup.as_deref(), data_loop_backup.as_deref()) } +/// Copies an existing canonical/Data Loop pair to new no-clobber destinations, +/// then removes only source files whose identity and contents still match the +/// files that were copied. +/// +/// The destination pair uses the same transactional promotion and rollback as +/// [`write_paired_output_files`]. Source cleanup quarantines and verifies each +/// file before deletion, so a concurrent replacement is restored rather than +/// removed. +/// +/// # Errors +/// +/// Returns an error when the source pair cannot be read, the destination pair +/// cannot be committed atomically, or verified source cleanup fails. +pub fn relocate_paired_output_files( + operation: &str, + canonical_source_path: &Path, + data_loop_source_path: &Path, + canonical_destination_path: &Path, + data_loop_destination_path: &Path, +) -> std::io::Result<()> { + if canonical_source_path == canonical_destination_path + && data_loop_source_path == data_loop_destination_path + { + return Ok(()); + } + if canonical_source_path == canonical_destination_path + || data_loop_source_path == data_loop_destination_path + { + return Err(std::io::Error::other(format!( + "{operation} cannot relocate only one member of an output pair" + ))); + } + + let mut canonical_source = std::fs::File::open(canonical_source_path)?; + let canonical_identity = PromotedOutput::from_file(canonical_source.try_clone()?)?; + let mut data_loop_source = std::fs::File::open(data_loop_source_path)?; + let data_loop_identity = PromotedOutput::from_file(data_loop_source.try_clone()?)?; + + write_paired_output_files( + operation, + canonical_destination_path, + data_loop_destination_path, + ExistingOutputPolicy::CreateNew, + |writer| { + canonical_source.rewind()?; + std::io::copy(&mut canonical_source, writer).map(|_| ()) + }, + |writer| { + data_loop_source.rewind()?; + std::io::copy(&mut data_loop_source, writer).map(|_| ()) + }, + )?; + + drop((canonical_source, data_loop_source)); + let data_loop_cleanup = + remove_promoted_output_if_owned(data_loop_source_path, data_loop_identity, None) + .and_then(cleanup_result); + let canonical_cleanup = + remove_promoted_output_if_owned(canonical_source_path, canonical_identity, None) + .and_then(cleanup_result); + match (data_loop_cleanup, canonical_cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(first), Err(second)) => Err(std::io::Error::other(format!( + "{first}; additionally failed to remove relocated canonical source {}: {second}", + canonical_source_path.display() + ))), + } +} + +fn cleanup_result(error: Option) -> std::io::Result<()> { + error.map_or(Ok(()), Err) +} + fn ensure_output_absent(operation: &str, path: &Path, kind: &str) -> std::io::Result<()> { match std::fs::symlink_metadata(path) { Ok(_) => Err(std::io::Error::new( @@ -577,6 +651,30 @@ mod tests { assert_eq!(std::fs::read(shared_path).unwrap(), b"previous"); } + #[test] + fn relocation_rolls_back_failed_second_promotion_and_keeps_sources() { + let directory = tempfile::tempdir().unwrap(); + let canonical_source = directory.path().join("source-denials.json"); + let data_loop_source = directory.path().join("source-denials.data-loop.json"); + let shared_destination = directory.path().join("denials.json"); + std::fs::write(&canonical_source, b"canonical").unwrap(); + std::fs::write(&data_loop_source, b"data-loop").unwrap(); + + let error = relocate_paired_output_files( + "test relocation", + &canonical_source, + &data_loop_source, + &shared_destination, + &shared_destination, + ) + .unwrap_err(); + + assert!(error.to_string().contains("failed to promote canonical")); + assert!(!shared_destination.exists()); + assert_eq!(std::fs::read(canonical_source).unwrap(), b"canonical"); + assert_eq!(std::fs::read(data_loop_source).unwrap(), b"data-loop"); + } + #[test] fn rollback_does_not_delete_unpromoted_collision() { let directory = tempfile::tempdir().unwrap(); diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index 7cf0199ac..50d6d9ddb 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -133,6 +133,7 @@ pub fn finalize( source_etl: &source_etl, final_etl: &final_etl, }, + learning_mode_core::relocate_paired_output_files, move_new_file, )?; remove_empty_managed_directory(source_etl.parent(), &context.log_dir)?; @@ -174,11 +175,10 @@ fn validate_metadata( /// on-disk state and the metadata paths mutually truthful even on partial /// failure. /// -/// All destinations are preflighted (no-clobber) before anything moves. The -/// primary denials JSON moves first and its final path is published to the -/// metadata immediately; its Data Loop sibling moves second; the retained ETL -/// moves last and its final path is published immediately on success. A later -/// move failure therefore leaves every metadata path truthful. +/// The JSON pair is relocated transactionally and its final canonical path is +/// published only after both siblings commit. The retained ETL moves last and +/// its final path is published immediately on success. A later move failure +/// therefore leaves every metadata path truthful. /// /// `move_file` is injected so tests can force a second-move failure /// deterministically; production passes [`move_new_file`]. @@ -194,21 +194,22 @@ struct ArtifactRelocation<'a> { fn relocate_artifacts( capture: &mut wxc_common::models::CaptureDenialsOutput, paths: &ArtifactRelocation<'_>, + relocate_pair: impl FnOnce(&str, &Path, &Path, &Path, &Path) -> std::io::Result<()>, mut move_file: impl FnMut(&Path, &Path) -> Result<(), String>, ) -> Result<(), String> { - ensure_destination_available(paths.source_denials, paths.final_denials)?; - ensure_destination_available(paths.source_data_loop, paths.final_data_loop)?; ensure_destination_available(paths.source_etl, paths.final_etl)?; - // Primary denials JSON first: publish its final path the instant it moves so - // a later ETL failure can never leave `output_path` referencing a file that - // has already been renamed away. - move_file(paths.source_denials, paths.final_denials)?; + relocate_pair( + "audit output relocation", + paths.source_denials, + paths.source_data_loop, + paths.final_denials, + paths.final_data_loop, + ) + .map_err(|error| format!("failed to relocate audit output pair: {error}"))?; capture.output_path = paths.final_denials.to_string_lossy().into_owned(); - move_file(paths.source_data_loop, paths.final_data_loop)?; - - // Retained ETL second: on failure it stays at its source and `etl_path` + // Retained ETL last: on failure it stays at its source and `etl_path` // still points there (unchanged), so each artifact remains individually // truthful even though the relocation as a whole failed. move_file(paths.source_etl, paths.final_etl)?; @@ -225,65 +226,37 @@ fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { Ok(()) => Ok(()), Err(rename_error) => { if let Err(copy_error) = copy_to_new_file(source, destination) { - return Err(match copy_error.cleanup_error { - Some(cleanup_error) => format!( - "failed to move audit artifact {} to {}: {rename_error}; \ - copy fallback failed: {}; partial destination cleanup failed: \ - {cleanup_error}", - source.display(), - destination.display(), - copy_error.error - ), - None => format!( - "failed to move audit artifact {} to {}: {rename_error}; \ - copy fallback failed: {}", - source.display(), - destination.display(), - copy_error.error - ), - }); + return Err(format!( + "failed to move audit artifact {} to {}: {rename_error}; \ + copy fallback failed: {copy_error}", + source.display(), + destination.display() + )); } - std::fs::remove_file(source).map_err(|remove_error| { - let _ = std::fs::remove_file(destination); - format!( - "copied audit artifact to {}, but failed to remove source {}: {remove_error}", + match std::fs::remove_file(source) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(remove_error) => Err(format!( + "copied audit artifact to {}, but failed to remove source {}: {remove_error}; \ + the complete no-clobber destination was retained", destination.display(), source.display() - ) - }) + )), + } } } } -struct CopyNewFileError { - error: std::io::Error, - cleanup_error: Option, -} - -fn copy_to_new_file(source: &Path, destination: &Path) -> Result<(), CopyNewFileError> { - let mut source_file = std::fs::File::open(source).map_err(|error| CopyNewFileError { - error, - cleanup_error: None, - })?; - let mut destination_file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(destination) - .map_err(|error| CopyNewFileError { - error, - cleanup_error: None, - })?; - if let Err(error) = std::io::copy(&mut source_file, &mut destination_file) { - drop(destination_file); - let cleanup_error = std::fs::remove_file(destination) - .err() - .filter(|cleanup_error| cleanup_error.kind() != std::io::ErrorKind::NotFound); - return Err(CopyNewFileError { - error, - cleanup_error, - }); - } - Ok(()) +fn copy_to_new_file(source: &Path, destination: &Path) -> std::io::Result<()> { + let parent = destination.parent().unwrap_or_else(|| Path::new(".")); + let mut source_file = std::fs::File::open(source)?; + let mut staged = tempfile::NamedTempFile::new_in(parent)?; + std::io::copy(&mut source_file, &mut staged)?; + staged.as_file().sync_all()?; + staged + .persist_noclobber(destination) + .map(|_| ()) + .map_err(|error| error.error) } fn ensure_destination_available(source: &Path, destination: &Path) -> Result<(), String> { @@ -512,7 +485,7 @@ mod tests { let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); - assert!(error.contains("destination already exists")); + assert!(error.contains("canonical output file already exists")); assert!(source_denials.exists()); assert!(learning_mode_core::data_loop_sibling_path(&source_denials) .unwrap() @@ -531,8 +504,7 @@ mod tests { let error = copy_to_new_file(&source, &destination).unwrap_err(); - assert_eq!(error.error.kind(), std::io::ErrorKind::AlreadyExists); - assert!(error.cleanup_error.is_none()); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); assert_eq!(std::fs::read(&source).unwrap(), b"new"); assert_eq!(std::fs::read(&destination).unwrap(), b"existing"); } @@ -596,12 +568,13 @@ mod tests { source_etl: &source_etl, final_etl: &final_etl, }, + learning_mode_core::relocate_paired_output_files, |source, destination| { calls += 1; - if calls <= 2 { - std::fs::rename(source, destination).map_err(|error| error.to_string()) - } else { + if calls == 1 { Err("injected ETL move failure".to_string()) + } else { + std::fs::rename(source, destination).map_err(|error| error.to_string()) } }, ) @@ -623,6 +596,50 @@ mod tests { ); } + #[test] + fn relocate_rolls_back_json_pair_before_publishing_metadata() { + let directory = tempfile::tempdir().unwrap(); + let source_denials = directory.path().join("denials.unique.json"); + let source_data_loop = directory.path().join("denials.unique.data-loop.json"); + let source_etl = directory.path().join("denials.unique.etl"); + let shared_destination = directory.path().join("denials.json"); + let final_etl = directory.path().join("trace.etl"); + std::fs::write(&source_denials, b"denials").unwrap(); + std::fs::write(&source_data_loop, b"data-loop").unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut capture = CaptureDenialsOutput { + kind: CaptureDenialsOutput::KIND.to_string(), + output_path: source_denials.to_string_lossy().into_owned(), + exit_code: 0, + total_denials: 0, + denied_resources_truncated: false, + etl_path: Some(source_etl.to_string_lossy().into_owned()), + }; + + let error = relocate_artifacts( + &mut capture, + &ArtifactRelocation { + source_denials: &source_denials, + final_denials: &shared_destination, + source_data_loop: &source_data_loop, + final_data_loop: &shared_destination, + source_etl: &source_etl, + final_etl: &final_etl, + }, + learning_mode_core::relocate_paired_output_files, + |_, _| panic!("ETL relocation must not run after JSON-pair failure"), + ) + .unwrap_err(); + + assert!(error.contains("failed to promote canonical")); + assert_eq!(Path::new(&capture.output_path), source_denials); + assert!(source_denials.is_file()); + assert!(source_data_loop.is_file()); + assert!(!shared_destination.exists()); + assert!(source_etl.is_file()); + assert!(!final_etl.exists()); + } + fn response_with_capture( denials_path: &Path, etl_path: &Path, From 46e4f3b1f9411efd61571204f3847c48588ca532 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 13:32:17 -0700 Subject: [PATCH 23/34] Address adversarial review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../appcontainer/common/src/capture_output.rs | 72 +--------- .../learning_mode/windows/src/extractors.rs | 116 ++++++++++----- src/core/learning_mode_core/src/analyze.rs | 135 ++++++++++++++++-- src/core/learning_mode_core/src/lib.rs | 3 +- .../learning_mode_core/src/paired_output.rs | 29 ++++ src/core/wxc/src/audit.rs | 91 ++++++------ 6 files changed, 281 insertions(+), 165 deletions(-) diff --git a/src/backends/appcontainer/common/src/capture_output.rs b/src/backends/appcontainer/common/src/capture_output.rs index 5a60a99b1..71773fac0 100644 --- a/src/backends/appcontainer/common/src/capture_output.rs +++ b/src/backends/appcontainer/common/src/capture_output.rs @@ -16,7 +16,6 @@ //! error type; callers map the plain `String` errors into their own error //! type at the call site. -use std::io::Write; use std::path::{Path, PathBuf}; use learning_mode_core::{ @@ -97,47 +96,6 @@ pub fn data_loop_output_path(output_path: &Path) -> std::io::Result { .map_err(|error| std::io::Error::other(format!("captureDenials {error}"))) } -/// Creates `output_path` (failing if it already exists) and writes through -/// `write`, cleaning up a partial file if `write` fails. -pub fn write_denials_output_file( - output_path: &Path, - write: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, -) -> std::io::Result<()> { - let file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(output_path) - .map_err(|error| { - std::io::Error::other(format!( - "captureDenials failed to create denials output file {}: {error}", - output_path.display() - )) - })?; - - let write_result = { - let mut writer = std::io::BufWriter::new(file); - write(&mut writer).and_then(|()| writer.flush()) - }; - if let Err(error) = write_result { - let write_error = std::io::Error::other(format!( - "captureDenials failed to write denials output file {}: {error}", - output_path.display() - )); - return match std::fs::remove_file(output_path) { - Ok(()) => Err(write_error), - Err(cleanup_error) if cleanup_error.kind() == std::io::ErrorKind::NotFound => { - Err(write_error) - } - Err(cleanup_error) => Err(std::io::Error::other(format!( - "{write_error}; additionally failed to remove incomplete output file {}: {cleanup_error}", - output_path.display() - ))), - }; - } - - Ok(()) -} - /// Inserts a per-run identifier into a denials output path's file stem so /// concurrent and sequential captures using the same configured `outputPath` /// produce distinct files instead of clobbering one another. @@ -327,6 +285,7 @@ mod tests { AccessType, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, DeniedResource, ResourceType, }; + use std::io::Write; #[test] fn write_denials_document_writes_summary_and_document() { @@ -448,35 +407,6 @@ mod tests { assert!(!data_loop_path.exists()); } - #[test] - fn failed_denials_write_removes_incomplete_output() { - let directory = tempfile::tempdir().expect("temp directory"); - let output_path = directory.path().join("denials.json"); - - let error = write_denials_output_file(&output_path, |writer| { - std::io::Write::write_all(writer, b"{\"partial\":")?; - Err(std::io::Error::other("simulated write failure")) - }) - .expect_err("write should fail"); - - assert!(error.to_string().contains("simulated write failure")); - assert!(!output_path.exists()); - } - - #[test] - fn denials_output_does_not_overwrite_an_existing_file() { - let directory = tempfile::tempdir().expect("temp directory"); - let output_path = directory.path().join("denials.json"); - std::fs::write(&output_path, b"existing").expect("seed output"); - - write_denials_output_file(&output_path, |_| Ok(())).expect_err("collision should fail"); - - assert_eq!( - std::fs::read(&output_path).expect("read existing output"), - b"existing" - ); - } - #[test] fn insert_run_id_into_stem_injects_id_before_extension() { let got = insert_run_id_into_stem(Path::new(r"C:\app\denials.json"), "1234_abcd"); diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index e00ea273d..ee5397a64 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -292,28 +292,74 @@ const SHA256_HEX_LEN: usize = 64; const BOUNDED_VALUE_MARKER_LEN: usize = "......".len(); const _: () = assert!(MAX_SIGNATURE_VALUE_LEN > BOUNDED_VALUE_MARKER_LEN); +#[derive(Clone, Copy)] +struct NormalizedPropertyName<'a>(&'a str); + +impl<'a> NormalizedPropertyName<'a> { + fn bytes(self) -> impl DoubleEndedIterator + Clone + 'a { + self.0 + .bytes() + .filter(|byte| !matches!(byte, b'_' | b'-')) + .map(|byte| byte.to_ascii_lowercase()) + } + + fn equals(self, expected: &str) -> bool { + self.bytes().eq(expected.bytes()) + } + + fn ends_with(self, suffix: &str) -> bool { + let mut bytes = self.bytes().rev(); + suffix + .bytes() + .rev() + .all(|expected| bytes.next() == Some(expected)) + } + + fn contains(self, needle: &str) -> bool { + let needle = needle.as_bytes(); + if needle.is_empty() { + return true; + } + let mut bytes = self.bytes(); + while let Some(byte) = bytes.next() { + if byte == needle[0] { + let mut remainder = bytes.clone(); + if needle[1..] + .iter() + .all(|expected| remainder.next() == Some(*expected)) + { + return true; + } + } + } + false + } +} + /// Returns whether `name` looks like it carries a timestamp, so it is /// dropped from a Data Loop signature entirely (never redacted or /// truncated) — exact timestamps must not prevent otherwise-identical /// events from deduplicating. fn is_timestamp_like_property(name: &str) -> bool { - let lower = name.to_ascii_lowercase().replace(['_', '-'], ""); - matches!( - lower.as_str(), - "time" - | "timestamp" - | "filetime" - | "systemtime" - | "eventtime" - | "timecreated" - | "creationtime" - | "createtime" - | "starttime" - | "endtime" - | "exittime" - | "lastwritetime" - ) || lower.ends_with("timestamp") - || lower.ends_with("filetime") + let normalized = NormalizedPropertyName(name); + [ + "time", + "timestamp", + "filetime", + "systemtime", + "eventtime", + "timecreated", + "creationtime", + "createtime", + "starttime", + "endtime", + "exittime", + "lastwritetime", + ] + .into_iter() + .any(|expected| normalized.equals(expected)) + || normalized.ends_with("timestamp") + || normalized.ends_with("filetime") } /// Returns whether `name` is itself a standalone user/account-identity @@ -321,33 +367,33 @@ fn is_timestamp_like_property(name: &str) -> bool { /// its value is replaced outright with [`REDACTED_USER`] regardless of /// content. fn is_identity_property(name: &str) -> bool { - let lower = name.to_ascii_lowercase().replace(['_', '-'], ""); - if lower.contains("sid") - || lower.contains("guid") - || lower.contains("identifier") - || lower.ends_with("id") + let normalized = NormalizedPropertyName(name); + if normalized.contains("sid") + || normalized.contains("guid") + || normalized.contains("identifier") + || normalized.ends_with("id") { return false; } - lower == "user" - || lower == "account" - || lower == "owner" - || lower.contains("username") - || lower.contains("accountname") - || lower.contains("ownername") - || lower.contains("principalname") - || lower.ends_with("account") - || lower == "upn" - || lower.ends_with("upn") + normalized.equals("user") + || normalized.equals("account") + || normalized.equals("owner") + || normalized.contains("username") + || normalized.contains("accountname") + || normalized.contains("ownername") + || normalized.contains("principalname") + || normalized.ends_with("account") + || normalized.equals("upn") + || normalized.ends_with("upn") } fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&str>) -> bool { - let normalized_name = name.to_ascii_lowercase().replace(['_', '-'], ""); - if normalized_name.ends_with("path") || normalized_name.ends_with("filename") { + let normalized = NormalizedPropertyName(name); + if normalized.ends_with("path") || normalized.ends_with("filename") { return true; } - if normalized_name != "objectname" && normalized_name != "resource" { + if !normalized.equals("objectname") && !normalized.equals("resource") { return false; } if let Some(object_type) = object_type { diff --git a/src/core/learning_mode_core/src/analyze.rs b/src/core/learning_mode_core/src/analyze.rs index 889a1e36a..4fff94fc9 100644 --- a/src/core/learning_mode_core/src/analyze.rs +++ b/src/core/learning_mode_core/src/analyze.rs @@ -16,7 +16,7 @@ use std::path::Path; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::data_loop::DataLoopSummary; +use crate::data_loop::{DataLoopAggregate, DataLoopSummary}; use crate::model::DeniedResource; /// Result of decoding a capture source into bounded, de-duplicated denials. @@ -102,25 +102,92 @@ impl AnalysisResult { self.data_loop.move_to_overflow(aggregate); } } + + let mut retained = std::mem::take(&mut self.data_loop.signatures); + let serialized_len = |retained_count: usize| { + serialized_analysis_len( + self, + &retained[..retained_count], + &retained[retained_count..], + ) + }; + let mut low = 0usize; + let mut high = retained.len(); + while low < high { + let middle = low + (high - low).div_ceil(2); + if serialized_len(middle)? <= max_bytes { + low = middle; + } else { + high = middle - 1; + } + } + let removed = retained.split_off(low); + self.data_loop.signatures = retained; + for aggregate in removed { + self.data_loop.move_to_overflow(aggregate); + } self.data_loop .signatures .sort_by(|left, right| left.signature.cmp(&right.signature)); + Ok(serde_json::to_vec(self)?.len() <= max_bytes) + } +} - while serde_json::to_vec(self)?.len() > max_bytes { - let index = self - .data_loop - .signatures - .iter() - .rposition(|aggregate| !aggregate.signature.reason.is_canonical_denial()) - .or_else(|| self.data_loop.signatures.len().checked_sub(1)); - let Some(index) = index else { - return Ok(false); - }; - let aggregate = self.data_loop.signatures.remove(index); - self.data_loop.move_to_overflow(aggregate); - } - Ok(true) +fn serialized_analysis_len( + analysis: &AnalysisResult, + retained: &[DataLoopAggregate], + removed: &[DataLoopAggregate], +) -> Result { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct AnalysisView<'a> { + denials: &'a [DeniedResource], + denied_resources_truncated: bool, + data_loop: DataLoopView<'a>, } + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct DataLoopView<'a> { + signatures: &'a [DataLoopAggregate], + total_occurrences: u64, + overflow_occurrences: u64, + canonical_overflow_occurrences: u64, + aggregate_groups_truncated: bool, + processed_events_truncated: bool, + canonical_denial_limit_reached: bool, + } + + let removed_occurrences = removed.iter().fold(0u64, |total, aggregate| { + total.saturating_add(aggregate.count) + }); + let removed_canonical_occurrences = removed + .iter() + .filter(|aggregate| aggregate.signature.reason.is_canonical_denial()) + .fold(0u64, |total, aggregate| { + total.saturating_add(aggregate.count) + }); + let view = AnalysisView { + denials: &analysis.denials, + denied_resources_truncated: analysis.denied_resources_truncated, + data_loop: DataLoopView { + signatures: retained, + total_occurrences: analysis.data_loop.total_occurrences, + overflow_occurrences: analysis + .data_loop + .overflow_occurrences + .saturating_add(removed_occurrences), + canonical_overflow_occurrences: analysis + .data_loop + .canonical_overflow_occurrences + .saturating_add(removed_canonical_occurrences), + aggregate_groups_truncated: analysis.data_loop.aggregate_groups_truncated + || !removed.is_empty(), + processed_events_truncated: analysis.data_loop.processed_events_truncated, + canonical_denial_limit_reached: analysis.data_loop.canonical_denial_limit_reached, + }, + }; + serde_json::to_vec(&view).map(|bytes| bytes.len()) } /// Failure modes when analysing a capture source into denials. @@ -284,4 +351,42 @@ mod tests { assert!(!result.fit_data_loop_within_serialized_bytes(1).unwrap()); assert_eq!(result.denials.len(), 1); } + + #[test] + fn serialized_size_limit_handles_maximum_signature_count() { + let signatures = (0..crate::data_loop::MAX_DATA_LOOP_GROUPS) + .map(|pid| DataLoopAggregate { + signature: DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "provider".to_string(), + event_id: 14, + reason: DataLoopExclusionReason::MissingObjectName, + pid: u32::try_from(pid).unwrap(), + access_type: Some(AccessType::Read), + resource_type: Some(ResourceType::File), + properties: vec![("ObjectName".to_string(), "x".repeat(256))], + }, + count: 1, + }) + .collect(); + let mut result = AnalysisResult { + denials: Vec::new(), + denied_resources_truncated: false, + data_loop: DataLoopSummary { + signatures, + total_occurrences: crate::data_loop::MAX_DATA_LOOP_GROUPS as u64, + ..Default::default() + }, + }; + + assert!(result + .fit_data_loop_within_serialized_bytes(64 * 1024) + .unwrap()); + assert!(serde_json::to_vec(&result).unwrap().len() <= 64 * 1024); + assert!(!result.data_loop.signatures.is_empty()); + assert_eq!( + result.data_loop.signatures.len() as u64 + result.data_loop.overflow_occurrences, + crate::data_loop::MAX_DATA_LOOP_GROUPS as u64 + ); + } } diff --git a/src/core/learning_mode_core/src/lib.rs b/src/core/learning_mode_core/src/lib.rs index 679f5eee8..d93cc4dea 100644 --- a/src/core/learning_mode_core/src/lib.rs +++ b/src/core/learning_mode_core/src/lib.rs @@ -48,6 +48,7 @@ pub use data_loop::{ pub use emit::{write_document, DenialsDocument, DenialsOutputPointer}; pub use model::{AccessType, DedupKey, DeniedResource, ResourceType}; pub use paired_output::{ - relocate_paired_output_files, write_paired_output_files, ExistingOutputPolicy, + relocate_output_file, relocate_paired_output_files, write_paired_output_files, + ExistingOutputPolicy, }; pub use summary::DenialSummary; diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index 9e451c7dd..a7bd19e73 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -260,6 +260,35 @@ pub fn relocate_paired_output_files( } } +/// Copies one existing output to a new no-clobber destination, then removes +/// the source only if its identity and contents still match the copied file. +/// +/// # Errors +/// +/// Returns an error when the source cannot be read, the destination cannot be +/// committed without clobbering, or verified source cleanup fails. +pub fn relocate_output_file( + operation: &str, + kind: &str, + source_path: &Path, + destination_path: &Path, +) -> std::io::Result<()> { + if source_path == destination_path { + return Ok(()); + } + + let mut source = std::fs::File::open(source_path)?; + let source_identity = PromotedOutput::from_file(source.try_clone()?)?; + ensure_output_absent(operation, destination_path, kind)?; + let temp = stage_output_file(operation, destination_path, kind, |writer| { + source.rewind()?; + std::io::copy(&mut source, writer).map(|_| ()) + })?; + let promoted = promote_output_file(operation, temp, destination_path, kind)?; + drop((source, promoted)); + remove_promoted_output_if_owned(source_path, source_identity, None).and_then(cleanup_result) +} + fn cleanup_result(error: Option) -> std::io::Result<()> { error.map_or(Ok(()), Err) } diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index 50d6d9ddb..96781e047 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -121,7 +121,8 @@ pub fn finalize( })?; let final_denials = context.log_dir.join("denials.json"); - let final_data_loop = context.log_dir.join("denials.data-loop.json"); + let final_data_loop = learning_mode_core::data_loop_sibling_path(&final_denials) + .map_err(|error| format!("failed to derive audit Data Loop output path: {error}"))?; let final_etl = context.log_dir.join("trace.etl"); relocate_artifacts( capture, @@ -218,45 +219,19 @@ fn relocate_artifacts( } fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { - if source == destination { - return Ok(()); - } - ensure_destination_available(source, destination)?; - match std::fs::rename(source, destination) { - Ok(()) => Ok(()), - Err(rename_error) => { - if let Err(copy_error) = copy_to_new_file(source, destination) { - return Err(format!( - "failed to move audit artifact {} to {}: {rename_error}; \ - copy fallback failed: {copy_error}", - source.display(), - destination.display() - )); - } - match std::fs::remove_file(source) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(remove_error) => Err(format!( - "copied audit artifact to {}, but failed to remove source {}: {remove_error}; \ - the complete no-clobber destination was retained", - destination.display(), - source.display() - )), - } - } - } -} - -fn copy_to_new_file(source: &Path, destination: &Path) -> std::io::Result<()> { - let parent = destination.parent().unwrap_or_else(|| Path::new(".")); - let mut source_file = std::fs::File::open(source)?; - let mut staged = tempfile::NamedTempFile::new_in(parent)?; - std::io::copy(&mut source_file, &mut staged)?; - staged.as_file().sync_all()?; - staged - .persist_noclobber(destination) - .map(|_| ()) - .map_err(|error| error.error) + learning_mode_core::relocate_output_file( + "audit artifact relocation", + "retained ETL", + source, + destination, + ) + .map_err(|error| { + format!( + "failed to move audit artifact {} to {}: {error}", + source.display(), + destination.display() + ) + }) } fn ensure_destination_available(source: &Path, destination: &Path) -> Result<(), String> { @@ -466,6 +441,36 @@ mod tests { assert!(source_etl.exists()); } + #[test] + fn finalize_rejects_malformed_data_loop_without_moving_artifacts() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&log_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_data_loop = learning_mode_core::data_loop_sibling_path(&source_denials).unwrap(); + let source_etl = log_dir.join("denials.unique.etl"); + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, false)); + std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 0); + std::fs::write(&source_data_loop, b"{").unwrap(); + let context = AuditContext { + log_dir: log_dir.clone(), + config_path: None, + }; + + let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); + + assert!(error.contains("Data Loop output")); + assert!(error.contains("not valid JSON")); + assert!(source_denials.is_file()); + assert!(source_data_loop.is_file()); + assert!(source_etl.is_file()); + assert!(!log_dir.join("denials.json").exists()); + assert!(!log_dir.join("denials.data-loop.json").exists()); + assert!(!log_dir.join("trace.etl").exists()); + } + #[test] fn finalize_preflights_both_destinations_before_moving_artifacts() { let directory = tempfile::tempdir().unwrap(); @@ -495,16 +500,16 @@ mod tests { } #[test] - fn copy_fallback_never_clobbers_an_existing_destination() { + fn etl_relocation_never_clobbers_an_existing_destination() { let directory = tempfile::tempdir().unwrap(); let source = directory.path().join("source.json"); let destination = directory.path().join("destination.json"); std::fs::write(&source, b"new").unwrap(); std::fs::write(&destination, b"existing").unwrap(); - let error = copy_to_new_file(&source, &destination).unwrap_err(); + let error = move_new_file(&source, &destination).unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert!(error.contains("already exists")); assert_eq!(std::fs::read(&source).unwrap(), b"new"); assert_eq!(std::fs::read(&destination).unwrap(), b"existing"); } From e2c3aca8be62ec801e0fba78b048d86ef3f621a6 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 13:57:40 -0700 Subject: [PATCH 24/34] Redact absolute paths for non-file events Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode/windows/src/extractors.rs | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index ee5397a64..c1595b38c 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -396,8 +396,8 @@ fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&s if !normalized.equals("objectname") && !normalized.equals("resource") { return false; } - if let Some(object_type) = object_type { - return object_type.eq_ignore_ascii_case("File"); + if object_type.is_some_and(|object_type| object_type.eq_ignore_ascii_case("File")) { + return true; } let bytes = value.as_bytes(); @@ -1435,6 +1435,11 @@ mod tests { r"\Device\HarddiskVolume3\secret.txt", Some("Section") )); + assert!(looks_like_file_path_property( + "ObjectName", + r"C:\Users\alice\secret.txt", + Some("Section") + )); assert!(!looks_like_file_path_property( "ObjectName", r"\BaseNamedObjects\shared-cache", @@ -1537,6 +1542,22 @@ mod tests { assert_eq!(value_for("ObjectName"), Some(REDACTED_PATH)); } + #[test] + fn sanitize_properties_redacts_absolute_path_despite_non_file_object_type() { + let props = vec![ + ("ObjectType".to_string(), "\"Section\"".to_string()), + ( + "ObjectName".to_string(), + "\"C:\\Users\\alice\\secret.txt\"".to_string(), + ), + ]; + + let out = sanitize_properties(&props); + let value_for = |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); + assert_eq!(value_for("ObjectName"), Some(REDACTED_PATH)); + assert_eq!(value_for("ObjectType"), Some("Section")); + } + #[test] fn sanitize_properties_still_redacts_username_in_non_file_identifier() { let props = vec![ From b6cbbe1b743a042e8cb981dbf686d0132becf88e Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 14:10:44 -0700 Subject: [PATCH 25/34] Redact NT filesystem paths in diagnostics Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode/windows/src/extractors.rs | 73 ++++++++++++++++--- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index c1595b38c..49c95fa23 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -410,6 +410,37 @@ fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&s || value.starts_with(r"\??\") || value.starts_with(r"\\?\") || value.starts_with(r"\\.\") + || looks_like_nt_filesystem_path(value) +} + +fn looks_like_nt_filesystem_path(value: &str) -> bool { + let Some(rest) = strip_prefix_ignore_ascii_case(value, r"\Device\") else { + return false; + }; + let Some((device, _path)) = rest.split_once('\\') else { + return false; + }; + if device.eq_ignore_ascii_case("Mup") || ends_with_ignore_ascii_case(device, "Redirector") { + return true; + } + + let Some(volume_number) = strip_prefix_ignore_ascii_case(device, "HarddiskVolume") else { + return false; + }; + !volume_number.is_empty() && volume_number.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn strip_prefix_ignore_ascii_case<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { + value + .get(..prefix.len()) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix)) + .then(|| &value[prefix.len()..]) +} + +fn ends_with_ignore_ascii_case(value: &str, suffix: &str) -> bool { + value + .get(value.len().saturating_sub(suffix.len())..) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(suffix)) } /// Bounds an already-sanitized property list to [`MAX_SIGNATURE_PROPERTIES`] @@ -1440,6 +1471,17 @@ mod tests { r"C:\Users\alice\secret.txt", Some("Section") )); + for path in [ + r"\Device\HarddiskVolume3\Users\alice\secret.txt", + r"\device\mup\server\share\Users\alice\secret.txt", + r"\Device\LanmanRedirector\;Z:0000000000001234\server\share\secret.txt", + ] { + assert!(looks_like_file_path_property( + "ObjectName", + path, + Some("Section") + )); + } assert!(!looks_like_file_path_property( "ObjectName", r"\BaseNamedObjects\shared-cache", @@ -1455,6 +1497,11 @@ mod tests { "internetClient", None )); + assert!(!looks_like_file_path_property( + "ObjectName", + r"\Device\MountPointManager", + Some("Section") + )); } #[test] @@ -1544,18 +1591,22 @@ mod tests { #[test] fn sanitize_properties_redacts_absolute_path_despite_non_file_object_type() { - let props = vec![ - ("ObjectType".to_string(), "\"Section\"".to_string()), - ( - "ObjectName".to_string(), - "\"C:\\Users\\alice\\secret.txt\"".to_string(), - ), - ]; + for path in [ + r"C:\Users\alice\secret.txt", + r"\Device\HarddiskVolume3\Users\alice\secret.txt", + r"\Device\Mup\server\share\Users\alice\secret.txt", + ] { + let props = vec![ + ("ObjectType".to_string(), "\"Section\"".to_string()), + ("ObjectName".to_string(), format!("\"{path}\"")), + ]; - let out = sanitize_properties(&props); - let value_for = |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); - assert_eq!(value_for("ObjectName"), Some(REDACTED_PATH)); - assert_eq!(value_for("ObjectType"), Some("Section")); + let out = sanitize_properties(&props); + let value_for = + |name: &str| out.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()); + assert_eq!(value_for("ObjectName"), Some(REDACTED_PATH)); + assert_eq!(value_for("ObjectType"), Some("Section")); + } } #[test] From c2e2d721771856deb413832106ec37cb84251ebb Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 14:21:17 -0700 Subject: [PATCH 26/34] Preserve signatures when size fit fails Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- src/core/learning_mode_core/src/analyze.rs | 50 ++++++++++++++++------ 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/src/core/learning_mode_core/src/analyze.rs b/src/core/learning_mode_core/src/analyze.rs index 4fff94fc9..834008a4b 100644 --- a/src/core/learning_mode_core/src/analyze.rs +++ b/src/core/learning_mode_core/src/analyze.rs @@ -81,13 +81,14 @@ impl AnalysisResult { return Ok(true); } - let original = std::mem::take(&mut self.data_loop.signatures); - let mut retained_bytes = 0usize; - const ENVELOPE_HEADROOM: usize = 4 * 1024; - let base_len = serde_json::to_vec(self)?.len(); + let base_len = serialized_analysis_len(self, &[], self.data_loop.signatures.as_slice())?; if base_len > max_bytes { return Ok(false); } + + let original = std::mem::take(&mut self.data_loop.signatures); + let mut retained_bytes = 0usize; + const ENVELOPE_HEADROOM: usize = 4 * 1024; let signature_budget = max_bytes .saturating_sub(base_len) .saturating_sub(ENVELOPE_HEADROOM); @@ -340,16 +341,41 @@ mod tests { #[test] fn serialized_size_limit_never_discards_canonical_denials() { - let mut result = AnalysisResult::complete(vec![DeniedResource { - resource: "canonical".repeat(512), - resource_type: ResourceType::File, - access_type: AccessType::Read, - pid: 1, - filetime: 2, - }]); + let mut result = AnalysisResult { + denials: vec![DeniedResource { + resource: "canonical".repeat(512), + resource_type: ResourceType::File, + access_type: AccessType::Read, + pid: 1, + filetime: 2, + }], + denied_resources_truncated: false, + data_loop: DataLoopSummary { + signatures: vec![DataLoopAggregate { + signature: DataLoopSignature { + provider: DataLoopProvider::KernelGeneral, + provider_guid: "provider".to_string(), + event_id: 14, + reason: DataLoopExclusionReason::CanonicalDenial, + pid: 1, + access_type: Some(AccessType::Read), + resource_type: Some(ResourceType::File), + properties: vec![("ObjectName".to_string(), "value".to_string())], + }, + count: 1, + }], + total_occurrences: 1, + ..Default::default() + }, + }; + let original = result.clone(); assert!(!result.fit_data_loop_within_serialized_bytes(1).unwrap()); - assert_eq!(result.denials.len(), 1); + assert_eq!(result, original); + assert!(result + .fit_data_loop_within_serialized_bytes(serde_json::to_vec(&original).unwrap().len()) + .unwrap()); + assert_eq!(result, original); } #[test] From 344acefd6e40a2cde71be8ae4e594fa579f47a20 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Tue, 18 Aug 2026 14:35:50 -0700 Subject: [PATCH 27/34] Distinguish DOS paths from device identifiers Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode/windows/src/extractors.rs | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index 49c95fa23..5a559bfa6 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -400,17 +400,36 @@ fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&s return true; } + crate::path_norm::is_user_visible_absolute(value) + || looks_like_dos_device_filesystem_path(value) + || looks_like_nt_filesystem_path(value) +} + +fn looks_like_dos_device_filesystem_path(value: &str) -> bool { + let rest = [r"\??\", r"\\?\", r"\\.\"] + .into_iter() + .find_map(|prefix| strip_prefix_ignore_ascii_case(value, prefix)); + let Some(rest) = rest else { + return false; + }; + if looks_like_drive_absolute_path(rest) + || strip_prefix_ignore_ascii_case(rest, r"UNC\").is_some() + { + return true; + } + + let Some(volume) = strip_prefix_ignore_ascii_case(rest, "Volume{") else { + return false; + }; + volume.contains(r"}\") +} + +fn looks_like_drive_absolute_path(value: &str) -> bool { let bytes = value.as_bytes(); - let drive_absolute = bytes.len() >= 3 + bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' - && matches!(bytes[2], b'\\' | b'/'); - drive_absolute - || value.starts_with(r"\\") - || value.starts_with(r"\??\") - || value.starts_with(r"\\?\") - || value.starts_with(r"\\.\") - || looks_like_nt_filesystem_path(value) + && matches!(bytes[2], b'\\' | b'/') } fn looks_like_nt_filesystem_path(value: &str) -> bool { @@ -1475,6 +1494,8 @@ mod tests { r"\Device\HarddiskVolume3\Users\alice\secret.txt", r"\device\mup\server\share\Users\alice\secret.txt", r"\Device\LanmanRedirector\;Z:0000000000001234\server\share\secret.txt", + r"\??\C:\Users\alice\secret.txt", + r"\??\UNC\server\share\Users\alice\secret.txt", ] { assert!(looks_like_file_path_property( "ObjectName", @@ -1502,6 +1523,13 @@ mod tests { r"\Device\MountPointManager", Some("Section") )); + for identifier in [r"\??\FDC#GENERIC_FLOPPY_DRIVE", r"\\.\PhysicalDrive0"] { + assert!(!looks_like_file_path_property( + "ObjectName", + identifier, + Some("SymbolicLink") + )); + } } #[test] From 486f8ce21aa59e20b9f70cf38a339572a1cb2674 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Wed, 19 Aug 2026 09:47:26 -0700 Subject: [PATCH 28/34] Decode additional learning mode denial shapes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- docs/learning-mode/capabilities.md | 12 +- .../learning_mode/windows/src/etl_decode.rs | 70 +++++++- .../learning_mode/windows/src/extractors.rs | 159 +++++++++++++++--- 3 files changed, 208 insertions(+), 33 deletions(-) diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index 0d8220ff4..c708b8c15 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -210,9 +210,12 @@ sandbox policy: - `resource` is the user-visible identifier for the denied resource, interpreted by `resourceType`: a canonical `C:\…` path for `file`, the AppContainer **capability name** (e.g. `internetClient`) for `capability`, - and the raw resource identifier otherwise. Well-known capability SIDs are + and the raw resource identifier otherwise. Named Section, SymbolicLink, and + Timer access checks are emitted as `other`. Well-known capability SIDs are resolved to their policy name; custom (hashed) capability SIDs that can't be - reversed fall back to the `S-1-15-3-…` SID string. + reversed fall back to the `S-1-15-3-…` SID string. Event 28 is + schema-discriminated: UI-shaped `Category`/`Detail` payloads emit `ui` + resources instead of treating the package SID as a capability. - `resourceType` is one of `file`, `ui`, `network`, `capability`, `other`; `accessType` is one of `read`, `write`, `execute`, `unknown`. Capability denials are recorded under `block`; current `allow` traces expose capability @@ -284,8 +287,9 @@ reason and their sanitized event properties: `\Device\MountPointManager` is useful Devices-namespace evidence, but it is not a directly authorable filesystem grant. - `unsupportedObjectType` means the event names a resource outside the - canonical policy model. Examples include `\BaseNamedObjects` as a Directory, - shared-cache Sections, SymbolicLinks, ALPC Ports such as + canonical policy model. Observed Section, SymbolicLink, and Timer checks are + canonical `other` resources; remaining examples include + `\BaseNamedObjects` as a Directory, ALPC Ports such as `ubpmtaskhostchannel`, and RPC Interface GUIDs. Property values longer than 256 characters retain bounded prefix and suffix diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index ec21a2c07..a14f0f3b6 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -1926,7 +1926,7 @@ mod tests { #[test] fn non_actionable_events_are_dropped() { let events = vec![ - kernel_event(14, 1, 1, &[("ObjectType", "\"Section\"")]), + kernel_event(14, 1, 1, &[("ObjectType", "\"Process\"")]), kernel_event(28, 0, 2, &[("ProcessId", "0x10"), ("Denied", "false")]), kernel_event(9999, 1, 3, &[("Foo", "\"bar\"")]), ]; @@ -1991,11 +1991,6 @@ mod tests { fn output_gap_object_types_are_individually_retained() { let cases = [ ("Directory", r"\BaseNamedObjects"), - ( - "Section", - r"\Sessions\1\AppContainerNamedObjects\S-1-15-2-1\cache.db", - ), - ("SymbolicLink", r"\??\FDC#GENERIC_FLOPPY_DRIVE"), ( "ALPC Port", r"\Sessions\1\AppContainerNamedObjects\S-1-15-2-1\RPC Control\ubpmtaskhostchannel", @@ -2072,15 +2067,74 @@ mod tests { let out = resources_from_events(&events); - assert!(out.denials.is_empty()); + assert_eq!(out.denials.len(), names.len()); + assert!(out.denials.iter().all(|denial| { + denial.resource_type == ResourceType::Other + && denial.access_type == AccessType::Write + && names.contains(&denial.resource) + })); assert_eq!(out.data_loop.signatures.len(), names.len()); assert!(out.data_loop.signatures.iter().all(|group| { - group.signature.reason == DataLoopExclusionReason::UnsupportedObjectType + group.signature.reason == DataLoopExclusionReason::CanonicalDenial && group.count == 1 && property(&group.signature, "ObjectName").contains(" { build_denial_from_learning_mode(parts, pid, filetime, provider) } + CAPABILITY_DENIAL_EVENT_ID if is_ui_violation_schema(parts) => { + build_denial_from_learning_mode(parts, pid, filetime, provider) + } CAPABILITY_DENIAL_EVENT_ID => build_denial_from_capability(parts, pid, filetime, provider), _ => Err(DataLoopExclusionReason::UnsupportedEventSchema), } @@ -178,11 +181,25 @@ pub(crate) fn data_loop_classification( ), Some(ResourceType::Other), ), + "Section" | "SymbolicLink" | "Timer" => ( + Some( + find_prop(&parts.props, "AccessMask") + .and_then(|value| parse_u32(value)) + .map(|mask| { + named_object_access_type(object_type.trim_matches('"'), mask) + }) + .unwrap_or(AccessType::Unknown), + ), + Some(ResourceType::Other), + ), "" => (Some(AccessType::Unknown), Some(ResourceType::Capability)), _ => (None, None), } } LEARNING_MODE_VIOLATION_EVENT_ID => (Some(AccessType::Unknown), Some(ResourceType::Ui)), + CAPABILITY_DENIAL_EVENT_ID if is_ui_violation_schema(parts) => { + (Some(AccessType::Unknown), Some(ResourceType::Ui)) + } CAPABILITY_DENIAL_EVENT_ID => (Some(AccessType::Unknown), Some(ResourceType::Capability)), _ => (None, None), } @@ -217,6 +234,12 @@ pub(crate) fn effective_event_pid(parts: &DecodedEventParts, header_pid: u32) -> } } +fn is_ui_violation_schema(parts: &DecodedEventParts) -> bool { + parts.event_id == CAPABILITY_DENIAL_EVENT_ID + && find_prop(&parts.props, "Detail").is_some() + && find_prop(&parts.props, "Category").is_some() +} + pub(crate) fn effective_capability_event_pid(process_id: Option<&str>) -> Option { process_id.and_then(parse_u32) } @@ -582,9 +605,10 @@ pub(crate) fn sanitize_properties(props: &[(String, String)]) -> Vec<(String, St /// /// The `ObjectType` field selects the resource type: `File` and `Key` /// (registry) map to concrete resources, an **empty** `ObjectType` is a -/// brokered-capability check, and any other object type (Section, Process, -/// Thread, ...) is dropped as not actionable via sandbox policy. An absent -/// `ObjectType` field drops the event. +/// brokered-capability check, and the observed named-object types `Section`, +/// `SymbolicLink`, and `Timer` map to [`ResourceType::Other`]. Other object +/// types are dropped until their access-mask vocabulary is understood. An +/// absent `ObjectType` field drops the event. /// /// For file/registry resources the [`AccessType`] is derived from the /// event's `AccessMask` field (the desired access the caller was denied; @@ -618,6 +642,7 @@ pub fn build_denial_from_access_check( let resource_type = match object_type_str { "File" => ResourceType::File, "Key" => ResourceType::Other, + "Section" | "SymbolicLink" | "Timer" => ResourceType::Other, // A present-but-empty object type is a brokered-capability check. "" => ResourceType::Capability, _ => return Err(DataLoopExclusionReason::UnsupportedObjectType), @@ -651,13 +676,13 @@ pub fn build_denial_from_access_check( // classifier over it. AccessType::Unknown } else { - // Registry keys and files share the standard/generic mask bits but - // assign different meanings to the object-specific low bits, so the - // classifier needs to know which vocabulary applies. - let is_registry = object_type_str == "Key"; find_prop(&parts.props, "AccessMask") .and_then(|v| parse_u32(v)) - .map(|mask| access_type_from_mask(mask, is_registry)) + .map(|mask| match object_type_str { + "Key" => access_type_from_mask(mask, true), + "File" => access_type_from_mask(mask, false), + _ => named_object_access_type(object_type_str, mask), + }) .unwrap_or(AccessType::Unknown) }; @@ -929,6 +954,37 @@ fn access_type_from_mask(mask: u32, is_registry: bool) -> AccessType { } } +fn named_object_access_type(object_type: &str, mask: u32) -> AccessType { + let (read_bits, write_bits, execute_bits) = match object_type { + // ntifs.h SECTION_* rights. + "Section" => (0x0001 | 0x0004, 0x0002 | 0x0010, 0x0008 | 0x0020), + // ntifs.h SYMBOLIC_LINK_* rights. + "SymbolicLink" => (0x0001, 0x0002, 0), + // winnt.h TIMER_* rights. + "Timer" => (0x0001, 0x0002, 0), + _ => return AccessType::Unknown, + }; + + // Standard and generic rights are object-type independent. + const DELETE: u32 = 0x0001_0000; + const WRITE_DAC: u32 = 0x0004_0000; + const WRITE_OWNER: u32 = 0x0008_0000; + const GENERIC_READ: u32 = 0x8000_0000; + const GENERIC_WRITE: u32 = 0x4000_0000; + const GENERIC_EXECUTE: u32 = 0x2000_0000; + const GENERIC_ALL: u32 = 0x1000_0000; + + if mask & (write_bits | DELETE | WRITE_DAC | WRITE_OWNER | GENERIC_WRITE | GENERIC_ALL) != 0 { + AccessType::Write + } else if mask & (execute_bits | GENERIC_EXECUTE) != 0 { + AccessType::Execute + } else if mask & (read_bits | GENERIC_READ) != 0 { + AccessType::Read + } else { + AccessType::Unknown + } +} + fn find_prop<'a>(props: &'a [(String, String)], name: &str) -> Option<&'a String> { props.iter().find(|(k, _)| k == name).map(|(_, v)| v) } @@ -1203,8 +1259,43 @@ mod tests { } #[test] - fn access_check_unknown_object_type_dropped() { - let p = parts(14, &[("ObjectType", "\"Section\"")]); + fn access_check_observed_named_object_types_are_retained() { + for (object_type, object_name, mask, expected_access) in [ + ( + "Section", + r"\Sessions\1\BaseNamedObjects\shared-section", + "0x6", + AccessType::Write, + ), + ("SymbolicLink", r"\??\C:", "0x1", AccessType::Read), + ( + "Timer", + r"\Sessions\1\BaseNamedObjects\deadline", + "0x1", + AccessType::Read, + ), + ] { + let p = parts( + 14, + &[ + ("ObjectType", object_type), + ("ObjectName", object_name), + ("AccessMask", mask), + ], + ); + let denial = extract_denial(&p, 1, FIXED_FILETIME).unwrap(); + assert_eq!(denial.resource_type, ResourceType::Other); + assert_eq!(denial.object_name, object_name); + assert_eq!(denial.access_type, expected_access); + } + } + + #[test] + fn access_check_unrecognized_object_type_is_dropped() { + let p = parts( + 14, + &[("ObjectType", "\"Process\""), ("ObjectName", "\"target\"")], + ); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), DataLoopExclusionReason::UnsupportedObjectType @@ -1342,6 +1433,32 @@ mod tests { assert_eq!(ev.provider, DataLoopProvider::KernelGeneral); } + #[test] + fn event_28_ui_violation_is_not_misclassified_as_capability() { + let p = parts( + 28, + &[ + ("ProcessName", "\"powershell.exe\""), + ("ProcessId", "0x1594"), + ("SequenceNumber", "302"), + ("Category", "2"), + ("Detail", "1"), + ("Denied", "false"), + ("UserSid", "S-1-5-21-1-2-3-1000"), + ("PackageSid", "S-1-15-2-1-2-3-4-5-6-7"), + ], + ); + + let denial = extract_denial(&p, 0x1594, FIXED_FILETIME).unwrap(); + assert_eq!(denial.resource_type, ResourceType::Ui); + assert_eq!(denial.object_name, "Handles"); + assert_eq!(denial.pid, 0x1594); + assert_eq!( + data_loop_classification(&p), + (Some(AccessType::Unknown), Some(ResourceType::Ui)) + ); + } + #[test] fn capability_denial_resolves_well_known_sid_to_name() { // PackageSid rendered by the TDH SID decoder as a legacy capability From afaf37afca20becdd121e02dfbaf4604c5e55d74 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Wed, 19 Aug 2026 10:02:10 -0700 Subject: [PATCH 29/34] Rename Data Loop to verbose logging Replace the public and internal Data Loop terminology with verbose logging, including the serialized analysis field and sibling artifact filename. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode_core/src/{data_loop.rs => verbose_logging.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/core/learning_mode_core/src/{data_loop.rs => verbose_logging.rs} (100%) diff --git a/src/core/learning_mode_core/src/data_loop.rs b/src/core/learning_mode_core/src/verbose_logging.rs similarity index 100% rename from src/core/learning_mode_core/src/data_loop.rs rename to src/core/learning_mode_core/src/verbose_logging.rs From 4151adce2d674d276e42013386cfda479dec1c81 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Wed, 19 Aug 2026 10:02:26 -0700 Subject: [PATCH 30/34] Use verbose logging terminology Replace Data Loop names in APIs, transport fields, artifact paths, diagnostics, tests, and documentation with verbose logging terminology. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- docs/learning-mode/capabilities.md | 24 +- .../appcontainer/common/src/capture_output.rs | 84 ++--- .../windows/src/capability_dacl.rs | 2 +- .../learning_mode/windows/src/etl_decode.rs | 307 ++++++++++-------- .../learning_mode/windows/src/extractors.rs | 162 ++++----- src/core/learning_mode_core/src/analyze.rs | 127 ++++---- src/core/learning_mode_core/src/lib.rs | 17 +- .../learning_mode_core/src/paired_output.rs | 150 +++++---- .../learning_mode_core/src/verbose_logging.rs | 164 +++++----- src/core/wxc/src/audit.rs | 130 ++++---- src/host/plm/src/analysis.rs | 34 +- src/host/plm/src/log.rs | 2 +- src/host/plm/src/stop.rs | 16 +- 13 files changed, 643 insertions(+), 576 deletions(-) diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index c708b8c15..6ad1ae277 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -105,8 +105,8 @@ stays enforced: hosts use native capture without PLM or UAC, while legacy or incompatible tiers use the session-scoped guarded-WPR fallback and elevate only its fixed-operation guardian. The CLI consumes the returned JSON and ETL paths, - relocates the canonical output, its Data Loop sibling, and the trace to - `denials.json`, `denials.data-loop.json`, and `trace.etl`, and generates the + relocates the canonical output, its verbose logging sibling, and the trace to + `denials.json`, `denials.verbose.json`, and `trace.etl`, and generates the source snapshot and `Adjusted_*.json` from canonical denials without decoding ETL again. Truncated analysis skips the adjusted config. @@ -224,12 +224,12 @@ sandbox policy: - `filetime` is a decimal string containing the Windows `FILETIME` value, so JavaScript consumers retain all 64 bits without numeric precision loss. -### Data Loop event signatures +### Verbose logging event signatures Every successful decode also writes a deterministic sibling file: -`denials..json` produces `denials..data-loop.json`. This Data -Loop artifact is a bounded, sensitive-value-redacted superset containing canonical -denial occurrences plus outcomes omitted from canonical denials: +`denials..json` produces `denials..verbose.json`. This verbose +logging artifact is a bounded, sensitive-value-redacted superset containing +canonical denial occurrences plus outcomes omitted from canonical denials: ```json { @@ -309,9 +309,9 @@ decoding, and `unsupportedPropertyEncoding` means the decoder cannot consume that property shape. When TDH exposes it, the schema-declared name is retained as the bounded `EventName` signature property. Free-form decoder errors are never serialized. Failure to obtain the event schema remains a fatal analysis -error rather than being represented as a Data Loop signature. +error rather than being represented as a verbose logging signature. -Data Loop retains at most 4,096 distinct signatures, 24 sorted properties per +Verbose logging retains at most 4,096 distinct signatures, 24 sorted properties per signature, 256 characters per property value, and 16 MiB of compact serialized signature data. Canonical-denial signatures take priority over diagnostic signatures at these bounds. Additional keys are collapsed into @@ -319,15 +319,15 @@ signatures at these bounds. Additional keys are collapsed into canonical occurrences that could not be represented even after diagnostic groups were evicted, and `aggregateGroupsTruncated` is set. Before returning analysis, the decoder also fits the complete canonical plus -Data Loop result within the guarded 64 MiB transport limit by moving additional -Data Loop groups into overflow; canonical denials are never discarded. +verbose logging result within the guarded 64 MiB transport limit by moving additional +verbose logging groups into overflow; canonical denials are never discarded. `processedEventsTruncated` indicates that the 1,000,000-event bound prevented complete accounting. Counts are candidate-level when extraction identifies individual denial candidates and one event-level outcome when decoding cannot determine candidate cardinality. -The canonical and Data Loop files fail together: MXC stages both and reports -capture failure unless both final artifacts are committed. The Data Loop path +The canonical and verbose logging files fail together: MXC stages both and reports +capture failure unless both final artifacts are committed. The verbose logging path is intentionally absent from stderr pointers and Rust, Node, C#, and FFI output metadata; callers derive it from the canonical path using the naming rule above. diff --git a/src/backends/appcontainer/common/src/capture_output.rs b/src/backends/appcontainer/common/src/capture_output.rs index 71773fac0..ed60aae41 100644 --- a/src/backends/appcontainer/common/src/capture_output.rs +++ b/src/backends/appcontainer/common/src/capture_output.rs @@ -7,7 +7,7 @@ //! decoding its own sealed ETL) and the guarded-WPR legacy-tier fallback //! (`appcontainer_runner`, consuming an already-decoded [`AnalysisResult`] //! handed back by the elevated PLM guardian) must emit byte-for-byte the same -//! [`DenialsDocument`] JSON shape, username-redacted Data Loop sibling, +//! [`DenialsDocument`] JSON shape, username-redacted verbose logging sibling, //! [`CaptureDenialsOutput`] summary, and resolved output-path convention. //! Centralizing that here is what guarantees the two paths can't drift. //! @@ -19,9 +19,9 @@ use std::path::{Path, PathBuf}; use learning_mode_core::{ - data_loop_sibling_path, write_data_loop_document, write_document, write_paired_output_files, - AnalysisResult, DataLoopDocument, DenialSummary, DenialsDocument, DenialsOutputPointer, - ExistingOutputPolicy, + verbose_logging_sibling_path, write_document, write_paired_output_files, + write_verbose_logging_document, AnalysisResult, DenialSummary, DenialsDocument, + DenialsOutputPointer, ExistingOutputPolicy, VerboseLoggingDocument, }; use wxc_common::models::CaptureDenialsOutput; @@ -51,7 +51,7 @@ pub struct DenialsOutputPaths { } /// Writes a bounded [`AnalysisResult`] as the canonical denials document and -/// deterministic Data Loop sibling, then returns caller-facing metadata for +/// deterministic verbose logging sibling, then returns caller-facing metadata for /// the canonical document. /// /// Never overwrites an existing file: a run whose output path collides with a @@ -61,22 +61,22 @@ pub fn write_denials_document( exit_code: i32, output_path: &Path, ) -> std::io::Result { - let data_loop_path = data_loop_output_path(output_path)?; + let verbose_logging_path = verbose_logging_output_path(output_path)?; let summary = DenialSummary::new( exit_code, analysis.denials.len(), analysis.denied_resources_truncated, ); let document = DenialsDocument::new(analysis.denials, summary); - let data_loop_document = DataLoopDocument::new(&analysis.data_loop); + let verbose_logging_document = VerboseLoggingDocument::new(&analysis.verbose_logging); write_paired_output_files( "captureDenials", output_path, - &data_loop_path, + &verbose_logging_path, ExistingOutputPolicy::CreateNew, |writer| write_document(writer, &document), - |writer| write_data_loop_document(writer, &data_loop_document), + |writer| write_verbose_logging_document(writer, &verbose_logging_document), )?; let pointer = DenialsOutputPointer::new(output_path.to_string_lossy(), &document.summary); @@ -90,9 +90,9 @@ pub fn write_denials_document( }) } -/// Derives the deterministic Data Loop sibling path for a denials output. -pub fn data_loop_output_path(output_path: &Path) -> std::io::Result { - data_loop_sibling_path(output_path) +/// Derives the deterministic verbose logging sibling path for a denials output. +pub fn verbose_logging_output_path(output_path: &Path) -> std::io::Result { + verbose_logging_sibling_path(output_path) .map_err(|error| std::io::Error::other(format!("captureDenials {error}"))) } @@ -282,8 +282,8 @@ pub fn write_stderr_line_best_effort(message: std::fmt::Arguments<'_>) { mod tests { use super::*; use learning_mode_core::{ - AccessType, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, DeniedResource, - ResourceType, + AccessType, DeniedResource, ResourceType, VerboseLoggingExclusionReason, + VerboseLoggingProvider, VerboseLoggingSignature, }; use std::io::Write; @@ -309,11 +309,11 @@ mod tests { let document: DenialsDocument = serde_json::from_slice(&std::fs::read(&output_path).unwrap()).unwrap(); assert_eq!(document.denials.len(), 1); - let data_loop: DataLoopDocument = serde_json::from_slice( - &std::fs::read(data_loop_output_path(&output_path).unwrap()).unwrap(), + let verbose_logging: VerboseLoggingDocument = serde_json::from_slice( + &std::fs::read(verbose_logging_output_path(&output_path).unwrap()).unwrap(), ) .unwrap(); - assert_eq!(data_loop.version, DataLoopDocument::VERSION); + assert_eq!(verbose_logging.version, VerboseLoggingDocument::VERSION); } #[test] @@ -327,19 +327,19 @@ mod tests { assert_eq!(metadata.total_denials, 0); assert!(output_path.exists()); - assert!(data_loop_output_path(&output_path).unwrap().exists()); + assert!(verbose_logging_output_path(&output_path).unwrap().exists()); } #[test] - fn write_denials_document_writes_data_loop_aggregates() { + fn write_denials_document_writes_verbose_logging_aggregates() { let directory = tempfile::tempdir().expect("temp directory"); let output_path = directory.path().join("denials.json"); let mut analysis = AnalysisResult::complete(Vec::new()); - analysis.data_loop.record(DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + analysis.verbose_logging.record(VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}".to_string(), event_id: 14, - reason: DataLoopExclusionReason::CanonicalDenial, + reason: VerboseLoggingExclusionReason::CanonicalDenial, pid: 42, access_type: Some(learning_mode_core::AccessType::Read), resource_type: Some(learning_mode_core::ResourceType::File), @@ -348,63 +348,65 @@ mod tests { write_denials_document(analysis, 0, &output_path).expect("write should succeed"); - let data_loop: DataLoopDocument = serde_json::from_slice( - &std::fs::read(data_loop_output_path(&output_path).unwrap()).unwrap(), + let verbose_logging: VerboseLoggingDocument = serde_json::from_slice( + &std::fs::read(verbose_logging_output_path(&output_path).unwrap()).unwrap(), ) .unwrap(); - assert_eq!(data_loop.signatures.len(), 1); - assert_eq!(data_loop.signatures[0].count, 1); - assert_eq!(data_loop.signatures[0].signature.pid, 42); + assert_eq!(verbose_logging.signatures.len(), 1); + assert_eq!(verbose_logging.signatures[0].count, 1); + assert_eq!(verbose_logging.signatures[0].signature.pid, 42); } #[test] - fn data_loop_path_is_a_deterministic_json_sibling() { + fn verbose_logging_path_is_a_deterministic_json_sibling() { assert_eq!( - data_loop_output_path(Path::new(r"C:\out\denials.123.json")).unwrap(), - PathBuf::from(r"C:\out\denials.123.data-loop.json") + verbose_logging_output_path(Path::new(r"C:\out\denials.123.json")).unwrap(), + PathBuf::from(r"C:\out\denials.123.verbose.json") ); assert_eq!( - data_loop_output_path(Path::new("denials")).unwrap(), - PathBuf::from("denials.data-loop.json") + verbose_logging_output_path(Path::new("denials")).unwrap(), + PathBuf::from("denials.verbose.json") ); } #[test] - fn data_loop_collision_leaves_canonical_output_absent() { + fn verbose_logging_collision_leaves_canonical_output_absent() { let directory = tempfile::tempdir().expect("temp directory"); let output_path = directory.path().join("denials.json"); - let data_loop_path = data_loop_output_path(&output_path).unwrap(); - std::fs::write(&data_loop_path, b"existing").expect("seed Data Loop output"); + let verbose_logging_path = verbose_logging_output_path(&output_path).unwrap(); + std::fs::write(&verbose_logging_path, b"existing").expect("seed verbose logging output"); write_denials_document(AnalysisResult::complete(Vec::new()), 0, &output_path) .expect_err("collision should fail"); assert!(!output_path.exists()); - assert_eq!(std::fs::read(data_loop_path).unwrap(), b"existing"); + assert_eq!(std::fs::read(verbose_logging_path).unwrap(), b"existing"); } #[test] fn second_staged_write_failure_leaves_no_final_output() { let directory = tempfile::tempdir().expect("temp directory"); let output_path = directory.path().join("denials.json"); - let data_loop_path = data_loop_output_path(&output_path).unwrap(); + let verbose_logging_path = verbose_logging_output_path(&output_path).unwrap(); let error = write_paired_output_files( "captureDenials", &output_path, - &data_loop_path, + &verbose_logging_path, ExistingOutputPolicy::CreateNew, |writer| writer.write_all(b"denials"), |writer| { writer.write_all(b"partial")?; - Err(std::io::Error::other("simulated Data Loop failure")) + Err(std::io::Error::other("simulated verbose logging failure")) }, ) .expect_err("paired write should fail"); - assert!(error.to_string().contains("simulated Data Loop failure")); + assert!(error + .to_string() + .contains("simulated verbose logging failure")); assert!(!output_path.exists()); - assert!(!data_loop_path.exists()); + assert!(!verbose_logging_path.exists()); } #[test] diff --git a/src/backends/learning_mode/windows/src/capability_dacl.rs b/src/backends/learning_mode/windows/src/capability_dacl.rs index 0fb9c90b2..91dab0565 100644 --- a/src/backends/learning_mode/windows/src/capability_dacl.rs +++ b/src/backends/learning_mode/windows/src/capability_dacl.rs @@ -269,7 +269,7 @@ pub(crate) fn extract_denials( filetime, event_id: parts.event_id, provider, - data_loop_properties: crate::extractors::sanitize_properties(&parts.props), + verbose_logging_properties: crate::extractors::sanitize_properties(&parts.props), }) .collect() } diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index a14f0f3b6..708864d8a 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -29,9 +29,9 @@ use std::os::windows::ffi::OsStrExt; use std::path::Path; use learning_mode_core::{ - AnalysisResult, AnalyzeError, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, - DataLoopSummary, DenialAnalyzer, DeniedResource, ProcessLifetime, - MAX_DATA_LOOP_SIGNATURE_BYTES, + AnalysisResult, AnalyzeError, DenialAnalyzer, DeniedResource, ProcessLifetime, + VerboseLoggingExclusionReason, VerboseLoggingProvider, VerboseLoggingSignature, + VerboseLoggingSummary, MAX_VERBOSE_LOGGING_SIGNATURE_BYTES, }; use windows::core::PWSTR; use windows::Win32::System::Diagnostics::Etw::{ @@ -142,8 +142,8 @@ struct Accumulator<'visitor> { decode_error: Option, panic_payload: Option>, schema_cache: tdh_decode::EventSchemaCache, - data_loop: DataLoopSummary, - data_loop_signature_bytes: usize, + verbose_logging: VerboseLoggingSummary, + verbose_logging_signature_bytes: usize, } impl<'visitor> Accumulator<'visitor> { @@ -165,8 +165,8 @@ impl<'visitor> Accumulator<'visitor> { decode_error: None, panic_payload: None, schema_cache: tdh_decode::EventSchemaCache::default(), - data_loop: DataLoopSummary::default(), - data_loop_signature_bytes: 0, + verbose_logging: VerboseLoggingSummary::default(), + verbose_logging_signature_bytes: 0, } } @@ -195,8 +195,8 @@ impl<'visitor> Accumulator<'visitor> { decode_error: None, panic_payload: None, schema_cache: tdh_decode::EventSchemaCache::default(), - data_loop: DataLoopSummary::default(), - data_loop_signature_bytes: 0, + verbose_logging: VerboseLoggingSummary::default(), + verbose_logging_signature_bytes: 0, } } @@ -218,8 +218,8 @@ impl<'visitor> Accumulator<'visitor> { decode_error: None, panic_payload: None, schema_cache: tdh_decode::EventSchemaCache::default(), - data_loop: DataLoopSummary::default(), - data_loop_signature_bytes: 0, + verbose_logging: VerboseLoggingSummary::default(), + verbose_logging_signature_bytes: 0, } } @@ -234,7 +234,7 @@ impl<'visitor> Accumulator<'visitor> { self.record_raw_denial_outcome( &raw, &candidate, - DataLoopExclusionReason::UnusableResourcePath, + VerboseLoggingExclusionReason::UnusableResourcePath, ); return; } @@ -246,7 +246,7 @@ impl<'visitor> Accumulator<'visitor> { self.record_raw_denial_outcome( &raw, &candidate, - DataLoopExclusionReason::UnusableResourcePath, + VerboseLoggingExclusionReason::UnusableResourcePath, ); return; } @@ -254,7 +254,11 @@ impl<'visitor> Accumulator<'visitor> { } else { path_norm::to_user_visible(&raw.object_name).unwrap_or_else(|| raw.object_name.clone()) }; - self.record_raw_denial_outcome(&raw, &resource, DataLoopExclusionReason::CanonicalDenial); + self.record_raw_denial_outcome( + &raw, + &resource, + VerboseLoggingExclusionReason::CanonicalDenial, + ); let dedup_resource = match raw.resource_type { learning_mode_core::ResourceType::File | learning_mode_core::ResourceType::Other => { resource.to_ascii_lowercase() @@ -269,11 +273,11 @@ impl<'visitor> Accumulator<'visitor> { } // The canonical unique-denial bound is reached: keep reading (up to // the processed-event bound) so every remaining outcome is still - // aggregated into the Data Loop summary, rather than stopping the + // aggregated into the verbose logging summary, rather than stopping the // trace early. if self.denials.len() >= MAX_UNIQUE_DENIALS { self.truncated = true; - self.data_loop.mark_canonical_denial_limit_reached(); + self.verbose_logging.mark_canonical_denial_limit_reached(); return; } self.seen.insert((dedup_resource, raw.access_type)); @@ -294,10 +298,10 @@ impl<'visitor> Accumulator<'visitor> { &mut self, raw: &RawDenial, resource: &str, - reason: DataLoopExclusionReason, + reason: VerboseLoggingExclusionReason, ) { let mut properties = raw - .data_loop_properties + .verbose_logging_properties .iter() .cloned() .collect::>(); @@ -340,15 +344,15 @@ impl<'visitor> Accumulator<'visitor> { ); } - /// Records one excluded outcome as a deduplicated Data Loop signature: + /// Records one excluded outcome as a deduplicated verbose logging signature: /// symbolic provider, provider GUID, event ID, reason, PID, and the /// already-sanitized/bounded property list all identify the group; /// repeats of the same signature only increment its `count`. fn record_exclusion( &mut self, - provider: DataLoopProvider, + provider: VerboseLoggingProvider, event_id: u16, - reason: DataLoopExclusionReason, + reason: VerboseLoggingExclusionReason, pid: u32, properties: Vec<(String, String)>, ) { @@ -357,9 +361,9 @@ impl<'visitor> Accumulator<'visitor> { fn record_outcome( &mut self, - provider: DataLoopProvider, + provider: VerboseLoggingProvider, event_id: u16, - reason: DataLoopExclusionReason, + reason: VerboseLoggingExclusionReason, pid: u32, classification: ( Option, @@ -368,7 +372,7 @@ impl<'visitor> Accumulator<'visitor> { properties: Vec<(String, String)>, ) { let (access_type, resource_type) = classification; - let signature = DataLoopSignature { + let signature = VerboseLoggingSignature { provider, provider_guid: crate::extractors::provider_guid_string(provider), event_id, @@ -378,10 +382,10 @@ impl<'visitor> Accumulator<'visitor> { resource_type, properties, }; - self.data_loop.record_with_byte_budget( + self.verbose_logging.record_with_byte_budget( signature, - &mut self.data_loop_signature_bytes, - MAX_DATA_LOOP_SIGNATURE_BYTES, + &mut self.verbose_logging_signature_bytes, + MAX_VERBOSE_LOGGING_SIGNATURE_BYTES, ); } @@ -395,7 +399,7 @@ impl<'visitor> Accumulator<'visitor> { if self.processed_event_count >= MAX_PROCESSED_EVENTS { self.processing_limit_reached = true; self.truncated = true; - self.data_loop.mark_processed_events_truncated(); + self.verbose_logging.mark_processed_events_truncated(); self.stop_requested = true; return false; } @@ -423,7 +427,7 @@ impl<'visitor> Accumulator<'visitor> { /// state is unreliable beyond this single event. Per-event decode /// failures are fatal only for the raw diagnostic visitor (which needs /// every event to succeed); in [`CollectionMode::Analyze`] they are - /// aggregated into the Data Loop summary for a known provider instead of + /// aggregated into the verbose logging summary for a known provider instead of /// silently dropped. fn record_event_decode_error( &mut self, @@ -443,13 +447,13 @@ impl<'visitor> Accumulator<'visitor> { if let Some(category) = crate::extractors::provider_category(provider) { let reason = match error.event_kind() { Some(tdh_decode::EventDecodeKind::PayloadMalformed) => { - DataLoopExclusionReason::EventPayloadMalformed + VerboseLoggingExclusionReason::EventPayloadMalformed } Some(tdh_decode::EventDecodeKind::DecoderLimitReached) => { - DataLoopExclusionReason::DecoderLimitReached + VerboseLoggingExclusionReason::DecoderLimitReached } Some(tdh_decode::EventDecodeKind::UnsupportedPropertyEncoding) => { - DataLoopExclusionReason::UnsupportedPropertyEncoding + VerboseLoggingExclusionReason::UnsupportedPropertyEncoding } None => return, }; @@ -482,9 +486,9 @@ impl<'visitor> Accumulator<'visitor> { let mut result = AnalysisResult { denials: self.denials, denied_resources_truncated: self.truncated, - data_loop: self.data_loop, + verbose_logging: self.verbose_logging, }; - match result.fit_data_loop_within_serialized_bytes( + match result.fit_verbose_logging_within_serialized_bytes( crate::guarded_wpr_protocol::MAX_ANALYSIS_BYTES as usize, ) { Ok(true) => Ok(result), @@ -550,7 +554,7 @@ impl DenialAnalyzer for EtlDenialAnalyzer { /// Runs the pure decode composition over already-collected events, using the /// same event-classification path ([`handle_decoded_event`]) as the real ETW /// callback: provider/vocabulary gating, extraction, capability-DACL -/// fallback, and Data Loop aggregation. Split out from +/// fallback, and verbose logging aggregation. Split out from /// [`EtlDenialAnalyzer::analyze`] so it can be tested with hand-built events /// that mirror real traces, without a live ETW/TDH read (which needs the /// provider manifests registered on the machine). @@ -828,7 +832,7 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu acc.record_exclusion( category, event_id, - DataLoopExclusionReason::UnsupportedEventSchema, + VerboseLoggingExclusionReason::UnsupportedEventSchema, header.ProcessId, Vec::new(), ); @@ -983,7 +987,7 @@ fn handle_decoded_event( acc.record_exclusion( category, parts.event_id, - DataLoopExclusionReason::UnsupportedEventSchema, + VerboseLoggingExclusionReason::UnsupportedEventSchema, header_pid, crate::extractors::sanitize_properties(&parts.props), ); @@ -995,9 +999,9 @@ fn handle_decoded_event( acc.record_outcome( category, parts.event_id, - DataLoopExclusionReason::EventPayloadMalformed, + VerboseLoggingExclusionReason::EventPayloadMalformed, header_pid, - crate::extractors::data_loop_classification(parts), + crate::extractors::verbose_logging_classification(parts), crate::extractors::sanitize_properties(&parts.props), ); } @@ -1021,7 +1025,7 @@ fn handle_decoded_event( match primary { Ok(raw) => acc.add_raw_denial(raw), Err(reason) => { - let recovered_by_dacl = reason == DataLoopExclusionReason::UnresolvedCapability + let recovered_by_dacl = reason == VerboseLoggingExclusionReason::UnresolvedCapability && !capability_candidates.is_empty(); if !recovered_by_dacl { acc.record_outcome( @@ -1029,7 +1033,7 @@ fn handle_decoded_event( parts.event_id, reason, pid, - crate::extractors::data_loop_classification(parts), + crate::extractors::verbose_logging_classification(parts), crate::extractors::sanitize_properties(&parts.props), ); } @@ -1184,12 +1188,12 @@ mod tests { assert_eq!( find_signature( - &analysis.data_loop, + &analysis.verbose_logging, crate::extractors::CAPABILITY_DENIAL_EVENT_ID ) .signature .reason, - DataLoopExclusionReason::EventPayloadMalformed + VerboseLoggingExclusionReason::EventPayloadMalformed ); } @@ -1342,8 +1346,9 @@ mod tests { access_type: access, filetime: 1, event_id: 4907, - provider: learning_mode_core::DataLoopProvider::PrivacyAuditingPermissiveLearningMode, - data_loop_properties: Vec::new(), + provider: + learning_mode_core::VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode, + verbose_logging_properties: Vec::new(), } } @@ -1401,10 +1406,12 @@ mod tests { assert_eq!(analysis.denials.len(), 1); assert_eq!(analysis.denials[0].resource, r"\\server\share\file.txt"); let excluded = analysis - .data_loop + .verbose_logging .signatures .iter() - .filter(|group| group.signature.reason == DataLoopExclusionReason::UnusableResourcePath) + .filter(|group| { + group.signature.reason == VerboseLoggingExclusionReason::UnusableResourcePath + }) .collect::>(); assert_eq!(excluded.len(), 1); assert!(excluded.iter().all(|group| { @@ -1443,10 +1450,10 @@ mod tests { #[test] fn unique_denial_bound_marks_truncated_without_stopping() { fn exclusion_count( - summary: &learning_mode_core::DataLoopSummary, - provider: learning_mode_core::DataLoopProvider, + summary: &learning_mode_core::VerboseLoggingSummary, + provider: learning_mode_core::VerboseLoggingProvider, event_id: u16, - reason: DataLoopExclusionReason, + reason: VerboseLoggingExclusionReason, ) -> u64 { summary .signatures @@ -1485,13 +1492,13 @@ mod tests { "hitting the unique-denial cap must not halt the trace early" ); assert!(accumulator.truncated); - assert!(accumulator.data_loop.canonical_denial_limit_reached); + assert!(accumulator.verbose_logging.canonical_denial_limit_reached); assert_eq!( exclusion_count( - &accumulator.data_loop, - learning_mode_core::DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + &accumulator.verbose_logging, + learning_mode_core::VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode, 4907, - DataLoopExclusionReason::CanonicalDenial + VerboseLoggingExclusionReason::CanonicalDenial ), 1 ); @@ -1509,10 +1516,10 @@ mod tests { assert!(!accumulator.stop_requested); assert_eq!( exclusion_count( - &accumulator.data_loop, - learning_mode_core::DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + &accumulator.verbose_logging, + learning_mode_core::VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode, 4907, - DataLoopExclusionReason::CanonicalDenial + VerboseLoggingExclusionReason::CanonicalDenial ), 3 ); @@ -1765,7 +1772,7 @@ mod tests { // by raw payload: one unresolved access-check event (14) plus two // unresolved capability denials (28). The two event-28 signatures stay // distinct because their retained ProcessId properties differ. - let groups = &out.data_loop.signatures; + let groups = &out.verbose_logging.signatures; assert_eq!(groups.len(), 3); let event_14_group = groups .iter() @@ -1773,11 +1780,11 @@ mod tests { .expect("event 14 exclusion group present"); assert_eq!( event_14_group.signature.provider, - DataLoopProvider::KernelGeneral + VerboseLoggingProvider::KernelGeneral ); assert_eq!( event_14_group.signature.reason, - DataLoopExclusionReason::UnresolvedCapability + VerboseLoggingExclusionReason::UnresolvedCapability ); assert_eq!(event_14_group.count, 1); let event_28_groups = groups @@ -1786,8 +1793,8 @@ mod tests { .collect::>(); assert_eq!(event_28_groups.len(), 2); assert!(event_28_groups.iter().all(|group| { - group.signature.provider == DataLoopProvider::KernelGeneral - && group.signature.reason == DataLoopExclusionReason::UnresolvedCapability + group.signature.provider == VerboseLoggingProvider::KernelGeneral + && group.signature.reason == VerboseLoggingExclusionReason::UnresolvedCapability && group.count == 1 })); } @@ -1848,17 +1855,19 @@ mod tests { assert_eq!(analysis.denials.len(), 1); assert_eq!(analysis.denials[0].resource, r"C:\owned.txt"); - assert_eq!(analysis.data_loop.signatures.len(), 2); + assert_eq!(analysis.verbose_logging.signatures.len(), 2); assert!(analysis - .data_loop + .verbose_logging .signatures .iter() .all(|group| group.signature.pid == 42)); let unsupported = analysis - .data_loop + .verbose_logging .signatures .iter() - .find(|group| group.signature.reason == DataLoopExclusionReason::UnsupportedEventSchema) + .find(|group| { + group.signature.reason == VerboseLoggingExclusionReason::UnsupportedEventSchema + }) .expect("owned unknown event retained"); assert_eq!(property(&unsupported.signature, "Marker"), "owned"); } @@ -1887,10 +1896,10 @@ mod tests { assert_eq!(analysis.denials[0].pid, 42); assert_eq!(analysis.denials[0].resource, "internetClient"); let group = analysis - .data_loop + .verbose_logging .signatures .iter() - .find(|group| group.signature.reason == DataLoopExclusionReason::CanonicalDenial) + .find(|group| group.signature.reason == VerboseLoggingExclusionReason::CanonicalDenial) .expect("canonical capability event retained"); assert_eq!(group.signature.pid, 42); assert_eq!(group.signature.access_type, Some(AccessType::Unknown)); @@ -1916,7 +1925,7 @@ mod tests { let analysis = resources_from_events_for_process_lifetimes(&events, Some(&[])); assert!(analysis.denials.is_empty()); - assert!(analysis.data_loop.is_empty()); + assert!(analysis.verbose_logging.is_empty()); } /// Non-actionable object types and not-denied capability records are @@ -1934,7 +1943,7 @@ mod tests { assert!(out.denials.is_empty()); let reason_for = |event_id: u16| { - out.data_loop + out.verbose_logging .signatures .iter() .find(|group| group.signature.event_id == event_id) @@ -1942,12 +1951,15 @@ mod tests { }; assert_eq!( reason_for(14), - Some(DataLoopExclusionReason::UnsupportedObjectType) + Some(VerboseLoggingExclusionReason::UnsupportedObjectType) + ); + assert_eq!( + reason_for(28), + Some(VerboseLoggingExclusionReason::NotActionable) ); - assert_eq!(reason_for(28), Some(DataLoopExclusionReason::NotActionable)); assert_eq!( reason_for(9999), - Some(DataLoopExclusionReason::UnsupportedEventSchema) + Some(VerboseLoggingExclusionReason::UnsupportedEventSchema) ); } @@ -1968,10 +1980,13 @@ mod tests { let out = resources_from_events(&events); assert!(out.denials.is_empty()); - assert_eq!(out.data_loop.signatures.len(), 1); - let group = &out.data_loop.signatures[0]; + assert_eq!(out.verbose_logging.signatures.len(), 1); + let group = &out.verbose_logging.signatures[0]; assert_eq!(group.count, 1); - assert_eq!(group.signature.provider, DataLoopProvider::KernelGeneral); + assert_eq!( + group.signature.provider, + VerboseLoggingProvider::KernelGeneral + ); assert_eq!( group.signature.provider_guid, "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}" @@ -1979,7 +1994,7 @@ mod tests { assert_eq!(group.signature.event_id, 14); assert_eq!( group.signature.reason, - DataLoopExclusionReason::UnusableResourcePath + VerboseLoggingExclusionReason::UnusableResourcePath ); assert_eq!(group.signature.pid, 1996); assert_eq!(group.signature.access_type, Some(AccessType::Read)); @@ -2019,17 +2034,17 @@ mod tests { let out = resources_from_events(&events); assert!(out.denials.is_empty()); - assert_eq!(out.data_loop.signatures.len(), cases.len()); + assert_eq!(out.verbose_logging.signatures.len(), cases.len()); for (object_type, object_name) in cases { let group = out - .data_loop + .verbose_logging .signatures .iter() .find(|group| property(&group.signature, "ObjectType") == object_type) - .unwrap_or_else(|| panic!("missing Data Loop signature for {object_type}")); + .unwrap_or_else(|| panic!("missing verbose logging signature for {object_type}")); assert_eq!( group.signature.reason, - DataLoopExclusionReason::UnsupportedObjectType + VerboseLoggingExclusionReason::UnsupportedObjectType ); assert_eq!(property(&group.signature, "ObjectName"), object_name); assert_eq!(group.count, 1); @@ -2073,9 +2088,9 @@ mod tests { && denial.access_type == AccessType::Write && names.contains(&denial.resource) })); - assert_eq!(out.data_loop.signatures.len(), names.len()); - assert!(out.data_loop.signatures.iter().all(|group| { - group.signature.reason == DataLoopExclusionReason::CanonicalDenial + assert_eq!(out.verbose_logging.signatures.len(), names.len()); + assert!(out.verbose_logging.signatures.iter().all(|group| { + group.signature.reason == VerboseLoggingExclusionReason::CanonicalDenial && group.count == 1 && property(&group.signature, "ObjectName").contains(" &learning_mode_core::DataLoopAggregate { + ) -> &learning_mode_core::VerboseLoggingAggregate { summary .signatures .iter() .find(|group| group.signature.event_id == event_id) - .unwrap_or_else(|| panic!("expected a Data Loop signature for event {event_id}")) + .unwrap_or_else(|| panic!("expected a verbose logging signature for event {event_id}")) } - fn property<'a>(signature: &'a learning_mode_core::DataLoopSignature, name: &str) -> &'a str { + fn property<'a>( + signature: &'a learning_mode_core::VerboseLoggingSignature, + name: &str, + ) -> &'a str { signature .properties .iter() @@ -2357,7 +2375,7 @@ mod tests { } #[test] - fn data_loop_byte_budget_preserves_guarded_protocol_headroom() { + fn verbose_logging_byte_budget_preserves_guarded_protocol_headroom() { let mut accumulator = Accumulator::analyze(); let properties = (0..crate::extractors::MAX_SIGNATURE_PROPERTIES) .map(|index| { @@ -2367,19 +2385,19 @@ mod tests { ) }) .collect::>(); - for pid in 0..learning_mode_core::MAX_DATA_LOOP_GROUPS as u32 { + for pid in 0..learning_mode_core::MAX_VERBOSE_LOGGING_GROUPS as u32 { accumulator.record_exclusion( - DataLoopProvider::KernelGeneral, + VerboseLoggingProvider::KernelGeneral, 14, - DataLoopExclusionReason::CanonicalDenial, + VerboseLoggingExclusionReason::CanonicalDenial, pid, properties.clone(), ); } - assert!(accumulator.data_loop.aggregate_groups_truncated); - assert!(accumulator.data_loop.overflow_occurrences > 0); - assert!(accumulator.data_loop_signature_bytes <= MAX_DATA_LOOP_SIGNATURE_BYTES); + assert!(accumulator.verbose_logging.aggregate_groups_truncated); + assert!(accumulator.verbose_logging.overflow_occurrences > 0); + assert!(accumulator.verbose_logging_signature_bytes <= MAX_VERBOSE_LOGGING_SIGNATURE_BYTES); } #[test] @@ -2393,10 +2411,10 @@ mod tests { let out = resources_from_events(&events); - let group = find_signature(&out.data_loop, 9999); + let group = find_signature(&out.verbose_logging, 9999); assert_eq!( group.signature.reason, - DataLoopExclusionReason::UnsupportedEventSchema + VerboseLoggingExclusionReason::UnsupportedEventSchema ); assert_eq!( property(&group.signature, "ObjectName"), @@ -2420,15 +2438,18 @@ mod tests { let out = resources_from_events(&events); - let group = find_signature(&out.data_loop, 28); + let group = find_signature(&out.verbose_logging, 28); assert_eq!( group.signature.reason, - DataLoopExclusionReason::UnresolvedCapability + VerboseLoggingExclusionReason::UnresolvedCapability + ); + assert_eq!( + group.signature.provider, + VerboseLoggingProvider::KernelGeneral ); - assert_eq!(group.signature.provider, DataLoopProvider::KernelGeneral); assert_eq!( group.signature.provider_guid, - crate::extractors::provider_guid_string(DataLoopProvider::KernelGeneral) + crate::extractors::provider_guid_string(VerboseLoggingProvider::KernelGeneral) ); assert_eq!(group.signature.pid, 42, "process identifier retained"); assert_eq!( @@ -2451,11 +2472,11 @@ mod tests { let out = resources_from_events(&events); assert_eq!( - out.data_loop.signatures.len(), + out.verbose_logging.signatures.len(), 1, "differing only by timestamp must dedupe to a single signature" ); - assert_eq!(find_signature(&out.data_loop, 9999).count, 2); + assert_eq!(find_signature(&out.verbose_logging, 9999).count, 2); } #[test] @@ -2472,7 +2493,7 @@ mod tests { let out = resources_from_events(&events); - let group = find_signature(&out.data_loop, 9999); + let group = find_signature(&out.verbose_logging, 9999); assert!( group .signature @@ -2521,22 +2542,22 @@ mod tests { .into_analysis() .expect("per-event decode failures are non-fatal in Analyze mode"); - let malformed = find_signature(&out.data_loop, 14); + let malformed = find_signature(&out.verbose_logging, 14); assert_eq!( malformed.signature.reason, - DataLoopExclusionReason::EventPayloadMalformed + VerboseLoggingExclusionReason::EventPayloadMalformed ); assert_eq!(malformed.signature.pid, 9); assert_eq!(property(&malformed.signature, "EventName"), "AccessCheck"); assert_eq!( - find_signature(&out.data_loop, 27).signature.reason, - DataLoopExclusionReason::DecoderLimitReached + find_signature(&out.verbose_logging, 27).signature.reason, + VerboseLoggingExclusionReason::DecoderLimitReached ); assert_eq!( - find_signature(&out.data_loop, 28).signature.reason, - DataLoopExclusionReason::UnsupportedPropertyEncoding + find_signature(&out.verbose_logging, 28).signature.reason, + VerboseLoggingExclusionReason::UnsupportedPropertyEncoding ); - assert!(out.data_loop.signatures.iter().all(|aggregate| { + assert!(out.verbose_logging.signatures.iter().all(|aggregate| { aggregate .signature .properties diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index 3936ea6ca..163bbe2a8 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -22,7 +22,7 @@ //! resource types we surface. This list grows as more denial sources are //! decoded; event IDs outside this vocabulary are excluded rather than //! extracted, and (for the known providers below) that exclusion is -//! aggregated into [`learning_mode_core::DataLoopSummary`] rather than +//! aggregated into [`learning_mode_core::VerboseLoggingSummary`] rather than //! silently dropped. The IDs handled today: //! //! - **14 / 4907 — access check** — the primary denial event @@ -47,7 +47,9 @@ //! capability SID via [`crate::capability_names`] (well-known SID → friendly //! name; custom hashed capabilities fall back to the SID string). -use learning_mode_core::{AccessType, DataLoopExclusionReason, DataLoopProvider, ResourceType}; +use learning_mode_core::{ + AccessType, ResourceType, VerboseLoggingExclusionReason, VerboseLoggingProvider, +}; use sha2::{Digest, Sha256}; use windows::core::GUID; @@ -107,19 +109,19 @@ pub struct RawDenial { pub filetime: u64, /// Originating ETW event ID (kept for diagnostics). pub event_id: u16, - /// Symbolic category of the originating provider, for Data Loop + /// Symbolic category of the originating provider, for verbose logging /// aggregation. Never a raw provider GUID. - pub provider: DataLoopProvider, - /// Bounded sensitive-value-redacted properties retained for Data Loop signatures. - pub data_loop_properties: Vec<(String, String)>, + pub provider: VerboseLoggingProvider, + /// Bounded sensitive-value-redacted properties retained for verbose logging signatures. + pub verbose_logging_properties: Vec<(String, String)>, } /// Routes a decoded event to the matching extractor by its event ID. /// -/// Returns `Err` with a closed [`DataLoopExclusionReason`] for events that +/// Returns `Err` with a closed [`VerboseLoggingExclusionReason`] for events that /// are not learning-mode denials, that carry an object type we don't /// surface, or that otherwise fail extraction. Callers aggregate the -/// returned reason into [`learning_mode_core::DataLoopSummary`] rather than +/// returned reason into [`learning_mode_core::VerboseLoggingSummary`] rather than /// discarding it, except when a fallback extractor (e.g. /// [`crate::capability_dacl`]) recovers an equivalent denial from the same /// event. @@ -127,14 +129,14 @@ pub fn extract_denial( parts: &DecodedEventParts, pid: u32, filetime: u64, -) -> Result { +) -> Result { // Callers on the real trace path gate on `is_learning_mode_event` before // decoding via TDH at all, so this branch only matters for direct/test // callers that skip that gate. - let provider = - provider_category(parts.provider).ok_or(DataLoopExclusionReason::UnsupportedEventSchema)?; + let provider = provider_category(parts.provider) + .ok_or(VerboseLoggingExclusionReason::UnsupportedEventSchema)?; if !is_learning_mode_event(parts.provider, parts.event_id) { - return Err(DataLoopExclusionReason::UnsupportedEventSchema); + return Err(VerboseLoggingExclusionReason::UnsupportedEventSchema); } match parts.event_id { @@ -148,13 +150,13 @@ pub fn extract_denial( build_denial_from_learning_mode(parts, pid, filetime, provider) } CAPABILITY_DENIAL_EVENT_ID => build_denial_from_capability(parts, pid, filetime, provider), - _ => Err(DataLoopExclusionReason::UnsupportedEventSchema), + _ => Err(VerboseLoggingExclusionReason::UnsupportedEventSchema), } } /// Returns typed denial classifications that can be determined independently /// of whether the event contains every property required for canonical output. -pub(crate) fn data_loop_classification( +pub(crate) fn verbose_logging_classification( parts: &DecodedEventParts, ) -> (Option, Option) { match parts.event_id { @@ -244,29 +246,29 @@ pub(crate) fn effective_capability_event_pid(process_id: Option<&str>) -> Option process_id.and_then(parse_u32) } -/// Maps a raw ETW provider GUID to its symbolic Data Loop category. +/// Maps a raw ETW provider GUID to its symbolic verbose logging category. /// /// Returns `None` for providers outside the Learning Mode vocabulary; those /// events are ignored entirely (not aggregated), since they are unrelated /// host traffic rather than an excluded Learning Mode outcome. -pub(crate) fn provider_category(provider: GUID) -> Option { +pub(crate) fn provider_category(provider: GUID) -> Option { if provider == KERNEL_GENERAL_PROVIDER { - Some(DataLoopProvider::KernelGeneral) + Some(VerboseLoggingProvider::KernelGeneral) } else if provider == PRIVACY_LEARNING_MODE_PROVIDER { - Some(DataLoopProvider::PrivacyAuditingPermissiveLearningMode) + Some(VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode) } else { None } } -/// Renders a symbolic Data Loop provider category back to its raw ETW +/// Renders a symbolic verbose logging provider category back to its raw ETW /// provider GUID, in canonical braced string form, for retention in a Data /// Loop signature. Provider GUIDs are stable component identifiers, not /// personal data, so unlike account/user values they are never redacted. -pub(crate) fn provider_guid_string(provider: DataLoopProvider) -> String { +pub(crate) fn provider_guid_string(provider: VerboseLoggingProvider) -> String { match provider { - DataLoopProvider::KernelGeneral => format_guid(KERNEL_GENERAL_PROVIDER), - DataLoopProvider::PrivacyAuditingPermissiveLearningMode => { + VerboseLoggingProvider::KernelGeneral => format_guid(KERNEL_GENERAL_PROVIDER), + VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode => { format_guid(PRIVACY_LEARNING_MODE_PROVIDER) } } @@ -289,9 +291,9 @@ fn format_guid(guid: GUID) -> String { ) } -// ---- Data Loop signature sanitization ------------------------------------- +// ---- verbose logging signature sanitization ------------------------------------- // -// A Data Loop signature retains identifiers useful for triage (SIDs, +// A verbose logging signature retains identifiers useful for triage (SIDs, // capability names, PIDs, provider/object GUIDs) but must never leak a // human account/user name, a file path, an exact timestamp, or a free-form // decoder error message. These bounds apply uniformly to every decoded @@ -304,7 +306,7 @@ fn format_guid(guid: GUID) -> String { pub(crate) const REDACTED_USER: &str = ""; /// Fixed replacement for a complete file path. pub(crate) const REDACTED_PATH: &str = ""; -/// Maximum number of `(name, value)` properties retained in one Data Loop +/// Maximum number of `(name, value)` properties retained in one verbose logging /// signature. Bounds pathological/huge TDH property lists; sanitization /// (never raw decoding) determines which survive, via deterministic /// (sorted-by-name) truncation. @@ -360,7 +362,7 @@ impl<'a> NormalizedPropertyName<'a> { } /// Returns whether `name` looks like it carries a timestamp, so it is -/// dropped from a Data Loop signature entirely (never redacted or +/// dropped from a verbose logging signature entirely (never redacted or /// truncated) — exact timestamps must not prevent otherwise-identical /// events from deduplicating. fn is_timestamp_like_property(name: &str) -> bool { @@ -490,7 +492,7 @@ fn ends_with_ignore_ascii_case(value: &str, suffix: &str) -> bool { /// /// Overlong values retain prefix and suffix context plus a digest of the full /// sanitized value. The digest prevents distinct values with a long shared -/// prefix from collapsing into one Data Loop signature. +/// prefix from collapsing into one verbose logging signature. /// /// Callers must sanitize (redact) *before* calling this so neither the retained /// context nor the digest is derived from sensitive identity content. @@ -619,24 +621,24 @@ pub(crate) fn sanitize_properties(props: &[(String, String)]) -> Vec<(String, St /// /// # Errors /// -/// Returns the closed [`DataLoopExclusionReason`] describing why no denial -/// could be built: a missing `ObjectType` ([`DataLoopExclusionReason::MissingObjectType`]), +/// Returns the closed [`VerboseLoggingExclusionReason`] describing why no denial +/// could be built: a missing `ObjectType` ([`VerboseLoggingExclusionReason::MissingObjectType`]), /// an object type this model can't represent -/// ([`DataLoopExclusionReason::UnsupportedObjectType`]), a missing/empty +/// ([`VerboseLoggingExclusionReason::UnsupportedObjectType`]), a missing/empty /// object name (registry/file: -/// [`DataLoopExclusionReason::MissingObjectName`]; capability: an +/// [`VerboseLoggingExclusionReason::MissingObjectName`]; capability: an /// unidentified brokered check, -/// [`DataLoopExclusionReason::UnresolvedCapability`] — [`crate::capability_dacl`] +/// [`VerboseLoggingExclusionReason::UnresolvedCapability`] — [`crate::capability_dacl`] /// may still recover it from the event's DACL payload), or a self-access -/// check that isn't actionable ([`DataLoopExclusionReason::NotActionable`]). +/// check that isn't actionable ([`VerboseLoggingExclusionReason::NotActionable`]). pub fn build_denial_from_access_check( parts: &DecodedEventParts, pid: u32, filetime: u64, - provider: DataLoopProvider, -) -> Result { - let object_type = - find_prop(&parts.props, "ObjectType").ok_or(DataLoopExclusionReason::MissingObjectType)?; + provider: VerboseLoggingProvider, +) -> Result { + let object_type = find_prop(&parts.props, "ObjectType") + .ok_or(VerboseLoggingExclusionReason::MissingObjectType)?; let object_type_str = object_type.trim_matches('"'); let resource_type = match object_type_str { @@ -645,7 +647,7 @@ pub fn build_denial_from_access_check( "Section" | "SymbolicLink" | "Timer" => ResourceType::Other, // A present-but-empty object type is a brokered-capability check. "" => ResourceType::Capability, - _ => return Err(DataLoopExclusionReason::UnsupportedObjectType), + _ => return Err(VerboseLoggingExclusionReason::UnsupportedObjectType), }; let object_name = find_prop(&parts.props, "ObjectName") @@ -655,9 +657,9 @@ pub fn build_denial_from_access_check( // An unidentified brokered-capability check: the identifier may // still be recoverable from the event's DACL payload. (ResourceType::Capability, None) => { - return Err(DataLoopExclusionReason::UnresolvedCapability) + return Err(VerboseLoggingExclusionReason::UnresolvedCapability) } - (_, None) => return Err(DataLoopExclusionReason::MissingObjectName), + (_, None) => return Err(VerboseLoggingExclusionReason::MissingObjectName), (_, Some(name)) => name, }; @@ -666,7 +668,7 @@ pub fn build_denial_from_access_check( .or_else(|| find_prop(&parts.props, "ApplicationPath")) .map(|value| value.trim_matches('"')); if app_path.is_some_and(|app_path| is_self_access(&object_name, app_path)) { - return Err(DataLoopExclusionReason::NotActionable); + return Err(VerboseLoggingExclusionReason::NotActionable); } } @@ -694,7 +696,7 @@ pub fn build_denial_from_access_check( filetime, event_id: parts.event_id, provider, - data_loop_properties: sanitize_properties(&parts.props), + verbose_logging_properties: sanitize_properties(&parts.props), }) } @@ -754,27 +756,27 @@ fn strip_dos_namespace_prefix(path: &str) -> &str { /// /// # Errors /// -/// Returns [`DataLoopExclusionReason::MissingObjectType`] when `Category` is -/// absent or unparseable, [`DataLoopExclusionReason::MissingObjectName`] +/// Returns [`VerboseLoggingExclusionReason::MissingObjectType`] when `Category` is +/// absent or unparseable, [`VerboseLoggingExclusionReason::MissingObjectName`] /// when the required `Detail` is absent or unparseable, and -/// [`DataLoopExclusionReason::NotActionable`] when the category/detail pair +/// [`VerboseLoggingExclusionReason::NotActionable`] when the category/detail pair /// describes no violation (`Category == 0`). pub fn build_denial_from_learning_mode( parts: &DecodedEventParts, pid: u32, filetime: u64, - provider: DataLoopProvider, -) -> Result { + provider: VerboseLoggingProvider, +) -> Result { let category = find_prop(&parts.props, "Category") .and_then(|value| parse_u32(value)) - .ok_or(DataLoopExclusionReason::MissingObjectType)?; + .ok_or(VerboseLoggingExclusionReason::MissingObjectType)?; let detail = match find_prop(&parts.props, "Detail") { - Some(value) => parse_u32(value).ok_or(DataLoopExclusionReason::MissingObjectName)?, + Some(value) => parse_u32(value).ok_or(VerboseLoggingExclusionReason::MissingObjectName)?, None if category == crate::ui::CONVERT_TO_GUI => 0, - None => return Err(DataLoopExclusionReason::MissingObjectName), + None => return Err(VerboseLoggingExclusionReason::MissingObjectName), }; - let object_name = - crate::ui::resource_name(category, detail).ok_or(DataLoopExclusionReason::NotActionable)?; + let object_name = crate::ui::resource_name(category, detail) + .ok_or(VerboseLoggingExclusionReason::NotActionable)?; Ok(RawDenial { pid, @@ -784,7 +786,7 @@ pub fn build_denial_from_learning_mode( filetime, event_id: parts.event_id, provider, - data_loop_properties: sanitize_properties(&parts.props), + verbose_logging_properties: sanitize_properties(&parts.props), }) } @@ -800,20 +802,20 @@ pub fn build_denial_from_learning_mode( /// /// # Errors /// -/// Returns [`DataLoopExclusionReason::NotActionable`] when `Denied` is -/// absent or not `true`, and [`DataLoopExclusionReason::UnresolvedCapability`] +/// Returns [`VerboseLoggingExclusionReason::NotActionable`] when `Denied` is +/// absent or not `true`, and [`VerboseLoggingExclusionReason::UnresolvedCapability`] /// when no usable capability identifier could be decoded. pub fn build_denial_from_capability( parts: &DecodedEventParts, pid: u32, filetime: u64, - provider: DataLoopProvider, -) -> Result { + provider: VerboseLoggingProvider, +) -> Result { // A partially decoded event must not become a policy recommendation. let denied = find_prop(&parts.props, "Denied") .is_some_and(|value| value.trim_matches('"').eq_ignore_ascii_case("true")); if !denied { - return Err(DataLoopExclusionReason::NotActionable); + return Err(VerboseLoggingExclusionReason::NotActionable); } let pid = find_prop(&parts.props, "ProcessId") @@ -839,7 +841,7 @@ pub fn build_denial_from_capability( && name != "" && name != "" }) - .ok_or(DataLoopExclusionReason::UnresolvedCapability)?; + .ok_or(VerboseLoggingExclusionReason::UnresolvedCapability)?; Ok(RawDenial { pid, @@ -849,7 +851,7 @@ pub fn build_denial_from_capability( filetime, event_id: parts.event_id, provider, - data_loop_properties: sanitize_properties(&parts.props), + verbose_logging_properties: sanitize_properties(&parts.props), }) } @@ -1097,7 +1099,7 @@ mod tests { ); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::NotActionable + VerboseLoggingExclusionReason::NotActionable ); } } @@ -1226,7 +1228,7 @@ mod tests { ); assert_eq!( extract_denial(&p, 5900, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::UnresolvedCapability + VerboseLoggingExclusionReason::UnresolvedCapability ); } @@ -1243,7 +1245,7 @@ mod tests { ); assert_eq!( extract_denial(&p, 5900, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::MissingObjectName + VerboseLoggingExclusionReason::MissingObjectName ); } } @@ -1298,7 +1300,7 @@ mod tests { ); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::UnsupportedObjectType + VerboseLoggingExclusionReason::UnsupportedObjectType ); } @@ -1307,7 +1309,7 @@ mod tests { let p = parts(14, &[("ObjectName", "\"x\"")]); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::MissingObjectType + VerboseLoggingExclusionReason::MissingObjectType ); } @@ -1390,13 +1392,13 @@ mod tests { let invalid_category = parts(27, &[("Category", "not-a-number"), ("Detail", "4")]); assert_eq!( extract_denial(&invalid_category, 9999, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::MissingObjectType + VerboseLoggingExclusionReason::MissingObjectType ); let invalid_detail = parts(27, &[("Category", "2"), ("Detail", "not-a-number")]); assert_eq!( extract_denial(&invalid_detail, 9999, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::MissingObjectName + VerboseLoggingExclusionReason::MissingObjectName ); } @@ -1405,7 +1407,7 @@ mod tests { let p = parts(27, &[("Category", "0"), ("Detail", "0")]); assert_eq!( extract_denial(&p, 9999, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::NotActionable + VerboseLoggingExclusionReason::NotActionable ); } @@ -1430,7 +1432,7 @@ mod tests { // pid comes from the payload ProcessId (0x1acc), not the header. assert_eq!(ev.pid, 0x1acc); assert_eq!(ev.object_name, "internetClient"); - assert_eq!(ev.provider, DataLoopProvider::KernelGeneral); + assert_eq!(ev.provider, VerboseLoggingProvider::KernelGeneral); } #[test] @@ -1454,7 +1456,7 @@ mod tests { assert_eq!(denial.object_name, "Handles"); assert_eq!(denial.pid, 0x1594); assert_eq!( - data_loop_classification(&p), + verbose_logging_classification(&p), (Some(AccessType::Unknown), Some(ResourceType::Ui)) ); } @@ -1499,7 +1501,7 @@ mod tests { let p = parts(28, &[("ProcessId", "0x10"), ("Denied", "false")]); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::NotActionable + VerboseLoggingExclusionReason::NotActionable ); } @@ -1508,7 +1510,7 @@ mod tests { let p = parts(28, &[("PackageSid", "S-1-15-3-1")]); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::NotActionable + VerboseLoggingExclusionReason::NotActionable ); } @@ -1517,7 +1519,7 @@ mod tests { let p = parts(28, &[("ProcessId", "0x10"), ("Denied", "true")]); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::UnresolvedCapability + VerboseLoggingExclusionReason::UnresolvedCapability ); } @@ -1530,7 +1532,7 @@ mod tests { ); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::UnsupportedEventSchema + VerboseLoggingExclusionReason::UnsupportedEventSchema ); assert!( provider_category(GUID::from_u128(0x12345678_1234_1234_1234_1234567890ab)).is_none() @@ -1560,7 +1562,7 @@ mod tests { let p = parts(9999, &[("Foo", "\"bar\"")]); assert_eq!( extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), - DataLoopExclusionReason::UnsupportedEventSchema + VerboseLoggingExclusionReason::UnsupportedEventSchema ); } @@ -1568,27 +1570,27 @@ mod tests { fn known_provider_category_is_symbolic_and_pii_free() { assert_eq!( provider_category(KERNEL_GENERAL_PROVIDER), - Some(DataLoopProvider::KernelGeneral) + Some(VerboseLoggingProvider::KernelGeneral) ); assert_eq!( provider_category(PRIVACY_LEARNING_MODE_PROVIDER), - Some(DataLoopProvider::PrivacyAuditingPermissiveLearningMode) + Some(VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode) ); assert_eq!(provider_category(GUID::from_u128(0)), None); } #[test] fn provider_guid_string_is_stable_and_round_trips_by_category() { - let kernel_guid = provider_guid_string(DataLoopProvider::KernelGeneral); + let kernel_guid = provider_guid_string(VerboseLoggingProvider::KernelGeneral); let privacy_guid = - provider_guid_string(DataLoopProvider::PrivacyAuditingPermissiveLearningMode); + provider_guid_string(VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode); assert_ne!(kernel_guid, privacy_guid); assert_eq!(kernel_guid, format_guid(KERNEL_GENERAL_PROVIDER)); assert_eq!(privacy_guid, format_guid(PRIVACY_LEARNING_MODE_PROVIDER)); assert!(kernel_guid.starts_with('{') && kernel_guid.ends_with('}')); } - // ---- Data Loop signature sanitization ---------------------------------- + // ---- verbose logging signature sanitization ---------------------------------- #[test] fn file_path_detection_preserves_non_file_identifiers() { diff --git a/src/core/learning_mode_core/src/analyze.rs b/src/core/learning_mode_core/src/analyze.rs index 834008a4b..ddb4b9413 100644 --- a/src/core/learning_mode_core/src/analyze.rs +++ b/src/core/learning_mode_core/src/analyze.rs @@ -16,8 +16,8 @@ use std::path::Path; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::data_loop::{DataLoopAggregate, DataLoopSummary}; use crate::model::DeniedResource; +use crate::verbose_logging::{VerboseLoggingAggregate, VerboseLoggingSummary}; /// Result of decoding a capture source into bounded, de-duplicated denials. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -30,7 +30,7 @@ pub struct AnalysisResult { pub denied_resources_truncated: bool, /// Username-redacted signatures for outcomes omitted from canonical denials. #[serde(default)] - pub data_loop: DataLoopSummary, + pub verbose_logging: VerboseLoggingSummary, } /// Inclusive process-lifetime window used to scope a host-wide capture. @@ -65,15 +65,15 @@ impl AnalysisResult { Self { denials, denied_resources_truncated: false, - data_loop: DataLoopSummary::default(), + verbose_logging: VerboseLoggingSummary::default(), } } - /// Trims Data Loop signatures so the complete compact JSON result fits. + /// Trims verbose logging signatures so the complete compact JSON result fits. /// - /// Returns `false` when the canonical denials and empty Data Loop envelope + /// Returns `false` when the canonical denials and empty verbose logging envelope /// alone exceed `max_bytes`; canonical denials are never discarded. - pub fn fit_data_loop_within_serialized_bytes( + pub fn fit_verbose_logging_within_serialized_bytes( &mut self, max_bytes: usize, ) -> Result { @@ -81,12 +81,13 @@ impl AnalysisResult { return Ok(true); } - let base_len = serialized_analysis_len(self, &[], self.data_loop.signatures.as_slice())?; + let base_len = + serialized_analysis_len(self, &[], self.verbose_logging.signatures.as_slice())?; if base_len > max_bytes { return Ok(false); } - let original = std::mem::take(&mut self.data_loop.signatures); + let original = std::mem::take(&mut self.verbose_logging.signatures); let mut retained_bytes = 0usize; const ENVELOPE_HEADROOM: usize = 4 * 1024; let signature_budget = max_bytes @@ -98,13 +99,13 @@ impl AnalysisResult { let aggregate_len = serde_json::to_vec(&aggregate)?.len().saturating_add(1); if retained_bytes.saturating_add(aggregate_len) <= signature_budget { retained_bytes += aggregate_len; - self.data_loop.signatures.push(aggregate); + self.verbose_logging.signatures.push(aggregate); } else { - self.data_loop.move_to_overflow(aggregate); + self.verbose_logging.move_to_overflow(aggregate); } } - let mut retained = std::mem::take(&mut self.data_loop.signatures); + let mut retained = std::mem::take(&mut self.verbose_logging.signatures); let serialized_len = |retained_count: usize| { serialized_analysis_len( self, @@ -123,11 +124,11 @@ impl AnalysisResult { } } let removed = retained.split_off(low); - self.data_loop.signatures = retained; + self.verbose_logging.signatures = retained; for aggregate in removed { - self.data_loop.move_to_overflow(aggregate); + self.verbose_logging.move_to_overflow(aggregate); } - self.data_loop + self.verbose_logging .signatures .sort_by(|left, right| left.signature.cmp(&right.signature)); Ok(serde_json::to_vec(self)?.len() <= max_bytes) @@ -136,21 +137,21 @@ impl AnalysisResult { fn serialized_analysis_len( analysis: &AnalysisResult, - retained: &[DataLoopAggregate], - removed: &[DataLoopAggregate], + retained: &[VerboseLoggingAggregate], + removed: &[VerboseLoggingAggregate], ) -> Result { #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct AnalysisView<'a> { denials: &'a [DeniedResource], denied_resources_truncated: bool, - data_loop: DataLoopView<'a>, + verbose_logging: VerboseLoggingView<'a>, } #[derive(Serialize)] #[serde(rename_all = "camelCase")] - struct DataLoopView<'a> { - signatures: &'a [DataLoopAggregate], + struct VerboseLoggingView<'a> { + signatures: &'a [VerboseLoggingAggregate], total_occurrences: u64, overflow_occurrences: u64, canonical_overflow_occurrences: u64, @@ -171,21 +172,21 @@ fn serialized_analysis_len( let view = AnalysisView { denials: &analysis.denials, denied_resources_truncated: analysis.denied_resources_truncated, - data_loop: DataLoopView { + verbose_logging: VerboseLoggingView { signatures: retained, - total_occurrences: analysis.data_loop.total_occurrences, + total_occurrences: analysis.verbose_logging.total_occurrences, overflow_occurrences: analysis - .data_loop + .verbose_logging .overflow_occurrences .saturating_add(removed_occurrences), canonical_overflow_occurrences: analysis - .data_loop + .verbose_logging .canonical_overflow_occurrences .saturating_add(removed_canonical_occurrences), - aggregate_groups_truncated: analysis.data_loop.aggregate_groups_truncated + aggregate_groups_truncated: analysis.verbose_logging.aggregate_groups_truncated || !removed.is_empty(), - processed_events_truncated: analysis.data_loop.processed_events_truncated, - canonical_denial_limit_reached: analysis.data_loop.canonical_denial_limit_reached, + processed_events_truncated: analysis.verbose_logging.processed_events_truncated, + canonical_denial_limit_reached: analysis.verbose_logging.canonical_denial_limit_reached, }, }; serde_json::to_vec(&view).map(|bytes| bytes.len()) @@ -233,10 +234,11 @@ pub trait DenialAnalyzer { #[cfg(test)] mod tests { use super::*; - use crate::data_loop::{ - DataLoopAggregate, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, - }; use crate::model::{AccessType, ResourceType}; + use crate::verbose_logging::{ + VerboseLoggingAggregate, VerboseLoggingExclusionReason, VerboseLoggingProvider, + VerboseLoggingSignature, + }; /// A trivial analyzer returning a fixed set, proving the trait is /// object-safe and usable behind a `dyn` reference. @@ -261,14 +263,14 @@ mod tests { let got = analyzer.analyze(Path::new("ignored.etl")).unwrap(); assert_eq!(got.denials, denials); assert!(!got.denied_resources_truncated); - assert!(got.data_loop.is_empty()); + assert!(got.verbose_logging.is_empty()); } #[test] - fn legacy_guarded_payload_defaults_data_loop_summary() { + fn legacy_guarded_payload_defaults_verbose_logging_summary() { let payload = br#"{"denials":[],"deniedResourcesTruncated":false}"#; let result: AnalysisResult = serde_json::from_slice(payload).unwrap(); - assert!(result.data_loop.is_empty()); + assert!(result.verbose_logging.is_empty()); } #[test] @@ -299,13 +301,13 @@ mod tests { } #[test] - fn serialized_size_limit_moves_data_loop_groups_to_overflow() { - let signature = |pid| DataLoopAggregate { - signature: DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + fn serialized_size_limit_moves_verbose_logging_groups_to_overflow() { + let signature = |pid| VerboseLoggingAggregate { + signature: VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "provider".to_string(), event_id: 14, - reason: DataLoopExclusionReason::CanonicalDenial, + reason: VerboseLoggingExclusionReason::CanonicalDenial, pid, access_type: Some(AccessType::Read), resource_type: Some(ResourceType::File), @@ -322,7 +324,7 @@ mod tests { filetime: 2, }], denied_resources_truncated: false, - data_loop: DataLoopSummary { + verbose_logging: VerboseLoggingSummary { signatures: vec![signature(1), signature(2)], total_occurrences: 6, ..Default::default() @@ -331,12 +333,12 @@ mod tests { let original_len = serde_json::to_vec(&result).unwrap().len(); assert!(result - .fit_data_loop_within_serialized_bytes(original_len - 1) + .fit_verbose_logging_within_serialized_bytes(original_len - 1) .unwrap()); assert!(serde_json::to_vec(&result).unwrap().len() < original_len); - assert!(result.data_loop.aggregate_groups_truncated); - assert_eq!(result.data_loop.total_occurrences, 6); - assert!(result.data_loop.overflow_occurrences > 0); + assert!(result.verbose_logging.aggregate_groups_truncated); + assert_eq!(result.verbose_logging.total_occurrences, 6); + assert!(result.verbose_logging.overflow_occurrences > 0); } #[test] @@ -350,13 +352,13 @@ mod tests { filetime: 2, }], denied_resources_truncated: false, - data_loop: DataLoopSummary { - signatures: vec![DataLoopAggregate { - signature: DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + verbose_logging: VerboseLoggingSummary { + signatures: vec![VerboseLoggingAggregate { + signature: VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "provider".to_string(), event_id: 14, - reason: DataLoopExclusionReason::CanonicalDenial, + reason: VerboseLoggingExclusionReason::CanonicalDenial, pid: 1, access_type: Some(AccessType::Read), resource_type: Some(ResourceType::File), @@ -370,23 +372,27 @@ mod tests { }; let original = result.clone(); - assert!(!result.fit_data_loop_within_serialized_bytes(1).unwrap()); + assert!(!result + .fit_verbose_logging_within_serialized_bytes(1) + .unwrap()); assert_eq!(result, original); assert!(result - .fit_data_loop_within_serialized_bytes(serde_json::to_vec(&original).unwrap().len()) + .fit_verbose_logging_within_serialized_bytes( + serde_json::to_vec(&original).unwrap().len() + ) .unwrap()); assert_eq!(result, original); } #[test] fn serialized_size_limit_handles_maximum_signature_count() { - let signatures = (0..crate::data_loop::MAX_DATA_LOOP_GROUPS) - .map(|pid| DataLoopAggregate { - signature: DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + let signatures = (0..crate::verbose_logging::MAX_VERBOSE_LOGGING_GROUPS) + .map(|pid| VerboseLoggingAggregate { + signature: VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "provider".to_string(), event_id: 14, - reason: DataLoopExclusionReason::MissingObjectName, + reason: VerboseLoggingExclusionReason::MissingObjectName, pid: u32::try_from(pid).unwrap(), access_type: Some(AccessType::Read), resource_type: Some(ResourceType::File), @@ -398,21 +404,22 @@ mod tests { let mut result = AnalysisResult { denials: Vec::new(), denied_resources_truncated: false, - data_loop: DataLoopSummary { + verbose_logging: VerboseLoggingSummary { signatures, - total_occurrences: crate::data_loop::MAX_DATA_LOOP_GROUPS as u64, + total_occurrences: crate::verbose_logging::MAX_VERBOSE_LOGGING_GROUPS as u64, ..Default::default() }, }; assert!(result - .fit_data_loop_within_serialized_bytes(64 * 1024) + .fit_verbose_logging_within_serialized_bytes(64 * 1024) .unwrap()); assert!(serde_json::to_vec(&result).unwrap().len() <= 64 * 1024); - assert!(!result.data_loop.signatures.is_empty()); + assert!(!result.verbose_logging.signatures.is_empty()); assert_eq!( - result.data_loop.signatures.len() as u64 + result.data_loop.overflow_occurrences, - crate::data_loop::MAX_DATA_LOOP_GROUPS as u64 + result.verbose_logging.signatures.len() as u64 + + result.verbose_logging.overflow_occurrences, + crate::verbose_logging::MAX_VERBOSE_LOGGING_GROUPS as u64 ); } } diff --git a/src/core/learning_mode_core/src/lib.rs b/src/core/learning_mode_core/src/lib.rs index d93cc4dea..5d59d8888 100644 --- a/src/core/learning_mode_core/src/lib.rs +++ b/src/core/learning_mode_core/src/lib.rs @@ -13,12 +13,12 @@ //! [`DenialAnalyzer`]; this crate owns the trait and the model. //! 3. **Emit** — the records plus a terminating [`DenialSummary`] are //! written to the canonical JSON output that host applications read to -//! regenerate policy. A deterministic Data Loop sibling contains bounded, +//! regenerate policy. A deterministic verbose logging sibling contains bounded, //! sensitive-value-redacted signatures for excluded decoder outcomes. //! //! This crate is the cross-platform hinge between stages 2 and 3: it //! defines the public [`DeniedResource`] model, the [`DenialSummary`] -//! terminator, the [`DenialsDocument`] and [`DataLoopDocument`] output shapes, +//! terminator, the [`DenialsDocument`] and [`VerboseLoggingDocument`] output shapes, //! and the [`DenialAnalyzer`] decode trait. It carries no OS-specific code so //! the wire format never encodes a platform assumption. //! @@ -33,18 +33,13 @@ #![deny(missing_docs)] pub mod analyze; -pub mod data_loop; pub mod emit; pub mod model; pub mod paired_output; pub mod summary; +pub mod verbose_logging; pub use analyze::{AnalysisResult, AnalyzeError, DenialAnalyzer, ProcessLifetime}; -pub use data_loop::{ - data_loop_sibling_path, write_data_loop_document, DataLoopAggregate, DataLoopDocument, - DataLoopDocumentSummary, DataLoopExclusionReason, DataLoopProvider, DataLoopSignature, - DataLoopSummary, MAX_DATA_LOOP_GROUPS, MAX_DATA_LOOP_SIGNATURE_BYTES, -}; pub use emit::{write_document, DenialsDocument, DenialsOutputPointer}; pub use model::{AccessType, DedupKey, DeniedResource, ResourceType}; pub use paired_output::{ @@ -52,3 +47,9 @@ pub use paired_output::{ ExistingOutputPolicy, }; pub use summary::DenialSummary; +pub use verbose_logging::{ + verbose_logging_sibling_path, write_verbose_logging_document, VerboseLoggingAggregate, + VerboseLoggingDocument, VerboseLoggingDocumentSummary, VerboseLoggingExclusionReason, + VerboseLoggingProvider, VerboseLoggingSignature, VerboseLoggingSummary, + MAX_VERBOSE_LOGGING_GROUPS, MAX_VERBOSE_LOGGING_SIGNATURE_BYTES, +}; diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index a7bd19e73..494037666 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Transactional staging for the canonical and Data Loop output pair. +//! Transactional staging for the canonical and verbose logging output pair. use sha2::{Digest, Sha256}; use std::io::{Read, Seek, Write}; @@ -102,7 +102,7 @@ pub enum ExistingOutputPolicy { Replace, } -/// Stages, flushes, and promotes a canonical output and its Data Loop sibling. +/// Stages, flushes, and promotes a canonical output and its verbose logging sibling. /// /// Both documents are serialized before either final path is changed. Replace /// mode retains the previous files until both new files are promoted, allowing @@ -110,35 +110,39 @@ pub enum ExistingOutputPolicy { pub fn write_paired_output_files( operation: &str, canonical_path: &Path, - data_loop_path: &Path, + verbose_logging_path: &Path, policy: ExistingOutputPolicy, write_canonical: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, - write_data_loop: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, + write_verbose_logging: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, ) -> std::io::Result<()> { let _pair_lock = OutputPairLock::acquire(operation, canonical_path)?; match policy { ExistingOutputPolicy::CreateNew => { ensure_output_absent(operation, canonical_path, "canonical")?; - ensure_output_absent(operation, data_loop_path, "Data Loop")?; + ensure_output_absent(operation, verbose_logging_path, "verbose logging")?; } ExistingOutputPolicy::Replace => { ensure_output_replaceable(operation, canonical_path, "canonical")?; - ensure_output_replaceable(operation, data_loop_path, "Data Loop")?; + ensure_output_replaceable(operation, verbose_logging_path, "verbose logging")?; } } let canonical_temp = stage_output_file(operation, canonical_path, "canonical", write_canonical)?; - let data_loop_temp = - stage_output_file(operation, data_loop_path, "Data Loop", write_data_loop)?; + let verbose_logging_temp = stage_output_file( + operation, + verbose_logging_path, + "verbose logging", + write_verbose_logging, + )?; let canonical_backup = if policy == ExistingOutputPolicy::Replace { backup_existing_output(operation, canonical_path, "canonical")? } else { None }; - let data_loop_backup = if policy == ExistingOutputPolicy::Replace { - match backup_existing_output(operation, data_loop_path, "Data Loop") { + let verbose_logging_backup = if policy == ExistingOutputPolicy::Replace { + match backup_existing_output(operation, verbose_logging_path, "verbose logging") { Ok(backup) => backup, Err(error) => { return Err(combine_error_with_rollback( @@ -151,23 +155,27 @@ pub fn write_paired_output_files( None }; - let data_loop_promoted = - match promote_output_file(operation, data_loop_temp, data_loop_path, "Data Loop") { - Ok(promoted) => promoted, - Err(error) => { - return Err(combine_error_with_rollback( - error, - restore_output_pair( - canonical_path, - canonical_backup.as_deref(), - None, - data_loop_path, - data_loop_backup.as_deref(), - None, - ), - )) - } - }; + let verbose_logging_promoted = match promote_output_file( + operation, + verbose_logging_temp, + verbose_logging_path, + "verbose logging", + ) { + Ok(promoted) => promoted, + Err(error) => { + return Err(combine_error_with_rollback( + error, + restore_output_pair( + canonical_path, + canonical_backup.as_deref(), + None, + verbose_logging_path, + verbose_logging_backup.as_deref(), + None, + ), + )) + } + }; let canonical_promoted = match promote_output_file(operation, canonical_temp, canonical_path, "canonical") { Ok(promoted) => promoted, @@ -178,19 +186,22 @@ pub fn write_paired_output_files( canonical_path, canonical_backup.as_deref(), None, - data_loop_path, - data_loop_backup.as_deref(), - Some(data_loop_promoted), + verbose_logging_path, + verbose_logging_backup.as_deref(), + Some(verbose_logging_promoted), ), )) } }; - drop((canonical_promoted, data_loop_promoted)); - remove_backups(canonical_backup.as_deref(), data_loop_backup.as_deref()) + drop((canonical_promoted, verbose_logging_promoted)); + remove_backups( + canonical_backup.as_deref(), + verbose_logging_backup.as_deref(), + ) } -/// Copies an existing canonical/Data Loop pair to new no-clobber destinations, +/// Copies an existing canonical/verbose logging pair to new no-clobber destinations, /// then removes only source files whose identity and contents still match the /// files that were copied. /// @@ -206,17 +217,17 @@ pub fn write_paired_output_files( pub fn relocate_paired_output_files( operation: &str, canonical_source_path: &Path, - data_loop_source_path: &Path, + verbose_logging_source_path: &Path, canonical_destination_path: &Path, - data_loop_destination_path: &Path, + verbose_logging_destination_path: &Path, ) -> std::io::Result<()> { if canonical_source_path == canonical_destination_path - && data_loop_source_path == data_loop_destination_path + && verbose_logging_source_path == verbose_logging_destination_path { return Ok(()); } if canonical_source_path == canonical_destination_path - || data_loop_source_path == data_loop_destination_path + || verbose_logging_source_path == verbose_logging_destination_path { return Err(std::io::Error::other(format!( "{operation} cannot relocate only one member of an output pair" @@ -225,32 +236,35 @@ pub fn relocate_paired_output_files( let mut canonical_source = std::fs::File::open(canonical_source_path)?; let canonical_identity = PromotedOutput::from_file(canonical_source.try_clone()?)?; - let mut data_loop_source = std::fs::File::open(data_loop_source_path)?; - let data_loop_identity = PromotedOutput::from_file(data_loop_source.try_clone()?)?; + let mut verbose_logging_source = std::fs::File::open(verbose_logging_source_path)?; + let verbose_logging_identity = PromotedOutput::from_file(verbose_logging_source.try_clone()?)?; write_paired_output_files( operation, canonical_destination_path, - data_loop_destination_path, + verbose_logging_destination_path, ExistingOutputPolicy::CreateNew, |writer| { canonical_source.rewind()?; std::io::copy(&mut canonical_source, writer).map(|_| ()) }, |writer| { - data_loop_source.rewind()?; - std::io::copy(&mut data_loop_source, writer).map(|_| ()) + verbose_logging_source.rewind()?; + std::io::copy(&mut verbose_logging_source, writer).map(|_| ()) }, )?; - drop((canonical_source, data_loop_source)); - let data_loop_cleanup = - remove_promoted_output_if_owned(data_loop_source_path, data_loop_identity, None) - .and_then(cleanup_result); + drop((canonical_source, verbose_logging_source)); + let verbose_logging_cleanup = remove_promoted_output_if_owned( + verbose_logging_source_path, + verbose_logging_identity, + None, + ) + .and_then(cleanup_result); let canonical_cleanup = remove_promoted_output_if_owned(canonical_source_path, canonical_identity, None) .and_then(cleanup_result); - match (data_loop_cleanup, canonical_cleanup) { + match (verbose_logging_cleanup, canonical_cleanup) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), (Err(first), Err(second)) => Err(std::io::Error::other(format!( @@ -432,13 +446,17 @@ fn restore_output_pair( canonical_path: &Path, canonical_backup: Option<&Path>, canonical_promoted: Option, - data_loop_path: &Path, - data_loop_backup: Option<&Path>, - data_loop_promoted: Option, + verbose_logging_path: &Path, + verbose_logging_backup: Option<&Path>, + verbose_logging_promoted: Option, ) -> std::io::Result<()> { - let data_loop_result = restore_output(data_loop_path, data_loop_backup, data_loop_promoted); + let verbose_logging_result = restore_output( + verbose_logging_path, + verbose_logging_backup, + verbose_logging_promoted, + ); let canonical_result = restore_output(canonical_path, canonical_backup, canonical_promoted); - match (data_loop_result, canonical_result) { + match (verbose_logging_result, canonical_result) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), (Err(first), Err(second)) => Err(std::io::Error::other(format!( @@ -586,7 +604,7 @@ fn persist_existing_file_noclobber(source_path: &Path, final_path: &Path) -> std fn remove_backups( canonical_backup: Option<&Path>, - data_loop_backup: Option<&Path>, + verbose_logging_backup: Option<&Path>, ) -> std::io::Result<()> { let canonical_result = canonical_backup.map_or(Ok(()), |path| { remove_if_present(path).map_err(|error| { @@ -596,19 +614,19 @@ fn remove_backups( )) }) }); - let data_loop_result = data_loop_backup.map_or(Ok(()), |path| { + let verbose_logging_result = verbose_logging_backup.map_or(Ok(()), |path| { remove_if_present(path).map_err(|error| { std::io::Error::other(format!( - "failed to remove Data Loop backup {}: {error}", + "failed to remove verbose logging backup {}: {error}", path.display() )) }) }); - match (canonical_result, data_loop_result) { + match (canonical_result, verbose_logging_result) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), (Err(first), Err(second)) => Err(std::io::Error::other(format!( - "{first}; additionally failed to remove Data Loop backup: {second}" + "{first}; additionally failed to remove verbose logging backup: {second}" ))), } } @@ -641,23 +659,23 @@ mod tests { fn replace_rejects_non_file_output_before_promoting_sibling() { let directory = tempfile::tempdir().unwrap(); let canonical_path = directory.path().join("denials.json"); - let data_loop_path = directory.path().join("denials.data-loop.json"); + let verbose_logging_path = directory.path().join("denials.verbose.json"); std::fs::create_dir(&canonical_path).unwrap(); let error = write_paired_output_files( "test", &canonical_path, - &data_loop_path, + &verbose_logging_path, ExistingOutputPolicy::Replace, |writer| writer.write_all(b"canonical"), - |writer| writer.write_all(b"data-loop"), + |writer| writer.write_all(b"verbose"), ) .unwrap_err(); assert!(error .to_string() .contains("cannot replace non-file canonical")); - assert!(!data_loop_path.exists()); + assert!(!verbose_logging_path.exists()); } #[test] @@ -672,7 +690,7 @@ mod tests { &shared_path, ExistingOutputPolicy::Replace, |writer| writer.write_all(b"canonical"), - |writer| writer.write_all(b"data-loop"), + |writer| writer.write_all(b"verbose"), ) .unwrap_err(); @@ -684,15 +702,15 @@ mod tests { fn relocation_rolls_back_failed_second_promotion_and_keeps_sources() { let directory = tempfile::tempdir().unwrap(); let canonical_source = directory.path().join("source-denials.json"); - let data_loop_source = directory.path().join("source-denials.data-loop.json"); + let verbose_logging_source = directory.path().join("source-denials.verbose.json"); let shared_destination = directory.path().join("denials.json"); std::fs::write(&canonical_source, b"canonical").unwrap(); - std::fs::write(&data_loop_source, b"data-loop").unwrap(); + std::fs::write(&verbose_logging_source, b"verbose").unwrap(); let error = relocate_paired_output_files( "test relocation", &canonical_source, - &data_loop_source, + &verbose_logging_source, &shared_destination, &shared_destination, ) @@ -701,7 +719,7 @@ mod tests { assert!(error.to_string().contains("failed to promote canonical")); assert!(!shared_destination.exists()); assert_eq!(std::fs::read(canonical_source).unwrap(), b"canonical"); - assert_eq!(std::fs::read(data_loop_source).unwrap(), b"data-loop"); + assert_eq!(std::fs::read(verbose_logging_source).unwrap(), b"verbose"); } #[test] diff --git a/src/core/learning_mode_core/src/verbose_logging.rs b/src/core/learning_mode_core/src/verbose_logging.rs index 9a528a6a9..1716ad3e1 100644 --- a/src/core/learning_mode_core/src/verbose_logging.rs +++ b/src/core/learning_mode_core/src/verbose_logging.rs @@ -10,17 +10,17 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; /// Maximum number of distinct aggregate groups retained in one analysis. -pub const MAX_DATA_LOOP_GROUPS: usize = 4_096; +pub const MAX_VERBOSE_LOGGING_GROUPS: usize = 4_096; /// Maximum compact-JSON bytes retained across distinct serialized signatures. /// /// This leaves substantial headroom under the guarded WPR analysis protocol's /// 64 MiB frame limit for canonical denials and envelope overhead. -pub const MAX_DATA_LOOP_SIGNATURE_BYTES: usize = 16 * 1024 * 1024; +pub const MAX_VERBOSE_LOGGING_SIGNATURE_BYTES: usize = 16 * 1024 * 1024; /// Stable category for a known Learning Mode ETW provider. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub enum DataLoopProvider { +pub enum VerboseLoggingProvider { /// Microsoft-Windows-Kernel-General. KernelGeneral, /// Microsoft-Windows-Privacy-Auditing-PermissiveLearningMode. @@ -30,7 +30,7 @@ pub enum DataLoopProvider { /// Closed reason why a decoder outcome was omitted from canonical denials. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub enum DataLoopExclusionReason { +pub enum VerboseLoggingExclusionReason { /// The event produced a valid canonical denial candidate. CanonicalDenial, /// The provider is known, but the event ID is not a supported denial schema. @@ -55,7 +55,7 @@ pub enum DataLoopExclusionReason { NotActionable, } -impl DataLoopExclusionReason { +impl VerboseLoggingExclusionReason { /// Returns whether this outcome represents a valid denial candidate. #[must_use] pub fn is_canonical_denial(self) -> bool { @@ -63,18 +63,18 @@ impl DataLoopExclusionReason { } } -/// One sanitized event signature used as a Data Loop deduplication key. +/// One sanitized event signature used as a verbose logging deduplication key. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct DataLoopSignature { +pub struct VerboseLoggingSignature { /// Symbolic provider category. - pub provider: DataLoopProvider, + pub provider: VerboseLoggingProvider, /// Stable ETW provider GUID. pub provider_guid: String, /// Provider-scoped ETW schema identifier. pub event_id: u16, /// Closed exclusion category. - pub reason: DataLoopExclusionReason, + pub reason: VerboseLoggingExclusionReason, /// Process identifier from the event header. pub pid: u32, /// Classified access type when the event produced a denial candidate. @@ -87,12 +87,12 @@ pub struct DataLoopSignature { pub properties: Vec<(String, String)>, } -/// One deduplicated Data Loop signature and its occurrence count. +/// One deduplicated verbose logging signature and its occurrence count. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct DataLoopAggregate { +pub struct VerboseLoggingAggregate { /// Sanitized signature shared by all counted occurrences. - pub signature: DataLoopSignature, + pub signature: VerboseLoggingSignature, /// Number of matching exclusion outcomes. pub count: u64, } @@ -100,9 +100,9 @@ pub struct DataLoopAggregate { /// Bounded aggregate state produced by one decoder pass. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct DataLoopSummary { - /// Sorted sanitized signatures retained within [`MAX_DATA_LOOP_GROUPS`]. - pub signatures: Vec, +pub struct VerboseLoggingSummary { + /// Sorted sanitized signatures retained within [`MAX_VERBOSE_LOGGING_GROUPS`]. + pub signatures: Vec, /// Total recorded event outcomes, including groups collapsed into overflow. pub total_occurrences: u64, /// Outcomes whose new aggregate key could not be retained at the group bound. @@ -117,9 +117,9 @@ pub struct DataLoopSummary { pub canonical_denial_limit_reached: bool, } -impl DataLoopSummary { +impl VerboseLoggingSummary { /// Records one event outcome while preserving deterministic group order. - pub fn record(&mut self, signature: DataLoopSignature) { + pub fn record(&mut self, signature: VerboseLoggingSignature) { self.total_occurrences = self.total_occurrences.saturating_add(1); match self .signatures @@ -128,10 +128,10 @@ impl DataLoopSummary { Ok(index) => { self.signatures[index].count = self.signatures[index].count.saturating_add(1); } - Err(index) if self.signatures.len() < MAX_DATA_LOOP_GROUPS => { + Err(index) if self.signatures.len() < MAX_VERBOSE_LOGGING_GROUPS => { self.signatures.insert( index, - DataLoopAggregate { + VerboseLoggingAggregate { signature, count: 1, }, @@ -146,7 +146,7 @@ impl DataLoopSummary { .unwrap_err(); self.signatures.insert( index, - DataLoopAggregate { + VerboseLoggingAggregate { signature, count: 1, }, @@ -162,7 +162,7 @@ impl DataLoopSummary { /// Records an outcome while bounding compact serialized signature bytes. pub fn record_with_byte_budget( &mut self, - signature: DataLoopSignature, + signature: VerboseLoggingSignature, retained_bytes: &mut usize, max_bytes: usize, ) { @@ -176,7 +176,7 @@ impl DataLoopSummary { } let serialized_len = Self::serialized_signature_len(&signature); - while self.signatures.len() >= MAX_DATA_LOOP_GROUPS + while self.signatures.len() >= MAX_VERBOSE_LOGGING_GROUPS || retained_bytes.saturating_add(serialized_len) > max_bytes { if !signature.reason.is_canonical_denial() @@ -193,7 +193,7 @@ impl DataLoopSummary { .unwrap_err(); self.signatures.insert( index, - DataLoopAggregate { + VerboseLoggingAggregate { signature, count: 1, }, @@ -214,7 +214,7 @@ impl DataLoopSummary { } /// Moves a retained aggregate into overflow accounting. - pub(crate) fn move_to_overflow(&mut self, aggregate: DataLoopAggregate) { + pub(crate) fn move_to_overflow(&mut self, aggregate: VerboseLoggingAggregate) { self.overflow_occurrences = self.overflow_occurrences.saturating_add(aggregate.count); if aggregate.signature.reason.is_canonical_denial() { self.canonical_overflow_occurrences = self @@ -263,8 +263,8 @@ impl DataLoopSummary { && !self.canonical_denial_limit_reached } - fn serialized_signature_len(signature: &DataLoopSignature) -> usize { - serde_json::to_vec(&DataLoopAggregate { + fn serialized_signature_len(signature: &VerboseLoggingSignature) -> usize { + serde_json::to_vec(&VerboseLoggingAggregate { signature: signature.clone(), count: 1, }) @@ -272,22 +272,22 @@ impl DataLoopSummary { } } -/// Versioned on-disk Data Loop document. +/// Versioned on-disk verbose logging document. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct DataLoopDocument { - /// Data Loop schema version. +pub struct VerboseLoggingDocument { + /// Verbose logging schema version. pub version: u32, /// Sorted deduplicated sanitized signatures. - pub signatures: Vec, + pub signatures: Vec, /// Aggregate summary and bounds state. - pub summary: DataLoopDocumentSummary, + pub summary: VerboseLoggingDocumentSummary, } -/// Bounds and count summary for a [`DataLoopDocument`]. +/// Bounds and count summary for a [`VerboseLoggingDocument`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct DataLoopDocumentSummary { +pub struct VerboseLoggingDocumentSummary { /// Total recorded event outcomes, including overflow. pub total_occurrences: u64, /// Outcomes collapsed because the group bound was reached. @@ -302,17 +302,17 @@ pub struct DataLoopDocumentSummary { pub canonical_denial_limit_reached: bool, } -impl DataLoopDocument { - /// Current Data Loop document schema version. +impl VerboseLoggingDocument { + /// Current verbose logging document schema version. pub const VERSION: u32 = 1; /// Builds an on-disk document from decoder aggregate state. #[must_use] - pub fn new(summary: &DataLoopSummary) -> Self { + pub fn new(summary: &VerboseLoggingSummary) -> Self { Self { version: Self::VERSION, signatures: summary.signatures.clone(), - summary: DataLoopDocumentSummary { + summary: VerboseLoggingDocumentSummary { total_occurrences: summary.total_occurrences, overflow_occurrences: summary.overflow_occurrences, canonical_overflow_occurrences: summary.canonical_overflow_occurrences, @@ -324,12 +324,12 @@ impl DataLoopDocument { } } -/// Derives the deterministic Data Loop sibling for a canonical denials path. +/// Derives the deterministic verbose logging sibling for a canonical denials path. /// /// # Errors /// /// Returns an error when `output_path` has no usable file name. -pub fn data_loop_sibling_path(output_path: &Path) -> io::Result { +pub fn verbose_logging_sibling_path(output_path: &Path) -> io::Result { let file_name = output_path .file_name() .and_then(|value| value.to_str()) @@ -343,21 +343,21 @@ pub fn data_loop_sibling_path(output_path: &Path) -> io::Result { .file_stem() .and_then(|value| value.to_str()) .unwrap_or(file_name); - let sibling = format!("{stem}.data-loop.json"); + let sibling = format!("{stem}.verbose.json"); Ok(match output_path.parent() { Some(parent) if !parent.as_os_str().is_empty() => parent.join(sibling), _ => PathBuf::from(sibling), }) } -/// Writes a pretty-printed Data Loop JSON document and trailing newline. +/// Writes a pretty-printed verbose logging JSON document and trailing newline. /// /// # Errors /// /// Returns serialization or underlying writer failures. -pub fn write_data_loop_document( +pub fn write_verbose_logging_document( writer: &mut W, - document: &DataLoopDocument, + document: &VerboseLoggingDocument, ) -> io::Result<()> { serde_json::to_writer_pretty(&mut *writer, document).map_err(io::Error::other)?; writer.write_all(b"\n")?; @@ -370,12 +370,12 @@ mod tests { #[test] fn aggregates_and_sorts_sanitized_signatures() { - let mut summary = DataLoopSummary::default(); - let signature = DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + let mut summary = VerboseLoggingSummary::default(); + let signature = VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}".to_string(), event_id: 14, - reason: DataLoopExclusionReason::MissingObjectName, + reason: VerboseLoggingExclusionReason::MissingObjectName, pid: 42, access_type: None, resource_type: None, @@ -392,7 +392,7 @@ mod tests { assert_eq!(summary.signatures[0].count, 2); let mut bytes = Vec::new(); - write_data_loop_document(&mut bytes, &DataLoopDocument::new(&summary)).unwrap(); + write_verbose_logging_document(&mut bytes, &VerboseLoggingDocument::new(&summary)).unwrap(); let text = String::from_utf8(bytes).unwrap(); assert!(text.contains("S-1-15-3-1")); assert!(text.contains("")); @@ -404,55 +404,55 @@ mod tests { #[test] fn group_bound_collapses_new_keys_into_overflow() { - let mut summary = DataLoopSummary::default(); - for event_id in 0..MAX_DATA_LOOP_GROUPS as u16 { - summary.record(DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + let mut summary = VerboseLoggingSummary::default(); + for event_id in 0..MAX_VERBOSE_LOGGING_GROUPS as u16 { + summary.record(VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "kernel".to_string(), event_id, - reason: DataLoopExclusionReason::UnsupportedEventSchema, + reason: VerboseLoggingExclusionReason::UnsupportedEventSchema, pid: 1, access_type: None, resource_type: None, properties: Vec::new(), }); } - summary.record(DataLoopSignature { - provider: DataLoopProvider::PrivacyAuditingPermissiveLearningMode, + summary.record(VerboseLoggingSignature { + provider: VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode, provider_guid: "privacy".to_string(), event_id: u16::MAX, - reason: DataLoopExclusionReason::UnsupportedEventSchema, + reason: VerboseLoggingExclusionReason::UnsupportedEventSchema, pid: 1, access_type: None, resource_type: None, properties: Vec::new(), }); - assert_eq!(summary.signatures.len(), MAX_DATA_LOOP_GROUPS); + assert_eq!(summary.signatures.len(), MAX_VERBOSE_LOGGING_GROUPS); assert_eq!(summary.overflow_occurrences, 1); assert!(summary.aggregate_groups_truncated); } #[test] fn canonical_signature_evicts_noncanonical_signature_at_group_bound() { - let mut summary = DataLoopSummary::default(); - for event_id in 0..MAX_DATA_LOOP_GROUPS as u16 { - summary.record(DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + let mut summary = VerboseLoggingSummary::default(); + for event_id in 0..MAX_VERBOSE_LOGGING_GROUPS as u16 { + summary.record(VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "kernel".to_string(), event_id, - reason: DataLoopExclusionReason::UnsupportedEventSchema, + reason: VerboseLoggingExclusionReason::UnsupportedEventSchema, pid: 1, access_type: None, resource_type: None, properties: Vec::new(), }); } - summary.record(DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + summary.record(VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "kernel".to_string(), event_id: u16::MAX, - reason: DataLoopExclusionReason::CanonicalDenial, + reason: VerboseLoggingExclusionReason::CanonicalDenial, pid: 1, access_type: Some(crate::AccessType::Read), resource_type: Some(crate::ResourceType::File), @@ -462,42 +462,42 @@ mod tests { assert!(summary .signatures .iter() - .any(|group| group.signature.reason == DataLoopExclusionReason::CanonicalDenial)); + .any(|group| group.signature.reason == VerboseLoggingExclusionReason::CanonicalDenial)); assert_eq!(summary.overflow_occurrences, 1); assert_eq!(summary.canonical_overflow_occurrences, 0); } #[test] fn byte_budget_leaves_guarded_analysis_protocol_headroom() { - let mut summary = DataLoopSummary::default(); + let mut summary = VerboseLoggingSummary::default(); let mut retained_bytes = 0; let properties = (0..24) .map(|index| (format!("Property{index:02}"), "x".repeat(256))) .collect::>(); - for pid in 0..MAX_DATA_LOOP_GROUPS as u32 { + for pid in 0..MAX_VERBOSE_LOGGING_GROUPS as u32 { summary.record_with_byte_budget( - DataLoopSignature { - provider: DataLoopProvider::KernelGeneral, + VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, provider_guid: "{A68CA8B7-004F-D7B6-A698-07E2DE0F1F5D}".to_string(), event_id: 14, - reason: DataLoopExclusionReason::CanonicalDenial, + reason: VerboseLoggingExclusionReason::CanonicalDenial, pid, access_type: Some(crate::AccessType::Read), resource_type: Some(crate::ResourceType::File), properties: properties.clone(), }, &mut retained_bytes, - MAX_DATA_LOOP_SIGNATURE_BYTES, + MAX_VERBOSE_LOGGING_SIGNATURE_BYTES, ); } assert!(summary.aggregate_groups_truncated); assert!(summary.overflow_occurrences > 0); - assert!(retained_bytes <= MAX_DATA_LOOP_SIGNATURE_BYTES); + assert!(retained_bytes <= MAX_VERBOSE_LOGGING_SIGNATURE_BYTES); let payload = serde_json::to_vec(&crate::AnalysisResult { denials: Vec::new(), denied_resources_truncated: false, - data_loop: summary, + verbose_logging: summary, }) .unwrap(); assert!(payload.len() < 64 * 1024 * 1024); @@ -505,25 +505,25 @@ mod tests { #[test] fn document_round_trips() { - let mut summary = DataLoopSummary::default(); + let mut summary = VerboseLoggingSummary::default(); summary.mark_canonical_denial_limit_reached(); summary.mark_processed_events_truncated(); - let document = DataLoopDocument::new(&summary); + let document = VerboseLoggingDocument::new(&summary); let mut bytes = Vec::new(); - write_data_loop_document(&mut bytes, &document).unwrap(); - let parsed: DataLoopDocument = serde_json::from_slice(&bytes).unwrap(); + write_verbose_logging_document(&mut bytes, &document).unwrap(); + let parsed: VerboseLoggingDocument = serde_json::from_slice(&bytes).unwrap(); assert_eq!(parsed, document); } #[test] fn sibling_path_replaces_the_canonical_extension() { assert_eq!( - data_loop_sibling_path(Path::new(r"C:\out\denials.123.json")).unwrap(), - PathBuf::from(r"C:\out\denials.123.data-loop.json") + verbose_logging_sibling_path(Path::new(r"C:\out\denials.123.json")).unwrap(), + PathBuf::from(r"C:\out\denials.123.verbose.json") ); assert_eq!( - data_loop_sibling_path(Path::new("denials")).unwrap(), - PathBuf::from("denials.data-loop.json") + verbose_logging_sibling_path(Path::new("denials")).unwrap(), + PathBuf::from("denials.verbose.json") ); } } diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index 96781e047..4b2d3773d 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -61,10 +61,10 @@ pub fn finalize( .as_mut() .ok_or_else(|| "captureDenials returned no successful output metadata".to_string())?; let source_denials = PathBuf::from(&capture.output_path); - let source_data_loop = - learning_mode_core::data_loop_sibling_path(&source_denials).map_err(|error| { + let source_verbose_logging = learning_mode_core::verbose_logging_sibling_path(&source_denials) + .map_err(|error| { format!( - "failed to derive Data Loop path from {}: {error}", + "failed to derive verbose logging path from {}: {error}", source_denials.display() ) })?; @@ -85,10 +85,10 @@ pub fn finalize( source_etl.display() )); } - if !source_data_loop.is_file() { + if !source_verbose_logging.is_file() { return Err(format!( - "captureDenials Data Loop output file is missing: {}", - source_data_loop.display() + "captureDenials verbose logging output file is missing: {}", + source_verbose_logging.display() )); } @@ -106,31 +106,31 @@ pub fn finalize( ) })?; validate_metadata(capture, &document)?; - let _: learning_mode_core::DataLoopDocument = - serde_json::from_slice(&std::fs::read(&source_data_loop).map_err(|error| { + let _: learning_mode_core::VerboseLoggingDocument = + serde_json::from_slice(&std::fs::read(&source_verbose_logging).map_err(|error| { format!( - "failed to read captureDenials Data Loop output {}: {error}", - source_data_loop.display() + "failed to read captureDenials verbose logging output {}: {error}", + source_verbose_logging.display() ) })?) .map_err(|error| { format!( - "captureDenials Data Loop output {} is not valid JSON: {error}", - source_data_loop.display() + "captureDenials verbose logging output {} is not valid JSON: {error}", + source_verbose_logging.display() ) })?; let final_denials = context.log_dir.join("denials.json"); - let final_data_loop = learning_mode_core::data_loop_sibling_path(&final_denials) - .map_err(|error| format!("failed to derive audit Data Loop output path: {error}"))?; + let final_verbose_logging = learning_mode_core::verbose_logging_sibling_path(&final_denials) + .map_err(|error| format!("failed to derive audit verbose logging output path: {error}"))?; let final_etl = context.log_dir.join("trace.etl"); relocate_artifacts( capture, &ArtifactRelocation { source_denials: &source_denials, final_denials: &final_denials, - source_data_loop: &source_data_loop, - final_data_loop: &final_data_loop, + source_verbose_logging: &source_verbose_logging, + final_verbose_logging: &final_verbose_logging, source_etl: &source_etl, final_etl: &final_etl, }, @@ -186,8 +186,8 @@ fn validate_metadata( struct ArtifactRelocation<'a> { source_denials: &'a Path, final_denials: &'a Path, - source_data_loop: &'a Path, - final_data_loop: &'a Path, + source_verbose_logging: &'a Path, + final_verbose_logging: &'a Path, source_etl: &'a Path, final_etl: &'a Path, } @@ -203,9 +203,9 @@ fn relocate_artifacts( relocate_pair( "audit output relocation", paths.source_denials, - paths.source_data_loop, + paths.source_verbose_logging, paths.final_denials, - paths.final_data_loop, + paths.final_verbose_logging, ) .map_err(|error| format!("failed to relocate audit output pair: {error}"))?; capture.output_path = paths.final_denials.to_string_lossy().into_owned(); @@ -279,7 +279,9 @@ fn remove_empty_managed_directory( #[cfg(test)] mod tests { use super::*; - use learning_mode_core::{DataLoopDocument, DataLoopSummary, DenialSummary, DenialsDocument}; + use learning_mode_core::{ + DenialSummary, DenialsDocument, VerboseLoggingDocument, VerboseLoggingSummary, + }; use wxc_common::models::{ CaptureDenialsErrorOutput, CaptureDenialsOutput, SandboxOutputMetadata, }; @@ -326,13 +328,15 @@ mod tests { finalize(&mut response, &context, directory.path(), false).unwrap(); assert!(!source_denials.exists()); - assert!(!learning_mode_core::data_loop_sibling_path(&source_denials) - .unwrap() - .exists()); + assert!( + !learning_mode_core::verbose_logging_sibling_path(&source_denials) + .unwrap() + .exists() + ); assert!(!source_etl.exists()); assert!(!retained_dir.exists()); assert_eq!(std::fs::read(log_dir.join("trace.etl")).unwrap(), b"etl"); - assert!(log_dir.join("denials.data-loop.json").is_file()); + assert!(log_dir.join("denials.verbose.json").is_file()); let capture = response .output_metadata .as_ref() @@ -385,7 +389,7 @@ mod tests { Some(log_dir.join("trace.etl").as_path()) ); assert!(log_dir.join("denials.json").is_file()); - assert!(log_dir.join("denials.data-loop.json").is_file()); + assert!(log_dir.join("denials.verbose.json").is_file()); assert!(log_dir.join("trace.etl").is_file()); } @@ -435,25 +439,28 @@ mod tests { assert!(error.contains("canonical denials JSON")); assert!(source_denials.exists()); - assert!(learning_mode_core::data_loop_sibling_path(&source_denials) - .unwrap() - .exists()); + assert!( + learning_mode_core::verbose_logging_sibling_path(&source_denials) + .unwrap() + .exists() + ); assert!(source_etl.exists()); } #[test] - fn finalize_rejects_malformed_data_loop_without_moving_artifacts() { + fn finalize_rejects_malformed_verbose_logging_without_moving_artifacts() { let directory = tempfile::tempdir().unwrap(); let log_dir = directory.path().join("audit"); std::fs::create_dir_all(&log_dir).unwrap(); let source_denials = log_dir.join("denials.unique.json"); - let source_data_loop = learning_mode_core::data_loop_sibling_path(&source_denials).unwrap(); + let source_verbose_logging = + learning_mode_core::verbose_logging_sibling_path(&source_denials).unwrap(); let source_etl = log_dir.join("denials.unique.etl"); let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, false)); std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); std::fs::write(&source_etl, b"etl").unwrap(); let mut response = response_with_capture(&source_denials, &source_etl, 0); - std::fs::write(&source_data_loop, b"{").unwrap(); + std::fs::write(&source_verbose_logging, b"{").unwrap(); let context = AuditContext { log_dir: log_dir.clone(), config_path: None, @@ -461,13 +468,13 @@ mod tests { let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); - assert!(error.contains("Data Loop output")); + assert!(error.contains("verbose logging output")); assert!(error.contains("not valid JSON")); assert!(source_denials.is_file()); - assert!(source_data_loop.is_file()); + assert!(source_verbose_logging.is_file()); assert!(source_etl.is_file()); assert!(!log_dir.join("denials.json").exists()); - assert!(!log_dir.join("denials.data-loop.json").exists()); + assert!(!log_dir.join("denials.verbose.json").exists()); assert!(!log_dir.join("trace.etl").exists()); } @@ -492,9 +499,11 @@ mod tests { assert!(error.contains("canonical output file already exists")); assert!(source_denials.exists()); - assert!(learning_mode_core::data_loop_sibling_path(&source_denials) - .unwrap() - .exists()); + assert!( + learning_mode_core::verbose_logging_sibling_path(&source_denials) + .unwrap() + .exists() + ); assert!(source_etl.exists()); assert!(!log_dir.join("trace.etl").exists()); } @@ -515,7 +524,7 @@ mod tests { } #[test] - fn finalize_rejects_missing_data_loop_output() { + fn finalize_rejects_missing_verbose_logging_output() { let directory = tempfile::tempdir().unwrap(); let log_dir = directory.path().join("audit"); std::fs::create_dir_all(&log_dir).unwrap(); @@ -525,8 +534,10 @@ mod tests { std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); std::fs::write(&source_etl, b"etl").unwrap(); let mut response = response_with_capture(&source_denials, &source_etl, 0); - std::fs::remove_file(learning_mode_core::data_loop_sibling_path(&source_denials).unwrap()) - .unwrap(); + std::fs::remove_file( + learning_mode_core::verbose_logging_sibling_path(&source_denials).unwrap(), + ) + .unwrap(); let context = AuditContext { log_dir, config_path: None, @@ -534,7 +545,7 @@ mod tests { let error = finalize(&mut response, &context, directory.path(), false).unwrap_err(); - assert!(error.contains("Data Loop output file is missing")); + assert!(error.contains("verbose logging output file is missing")); assert!(source_denials.exists()); assert!(source_etl.exists()); } @@ -545,13 +556,13 @@ mod tests { let log_dir = directory.path().join("audit"); std::fs::create_dir_all(&log_dir).unwrap(); let source_denials = log_dir.join("denials.unique.json"); - let source_data_loop = log_dir.join("denials.unique.data-loop.json"); + let source_verbose_logging = log_dir.join("denials.unique.verbose.json"); let source_etl = log_dir.join("denials.unique.etl"); let final_denials = log_dir.join("denials.json"); - let final_data_loop = log_dir.join("denials.data-loop.json"); + let final_verbose_logging = log_dir.join("denials.verbose.json"); let final_etl = log_dir.join("trace.etl"); std::fs::write(&source_denials, b"denials").unwrap(); - std::fs::write(&source_data_loop, b"data-loop").unwrap(); + std::fs::write(&source_verbose_logging, b"verbose").unwrap(); std::fs::write(&source_etl, b"etl").unwrap(); let mut capture = CaptureDenialsOutput { kind: CaptureDenialsOutput::KIND.to_string(), @@ -568,8 +579,8 @@ mod tests { &ArtifactRelocation { source_denials: &source_denials, final_denials: &final_denials, - source_data_loop: &source_data_loop, - final_data_loop: &final_data_loop, + source_verbose_logging: &source_verbose_logging, + final_verbose_logging: &final_verbose_logging, source_etl: &source_etl, final_etl: &final_etl, }, @@ -590,8 +601,8 @@ mod tests { assert!(final_denials.is_file()); assert!(!source_denials.exists()); assert_eq!(Path::new(&capture.output_path), final_denials); - assert!(final_data_loop.is_file()); - assert!(!source_data_loop.exists()); + assert!(final_verbose_logging.is_file()); + assert!(!source_verbose_logging.exists()); // ETL untouched; metadata still truthfully points at the source. assert!(source_etl.is_file()); assert!(!final_etl.exists()); @@ -605,12 +616,12 @@ mod tests { fn relocate_rolls_back_json_pair_before_publishing_metadata() { let directory = tempfile::tempdir().unwrap(); let source_denials = directory.path().join("denials.unique.json"); - let source_data_loop = directory.path().join("denials.unique.data-loop.json"); + let source_verbose_logging = directory.path().join("denials.unique.verbose.json"); let source_etl = directory.path().join("denials.unique.etl"); let shared_destination = directory.path().join("denials.json"); let final_etl = directory.path().join("trace.etl"); std::fs::write(&source_denials, b"denials").unwrap(); - std::fs::write(&source_data_loop, b"data-loop").unwrap(); + std::fs::write(&source_verbose_logging, b"verbose").unwrap(); std::fs::write(&source_etl, b"etl").unwrap(); let mut capture = CaptureDenialsOutput { kind: CaptureDenialsOutput::KIND.to_string(), @@ -626,8 +637,8 @@ mod tests { &ArtifactRelocation { source_denials: &source_denials, final_denials: &shared_destination, - source_data_loop: &source_data_loop, - final_data_loop: &shared_destination, + source_verbose_logging: &source_verbose_logging, + final_verbose_logging: &shared_destination, source_etl: &source_etl, final_etl: &final_etl, }, @@ -639,7 +650,7 @@ mod tests { assert!(error.contains("failed to promote canonical")); assert_eq!(Path::new(&capture.output_path), source_denials); assert!(source_denials.is_file()); - assert!(source_data_loop.is_file()); + assert!(source_verbose_logging.is_file()); assert!(!shared_destination.exists()); assert!(source_etl.is_file()); assert!(!final_etl.exists()); @@ -650,9 +661,14 @@ mod tests { etl_path: &Path, exit_code: i32, ) -> ScriptResponse { - let data_loop_path = learning_mode_core::data_loop_sibling_path(denials_path).unwrap(); - let data_loop = DataLoopDocument::new(&DataLoopSummary::default()); - std::fs::write(data_loop_path, serde_json::to_vec(&data_loop).unwrap()).unwrap(); + let verbose_logging_path = + learning_mode_core::verbose_logging_sibling_path(denials_path).unwrap(); + let verbose_logging = VerboseLoggingDocument::new(&VerboseLoggingSummary::default()); + std::fs::write( + verbose_logging_path, + serde_json::to_vec(&verbose_logging).unwrap(), + ) + .unwrap(); ScriptResponse { exit_code, output_metadata: Some(Box::new(SandboxOutputMetadata { diff --git a/src/host/plm/src/analysis.rs b/src/host/plm/src/analysis.rs index 4efe83803..133036438 100644 --- a/src/host/plm/src/analysis.rs +++ b/src/host/plm/src/analysis.rs @@ -8,9 +8,9 @@ use std::path::Path; use anyhow::{Context, Result}; use learning_mode_core::{ - data_loop_sibling_path, write_data_loop_document, write_document, write_paired_output_files, - AccessType, AnalysisResult, DataLoopDocument, DenialAnalyzer, DenialSummary, DenialsDocument, - DeniedResource, ExistingOutputPolicy, ResourceType, + verbose_logging_sibling_path, write_document, write_paired_output_files, + write_verbose_logging_document, AccessType, AnalysisResult, DenialAnalyzer, DenialSummary, + DenialsDocument, DeniedResource, ExistingOutputPolicy, ResourceType, VerboseLoggingDocument, }; use learning_mode_windows::EtlDenialAnalyzer; @@ -24,7 +24,7 @@ pub fn analyze_trace(trace_file: &Path) -> Result { .with_context(|| format!("failed to analyze {}", trace_file.display())) } -/// Write canonical denials JSON and its Data Loop sibling, then return the +/// Write canonical denials JSON and its verbose logging sibling, then return the /// canonical document. pub fn write_denials( output_path: &Path, @@ -41,16 +41,16 @@ pub fn write_denials( analysis.denied_resources_truncated, ); let document = DenialsDocument::new(analysis.denials.clone(), summary); - let data_loop_document = DataLoopDocument::new(&analysis.data_loop); - let data_loop_path = data_loop_sibling_path(output_path) + let verbose_logging_document = VerboseLoggingDocument::new(&analysis.verbose_logging); + let verbose_logging_path = verbose_logging_sibling_path(output_path) .with_context(|| format!("failed to derive sibling for {}", output_path.display()))?; write_paired_output_files( "plm stop", output_path, - &data_loop_path, + &verbose_logging_path, ExistingOutputPolicy::Replace, |writer| write_document(writer, &document), - |writer| write_data_loop_document(writer, &data_loop_document), + |writer| write_verbose_logging_document(writer, &verbose_logging_document), )?; Ok(document) } @@ -294,31 +294,31 @@ mod tests { let analysis = AnalysisResult { denials: vec![denial(r"C:\read.txt", ResourceType::File, AccessType::Read)], denied_resources_truncated: true, - data_loop: Default::default(), + verbose_logging: Default::default(), }; write_denials(&path, &analysis, 7).unwrap(); let document: DenialsDocument = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); - let data_loop_path = learning_mode_core::data_loop_sibling_path(&path).unwrap(); - let data_loop_document: learning_mode_core::DataLoopDocument = - serde_json::from_slice(&std::fs::read(data_loop_path).unwrap()).unwrap(); + let verbose_logging_path = learning_mode_core::verbose_logging_sibling_path(&path).unwrap(); + let verbose_logging_document: learning_mode_core::VerboseLoggingDocument = + serde_json::from_slice(&std::fs::read(verbose_logging_path).unwrap()).unwrap(); assert_eq!(document.denials, analysis.denials); assert_eq!(document.summary.exit_code, 7); assert!(document.summary.denied_resources_truncated); - assert_eq!(data_loop_document.summary.total_occurrences, 0); + assert_eq!(verbose_logging_document.summary.total_occurrences, 0); } #[test] - fn canonical_promotion_failure_removes_data_loop_sibling() { + fn canonical_promotion_failure_removes_verbose_logging_sibling() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("denials.json"); std::fs::create_dir(&path).unwrap(); - let data_loop_path = learning_mode_core::data_loop_sibling_path(&path).unwrap(); + let verbose_logging_path = learning_mode_core::verbose_logging_sibling_path(&path).unwrap(); let analysis = AnalysisResult { denials: Vec::new(), denied_resources_truncated: false, - data_loop: Default::default(), + verbose_logging: Default::default(), }; let error = write_denials(&path, &analysis, 0).unwrap_err(); @@ -326,6 +326,6 @@ mod tests { assert!(error .to_string() .contains("cannot replace non-file canonical")); - assert!(!data_loop_path.exists()); + assert!(!verbose_logging_path.exists()); } } diff --git a/src/host/plm/src/log.rs b/src/host/plm/src/log.rs index 5d4998fb4..559370144 100644 --- a/src/host/plm/src/log.rs +++ b/src/host/plm/src/log.rs @@ -98,7 +98,7 @@ mod tests { assert!(!can_generate_policy_preview(&AnalysisResult { denials: Vec::new(), denied_resources_truncated: true, - data_loop: Default::default(), + verbose_logging: Default::default(), })); } } diff --git a/src/host/plm/src/stop.rs b/src/host/plm/src/stop.rs index 392aedbcf..006dffe8c 100644 --- a/src/host/plm/src/stop.rs +++ b/src/host/plm/src/stop.rs @@ -89,10 +89,10 @@ fn prepare_config_output_paths( trace_path: &Path, denials_path: &Path, ) -> Result> { - let data_loop_path = learning_mode_core::data_loop_sibling_path(denials_path) - .context("failed to derive Data Loop output path")?; + let verbose_logging_path = learning_mode_core::verbose_logging_sibling_path(denials_path) + .context("failed to derive verbose logging output path")?; if same_config_target(trace_path, denials_path) - || same_config_target(trace_path, &data_loop_path) + || same_config_target(trace_path, &verbose_logging_path) { anyhow::bail!( "trace output {} would be overwritten by a denials output derived from {}", @@ -118,7 +118,7 @@ fn prepare_config_output_paths( ] { if same_config_target(path, trace_path) || same_config_target(path, denials_path) - || same_config_target(path, &data_loop_path) + || same_config_target(path, &verbose_logging_path) { anyhow::bail!( "{label} path {} collides with a capture output", @@ -667,8 +667,8 @@ mod tests { } #[test] - fn trace_input_cannot_collide_with_data_loop_output() { - let trace = Path::new(r"C:\captures\denials.data-loop.json"); + fn trace_input_cannot_collide_with_verbose_logging_output() { + let trace = Path::new(r"C:\captures\denials.verbose.json"); let error = prepare_config_output_paths( None, Path::new(r"C:\captures"), @@ -680,8 +680,8 @@ mod tests { } #[test] - fn config_snapshot_cannot_collide_with_data_loop_output() { - let source = Path::new(r"C:\source\denials.data-loop.json"); + fn config_snapshot_cannot_collide_with_verbose_logging_output() { + let source = Path::new(r"C:\source\denials.verbose.json"); let error = prepare_config_output_paths( Some(source), Path::new(r"C:\captures"), From daf8f563ceaffc119d5d1572d993e447bf8c36c8 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Wed, 19 Aug 2026 10:04:57 -0700 Subject: [PATCH 31/34] Align fixture event accounting Route decoded test fixtures directly through the production handler so scope filtering and processing limits use the same single accounting path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode/windows/src/etl_decode.rs | 63 +++++++++++++------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 708864d8a..02cde102c 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -572,15 +572,20 @@ fn resources_from_events_for_process_lifetimes( Some(lifetimes) => Accumulator::analyze_for_process_lifetimes(lifetimes), None => Accumulator::analyze(), }; + accumulate_collected_events(events, &mut accumulator); + accumulator + .into_analysis() + .expect("pure denial accumulation cannot decode-fail") +} + +#[cfg(test)] +fn accumulate_collected_events(events: &[CollectedEvent], accumulator: &mut Accumulator<'_>) { for event in events { - if !accumulator.begin_event() { + handle_decoded_event(&event.parts, event.pid, event.filetime, accumulator); + if accumulator.stop_requested { break; } - handle_decoded_event(&event.parts, event.pid, event.filetime, &mut accumulator); } - accumulator - .into_analysis() - .expect("pure denial accumulation cannot decode-fail") } /// Streams every decoded event in the ETL to `visitor` for schema discovery @@ -1542,23 +1547,45 @@ mod tests { #[test] fn out_of_scope_events_are_excluded_before_consuming_the_budget() { - // Lifetime scoping gates the shared processing budget: an event whose - // PID/time falls outside the attested sandbox lifetimes is not in - // scope, so `process_event_record` skips it before ever calling - // `begin_event`. This asserts the scope predicate that drives that - // early return; legacy (no-lifetime) analysis treats everything as in - // scope. - let scoped = Accumulator::analyze_for_process_lifetimes(&[ProcessLifetime { + let lifetimes = [ProcessLifetime { pid: 7, start_filetime: 100, end_filetime: 200, - }]); - assert!(scoped.event_in_scope(7, 150)); - assert!(!scoped.event_in_scope(7, 250), "outside the time range"); - assert!(!scoped.event_in_scope(9, 150), "unrelated PID"); + }]; + let events = [ + kernel_event( + 14, + 9, + 150, + &[ + ("ObjectType", "\"File\""), + ("ObjectName", "\"C:\\unrelated.txt\""), + ("AccessMask", "0x1"), + ], + ), + kernel_event( + 14, + 7, + 150, + &[ + ("ObjectType", "\"File\""), + ("ObjectName", "\"C:\\owned.txt\""), + ("AccessMask", "0x1"), + ], + ), + ]; + let mut accumulator = Accumulator { + processed_event_count: MAX_PROCESSED_EVENTS - 1, + ..Accumulator::analyze_for_process_lifetimes(&lifetimes) + }; + + accumulate_collected_events(&events, &mut accumulator); - let legacy = Accumulator::analyze(); - assert!(legacy.event_in_scope(9, 150), "no lifetime filter"); + assert_eq!(accumulator.processed_event_count, MAX_PROCESSED_EVENTS); + assert!(!accumulator.processing_limit_reached); + assert!(!accumulator.stop_requested); + assert_eq!(accumulator.denials.len(), 1); + assert_eq!(accumulator.denials[0].resource, r"C:\owned.txt"); } #[test] From 7cd698c2fc74344ddbaa8c6a035ae47e10303971 Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Wed, 19 Aug 2026 10:49:01 -0700 Subject: [PATCH 32/34] Harden verbose output relocation Avoid per-run create-new lock artifacts, preserve committed relocation metadata while surfacing source-cleanup warnings, classify malformed event 28 records by schema name, and update the documented output architecture. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .github/copilot-instructions.md | 2 +- .../learning_mode/windows/src/etl_decode.rs | 64 +++++++- src/core/learning_mode_core/src/lib.rs | 2 +- .../learning_mode_core/src/paired_output.rs | 139 +++++++++++++++--- src/core/wxc/src/audit.rs | 35 +++-- src/core/wxc/src/main.rs | 31 ++-- 6 files changed, 221 insertions(+), 52 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3a1ad851a..96a354850 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -280,7 +280,7 @@ The workspace is organized into six top-level directories under `src/`: - The lower-level execution surface lives in `wxc_common::sandbox_process`: the `SandboxBackend` trait (`validate` + `spawn(request, logger, StdioMode) -> Box` + a `diagnose_exit` hook) and the generic `Runner` adapter that bridges any `SandboxBackend` to the run-to-completion `ScriptRunner` (via `spawn(StdioMode::Inherit)` then `wait()`). `SandboxProcess::output_metadata()` carries backend-produced structured outputs after terminal teardown without writing to process-global stdio. `StdioMode::Pipes` hands the caller live stdin/stdout/stderr (what the `mxc-sdk` streaming path uses); `StdioMode::Inherit` lets the child inherit the host's stdio (what the executor binaries use, preserving the TTY under a pty). `SandboxBackend` is implemented for Seatbelt, Bubblewrap, Windows ProcessContainer, and WSLC (on `wslc_common::WSLContainerRunner` itself, which shares one container lifecycle — `start_container` — between its streaming `SandboxBackend` and run-to-completion `ScriptRunner` impls, differing only in where the WSLC SDK's output callbacks write). - `mxc_ffi` (`ffi/mxc_ffi`, `crate-type = ["cdylib", "staticlib", "lib"]`) is a flat, panic-safe **C ABI over `mxc-sdk`** for language bindings. `mxc_run(policyJson, command, out)` runs a sandbox to completion, filling a `#[repr(C)] MxcRunResult` (status + exit_code + timed_out + owned stdout/stderr/error/output-metadata C strings); every entry point is `catch_unwind`-wrapped so a panic becomes a status code, never an unwind. Its `build.rs` runs **csbindgen** to generate the C# P/Invoke (`sdk/dotnet/Microsoft.Mxc.Sdk/Native/NativeMethods.g.cs`), gated behind the crate's **`dotnetsdk`** feature (off by default, so the whole-workspace backend build matrix doesn't compile csbindgen). The generated file is **not committed** (gitignored); the C# csproj regenerates it at build time and `scripts/check-dotnet-bindings-codegen.js` runs the codegen in CI and asserts the expected entry points are produced. The C ABI is **not a stable external contract** (native + binding are co-versioned and generated together; see the crate docs). It exposes three surfaces: **run-to-completion** (`mxc_run`), **streaming** (`mxc_spawn` → opaque `MxcSandbox` handle; `mxc_stream_read`/`write`/`flush`, `mxc_sandbox_take_stdin`/`stdout`/`stderr`, `mxc_sandbox_id`/`try_wait`/`wait`/`kill`/`output_metadata_json`/`free`, in `src/streaming.rs`), and the **state-aware lifecycle** (`mxc_state_aware` for the envelope phases + `mxc_state_aware_exec` returning a live streaming handle, in `src/state_aware.rs`). All three `.rs` files are csbindgen inputs in `build.rs`; the `MXC_STATUS_*` space already reserves the state-aware phase codes. - `mxc_pty` is the shared pty bridge used by the LXC backend (`lxc_common::lxc_bindings::attach_run`) so the inner shell sees a real TTY and host stdio is streamed live. (Seatbelt and Bubblewrap no longer use it: they spawn directly and let the child inherit the host's stdio — a TTY when the executor binary runs under a pty — via `SandboxBackend::spawn(StdioMode::Inherit)`.) -- `learning_mode_core` is the **cross-platform learning-mode / captureDenials model + output emitter**: `DeniedResource` (+ `ResourceType`/`AccessType`), `DenialSummary`, the `DenialAnalyzer` decode trait, and `emit` — which writes the on-disk denials deliverable as a **single JSON document** `{ "denials": [...], "summary": {...} }` (`write_document` / `DenialsDocument`) and defines the serializable `DenialsOutputPointer`. It carries no OS-specific code (must not depend on any `backends/*` crate); the Windows ETL decoder implementing `DenialAnalyzer` lives in `backends/learning_mode/windows`. When `processContainer.captureDenials` is set, native PSEC/V2 seals and decodes a managed ETL locally, while guarded WPR relogs its host-wide source into a process-scoped ETL before analysis; both routes write the same canonical JSON through shared output plumbing and return neutral `wxc_common` metadata. Explicit `retainEtl` preserves the native sealed trace or the guarded process-scoped relogged trace after a terminal wait; abandonment discards it. `wxc-exec` serializes the metadata as the one-line stderr pointer at the CLI boundary; Rust/C#/FFI callers receive it programmatically. Each denial's `resource` field holds the file path or the AppContainer capability name; capability denials resolve their capability SID to a friendly name via `backends/learning_mode/windows`'s `capability_names` (well-known `S-1-15-3-…` SID → policy name; custom hashed SIDs fall back to the SID string). +- `learning_mode_core` is the **cross-platform learning-mode / captureDenials model + output emitter**: `DeniedResource` (+ `ResourceType`/`AccessType`), `DenialSummary`, the `DenialAnalyzer` decode trait, canonical `DenialsDocument` emission, bounded sensitive-value-redacted `VerboseLoggingDocument` emission, and transactional paired-file write/relocation helpers. Every successful analysis writes the canonical policy artifact plus its deterministic `*.verbose.json` sibling; relocation reports destination commit separately from source-cleanup warnings so callers keep metadata truthful. The crate also defines the serializable `DenialsOutputPointer` and carries no OS-specific code (it must not depend on any `backends/*` crate). The Windows ETL decoder implementing `DenialAnalyzer` lives in `backends/learning_mode/windows`. When `processContainer.captureDenials` is set, native PSEC/V2 seals and decodes a managed ETL locally, while guarded WPR relogs its host-wide source into a process-scoped ETL before analysis; both routes write the same output pair through shared plumbing and return neutral `wxc_common` metadata. Explicit `retainEtl` preserves the native sealed trace or the guarded process-scoped relogged trace after a terminal wait; abandonment discards it. `wxc-exec` serializes the metadata as the one-line stderr pointer at the CLI boundary; Rust/C#/FFI callers receive it programmatically. Each canonical denial's `resource` field holds the file path, named object, UI category, or AppContainer capability name; capability denials resolve their capability SID to a friendly name via `backends/learning_mode/windows`'s `capability_names` (well-known `S-1-15-3-…` SID → policy name; custom hashed SIDs fall back to the SID string). - `mxc_build_common` is a build-time helper crate — all Windows binary crates use it in their `build.rs` to embed VersionInfo (ProductName, FileDescription, copyright, version+commit). When adding a new Windows binary crate, add `mxc_build_common` as a build-dependency and call `mxc_build_common::embed_version_info()` from `build.rs` - `nanvix_build_common` is a **build-only** helper crate (never linked into the runtime): it stages NanVix binaries next to the executable and resolves the `NANVIX_BIN` prefetch directory. The `nanvix_binaries`, `wxc`, and `lxc` build scripts consume it as a `[build-dependencies]` entry. Runtime constants it needs (binary/snapshot filenames) stay in `nanvix_common`. Keep build-only file-staging logic here, not in `nanvix_common` (which is a runtime dependency of `nanvix_runner`). - Platform-specific modules use `#[cfg(target_os = "windows")]` / `#[cfg(target_os = "linux")]` diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 02cde102c..94b486dfb 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -469,10 +469,17 @@ impl<'visitor> Accumulator<'visitor> { Some(learning_mode_core::AccessType::Unknown), Some(learning_mode_core::ResourceType::Ui), ), - crate::extractors::CAPABILITY_DENIAL_EVENT_ID => ( - Some(learning_mode_core::AccessType::Unknown), - Some(learning_mode_core::ResourceType::Capability), - ), + crate::extractors::CAPABILITY_DENIAL_EVENT_ID => match error.event_name() { + Some(name) if name.eq_ignore_ascii_case("CapabilityDenial") => ( + Some(learning_mode_core::AccessType::Unknown), + Some(learning_mode_core::ResourceType::Capability), + ), + Some(name) if name.eq_ignore_ascii_case("LearningModeViolation") => ( + Some(learning_mode_core::AccessType::Unknown), + Some(learning_mode_core::ResourceType::Ui), + ), + _ => (None, None), + }, _ => (None, None), }; self.record_outcome(category, event_id, reason, pid, classification, properties); @@ -2584,6 +2591,12 @@ mod tests { find_signature(&out.verbose_logging, 28).signature.reason, VerboseLoggingExclusionReason::UnsupportedPropertyEncoding ); + assert_eq!( + find_signature(&out.verbose_logging, 28) + .signature + .resource_type, + Some(ResourceType::Capability) + ); assert!(out.verbose_logging.signatures.iter().all(|aggregate| { aggregate .signature @@ -2592,4 +2605,47 @@ mod tests { .all(|(name, _)| name == "EventName") })); } + + #[test] + fn event_28_decode_failure_classification_requires_a_known_schema_name() { + let analyze = |event_name: Option<&str>| { + let mut accumulator = Accumulator::analyze(); + accumulator.record_event_decode_error( + crate::extractors::KERNEL_GENERAL_PROVIDER, + 28, + 11, + tdh_decode::DecodeError::event( + tdh_decode::EventDecodeKind::PayloadMalformed, + "property payload is malformed".to_string(), + event_name.map(str::to_string), + ), + ); + accumulator.into_analysis().unwrap() + }; + + assert_eq!( + find_signature(&analyze(Some("CapabilityDenial")).verbose_logging, 28) + .signature + .resource_type, + Some(ResourceType::Capability) + ); + assert_eq!( + find_signature(&analyze(Some("LearningModeViolation")).verbose_logging, 28) + .signature + .resource_type, + Some(ResourceType::Ui) + ); + assert_eq!( + find_signature(&analyze(Some("UnknownEvent28")).verbose_logging, 28) + .signature + .resource_type, + None + ); + assert_eq!( + find_signature(&analyze(None).verbose_logging, 28) + .signature + .resource_type, + None + ); + } } diff --git a/src/core/learning_mode_core/src/lib.rs b/src/core/learning_mode_core/src/lib.rs index 5d59d8888..a505b767e 100644 --- a/src/core/learning_mode_core/src/lib.rs +++ b/src/core/learning_mode_core/src/lib.rs @@ -44,7 +44,7 @@ pub use emit::{write_document, DenialsDocument, DenialsOutputPointer}; pub use model::{AccessType, DedupKey, DeniedResource, ResourceType}; pub use paired_output::{ relocate_output_file, relocate_paired_output_files, write_paired_output_files, - ExistingOutputPolicy, + ExistingOutputPolicy, RelocationOutcome, }; pub use summary::DenialSummary; pub use verbose_logging::{ diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index 494037666..20fd19478 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -102,6 +102,35 @@ pub enum ExistingOutputPolicy { Replace, } +/// Result of a successfully committed output relocation. +/// +/// Source cleanup happens after the destination is committed. Cleanup failures +/// are warnings rather than relocation failures so callers can publish the +/// authoritative destination path. +#[derive(Debug, Default)] +pub struct RelocationOutcome { + cleanup_warnings: Vec, +} + +impl RelocationOutcome { + /// Returns warnings produced while removing verified source artifacts. + pub fn cleanup_warnings(&self) -> &[String] { + &self.cleanup_warnings + } + + /// Consumes the outcome and returns its source-cleanup warnings. + pub fn into_cleanup_warnings(self) -> Vec { + self.cleanup_warnings + } + + fn record_cleanup_error(&mut self, kind: &str, path: &Path, error: std::io::Error) { + self.cleanup_warnings.push(format!( + "relocation committed, but failed to remove {kind} source {}: {error}", + path.display() + )); + } +} + /// Stages, flushes, and promotes a canonical output and its verbose logging sibling. /// /// Both documents are serialized before either final path is changed. Replace @@ -115,7 +144,10 @@ pub fn write_paired_output_files( write_canonical: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, write_verbose_logging: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, ) -> std::io::Result<()> { - let _pair_lock = OutputPairLock::acquire(operation, canonical_path)?; + let _pair_lock = match policy { + ExistingOutputPolicy::CreateNew => None, + ExistingOutputPolicy::Replace => Some(OutputPairLock::acquire(operation, canonical_path)?), + }; match policy { ExistingOutputPolicy::CreateNew => { ensure_output_absent(operation, canonical_path, "canonical")?; @@ -212,19 +244,20 @@ pub fn write_paired_output_files( /// /// # Errors /// -/// Returns an error when the source pair cannot be read, the destination pair -/// cannot be committed atomically, or verified source cleanup fails. +/// Returns an error when the source pair cannot be read or the destination pair +/// cannot be committed atomically. Source-cleanup failures are returned as +/// warnings in [`RelocationOutcome`] after destination commit. pub fn relocate_paired_output_files( operation: &str, canonical_source_path: &Path, verbose_logging_source_path: &Path, canonical_destination_path: &Path, verbose_logging_destination_path: &Path, -) -> std::io::Result<()> { +) -> std::io::Result { if canonical_source_path == canonical_destination_path && verbose_logging_source_path == verbose_logging_destination_path { - return Ok(()); + return Ok(RelocationOutcome::default()); } if canonical_source_path == canonical_destination_path || verbose_logging_source_path == verbose_logging_destination_path @@ -264,14 +297,14 @@ pub fn relocate_paired_output_files( let canonical_cleanup = remove_promoted_output_if_owned(canonical_source_path, canonical_identity, None) .and_then(cleanup_result); - match (verbose_logging_cleanup, canonical_cleanup) { - (Ok(()), Ok(())) => Ok(()), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Err(first), Err(second)) => Err(std::io::Error::other(format!( - "{first}; additionally failed to remove relocated canonical source {}: {second}", - canonical_source_path.display() - ))), + let mut outcome = RelocationOutcome::default(); + if let Err(error) = verbose_logging_cleanup { + outcome.record_cleanup_error("verbose logging", verbose_logging_source_path, error); + } + if let Err(error) = canonical_cleanup { + outcome.record_cleanup_error("canonical", canonical_source_path, error); } + Ok(outcome) } /// Copies one existing output to a new no-clobber destination, then removes @@ -279,16 +312,17 @@ pub fn relocate_paired_output_files( /// /// # Errors /// -/// Returns an error when the source cannot be read, the destination cannot be -/// committed without clobbering, or verified source cleanup fails. +/// Returns an error when the source cannot be read or the destination cannot be +/// committed without clobbering. Source-cleanup failures are returned as +/// warnings in [`RelocationOutcome`] after destination commit. pub fn relocate_output_file( operation: &str, kind: &str, source_path: &Path, destination_path: &Path, -) -> std::io::Result<()> { +) -> std::io::Result { if source_path == destination_path { - return Ok(()); + return Ok(RelocationOutcome::default()); } let mut source = std::fs::File::open(source_path)?; @@ -300,7 +334,13 @@ pub fn relocate_output_file( })?; let promoted = promote_output_file(operation, temp, destination_path, kind)?; drop((source, promoted)); - remove_promoted_output_if_owned(source_path, source_identity, None).and_then(cleanup_result) + let mut outcome = RelocationOutcome::default(); + if let Err(error) = + remove_promoted_output_if_owned(source_path, source_identity, None).and_then(cleanup_result) + { + outcome.record_cleanup_error(kind, source_path, error); + } + Ok(outcome) } fn cleanup_result(error: Option) -> std::io::Result<()> { @@ -516,12 +556,7 @@ fn remove_promoted_output_if_owned( }); drop(promoted); match matches { - Ok(true) => Ok(remove_if_present(&quarantine_path).err().map(|error| { - std::io::Error::other(format!( - "failed to remove quarantined promoted output {}: {error}", - quarantine_path.display() - )) - })), + Ok(true) => remove_quarantine_or_restore(final_path, &quarantine_path, remove_if_present), Ok(false) => { let restore_result = persist_existing_file_noclobber(&quarantine_path, final_path); if let Err(restore_error) = restore_result { @@ -548,6 +583,25 @@ fn remove_promoted_output_if_owned( } } +fn remove_quarantine_or_restore( + final_path: &Path, + quarantine_path: &Path, + remove: impl FnOnce(&Path) -> std::io::Result<()>, +) -> std::io::Result> { + match remove(quarantine_path) { + Ok(()) => Ok(None), + Err(error) => combine_error_with_restore( + std::io::Error::other(format!( + "failed to remove quarantined promoted output {}: {error}", + quarantine_path.display() + )), + persist_existing_file_noclobber(quarantine_path, final_path), + quarantine_path, + final_path, + ), + } +} + fn combine_error_with_restore( error: std::io::Error, restore: std::io::Result<()>, @@ -902,4 +956,43 @@ mod tests { drop(first); OutputPairLock::acquire("test", &canonical_path).unwrap(); } + + #[test] + fn create_new_pair_does_not_leave_a_lock_artifact() { + let directory = tempfile::tempdir().unwrap(); + let canonical_path = directory.path().join("denials.json"); + let verbose_logging_path = directory.path().join("denials.verbose.json"); + + write_paired_output_files( + "test", + &canonical_path, + &verbose_logging_path, + ExistingOutputPolicy::CreateNew, + |writer| writer.write_all(b"canonical"), + |writer| writer.write_all(b"verbose"), + ) + .unwrap(); + + assert!(!directory.path().join(".denials.json.pair.lock").exists()); + } + + #[test] + fn quarantine_cleanup_failure_restores_the_original_path() { + let directory = tempfile::tempdir().unwrap(); + let final_path = directory.path().join("source.json"); + let quarantine_path = directory.path().join("source.quarantine"); + std::fs::write(&quarantine_path, b"source").unwrap(); + + let error = remove_quarantine_or_restore(&final_path, &quarantine_path, |_| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected cleanup failure", + )) + }) + .unwrap_err(); + + assert!(error.to_string().contains("injected cleanup failure")); + assert_eq!(std::fs::read(final_path).unwrap(), b"source"); + assert!(!quarantine_path.exists()); + } } diff --git a/src/core/wxc/src/audit.rs b/src/core/wxc/src/audit.rs index 4b2d3773d..e3290a9e8 100644 --- a/src/core/wxc/src/audit.rs +++ b/src/core/wxc/src/audit.rs @@ -45,7 +45,7 @@ pub fn finalize( context: &AuditContext, exe_dir: &Path, verbose: bool, -) -> Result<(), String> { +) -> Result, String> { let metadata = response .output_metadata .as_mut() @@ -124,7 +124,7 @@ pub fn finalize( let final_verbose_logging = learning_mode_core::verbose_logging_sibling_path(&final_denials) .map_err(|error| format!("failed to derive audit verbose logging output path: {error}"))?; let final_etl = context.log_dir.join("trace.etl"); - relocate_artifacts( + let cleanup_warnings = relocate_artifacts( capture, &ArtifactRelocation { source_denials: &source_denials, @@ -153,7 +153,7 @@ pub fn finalize( ) .map_err(|error| format!("failed to generate audit compatibility artifacts: {error:#}"))?; - Ok(()) + Ok(cleanup_warnings) } fn validate_metadata( @@ -195,12 +195,18 @@ struct ArtifactRelocation<'a> { fn relocate_artifacts( capture: &mut wxc_common::models::CaptureDenialsOutput, paths: &ArtifactRelocation<'_>, - relocate_pair: impl FnOnce(&str, &Path, &Path, &Path, &Path) -> std::io::Result<()>, - mut move_file: impl FnMut(&Path, &Path) -> Result<(), String>, -) -> Result<(), String> { + relocate_pair: impl FnOnce( + &str, + &Path, + &Path, + &Path, + &Path, + ) -> std::io::Result, + mut move_file: impl FnMut(&Path, &Path) -> Result, +) -> Result, String> { ensure_destination_available(paths.source_etl, paths.final_etl)?; - relocate_pair( + let pair_outcome = relocate_pair( "audit output relocation", paths.source_denials, paths.source_verbose_logging, @@ -209,16 +215,21 @@ fn relocate_artifacts( ) .map_err(|error| format!("failed to relocate audit output pair: {error}"))?; capture.output_path = paths.final_denials.to_string_lossy().into_owned(); + let mut cleanup_warnings = pair_outcome.into_cleanup_warnings(); // Retained ETL last: on failure it stays at its source and `etl_path` // still points there (unchanged), so each artifact remains individually // truthful even though the relocation as a whole failed. - move_file(paths.source_etl, paths.final_etl)?; + let etl_outcome = move_file(paths.source_etl, paths.final_etl)?; capture.etl_path = Some(paths.final_etl.to_string_lossy().into_owned()); - Ok(()) + cleanup_warnings.extend(etl_outcome.into_cleanup_warnings()); + Ok(cleanup_warnings) } -fn move_new_file(source: &Path, destination: &Path) -> Result<(), String> { +fn move_new_file( + source: &Path, + destination: &Path, +) -> Result { learning_mode_core::relocate_output_file( "audit artifact relocation", "retained ETL", @@ -590,7 +601,9 @@ mod tests { if calls == 1 { Err("injected ETL move failure".to_string()) } else { - std::fs::rename(source, destination).map_err(|error| error.to_string()) + std::fs::rename(source, destination) + .map(|_| learning_mode_core::RelocationOutcome::default()) + .map_err(|error| error.to_string()) } }, ) diff --git a/src/core/wxc/src/main.rs b/src/core/wxc/src/main.rs index 52ef9e01a..4a08e6dcb 100644 --- a/src/core/wxc/src/main.rs +++ b/src/core/wxc/src/main.rs @@ -1225,18 +1225,25 @@ fn main() { #[cfg(target_os = "windows")] if let Some(context) = audit_context.as_ref() { if !cli.dry_run { - if let Err(error) = audit::finalize(&mut response, context, &exe_dir, cli.audit_verbose) - { - let message = format!("audit finalization failed: {error}"); - let _ = writeln!(logger, "[audit] {message}"); - eprintln!("error: {message}"); - response.exit_code = -1; - response.error_message = match response.error_message.is_empty() { - true => message, - false => format!("{}; {message}", response.error_message), - }; - } else { - eprintln!("[audit] artifacts written to {}", context.log_dir.display()); + match audit::finalize(&mut response, context, &exe_dir, cli.audit_verbose) { + Ok(cleanup_warnings) => { + for warning in cleanup_warnings { + let message = format!("[audit] warning: {warning}"); + let _ = writeln!(logger, "{message}"); + eprintln!("{message}"); + } + eprintln!("[audit] artifacts written to {}", context.log_dir.display()); + } + Err(error) => { + let message = format!("audit finalization failed: {error}"); + let _ = writeln!(logger, "[audit] {message}"); + eprintln!("error: {message}"); + response.exit_code = -1; + response.error_message = match response.error_message.is_empty() { + true => message, + false => format!("{}; {message}", response.error_message), + }; + } } } } From fe98baf1ec2c9962744015e4b5926218c319547b Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Thu, 20 Aug 2026 14:55:53 -0700 Subject: [PATCH 33/34] Address Learning Mode review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- docs/learning-mode/capabilities.md | 57 +++--- .../windows/src/capability_dacl.rs | 3 +- .../learning_mode/windows/src/etl_decode.rs | 56 +++--- .../learning_mode/windows/src/etl_filter.rs | 12 +- .../learning_mode/windows/src/extractors.rs | 186 ++++++++++-------- .../learning_mode/windows/src/tdh_decode.rs | 5 +- src/core/learning_mode_core/src/analyze.rs | 42 +++- 7 files changed, 218 insertions(+), 143 deletions(-) diff --git a/docs/learning-mode/capabilities.md b/docs/learning-mode/capabilities.md index 6ad1ae277..de1081697 100644 --- a/docs/learning-mode/capabilities.md +++ b/docs/learning-mode/capabilities.md @@ -105,9 +105,9 @@ stays enforced: hosts use native capture without PLM or UAC, while legacy or incompatible tiers use the session-scoped guarded-WPR fallback and elevate only its fixed-operation guardian. The CLI consumes the returned JSON and ETL paths, - relocates the canonical output, its verbose logging sibling, and the trace to + relocates the policy output, its verbose logging sibling, and the trace to `denials.json`, `denials.verbose.json`, and `trace.etl`, and generates the - source snapshot and `Adjusted_*.json` from canonical denials without decoding + source snapshot and `Adjusted_*.json` from the policy denials without decoding ETL again. Truncated analysis skips the adjusted config. ``` @@ -172,7 +172,7 @@ ungranted access is handled while it is recorded: ### Output file the caller consumes After the sandboxed workload exits, MXC decodes the captured denials and writes -the canonical JSON deliverable a host application reads to regenerate its +the policy JSON deliverable a host application reads to regenerate its sandbox policy: ```json @@ -204,11 +204,11 @@ sandbox policy: - `denials` is already de-duplicated per `(resource, accessType)`, so `summary.totalDenials` equals `denials.length`. - Analysis retains at most 10,000 unique denials and processes at most - 1,000,000 ETW events. Reaching the unique-denial bound stops adding canonical - entries but continues bounded aggregate accounting; reaching either bound + 1,000,000 ETW events. Reaching the unique-denial bound stops adding policy + entries but continues bounded diagnostic accounting; reaching either bound sets `summary.deniedResourcesTruncated` to `true`. - `resource` is the user-visible identifier for the denied resource, - interpreted by `resourceType`: a canonical `C:\…` path for `file`, the + interpreted by `resourceType`: an absolute `C:\…` path for `file`, the AppContainer **capability name** (e.g. `internetClient`) for `capability`, and the raw resource identifier otherwise. Named Section, SymbolicLink, and Timer access checks are emitted as `other`. Well-known capability SIDs are @@ -229,7 +229,7 @@ sandbox policy: Every successful decode also writes a deterministic sibling file: `denials..json` produces `denials..verbose.json`. This verbose logging artifact is a bounded, sensitive-value-redacted superset containing -canonical denial occurrences plus outcomes omitted from canonical denials: +policy denial occurrences plus diagnostic outcomes omitted from the policy file: ```json { @@ -272,14 +272,14 @@ non-file resource values are retained. Complete file paths are replaced with Exact header timestamps and timestamp-like properties are omitted so otherwise identical events deduplicate, and free-form decoder errors are never serialized. -Every valid denial candidate is classified as `canonicalDenial`, including its -first canonical occurrence, later duplicates, and candidates observed after -the canonical unique-denial bound. Those occurrences deduplicate under the +Every valid denial candidate is classified as `canonicalDenial` in the verbose +file, including its first occurrence, later duplicates, and candidates observed +after the policy file's unique-denial bound. Those occurrences deduplicate under the same signature and increment its count. `accessType` and `resourceType` are included when denial extraction determined them; diagnostic outcomes without those classifications omit the fields. -Candidates excluded from canonical policy output retain a closed diagnostic +Candidates excluded from the policy output retain a closed diagnostic reason and their sanitized event properties: - `unusableResourcePath` means a File access-check resource could not be @@ -287,8 +287,8 @@ reason and their sanitized event properties: `\Device\MountPointManager` is useful Devices-namespace evidence, but it is not a directly authorable filesystem grant. - `unsupportedObjectType` means the event names a resource outside the - canonical policy model. Observed Section, SymbolicLink, and Timer checks are - canonical `other` resources; remaining examples include + supported policy model. Observed Section, SymbolicLink, and Timer checks are + retained as `other` resources; remaining examples include `\BaseNamedObjects` as a Directory, ALPC Ports such as `ubpmtaskhostchannel`, and RPC Interface GUIDs. @@ -311,25 +311,18 @@ as the bounded `EventName` signature property. Free-form decoder errors are never serialized. Failure to obtain the event schema remains a fatal analysis error rather than being represented as a verbose logging signature. -Verbose logging retains at most 4,096 distinct signatures, 24 sorted properties per -signature, 256 characters per property value, and 16 MiB of compact serialized -signature data. Canonical-denial signatures take priority over diagnostic -signatures at these bounds. Additional keys are collapsed into -`overflowOccurrences`; `canonicalOverflowOccurrences` reports the subset of -canonical occurrences that could not be represented even after diagnostic -groups were evicted, and `aggregateGroupsTruncated` is set. -Before returning analysis, the decoder also fits the complete canonical plus -verbose logging result within the guarded 64 MiB transport limit by moving additional -verbose logging groups into overflow; canonical denials are never discarded. -`processedEventsTruncated` indicates that the 1,000,000-event bound prevented -complete accounting. Counts are candidate-level when extraction identifies -individual denial candidates and one event-level outcome when decoding cannot -determine candidate cardinality. - -The canonical and verbose logging files fail together: MXC stages both and reports +To keep diagnostics bounded, verbose logging retains at most 4,096 distinct +signatures, 24 sorted properties per signature, and 256 characters per property +value. `overflowOccurrences` and `aggregateGroupsTruncated` indicate that +additional diagnostic groups were omitted. `canonicalOverflowOccurrences` +counts omitted policy-denial occurrences, while `processedEventsTruncated` +indicates that the 1,000,000-event limit prevented complete accounting. The +policy file itself is never reduced to make room for verbose logging. + +The policy and verbose logging files fail together: MXC stages both and reports capture failure unless both final artifacts are committed. The verbose logging path is intentionally absent from stderr pointers and Rust, Node, C#, and FFI output -metadata; callers derive it from the canonical path using the naming rule +metadata; callers derive it from the policy output path using the naming rule above. **Locating the file.** Set `captureDenials.outputPath` to name the file @@ -345,8 +338,8 @@ so CLI callers can locate the deliverable without scanning the filesystem: {"type":"captureDenials","outputPath":"C:\\logs\\denials.4321_0123456789abcdef0123456789abcdef.json","exitCode":0,"totalDenials":2,"deniedResourcesTruncated":false} ``` -The pointer echoes the canonical file's `summary`; the authoritative policy -record is that file. In-process Rust callers receive the same information through +The pointer echoes the policy file's `summary`; that file is the authoritative +record of denials. In-process Rust callers receive the same summary information through `Output::output_metadata` or `Sandbox::output_metadata()` after waiting. The C# SDK exposes it through `RunResult.OutputMetadata` and `MxcSandboxProcess.OutputMetadata`. diff --git a/src/backends/learning_mode/windows/src/capability_dacl.rs b/src/backends/learning_mode/windows/src/capability_dacl.rs index 91dab0565..bee880627 100644 --- a/src/backends/learning_mode/windows/src/capability_dacl.rs +++ b/src/backends/learning_mode/windows/src/capability_dacl.rs @@ -254,7 +254,8 @@ pub(crate) fn extract_denials( } // `is_supported_event` above only matches the two known Learning Mode // providers, so this always resolves. - let Some(provider) = crate::extractors::provider_category(parts.provider) else { + let Some(provider) = crate::extractors::verbose_logging_provider_for_guid(parts.provider) + else { return Vec::new(); }; diff --git a/src/backends/learning_mode/windows/src/etl_decode.rs b/src/backends/learning_mode/windows/src/etl_decode.rs index 94b486dfb..6f021869a 100644 --- a/src/backends/learning_mode/windows/src/etl_decode.rs +++ b/src/backends/learning_mode/windows/src/etl_decode.rs @@ -374,7 +374,7 @@ impl<'visitor> Accumulator<'visitor> { let (access_type, resource_type) = classification; let signature = VerboseLoggingSignature { provider, - provider_guid: crate::extractors::provider_guid_string(provider), + provider_guid: crate::extractors::verbose_logging_provider_guid(provider), event_id, reason, pid, @@ -444,7 +444,7 @@ impl<'visitor> Accumulator<'visitor> { } return; } - if let Some(category) = crate::extractors::provider_category(provider) { + if let Some(category) = crate::extractors::verbose_logging_provider_for_guid(provider) { let reason = match error.event_kind() { Some(tdh_decode::EventDecodeKind::PayloadMalformed) => { VerboseLoggingExclusionReason::EventPayloadMalformed @@ -610,7 +610,8 @@ pub fn visit_raw_events( process_trace_file(source_path, &mut accumulator)?; if accumulator.processing_limit_reached { return Err(AnalyzeError::Decode(format!( - "trace exceeded the {MAX_PROCESSED_EVENTS}-event processing limit" + "trace exceeded the {MAX_PROCESSED_EVENTS}-event processing limit; \ + rerun a smaller workload or split it into multiple captureDenials runs" ))); } if let Some(error) = accumulator.decode_error { @@ -785,7 +786,7 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let event_id = header.EventDescriptor.Id; if matches!(acc.mode, CollectionMode::SelectForRelogging) { - if crate::extractors::provider_category(provider).is_none() { + if crate::extractors::verbose_logging_provider_for_guid(provider).is_none() { return; } let event_index = acc.relog_event_count; @@ -825,7 +826,7 @@ unsafe fn process_event_record(event_record: *mut EVENT_RECORD, acc: &mut Accumu let mut analyze_filetime = None; if matches!(acc.mode, CollectionMode::Analyze) { - let Some(category) = crate::extractors::provider_category(provider) else { + let Some(category) = crate::extractors::verbose_logging_provider_for_guid(provider) else { // Unrelated provider: unrelated host traffic, ignored entirely // (not aggregated as an excluded Learning Mode outcome). return; @@ -970,7 +971,8 @@ fn select_event_for_relogging( } if acc.relog_selected_event_indices.len() >= MAX_PROCESSED_EVENTS { acc.decode_error = Some(format!( - "trace exceeded the {MAX_PROCESSED_EVENTS}-event process-scoped relogging limit" + "trace exceeded the {MAX_PROCESSED_EVENTS}-event process-scoped relogging limit; \ + rerun a smaller workload or split it into multiple captureDenials runs" )); acc.stop_requested = true; return; @@ -991,7 +993,8 @@ fn handle_decoded_event( filetime: u64, acc: &mut Accumulator<'_>, ) { - let Some(category) = crate::extractors::provider_category(parts.provider) else { + let Some(category) = crate::extractors::verbose_logging_provider_for_guid(parts.provider) + else { return; }; if !is_learning_mode_event(parts.provider, parts.event_id) { @@ -1024,11 +1027,11 @@ fn handle_decoded_event( } let primary = extract_denial(parts, pid, filetime); - // Some permissive capability checks leave the primary access-check event - // unidentified (`ObjectType`/`ObjectName` both empty); the identifier may - // still be recoverable from the event's DACL payload. Each recovered - // candidate is fed independently, and the primary `UnresolvedCapability` - // outcome is only counted if none of them panned out. + // Some permissive Event 14 capability checks leave `ObjectType` and + // `ObjectName` empty. `capability_dacl::KNOWN_CAPABILITIES` defines the + // capability names recoverable from ACE SIDs in the DACL payload. Each + // recovered candidate is fed independently, and the primary + // `UnresolvedCapability` outcome is counted only when none are recovered. let capability_candidates = crate::capability_dacl::extract_denials(parts, pid, filetime); for raw in capability_candidates.iter().cloned() { acc.add_raw_denial(raw); @@ -1072,6 +1075,11 @@ mod tests { use super::*; use learning_mode_core::{AccessType, ResourceType}; + const SCOPED_PID: u32 = 42; + const SCOPED_START_FILETIME: u64 = 100; + const SCOPED_END_FILETIME: u64 = 200; + const SCOPED_EVENT_FILETIME: u64 = 150; + #[test] fn process_lifetime_index_matches_pid_and_merged_time_ranges() { let index = ProcessLifetimeIndex::new(&[ @@ -1185,9 +1193,9 @@ mod tests { #[test] fn scoped_analysis_classifies_malformed_capability_from_in_scope_emitter() { let mut accumulator = Accumulator::analyze_for_process_lifetimes(&[ProcessLifetime { - pid: 42, - start_filetime: 100, - end_filetime: 200, + pid: SCOPED_PID, + start_filetime: SCOPED_START_FILETIME, + end_filetime: SCOPED_END_FILETIME, }]); let parts = DecodedEventParts { provider: crate::extractors::KERNEL_GENERAL_PROVIDER, @@ -1195,7 +1203,7 @@ mod tests { props: Vec::new(), }; - handle_decoded_event(&parts, 42, 150, &mut accumulator); + handle_decoded_event(&parts, SCOPED_PID, SCOPED_EVENT_FILETIME, &mut accumulator); let analysis = accumulator.into_analysis().unwrap(); assert_eq!( @@ -1415,8 +1423,13 @@ mod tests { let analysis = dedup_to_resources(denials); - assert_eq!(analysis.denials.len(), 1); - assert_eq!(analysis.denials[0].resource, r"\\server\share\file.txt"); + assert_eq!( + analysis + .denials + .first() + .map(|denial| denial.resource.as_str()), + Some(r"\\server\share\file.txt") + ); let excluded = analysis .verbose_logging .signatures @@ -1740,7 +1753,8 @@ mod tests { /// empty-`ObjectType` event 14 (there is no event 28 in this mode). #[test] fn allow_shape_recovers_capability_from_dacl() { - // DWORD-padded allow ACE carrying S-1-15-3-1 (internetClient). + // DWORD-padded allow ACE containing the decimal SID S-1-15-3-1 + // (`internetClient`). let dacl = "hex:000000000000000001000000010200000000000F0300000001000000"; let events = vec![ permissive_event( @@ -2382,8 +2396,6 @@ mod tests { assert_eq!(signature.resource_type, Some(ResourceType::Capability)); } - // ---- verbose logging signature retention (identifiers + redaction + dedup) -- - fn find_signature( summary: &learning_mode_core::VerboseLoggingSummary, event_id: u16, @@ -2483,7 +2495,7 @@ mod tests { ); assert_eq!( group.signature.provider_guid, - crate::extractors::provider_guid_string(VerboseLoggingProvider::KernelGeneral) + crate::extractors::verbose_logging_provider_guid(VerboseLoggingProvider::KernelGeneral) ); assert_eq!(group.signature.pid, 42, "process identifier retained"); assert_eq!( diff --git a/src/backends/learning_mode/windows/src/etl_filter.rs b/src/backends/learning_mode/windows/src/etl_filter.rs index 746916106..1c61e2b90 100644 --- a/src/backends/learning_mode/windows/src/etl_filter.rs +++ b/src/backends/learning_mode/windows/src/etl_filter.rs @@ -19,7 +19,7 @@ use windows::Win32::System::Diagnostics::Etw::{ }; use crate::etl_decode::select_learning_mode_events_for_relogging; -use crate::extractors::{is_learning_mode_event, provider_category}; +use crate::extractors::{is_learning_mode_event, verbose_logging_provider_for_guid}; use crate::process_lifetime::{attested_process_lifetimes, JobMembershipSnapshot}; use crate::tdh_decode; @@ -101,7 +101,7 @@ impl RelogSelectionState { // lifetime bounds. Revalidate the second pass's current PID before // injection so a different equal-timestamp ordering cannot substitute // a foreign process's event at the same ordinal. Do not advance the - // selected cursor on mismatch: the final count reconciliation then + // selected-event index on mismatch: the final count reconciliation then // fails closed and the partial destination is deleted. if self.selected_event_pids.get(selected_index) != Some(&pid) || !self.attested_pids.contains(&pid) @@ -156,13 +156,13 @@ impl ITraceEventCallback_Impl for ProcessScopedTraceFilter_Impl { )); }; let header = &record.EventHeader; - if provider_category(header.ProviderId).is_none() { + if verbose_logging_provider_for_guid(header.ProviderId).is_none() { return Ok(()); } - let effective_pid = if header.EventDescriptor.Id + let is_supported_capability_event = header.EventDescriptor.Id == crate::extractors::CAPABILITY_DENIAL_EVENT_ID - && is_learning_mode_event(header.ProviderId, header.EventDescriptor.Id) - { + && is_learning_mode_event(header.ProviderId, header.EventDescriptor.Id); + let effective_pid = if is_supported_capability_event { let payload_pid = self .schema_cache .lock() diff --git a/src/backends/learning_mode/windows/src/extractors.rs b/src/backends/learning_mode/windows/src/extractors.rs index 163bbe2a8..5941a526c 100644 --- a/src/backends/learning_mode/windows/src/extractors.rs +++ b/src/backends/learning_mode/windows/src/extractors.rs @@ -133,7 +133,7 @@ pub fn extract_denial( // Callers on the real trace path gate on `is_learning_mode_event` before // decoding via TDH at all, so this branch only matters for direct/test // callers that skip that gate. - let provider = provider_category(parts.provider) + let provider = verbose_logging_provider_for_guid(parts.provider) .ok_or(VerboseLoggingExclusionReason::UnsupportedEventSchema)?; if !is_learning_mode_event(parts.provider, parts.event_id) { return Err(VerboseLoggingExclusionReason::UnsupportedEventSchema); @@ -251,7 +251,7 @@ pub(crate) fn effective_capability_event_pid(process_id: Option<&str>) -> Option /// Returns `None` for providers outside the Learning Mode vocabulary; those /// events are ignored entirely (not aggregated), since they are unrelated /// host traffic rather than an excluded Learning Mode outcome. -pub(crate) fn provider_category(provider: GUID) -> Option { +pub(crate) fn verbose_logging_provider_for_guid(provider: GUID) -> Option { if provider == KERNEL_GENERAL_PROVIDER { Some(VerboseLoggingProvider::KernelGeneral) } else if provider == PRIVACY_LEARNING_MODE_PROVIDER { @@ -261,20 +261,24 @@ pub(crate) fn provider_category(provider: GUID) -> Option String { +pub(crate) fn verbose_logging_provider_guid(provider: VerboseLoggingProvider) -> String { match provider { - VerboseLoggingProvider::KernelGeneral => format_guid(KERNEL_GENERAL_PROVIDER), + VerboseLoggingProvider::KernelGeneral => { + format_guid_braced_uppercase(KERNEL_GENERAL_PROVIDER) + } VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode => { - format_guid(PRIVACY_LEARNING_MODE_PROVIDER) + format_guid_braced_uppercase(PRIVACY_LEARNING_MODE_PROVIDER) } } } -fn format_guid(guid: GUID) -> String { +// Keep the serialized spelling independent of formatting changes in the +// `windows` crate: verbose signatures require braces and uppercase hex. +fn format_guid_braced_uppercase(guid: GUID) -> String { format!( "{{{:08X}-{:04X}-{:04X}-{:02X}{:02X}-{:02X}{:02X}{:02X}{:02X}{:02X}{:02X}}}", guid.data1, @@ -316,6 +320,30 @@ pub(crate) const MAX_SIGNATURE_VALUE_LEN: usize = 256; const SHA256_HEX_LEN: usize = 64; const BOUNDED_VALUE_MARKER_LEN: usize = "......".len(); const _: () = assert!(MAX_SIGNATURE_VALUE_LEN > BOUNDED_VALUE_MARKER_LEN); +const TIMESTAMP_PROPERTY_NAMES: &[&str] = &[ + "time", + "timestamp", + "filetime", + "systemtime", + "eventtime", + "timecreated", + "creationtime", + "createtime", + "starttime", + "endtime", + "exittime", + "lastwritetime", +]; +const RETAINED_IDENTIFIER_SUFFIXES: &[&str] = &["sid", "guid", "identifier", "id"]; +const IDENTITY_PROPERTY_NAMES: &[&str] = &["user", "account", "owner", "upn"]; +const IDENTITY_PROPERTY_SUFFIXES: &[&str] = &[ + "username", + "accountname", + "ownername", + "principalname", + "account", + "upn", +]; #[derive(Clone, Copy)] struct NormalizedPropertyName<'a>(&'a str); @@ -339,50 +367,20 @@ impl<'a> NormalizedPropertyName<'a> { .rev() .all(|expected| bytes.next() == Some(expected)) } - - fn contains(self, needle: &str) -> bool { - let needle = needle.as_bytes(); - if needle.is_empty() { - return true; - } - let mut bytes = self.bytes(); - while let Some(byte) = bytes.next() { - if byte == needle[0] { - let mut remainder = bytes.clone(); - if needle[1..] - .iter() - .all(|expected| remainder.next() == Some(*expected)) - { - return true; - } - } - } - false - } } /// Returns whether `name` looks like it carries a timestamp, so it is /// dropped from a verbose logging signature entirely (never redacted or /// truncated) — exact timestamps must not prevent otherwise-identical -/// events from deduplicating. +/// events from deduplicating. These are trusted ETW schema property names, so +/// matching is deliberately limited to known names and suffixes rather than +/// fuzzy spellings that could discard unrelated properties. fn is_timestamp_like_property(name: &str) -> bool { let normalized = NormalizedPropertyName(name); - [ - "time", - "timestamp", - "filetime", - "systemtime", - "eventtime", - "timecreated", - "creationtime", - "createtime", - "starttime", - "endtime", - "exittime", - "lastwritetime", - ] - .into_iter() - .any(|expected| normalized.equals(expected)) + TIMESTAMP_PROPERTY_NAMES + .iter() + .copied() + .any(|expected| normalized.equals(expected)) || normalized.ends_with("timestamp") || normalized.ends_with("filetime") } @@ -393,23 +391,21 @@ fn is_timestamp_like_property(name: &str) -> bool { /// content. fn is_identity_property(name: &str) -> bool { let normalized = NormalizedPropertyName(name); - if normalized.contains("sid") - || normalized.contains("guid") - || normalized.contains("identifier") - || normalized.ends_with("id") + if RETAINED_IDENTIFIER_SUFFIXES + .iter() + .copied() + .any(|suffix| normalized.equals(suffix) || normalized.ends_with(suffix)) { return false; } - normalized.equals("user") - || normalized.equals("account") - || normalized.equals("owner") - || normalized.contains("username") - || normalized.contains("accountname") - || normalized.contains("ownername") - || normalized.contains("principalname") - || normalized.ends_with("account") - || normalized.equals("upn") - || normalized.ends_with("upn") + IDENTITY_PROPERTY_NAMES + .iter() + .copied() + .any(|expected| normalized.equals(expected)) + || IDENTITY_PROPERTY_SUFFIXES + .iter() + .copied() + .any(|suffix| normalized.ends_with(suffix)) } fn looks_like_file_path_property(name: &str, value: &str, object_type: Option<&str>) -> bool { @@ -487,8 +483,9 @@ fn ends_with_ignore_ascii_case(value: &str, suffix: &str) -> bool { .is_some_and(|candidate| candidate.eq_ignore_ascii_case(suffix)) } -/// Bounds an already-sanitized property list to [`MAX_SIGNATURE_PROPERTIES`] -/// entries and [`MAX_SIGNATURE_VALUE_LEN`] characters per value. +/// Deterministically discards sorted properties after +/// [`MAX_SIGNATURE_PROPERTIES`] and bounds retained values to +/// [`MAX_SIGNATURE_VALUE_LEN`] characters. /// /// Overlong values retain prefix and suffix context plus a digest of the full /// sanitized value. The digest prevents distinct values with a long shared @@ -522,8 +519,8 @@ fn bound_property_value(value: &str, char_count: usize) -> String { format!("{prefix}{marker}{suffix}") } -/// Produces the deterministic, sanitized, bounded property list for a Data -/// Loop signature from one decoded event's raw TDH properties. +/// Produces the deterministic, sanitized, bounded property list for a verbose +/// logging signature from one decoded event's raw TDH properties. /// /// Never includes a free-form decoder error message (decode failures are /// recorded with an empty property list by the caller, before this @@ -532,7 +529,9 @@ fn bound_property_value(value: &str, char_count: usize) -> String { /// prevent their signatures from deduplicating. SIDs, capability names, /// PIDs carried as properties, and GUID-shaped values are retained /// verbatim; account/user-identity content and complete file paths are -/// redacted. +/// redacted. Classification is schema-independent: newly decoded properties +/// automatically pass through these name/value checks without requiring a new +/// extractor branch. pub(crate) fn sanitize_properties(props: &[(String, String)]) -> Vec<(String, String)> { let usernames = props .iter() @@ -1534,9 +1533,10 @@ mod tests { extract_denial(&p, 1, FIXED_FILETIME).unwrap_err(), VerboseLoggingExclusionReason::UnsupportedEventSchema ); - assert!( - provider_category(GUID::from_u128(0x12345678_1234_1234_1234_1234567890ab)).is_none() - ); + assert!(verbose_logging_provider_for_guid(GUID::from_u128( + 0x12345678_1234_1234_1234_1234567890ab + )) + .is_none()); } #[test] @@ -1567,26 +1567,33 @@ mod tests { } #[test] - fn known_provider_category_is_symbolic_and_pii_free() { + fn known_provider_guid_maps_to_symbolic_pii_free_category() { assert_eq!( - provider_category(KERNEL_GENERAL_PROVIDER), + verbose_logging_provider_for_guid(KERNEL_GENERAL_PROVIDER), Some(VerboseLoggingProvider::KernelGeneral) ); assert_eq!( - provider_category(PRIVACY_LEARNING_MODE_PROVIDER), + verbose_logging_provider_for_guid(PRIVACY_LEARNING_MODE_PROVIDER), Some(VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode) ); - assert_eq!(provider_category(GUID::from_u128(0)), None); + assert_eq!(verbose_logging_provider_for_guid(GUID::from_u128(0)), None); } #[test] - fn provider_guid_string_is_stable_and_round_trips_by_category() { - let kernel_guid = provider_guid_string(VerboseLoggingProvider::KernelGeneral); - let privacy_guid = - provider_guid_string(VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode); + fn verbose_logging_provider_guid_has_stable_spelling() { + let kernel_guid = verbose_logging_provider_guid(VerboseLoggingProvider::KernelGeneral); + let privacy_guid = verbose_logging_provider_guid( + VerboseLoggingProvider::PrivacyAuditingPermissiveLearningMode, + ); assert_ne!(kernel_guid, privacy_guid); - assert_eq!(kernel_guid, format_guid(KERNEL_GENERAL_PROVIDER)); - assert_eq!(privacy_guid, format_guid(PRIVACY_LEARNING_MODE_PROVIDER)); + assert_eq!( + kernel_guid, + format_guid_braced_uppercase(KERNEL_GENERAL_PROVIDER) + ); + assert_eq!( + privacy_guid, + format_guid_braced_uppercase(PRIVACY_LEARNING_MODE_PROVIDER) + ); assert!(kernel_guid.starts_with('{') && kernel_guid.ends_with('}')); } @@ -1835,6 +1842,29 @@ mod tests { } } + #[test] + fn identity_detection_does_not_match_embedded_name_fragments() { + let props = vec![ + ( + "AccountNamePolicy".to_string(), + "\"retain-this-value\"".to_string(), + ), + ( + "ConsiderationUserName".to_string(), + "\"redact-this-value\"".to_string(), + ), + ]; + + let out = sanitize_properties(&props); + let value_for = |name: &str| { + out.iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!(value_for("AccountNamePolicy"), Some("retain-this-value")); + assert_eq!(value_for("ConsiderationUserName"), Some(REDACTED_USER)); + } + #[test] fn sanitize_properties_redacts_unicode_username_case_variants() { let props = vec![ diff --git a/src/backends/learning_mode/windows/src/tdh_decode.rs b/src/backends/learning_mode/windows/src/tdh_decode.rs index 68d46bfb7..63ce8ebaa 100644 --- a/src/backends/learning_mode/windows/src/tdh_decode.rs +++ b/src/backends/learning_mode/windows/src/tdh_decode.rs @@ -406,8 +406,9 @@ fn decode_properties( event_record: *mut EVENT_RECORD, pointer_size: usize, ) -> Result, PropertyDecodeError> { - // SAFETY: caller passes a valid EVENT_RECORD; the field accesses - // are reads of POD fields. + // SAFETY: `decode_event_property` and the ETW callback supply a non-null + // `EVENT_RECORD` that remains valid for this synchronous decode. This + // borrow reads only its fixed-size header fields. let event = unsafe { &*event_record }; let user_data = event.UserData as *const u8; let user_data_len = event.UserDataLength as usize; diff --git a/src/core/learning_mode_core/src/analyze.rs b/src/core/learning_mode_core/src/analyze.rs index ddb4b9413..644db4b83 100644 --- a/src/core/learning_mode_core/src/analyze.rs +++ b/src/core/learning_mode_core/src/analyze.rs @@ -69,7 +69,8 @@ impl AnalysisResult { } } - /// Trims verbose logging signatures so the complete compact JSON result fits. + /// Trims verbose logging signatures so the complete compact JSON + /// serialization is no larger than `max_bytes`. /// /// Returns `false` when the canonical denials and empty verbose logging envelope /// alone exceed `max_bytes`; canonical denials are never discarded. @@ -267,7 +268,7 @@ mod tests { } #[test] - fn legacy_guarded_payload_defaults_verbose_logging_summary() { + fn analysis_payload_without_verbose_logging_defaults_summary() { let payload = br#"{"denials":[],"deniedResourcesTruncated":false}"#; let result: AnalysisResult = serde_json::from_slice(payload).unwrap(); assert!(result.verbose_logging.is_empty()); @@ -341,6 +342,43 @@ mod tests { assert!(result.verbose_logging.overflow_occurrences > 0); } + #[test] + fn serialized_size_limit_saturates_overflow_counters() { + let mut result = AnalysisResult { + denials: Vec::new(), + denied_resources_truncated: false, + verbose_logging: VerboseLoggingSummary { + signatures: vec![VerboseLoggingAggregate { + signature: VerboseLoggingSignature { + provider: VerboseLoggingProvider::KernelGeneral, + provider_guid: "provider".to_string(), + event_id: 14, + reason: VerboseLoggingExclusionReason::CanonicalDenial, + pid: 1, + access_type: Some(AccessType::Read), + resource_type: Some(ResourceType::File), + properties: vec![("ObjectName".to_string(), "x".repeat(4_096))], + }, + count: 3, + }], + total_occurrences: u64::MAX, + overflow_occurrences: u64::MAX - 1, + canonical_overflow_occurrences: u64::MAX - 1, + ..Default::default() + }, + }; + let original_len = serde_json::to_vec(&result).unwrap().len(); + + assert!(result + .fit_verbose_logging_within_serialized_bytes(original_len - 1) + .unwrap()); + assert_eq!(result.verbose_logging.overflow_occurrences, u64::MAX); + assert_eq!( + result.verbose_logging.canonical_overflow_occurrences, + u64::MAX + ); + } + #[test] fn serialized_size_limit_never_discards_canonical_denials() { let mut result = AnalysisResult { From 57c678c086b2833e4984099334432aa04b7c87be Mon Sep 17 00:00:00 2001 From: "Richie Gomez (he/him)" Date: Thu, 20 Aug 2026 15:11:25 -0700 Subject: [PATCH 34/34] Harden output backup ownership Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f9ac49b1-fdfe-4cca-9412-9faa99cef2ab --- .../learning_mode_core/src/paired_output.rs | 86 +++++++++++++++++-- src/host/plm/src/stop.rs | 5 +- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/core/learning_mode_core/src/paired_output.rs b/src/core/learning_mode_core/src/paired_output.rs index 20fd19478..19bc8927e 100644 --- a/src/core/learning_mode_core/src/paired_output.rs +++ b/src/core/learning_mode_core/src/paired_output.rs @@ -445,23 +445,71 @@ fn backup_existing_output( final_path: &Path, kind: &str, ) -> std::io::Result> { - if !final_path.try_exists().map_err(|error| { + backup_existing_output_with(operation, final_path, kind, persist_existing_file_noclobber) +} + +fn backup_existing_output_with( + operation: &str, + final_path: &Path, + kind: &str, + persist: impl FnOnce(&Path, &Path) -> std::io::Result<()>, +) -> std::io::Result> { + let file = match std::fs::File::open(final_path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(std::io::Error::other(format!( + "{operation} failed to inspect {kind} output file {}: {error}", + final_path.display() + ))) + } + }; + let expected = PromotedOutput::from_file(file).map_err(|error| { std::io::Error::other(format!( - "{operation} failed to inspect {kind} output file {}: {error}", + "{operation} failed to identify existing {kind} output file {}: {error}", final_path.display() )) - })? { - return Ok(None); - } - + })?; let backup_path = vacant_sibling_path(operation, final_path, kind)?; - persist_existing_file_noclobber(final_path, &backup_path).map_err(|error| { + persist(final_path, &backup_path).map_err(|error| { std::io::Error::other(format!( "{operation} failed to back up {kind} output file {}: {error}", final_path.display() )) })?; - Ok(Some(backup_path)) + let matches = expected.matches_path(&backup_path).map_err(|error| { + std::io::Error::other(format!( + "{operation} failed to verify backed-up {kind} output file {}: {error}", + backup_path.display() + )) + }); + drop(expected); + + match matches { + Ok(true) => Ok(Some(backup_path)), + Ok(false) => { + let error = std::io::Error::other(format!( + "{operation} detected that {kind} output {} was replaced while it was being backed up", + final_path.display() + )); + match persist_existing_file_noclobber(&backup_path, final_path) { + Ok(()) => Err(error), + Err(restore_error) => Err(std::io::Error::other(format!( + "{error}; additionally failed to restore unowned file {} to {}: {restore_error}", + backup_path.display(), + final_path.display() + ))), + } + } + Err(error) => match persist_existing_file_noclobber(&backup_path, final_path) { + Ok(()) => Err(error), + Err(restore_error) => Err(std::io::Error::other(format!( + "{error}; additionally failed to restore unverifiable file {} to {}: {restore_error}", + backup_path.display(), + final_path.display() + ))), + }, + } } fn vacant_sibling_path(operation: &str, final_path: &Path, kind: &str) -> std::io::Result { @@ -877,6 +925,28 @@ mod tests { assert!(error.to_string().contains("restore failed")); } + #[test] + fn backup_restores_concurrent_replacement_instead_of_claiming_it() { + let directory = tempfile::tempdir().unwrap(); + let final_path = directory.path().join("denials.json"); + std::fs::write(&final_path, b"original").unwrap(); + + let error = backup_existing_output_with( + "test", + &final_path, + "canonical", + |source_path, backup_path| { + std::fs::remove_file(source_path)?; + std::fs::write(source_path, b"concurrent")?; + persist_existing_file_noclobber(source_path, backup_path) + }, + ) + .unwrap_err(); + + assert!(error.to_string().contains("was replaced")); + assert_eq!(std::fs::read(&final_path).unwrap(), b"concurrent"); + } + #[test] fn promotion_error_describes_rollback_failure() { let error = combine_error_with_rollback( diff --git a/src/host/plm/src/stop.rs b/src/host/plm/src/stop.rs index 006dffe8c..41a426b22 100644 --- a/src/host/plm/src/stop.rs +++ b/src/host/plm/src/stop.rs @@ -667,7 +667,10 @@ mod tests { } #[test] - fn trace_input_cannot_collide_with_verbose_logging_output() { + fn rejects_user_selected_trace_path_that_matches_derived_verbose_output() { + // `--trace-file` is caller-controlled, while this sibling path is + // derived from denials.json. Without this check, publishing the verbose + // output would overwrite the retained ETL before post-processing ends. let trace = Path::new(r"C:\captures\denials.verbose.json"); let error = prepare_config_output_paths( None,