diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..802dfc1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "08:00" + timezone: "UTC" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + labels: + - "dependencies" + - "rust" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "08:30" + timezone: "UTC" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps(ci)" + labels: + - "dependencies" + - "ci" diff --git a/.github/workflows/auto-bump-version.yml b/.github/workflows/auto-bump-version.yml new file mode 100644 index 0000000..a62a419 --- /dev/null +++ b/.github/workflows/auto-bump-version.yml @@ -0,0 +1,63 @@ +name: auto-bump-version + +on: + push: + branches: + - main + paths: + - Cargo.lock + +permissions: + contents: write + +jobs: + bump: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Bump patch version when dependencies changed without version update + shell: bash + run: | + set -euo pipefail + + before="${{ github.event.before }}" + if [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; then + echo "No valid previous commit SHA, skipping." + exit 0 + fi + + if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then + git fetch --no-tags --depth=1 origin "$before" + fi + + prev_version="$(git show "$before:Cargo.toml" | sed -n 's/^version = "\(.*\)"/\1/p' | head -n1 || true)" + curr_version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n1 || true)" + + if [ -z "$prev_version" ] || [ -z "$curr_version" ]; then + echo "Unable to determine versions, skipping." + exit 0 + fi + + if [ "$prev_version" != "$curr_version" ]; then + echo "Version already changed ($prev_version -> $curr_version), skipping." + exit 0 + fi + + changed_files="$(git diff --name-only "$before" "$GITHUB_SHA")" + if ! printf '%s\n' "$changed_files" | grep -qx "Cargo.lock"; then + echo "No Cargo.lock change detected, skipping." + exit 0 + fi + + new_version="$(ci/bump_patch_version.sh Cargo.toml)" + echo "Bumped patch version to $new_version" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add Cargo.toml + git commit -m "chore(release): bump version to $new_version after dependency updates" + git push diff --git a/.github/workflows/auto-tag-release.yml b/.github/workflows/auto-tag-release.yml new file mode 100644 index 0000000..3ab6d80 --- /dev/null +++ b/.github/workflows/auto-tag-release.yml @@ -0,0 +1,58 @@ +name: auto-tag-release + +on: + push: + branches: + - main + paths: + - Cargo.toml + +permissions: + contents: write + +jobs: + tag: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create release tag from Cargo.toml version bump + shell: bash + run: | + set -euo pipefail + + before="${{ github.event.before }}" + if [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; then + echo "No valid previous commit SHA, skipping." + exit 0 + fi + + if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then + git fetch --no-tags --depth=1 origin "$before" + fi + + prev_version="$(git show "$before:Cargo.toml" | sed -n 's/^version = "\(.*\)"/\1/p' | head -n1 || true)" + curr_version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n1 || true)" + + if [ -z "$prev_version" ] || [ -z "$curr_version" ]; then + echo "Unable to determine versions, skipping." + exit 0 + fi + + if [ "$prev_version" = "$curr_version" ]; then + echo "Version did not change, skipping." + exit 0 + fi + + if git rev-parse -q --verify "refs/tags/$curr_version" >/dev/null; then + echo "Tag $curr_version already exists, skipping." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$curr_version" -m "$curr_version" + git push origin "$curr_version" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fb5997..75972ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,3 +159,36 @@ jobs: env: RUSTDOCFLAGS: -D warnings run: cargo doc --no-deps --document-private-items --workspace + + mrshv2-ffi-smoke: + name: mrshv2-ffi-smoke + runs-on: ubuntu-latest + env: + RUST_BACKTRACE: 1 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install packages (Ubuntu) + run: | + ci/ubuntu-install-packages + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - name: Build MRSHv2 mock adapter + shell: bash + run: | + set -euo pipefail + mock_dir="$RUNNER_TEMP/mrshv2-mock" + mkdir -p "$mock_dir" + ci/build_mrshv2_mock.sh "$mock_dir" + echo "PRECURSOR_MRSHV2_LIB_DIR=$mock_dir" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=$mock_dir:${LD_LIBRARY_PATH:-}" >> "$GITHUB_ENV" + + - name: Run tests with MRSHv2 feature enabled + shell: bash + run: | + cargo test --verbose --workspace --features similarity-mrshv2 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..39eca87 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,45 @@ +name: pages + +on: + push: + branches: + - main + paths: + - site/** + - .github/workflows/pages.yml + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Upload static site artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5996610..736da6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,7 @@ name: release on: push: tags: - - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]*.[0-9]*.[0-9]*" # We need this to be able to create releases. permissions: @@ -20,8 +20,12 @@ jobs: steps: - uses: actions/checkout@v4 - name: Get the release version from the tag - if: env.VERSION == '' - run: echo "VERSION=${{ github.ref_name }}" >> $GITHUB_ENV + id: release_version + shell: bash + run: | + version="${{ github.ref_name }}" + echo "VERSION=$version" >> $GITHUB_ENV + echo "version=$version" >> $GITHUB_OUTPUT - name: Show the version run: | echo "version is: $VERSION" @@ -37,11 +41,47 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release create $VERSION --draft --verify-tag --title $VERSION outputs: - version: ${{ env.VERSION }} + version: ${{ steps.release_version.outputs.version }} + + generate-cli-assets: + name: generate-cli-assets + needs: ["create-release"] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - name: Install help2man + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y help2man + + - name: Build precursor (host) + shell: bash + run: cargo build --release + + - name: Generate completions and man page + shell: bash + run: | + set -euo pipefail + ci/generate_cli_assets.sh dist/cli-assets ./target/release/precursor + + - name: Upload CLI assets + uses: actions/upload-artifact@v4 + with: + name: precursor-cli-assets + path: dist/cli-assets + if-no-files-found: error build-release: name: build-release - needs: ["create-release"] + needs: ["create-release", "generate-cli-assets"] runs-on: ${{ matrix.os }} env: # For some builds, we use cross to test on 32-bit and big-endian @@ -197,50 +237,11 @@ jobs: cp {README.md,COPYING,UNLICENSE,LICENSE-MIT} "$ARCHIVE"/ cp CHANGELOG.md "$ARCHIVE"/doc/ - #- name: Generate man page and completions (no emulation) - # if: matrix.qemu == '' - # shell: bash - # run: | - # "$BIN" --version - # "$BIN" --generate complete-bash > "$ARCHIVE/complete/precursor.bash" - # "$BIN" --generate complete-fish > "$ARCHIVE/complete/precursor.fish" - # "$BIN" --generate complete-powershell > "$ARCHIVE/complete/_precursor.ps1" - # "$BIN" --generate complete-zsh > "$ARCHIVE/complete/_precursor" - # "$BIN" --generate man > "$ARCHIVE/doc/precursor.1" - - #- name: Generate man page and completions (emulation) - # if: matrix.qemu != '' - # shell: bash - # run: | - # docker run --rm -v \ - # "$PWD/target:/target:Z" \ - # "rustembedded/cross:${{ matrix.target }}" \ - # "${{ matrix.qemu }}" "/$BIN" --version - # docker run --rm -v \ - # "$PWD/target:/target:Z" \ - # "rustembedded/cross:${{ matrix.target }}" \ - # "${{ matrix.qemu }}" "/$BIN" \ - # --generate complete-bash > "$ARCHIVE/complete/precursor.bash" - # docker run --rm -v \ - # "$PWD/target:/target:Z" \ - # "rustembedded/cross:${{ matrix.target }}" \ - # "${{ matrix.qemu }}" "/$BIN" \ - # --generate complete-fish > "$ARCHIVE/complete/precursor.fish" - # docker run --rm -v \ - # "$PWD/target:/target:Z" \ - # "rustembedded/cross:${{ matrix.target }}" \ - # "${{ matrix.qemu }}" "/$BIN" \ - # --generate complete-powershell > "$ARCHIVE/complete/_precursor.ps1" - # docker run --rm -v \ - # "$PWD/target:/target:Z" \ - # "rustembedded/cross:${{ matrix.target }}" \ - # "${{ matrix.qemu }}" "/$BIN" \ - # --generate complete-zsh > "$ARCHIVE/complete/_precursor" - # docker run --rm -v \ - # "$PWD/target:/target:Z" \ - # "rustembedded/cross:${{ matrix.target }}" \ - # "${{ matrix.qemu }}" "/$BIN" \ - # --generate man > "$ARCHIVE/doc/precursor.1" + - name: Download CLI assets + uses: actions/download-artifact@v4 + with: + name: precursor-cli-assets + path: ${{ env.ARCHIVE }} - name: Build archive (Windows) shell: bash @@ -300,37 +301,6 @@ jobs: run: | cargo install cargo-deb - # 'cargo deb' does not seem to provide a way to specify an asset that is - # created at build time, such as precursor's man page. To work around this, - # we force a debug build, copy out the man page (and shell completions) - # produced from that build, put it into a predictable location and then - # build the deb, which knows where to look. - - name: Build debug binary to create release assets - shell: bash - run: | - cargo build --target ${{ env.TARGET }} - bin="target/${{ env.TARGET }}/debug/precursor" - echo "BIN=$bin" >> $GITHUB_ENV - - - name: Create deployment directory - shell: bash - run: | - dir=deployment/deb - mkdir -p "$dir" - echo "DEPLOY_DIR=$dir" >> $GITHUB_ENV - - #- name: Generate man page - # shell: bash - # run: | - # "$BIN" --generate man > "$DEPLOY_DIR/precursor.1" - - #- name: Generate shell completions - # shell: bash - # run: | - # "$BIN" --generate complete-bash > "$DEPLOY_DIR/precursor.bash" - # "$BIN" --generate complete-fish > "$DEPLOY_DIR/precursor.fish" - # "$BIN" --generate complete-zsh > "$DEPLOY_DIR/_precursor" - - name: Build release binary shell: bash run: | diff --git a/.gitignore b/.gitignore index bbc72cc..d6c2e27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ target/ samples/*.json -README_files/* \ No newline at end of file +README_files/* +.env +.env.* +*.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2818d8e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,21 @@ +# AGENTS.md + +## Mission +Keep `precursor` reliable and fast as a PCRE2 + TLSH CLI for payload labeling and similarity analysis. + +## Project Memory +Read these files first when starting new work: +- `ai/MEMORY.md` +- `ai/PROMPT_STRATEGY.md` +- `ai/REPO_REVIEW.md` +- `ai/LLM_DISCOVERY_LOOP.md` + +## Repo-Local Skills +- `precursor-maintainer`: Maintain and evolve the Rust CLI, CI, and release workflow. (file: `skills/precursor-maintainer/SKILL.md`) +- `precursor-pattern-lab`: Design and validate PCRE2 pattern packs and tagging rules. (file: `skills/precursor-pattern-lab/SKILL.md`) + +## Working Rules +- Preserve output compatibility unless a breaking change is explicitly requested. +- Prefer returning structured errors over panics in ingestion and matching paths. +- Keep pattern rules centered on named capture groups; tags come from capture names. +- Update `ai/MEMORY.md` and `ai/REPO_REVIEW.md` when major behavior changes land. diff --git a/CHANGELOG.md b/CHANGELOG.md index ee7eb9b..dcec0bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,50 @@ -TBD -=== -Unreleased changes. Release notes have not yet been written. +0.2.0 - 2026-02-13 +=================== + +## Added +- LZJD similarity backend support: + - `--similarity-mode lzjd` + - backend-agnostic `similarity_hash` output remains stable +- Feature-gated MRSHv2 native adapter backend: + - `--similarity-mode mrshv2` + - compile with `--features similarity-mrshv2` + - link native adapter via `PRECURSOR_MRSHV2_LIB_DIR`/`PRECURSOR_MRSHV2_LIB_NAME` +- Single-packet protocol inference mode for matched payloads: + - `-P, --single-packet` + - `-A, --abstain-threshold <0.0-1.0>` + - `-k, --protocol-top-k ` +- New per-record inference fields: + - `protocol_label` + - `protocol_confidence` + - `protocol_abstained` + - `protocol_candidates` +- Optional blob ingestion mode: + - `-z, --input-blob` processes each file/stdin stream as a single payload record. +- Integration tests for CLI output contract: + - protocol inference fields + - protocol hint JSON emission + - multiline blob matching behavior +- Scenario corpus and scenario integration coverage: + - `samples/scenarios/` (packet triage, firmware fragments, ICS Modbus) + - `tests/scenario_corpus_contract.rs` +- Scenario benchmark harness and baseline snapshot: + - `ci/benchmark_scenarios.sh` + - `benchmarks/baseline-2026-02-13.md` +- GitHub Pages site for demos: + - `site/` + - `.github/workflows/pages.yml` +- Repository roadmap: `ROADMAP.md` + +## Changed +- Protocol hints now include inference context fields (`protocol_label`, `protocol_confidence`, `protocol_abstained`) when present. +- README and architecture diagram were updated to reflect current CLI behavior and data flow. +- Release workflow now exports tag version through a step output to avoid empty downstream version values. +- Release workflow now generates shell completions and man page once and includes them in release archives. + +## Fixed +- Replaced panic-prone file ingestion `expect(...)` paths with recoverable error handling. + +## Known limitations +- Blob mode supports raw bytes in `string` mode, and UTF-8 encoded `base64`/`hex` wrappers for encoded modes. +- FBHash backend mode remains scaffolded. +- MRSHv2 depends on a native adapter library when `similarity-mrshv2` is enabled. diff --git a/Cargo.lock b/Cargo.lock index b545080..c0dbad9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,16 +182,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" -[[package]] -name = "clap_mangen" -version = "0.2.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "724842fa9b144f9b89b3f3d371a89f3455eea660361d13a554f68f8ae5d6c13a" -dependencies = [ - "clap", - "roff", -] - [[package]] name = "colorchoice" version = "1.0.3" @@ -566,12 +556,11 @@ checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "precursor" -version = "0.1.0" +version = "0.2.0" dependencies = [ "atomic-counter", "base64", "clap", - "clap_mangen", "dashmap", "hex", "indicatif", @@ -667,12 +656,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "roff" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" - [[package]] name = "ryu" version = "1.0.20" diff --git a/Cargo.toml b/Cargo.toml index 569176c..97bd247 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,29 @@ [package] name = "precursor" build = "build.rs" -version = "0.1.1" +version = "0.2.0" +edition = "2021" +rust-version = "1.86" authors = ["Matt Lehman "] -homepage = "https://github.com/GreyNoise-Intelligence/precursor" -repository = "https://github.com/GreyNoise-Intelligence/precursor" -description = "A data analysis tool for text and binary tagging and filtering with similarity comparisons." +homepage = "https://precursor.hashdb.io" +repository = "https://github.com/Obsecurus/precursor" +documentation = "https://github.com/Obsecurus/precursor#readme" +description = "Pre-protocol payload tagging, similarity clustering, and packet/firmware triage CLI." readme = "README.md" -keywords = ["binary", "tool", "analysis", "similarity", "hashing"] -license = "MIT" +keywords = ["ids", "forensics", "similarity", "packet", "firmware"] +license = "MIT OR Unlicense" categories = ["command-line-utilities", "filesystem", "datascience"] +exclude = [ + ".env", + ".env.*", + ".DS_Store", + "samples/.DS_Store", + "README_files/*", +] + +[features] +default = [] +similarity-mrshv2 = [] [dependencies] xxhash-rust = { version = "0.8.0", features = ["xxh3", "const_xxh3"] } @@ -33,6 +47,3 @@ path = "src/main.rs" [profile.release] debug = false - -[build-dependencies] -clap_mangen = "0.2.14" diff --git a/README.md b/README.md index 0f217d6..415d2c8 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,283 @@ -# UNDER CONSTRUCTION -![UNDER CONSTRUCTION](https://github.com/GreyNoise-Intelligence/precursor/assets/30487781/066ff068-68c0-46c3-8c28-32a3f68cd14e) +# precursor -Please pardon the our rust! ;) +

+ Precursor logo +

-## SEVERLY LACKING DOCUMENTATION AND WILL LIKELY BREAK! +`precursor` is a CLI for **pre-protocol payload tagging + similarity clustering**. +It combines PCRE2 named-capture matching, TLSH/LZJD similarity, optional MRSHv2 adapter mode, and JSON outputs that are easy to feed into detection engineering and LLM-assisted protocol discovery loops. +Project page: https://precursor.hashdb.io -![precursor](./architecture.png) +## Release 0.2.0 Highlights -# precursor +| Area | What landed | +| --- | --- | +| Packet Inference | Single-packet protocol scoring via `-P` / `-A` / `-k` | +| Blob Processing | `-z, --input-blob` for multiline or stream-as-one-record analysis | +| Similarity Workflows | TLSH or LZJD clustering + protocol hints (`--protocol-hints`) for discovery loops | +| Output Contract | Stable `protocol_*`, `similarity_hash`, `tags`, `xxh3_64_sum` JSON fields | +| Reliability | Runtime ingest path no longer relies on panic-prone `expect(...)` calls | +| Scenario Corpus | Versioned packet/firmware/ICS samples in `samples/scenarios/` | +| Release Ops | Dependency auto-bump/tag workflows + benchmark harness + Pages site | + +> [!IMPORTANT] +> **Known limitation:** in blob mode (`-z`), raw bytes are fully supported in `string` mode, while `base64` and `hex` blob decoding currently expects UTF-8 wrapper text. + +## 60-second teaser + +```bash +cat samples/scenarios/pre-protocol-packet-triage/payloads.b64 \ + | precursor -p samples/scenarios/pre-protocol-packet-triage/patterns.pcre \ + -m base64 -t -d --similarity-mode lzjd -P --protocol-hints +``` + +Representative output shape: -`precursor` is a command-line tool for searching files and directories using regular expressions. It supports searching against a rules file with named rules/patterns and outputting each match with both the name of the rule and the matched bytes. It also supports searching against a single pattern in the named format and optionally reading from STDIN for the input to search against. Additionally, it can output matches in JSON format and optionally summarize the matches by rule name and count. +```json +{ + "tags": ["http_method"], + "similarity_hash": "lzjd:128:...", + "protocol_label": "http", + "protocol_confidence": 0.93, + "protocol_candidates": [ + {"protocol": "http", "score": 0.93, "evidence": ["matched HTTP request/headers"]} + ], + "xxh3_64_sum": "..." +} +``` -## TODO - 1. Better comments and documentation - 2. Maybe support PCAP with some mechanism that parses the PCAP into some consistent newline structure for specific protocols? - 3. Add a mode to disable pattern matching, which could just essentially inject a `(?.*)` pattern but that may be much slower. - 4. Add a training mode that uses a sample to find the optimal TLSH algorithm and distance for finding similairites within the supplied input. - 5. Add support for binary inputs - 6. Add support for processing entire input vs newline split (requirement for binary input) - 7. Use roaring bitmap for persistance - 8. Add ability to mask chunks based on a pattern for TLSH/FBHash - 9. Some very basic tests - 10. Refactor things so that CLI is isolated from using `precursor` as a library +Why this matters: +- Fast pre-protocol triage when DPI/parsers are unavailable. +- Stable JSON fields for SOC pipelines and enrichment tooling. +- Built-in cluster context for LLM-assisted protocol discovery. -## Installation +## Best Fit -To install `precursor`, you need to have Rust installed on your system. You can download and install Rust from the official website[1]. Once Rust is installed, you can install `precursor` using the following command: +- Rapid triage of payload lines from logs, brokers, sensors, or ad-hoc captures. +- Label-first workflows where regex capture names become downstream tags. +- Similarity-first clustering when full protocol parsers are unavailable or too brittle. +- Early-stage protocol discovery where hints are fed to humans or LLM tooling. +## Non-Goals +- Not a replacement for full IDS/NSM stacks (Suricata, Zeek). +- Not a malware rule engine replacement (YARA / YARA-X). +- Not yet a full raw-binary parser framework; blob mode currently expects UTF-8 wrappers for `base64`/`hex` decode modes. + +## Architecture + +

+ Precursor architecture diagram +

+ +## Install + +### Cargo + +```bash cargo install precursor +``` -This will download and install the latest version of `precursor` from the official Rust package registry. +### From source -## Usage +```bash +git clone https://github.com/Obsecurus/precursor.git +cd precursor +cargo build --release +./target/release/precursor --help +``` -To use `precursor`, run the following command: +Release archives include generated shell completion files (`bash`, `fish`, `zsh`, `powershell`) and a `precursor.1` man page. +### Optional: Build with MRSHv2 adapter mode -precursor +```bash +mock_dir="$(mktemp -d)" +ci/build_mrshv2_mock.sh "$mock_dir" +PRECURSOR_MRSHV2_LIB_DIR="$mock_dir" cargo build --features similarity-mrshv2 +``` -Replace `` with the pattern to search for in the named format, and `` with the name of the file to search in. For example: +## Quick start +### 1) Match string payloads from stdin -precursor "rule1:foo\d+" input.txt +```bash +printf 'hello world\nbye world\n' \ + | precursor '(?hello)' -m string +``` -This will search for the specified pattern in the input file and output the matches to the console. +### 2) Match base64 payloads + +```bash +printf 'aGVsbG8gd29ybGQ=\n' \ + | precursor '(?hello)' -m base64 +``` + +### 3) Load patterns from file + +```bash +printf 'aGVsbG8gd29ybGQ=\n' \ + | precursor -p patterns/new -m base64 +``` + +### 4) Extract payload from JSON before matching + +```bash +printf '{"payload":"aGVsbG8gd29ybGQ="}\n' \ + | precursor '(?hello)' -j '.payload' -m base64 +``` -Alternatively, you can run `precursor` with the `-p` flag to specify a pattern file in the named format: +### 5) Enable similarity diffing (TLSH default) +```bash +cat payloads.b64 \ + | precursor -p patterns/new -m base64 -t -d -x 80 +``` -precursor -p pattern.txt input.txt +### 6) Switch to LZJD similarity mode -Replace `pattern.txt` with the name of the pattern file. +```bash +cat payloads.raw \ + | precursor -p patterns/new -m string -t -d --similarity-mode lzjd -x 80 +``` -By default, `precursor` outputs JSON with the rule name and the match value. You can use the `--only-matches` flag to output only the matching string along with the rule name it matched on. You can use the `--json` flag to output a single JSON file summarizing just the rule name and match counts. For example: +### 7) Emit protocol-discovery hints for an LLM loop +```bash +cat payloads.b64 \ + | precursor -p patterns/new -m base64 -t -d --protocol-hints --protocol-hints-limit 20 +``` -precursor "rule1:foo\d+" input.txt --only-matches --json > output.json +### 8) Enable single-packet protocol inference output -This will search for the specified pattern in the input file with the specified options and output the results to a file named `output.json`. +```bash +cat payloads.b64 \ + | precursor -p patterns/new -m base64 -P -A 0.7 -k 5 +``` -## License +### 9) Match a multiline payload as one blob -`precursor` is is dual-licensed under the Unlicense and MIT licenses. +```bash +printf 'GET /blob HTTP/1.1\nHost: blob.example\n' \ + | precursor '(?GET /blob HTTP/1\.1\nHost: blob\.example)' -m string -z +``` -You may use this code under the terms of either license. +## CLI reference -## Acknowledgements +```text +precursor [PATTERN] [OPTIONS] +``` -`precursor` is inspired by various Rust command-line tools and libraries, including `ripgrep`, `grep`, and `pcre2`. Special thanks to the Rust community for creating and maintaining these amazing tools and libraries. +Pattern source: +- positional `PATTERN` (single named-capture regex) +- `-p, --pattern-file ` (one named-capture pattern per line) -In fact much of the build system is verbatim from `ripgrep` as I thought there may be a future where this gets folded in as a module some how. Thank you [BurntSushi](https://github.com/BurntSushi)! +Input: +- `-f, --input-folder `: read newline-delimited content from files +- stdin: read newline-delimited input from standard input +- `-z, --input-blob`: process each input source as one blob instead of line splitting +- `-m, --input-mode `: decode mode (default: `base64`) +- `-j, --input-json-key `: extract payload from JSON input first -## Contributing +Similarity: +- `-t, --tlsh`: compute TLSH hash for matched payloads +- `-d, --tlsh-diff`: compute pairwise TLSH distance among matched payloads +- `-a, --tlsh-algorithm <48_1|128_1|128_3|256_1|256_3>` +- `-x, --tlsh-distance `: max distance threshold (default: `100`) +- `-l, --tlsh-length`: include payload length in diff scoring +- `-y, --tlsh-sim-only`: only output payloads that have TLSH similarities +- `--similarity-mode `: + - `tlsh` and `lzjd` are implemented in default builds + - `mrshv2` is implemented behind `--features similarity-mrshv2` and native adapter linking + - `fbhash` remains scaffolded +- `--protocol-hints`: emit LLM-oriented protocol-discovery hint JSON to `stderr` +- `--protocol-hints-limit `: limit hint candidate count (default: `25`) +- `-P, --single-packet`: enable heuristic protocol inference on each matched payload +- `-A, --abstain-threshold <0.0-1.0>`: minimum confidence required to emit a non-`unknown` label (default: `0.65`) +- `-k, --protocol-top-k `: candidate count included in `protocol_candidates` (default: `3`) + +Other: +- `-s, --stats`: emit run statistics JSON to `stderr` + +## Output model + +Each matched payload is emitted as JSON on `stdout` with fields such as: +- `tags`: array of matched capture names +- `tlsh`: active similarity hash when enabled (legacy field name preserved for compatibility) +- `similarity_hash`: backend-agnostic similarity hash field +- `xxh3_64_sum`: stable payload key for report correlation +- `tlsh_similarities`: distance map when `--tlsh-diff` is enabled +- `protocol_label`: top protocol guess (or `unknown` when abstaining) +- `protocol_confidence`: confidence score for `protocol_label` +- `protocol_abstained`: whether inference abstained under threshold +- `protocol_candidates`: scored candidate list with evidence strings + +When `--stats` is enabled, a summary JSON object is emitted to `stderr`. +When `--protocol-hints` is enabled, an additional hint JSON block is emitted to `stderr` for LLM-guided protocol discovery workflows, including `protocol_*` fields when single-packet inference is enabled. +When both `--single-packet` and `--tlsh-diff` are enabled, protocol confidence is cluster-boosted using similarity neighbor counts. +When `--input-blob` is enabled, each file/stdin stream is treated as a single candidate payload. + +## Positioning vs adjacent tools + +- Use **Suricata/Zeek** for full protocol-aware IDS/NSM and rich ecosystem integrations. +- Use **YARA/YARA-X** for signature-based scanning of files and malware-centric workflows. +- Use **Precursor** when you need lightweight, custom payload tagging plus TLSH/LZJD similarity in one CLI pipeline. + +## Scenario corpus and demos + +- Scenario corpus: `samples/scenarios/` +- Demo runner: `samples/scenarios/run_all.sh` +- Static demo site source: `site/` + +```bash +samples/scenarios/run_all.sh ./target/release/precursor +``` + +## Benchmarks + +Generate a reproducible scenario snapshot: + +```bash +cargo build --release +ci/benchmark_scenarios.sh ./target/release/precursor benchmarks/latest.md +``` + +Committed baseline: +- `benchmarks/baseline-2026-02-13.md` + +## GitHub Pages + precursor.hashdb.io + +- Pages workflow: `.github/workflows/pages.yml` +- Site content: `site/` +- Custom domain file: `site/CNAME` +- Configure DNS at your provider with: + - record type: `CNAME` + - host: `precursor` + - value: `obsecurus.github.io` + +## Current roadmap + +See `ROADMAP.md` for prioritized milestones and release criteria. +See `SIMILARITY_BACKENDS.md` for MRSHv2/FBHash feasibility and backend sequencing. + +## Development + +```bash +cargo fmt --all --check +cargo test --workspace +mock_dir="$(mktemp -d)" +ci/build_mrshv2_mock.sh "$mock_dir" >/dev/null +PRECURSOR_MRSHV2_LIB_DIR="$mock_dir" LD_LIBRARY_PATH="$mock_dir:${LD_LIBRARY_PATH:-}" cargo test --workspace --features similarity-mrshv2 +``` + +CI currently tests multiple toolchains and targets, including a pinned Rust `1.86.0` lane. +Dependabot is configured for weekly Cargo and GitHub Actions updates, with auto patch-version bump + auto-tag workflows so dependency updates can flow into release builds. + +## Background + +- GreyNoise blog: https://www.greynoise.io/blog/precursor-a-quantum-leap-in-arbitrary-payload-similarity-analysis +- GreyNoise Labs writeup: https://www.labs.greynoise.io/grimoire/2023-10-11-precursor/ + +## License -If you find a bug or have a feature request, please open an issue on the GitHub repository[2]. Pull requests are also welcome! +Dual-licensed under MIT and Unlicense. diff --git a/RELEASE-CHECKLIST.md b/RELEASE-CHECKLIST.md index 1200a67..1189927 100644 --- a/RELEASE-CHECKLIST.md +++ b/RELEASE-CHECKLIST.md @@ -1,59 +1,48 @@ # Release Checklist -* Ensure local `main` is up to date with respect to `origin/main`. -* Run `cargo update` and review dependency updates. Commit updated - `Cargo.lock`. -* Run `cargo outdated` and review semver incompatible updates. Unless there is - a strong motivation otherwise, review and update every dependency. Also - run `--aggressive`, but don't update to crates that are still in beta. -* Update date in `crates/core/flags/doc/template.rg.1`. -* Review changes for every crate in `crates` since the last ripgrep release. - If the set of changes is non-empty, issue a new release for that crate. Check - crates in the following order. After updating a crate, ensure minimal - versions are updated as appropriate in dependents. If an update is required, - run `cargo-up --no-push crates/{CRATE}/Cargo.toml`. - * crates/globset - * crates/ignore - * crates/cli - * crates/matcher - * crates/regex - * crates/pcre2 - * crates/searcher - * crates/printer - * crates/grep (bump minimal versions as necessary) - * crates/core (do **not** bump version, but update dependencies as needed) -* Update the CHANGELOG as appropriate. -* Edit the `Cargo.toml` to set the new ripgrep version. Run - `cargo update -p ripgrep` so that the `Cargo.lock` is updated. Commit the - changes and create a new signed tag. Alternatively, use - `cargo-up --no-push --no-release Cargo.toml {VERSION}` to automate this. -* Run `cargo package` and ensure it succeeds. -* Push changes to GitHub, NOT including the tag. (But do not publish a new - version of ripgrep to crates.io yet.) -* Once CI for `main` finishes successfully, push the version tag. (Trying to - do this in one step seems to result in GitHub Actions not seeing the tag - push and thus not running the release workflow.) -* Wait for CI to finish creating the release. If the release build fails, then - delete the tag from GitHub, make fixes, re-tag, delete the release and push. -* Copy the relevant section of the CHANGELOG to the tagged release notes. - Include this blurb describing what ripgrep is: - > In case you haven't heard of it before, ripgrep is a line-oriented search - > tool that recursively searches the current directory for a regex pattern. - > By default, ripgrep will respect gitignore rules and automatically skip - > hidden files/directories and binary files. -* Run `git checkout {VERSION} && ci/build-and-publish-m2 {VERSION}` on a macOS - system with Apple silicon. -* Run `cargo publish`. -* Run `ci/sha256-releases {VERSION} >> pkg/brew/ripgrep-bin.rb`. Then edit - `pkg/brew/ripgrep-bin.rb` to update the version number and sha256 hashes. - Remove extraneous stuff added by `ci/sha256-releases`. Commit changes. -* Add TBD section to the top of the CHANGELOG: - ``` - TBD - === - Unreleased changes. Release notes have not yet been written. - ``` - -Note that [`cargo-up` can be found in BurntSushi's dotfiles][dotfiles]. - -[dotfiles]: https://github.com/BurntSushi/dotfiles/blob/master/bin/cargo-up +## 1) Prepare + +- Ensure local `main` is up to date with `origin/main`. +- Confirm `Cargo.toml` version is the intended release version. +- Review dependency updates (`cargo update`) and commit `Cargo.lock` changes if needed. +- Update `CHANGELOG.md` with release notes for this version. + +## 2) Validate + +- Confirm toolchain pin is available locally: `rustup toolchain install 1.86.0 --profile minimal`. +- Run formatting: `cargo fmt --all --check`. +- Run tests (unit + integration): `cargo test --workspace`. +- Build release binary: `cargo build --release`. +- Verify CLI assets script output: `ci/generate_cli_assets.sh /tmp/precursor-cli-assets ./target/release/precursor`. +- Verify CLI help renders: `./target/release/precursor --help`. +- Smoke test implemented similarity modes: + - TLSH: `printf 'aGVsbG8=\n' | ./target/release/precursor '(?hello)' -m base64 -t --similarity-mode tlsh` + - LZJD: `printf 'GET / HTTP/1.1\n' | ./target/release/precursor '(?GET)' -m string -t --similarity-mode lzjd` +- Smoke test MRSHv2 adapter mode: + - `mock_dir="$(mktemp -d)" && ci/build_mrshv2_mock.sh "$mock_dir"` + - `PRECURSOR_MRSHV2_LIB_DIR="$mock_dir" LD_LIBRARY_PATH="$mock_dir:${LD_LIBRARY_PATH:-}" cargo test --workspace --features similarity-mrshv2` +- Refresh benchmark snapshot: + - `ci/benchmark_scenarios.sh ./target/release/precursor benchmarks/latest.md` + +## 3) Tag and push + +- Commit release prep changes. +- If releasing manually, create and sign tag: `git tag -s -m ""`. +- Push branch first, then push tag after CI on branch passes. +- If dependency updates were merged without a version bump: + - `.github/workflows/auto-bump-version.yml` bumps patch version automatically. + - `.github/workflows/auto-tag-release.yml` tags the new version automatically. + +## 4) CI release + +- Confirm GitHub Actions release workflow completed successfully. +- Confirm MRSHv2 feature smoke job passed in CI (`mrshv2-ffi-smoke`). +- Confirm archives/checksums were attached to the GitHub release draft. +- Promote draft release after artifact validation. + +## 5) Post-release + +- Verify install path(s) (`cargo install precursor` and release binaries). +- Verify GitHub Pages demo site deployment and custom domain (`precursor.hashdb.io`) health. +- Add next `TBD`/unreleased section to `CHANGELOG.md`. +- Announce release with notable changes and known limitations. diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..b388e66 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,57 @@ +# Roadmap + +Last updated: February 13, 2026 + +## Release focus: `0.2.x` + +## Guiding priorities +- Keep Precursor fast and scriptable for payload triage. +- Improve protocol discovery workflows without hard protocol dependencies. +- Preserve stable JSON output fields for downstream tooling. + +## Near-term milestones + +### 1) Binary/blob depth improvements +Status: `in progress` +- Expand blob mode beyond UTF-8 wrappers for `base64`/`hex`. +- Add explicit raw-binary mode semantics for firmware and packet stream chunks. +- Added corpus fixtures for binary-like and mixed-encoding payloads in `samples/scenarios/`. + +### 2) Similarity backend expansion +Status: `in progress` +- Implemented `lzjd` backend behind `--similarity-mode lzjd`. +- Implemented `mrshv2` backend path behind `--similarity-mode mrshv2` with feature gate + native adapter ABI (`similarity-mrshv2`). +- Prototype `fbhash` backend behind `--similarity-mode fbhash`. +- Keep output contract backend-agnostic via `similarity_hash`. + +### 3) Inference quality hardening +Status: `in progress` +- Add more protocol-family heuristics for single-packet inference. +- Add ambiguity/abstention tests to reduce false confidence. +- Added regression corpus coverage for packet/firmware/ICS scenarios. + +## Mid-term milestones + +### 4) Library/CLI separation +Status: `planned` +- Introduce `src/lib.rs` for reusable pipeline components. +- Keep CLI as thin orchestration layer. +- Add integration tests that cover both library and CLI entry points. + +### 5) Performance and scaling +Status: `in progress` +- Reduce overhead when `--stats` is disabled. +- Added scenario benchmark harness (`ci/benchmark_scenarios.sh`) and baseline snapshot. +- Improve large-cluster comparison ergonomics around O(n^2) diff behavior. + +## Release criteria for `0.3.0` +- `lzjd` production-hardening completed (corpus validation + benchmark baseline). +- `mrshv2` native adapter path validated on CI and documented for production adapter wiring. +- Blob mode supports raw binary stream workflows beyond UTF-8 wrappers. +- Integration corpus expanded with realistic packet/firmware samples. +- Stable JSON schema documented with examples for all major modes. + +## Backlog candidates +- Optional match-mask support before similarity hashing. +- Richer protocol hint output tuned for human + LLM triage loops. +- Packaging improvements for downstream distro ecosystems. diff --git a/SIMILARITY_BACKENDS.md b/SIMILARITY_BACKENDS.md new file mode 100644 index 0000000..bbe0c66 --- /dev/null +++ b/SIMILARITY_BACKENDS.md @@ -0,0 +1,109 @@ +# Similarity Backend Feasibility + +Last updated: February 13, 2026 + +## Goal + +Determine whether Precursor should add MRSHv2 and/or FBHash support, and whether a newer algorithm should be prioritized first. + +## Summary Recommendation + +1. Keep `lzjd` as the first non-TLSH backend (now implemented in-tree in this repo). +2. Keep MRSHv2 in feature-gated native adapter mode and harden with production adapter coverage. +3. Treat FBHash as optional/experimental unless we commit to corpus-level indexing and TF-IDF state management. + +## Evidence Snapshot + +### MRSHv2 +- Frank Breitinger's tools page still lists `mrsh_v2.0` (last update 2013-10-04), `mrsh_net` (2014-11-12), and `mrsh_cuckoo` (2015-04-10): + - https://fbreitinger.de/?page_id=218 +- A current mirror/development repo exists (`w4term3loon/mrsh`), with release `v1.0.0` dated October 13, 2025 and Apache-2.0 license: + - https://github.com/w4term3loon/mrsh + - (discovered via PyPI project linking and release metadata) +- Python bindings (`mrshw`) were released as `1.0.0` on October 13, 2025 and explicitly wrap the MRSH CLI: + - https://pypi.org/project/mrshw/ + +### FBHash +- Rust implementation exists with recent release metadata (`0.1.5` latest release June 25, 2025), but low ecosystem traction (very low stars/forks): + - https://github.com/erwinvaneijk/fbhash +- Repo README content confirms algorithm design is TF-IDF/cosine based over document chunks, which implies corpus-level state rather than simple per-record digesting. + +### Recent Forensic Direction +- 2024 temporal Android malware evaluation reports fuzzy hashing remains useful and robust over long horizons (10-year detection rates over 80%), comparing multiple algorithm families: + - https://doi.org/10.1016/j.fsidi.2024.301770 + - landing/details: https://pure.qub.ac.uk/en/publications/a-temporal-analysis-and-evaluation-of-fuzzy-hashing-algorithms-fo/ +- 2025 Windows-system-binary dataset article includes TLSH, ssdeep, sdhash, and LZJD digests, indicating LZJD remains operationally relevant in recent forensic workflows: + - https://doi.org/10.1016/j.dib.2025.111993 + - PubMed entry: https://pubmed.ncbi.nlm.nih.gov/40955418/ +- Rust LZJD implementation is available as a maintained crate entry: + - https://docs.rs/lzjd/latest/lzjd/ + +## Engineering Fit vs Current Precursor Pipeline + +Precursor currently assumes: +- a per-payload hash representation (`similarity_hash`) +- pairwise diff function for in-memory comparisons + +### MRSHv2 fit +- Good fit for file/blob similarity and fragment detection. +- Requires either: + - C FFI integration, or + - shelling out to CLI and parsing output (not preferred for production path). +- Complexity: medium-high. + +### FBHash fit +- Weaker fit for current architecture because FBHash relies on corpus document-frequency context. +- A correct implementation needs: + - corpus build stage + - stored global DF model + - vector representation per payload + - cosine similarity, not just digest-distance semantics +- Complexity: high. + +### LZJD fit +- Strong fit to current architecture. +- Pure Rust implementation path. +- Can be used for pairwise distance without external native dependencies. +- Complexity: medium. + +## Proposed Implementation Plan + +## Phase 1 (completed in repo) +- Added `lzjd` backend to `--similarity-mode`. +- Implemented: + - hash creation from payload bytes + - pairwise distance scoring + - report output field continuity (`similarity_hash`, diff maps) +- Added mode-specific unit/integration tests. + +## Phase 2 (in progress) +- Added `mrshv2` backend path behind Cargo feature: + - `similarity-mrshv2` +- Added native C adapter ABI contract: + - `ffi/mrshv2_adapter.h` +- Added CI smoke validation with a mock native adapter: + - `ci/build_mrshv2_mock.sh` + - `.github/workflows/ci.yml` (`mrshv2-ffi-smoke`) +- Remaining work: + - wire adapter against production MRSHv2 core implementation + - validate adapter semantics against a real MRSHv2 corpus + +## Phase 3 (optional/experimental) +- Add FBHash in a separate mode family that explicitly supports corpus-state workflows: + - `--similarity-mode fbhash` + - plus corpus/index path inputs +- Do not force FBHash into the simple "single digest + pairwise diff" model. + +## Release Criteria for Backend Expansion + +Before enabling non-TLSH mode by default: +- deterministic fixtures for each mode +- runtime and memory benchmarks for line mode and blob mode +- docs that state minimum payload size and failure behavior +- clear provenance and license tracking for any external implementation + +## Open Risks + +- Supply-chain risk from low-adoption crates/repos: pin versions, verify source, and prefer reproducible builds. +- API-shape mismatch between digest-distance tools and corpus-vector tools. +- Native dependency complexity for MRSHv2 if static linking is required across platforms. diff --git a/ai/LLM_DISCOVERY_LOOP.md b/ai/LLM_DISCOVERY_LOOP.md new file mode 100644 index 0000000..075d66f --- /dev/null +++ b/ai/LLM_DISCOVERY_LOOP.md @@ -0,0 +1,42 @@ +# LLM Discovery Loop (Scaffold) + +## Goal +Use Precursor's pre-protocol similarity clustering to bootstrap protocol awareness and rule discovery. + +## Current hooks +- Similarity backend selector: `--similarity-mode ` +- Hint export for LLM input: `--protocol-hints --protocol-hints-limit ` +- Single-packet protocol inference: `--single-packet --abstain-threshold --protocol-top-k ` + +## Suggested loop +1. Start with versioned corpora in `samples/scenarios/`, then run on raw payload streams with high-recall pattern gates. +2. Enable similarity diffing, hint export, and packet inference: + - `-t -d --single-packet --protocol-hints --protocol-hints-limit 50` +3. Feed the protocol hint JSON to an LLM prompt that asks for: + - candidate protocol families + - likely field boundaries / delimiters + - discriminating regex capture ideas +4. Also feed `protocol_candidates` from payload output to let the model compare top heuristic hypotheses vs cluster context. +5. Convert model output into proposed named-capture patterns. +6. Validate those patterns against positive/negative corpora. +7. Repeat until precision/recall targets are met. + +## Prompt seed +```text +You are analyzing pre-protocol payload clusters. +Given this Precursor protocol hint JSON: +- propose likely protocol/message families, +- infer stable token/field structures, +- draft named-capture regexes for high-signal tags, +- list confidence and ambiguity for each proposal. +``` + +## Notes +- `tlsh` and `lzjd` are implemented in default builds. +- `mrshv2` is implemented behind `similarity-mrshv2` and native adapter linking. +- `fbhash` remains scaffolded. +- With `--tlsh-diff`, inference confidence can be boosted by similarity neighbor count. +- Keep generated regex tags stable and snake_case to preserve downstream compatibility. +- MRSHv2 mode requires a feature build plus native adapter linkage: + - `--features similarity-mrshv2` + - `PRECURSOR_MRSHV2_LIB_DIR=` diff --git a/ai/MEMORY.md b/ai/MEMORY.md new file mode 100644 index 0000000..189ec91 --- /dev/null +++ b/ai/MEMORY.md @@ -0,0 +1,75 @@ +# Precursor Memory + +Last updated: February 13, 2026 + +## Product Snapshot +- Language: Rust +- Binary: `precursor` (`src/main.rs`) +- Core purpose: tag payloads with PCRE2 named-capture patterns, optionally compute similarity hashes (TLSH/LZJD/feature-gated MRSHv2) and pairwise distances, emit JSON records to STDOUT and optional run stats to STDERR. + +## Repository Map +- `src/main.rs`: CLI, ingest loop, matching pipeline, TLSH diff stage, stats/report output. +- `src/precursor/similarity.rs`: similarity backend selector and backend-agnostic hash/diff dispatch. +- `src/precursor/lzjd.rs`: in-tree LZJD-style hashing backend for pairwise similarity mode. +- `src/precursor/mrshv2.rs`: feature-gated MRSHv2 native adapter bindings and hash/diff wrapper. +- `src/precursor/util.rs`: payload decoding, regex builder, pattern file loader, utility functions and unit tests. +- `src/precursor/tlsh.rs`: TLSH wrapper enums/builders and hash/diff logic. +- `samples/scenarios/`: versioned packet/firmware/ICS corpus and scenario runner script. +- `site/`: GitHub Pages static demo content for `precursor.hashdb.io`. +- `patterns/`: rule packs and pattern definitions. +- `ci/` and `.github/workflows/`: multi-target build/release workflow. + +## Execution Model +1. Parse args and read patterns from `-p` file or positional pattern. +2. Compile regexes once before processing input lines. +3. Read stdin lines (parallel) or files from `-f` directory. +4. Decode payload (`base64`/`string`/`hex`) and optionally extract from JSON path. +5. Apply PCRE2 rules and collect matching capture names as tags. +6. For matched payloads, optionally compute selected similarity hashes and optional pairwise diffs. +7. Emit per-payload JSON to STDOUT and optional stats JSON to STDERR. + +## Known Constraints +- Blob mode (`--input-blob`) is implemented, but encoded blob decoding (`base64`/`hex`) currently expects UTF-8 wrapper text. +- Pairwise similarity diff is O(n^2) by number of matched payload hashes. + +## Recently Landed Improvements +- Pattern regex compilation moved out of per-line hot path. +- File input now increments input counters consistently. +- Stats path handles empty vectors and zero-duration runs safely. +- Payload decoding and JSON extraction failures are now recoverable per-line errors. +- TLSH diffing/report output now handle incompatible hash types, lock poisoning, and output serialization failures without panicking. +- Similarity backend support now includes: + - `tlsh` (existing) + - `lzjd` (implemented) + - `mrshv2` (implemented behind `similarity-mrshv2` + native adapter ABI) + - `fbhash` (scaffolded for future work) +- Protocol-hint export (`--protocol-hints`) now emits LLM-oriented candidate clusters to `stderr`. +- Single-packet protocol inference mode was added: + - `--single-packet` + - `--abstain-threshold` + - `--protocol-top-k` + - output fields: `protocol_label`, `protocol_confidence`, `protocol_abstained`, `protocol_candidates` +- Inference confidence can now be cluster-boosted from similarity neighbor counts when `--single-packet` and `--tlsh-diff` are both enabled. +- Blob mode is now implemented with `--input-blob` for one-record ingestion from stdin/file streams. +- Ingestion now handles file/line errors without panic in runtime paths. +- CLI integration tests now validate: + - single-packet protocol fields + - protocol-hint stderr JSON + - multiline blob matching +- Scenario integration tests now validate: + - pre-protocol packet corpus behavior + - firmware-fragment inference behavior + - ICS Modbus hint emission +- README was rewritten to match actual CLI behavior and project positioning. +- Release checklist now reflects Precursor's actual release process. +- CI/CD now includes Dependabot plus auto patch-version bump and auto-tag workflows for dependency-driven releases. +- CI now includes MRSHv2 feature smoke coverage with a compiled mock native adapter. +- Benchmark harness and baseline snapshot added: + - `ci/benchmark_scenarios.sh` + - `benchmarks/baseline-2026-02-13.md` + +## Current Priorities +1. Expand realistic payload corpora and broaden integration fixture coverage. +2. Extend inference for binary stream/firmware-first workflows (file magic, container formats, stream framing). +3. Evaluate library/CLI split (`src/lib.rs`) for embeddability. +4. Reduce stats-related overhead when `--stats` is disabled. diff --git a/ai/PROMPT_STRATEGY.md b/ai/PROMPT_STRATEGY.md new file mode 100644 index 0000000..fd0b065 --- /dev/null +++ b/ai/PROMPT_STRATEGY.md @@ -0,0 +1,106 @@ +# Prompt Strategy + +## Objective +Drive faster, safer iteration on `precursor` by using narrow prompts with explicit acceptance checks. + +## Default Agent Loop +1. Load only required context: `ai/MEMORY.md` plus target files. +2. Restate goal and non-goals in one short paragraph. +3. Propose smallest safe change plan. +4. Implement. +5. Run verification commands. +6. Report findings, risks, and follow-up options. + +## Prompt Templates + +### 1) Bug Fix Prompt +Use when behavior is wrong or unstable. + +```text +Use $precursor-maintainer. +Goal: Fix in . +Constraints: Keep CLI/output backward compatible; avoid broad refactors. +Validate with: . +Done when: . +``` + +### 2) Performance Prompt +Use when latency/throughput is the main concern. + +```text +Use $precursor-maintainer. +Goal: Improve throughput. +Scope: only . +Measure: before/after using . +Guardrails: no correctness regressions; keep JSON schema stable. +``` + +### 3) Pattern Engineering Prompt +Use when creating or tuning pattern packs. + +```text +Use $precursor-pattern-lab. +Goal: Detect/tag . +Inputs: , , . +Quality target: maximize precision, keep recall acceptable. +Deliver: updated pattern file + validation output + known limitations. +``` + +### 4) Architecture Prompt +Use when planning larger changes. + +```text +Use $precursor-maintainer. +Task: Propose a staged plan for . +Include: migration steps, risk list, test strategy, and rollback point. +Avoid: changing public CLI semantics in phase 1. +``` + +### 5) Protocol Inference Prompt +Use when tuning `--single-packet` behavior. + +```text +Use $precursor-maintainer. +Goal: Improve single-packet inference for . +Inputs: , current `--abstain-threshold`, current patterns. +Deliver: +- heuristic changes with rationale, +- expected label/confidence shifts, +- tests for positive + ambiguous payloads. +Guardrails: keep existing JSON fields stable (`protocol_*` schema). +``` + +### 6) Scenario Regression Prompt +Use when validating changes against versioned corpus fixtures. + +```text +Goal: Validate against scenario corpus. +Run: +- cargo test --workspace +- cargo test --workspace --test scenario_corpus_contract +- ci/benchmark_scenarios.sh ./target/release/precursor benchmarks/latest.md +Check: +- JSON output contract unchanged +- no confidence regressions on firmware/ICS packet examples +- benchmark deltas called out with rationale +``` + +### 7) MRSHv2 Adapter Prompt +Use when changing native adapter compatibility. + +```text +Goal: Keep MRSHv2 feature build stable and testable. +Build: +- mock_dir="$(mktemp -d)" +- ci/build_mrshv2_mock.sh "$mock_dir" +- PRECURSOR_MRSHV2_LIB_DIR="$mock_dir" LD_LIBRARY_PATH="$mock_dir:${LD_LIBRARY_PATH:-}" cargo test --workspace --features similarity-mrshv2 +Done when: +- similarity hashes emit `mrshv2:` prefix in CLI tests +- CI path remains green for mrshv2-ffi-smoke +``` + +## Prompt Hygiene Rules +- Provide explicit file paths and expected outputs. +- Ask for one deployable step at a time when uncertainty is high. +- Demand line-referenced findings for reviews. +- Require a verification section in every response. diff --git a/ai/REPO_REVIEW.md b/ai/REPO_REVIEW.md new file mode 100644 index 0000000..168a301 --- /dev/null +++ b/ai/REPO_REVIEW.md @@ -0,0 +1,28 @@ +# Repository Review + +Date: February 13, 2026 + +## Resolved In This Iteration +- Empty-input stats panic path was hardened in `src/main.rs` (safe avg/p95 handling and zero-duration guards). +- Ingestion paths now handle malformed JSON/base64/hex without process abort in `src/main.rs` and `src/precursor/util.rs`. +- Regexes are now compiled once before line processing in `src/main.rs`. +- File-mode input counting now increments `Input.Count` in `src/main.rs`. +- TLSH/reporting paths now avoid panic on incompatible hash types, poisoned locks, and report serialization/flush failures in `src/main.rs` and `src/precursor/tlsh.rs`. +- Similarity backend plumbing (`--similarity-mode`) plus implemented `lzjd` mode and protocol hint export (`--protocol-hints`) were added in `src/main.rs`, `src/precursor/similarity.rs`, and `src/precursor/lzjd.rs`. +- MRSHv2 feature-gated native adapter backend was added in `src/precursor/mrshv2.rs` with build/CI integration in `build.rs`, `ci/build_mrshv2_mock.sh`, and `.github/workflows/ci.yml`. +- Scenario corpus + regression tests were added in `samples/scenarios/` and `tests/scenario_corpus_contract.rs`. +- Benchmark harness and baseline snapshot were added in `ci/benchmark_scenarios.sh` and `benchmarks/baseline-2026-02-13.md`. +- Static GitHub Pages demo site and deployment workflow were added in `site/` and `.github/workflows/pages.yml`. +- README drift was corrected in `README.md` and aligned with implemented flags. +- Release checklist was rewritten for actual `precursor` release flow in `RELEASE-CHECKLIST.md`. +- Automated dependency release plumbing was added via `.github/dependabot.yml`, `.github/workflows/auto-bump-version.yml`, and `.github/workflows/auto-tag-release.yml`. + +## Medium +- `src/main.rs` still allocates and updates stats-tracking structures even when `--stats` is off; this adds avoidable hot-path overhead. +- Default `--input-mode base64` can surprise users on plain text streams when `-m string` is not explicitly provided. + +## Low +- MRSHv2 currently relies on adapter ABI compatibility; production adapter validation against upstream MRSHv2 corpus still needs follow-up. + +## Verification Note +- `cargo test --workspace` passes in this workspace. diff --git a/architecture.png b/architecture.png index 5255fa7..6ecb131 100644 Binary files a/architecture.png and b/architecture.png differ diff --git a/architecture.puml b/architecture.puml index 4c40a9d..72fcad1 100644 --- a/architecture.puml +++ b/architecture.puml @@ -1,46 +1,58 @@ @startuml -skinparam arrowThickness 4 -skinparam nodesep 10 -skinparam ranksep 20 +left to right direction +skinparam backgroundColor #f8fafc +skinparam linetype ortho +skinparam shadowing false +skinparam roundCorner 12 +skinparam defaultFontName Helvetica +skinparam nodesep 28 +skinparam ranksep 36 +skinparam rectangle { + BorderColor #334155 + FontColor #0f172a +} +skinparam arrow { + Color #1e293b + Thickness 1.5 +} -rectangle "Input (STDIN or File)" as input { - rectangle "Base64" as rawBase64 - rectangle "Hex" as rawHex - rectangle "String" as rawString - rectangle "Binary" as binary +rectangle "Inputs" as inputs #dbeafe { + rectangle "STDIN\n(line or blob mode)" as stdin #eff6ff + rectangle "Folder files\n(line or blob mode)" as folder #eff6ff } -rectangle "Patterns (PCRE2)" as patterns { - rectangle "File/s" as patternfile - rectangle "Argument" as patternargument +rectangle "Decode + Extract" as decode #fef3c7 { + rectangle "Decode modes\nbase64 | string | hex" as modes #fffbeb + rectangle "Optional JSON extraction\n--input-json-key" as json_extract #fffbeb } -rectangle "Filter" as filter { - rectangle "Remove" as remove - rectangle "Label" as label +rectangle "Tagging" as tagging #dcfce7 { + rectangle "PCRE2 named captures" as pcre #f0fdf4 + rectangle "Tag array\n(tags[])" as tags #f0fdf4 } -rectangle "Compare (TLSH)" as compare { - rectangle "Algorithms" as tlshAlgorithms - rectangle "Threshold" as distanceThreshold +rectangle "Similarity" as similarity #fee2e2 { + rectangle "TLSH hash\n(similarity_hash)" as hash #fff1f2 + rectangle "Pairwise diff\n--tlsh-diff / --tlsh-distance" as diff #fff1f2 } -rectangle "Output (STDOUT)" as stdout { - rectangle "Merged JSON" as jsonOutput - rectangle "JSON" as nonJsonOutput +rectangle "Inference" as inference #ede9fe { + rectangle "Single-packet scoring\n-P / -A / -k" as packet #f5f3ff + rectangle "Neighbor confidence boost\n(cluster size from diff graph)" as boost #f5f3ff } -rectangle "Statistics (STDERR)" as stderr { - rectangle "Input" as inputStats - rectangle "Pattern" as inputStats - rectangle "Filter" as filterStats - rectangle "Compare" as compareStats - rectangle "Environment" as runStats +rectangle "Outputs" as outputs #e0f2fe { + rectangle "STDOUT\nNDJSON per matched payload" as stdout #f0f9ff + rectangle "STDERR\nstats JSON (--stats)" as stderr_stats #f0f9ff + rectangle "STDERR\nprotocol hints (--protocol-hints)" as stderr_hints #f0f9ff } -input -down[thickness=10]-> filter -patterns -down-> filter -filter -down[thickness=5]-> compare -compare -down-> stdout -compare -down-> stderr -@enduml \ No newline at end of file +inputs --> decode +decode --> tagging +tagging --> similarity +similarity --> inference +tagging --> outputs +similarity --> outputs +inference --> outputs + +@enduml diff --git a/architecture.svg b/architecture.svg new file mode 100644 index 0000000..20bb725 --- /dev/null +++ b/architecture.svg @@ -0,0 +1 @@ +InputsDecode + ExtractTaggingSimilarityInferenceOutputsSTDIN(line or blob mode)Folder files(line or blob mode)Decode modesbase64 | string | hexOptional JSON extraction--input-json-keyPCRE2 named capturesTag array(tags[])TLSH hash(similarity_hash)Pairwise difftlsh-diff /tlsh-distanceSingle-packet scoring-P / -A / -kNeighbor confidence boost(cluster size from diff graph)STDOUTNDJSON per matched payloadSTDERRstats JSON (--stats)STDERRprotocol hints (--protocol-hints) \ No newline at end of file diff --git a/assets/logo/precursor-logo.svg b/assets/logo/precursor-logo.svg new file mode 100644 index 0000000..201ddf7 --- /dev/null +++ b/assets/logo/precursor-logo.svg @@ -0,0 +1,52 @@ + + Precursor logo + A radar-inspired emblem and wordmark for the Precursor project. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PRECURSOR + + + PAYLOAD TAGGING AND SIMILARITY ANALYSIS + + + diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..a2c89cd --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,25 @@ +# Benchmarks + +This folder stores lightweight, reproducible benchmark snapshots based on the +versioned scenario corpus under `samples/scenarios/`. + +## Generate a fresh snapshot + +```bash +cargo build --release +ci/benchmark_scenarios.sh ./target/release/precursor benchmarks/latest.md +``` + +## Include optional MRSHv2 mode + +Build with the MRSHv2 feature and native adapter first, then: + +```bash +PRECURSOR_BENCH_INCLUDE_MRSHV2=1 \ + ci/benchmark_scenarios.sh ./target/release/precursor benchmarks/latest.md +``` + +## Notes + +- These are smoke-level comparative numbers, not full microbenchmarks. +- Use them to detect meaningful regressions between commits/releases. diff --git a/benchmarks/baseline-2026-02-13.md b/benchmarks/baseline-2026-02-13.md new file mode 100644 index 0000000..9eca3a8 --- /dev/null +++ b/benchmarks/baseline-2026-02-13.md @@ -0,0 +1,12 @@ +# Scenario Benchmark Snapshot + +Date: 2026-02-13 22:42:24Z +Binary: `./target/release/precursor` +Repeat factor: `200` + +| Case | Similarity | Reports | Matches | DurationSeconds | +| --- | --- | ---: | ---: | ---: | +| Pre-protocol packet triage | tlsh | 4 | 800 | 0.02 | +| Pre-protocol packet triage | lzjd | 4 | 800 | 0.26 | +| Firmware fragment triage | lzjd | 5 | 1200 | 0.36 | +| ICS Modbus single-packet | lzjd | 5 | 1000 | 0.34 | diff --git a/build.rs b/build.rs index ded4653..3e5239b 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,7 @@ fn main() { set_git_revision_hash(); set_windows_exe_options(); + set_mrshv2_linking(); } /// Embed a Windows manifest and set some linker options. @@ -54,3 +55,32 @@ fn set_git_revision_hash() { } println!("cargo:rustc-env=PRECURSOR_BUILD_GIT_HASH={}", rev); } + +/// Configure optional MRSHv2 adapter linking when the feature is enabled. +/// +/// The Rust MRSHv2 backend links against a tiny adapter ABI: +/// `precursor_mrshv2_hash`, `precursor_mrshv2_diff`, `precursor_mrshv2_free`, +/// and `precursor_mrshv2_last_error`. +/// +/// The library name defaults to `precursor_mrshv2` and can be overridden with +/// `PRECURSOR_MRSHV2_LIB_NAME`. An extra search directory can be provided via +/// `PRECURSOR_MRSHV2_LIB_DIR`. +fn set_mrshv2_linking() { + if std::env::var_os("CARGO_FEATURE_SIMILARITY_MRSHV2").is_none() { + return; + } + println!("cargo:rerun-if-env-changed=PRECURSOR_MRSHV2_LIB_DIR"); + println!("cargo:rerun-if-env-changed=PRECURSOR_MRSHV2_LIB_NAME"); + + if let Ok(lib_dir) = std::env::var("PRECURSOR_MRSHV2_LIB_DIR") { + if !lib_dir.is_empty() { + println!("cargo:rustc-link-search=native={}", lib_dir); + } + } + + let lib_name = std::env::var("PRECURSOR_MRSHV2_LIB_NAME") + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "precursor_mrshv2".to_string()); + println!("cargo:rustc-link-lib=dylib={}", lib_name); +} diff --git a/ci/benchmark_scenarios.sh b/ci/benchmark_scenarios.sh new file mode 100755 index 0000000..833de7b --- /dev/null +++ b/ci/benchmark_scenarios.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +set -euo pipefail + +bin_path="${1:-./target/release/precursor}" +output_path="${2:-benchmarks/latest.md}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repeat_factor="${PRECURSOR_BENCH_REPEAT:-200}" + +if [ ! -x "$bin_path" ]; then + echo "missing executable precursor binary: $bin_path" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$output_path")" + +run_case() { + local case_name="$1" + local pattern_file="$2" + local input_mode="$3" + local input_file="$4" + local similarity_mode="$5" + local extra_flags="${6:-}" + local run_input="$input_file" + + local stdout_file + local stderr_file + stdout_file="$(mktemp)" + stderr_file="$(mktemp)" + local expanded_input + expanded_input="$(mktemp)" + + if [ "$repeat_factor" -gt 1 ]; then + : > "$expanded_input" + for _ in $(seq 1 "$repeat_factor"); do + cat "$input_file" >> "$expanded_input" + printf '\n' >> "$expanded_input" + done + run_input="$expanded_input" + fi + + # shellcheck disable=SC2086 + "$bin_path" -p "$pattern_file" -m "$input_mode" -t -d --similarity-mode "$similarity_mode" -s $extra_flags \ + < "$run_input" > "$stdout_file" 2> "$stderr_file" + + local reports + reports="$(wc -l < "$stdout_file" | tr -d ' ')" + local total_matches + total_matches="$(rg -o '"TotalMatches": [0-9]+' "$stderr_file" | head -n1 | sed -E 's/.*: ([0-9]+)/\1/' || true)" + local duration + duration="$(rg -o '"DurationSeconds": "[^"]+"' "$stderr_file" | head -n1 | sed -E 's/.*"([0-9.]+)".*/\1/' || true)" + + if [ -z "$total_matches" ]; then + total_matches="0" + fi + if [ -z "$duration" ]; then + duration="n/a" + fi + + printf '| %s | %s | %s | %s | %s |\n' \ + "$case_name" "$similarity_mode" "$reports" "$total_matches" "$duration" + + rm -f "$stdout_file" "$stderr_file" "$expanded_input" +} + +{ + echo "# Scenario Benchmark Snapshot" + echo + echo "Date: $(date -u '+%Y-%m-%d %H:%M:%SZ')" + echo "Binary: \`$bin_path\`" + echo "Repeat factor: \`$repeat_factor\`" + echo + echo '| Case | Similarity | Reports | Matches | DurationSeconds |' + echo '| --- | --- | ---: | ---: | ---: |' + run_case \ + "Pre-protocol packet triage" \ + "$repo_root/samples/scenarios/pre-protocol-packet-triage/patterns.pcre" \ + "base64" \ + "$repo_root/samples/scenarios/pre-protocol-packet-triage/payloads.b64" \ + "tlsh" + run_case \ + "Pre-protocol packet triage" \ + "$repo_root/samples/scenarios/pre-protocol-packet-triage/patterns.pcre" \ + "base64" \ + "$repo_root/samples/scenarios/pre-protocol-packet-triage/payloads.b64" \ + "lzjd" + run_case \ + "Firmware fragment triage" \ + "$repo_root/samples/scenarios/firmware-fragment-triage/patterns.pcre" \ + "hex" \ + "$repo_root/samples/scenarios/firmware-fragment-triage/payloads.hex" \ + "lzjd" + run_case \ + "ICS Modbus single-packet" \ + "$repo_root/samples/scenarios/ics-modbus-single-packet/patterns.pcre" \ + "hex" \ + "$repo_root/samples/scenarios/ics-modbus-single-packet/payloads.hex" \ + "lzjd" \ + "-P" +} > "$output_path" + +if [ "${PRECURSOR_BENCH_INCLUDE_MRSHV2:-0}" = "1" ]; then + { + echo + echo "## Optional MRSHv2" + echo + echo '| Case | Similarity | Reports | Matches | DurationSeconds |' + echo '| --- | --- | ---: | ---: | ---: |' + run_case \ + "Pre-protocol packet triage" \ + "$repo_root/samples/scenarios/pre-protocol-packet-triage/patterns.pcre" \ + "base64" \ + "$repo_root/samples/scenarios/pre-protocol-packet-triage/payloads.b64" \ + "mrshv2" + } >> "$output_path" +fi + +echo "wrote benchmark snapshot to $output_path" diff --git a/ci/build_mrshv2_mock.sh b/ci/build_mrshv2_mock.sh new file mode 100755 index 0000000..f0a2f87 --- /dev/null +++ b/ci/build_mrshv2_mock.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +out_dir="$1" +mkdir -p "$out_dir" +cc_bin="${CC:-cc}" +src="ffi/mrshv2_mock.c" +lib_base="precursor_mrshv2" + +if [ ! -f "$src" ]; then + echo "missing source file: $src" >&2 + exit 1 +fi + +case "$(uname -s)" in + Darwin) + lib_path="$out_dir/lib${lib_base}.dylib" + "$cc_bin" -dynamiclib -O2 -fPIC -Iffi "$src" -o "$lib_path" + ;; + Linux) + lib_path="$out_dir/lib${lib_base}.so" + "$cc_bin" -shared -O2 -fPIC -Iffi "$src" -o "$lib_path" + ;; + MINGW*|MSYS*|CYGWIN*) + lib_path="$out_dir/${lib_base}.dll" + "$cc_bin" -shared -O2 -Iffi "$src" -o "$lib_path" + ;; + *) + echo "unsupported platform: $(uname -s)" >&2 + exit 1 + ;; +esac + +echo "$lib_path" diff --git a/ci/bump_patch_version.sh b/ci/bump_patch_version.sh new file mode 100755 index 0000000..4fa62e6 --- /dev/null +++ b/ci/bump_patch_version.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +cargo_toml="${1:-Cargo.toml}" + +current_version="$(sed -n 's/^version = "\(.*\)"/\1/p' "$cargo_toml" | head -n1)" +if [ -z "$current_version" ]; then + echo "unable to read package version from $cargo_toml" >&2 + exit 1 +fi + +IFS='.' read -r major minor patch <&2 + exit 1 +fi + +new_patch=$((patch + 1)) +new_version="${major}.${minor}.${new_patch}" + +tmp_file="$(mktemp)" +awk -v new_version="$new_version" ' + BEGIN { replaced = 0 } + /^version = "/ && replaced == 0 { + print "version = \"" new_version "\"" + replaced = 1 + next + } + { print } +' "$cargo_toml" > "$tmp_file" +mv "$tmp_file" "$cargo_toml" + +echo "$new_version" diff --git a/ci/generate_cli_assets.sh b/ci/generate_cli_assets.sh new file mode 100755 index 0000000..05fedbb --- /dev/null +++ b/ci/generate_cli_assets.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -euo pipefail + +out_dir="${1:-dist/cli-assets}" +bin_path="${2:-./target/release/precursor}" + +mkdir -p "${out_dir}/complete" "${out_dir}/doc" + +common_options="-f --input-folder -z --input-blob -p --pattern-file -t --tlsh -a --tlsh-algorithm -d --tlsh-diff -y --tlsh-sim-only -x --tlsh-distance -l --tlsh-length --similarity-mode --protocol-hints --protocol-hints-limit -P --single-packet -A --abstain-threshold -k --protocol-top-k -m --input-mode -j --input-json-key -s --stats -h --help" + +cat > "${out_dir}/complete/precursor.bash" < "${out_dir}/complete/precursor.fish" <<'EOF' +complete -c precursor -f +complete -c precursor -s f -l input-folder -d "Specify the path to the input folder." +complete -c precursor -s z -l input-blob -d "Process each input source as a single blob." +complete -c precursor -s p -l pattern-file -d "Pattern file path." +complete -c precursor -s t -l tlsh -d "Calculate TLSH hash." +complete -c precursor -s a -l tlsh-algorithm -d "TLSH algorithm." -a "48_1 128_1 128_3 256_1 256_3" +complete -c precursor -s d -l tlsh-diff -d "Perform TLSH distance calculations." +complete -c precursor -s y -l tlsh-sim-only -d "Only output similar payloads." +complete -c precursor -s x -l tlsh-distance -d "TLSH distance threshold." +complete -c precursor -s l -l tlsh-length -d "Include length in TLSH diff." +complete -c precursor -l similarity-mode -d "Similarity backend." -a "tlsh mrshv2 fbhash" +complete -c precursor -l protocol-hints -d "Emit protocol hint JSON to STDERR." +complete -c precursor -l protocol-hints-limit -d "Limit protocol hint candidates." +complete -c precursor -s P -l single-packet -d "Enable single-packet protocol inference." +complete -c precursor -s A -l abstain-threshold -d "Protocol inference abstain threshold." +complete -c precursor -s k -l protocol-top-k -d "Protocol candidate count." +complete -c precursor -s m -l input-mode -d "Input mode." -a "base64 string hex" +complete -c precursor -s j -l input-json-key -d "JSON extraction key." +complete -c precursor -s s -l stats -d "Emit statistics to STDERR." +complete -c precursor -s h -l help -d "Show help." +EOF + +cat > "${out_dir}/complete/_precursor.ps1" < "${out_dir}/complete/_precursor" </dev/null 2>&1; then + help2man -N --name "pre-protocol payload tagging and similarity analysis" "${bin_path}" > "${out_dir}/doc/precursor.1" +else + { + echo ".TH precursor 1" + echo ".SH NAME" + echo "precursor - pre-protocol payload tagging and similarity analysis" + echo ".SH SYNOPSIS" + echo "precursor [PATTERN] [OPTIONS]" + echo ".SH DESCRIPTION" + "${bin_path}" --help + } > "${out_dir}/doc/precursor.1" +fi diff --git a/ffi/README.md b/ffi/README.md new file mode 100644 index 0000000..a51791c --- /dev/null +++ b/ffi/README.md @@ -0,0 +1,26 @@ +# MRSHv2 Native Adapter ABI + +Precursor's optional MRSHv2 mode (`--features similarity-mrshv2`) links against +a native library that exports the ABI defined in `mrshv2_adapter.h`. + +## Required symbols + +- `precursor_mrshv2_hash` +- `precursor_mrshv2_diff` +- `precursor_mrshv2_free` +- `precursor_mrshv2_last_error` + +## Build/test with mock adapter + +```bash +mock_dir="$(mktemp -d)" +ci/build_mrshv2_mock.sh "$mock_dir" +PRECURSOR_MRSHV2_LIB_DIR="$mock_dir" LD_LIBRARY_PATH="$mock_dir:${LD_LIBRARY_PATH:-}" \ + cargo test --workspace --features similarity-mrshv2 +``` + +## Production adapter notes + +- Keep `precursor_mrshv2_diff` output normalized to `[0,100]` where `0` means identical. +- Return clear thread-local text from `precursor_mrshv2_last_error`. +- Preserve backward compatibility for symbol names and argument types. diff --git a/ffi/mrshv2_adapter.h b/ffi/mrshv2_adapter.h new file mode 100644 index 0000000..51b1f03 --- /dev/null +++ b/ffi/mrshv2_adapter.h @@ -0,0 +1,42 @@ +#ifndef PRECURSOR_MRSHV2_ADAPTER_H +#define PRECURSOR_MRSHV2_ADAPTER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Hash payload bytes into a stable digest string. + * Returns 0 on success and writes a heap-allocated C string to out_digest. + * Caller must release out_digest with precursor_mrshv2_free. + */ +int precursor_mrshv2_hash(const uint8_t *payload, size_t payload_len, char **out_digest); + +/* + * Compute a normalized distance [0,100] between two digest strings. + * 0 means identical and higher values are less similar. + * Returns 0 on success. + */ +int precursor_mrshv2_diff( + const char *left_digest, + const char *right_digest, + int *out_distance +); + +/* Free heap-allocated digest strings returned by precursor_mrshv2_hash. */ +void precursor_mrshv2_free(char *value); + +/* + * Return a pointer to thread-local error text for the last failure. + * The returned pointer is borrowed and must not be freed. + */ +const char *precursor_mrshv2_last_error(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ffi/mrshv2_mock.c b/ffi/mrshv2_mock.c new file mode 100644 index 0000000..8c03ba0 --- /dev/null +++ b/ffi/mrshv2_mock.c @@ -0,0 +1,86 @@ +#include "mrshv2_adapter.h" + +#include +#include +#include + +static _Thread_local char g_last_error[256]; + +static void set_last_error(const char *message) { + if (message == NULL) { + g_last_error[0] = '\0'; + return; + } + snprintf(g_last_error, sizeof(g_last_error), "%s", message); +} + +const char *precursor_mrshv2_last_error(void) { + return g_last_error; +} + +void precursor_mrshv2_free(char *value) { + free(value); +} + +int precursor_mrshv2_hash(const uint8_t *payload, size_t payload_len, char **out_digest) { + if (payload == NULL || payload_len == 0 || out_digest == NULL) { + set_last_error("invalid hash input"); + return -1; + } + + /* Mock FNV-1a digest for CI/smoke tests. */ + unsigned long long hash = 1469598103934665603ULL; + for (size_t i = 0; i < payload_len; i++) { + hash ^= (unsigned long long)payload[i]; + hash *= 1099511628211ULL; + } + + char *digest = (char *)malloc(64); + if (digest == NULL) { + set_last_error("unable to allocate digest buffer"); + return -1; + } + snprintf(digest, 64, "mrshv2:%zu:%016llx", payload_len, hash); + *out_digest = digest; + set_last_error(""); + return 0; +} + +int precursor_mrshv2_diff( + const char *left_digest, + const char *right_digest, + int *out_distance +) { + if (left_digest == NULL || right_digest == NULL || out_distance == NULL) { + set_last_error("invalid diff input"); + return -1; + } + + size_t left_len = strlen(left_digest); + size_t right_len = strlen(right_digest); + size_t max_len = left_len > right_len ? left_len : right_len; + if (max_len == 0) { + *out_distance = 0; + set_last_error(""); + return 0; + } + + size_t min_len = left_len < right_len ? left_len : right_len; + size_t mismatch = left_len > right_len ? left_len - right_len : right_len - left_len; + for (size_t i = 0; i < min_len; i++) { + if (left_digest[i] != right_digest[i]) { + mismatch += 1; + } + } + + int distance = (int)((mismatch * 100 + (max_len / 2)) / max_len); + if (distance < 0) { + distance = 0; + } + if (distance > 100) { + distance = 100; + } + *out_distance = distance; + set_last_error(""); + return 0; +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..4fef3ca --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.86.0" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/samples/scenarios/README.md b/samples/scenarios/README.md new file mode 100644 index 0000000..ed96b9b --- /dev/null +++ b/samples/scenarios/README.md @@ -0,0 +1,43 @@ +# Scenario Corpus + +This corpus is intentionally small and deterministic so it can be versioned, +tested, and benchmarked directly in this repository. + +## Scenarios + +1. `pre-protocol-packet-triage` +Purpose: cluster single packets before full parser selection. +Data: +- `payloads.b64`: mixed HTTP/TLS/SSH/DNS/Modbus payloads +- `patterns.pcre`: named-capture tags for cross-protocol triage + +2. `firmware-fragment-triage` +Purpose: classify likely firmware or compressed fragments from arbitrary blobs. +Data: +- `payloads.hex`: ELF/PE/uImage/gzip-like fragments + one opaque sample +- `patterns.pcre`: file-magic style tags + +3. `ics-modbus-single-packet` +Purpose: tag and cluster Modbus/TCP request/response messages from single packets. +Data: +- `payloads.hex`: short Modbus/TCP frames +- `patterns.pcre`: function code and exception tags + +## Provenance + +Samples are either: +- protocol-shape examples derived from public standards and protocol docs, or +- synthetic test vectors assembled to exercise Precursor behavior. + +Reference docs used when assembling payload shapes: +- RFC 9112 (HTTP/1.1 messaging) +- RFC 4253 (SSH transport) +- RFC 8446 (TLS 1.3 record framing) +- RFC 1035 (DNS message format) +- Modbus Application Protocol Specification v1.1b3 + +## Quick run + +```bash +samples/scenarios/run_all.sh ./target/release/precursor +``` diff --git a/samples/scenarios/firmware-fragment-triage/patterns.pcre b/samples/scenarios/firmware-fragment-triage/patterns.pcre new file mode 100644 index 0000000..14478f8 --- /dev/null +++ b/samples/scenarios/firmware-fragment-triage/patterns.pcre @@ -0,0 +1,5 @@ +(?^\x7fELF) +(?^MZ) +(?^\x27\x05\x19\x56) +(?^\x1f\x8b) +(?^[\x00-\xff]{24,}$) diff --git a/samples/scenarios/firmware-fragment-triage/payloads.hex b/samples/scenarios/firmware-fragment-triage/payloads.hex new file mode 100644 index 0000000..251ba8f --- /dev/null +++ b/samples/scenarios/firmware-fragment-triage/payloads.hex @@ -0,0 +1,5 @@ +7f454c4602010100000000000000000002003e0001000000 +4d5a90000300000004000000ffff0000b8000000 +27051956000000005f3759df0000000000000000 +1f8b08000000000000034b4c4a0600425d5c1c05000000 +8f3ca1b57e4429ff01928374655aaabbeeff00112233445566778899aabbccdd diff --git a/samples/scenarios/ics-modbus-single-packet/patterns.pcre b/samples/scenarios/ics-modbus-single-packet/patterns.pcre new file mode 100644 index 0000000..753e6da --- /dev/null +++ b/samples/scenarios/ics-modbus-single-packet/patterns.pcre @@ -0,0 +1,5 @@ +(?^\x00\x01\x00\x00\x00\x06\x11\x03) +(?^\x00\x01\x00\x00\x00\x09\x11\x03) +(?^\x00\x02\x00\x00\x00\x06\x11\x05) +(?^\x00\x02\x00\x00\x00\x03\x11\x85) +(?^\x00\x03\x00\x00\x00\x06\x11\x01) diff --git a/samples/scenarios/ics-modbus-single-packet/payloads.hex b/samples/scenarios/ics-modbus-single-packet/payloads.hex new file mode 100644 index 0000000..3cc834b --- /dev/null +++ b/samples/scenarios/ics-modbus-single-packet/payloads.hex @@ -0,0 +1,5 @@ +0001000000061103006b0003 +000100000009110306022b00000064 +000200000006110500acff00 +000200000003118502 +000300000006110100130025 diff --git a/samples/scenarios/pre-protocol-packet-triage/patterns.pcre b/samples/scenarios/pre-protocol-packet-triage/patterns.pcre new file mode 100644 index 0000000..efb90dc --- /dev/null +++ b/samples/scenarios/pre-protocol-packet-triage/patterns.pcre @@ -0,0 +1,5 @@ +(?(GET|POST|HEAD|PUT|DELETE) ) +(?^\x16\x03[\x00-\x04]) +(?^SSH-\d\.\d-) +(?example\.com) +(?^\x00[\x00-\xff]\x00\x00\x00[\x02-\x10]\x11[\x01-\x7f]) diff --git a/samples/scenarios/pre-protocol-packet-triage/payloads.b64 b/samples/scenarios/pre-protocol-packet-triage/payloads.b64 new file mode 100644 index 0000000..4dcd094 --- /dev/null +++ b/samples/scenarios/pre-protocol-packet-triage/payloads.b64 @@ -0,0 +1,5 @@ +R0VUIC9hZG1pbi9sb2dpbiBIVFRQLzEuMQ0KSG9zdDogZWRnZS1nYXRld2F5LmV4YW1wbGUNClVzZXItQWdlbnQ6IHNjYW5uZXItYm90DQoNCg== +FgMBAC8BAAArAwNYWVoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAvAAACAC8= +U1NILTIuMC1PcGVuU1NIXzkuNg0K +q80BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE= +AAEAAAAGEQMAawAD diff --git a/samples/scenarios/run_all.sh b/samples/scenarios/run_all.sh new file mode 100755 index 0000000..bb369c6 --- /dev/null +++ b/samples/scenarios/run_all.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +bin_path="${1:-precursor}" +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "== pre-protocol packet triage (lzjd) ==" +"$bin_path" \ + -p "$root_dir/pre-protocol-packet-triage/patterns.pcre" \ + -m base64 \ + -t -d \ + --similarity-mode lzjd \ + -P \ + --protocol-hints \ + < "$root_dir/pre-protocol-packet-triage/payloads.b64" + +echo +echo "== firmware fragment triage (lzjd) ==" +"$bin_path" \ + -p "$root_dir/firmware-fragment-triage/patterns.pcre" \ + -m hex \ + -t \ + --similarity-mode lzjd \ + -P \ + < "$root_dir/firmware-fragment-triage/payloads.hex" + +echo +echo "== ics modbus single-packet (lzjd) ==" +"$bin_path" \ + -p "$root_dir/ics-modbus-single-packet/patterns.pcre" \ + -m hex \ + -t -d \ + --similarity-mode lzjd \ + -P \ + --protocol-hints \ + < "$root_dir/ics-modbus-single-packet/payloads.hex" diff --git a/site/CNAME b/site/CNAME new file mode 100644 index 0000000..f3e5986 --- /dev/null +++ b/site/CNAME @@ -0,0 +1 @@ +precursor.hashdb.io diff --git a/site/DEPLOY.md b/site/DEPLOY.md new file mode 100644 index 0000000..553cd7d --- /dev/null +++ b/site/DEPLOY.md @@ -0,0 +1,39 @@ +# GitHub Pages Deployment + +## 1) Enable GitHub Pages + +1. Open repository settings. +2. Under Pages, set source to GitHub Actions. +3. Merge/push changes containing `.github/workflows/pages.yml` and `site/`. + +## 2) Configure custom domain + +This repository includes `site/CNAME` with: + +```text +precursor.hashdb.io +``` + +GitHub Pages should detect this automatically after deployment. + +## 3) Create/update DNS record at your DNS provider + +Create this record: +- Type: `CNAME` +- Host: `precursor` +- Value: `obsecurus.github.io` +- TTL: `300` (or automatic) + +## 4) Verify + +After DNS propagation: + +```bash +dig +short precursor.hashdb.io +``` + +Then visit: + +```text +https://precursor.hashdb.io +``` diff --git a/site/app.js b/site/app.js new file mode 100644 index 0000000..7600097 --- /dev/null +++ b/site/app.js @@ -0,0 +1,83 @@ +const scenarios = [ + { + id: "packet-triage", + label: "Packet Triage", + title: "Pre-Protocol Packet Triage", + description: + "Cluster mixed HTTP/TLS/SSH/DNS/Modbus payloads before parser commitment. Useful for scanner traffic and unknown service discovery.", + command: `cat samples/scenarios/pre-protocol-packet-triage/payloads.b64 \\ + | precursor -p samples/scenarios/pre-protocol-packet-triage/patterns.pcre \\ + -m base64 -t -d --similarity-mode lzjd -P --protocol-hints`, + output: `stdout: similarity_hash + protocol_* fields +stderr: ---PRECURSOR_PROTOCOL_HINTS--- with top candidate clusters`, + }, + { + id: "firmware-fragments", + label: "Firmware", + title: "Firmware Fragment Sorting", + description: + "Tag likely ELF/PE/uImage/gzip fragments from arbitrary blob streams and route high-entropy unknowns for deeper reverse engineering.", + command: `cat samples/scenarios/firmware-fragment-triage/payloads.hex \\ + | precursor -p samples/scenarios/firmware-fragment-triage/patterns.pcre \\ + -m hex -t --similarity-mode lzjd -P`, + output: `protocol_label typically includes firmware_binary or compressed_binary +tags include file-magic style markers`, + }, + { + id: "ics-single-packet", + label: "ICS/OT", + title: "ICS Modbus Single-Packet Discovery", + description: + "Detect Modbus request/response function families from single packets where full DPI context is unavailable.", + command: `cat samples/scenarios/ics-modbus-single-packet/payloads.hex \\ + | precursor -p samples/scenarios/ics-modbus-single-packet/patterns.pcre \\ + -m hex -t -d --similarity-mode lzjd -P --protocol-hints`, + output: `cluster boosts improve confidence when payload families repeat +hint candidates can be fed into LLM-assisted rule authoring loops`, + }, +]; + +const tabContainer = document.getElementById("scenario-tabs"); +const title = document.getElementById("scenario-title"); +const description = document.getElementById("scenario-description"); +const command = document.getElementById("scenario-command"); +const output = document.getElementById("scenario-output"); +const copyButton = document.getElementById("copy-command"); + +function renderScenario(scenarioId) { + const selected = scenarios.find((scenario) => scenario.id === scenarioId) || scenarios[0]; + title.textContent = selected.title; + description.textContent = selected.description; + command.textContent = selected.command; + output.textContent = selected.output; + tabContainer.querySelectorAll("button").forEach((button) => { + button.setAttribute("aria-selected", button.dataset.scenarioId === selected.id ? "true" : "false"); + }); +} + +scenarios.forEach((scenario, idx) => { + const button = document.createElement("button"); + button.type = "button"; + button.dataset.scenarioId = scenario.id; + button.textContent = scenario.label; + button.setAttribute("aria-selected", idx === 0 ? "true" : "false"); + button.addEventListener("click", () => renderScenario(scenario.id)); + tabContainer.appendChild(button); +}); + +copyButton.addEventListener("click", async () => { + try { + await navigator.clipboard.writeText(command.textContent || ""); + copyButton.textContent = "Copied"; + setTimeout(() => { + copyButton.textContent = "Copy"; + }, 1000); + } catch (_error) { + copyButton.textContent = "Manual copy"; + setTimeout(() => { + copyButton.textContent = "Copy"; + }, 1200); + } +}); + +renderScenario(scenarios[0].id); diff --git a/site/index.html b/site/index.html new file mode 100644 index 0000000..83467da --- /dev/null +++ b/site/index.html @@ -0,0 +1,159 @@ + + + + + + Precursor | Pre-Protocol Similarity Triage + + + + + + + +
+
+
+

precursor.hashdb.io

+

Pre-Protocol Payload Triage for Packets, Logs, and Firmware Fragments

+

+ Precursor tags payloads with named captures, clusters near-matches with + TLSH/LZJD (and optional MRSHv2 adapter mode), and emits JSON designed + for SOC pipelines and LLM-guided protocol discovery loops. +

+
+ GitHub + Try Demo Scenarios +
+
+ +
+
+

Dual Input Shapes

+

Text/base64/hex now, raw-binary expansion in active roadmap.

+
+
+

Similarity Modes

+

TLSH + LZJD implemented, MRSHv2 available behind native adapter feature.

+
+
+

Discovery Loop

+

Protocol hints + single-packet inference for human and LLM analysis loops.

+
+
+ +
+
+

Why Install Precursor

+

+ One command turns opaque payload streams into tags, clusters, and + protocol confidence output you can act on immediately. +

+
+
+
+

Input stream

+
+ GET /admin HTTP/1.1 + 16 03 03 ... + 00 01 00 00 00 06 11 03 ... +
+
+ +
+

Tag + similarity

+
    +
  • tags: ["http_method"]
    • +
  • similarity_hash: "lzjd:..."
    • +
  • tlsh_similarities: {...}
    • +
+
+ +
+

Actionable triage

+
+
+ http + 0.93 +
+
+ tls + 0.90 +
+
+ firmware_binary + 0.86 +
+
+
+
+
+
+ Sample JSON line +
+ 
{"protocol_label":"http","protocol_confidence":0.93,"similarity_hash":"lzjd:128:...","tags":["http_method"]}
+
+
+ +
+
+

Scenario Demos

+

These examples map directly to files committed under samples/scenarios/.

+
+ +
+

+

+
+
+ Command + +
+
+
+
+
+ Expected signal +
+
+
+
+
+ +
+

High-Impact Use Cases

+
+
+

Exploit Spray Triage

+

Cluster scanner traffic before parser development and identify repeated payload families quickly.

+
+
+

ICS/OT Packet Discovery

+

Start from single packets when DPI fails or protocol metadata is missing.

+
+
+

Firmware Fragment Sorting

+

Tag binary fragments by magic + similarity to route unknown blobs to the right analyst workflow.

+
+
+
+ +
+

Deploy This Site to GitHub Pages

+
    +
  1. Enable Pages in repository settings and select GitHub Actions as source.
    2. +
  2. Create DNS record: precursor.hashdb.io CNAME obsecurus.github.io.
    3. +
  3. Push to main; workflow pages.yml publishes site/.
    4. +
+
+
+ + + + diff --git a/site/styles.css b/site/styles.css new file mode 100644 index 0000000..31f2384 --- /dev/null +++ b/site/styles.css @@ -0,0 +1,423 @@ +:root { + --paper: #f9f7ef; + --ink: #0f1f2f; + --muted: #4f5f6f; + --accent: #ff5f2e; + --accent-soft: #ffd67c; + --teal: #0f8f8f; + --card: rgba(255, 255, 255, 0.84); + --line: rgba(15, 31, 47, 0.14); +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + font-family: "Space Grotesk", sans-serif; + color: var(--ink); + background: var(--paper); +} + +.bg-layer { + position: fixed; + inset: 0; + z-index: -1; + background: + radial-gradient(circle at 12% 12%, rgba(255, 214, 124, 0.45) 0, transparent 42%), + radial-gradient(circle at 86% 0%, rgba(15, 143, 143, 0.24) 0, transparent 38%), + linear-gradient(130deg, #fff9e9 0%, #f1f8f7 52%, #fdf1e6 100%); +} + +main { + width: min(1080px, 92vw); + margin: 0 auto; + padding: 2.8rem 0 4.2rem; +} + +.hero { + display: grid; + gap: 1rem; + padding: 2rem; + border: 1px solid var(--line); + border-radius: 28px; + background: var(--card); + backdrop-filter: blur(5px); + animation: rise-in 0.7s ease-out both; +} + +.kicker { + margin: 0; + font-family: "IBM Plex Mono", monospace; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--teal); +} + +h1 { + margin: 0; + font-size: clamp(1.9rem, 4.6vw, 3.15rem); + line-height: 1.07; +} + +.hero-copy { + margin: 0; + max-width: 72ch; + color: var(--muted); + font-size: 1.05rem; +} + +.hero-cta { + display: flex; + flex-wrap: wrap; + gap: 0.75rem; +} + +.hero-cta a { + text-decoration: none; + color: var(--ink); + border: 1px solid var(--line); + border-radius: 999px; + padding: 0.54rem 1rem; + background: #fff; + transition: transform 0.2s ease, border-color 0.2s ease; +} + +.hero-cta a:hover { + transform: translateY(-1px); + border-color: var(--accent); +} + +.stats { + margin-top: 1.3rem; + display: grid; + gap: 0.9rem; + grid-template-columns: repeat(3, minmax(0, 1fr)); +} + +.stats article { + border: 1px solid var(--line); + border-radius: 20px; + background: rgba(255, 255, 255, 0.72); + padding: 1rem; + animation: rise-in 0.7s ease-out both; +} + +.stats h2 { + margin: 0; + font-size: 1rem; +} + +.stats p { + margin: 0.4rem 0 0; + color: var(--muted); + font-size: 0.95rem; +} + +.teaser { + border: 1px solid var(--line); + border-radius: 24px; + padding: 1rem; + background: rgba(255, 255, 255, 0.8); +} + +.teaser-head h2 { + margin: 0; + font-size: 1.4rem; +} + +.teaser-head p { + margin: 0.4rem 0 0; + color: var(--muted); +} + +.teaser-stage { + margin-top: 0.9rem; + display: grid; + grid-template-columns: minmax(0, 1fr) auto minmax(0, 1fr) auto minmax(0, 1fr); + gap: 0.6rem; + align-items: stretch; +} + +.teaser-lane { + border: 1px solid var(--line); + border-radius: 16px; + background: rgba(255, 255, 255, 0.95); + padding: 0.75rem; +} + +.teaser-lane h3 { + margin: 0; + font-size: 0.95rem; +} + +.teaser-arrow { + align-self: center; + font-family: "IBM Plex Mono", monospace; + color: var(--teal); + font-size: 1.1rem; +} + +.chip-stack { + margin-top: 0.55rem; + display: grid; + gap: 0.42rem; +} + +.chip { + display: inline-flex; + width: fit-content; + max-width: 100%; + font-family: "IBM Plex Mono", monospace; + font-size: 0.72rem; + padding: 0.28rem 0.44rem; + border-radius: 9px; + border: 1px solid rgba(15, 31, 47, 0.15); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + animation: chip-drift 2.8s ease-in-out infinite; +} + +.chip-http { + background: rgba(255, 95, 46, 0.14); + animation-delay: 0s; +} + +.chip-tls { + background: rgba(15, 143, 143, 0.14); + animation-delay: 0.4s; +} + +.chip-ics { + background: rgba(255, 214, 124, 0.35); + animation-delay: 0.8s; +} + +.teaser-list { + margin: 0.58rem 0 0; + padding-left: 1rem; + display: grid; + gap: 0.28rem; +} + +.teaser-list li { + color: var(--muted); + font-size: 0.85rem; +} + +.teaser-list code { + color: #1f2a35; + font-size: 0.76rem; +} + +.cluster-view { + margin-top: 0.58rem; + display: grid; + gap: 0.3rem; +} + +.cluster-row { + display: flex; + justify-content: space-between; + align-items: center; + border: 1px solid var(--line); + border-radius: 10px; + padding: 0.34rem 0.5rem; + font-family: "IBM Plex Mono", monospace; + font-size: 0.78rem; + background: linear-gradient(90deg, rgba(15, 143, 143, 0.11), rgba(255, 214, 124, 0.14)); +} + +.teaser-terminal { + margin-top: 0.75rem; + border: 1px solid var(--line); + border-radius: 12px; + overflow: hidden; +} + +.teaser-terminal pre { + background: #131b24; +} + +section { + margin-top: 1.9rem; +} + +.demo-head h2, +.use-cases h2, +.deploy h2 { + margin: 0; + font-size: 1.4rem; +} + +.demo-head p { + margin: 0.45rem 0 0; + color: var(--muted); +} + +.scenario-tabs { + margin-top: 0.95rem; + display: flex; + flex-wrap: wrap; + gap: 0.65rem; +} + +.scenario-tabs button { + border: 1px solid var(--line); + border-radius: 999px; + background: #fff; + font-family: "IBM Plex Mono", monospace; + font-size: 0.78rem; + padding: 0.42rem 0.78rem; + color: var(--ink); + cursor: pointer; +} + +.scenario-tabs button[aria-selected="true"] { + border-color: var(--accent); + background: var(--accent-soft); +} + +.scenario-panel { + margin-top: 0.9rem; + border: 1px solid var(--line); + border-radius: 22px; + background: rgba(255, 255, 255, 0.82); + padding: 1rem; +} + +.scenario-panel h3 { + margin: 0; +} + +.scenario-panel p { + margin: 0.52rem 0 0.8rem; + color: var(--muted); +} + +.code-wrap { + margin-top: 0.8rem; + border: 1px solid var(--line); + border-radius: 14px; + background: #131b24; + overflow: hidden; +} + +.code-head { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.5rem 0.7rem; + background: #1f2a35; + color: #d9e5f2; + font-family: "IBM Plex Mono", monospace; + font-size: 0.72rem; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.code-head button { + border: 1px solid #395167; + border-radius: 8px; + background: #111821; + color: #d9e5f2; + font-family: "IBM Plex Mono", monospace; + font-size: 0.72rem; + padding: 0.3rem 0.52rem; + cursor: pointer; +} + +pre { + margin: 0; + padding: 0.88rem 0.95rem; + overflow: auto; +} + +code { + font-family: "IBM Plex Mono", monospace; + color: #d9e5f2; + font-size: 0.84rem; + line-height: 1.45; +} + +.cards { + margin-top: 0.95rem; + display: grid; + gap: 0.9rem; + grid-template-columns: repeat(3, minmax(0, 1fr)); +} + +.cards article { + border: 1px solid var(--line); + border-radius: 16px; + background: rgba(255, 255, 255, 0.84); + padding: 0.85rem; +} + +.cards h3 { + margin: 0; + font-size: 1rem; +} + +.cards p { + margin: 0.4rem 0 0; + color: var(--muted); + font-size: 0.95rem; +} + +.deploy ol { + margin: 0.85rem 0 0; + padding-left: 1.2rem; + color: var(--muted); + display: grid; + gap: 0.35rem; +} + +.deploy code { + color: #f5bd7d; +} + +@keyframes rise-in { + from { + transform: translateY(10px); + opacity: 0; + } + to { + transform: translateY(0); + opacity: 1; + } +} + +@keyframes chip-drift { + 0% { + transform: translateX(0); + } + 50% { + transform: translateX(5px); + } + 100% { + transform: translateX(0); + } +} + +@media (max-width: 900px) { + .stats, + .cards { + grid-template-columns: 1fr; + } + + .teaser-stage { + grid-template-columns: 1fr; + } + + .teaser-arrow { + transform: rotate(90deg); + justify-self: center; + } + + main { + padding-top: 1.3rem; + } +} diff --git a/skills/precursor-maintainer/SKILL.md b/skills/precursor-maintainer/SKILL.md new file mode 100644 index 0000000..f1fc26e --- /dev/null +++ b/skills/precursor-maintainer/SKILL.md @@ -0,0 +1,42 @@ +--- +name: precursor-maintainer +description: Maintain and evolve the precursor Rust CLI for PCRE2/TLSH labeling and similarity workflows. Use when working in this repository on bug fixes, refactors, performance tuning, CI/release hygiene, CLI or JSON behavior changes, architecture reviews, or reliability hardening. +--- + +# Precursor Maintainer + +## Quick Start +1. Read `ai/MEMORY.md` for current architecture and priorities. +2. Read `ai/REPO_REVIEW.md` for known risks before editing hot paths. +3. Load only relevant source files (`src/main.rs`, `src/precursor/*.rs`, workflow files) for the task. + +## Workflow + +### 1) Scope the change +- Confirm the user-visible behavior that must stay stable. +- Identify whether the task affects ingest, matching, TLSH comparison, stats, or release/CI. +- Prefer the smallest patch that resolves the target issue. + +### 2) Baseline quickly +- Run `scripts/scan_hotspots.sh` to surface panic/unwrap/TODO hotspots. +- Run `scripts/run_checks.sh` when toolchain supports it. +- If checks cannot run, capture the blocking reason in the final report. + +### 3) Implement safely +- Keep JSON shape stable unless a breaking change is requested. +- Replace panic paths on untrusted input with recoverable error handling. +- Avoid introducing per-line allocations or regex recompilation in hot loops. + +### 4) Validate +- Re-run focused checks for touched behavior. +- Re-run `scripts/scan_hotspots.sh` if touching ingest or matching code. +- Update docs when CLI flags, output shape, or release flow changes. + +### 5) Persist memory +- Update `ai/MEMORY.md` for architectural or process changes. +- Update `ai/REPO_REVIEW.md` when risks are fixed or new ones are discovered. + +## References +- Use `references/repo-map.md` for file ownership and module boundaries. +- Use `references/quality-gates.md` for preferred verification sequence. +- Use `references/backlog.md` for prioritized roadmap candidates. diff --git a/skills/precursor-maintainer/agents/openai.yaml b/skills/precursor-maintainer/agents/openai.yaml new file mode 100644 index 0000000..f20d224 --- /dev/null +++ b/skills/precursor-maintainer/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Precursor Maintainer" + short_description: "Maintain and evolve the Precursor codebase" + default_prompt: "Use $precursor-maintainer to review and implement robust improvements in this repository." diff --git a/skills/precursor-maintainer/references/backlog.md b/skills/precursor-maintainer/references/backlog.md new file mode 100644 index 0000000..aa4591c --- /dev/null +++ b/skills/precursor-maintainer/references/backlog.md @@ -0,0 +1,16 @@ +# Backlog (Prioritized) + +## P0 +- Remove panic/unwrap from untrusted input paths and return structured per-line errors. +- Precompile regexes once before line iteration. +- Harden stats calculations for empty input and no-match cases. + +## P1 +- Introduce integration tests with realistic corpora under `samples/`. +- Fix release checklist drift and ensure release docs reference `precursor` only. +- Separate CLI orchestration from reusable library logic (`src/lib.rs`). + +## P2 +- Add binary/blob ingest mode and corresponding tests. +- Add tuning mode for TLSH algorithm/distance selection. +- Evaluate memory-efficient similarity indexing for large datasets. diff --git a/skills/precursor-maintainer/references/quality-gates.md b/skills/precursor-maintainer/references/quality-gates.md new file mode 100644 index 0000000..cbd1e36 --- /dev/null +++ b/skills/precursor-maintainer/references/quality-gates.md @@ -0,0 +1,16 @@ +# Quality Gates + +## Fast local gates +1. Run `skills/precursor-maintainer/scripts/scan_hotspots.sh`. +2. Run `cargo fmt --all --check`. +3. Run `cargo test --workspace`. + +## If Cargo lockfile/toolchain mismatch blocks tests +- Record the exact error. +- Run static checks (`scan_hotspots.sh`) and line-level review. +- Avoid claiming runtime verification succeeded. + +## Change-specific checks +- Ingest/decoder changes: test malformed base64, malformed hex, malformed JSON lines. +- Pattern pipeline changes: test empty pattern file, bad pattern syntax, high-volume input. +- TLSH changes: test small payload path (<49 bytes), distance threshold behavior, and `--tlsh-sim-only` output filtering. diff --git a/skills/precursor-maintainer/references/repo-map.md b/skills/precursor-maintainer/references/repo-map.md new file mode 100644 index 0000000..e6c3f35 --- /dev/null +++ b/skills/precursor-maintainer/references/repo-map.md @@ -0,0 +1,21 @@ +# Repo Map + +## Runtime pipeline +- `src/main.rs`: orchestrates CLI parsing, ingest loops, match execution, TLSH diff, and output. +- `src/precursor/util.rs`: helpers for decoding payloads, reading pattern files, regex builder, and utility tests. +- `src/precursor/tlsh.rs`: TLSH abstraction over algorithm families and error struct. + +## Patterns and examples +- `patterns/definitions`: shared named-pattern definitions. +- `patterns/*`: specialized rule packs (`fortinet`, `ics`, `suspicious`, etc.). +- `samples/`: intended for corpus data (currently almost empty). + +## Release and CI +- `.github/workflows/ci.yml`: multi-platform build, test, fmt, docs checks. +- `.github/workflows/release.yml`: release artifact build and upload. +- `RELEASE-CHECKLIST.md`: needs cleanup from ripgrep leftovers. + +## Documentation memory +- `ai/MEMORY.md`: stable project memory. +- `ai/PROMPT_STRATEGY.md`: recommended prompting patterns. +- `ai/REPO_REVIEW.md`: current risk register. diff --git a/skills/precursor-maintainer/scripts/run_checks.sh b/skills/precursor-maintainer/scripts/run_checks.sh new file mode 100755 index 0000000..7c9d731 --- /dev/null +++ b/skills/precursor-maintainer/scripts/run_checks.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$repo_root" + +echo "== Toolchain ==" +rustc --version || true +cargo --version || true + +echo "== Cargo metadata ==" +if ! cargo metadata --format-version 1 --no-deps >/tmp/precursor-cargo-metadata.json 2>/tmp/precursor-cargo-metadata.err; then + cat /tmp/precursor-cargo-metadata.err + echo "cargo metadata failed. Toolchain likely too old for this lockfile." + exit 2 +fi + +echo "== rustfmt ==" +cargo fmt --all --check + +echo "== tests ==" +cargo test --workspace + +echo "== done ==" diff --git a/skills/precursor-maintainer/scripts/scan_hotspots.sh b/skills/precursor-maintainer/scripts/scan_hotspots.sh new file mode 100755 index 0000000..442d11d --- /dev/null +++ b/skills/precursor-maintainer/scripts/scan_hotspots.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$repo_root" + +echo "== Panic/unwrap/TODO hotspots ==" +rg -n --no-heading "NOT IMPLEMENTED|TODO|panic!|unwrap\\(|expect\\(" src README.md RELEASE-CHECKLIST.md || true + +echo +echo "== Rust file sizes ==" +wc -l src/main.rs src/precursor/*.rs | sort -n diff --git a/skills/precursor-pattern-lab/SKILL.md b/skills/precursor-pattern-lab/SKILL.md new file mode 100644 index 0000000..92be21d --- /dev/null +++ b/skills/precursor-pattern-lab/SKILL.md @@ -0,0 +1,42 @@ +--- +name: precursor-pattern-lab +description: Design, tune, and validate PCRE2 named-capture rule packs for precursor. Use when creating new detection tags, reducing false positives, reorganizing pattern files, or validating pattern quality before merging. +--- + +# Precursor Pattern Lab + +## Quick Start +1. Load target rule file(s) under `patterns/`. +2. Read `references/pattern-authoring.md` for naming and quality conventions. +3. Run `scripts/validate_pattern_file.sh ` before proposing changes. + +## Workflow + +### 1) Define detection intent +- State what behavior is being tagged. +- Define expected true positives and likely false positives. +- Choose input encoding assumptions (`base64`, `string`, `hex`). + +### 2) Author or revise patterns +- Use named capture groups because tag extraction uses capture names. +- Keep each line focused on one logical detection purpose. +- Prefer precise anchors and context constraints over broad `.*` greed. + +### 3) Perform static validation +- Run `scripts/validate_pattern_file.sh` on modified files. +- Run `scripts/list_pattern_tags.sh` to inspect resulting tag inventory. +- Resolve malformed lines before runtime testing. + +### 4) Evaluate on corpus +- Use positive and negative samples as described in `references/test-corpus-guidance.md`. +- Track precision problems and adjust pattern specificity first. +- Document known blind spots explicitly in the change summary. + +### 5) Deliver merge-ready output +- Include updated pattern files. +- Include validation command outputs. +- Call out migration or naming impacts if tags were renamed. + +## References +- Use `references/pattern-authoring.md` for capture naming and regex practices. +- Use `references/test-corpus-guidance.md` for repeatable eval setup. diff --git a/skills/precursor-pattern-lab/agents/openai.yaml b/skills/precursor-pattern-lab/agents/openai.yaml new file mode 100644 index 0000000..317d659 --- /dev/null +++ b/skills/precursor-pattern-lab/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Precursor Pattern Lab" + short_description: "Design and validate PCRE2 pattern packs" + default_prompt: "Use $precursor-pattern-lab to create, test, and tune high-signal pattern files." diff --git a/skills/precursor-pattern-lab/references/pattern-authoring.md b/skills/precursor-pattern-lab/references/pattern-authoring.md new file mode 100644 index 0000000..601d039 --- /dev/null +++ b/skills/precursor-pattern-lab/references/pattern-authoring.md @@ -0,0 +1,22 @@ +# Pattern Authoring Guide + +## Tag naming +- Name captures in lowercase snake_case. +- Treat capture names as public tag IDs. +- Rename tags only when necessary; call out breaking changes. + +## Pattern construction +- Use explicit boundaries where possible. +- Minimize broad `.*` prefixes/suffixes unless unavoidable. +- Keep per-line complexity reasonable to avoid catastrophic backtracking. +- Prefer one intent per rule line. + +## Safety checks +- Ensure each non-empty rule line has at least one named capture group. +- Keep comments in separate lines if needed. +- Validate syntax before runtime execution. + +## Practical tuning +- Start with high precision, then relax for recall if needed. +- Maintain a negative corpus to guard against regressions. +- Track additions/removals of tags with `list_pattern_tags.sh`. diff --git a/skills/precursor-pattern-lab/references/test-corpus-guidance.md b/skills/precursor-pattern-lab/references/test-corpus-guidance.md new file mode 100644 index 0000000..cb868bc --- /dev/null +++ b/skills/precursor-pattern-lab/references/test-corpus-guidance.md @@ -0,0 +1,17 @@ +# Test Corpus Guidance + +## Corpus layout suggestion +- `samples/positive/.txt`: lines expected to match at least one target tag. +- `samples/negative/.txt`: lines expected to avoid those tags. +- Keep samples representative of production payload formats. + +## Evaluation loop +1. Validate syntax and named captures. +2. Run tag inventory and verify expected names appear. +3. Execute runtime checks against positive/negative sets. +4. Record false positives/false negatives and iterate. + +## Minimum evidence for pattern changes +- At least one positive sample per new tag. +- At least one negative sample that is close-but-should-not-match. +- Command transcript in PR notes for reproducibility. diff --git a/skills/precursor-pattern-lab/scripts/list_pattern_tags.sh b/skills/precursor-pattern-lab/scripts/list_pattern_tags.sh new file mode 100755 index 0000000..da4cc43 --- /dev/null +++ b/skills/precursor-pattern-lab/scripts/list_pattern_tags.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -lt 1 ]; then + echo "usage: $0 [pattern-file ...]" >&2 + exit 1 +fi + +for pattern_file in "$@"; do + if [ ! -f "$pattern_file" ]; then + echo "error: file not found: $pattern_file" >&2 + exit 1 + fi +done + +rg -o '\(\?<[^>]+>' "$@" \ + | sed -E 's/^\(\?<([^>]+)>$/\1/' \ + | sort -u diff --git a/skills/precursor-pattern-lab/scripts/validate_pattern_file.sh b/skills/precursor-pattern-lab/scripts/validate_pattern_file.sh new file mode 100755 index 0000000..5b8339c --- /dev/null +++ b/skills/precursor-pattern-lab/scripts/validate_pattern_file.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +pattern_file="$1" +if [ ! -f "$pattern_file" ]; then + echo "error: file not found: $pattern_file" >&2 + exit 1 +fi + +line_no=0 +valid_lines=0 +errors=0 +declare -A seen + +while IFS= read -r raw_line || [ -n "$raw_line" ]; do + line_no=$((line_no + 1)) + line="$(printf '%s' "$raw_line" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')" + + if [ -z "$line" ] || [[ "$line" =~ ^# ]]; then + continue + fi + + if ! printf '%s' "$line" | rg -q '\(\?<[^>]+>'; then + echo "error:$line_no: missing named capture group: $line" >&2 + errors=$((errors + 1)) + continue + fi + + while IFS= read -r cap; do + [ -z "$cap" ] && continue + if [ -n "${seen[$cap]:-}" ]; then + echo "warn:$line_no: duplicate capture name '$cap' (first seen on line ${seen[$cap]})" >&2 + else + seen[$cap]="$line_no" + fi + done < <(printf '%s' "$line" | rg -o '\(\?<[^>]+>' | sed -E 's/^\(\?<([^>]+)>$/\1/') + + valid_lines=$((valid_lines + 1)) +done < "$pattern_file" + +if [ "$errors" -gt 0 ]; then + echo "validation failed: $errors error(s), $valid_lines valid rule line(s)" >&2 + exit 2 +fi + +echo "validation passed: $valid_lines valid rule line(s), ${#seen[@]} unique capture name(s)" diff --git a/src/main.rs b/src/main.rs index 038ae79..2b918ea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ mod precursor; use std::collections::HashSet; -use std::io::{self, BufRead, Write}; +use std::io::{self, BufRead, Read, Write}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -17,7 +17,8 @@ extern crate rayon; extern crate serde_json; extern crate xxhash_rust; -use crate::precursor::tlsh::*; +use crate::precursor::inference::infer_protocol_candidates; +use crate::precursor::similarity::*; use crate::precursor::util::*; use atomic_counter::{AtomicCounter, ConsistentCounter}; @@ -47,6 +48,16 @@ const INPUT_MODE_HEX: &str = "hex"; const INPUT_JSON_KEY: &str = "input-json-key"; const PATTERN_FILE: &str = "pattern-file"; const PATTERN: &str = "pattern"; +const SIMILARITY_MODE: &str = "similarity-mode"; +const SIMILARITY_MODE_TLSH: &str = "tlsh"; +const SIMILARITY_MODE_LZJD: &str = "lzjd"; +const SIMILARITY_MODE_MRSHV2: &str = "mrshv2"; +const SIMILARITY_MODE_FBHASH: &str = "fbhash"; +const PROTOCOL_HINTS: &str = "protocol-hints"; +const PROTOCOL_HINTS_LIMIT: &str = "protocol-hints-limit"; +const SINGLE_PACKET: &str = "single-packet"; +const ABSTAIN_THRESHOLD: &str = "abstain-threshold"; +const PROTOCOL_TOP_K: &str = "protocol-top-k"; fn main() { // Start execution timer @@ -65,17 +76,22 @@ fn main() { let vec_tlsh_disance: Arc>> = Arc::new(Mutex::new(Vec::new())); // Create a list to store tlsh::Tlsh objects - let tlsh_list: Vec = Vec::new(); + let tlsh_list: Vec = Vec::new(); // Create map store payload reports by xxh3_64 hash let payload_reports = Map::new(); // Create map store to store tlsh_reports by tlsh let tlsh_reports: DashMap = DashMap::new(); + let similarity_mode_help = if cfg!(feature = "similarity-mrshv2") { + "Select the similarity backend. TLSH/LZJD/MRSHv2 are implemented; FBHash is scaffolded." + } else { + "Select the similarity backend. TLSH and LZJD are implemented; MRSHv2/FBHash are scaffolded." + }; // Create a clap::ArgMatches object to store the CLI arguments let cmd = Command::new("precursor") - .about("Precursor is a regex (PCRE2) and locality sensitive hasing (TLSH) tool for labeling and finding similairites between text, hex, or base64 encoded data.") + .about("Precursor is a PCRE2 payload tagging and similarity hashing CLI (TLSH/LZJD) for text, hex, or base64 input.") .color(ColorChoice::Auto) .long_about("Precursor currently supports the following TLSH algorithms:\n 1. Tlsh48_1\n @@ -83,11 +99,12 @@ fn main() { 3. Tlsh128_3\n 4. Tlsh256_1\n 5. Tlsh256_3\n + 6. LZJD-style sketching (`--similarity-mode lzjd`)\n \n - The -d flag performs TLSH distance calculations between every line of input provided. This is an expensive O(2^n) operation and can consume significant amounts of memory. You can optimize this by using appropriate PCRE2 pre-filters and chosing a smaller TLSH algorithm.") + The -d flag performs pairwise distance calculations between every line of input provided. This is an expensive O(2^n) operation and can consume significant amounts of memory. You can optimize this by using appropriate PCRE2 pre-filters and choosing a smaller TLSH algorithm/sketch.") .arg(Arg::new(PATTERN) .help("Specify the PCRE2 pattern to be used, it must contain a single named capture group.") - .required(false) + .required_unless_present(PATTERN_FILE) .index(1)) .arg(Arg::new(INPUT_FOLDER) .short('f') @@ -98,13 +115,14 @@ fn main() { .arg(Arg::new(INPUT_BLOB) .short('z') .long(INPUT_BLOB) - .help("NOT IMPLEMENTED! - Process input as single blob instead of splitting on newlines.") + .help("Process each input source as a single blob instead of splitting on newlines.") .action(ArgAction::SetTrue)) .arg(Arg::new(PATTERN_FILE) .short('p') .long(PATTERN_FILE) .value_parser(PathBufValueParser::new()) .help("Specify the path to the file containing PCRE2 patterns, one per line, each must contain a single named capture group.") + .conflicts_with(PATTERN) .action(ArgAction::Set)) .arg(Arg::new(TLSH) .short('t') @@ -140,6 +158,46 @@ fn main() { .long(TLSH_LENGTH) .help("This uses a TLSH algorithm that considered the payload length.") .action(ArgAction::SetTrue)) + .arg(Arg::new(SIMILARITY_MODE) + .long(SIMILARITY_MODE) + .help(similarity_mode_help) + .value_parser([ + SIMILARITY_MODE_TLSH, + SIMILARITY_MODE_LZJD, + SIMILARITY_MODE_MRSHV2, + SIMILARITY_MODE_FBHASH, + ]) + .action(ArgAction::Set) + .default_value(SIMILARITY_MODE_TLSH)) + .arg(Arg::new(PROTOCOL_HINTS) + .long(PROTOCOL_HINTS) + .help("Emit protocol-discovery hint JSON to STDERR for LLM-guided analysis loops.") + .action(ArgAction::SetTrue)) + .arg(Arg::new(PROTOCOL_HINTS_LIMIT) + .long(PROTOCOL_HINTS_LIMIT) + .help("Limit the number of protocol hint candidates emitted.") + .value_parser(value_parser!(usize)) + .default_value("25") + .action(ArgAction::Set)) + .arg(Arg::new(SINGLE_PACKET) + .short('P') + .long(SINGLE_PACKET) + .help("Enable single-packet protocol inference heuristics for matched payloads.") + .action(ArgAction::SetTrue)) + .arg(Arg::new(ABSTAIN_THRESHOLD) + .short('A') + .long(ABSTAIN_THRESHOLD) + .help("Confidence threshold below which protocol inference abstains as unknown.") + .value_parser(value_parser!(f64)) + .default_value("0.65") + .action(ArgAction::Set)) + .arg(Arg::new(PROTOCOL_TOP_K) + .short('k') + .long(PROTOCOL_TOP_K) + .help("Maximum number of protocol candidates to include per report.") + .value_parser(value_parser!(usize)) + .default_value("3") + .action(ArgAction::Set)) .arg(Arg::new(INPUT_MODE) .short('m') .long(INPUT_MODE) @@ -159,74 +217,154 @@ fn main() { .action(ArgAction::SetTrue)); let args = cmd.get_matches(); + let similarity_mode_value = args + .get_one::(SIMILARITY_MODE) + .map_or(SIMILARITY_MODE_TLSH, String::as_str); + let similarity_mode = match SimilarityMode::from_str(similarity_mode_value) { + Ok(mode) => mode, + Err(err) => { + eprintln!("Unable to parse similarity mode: {}", err); + std::process::exit(2); + } + }; + + let similarity_requested = + args.get_flag(TLSH) || args.get_flag(TLSH_DIFF) || args.get_flag(TLSH_LENGTH); + if similarity_requested { + let mrshv2_enabled = cfg!(feature = "similarity-mrshv2"); + if similarity_mode == SimilarityMode::FbHash { + eprintln!( + "Similarity mode '{}' is scaffolded but not implemented yet. Use --{} {} or --{} {} for active hashing.", + similarity_mode.as_str(), + SIMILARITY_MODE, + SIMILARITY_MODE_TLSH, + SIMILARITY_MODE, + SIMILARITY_MODE_LZJD + ); + std::process::exit(2); + } + if similarity_mode == SimilarityMode::Mrshv2 && !mrshv2_enabled { + eprintln!( + "Similarity mode '{}' requires compiling with `--features similarity-mrshv2` and linking a native adapter. Use --{} {} or --{} {} for active hashing in this build.", + similarity_mode.as_str(), + SIMILARITY_MODE, + SIMILARITY_MODE_TLSH, + SIMILARITY_MODE, + SIMILARITY_MODE_LZJD + ); + std::process::exit(2); + } + } let tlsh_list = Mutex::new(tlsh_list); let payload_reports = Mutex::new(payload_reports); - #[allow(unused_assignments)] - // This is valid because of the rayon usesage via the par_iter() method - let mut patterns: Vec = Vec::new(); - if args.contains_id(PATTERN_FILE) { - let pattern_file = args - .get_one::(PATTERN_FILE) - .expect("Unable to read pattern file"); - patterns = read_patterns(Some(pattern_file)); - } else { - let pattern = args - .get_one::(PATTERN) - .expect("Unable to read pattern"); - patterns = vec![pattern.to_string()]; - } - - counter_pcre_patterns.add(patterns.len()); - - if args.contains_id(INPUT_FOLDER) { - let path = args - .get_one::(INPUT_FOLDER) - .expect("Unable to read input folder"); - if path.is_dir() { - for entry in std::fs::read_dir(path).expect("Unable to read directory") { - let entry = entry.expect("Unable to read entry"); - let file_path: PathBuf = entry.path(); - println!("Processing file: {}", file_path.display()); - if file_path.is_file() { - let file = std::fs::File::open(&file_path).expect("Unable to open file"); - let reader = std::io::BufReader::new(file); - for line in reader.lines() { - let line = line.expect("Unable to read line"); - handle_line( - &line, - &patterns, - &args, - &tlsh_list, - &payload_reports, - &counter_pcre_matches, - &counter_tlsh_hashes, - &vec_payload_size, - &vec_payload_size_matched, - &counter_unique_payloads, - &counter_pcre_matches_total, - ); - } + let patterns: Vec = + if let Some(pattern_file) = args.get_one::(PATTERN_FILE) { + match read_patterns(Some(pattern_file)) { + Ok(patterns) => patterns, + Err(err) => { + eprintln!( + "Unable to read pattern file {}: {}", + pattern_file.display(), + err + ); + std::process::exit(2); } } + } else if let Some(pattern) = args.get_one::(PATTERN) { + vec![pattern.to_string()] } else { - println!("-f path must be a folder"); + eprintln!("Either a positional pattern or --pattern-file must be provided."); + std::process::exit(2); + }; + + let mut compiled_patterns = Vec::with_capacity(patterns.len()); + for pattern in &patterns { + match build_regex(pattern) { + Ok(re) => compiled_patterns.push(re), + Err(err) => { + eprintln!("Invalid PCRE2 pattern '{}': {}", pattern, err); + std::process::exit(2); + } } - } else { - let stdin = io::stdin(); - stdin - .lock() - .lines() - .filter_map(Result::ok) - .collect::>() - .par_iter() - .for_each(|line| { + } + counter_pcre_patterns.add(compiled_patterns.len()); + + if let Some(path) = args.get_one::(INPUT_FOLDER) { + if !path.is_dir() { + eprintln!("-f path must be a folder: {}", path.display()); + return; + } + let entries = match std::fs::read_dir(path) { + Ok(entries) => entries, + Err(err) => { + eprintln!("Unable to read directory {}: {}", path.display(), err); + return; + } + }; + + for entry_result in entries { + let entry = match entry_result { + Ok(entry) => entry, + Err(err) => { + eprintln!("Unable to read directory entry: {}", err); + continue; + } + }; + let file_path: PathBuf = entry.path(); + if !file_path.is_file() { + continue; + } + + if args.get_flag(INPUT_BLOB) { + let blob = match std::fs::read(&file_path) { + Ok(blob) => blob, + Err(err) => { + eprintln!("Unable to read blob file {}: {}", file_path.display(), err); + continue; + } + }; + counter_inputs.inc(); + handle_blob( + blob.as_slice(), + &compiled_patterns, + &args, + &similarity_mode, + &tlsh_list, + &payload_reports, + &counter_pcre_matches, + &counter_tlsh_hashes, + &vec_payload_size, + &vec_payload_size_matched, + &counter_unique_payloads, + &counter_pcre_matches_total, + ); + continue; + } + + let file = match std::fs::File::open(&file_path) { + Ok(file) => file, + Err(err) => { + eprintln!("Unable to open file {}: {}", file_path.display(), err); + continue; + } + }; + let reader = std::io::BufReader::new(file); + for line_result in reader.lines() { + let line = match line_result { + Ok(line) => line, + Err(err) => { + eprintln!("Unable to read line from {}: {}", file_path.display(), err); + continue; + } + }; counter_inputs.inc(); handle_line( - line, - &patterns, + &line, + &compiled_patterns, &args, + &similarity_mode, &tlsh_list, &payload_reports, &counter_pcre_matches, @@ -236,13 +374,70 @@ fn main() { &counter_unique_payloads, &counter_pcre_matches_total, ); - }); + } + } + } else { + let stdin = io::stdin(); + if args.get_flag(INPUT_BLOB) { + let mut blob = Vec::new(); + let mut lock = stdin.lock(); + if let Err(err) = lock.read_to_end(&mut blob) { + eprintln!("Unable to read blob from STDIN: {}", err); + return; + } + counter_inputs.inc(); + handle_blob( + blob.as_slice(), + &compiled_patterns, + &args, + &similarity_mode, + &tlsh_list, + &payload_reports, + &counter_pcre_matches, + &counter_tlsh_hashes, + &vec_payload_size, + &vec_payload_size_matched, + &counter_unique_payloads, + &counter_pcre_matches_total, + ); + } else { + stdin + .lock() + .lines() + .filter_map(|line| match line { + Ok(value) => Some(value), + Err(err) => { + eprintln!("Unable to read line from STDIN: {}", err); + None + } + }) + .collect::>() + .par_iter() + .for_each(|line| { + counter_inputs.inc(); + handle_line( + line, + &compiled_patterns, + &args, + &similarity_mode, + &tlsh_list, + &payload_reports, + &counter_pcre_matches, + &counter_tlsh_hashes, + &vec_payload_size, + &vec_payload_size_matched, + &counter_unique_payloads, + &counter_pcre_matches_total, + ); + }); + } } if args.get_flag(TLSH_DIFF) { run_hash_diffs( &tlsh_list, &args, + &similarity_mode, &tlsh_reports, &counter_tlsh_similarites, &vec_tlsh_disance, @@ -250,6 +445,9 @@ fn main() { } generate_reports(&tlsh_reports, &payload_reports, &args); + if args.get_flag(PROTOCOL_HINTS) { + emit_protocol_hints(&payload_reports, &tlsh_reports, &args, &similarity_mode); + } if args.get_flag(STATS) { // TODO: Potentially optimize so that we don't waist CPU on creation of stats (counter, incrementers, etc.) unless this flag is passed. @@ -260,54 +458,102 @@ fn main() { let formated_duration: String = format!("{:.2}", duration_in_seconds); // Payloads Matched - let payload_sizes_matched = vec_payload_size_matched.lock().unwrap(); - let avg_payload_size_matched = - payload_sizes_matched.iter().sum::() as f64 / payload_sizes_matched.len() as f64; - let min_payload_size_matched = payload_sizes_matched.iter().min().unwrap_or(&default_empty); - let max_payload_size_matched = payload_sizes_matched.iter().max().unwrap_or(&default_empty); - let mut sorted_payload_sizes_matched = payload_sizes_matched.clone(); - sorted_payload_sizes_matched.sort(); - let payload_sizes_matched_len = payload_sizes_matched.len(); - let p95_payload_size_matched = if payload_sizes_matched_len > 1 { - sorted_payload_sizes_matched[(payload_sizes_matched_len * 95 / 100) - 1] - } else if payload_sizes_matched_len == 1 { - sorted_payload_sizes_matched[0] - } else { - default_empty + let ( + avg_payload_size_matched, + min_payload_size_matched, + max_payload_size_matched, + p95_payload_size_matched, + total_payload_size_matched, + ) = match vec_payload_size_matched.lock() { + Ok(payload_sizes_matched) => { + let payload_sizes_matched_len = payload_sizes_matched.len(); + let avg_payload_size_matched = if payload_sizes_matched_len == 0 { + 0.0 + } else { + payload_sizes_matched.iter().sum::() as f64 + / payload_sizes_matched_len as f64 + }; + let min_payload_size_matched = + *payload_sizes_matched.iter().min().unwrap_or(&default_empty); + let max_payload_size_matched = + *payload_sizes_matched.iter().max().unwrap_or(&default_empty); + let mut sorted_payload_sizes_matched = payload_sizes_matched.clone(); + sorted_payload_sizes_matched.sort(); + let p95_payload_size_matched = if payload_sizes_matched_len > 1 { + sorted_payload_sizes_matched[(payload_sizes_matched_len * 95 / 100) - 1] + } else if payload_sizes_matched_len == 1 { + sorted_payload_sizes_matched[0] + } else { + default_empty + }; + let total_payload_size_matched = payload_sizes_matched.iter().sum::(); + ( + avg_payload_size_matched, + min_payload_size_matched, + max_payload_size_matched, + p95_payload_size_matched, + total_payload_size_matched, + ) + } + Err(err) => { + eprintln!( + "Unable to read matched payload sizes due to poisoned lock: {}", + err + ); + (0.0, default_empty, default_empty, default_empty, 0) + } }; - let total_payload_size_matched = payload_sizes_matched.iter().sum::(); // Raw Payloads - let payload_sizes = vec_payload_size.lock().unwrap(); - let avg_payload_size = - payload_sizes.iter().sum::() as f64 / payload_sizes.len() as f64; - let min_payload_size = payload_sizes.iter().min().unwrap_or(&default_empty); - let max_payload_size = payload_sizes.iter().max().unwrap_or(&default_empty); - let mut sorted_payload_sizes = payload_sizes.clone(); - sorted_payload_sizes.sort(); - let payload_sizes_len = payload_sizes.len(); - let p95_payload_size = if payload_sizes_len > 1 { - sorted_payload_sizes[(payload_sizes_len * 95 / 100) - 1] - } else { - sorted_payload_sizes[0] + let ( + avg_payload_size, + min_payload_size, + max_payload_size, + p95_payload_size, + total_payload_size, + ) = match vec_payload_size.lock() { + Ok(payload_sizes) => { + let payload_sizes_len = payload_sizes.len(); + let avg_payload_size = if payload_sizes_len == 0 { + 0.0 + } else { + payload_sizes.iter().sum::() as f64 / payload_sizes_len as f64 + }; + let min_payload_size = *payload_sizes.iter().min().unwrap_or(&default_empty); + let max_payload_size = *payload_sizes.iter().max().unwrap_or(&default_empty); + let mut sorted_payload_sizes = payload_sizes.clone(); + sorted_payload_sizes.sort(); + let p95_payload_size = if payload_sizes_len > 1 { + sorted_payload_sizes[(payload_sizes_len * 95 / 100) - 1] + } else if payload_sizes_len == 1 { + sorted_payload_sizes[0] + } else { + default_empty + }; + let total_payload_size = payload_sizes.iter().sum::(); + ( + avg_payload_size, + min_payload_size, + max_payload_size, + p95_payload_size, + total_payload_size, + ) + } + Err(err) => { + eprintln!("Unable to read payload sizes due to poisoned lock: {}", err); + (0.0, default_empty, default_empty, default_empty, 0) + } }; - let total_payload_size = payload_sizes.iter().sum::(); - let processing_rate: String; if duration.as_secs() < 1 { - processing_rate = format!( - "{}/ms", - format_size(total_payload_size / duration.as_millis() as i64) - ); + let elapsed_millis = std::cmp::max(duration.as_millis() as i64, 1); + processing_rate = format!("{}/ms", format_size(total_payload_size / elapsed_millis)); } else { - processing_rate = format!( - "{}/s", - format_size(total_payload_size / duration.as_secs() as i64) - ); + let elapsed_seconds = std::cmp::max(duration.as_secs() as i64, 1); + processing_rate = format!("{}/s", format_size(total_payload_size / elapsed_seconds)); } let default_empty_32 = 0_i32; - let default_empty_str = std::string::String::new(); // TLSH Hashes let mut compare_json: Value = Value::Null; let mut matches_json_array = Vec::new(); @@ -321,38 +567,62 @@ fn main() { matches_json_array.push(json_object); } let matches_json = Value::Array(matches_json_array); - let tlsh_distances: std::sync::MutexGuard<'_, Vec> = vec_tlsh_disance.lock().unwrap(); - if tlsh_distances.len() > 2 { - let avg_tlsh_distance = - tlsh_distances.iter().sum::() as f32 / tlsh_distances.len() as f32; - let min_tlsh_distance = tlsh_distances.iter().min().unwrap_or(&default_empty_32); - let max_tlsh_distance = tlsh_distances.iter().max().unwrap_or(&default_empty_32); - let mut sorted_tlsh_distances = tlsh_distances.clone(); - sorted_tlsh_distances.sort(); - let tlsh_distances_len = tlsh_distances.len(); - let p95_tlsh_distance = if tlsh_distances_len > 1 { - sorted_tlsh_distances[(tlsh_distances_len * 95 / 100) - 1] - } else { - sorted_tlsh_distances[0] - }; - compare_json = json!({ - "Similarities": counter_tlsh_similarites.get(), - "AvgDistance": format!("{:.0}", avg_tlsh_distance), - "MinDistance": *min_tlsh_distance, - "MaxDistance": *max_tlsh_distance, - "P95Distance": p95_tlsh_distance, - }); + if let Ok(tlsh_distances) = vec_tlsh_disance.lock() { + if tlsh_distances.len() > 2 { + let avg_tlsh_distance = + tlsh_distances.iter().sum::() as f32 / tlsh_distances.len() as f32; + let min_tlsh_distance = tlsh_distances.iter().min().unwrap_or(&default_empty_32); + let max_tlsh_distance = tlsh_distances.iter().max().unwrap_or(&default_empty_32); + let mut sorted_tlsh_distances = tlsh_distances.clone(); + sorted_tlsh_distances.sort(); + let tlsh_distances_len = tlsh_distances.len(); + let p95_tlsh_distance = if tlsh_distances_len > 1 { + sorted_tlsh_distances[(tlsh_distances_len * 95 / 100) - 1] + } else { + sorted_tlsh_distances[0] + }; + compare_json = json!({ + "Similarities": counter_tlsh_similarites.get(), + "AvgDistance": format!("{:.0}", avg_tlsh_distance), + "MinDistance": *min_tlsh_distance, + "MaxDistance": *max_tlsh_distance, + "P95Distance": p95_tlsh_distance, + }); + } + } else { + eprintln!("Unable to read TLSH distances due to poisoned lock"); } + let unique_payload_count = match counter_unique_payloads.lock() { + Ok(unique_payloads) => unique_payloads.len(), + Err(err) => { + eprintln!( + "Unable to read unique payload count due to poisoned lock: {}", + err + ); + 0 + } + }; + let input_mode = args + .get_one::(INPUT_MODE) + .map_or(INPUT_MODE_BASE64, String::as_str); + let hash_function = args + .get_one::(TLSH_ALGORITHM) + .map_or("48_1", String::as_str); + let distance_threshold = args.get_one::(TLSH_DISTANCE).copied().unwrap_or(100); + let input_json_key = args + .get_one::(INPUT_JSON_KEY) + .map_or("", String::as_str); + // Create a JSON object for the stats let stats = json!({ "---PRECURSOR_STATISTICS---": "This JSON is output to STDERR so that you can parse stats seperate from the primary output.", "Input": { "Count": counter_inputs.get(), - "Unique": counter_unique_payloads.lock().unwrap().len(), + "Unique": unique_payload_count, "AvgSize": format!("{:.0}", avg_payload_size), - "MinSize": *min_payload_size, - "MaxSize": *max_payload_size, + "MinSize": min_payload_size, + "MaxSize": max_payload_size, "P95Size": p95_payload_size, "TotalSize": format_size(total_payload_size),}, "Match": { @@ -361,8 +631,8 @@ fn main() { "Matches": matches_json, "HashesGenerated": counter_tlsh_hashes.get(), "AvgSize": format!("{:.0}", avg_payload_size_matched), - "MinSize": *min_payload_size_matched, - "MaxSize": *max_payload_size_matched, + "MinSize": min_payload_size_matched, + "MaxSize": max_payload_size_matched, "P95Size": p95_payload_size_matched, "TotalSize": format_size(total_payload_size_matched),}, "Compare": compare_json, @@ -370,118 +640,409 @@ fn main() { "Version": env!("CARGO_PKG_VERSION"), "DurationSeconds": formated_duration, "ProcessingRate": processing_rate, - "InputMode": args.get_one::(INPUT_MODE).unwrap(), - "HashFunction": args.get_one::(TLSH_ALGORITHM).unwrap(), - "DistanceThreshold": args.get_one::(TLSH_DISTANCE).unwrap(), + "SimilarityMode": similarity_mode.as_str(), + "InputMode": input_mode, + "HashFunction": hash_function, + "DistanceThreshold": distance_threshold, "DiffEnabled": args.get_flag(TLSH_DIFF), "OnlyOutputSimilar": args.get_flag(TLSH_SIM_ONLY), "LengthEnabled": args.get_flag(TLSH_LENGTH), - "InputJSONKey": args.get_one::(INPUT_JSON_KEY).unwrap_or(&default_empty_str), + "InputJSONKey": input_json_key, + "SinglePacketInference": args.get_flag(SINGLE_PACKET), + "AbstainThreshold": args.get_one::(ABSTAIN_THRESHOLD).copied().unwrap_or(0.65), + "ProtocolTopK": args.get_one::(PROTOCOL_TOP_K).copied().unwrap_or(3), }, } ); // Serialize the JSON object as a pretty-printed String - let pretty_json = serde_json::to_string_pretty(&stats) - .expect("Error converting JSON object to pretty-printed String"); - - // Print the pretty-printed JSON to STDERR - let mut stderr = std::io::stderr(); - writeln!(&mut stderr, "{}", pretty_json).expect("Error printing JSON to STDERR"); - stderr.flush().expect("Error flushing STDERR buffer"); + match serde_json::to_string_pretty(&stats) { + Ok(pretty_json) => { + let mut stderr = std::io::stderr(); + if let Err(err) = writeln!(&mut stderr, "{}", pretty_json) { + eprintln!("Error printing JSON to STDERR: {}", err); + return; + } + if let Err(err) = stderr.flush() { + eprintln!("Error flushing STDERR buffer: {}", err); + } + } + Err(err) => { + eprintln!( + "Error converting JSON object to pretty-printed String: {}", + err + ); + } + } } } // Unpacks the reports from the shared mutex // and performs TLSH hash lookups for the matches from the tlsh in the payload report./ +fn emit_report(report: &Value) { + let report_json = match to_string(report) { + Ok(serialized) => serialized, + Err(err) => { + eprintln!("Unable to serialize report to JSON: {}", err); + return; + } + }; + let mut stdout = io::stdout(); + if let Err(err) = writeln!(&mut stdout, "{}", report_json) { + eprintln!("Error writing report to STDOUT: {}", err); + return; + } + if let Err(err) = stdout.flush() { + eprintln!("Error flushing STDOUT buffer: {}", err); + } +} + +fn apply_similarity_neighbor_boost( + report: &mut Value, + neighbor_count: usize, + abstain_threshold: f64, +) { + if neighbor_count == 0 { + return; + } + let boost = ((neighbor_count as f64).ln_1p() * 0.08).min(0.25); + if boost <= 0.0 { + return; + } + let Some(candidates) = report + .get_mut("protocol_candidates") + .and_then(Value::as_array_mut) + else { + return; + }; + + for candidate in candidates.iter_mut() { + let current = candidate + .get("score") + .and_then(Value::as_f64) + .unwrap_or(0.0); + let boosted = (current + boost).clamp(0.0, 0.99); + if let Some(score) = Number::from_f64(boosted) { + candidate["score"] = Value::Number(score); + } + if let Some(evidence) = candidate.get_mut("evidence").and_then(Value::as_array_mut) { + evidence.push(Value::String(format!( + "similarity cluster boost from {} neighbors", + neighbor_count + ))); + } + } + + candidates.sort_by(|left, right| { + let left_score = left.get("score").and_then(Value::as_f64).unwrap_or(0.0); + let right_score = right.get("score").and_then(Value::as_f64).unwrap_or(0.0); + right_score + .partial_cmp(&left_score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let top_summary = candidates.first().map(|top| { + let top_score = top.get("score").and_then(Value::as_f64).unwrap_or(0.0); + let top_protocol = top + .get("protocol") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(); + (top_score, top_protocol) + }); + + if let Some((top_score, top_protocol)) = top_summary { + let abstained = top_score < abstain_threshold.clamp(0.0, 1.0); + if let Some(score) = Number::from_f64(top_score) { + report["protocol_confidence"] = Value::Number(score); + } + report["protocol_abstained"] = Value::Bool(abstained); + if abstained { + report["protocol_label"] = Value::String("unknown".to_string()); + } else { + report["protocol_label"] = Value::String(top_protocol); + } + } +} + fn generate_reports( tlsh_reports: &DashMap, payload_reports: &Mutex>, args: &ArgMatches, ) { - for (xxh3_64_sum, report) in payload_reports - .lock() - .expect("unable to get payload_reports") - .iter() - { - if report["tlsh"] != "" && args.get_flag(TLSH_DIFF) { + let payload_reports_guard = match payload_reports.lock() { + Ok(guard) => guard, + Err(err) => { + eprintln!("Unable to acquire payload report lock: {}", err); + return; + } + }; + for (xxh3_64_sum, report) in payload_reports_guard.iter() { + let similarity_hash = report["similarity_hash"] + .as_str() + .or_else(|| report["tlsh"].as_str()) + .unwrap_or(""); + if !similarity_hash.is_empty() && args.get_flag(TLSH_DIFF) { let mut report_clone = report.clone(); - let tlsh_hash: Option<&str> = report["tlsh"].as_str(); report_clone["xxh3_64_sum"] = json!(xxh3_64_sum.as_str()); - if let Some(tlsh_hash) = tlsh_hash { - if let Some(tlsh_similarities) = tlsh_reports.get(tlsh_hash) { - report_clone["tlsh_similarities"] = tlsh_similarities.value().clone(); - // Print reports with TLSH hash and TLSH similarities. - println!( - "{}", - to_string(&report_clone).expect("unable to print report to string") + if let Some(tlsh_similarities) = tlsh_reports.get(similarity_hash) { + report_clone["tlsh_similarities"] = tlsh_similarities.value().clone(); + if args.get_flag(SINGLE_PACKET) { + let neighbor_count = tlsh_similarities + .value() + .as_object() + .map(|obj| obj.len()) + .unwrap_or(0); + let abstain_threshold = args + .get_one::(ABSTAIN_THRESHOLD) + .copied() + .unwrap_or(0.65); + apply_similarity_neighbor_boost( + &mut report_clone, + neighbor_count, + abstain_threshold, ); - io::stdout().flush().expect("Error flushing STDOUT buffer"); - } else if !args.get_flag(TLSH_SIM_ONLY) { - // Print reports with TLSH hash but no TLSH similarities. - println!( - "{}", - to_string(&report_clone).expect("unable to print report to string") - ); - io::stdout().flush().expect("Error flushing STDOUT buffer"); } + // Print reports with TLSH hash and TLSH similarities. + emit_report(&report_clone); + } else if !args.get_flag(TLSH_SIM_ONLY) { + // Print reports with TLSH hash but no TLSH similarities. + emit_report(&report_clone); } } else if !args.get_flag(TLSH_SIM_ONLY) { // Print reports empty TLSH hashes let mut report_clone = report.clone(); report_clone["xxh3_64_sum"] = json!(xxh3_64_sum.as_str()); - println!( - "{}", - to_string(&report_clone).expect("unable to print report to string") + emit_report(&report_clone); + } + } +} + +fn emit_protocol_hints( + payload_reports: &Mutex>, + tlsh_reports: &DashMap, + args: &ArgMatches, + similarity_mode: &SimilarityMode, +) { + let limit = args + .get_one::(PROTOCOL_HINTS_LIMIT) + .copied() + .unwrap_or(25); + let mut candidates: Vec<(usize, Value)> = Vec::new(); + let payload_reports_guard = match payload_reports.lock() { + Ok(guard) => guard, + Err(err) => { + eprintln!( + "Unable to acquire payload report lock for protocol hints: {}", + err ); + return; + } + }; + + for (xxh3_64_sum, report) in payload_reports_guard.iter() { + let similarity_hash = report["similarity_hash"] + .as_str() + .or_else(|| report["tlsh"].as_str()) + .unwrap_or(""); + if similarity_hash.is_empty() { + continue; + } + let neighbor_count = tlsh_reports + .get(similarity_hash) + .and_then(|map| map.value().as_object().map(|obj| obj.len())) + .unwrap_or(0); + let tags = report + .get("tags") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())); + let protocol_label = report.get("protocol_label").cloned().unwrap_or(Value::Null); + let protocol_confidence = report + .get("protocol_confidence") + .cloned() + .unwrap_or(Value::Null); + let protocol_abstained = report + .get("protocol_abstained") + .cloned() + .unwrap_or(Value::Null); + candidates.push(( + neighbor_count, + json!({ + "xxh3_64_sum": xxh3_64_sum.as_str(), + "similarity_hash": similarity_hash, + "neighbor_count": neighbor_count, + "tags": tags, + "protocol_label": protocol_label, + "protocol_confidence": protocol_confidence, + "protocol_abstained": protocol_abstained, + }), + )); + } + candidates.sort_by(|left, right| right.0.cmp(&left.0)); + let candidate_json: Vec = candidates + .into_iter() + .take(limit) + .map(|(_, value)| value) + .collect(); + let hints = json!({ + "---PRECURSOR_PROTOCOL_HINTS---": "Candidate payload clusters for LLM-guided protocol discovery.", + "SimilarityMode": similarity_mode.as_str(), + "DistanceThreshold": args.get_one::(TLSH_DISTANCE).copied().unwrap_or(100), + "Candidates": candidate_json + }); + + match serde_json::to_string_pretty(&hints) { + Ok(serialized) => { + let mut stderr = io::stderr(); + if let Err(err) = writeln!(&mut stderr, "{}", serialized) { + eprintln!("Unable to emit protocol hints to STDERR: {}", err); + } + } + Err(err) => { + eprintln!("Unable to serialize protocol hints: {}", err); } } } fn run_hash_diffs( - tlsh_list: &Mutex>, + tlsh_list: &Mutex>, args: &ArgMatches, + similarity_mode: &SimilarityMode, tlsh_reports: &DashMap, counter_tlsh_similarites: &Arc, vec_tlsh_disance: &std::sync::Mutex>, ) { - let tlsh_list_guard = tlsh_list.lock().unwrap(); + let tlsh_list_guard = match tlsh_list.lock() { + Ok(guard) => guard, + Err(err) => { + eprintln!("Unable to acquire TLSH list lock: {}", err); + return; + } + }; + let distance_threshold = match args.get_one::(TLSH_DISTANCE) { + Some(distance) => *distance, + None => { + eprintln!("Unable to read TLSH distance threshold argument"); + return; + } + }; + let include_file_length_in_calculation = args.get_flag(TLSH_LENGTH); + let similarity_mode_name = similarity_mode.as_str(); tlsh_list_guard .par_iter() .enumerate() .for_each(|(i, tlsh_i)| { let mut local_tlsh_map = Map::new(); - for (_j, tlsh_j) in tlsh_list_guard.iter().enumerate().skip(i + 1) { - let include_file_length_in_calculation = args.get_flag(TLSH_LENGTH); - let diff = tlsh_i.diff(tlsh_j, include_file_length_in_calculation); - vec_tlsh_disance.lock().unwrap().push(diff); - if diff - <= *args - .get_one(TLSH_DISTANCE) - .expect("unable to get TLSH distance argument") - { + for tlsh_j in tlsh_list_guard.iter().skip(i + 1) { + let diff = match diff_similarity_hash( + tlsh_i, + tlsh_j, + include_file_length_in_calculation, + ) { + Ok(distance) => distance, + Err(err) => { + eprintln!("Skipping {} diff: {}", similarity_mode_name, err); + continue; + } + }; + if let Ok(mut diff_vec) = vec_tlsh_disance.lock() { + diff_vec.push(diff); + } else { + eprintln!("Unable to record TLSH distance due to poisoned lock"); + } + if diff <= distance_threshold { counter_tlsh_similarites.inc(); - let tlsh_hash_lowercase = tlsh_j.hash().to_ascii_lowercase(); - let tlsh_hash_string = String::from_utf8(tlsh_hash_lowercase); + let tlsh_hash_string = match tlsh_j.as_string() { + Ok(hash_string) => hash_string, + Err(err) => { + eprintln!("Unable to convert similarity hash to string: {}", err); + continue; + } + }; let diff_number: Number = diff.into(); - local_tlsh_map.insert( - tlsh_hash_string.expect("unable to convert TLSH hash to string from UTF8"), - Value::Number(diff_number), - ); + local_tlsh_map.insert(tlsh_hash_string, Value::Number(diff_number)); } } - let tlsh_hash_lowercase = tlsh_i.hash().to_ascii_lowercase(); - let tlsh_hash_string = String::from_utf8(tlsh_hash_lowercase); - tlsh_reports.insert(tlsh_hash_string.unwrap(), Value::Object(local_tlsh_map)); + let tlsh_hash_string = match tlsh_i.as_string() { + Ok(hash_string) => hash_string, + Err(err) => { + eprintln!("Unable to convert similarity hash to string: {}", err); + return; + } + }; + tlsh_reports.insert(tlsh_hash_string, Value::Object(local_tlsh_map)); }); } -fn handle_line( - line: &String, - patterns: &[String], +fn decode_payload_from_json_expression( + raw_json: &str, + payload_key: &str, + input_mode: &str, +) -> Result<(Vec, Value), String> { + let line_json: Value = + from_str(raw_json).map_err(|err| format!("Unable to parse input as JSON: {}", err))?; + + let json_clone = if line_json.is_object() { + line_json.clone() + } else { + let mut wrapped = Map::new(); + wrapped.insert("input".to_string(), line_json.clone()); + Value::Object(wrapped) + }; + + let defs = Definitions::core(); + let mut errs = Vec::new(); + let Some(parsed_filter) = parse::parse(payload_key, parse::main()).0 else { + return Err(format!( + "Unable to parse JSON key expression: {:?}", + payload_key + )); + }; + let f = defs.finish(parsed_filter, Vec::new(), &mut errs); + if !errs.is_empty() { + return Err(format!( + "Unable to compile JSON key expression {:?}: {:?}", + payload_key, errs + )); + } + + let inputs = RcIter::new(core::iter::empty()); + let mut out = f.run(Ctx::new([], &inputs), Val::from(line_json)); + let payload = match out.next() { + Some(Ok(v)) => { + let v_str = v.to_string(); + get_payload(&v_str, input_mode).map_err(|err| { + format!( + "Unable to decode payload from JSON key {:?}: {}", + payload_key, err + ) + })? + } + Some(Err(e)) => { + return Err(format!( + "Unable to parse JSON pattern: {:?} with error: {:?}", + payload_key, e + )); + } + None => { + return Err(format!( + "No valid JSON was found for pattern: {:?}", + payload_key + )); + } + }; + + Ok((payload, json_clone)) +} + +fn process_decoded_payload( + payload: Vec, + mut json_clone: Value, + patterns: &[pcre2::bytes::Regex], args: &ArgMatches, - tlsh_list: &Mutex>, + similarity_mode: &SimilarityMode, + tlsh_list: &Mutex>, payload_reports: &Mutex>, counter_pcre_matches: &Arc>, counter_tlsh_hashes: &Arc, @@ -490,92 +1051,47 @@ fn handle_line( counter_unique_payloads: &Arc>>, counter_pcre_matches_total: &Arc, ) { - #[allow(unused_assignments)] - let mut payload: Vec = Vec::new(); - #[allow(unused_assignments)] - let mut json_clone: Value = Value::Null; - #[allow(unused_assignments)] - let mut line_json = Value::Null; - if let Some(payload_key) = args.get_one::(INPUT_JSON_KEY) { - if args.contains_id(INPUT_JSON_KEY) { - // WARNING: This logic should probably move up so we don't have to parse the input - // JSON twice from the line. - line_json = from_str(line).unwrap(); - } else { - //let json_tlsh_hash_clone = json_tlsh_hash.clone(); - line_json = Value::Object(Map::new()); - } - json_clone = line_json.clone(); - - // JQ Like parsing - let defs = Definitions::core(); - let mut errs = Vec::new(); - let f = parse::parse(payload_key, parse::main()).0.unwrap(); - let f = defs.finish(f, Vec::new(), &mut errs); - assert_eq!(errs, Vec::new()); - let inputs = RcIter::new(core::iter::empty()); - let mut out = f.run(Ctx::new([], &inputs), Val::from(line_json)); - match out.next() { - Some(Ok(v)) => { - let v_str = v.to_string(); - if args.contains_id(INPUT_MODE) { - let input_mode = args.get_one::(INPUT_MODE).unwrap(); - payload = get_payload(&v_str, input_mode) - // This is the only path because INPUT_MODE has a clap default value of base64. - } - } - Some(Err(e)) => { - eprintln!( - "Unable to parse JSON pattern: {:?} with error: {:?}", - payload_key, e - ); - } - None => { - eprintln!("No valid JSON was found for pattern: {:?}", payload_key); - } - } + if let Ok(mut payload_sizes) = vec_payload_size.lock() { + payload_sizes.push(payload.len() as i64); } else { - #[allow(unused_assignments)] // this is used below - if args.contains_id(INPUT_MODE) { - let input_mode = args.get_one::(INPUT_MODE).unwrap(); - payload = get_payload(line, input_mode) - // This is the only path because INPUT_MODE has a clap default value of base64. - } + eprintln!("Unable to record payload size due to poisoned lock"); + return; } - vec_payload_size.lock().unwrap().push(payload.len() as i64); let (xxh3_64_sum, xxh3_64_sum_string) = xxh3_64_hex(payload.clone()); - counter_unique_payloads.lock().unwrap().insert(xxh3_64_sum); - - let matched_capture_groups = Mutex::new(Value::Array(Vec::new())); + if let Ok(mut unique_payloads) = counter_unique_payloads.lock() { + unique_payloads.insert(xxh3_64_sum); + } else { + eprintln!("Unable to record unique payload due to poisoned lock"); + return; + } - let match_exists = Arc::new(Mutex::new(false)); + let mut matched_capture_groups: Vec = Vec::new(); + let mut matched_tag_names: Vec = Vec::new(); + let mut match_exists = false; - patterns.par_iter().for_each(|pattern: &String| { - let re = - build_regex(pattern).unwrap_or_else(|_| panic!("invalid PCRE2 found: {}", pattern)); + for re in patterns.iter() { let result = re .captures_iter(payload.as_slice()) .filter_map(|res| res.ok()) .any(|caps| { - vec_payload_size_matched - .lock() - .unwrap() - .push(payload.len() as i64); + if let Ok(mut payload_sizes_matched) = vec_payload_size_matched.lock() { + payload_sizes_matched.push(payload.len() as i64); + } else { + eprintln!("Unable to record matched payload size due to poisoned lock"); + } counter_pcre_matches_total.inc(); let mut found_match = false; for name in re.capture_names() { if let Some(name) = name { if caps.name(name).is_some() { // Here we increment a counter for each of the capture group names from the PCRE2 patterns. + let tag_name = name.to_string(); let mut count = - counter_pcre_matches.entry(name.to_string()).or_insert(0); + counter_pcre_matches.entry(tag_name.clone()).or_insert(0); *count += 1; - let mut matched_capture_groups = matched_capture_groups.lock().unwrap(); - matched_capture_groups - .as_array_mut() - .unwrap() - .push(Value::String(name.to_string())); + matched_capture_groups.push(Value::String(tag_name.clone())); + matched_tag_names.push(tag_name); found_match = true; } } @@ -583,29 +1099,45 @@ fn handle_line( found_match }); if result { - *match_exists.lock().unwrap() = true; + match_exists = true; } - }); + } let mut json_tlsh_hash: Value = Value::String(String::new()); - let tlsh_algorithm = args.get_one::(TLSH_ALGORITHM).unwrap(); - if *match_exists.lock().unwrap() { + let tlsh_algorithm = match args.get_one::(TLSH_ALGORITHM) { + Some(algorithm) => algorithm, + None => { + eprintln!("Unable to read TLSH algorithm argument"); + return; + } + }; + if match_exists { // We only calculate TLSH hashes and push to the global TLSH list // If the payload passes the pattern_match gate // This helps us acchieve a massive reduction in work for TLSH computation if args.get_flag(TLSH) || args.get_flag(TLSH_DIFF) || args.get_flag(TLSH_LENGTH) { - match calculate_tlsh_hash(payload.as_slice(), tlsh_algorithm) { + match calculate_similarity_hash(payload.as_slice(), similarity_mode, tlsh_algorithm) { Ok(hash) => { counter_tlsh_hashes.inc(); - let cloned_hash = hash.hash().clone(); - tlsh_list.lock().unwrap().push(hash); - let tlsh_hash_lowercase = cloned_hash.to_ascii_lowercase(); - let tlsh_hash_string = String::from_utf8(tlsh_hash_lowercase); - json_tlsh_hash = Value::String(tlsh_hash_string.unwrap()); + let hash_as_string = hash.as_string(); + if let Ok(mut tlsh_hashes) = tlsh_list.lock() { + tlsh_hashes.push(hash); + } else { + eprintln!("Unable to record TLSH hash due to poisoned lock"); + return; + } + if let Ok(tlsh_hash_string) = hash_as_string { + json_tlsh_hash = Value::String(tlsh_hash_string); + } else { + eprintln!("Unable to convert similarity hash to UTF-8 string"); + } } - Err(_err) => { - // Handle the error by printing an error message - //println!("Error calculating TLSH hash: {}", err); + Err(err) => { + eprintln!( + "Unable to calculate similarity hash using mode {}: {}", + similarity_mode.as_str(), + err + ); } }; } @@ -616,14 +1148,246 @@ fn handle_line( let json_tlsh_hash_clone = json_tlsh_hash.clone(); if json_tlsh_hash_clone.as_str().is_none() { json_clone["tlsh"] = Value::String(String::new()); + json_clone["similarity_hash"] = Value::String(String::new()); } else { json_clone["tlsh"] = json_tlsh_hash.clone(); + json_clone["similarity_hash"] = json_tlsh_hash.clone(); + } + json_clone["tags"] = Value::Array(matched_capture_groups); + if args.get_flag(SINGLE_PACKET) { + let abstain_threshold = args + .get_one::(ABSTAIN_THRESHOLD) + .copied() + .unwrap_or(0.65); + let protocol_top_k = args.get_one::(PROTOCOL_TOP_K).copied().unwrap_or(3); + let inference = infer_protocol_candidates( + payload.as_slice(), + &matched_tag_names, + 0, + protocol_top_k, + abstain_threshold, + ); + + json_clone["protocol_label"] = Value::String(inference.label); + json_clone["protocol_abstained"] = Value::Bool(inference.abstained); + json_clone["protocol_confidence"] = Number::from_f64(inference.confidence) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(0))); + let protocol_candidates = inference + .candidates + .into_iter() + .map(|candidate| { + let score_value = Number::from_f64(candidate.score) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(0))); + json!({ + "protocol": candidate.protocol, + "score": score_value, + "evidence": candidate.evidence + }) + }) + .collect::>(); + json_clone["protocol_candidates"] = Value::Array(protocol_candidates); } - json_clone["tags"] = matched_capture_groups.lock().unwrap().clone(); // This is where we insert the finished per-payload report - payload_reports - .lock() - .unwrap() - .insert(xxh3_64_sum_string, json_clone); + if let Ok(mut reports) = payload_reports.lock() { + reports.insert(xxh3_64_sum_string, json_clone); + } else { + eprintln!("Unable to record payload report due to poisoned lock"); + } + } +} + +fn handle_blob( + blob: &[u8], + patterns: &[pcre2::bytes::Regex], + args: &ArgMatches, + similarity_mode: &SimilarityMode, + tlsh_list: &Mutex>, + payload_reports: &Mutex>, + counter_pcre_matches: &Arc>, + counter_tlsh_hashes: &Arc, + vec_payload_size: &std::sync::Mutex>, + vec_payload_size_matched: &std::sync::Mutex>, + counter_unique_payloads: &Arc>>, + counter_pcre_matches_total: &Arc, +) { + let input_mode = args + .get_one::(INPUT_MODE) + .map_or(INPUT_MODE_BASE64, String::as_str); + + let (payload, json_clone) = if let Some(payload_key) = args.get_one::(INPUT_JSON_KEY) { + let blob_as_utf8 = match std::str::from_utf8(blob) { + Ok(text) => text, + Err(err) => { + eprintln!( + "Unable to decode input blob as UTF-8 for JSON extraction: {}", + err + ); + return; + } + }; + match decode_payload_from_json_expression(blob_as_utf8, payload_key, input_mode) { + Ok(decoded) => decoded, + Err(err) => { + eprintln!("{}", err); + return; + } + } + } else { + let payload = match input_mode { + INPUT_MODE_STRING => blob.to_vec(), + INPUT_MODE_BASE64 | INPUT_MODE_HEX => { + let blob_as_utf8 = match std::str::from_utf8(blob) { + Ok(text) => text, + Err(err) => { + eprintln!( + "Unable to decode blob using input mode {}: {}", + input_mode, err + ); + return; + } + }; + let normalized: String = blob_as_utf8 + .chars() + .filter(|ch| !ch.is_whitespace()) + .collect(); + match get_payload(&normalized, input_mode) { + Ok(decoded) => decoded, + Err(err) => { + eprintln!( + "Unable to decode blob using input mode {}: {}", + input_mode, err + ); + return; + } + } + } + _ => { + eprintln!("{} not a supported input mode.", input_mode); + return; + } + }; + (payload, Value::Object(Map::new())) + }; + + process_decoded_payload( + payload, + json_clone, + patterns, + args, + similarity_mode, + tlsh_list, + payload_reports, + counter_pcre_matches, + counter_tlsh_hashes, + vec_payload_size, + vec_payload_size_matched, + counter_unique_payloads, + counter_pcre_matches_total, + ); +} + +fn handle_line( + line: &str, + patterns: &[pcre2::bytes::Regex], + args: &ArgMatches, + similarity_mode: &SimilarityMode, + tlsh_list: &Mutex>, + payload_reports: &Mutex>, + counter_pcre_matches: &Arc>, + counter_tlsh_hashes: &Arc, + vec_payload_size: &std::sync::Mutex>, + vec_payload_size_matched: &std::sync::Mutex>, + counter_unique_payloads: &Arc>>, + counter_pcre_matches_total: &Arc, +) { + let input_mode = args + .get_one::(INPUT_MODE) + .map_or(INPUT_MODE_BASE64, String::as_str); + let (payload, json_clone) = if let Some(payload_key) = args.get_one::(INPUT_JSON_KEY) { + match decode_payload_from_json_expression(line, payload_key, input_mode) { + Ok(decoded) => decoded, + Err(err) => { + eprintln!("{}", err); + return; + } + } + } else { + let payload = match get_payload(line, input_mode) { + Ok(payload) => payload, + Err(err) => { + eprintln!( + "Unable to decode payload using input mode {}: {}", + input_mode, err + ); + return; + } + }; + (payload, Value::Object(Map::new())) + }; + + process_decoded_payload( + payload, + json_clone, + patterns, + args, + similarity_mode, + tlsh_list, + payload_reports, + counter_pcre_matches, + counter_tlsh_hashes, + vec_payload_size, + vec_payload_size_matched, + counter_unique_payloads, + counter_pcre_matches_total, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_apply_similarity_neighbor_boost_updates_top_candidate() { + let mut report = json!({ + "protocol_label": "unknown", + "protocol_confidence": 0.50, + "protocol_abstained": true, + "protocol_candidates": [ + { "protocol": "http", "score": 0.50, "evidence": [] }, + { "protocol": "tls", "score": 0.40, "evidence": [] } + ] + }); + + apply_similarity_neighbor_boost(&mut report, 10, 0.60); + + assert_eq!(report["protocol_label"], json!("http")); + assert_eq!(report["protocol_abstained"], json!(false)); + let confidence = report["protocol_confidence"].as_f64().unwrap_or(0.0); + assert!(confidence > 0.60); + let evidence_len = report["protocol_candidates"][0]["evidence"] + .as_array() + .map_or(0, |e| e.len()); + assert!(evidence_len > 0); + } + + #[test] + fn test_apply_similarity_neighbor_boost_is_noop_for_empty_neighbors() { + let mut report = json!({ + "protocol_label": "http", + "protocol_confidence": 0.80, + "protocol_abstained": false, + "protocol_candidates": [ + { "protocol": "http", "score": 0.80, "evidence": [] } + ] + }); + + apply_similarity_neighbor_boost(&mut report, 0, 0.60); + + assert_eq!(report["protocol_label"], json!("http")); + assert_eq!(report["protocol_abstained"], json!(false)); + let confidence = report["protocol_confidence"].as_f64().unwrap_or(0.0); + assert!((confidence - 0.80).abs() < f64::EPSILON); } } diff --git a/src/precursor/inference.rs b/src/precursor/inference.rs new file mode 100644 index 0000000..59ce374 --- /dev/null +++ b/src/precursor/inference.rs @@ -0,0 +1,331 @@ +use std::collections::HashMap; + +#[derive(Clone, Debug)] +pub struct ProtocolCandidate { + pub protocol: String, + pub score: f64, + pub evidence: Vec, +} + +#[derive(Clone, Debug)] +pub struct ProtocolInference { + pub label: String, + pub confidence: f64, + pub abstained: bool, + pub candidates: Vec, +} + +fn shannon_entropy(payload: &[u8]) -> f64 { + if payload.is_empty() { + return 0.0; + } + let mut counts = [0usize; 256]; + for byte in payload { + counts[*byte as usize] += 1; + } + let payload_len = payload.len() as f64; + let mut entropy = 0.0; + for count in counts { + if count == 0 { + continue; + } + let probability = count as f64 / payload_len; + entropy -= probability * probability.log2(); + } + entropy +} + +fn printable_ratio(payload: &[u8]) -> f64 { + if payload.is_empty() { + return 0.0; + } + let printable = payload + .iter() + .filter(|byte| matches!(**byte, b'\n' | b'\r' | b'\t' | 0x20..=0x7e)) + .count() as f64; + printable / payload.len() as f64 +} + +fn has_magic(payload: &[u8], magic: &[u8]) -> bool { + payload.starts_with(magic) +} + +fn add_score( + scores: &mut HashMap)>, + protocol: &str, + score: f64, + evidence: &str, +) { + let entry = scores + .entry(protocol.to_string()) + .or_insert((0.0, Vec::::new())); + entry.0 += score; + entry.1.push(evidence.to_string()); +} + +fn lowercase_payload(payload: &[u8]) -> String { + String::from_utf8_lossy(payload).to_ascii_lowercase() +} + +pub fn infer_protocol_candidates( + payload: &[u8], + tags: &[String], + neighbor_count: usize, + top_k: usize, + abstain_threshold: f64, +) -> ProtocolInference { + let mut scores: HashMap)> = HashMap::new(); + let lower_payload = lowercase_payload(payload); + let entropy = shannon_entropy(payload); + let printable = printable_ratio(payload); + let payload_len = payload.len(); + + if lower_payload.starts_with("get ") + || lower_payload.starts_with("post ") + || lower_payload.starts_with("head ") + || lower_payload.starts_with("put ") + || lower_payload.starts_with("delete ") + || lower_payload.contains(" http/1.") + || lower_payload.contains("host:") + { + add_score(&mut scores, "http", 0.85, "matched HTTP request/headers"); + } + + if payload_len >= 3 && payload[0] == 0x16 && payload[1] == 0x03 && payload[2] <= 0x04 { + add_score( + &mut scores, + "tls", + 0.9, + "matched TLS handshake prefix 16 03 xx", + ); + } + + if lower_payload.starts_with("ssh-") { + add_score( + &mut scores, + "ssh", + 0.95, + "matched SSH identification banner", + ); + } + + if lower_payload.starts_with("ehlo ") + || lower_payload.starts_with("helo ") + || lower_payload.starts_with("mail from:") + || lower_payload.starts_with("rcpt to:") + || lower_payload.starts_with("220 ") + || lower_payload.starts_with("250 ") + { + add_score( + &mut scores, + "smtp", + 0.78, + "matched SMTP command/response markers", + ); + } + + if lower_payload.starts_with("user ") + || lower_payload.starts_with("pass ") + || lower_payload.starts_with("+ok") + || lower_payload.starts_with("-err") + { + add_score( + &mut scores, + "pop3_or_ftp", + 0.66, + "matched POP3/FTP style tokens", + ); + } + + if lower_payload.starts_with("{") && lower_payload.contains(':') && printable > 0.95 { + add_score( + &mut scores, + "json_application", + 0.52, + "high-printable JSON-like payload shape", + ); + } + + if has_magic(payload, b"\x7fELF") { + add_score(&mut scores, "firmware_binary", 0.98, "ELF magic header"); + } + if has_magic(payload, b"MZ") { + add_score(&mut scores, "firmware_binary", 0.85, "PE/COFF MZ header"); + } + if has_magic(payload, b"\x1f\x8b") { + add_score(&mut scores, "compressed_binary", 0.88, "gzip magic header"); + } + if has_magic(payload, b"PK\x03\x04") { + add_score(&mut scores, "compressed_binary", 0.8, "zip magic header"); + } + if payload_len >= 4 && payload[0..4] == [0x27, 0x05, 0x19, 0x56] { + add_score( + &mut scores, + "firmware_binary", + 0.86, + "uImage magic header (0x27051956)", + ); + } + + if printable < 0.35 && entropy > 6.2 { + add_score( + &mut scores, + "opaque_binary_stream", + 0.6, + "low-printable/high-entropy binary characteristics", + ); + } + + if lower_payload.contains("/bin/sh") + || lower_payload.starts_with("wget ") + || lower_payload.starts_with("curl ") + || lower_payload.starts_with("busybox ") + || lower_payload.starts_with("chmod ") + || lower_payload.starts_with("powershell ") + { + add_score( + &mut scores, + "shell_command", + 0.72, + "matched command execution markers", + ); + } + + let dot_count = lower_payload.matches('.').count(); + if printable > 0.9 && dot_count >= 2 && !lower_payload.contains(' ') { + add_score( + &mut scores, + "dns_or_domain_payload", + 0.44, + "domain-like token shape", + ); + } + + for tag in tags { + let tag_lower = tag.to_ascii_lowercase(); + if tag_lower.contains("http") { + add_score(&mut scores, "http", 0.2, "tag evidence: http"); + } + if tag_lower.contains("tls") || tag_lower.contains("ssl") { + add_score(&mut scores, "tls", 0.2, "tag evidence: tls/ssl"); + } + if tag_lower.contains("dns") { + add_score( + &mut scores, + "dns_or_domain_payload", + 0.2, + "tag evidence: dns", + ); + } + if tag_lower.contains("ssh") { + add_score(&mut scores, "ssh", 0.2, "tag evidence: ssh"); + } + if tag_lower.contains("firmware") || tag_lower.contains("elf") { + add_score( + &mut scores, + "firmware_binary", + 0.2, + "tag evidence: firmware/elf", + ); + } + } + + let neighbor_boost = (neighbor_count as f64).ln_1p() * 0.08; + if neighbor_boost > 0.0 { + for (_protocol, (score, evidence)) in scores.iter_mut() { + *score += neighbor_boost.min(0.25); + evidence.push(format!( + "similarity cluster boost from {} neighbors", + neighbor_count + )); + } + } + + let mut candidates: Vec = scores + .into_iter() + .map(|(protocol, (score, evidence))| ProtocolCandidate { + protocol, + score: score.clamp(0.0, 0.99), + evidence, + }) + .collect(); + + candidates.sort_by(|left, right| { + right + .score + .partial_cmp(&left.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + if candidates.is_empty() { + return ProtocolInference { + label: "unknown".to_string(), + confidence: 0.0, + abstained: true, + candidates: vec![ProtocolCandidate { + protocol: "unknown".to_string(), + score: 0.0, + evidence: vec!["no protocol heuristics matched".to_string()], + }], + }; + } + + let top = candidates[0].clone(); + let abstained = top.score < abstain_threshold.clamp(0.0, 1.0); + let label = if abstained { + "unknown".to_string() + } else { + top.protocol.clone() + }; + + ProtocolInference { + label, + confidence: top.score, + abstained, + candidates: candidates.into_iter().take(top_k.max(1)).collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_http_candidate() { + let payload = b"GET /index.html HTTP/1.1\r\nHost: example.org\r\n\r\n"; + let inference = infer_protocol_candidates(payload, &[], 0, 3, 0.6); + assert_eq!(inference.label, "http"); + assert!(!inference.abstained); + } + + #[test] + fn test_tls_candidate() { + let payload = vec![0x16, 0x03, 0x03, 0x00, 0x2f, 0x01, 0x00, 0x00, 0x2b]; + let inference = infer_protocol_candidates(&payload, &[], 0, 3, 0.6); + assert_eq!(inference.label, "tls"); + assert!(!inference.abstained); + } + + #[test] + fn test_firmware_magic_candidate() { + let payload = b"\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00"; + let inference = infer_protocol_candidates(payload, &[], 0, 3, 0.6); + assert_eq!(inference.label, "firmware_binary"); + } + + #[test] + fn test_abstain_on_ambiguous_payload() { + let payload = b"abc"; + let inference = infer_protocol_candidates(payload, &[], 0, 3, 0.8); + assert_eq!(inference.label, "unknown"); + assert!(inference.abstained); + } + + #[test] + fn test_neighbor_boost_changes_confidence() { + let payload = b"GET / HTTP/1.1\r\nHost: x\r\n\r\n"; + let without_neighbors = infer_protocol_candidates(payload, &[], 0, 3, 0.95); + let with_neighbors = infer_protocol_candidates(payload, &[], 20, 3, 0.95); + assert!(with_neighbors.confidence > without_neighbors.confidence); + } +} diff --git a/src/precursor/lzjd.rs b/src/precursor/lzjd.rs new file mode 100644 index 0000000..48a0738 --- /dev/null +++ b/src/precursor/lzjd.rs @@ -0,0 +1,163 @@ +use sha2::{Digest, Sha256}; +use std::collections::HashSet; + +const LZJD_SKETCH_SIZE: usize = 128; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LzjdHash { + sketch: Vec, + payload_len: usize, +} + +impl LzjdHash { + pub fn as_string(&self) -> String { + let mut sketch_bytes = Vec::with_capacity((self.sketch.len() + 1) * 8); + sketch_bytes.extend_from_slice(&(self.payload_len as u64).to_be_bytes()); + for bucket in self.sketch.iter() { + sketch_bytes.extend_from_slice(&bucket.to_be_bytes()); + } + let digest = Sha256::digest(sketch_bytes.as_slice()); + let short_fingerprint = hex::encode(&digest[..16]); + format!("lzjd:{}:{}", self.sketch.len(), short_fingerprint) + } + + pub fn diff(&self, right: &Self, include_file_length: bool) -> i32 { + let jaccard_similarity = + jaccard_similarity(self.sketch.as_slice(), right.sketch.as_slice()); + let mut distance = ((1.0 - jaccard_similarity) * 100.0).round() as i32; + + if include_file_length { + let max_len = self.payload_len.max(right.payload_len) as f64; + if max_len > 0.0 { + let len_delta = self.payload_len.abs_diff(right.payload_len) as f64; + let len_penalty = ((len_delta / max_len) * 10.0).round() as i32; + distance = (distance + len_penalty).clamp(0, 100); + } + } + + distance + } +} + +pub fn calculate_lzjd_hash(payload: &[u8]) -> Result { + if payload.is_empty() { + return Err("LZJD hash requires a non-empty payload".to_string()); + } + + let phrases = lz78_phrases(payload); + if phrases.is_empty() { + return Err("LZJD hash produced an empty phrase set".to_string()); + } + + let mut sketch: Vec = phrases + .into_iter() + .map(|phrase| hash_phrase_to_bucket(phrase.as_slice())) + .collect(); + sketch.sort_unstable(); + sketch.dedup(); + sketch.truncate(LZJD_SKETCH_SIZE); + + if sketch.is_empty() { + return Err("LZJD hash produced an empty sketch".to_string()); + } + + Ok(LzjdHash { + sketch, + payload_len: payload.len(), + }) +} + +fn lz78_phrases(payload: &[u8]) -> HashSet> { + let mut dictionary: HashSet> = HashSet::new(); + let mut start = 0usize; + + while start < payload.len() { + let mut end = start + 1; + while end <= payload.len() && dictionary.contains(&payload[start..end]) { + end += 1; + } + + if end <= payload.len() { + dictionary.insert(payload[start..end].to_vec()); + start = end; + } else { + dictionary.insert(payload[start..payload.len()].to_vec()); + break; + } + } + + dictionary +} + +fn hash_phrase_to_bucket(phrase: &[u8]) -> u64 { + let digest = Sha256::digest(phrase); + let mut bucket = [0u8; 8]; + bucket.copy_from_slice(&digest[..8]); + u64::from_be_bytes(bucket) +} + +fn jaccard_similarity(left: &[u64], right: &[u64]) -> f64 { + if left.is_empty() && right.is_empty() { + return 1.0; + } + + let mut i = 0usize; + let mut j = 0usize; + let mut intersection = 0usize; + + while i < left.len() && j < right.len() { + match left[i].cmp(&right[j]) { + std::cmp::Ordering::Less => i += 1, + std::cmp::Ordering::Greater => j += 1, + std::cmp::Ordering::Equal => { + intersection += 1; + i += 1; + j += 1; + } + } + } + + let union = left.len() + right.len() - intersection; + if union == 0 { + return 1.0; + } + intersection as f64 / union as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_calculate_lzjd_hash_returns_stable_prefix() { + let hash = calculate_lzjd_hash(b"GET / HTTP/1.1\r\nHost: example.org\r\n") + .expect("expected lzjd hash"); + let rendered = hash.as_string(); + assert!(rendered.starts_with("lzjd:")); + } + + #[test] + fn test_diff_identical_payloads_is_zero() { + let left = calculate_lzjd_hash(b"AAAAABBBBBCCCCCDDDD").expect("expected left hash"); + let right = calculate_lzjd_hash(b"AAAAABBBBBCCCCCDDDD").expect("expected right hash"); + assert_eq!(left.diff(&right, false), 0); + } + + #[test] + fn test_diff_changes_with_different_payloads() { + let left = calculate_lzjd_hash(b"AAAAABBBBBCCCCCDDDD").expect("expected left hash"); + let right = calculate_lzjd_hash(b"\x7fELF\x02\x01\x01\x00\xAA\xBB\xCC\xDD") + .expect("expected right hash"); + assert!(left.diff(&right, false) > 0); + } + + #[test] + fn test_diff_with_length_penalty() { + let short = calculate_lzjd_hash(b"GET /short HTTP/1.1").expect("expected short hash"); + let long = calculate_lzjd_hash( + b"GET /a/very/long/path HTTP/1.1\r\nHost: example.org\r\nUser-Agent: precursor\r\n", + ) + .expect("expected long hash"); + assert!(short.diff(&long, true) >= short.diff(&long, false)); + } +} diff --git a/src/precursor/mod.rs b/src/precursor/mod.rs index 8c31dca..0fdfbc6 100644 --- a/src/precursor/mod.rs +++ b/src/precursor/mod.rs @@ -1,3 +1,7 @@ +pub mod inference; +pub mod lzjd; +pub mod mrshv2; +pub mod similarity; pub mod tlsh; pub mod util; diff --git a/src/precursor/mrshv2.rs b/src/precursor/mrshv2.rs new file mode 100644 index 0000000..31c482d --- /dev/null +++ b/src/precursor/mrshv2.rs @@ -0,0 +1,192 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Mrshv2Hash { + digest: String, + payload_len: usize, +} + +impl Mrshv2Hash { + pub fn as_string(&self) -> &str { + self.digest.as_str() + } + + #[cfg(feature = "similarity-mrshv2")] + pub fn payload_len(&self) -> usize { + self.payload_len + } +} + +#[cfg(feature = "similarity-mrshv2")] +mod native { + use super::Mrshv2Hash; + use std::ffi::{CStr, CString}; + use std::os::raw::{c_char, c_int, c_uchar}; + use std::sync::{Mutex, OnceLock}; + + extern "C" { + fn precursor_mrshv2_hash( + payload: *const c_uchar, + payload_len: usize, + out_digest: *mut *mut c_char, + ) -> c_int; + fn precursor_mrshv2_diff( + left_digest: *const c_char, + right_digest: *const c_char, + out_distance: *mut c_int, + ) -> c_int; + fn precursor_mrshv2_free(value: *mut c_char); + fn precursor_mrshv2_last_error() -> *const c_char; + } + + fn ffi_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + fn last_error_message(default_message: &str) -> String { + unsafe { + let ptr = precursor_mrshv2_last_error(); + if ptr.is_null() { + return default_message.to_string(); + } + let message = CStr::from_ptr(ptr).to_string_lossy().trim().to_string(); + if message.is_empty() { + default_message.to_string() + } else { + message + } + } + } + + pub fn calculate_mrshv2_hash(payload: &[u8]) -> Result { + if payload.is_empty() { + return Err("MRSHv2 hash requires a non-empty payload".to_string()); + } + + let _guard = ffi_lock() + .lock() + .map_err(|_| "MRSHv2 adapter lock is poisoned".to_string())?; + + let mut out_digest: *mut c_char = std::ptr::null_mut(); + let rc = unsafe { + precursor_mrshv2_hash( + payload.as_ptr() as *const c_uchar, + payload.len(), + &mut out_digest as *mut *mut c_char, + ) + }; + if rc != 0 { + if !out_digest.is_null() { + unsafe { + precursor_mrshv2_free(out_digest); + } + } + return Err(last_error_message( + "MRSHv2 adapter failed to compute hash; check linked native adapter", + )); + } + if out_digest.is_null() { + return Err("MRSHv2 adapter returned an empty digest pointer".to_string()); + } + + let digest_result = unsafe { CStr::from_ptr(out_digest) } + .to_str() + .map(|value| value.to_string()) + .map_err(|err| format!("MRSHv2 adapter returned non UTF-8 digest: {}", err)); + + unsafe { + precursor_mrshv2_free(out_digest); + } + let digest = digest_result?; + + Ok(Mrshv2Hash { + digest, + payload_len: payload.len(), + }) + } + + pub fn diff_mrshv2_hash( + left: &Mrshv2Hash, + right: &Mrshv2Hash, + include_file_length: bool, + ) -> Result { + let _guard = ffi_lock() + .lock() + .map_err(|_| "MRSHv2 adapter lock is poisoned".to_string())?; + + let left_digest = CString::new(left.digest.as_str()) + .map_err(|err| format!("MRSHv2 left digest contains embedded NUL: {}", err))?; + let right_digest = CString::new(right.digest.as_str()) + .map_err(|err| format!("MRSHv2 right digest contains embedded NUL: {}", err))?; + + let mut distance: c_int = 0; + let rc = unsafe { + precursor_mrshv2_diff( + left_digest.as_ptr(), + right_digest.as_ptr(), + &mut distance as *mut c_int, + ) + }; + if rc != 0 { + return Err(last_error_message( + "MRSHv2 adapter failed to diff digests; check linked native adapter", + )); + } + + let mut normalized = (distance as i32).clamp(0, 100); + if include_file_length { + let max_len = left.payload_len().max(right.payload_len()) as f64; + if max_len > 0.0 { + let len_delta = left.payload_len().abs_diff(right.payload_len()) as f64; + let len_penalty = ((len_delta / max_len) * 10.0).round() as i32; + normalized = (normalized + len_penalty).clamp(0, 100); + } + } + + Ok(normalized) + } +} + +#[cfg(not(feature = "similarity-mrshv2"))] +mod native { + use super::Mrshv2Hash; + + pub fn calculate_mrshv2_hash(_payload: &[u8]) -> Result { + Err( + "MRSHv2 support is disabled in this build. Recompile with `--features similarity-mrshv2` and provide a native adapter library." + .to_string(), + ) + } + + pub fn diff_mrshv2_hash( + _left: &Mrshv2Hash, + _right: &Mrshv2Hash, + _include_file_length: bool, + ) -> Result { + Err( + "MRSHv2 support is disabled in this build. Recompile with `--features similarity-mrshv2` and provide a native adapter library." + .to_string(), + ) + } +} + +pub use native::{calculate_mrshv2_hash, diff_mrshv2_hash}; + +#[cfg(all(test, feature = "similarity-mrshv2"))] +mod tests { + use super::*; + + #[test] + fn test_calculate_mrshv2_hash_prefix() { + let hash = calculate_mrshv2_hash(b"GET / HTTP/1.1\r\nHost: example.org\r\n") + .expect("expected mrshv2 hash"); + assert!(hash.as_string().starts_with("mrshv2:")); + } + + #[test] + fn test_diff_mrshv2_hash_identical_is_zero() { + let left = calculate_mrshv2_hash(b"AAAAABBBBB").expect("expected left hash"); + let right = calculate_mrshv2_hash(b"AAAAABBBBB").expect("expected right hash"); + let distance = diff_mrshv2_hash(&left, &right, false).expect("expected distance"); + assert_eq!(distance, 0); + } +} diff --git a/src/precursor/similarity.rs b/src/precursor/similarity.rs new file mode 100644 index 0000000..bd9b7f6 --- /dev/null +++ b/src/precursor/similarity.rs @@ -0,0 +1,150 @@ +use crate::precursor::lzjd::{calculate_lzjd_hash, LzjdHash}; +use crate::precursor::mrshv2::{calculate_mrshv2_hash, diff_mrshv2_hash, Mrshv2Hash}; +use crate::precursor::tlsh::{calculate_tlsh_hash, TlshHashInstance}; +use std::error::Error; +use std::fmt; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SimilarityMode { + Tlsh, + Lzjd, + Mrshv2, + FbHash, +} + +impl SimilarityMode { + pub fn from_str(value: &str) -> Result { + match value { + "tlsh" => Ok(Self::Tlsh), + "lzjd" => Ok(Self::Lzjd), + "mrshv2" => Ok(Self::Mrshv2), + "fbhash" => Ok(Self::FbHash), + _ => Err(SimilarityError::new(format!( + "Unsupported similarity mode '{}'", + value + ))), + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Tlsh => "tlsh", + Self::Lzjd => "lzjd", + Self::Mrshv2 => "mrshv2", + Self::FbHash => "fbhash", + } + } +} + +pub enum SimilarityHash { + Tlsh(TlshHashInstance), + Lzjd(LzjdHash), + Mrshv2(Mrshv2Hash), +} + +impl SimilarityHash { + pub fn as_string(&self) -> Result { + match self { + SimilarityHash::Tlsh(hash) => String::from_utf8(hash.hash().to_ascii_lowercase()) + .map_err(|err| SimilarityError::new(format!("Invalid TLSH hash UTF-8: {}", err))), + SimilarityHash::Lzjd(hash) => Ok(hash.as_string()), + SimilarityHash::Mrshv2(hash) => Ok(hash.as_string().to_string()), + } + } +} + +#[derive(Debug)] +pub struct SimilarityError { + message: String, +} + +impl SimilarityError { + pub fn new(message: String) -> Self { + Self { message } + } +} + +impl fmt::Display for SimilarityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl Error for SimilarityError {} + +pub fn calculate_similarity_hash( + payload: &[u8], + mode: &SimilarityMode, + tlsh_algorithm: &str, +) -> Result { + match mode { + SimilarityMode::Tlsh => calculate_tlsh_hash(payload, &tlsh_algorithm.to_string()) + .map(SimilarityHash::Tlsh) + .map_err(|err| SimilarityError::new(err.to_string())), + SimilarityMode::Lzjd => calculate_lzjd_hash(payload) + .map(SimilarityHash::Lzjd) + .map_err(SimilarityError::new), + SimilarityMode::Mrshv2 => calculate_mrshv2_hash(payload) + .map(SimilarityHash::Mrshv2) + .map_err(SimilarityError::new), + SimilarityMode::FbHash => Err(SimilarityError::new( + "FBHash similarity mode is scaffolded but not implemented yet".to_string(), + )), + } +} + +pub fn diff_similarity_hash( + left: &SimilarityHash, + right: &SimilarityHash, + include_file_length: bool, +) -> Result { + match (left, right) { + (SimilarityHash::Tlsh(left_hash), SimilarityHash::Tlsh(right_hash)) => left_hash + .diff(right_hash, include_file_length) + .ok_or_else(|| { + SimilarityError::new("Incompatible TLSH hash algorithm types".to_string()) + }), + (SimilarityHash::Lzjd(left_hash), SimilarityHash::Lzjd(right_hash)) => { + Ok(left_hash.diff(right_hash, include_file_length)) + } + (SimilarityHash::Mrshv2(left_hash), SimilarityHash::Mrshv2(right_hash)) => { + diff_mrshv2_hash(left_hash, right_hash, include_file_length) + .map_err(SimilarityError::new) + } + _ => Err(SimilarityError::new( + "Incompatible similarity hash algorithm types".to_string(), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_similarity_mode_lzjd_roundtrip() { + let mode = SimilarityMode::from_str("lzjd").expect("expected mode"); + assert_eq!(mode, SimilarityMode::Lzjd); + assert_eq!(mode.as_str(), "lzjd"); + } + + #[test] + fn test_calculate_and_diff_lzjd() { + let payload = b"GET /index HTTP/1.1\r\nHost: example.org\r\n"; + let left = calculate_similarity_hash(payload, &SimilarityMode::Lzjd, "48_1") + .expect("expected left hash"); + let right = calculate_similarity_hash(payload, &SimilarityMode::Lzjd, "48_1") + .expect("expected right hash"); + let rendered = left.as_string().expect("expected string form"); + assert!(rendered.starts_with("lzjd:")); + let distance = diff_similarity_hash(&left, &right, false).expect("expected distance"); + assert_eq!(distance, 0); + } + + #[test] + fn test_similarity_mode_mrshv2_roundtrip() { + let mode = SimilarityMode::from_str("mrshv2").expect("expected mode"); + assert_eq!(mode, SimilarityMode::Mrshv2); + assert_eq!(mode.as_str(), "mrshv2"); + } +} diff --git a/src/precursor/tlsh.rs b/src/precursor/tlsh.rs index c6ccaf5..9a0b513 100644 --- a/src/precursor/tlsh.rs +++ b/src/precursor/tlsh.rs @@ -18,24 +18,24 @@ pub enum TlshBuilderInstance { } impl TlshHashInstance { - pub fn diff(&self, other: &Self, include_file_length: bool) -> i32 { + pub fn diff(&self, other: &Self, include_file_length: bool) -> Option { match (self, other) { (TlshHashInstance::Tlsh48_1(hash1), TlshHashInstance::Tlsh48_1(hash2)) => { - hash1.diff(hash2, include_file_length) + Some(hash1.diff(hash2, include_file_length)) } (TlshHashInstance::Tlsh128_1(hash1), TlshHashInstance::Tlsh128_1(hash2)) => { - hash1.diff(hash2, include_file_length) + Some(hash1.diff(hash2, include_file_length)) } (TlshHashInstance::Tlsh128_3(hash1), TlshHashInstance::Tlsh128_3(hash2)) => { - hash1.diff(hash2, include_file_length) + Some(hash1.diff(hash2, include_file_length)) } (TlshHashInstance::Tlsh256_1(hash1), TlshHashInstance::Tlsh256_1(hash2)) => { - hash1.diff(hash2, include_file_length) + Some(hash1.diff(hash2, include_file_length)) } (TlshHashInstance::Tlsh256_3(hash1), TlshHashInstance::Tlsh256_3(hash2)) => { - hash1.diff(hash2, include_file_length) + Some(hash1.diff(hash2, include_file_length)) } - _ => panic!("Incompatible hash types"), + _ => None, } } diff --git a/src/precursor/util.rs b/src/precursor/util.rs index 59369f4..a10a1ab 100644 --- a/src/precursor/util.rs +++ b/src/precursor/util.rs @@ -14,13 +14,16 @@ pub fn remove_wrapped_quotes(input: &str) -> &str { .trim_end_matches(|c| c == '"' || c == '\'') } -pub fn get_payload(line: &str, input_mode: &str) -> Vec { +pub fn get_payload(line: &str, input_mode: &str) -> Result, String> { let line_with_no_wrapped_quotes = remove_wrapped_quotes(line); match input_mode { - "base64" => STANDARD.decode(line_with_no_wrapped_quotes).unwrap(), - "string" => line_with_no_wrapped_quotes.as_bytes().to_vec(), - "hex" => hex::decode(line_with_no_wrapped_quotes).unwrap(), - _ => panic!("{} not a supported input mode.", input_mode), + "base64" => STANDARD + .decode(line_with_no_wrapped_quotes) + .map_err(|err| format!("invalid base64 payload: {}", err)), + "string" => Ok(line_with_no_wrapped_quotes.as_bytes().to_vec()), + "hex" => hex::decode(line_with_no_wrapped_quotes) + .map_err(|err| format!("invalid hex payload: {}", err)), + _ => Err(format!("{} not a supported input mode.", input_mode)), } } @@ -43,18 +46,18 @@ pub fn format_size(size: i64) -> String { } } -pub fn read_patterns(pattern_file: Option<&PathBuf>) -> Vec { +pub fn read_patterns(pattern_file: Option<&PathBuf>) -> Result, std::io::Error> { let mut patterns = Vec::new(); if let Some(path) = pattern_file { - let file_contents = std::fs::read_to_string(path).unwrap(); + let file_contents = std::fs::read_to_string(path)?; for line in file_contents.lines() { patterns.push(line.to_owned()); } } - patterns + Ok(patterns) } -pub fn build_regex(pattern: &String) -> Result> { +pub fn build_regex(pattern: &str) -> Result> { let re = RegexBuilder::new() // NOTE: We should only enable JIT if we're going to compile all patterns into one large PCRE2 statement // TODO: Pass CLI flags for certain REGEX settings down to the builder. @@ -96,11 +99,14 @@ mod tests { // Test for `get_payload` function #[test] fn test_get_payload() { - assert_eq!(get_payload("aGVsbG8=", "base64"), b"hello".to_vec()); - assert_eq!(get_payload("hello", "string"), b"hello".to_vec()); - assert_eq!(get_payload("68656c6c6f", "hex"), b"hello".to_vec()); - - let result = std::panic::catch_unwind(|| get_payload("hello", "invalid_mode")); + assert_eq!( + get_payload("aGVsbG8=", "base64").unwrap(), + b"hello".to_vec() + ); + assert_eq!(get_payload("hello", "string").unwrap(), b"hello".to_vec()); + assert_eq!(get_payload("68656c6c6f", "hex").unwrap(), b"hello".to_vec()); + + let result = get_payload("hello", "invalid_mode"); assert!(result.is_err()); } @@ -129,7 +135,7 @@ mod tests { writeln!(temp_file, "pattern1\npattern2").expect("Failed to write to temp file"); // Test: Read patterns from the file - let patterns = read_patterns(Some(&temp_file_path.to_path_buf())); + let patterns = read_patterns(Some(&temp_file_path.to_path_buf())).unwrap(); assert_eq!(patterns, vec!["pattern1", "pattern2"]); // Clean up: Remove the temporary file @@ -139,7 +145,7 @@ mod tests { // Test for `build_regex` function #[test] fn test_build_regex() { - assert!(build_regex(&"\\d+".to_string()).is_ok()); - assert!(build_regex(&"[InvalidRegex".to_string()).is_err()); + assert!(build_regex("\\d+").is_ok()); + assert!(build_regex("[InvalidRegex").is_err()); } } diff --git a/tests/cli_contract.rs b/tests/cli_contract.rs new file mode 100644 index 0000000..d5fd980 --- /dev/null +++ b/tests/cli_contract.rs @@ -0,0 +1,195 @@ +use serde_json::Value; +use std::io::Write; +use std::process::{Command, Output, Stdio}; + +fn run_precursor(args: &[&str], stdin_payload: &str) -> Output { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_precursor")); + cmd.args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = cmd.spawn().expect("failed to spawn precursor"); + if let Some(stdin) = child.stdin.as_mut() { + stdin + .write_all(stdin_payload.as_bytes()) + .expect("failed to write stdin"); + } + let output = child.wait_with_output().expect("failed to wait on process"); + assert!( + output.status.success(), + "process failed with status {:?}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); + output +} + +fn parse_ndjson(stdout: &[u8]) -> Vec { + String::from_utf8_lossy(stdout) + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + serde_json::from_str::(line) + .unwrap_or_else(|err| panic!("invalid JSON line {:?}: {}", line, err)) + }) + .collect() +} + +#[test] +fn single_packet_emits_protocol_fields() { + let output = run_precursor( + &["(?GET)", "-m", "string", "-P"], + "GET /index.html HTTP/1.1 Host: example.org User-Agent: precursor-test\n", + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 1); + let report = &reports[0]; + assert!(report.get("protocol_label").is_some()); + assert!(report.get("protocol_confidence").is_some()); + assert!(report.get("protocol_abstained").is_some()); + assert!(report + .get("protocol_candidates") + .and_then(Value::as_array) + .map(|candidates| !candidates.is_empty()) + .unwrap_or(false)); +} + +#[test] +fn protocol_hints_include_inference_context() { + let line_one = + "GET /one HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-aaaaaaaaaa"; + let line_two = + "GET /two HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-bbbbbbbbbb"; + let stdin_payload = format!("{}\n{}\n", line_one, line_two); + + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-P", + "-t", + "-d", + "--protocol-hints", + "--protocol-hints-limit", + "5", + ], + &stdin_payload, + ); + + let stderr = String::from_utf8_lossy(&output.stderr); + let start = stderr + .find('{') + .unwrap_or_else(|| panic!("expected protocol hint JSON in stderr, got: {}", stderr)); + let end = stderr + .rfind('}') + .unwrap_or_else(|| panic!("expected protocol hint JSON in stderr, got: {}", stderr)); + let hints: Value = serde_json::from_str(&stderr[start..=end]) + .unwrap_or_else(|err| panic!("unable to parse hint JSON: {} in {}", err, stderr)); + + assert!(hints.get("---PRECURSOR_PROTOCOL_HINTS---").is_some()); + assert!(hints.get("Candidates").and_then(Value::as_array).is_some()); +} + +#[test] +fn input_blob_mode_supports_multiline_patterns() { + let pattern = "(?GET /blob HTTP/1\\.1\\nHost: blob\\.example)"; + let blob_payload = "GET /blob HTTP/1.1\nHost: blob.example\n"; + + let line_output = run_precursor(&[pattern, "-m", "string"], blob_payload); + let line_reports = parse_ndjson(&line_output.stdout); + assert_eq!( + line_reports.len(), + 0, + "line mode should not match cross-line pattern" + ); + + let blob_output = run_precursor(&[pattern, "-m", "string", "-z"], blob_payload); + let blob_reports = parse_ndjson(&blob_output.stdout); + assert_eq!(blob_reports.len(), 1); + assert!(blob_reports[0] + .get("tags") + .and_then(Value::as_array) + .map(|tags| tags.iter().any(|tag| tag.as_str() == Some("multi"))) + .unwrap_or(false)); +} + +#[test] +fn lzjd_similarity_mode_emits_backend_hashes() { + let line_one = + "GET /one HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-aaaaaaaaaa"; + let line_two = + "GET /two HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-bbbbbbbbbb"; + let stdin_payload = format!("{}\n{}\n", line_one, line_two); + + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "lzjd", + "-x", + "100", + ], + &stdin_payload, + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 2); + + assert!(reports.iter().all(|report| { + report + .get("similarity_hash") + .and_then(Value::as_str) + .map(|value| value.starts_with("lzjd:")) + .unwrap_or(false) + })); + + assert!(reports.iter().any(|report| { + report + .get("tlsh_similarities") + .and_then(Value::as_object) + .map(|obj| !obj.is_empty()) + .unwrap_or(false) + })); +} + +#[cfg(feature = "similarity-mrshv2")] +#[test] +fn mrshv2_similarity_mode_emits_backend_hashes() { + let line_one = + "GET /one HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-aaaaaaaaaa"; + let line_two = + "GET /two HTTP/1.1 Host: example.org User-Agent: precursor-long-test-agent-bbbbbbbbbb"; + let stdin_payload = format!("{}\n{}\n", line_one, line_two); + + let output = run_precursor( + &[ + "(?GET)", + "-m", + "string", + "-t", + "-d", + "--similarity-mode", + "mrshv2", + "-x", + "100", + ], + &stdin_payload, + ); + + let reports = parse_ndjson(&output.stdout); + assert_eq!(reports.len(), 2); + + assert!(reports.iter().all(|report| { + report + .get("similarity_hash") + .and_then(Value::as_str) + .map(|value| value.starts_with("mrshv2:")) + .unwrap_or(false) + })); +} diff --git a/tests/scenario_corpus_contract.rs b/tests/scenario_corpus_contract.rs new file mode 100644 index 0000000..89b0199 --- /dev/null +++ b/tests/scenario_corpus_contract.rs @@ -0,0 +1,158 @@ +use serde_json::Value; +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Output, Stdio}; + +fn run_precursor(args: &[&str], stdin_payload: &str) -> Output { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_precursor")); + cmd.args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = cmd.spawn().expect("failed to spawn precursor"); + if let Some(stdin) = child.stdin.as_mut() { + stdin + .write_all(stdin_payload.as_bytes()) + .expect("failed to write stdin"); + } + let output = child.wait_with_output().expect("failed to wait on process"); + assert!( + output.status.success(), + "process failed with status {:?}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); + output +} + +fn parse_ndjson(stdout: &[u8]) -> Vec { + String::from_utf8_lossy(stdout) + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + serde_json::from_str::(line) + .unwrap_or_else(|err| panic!("invalid JSON line {:?}: {}", line, err)) + }) + .collect() +} + +fn scenario_paths() -> (PathBuf, PathBuf, PathBuf) { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("samples/scenarios"); + let pre = root.join("pre-protocol-packet-triage"); + let firmware = root.join("firmware-fragment-triage"); + let modbus = root.join("ics-modbus-single-packet"); + (pre, firmware, modbus) +} + +#[test] +fn pre_protocol_packet_scenario_emits_clusterable_hashes() { + let (pre, _, _) = scenario_paths(); + let pattern_file = pre.join("patterns.pcre"); + let payloads = std::fs::read_to_string(pre.join("payloads.b64")).expect("read payloads"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-m", + "base64", + "-t", + "-d", + "-x", + "100", + "-P", + "--similarity-mode", + "lzjd", + ], + payloads.as_str(), + ); + + let reports = parse_ndjson(&output.stdout); + assert!( + reports.len() >= 4, + "expected at least 4 reports, got {}", + reports.len() + ); + assert!(reports.iter().all(|report| { + report + .get("similarity_hash") + .and_then(Value::as_str) + .map(|value| value.starts_with("lzjd:")) + .unwrap_or(false) + })); +} + +#[test] +fn firmware_fragment_scenario_hits_firmware_inference() { + let (_, firmware, _) = scenario_paths(); + let pattern_file = firmware.join("patterns.pcre"); + let payloads = std::fs::read_to_string(firmware.join("payloads.hex")).expect("read payloads"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-m", + "hex", + "-t", + "-P", + "--similarity-mode", + "lzjd", + ], + payloads.as_str(), + ); + + let reports = parse_ndjson(&output.stdout); + assert!( + reports.iter().any(|report| { + report + .get("protocol_label") + .and_then(Value::as_str) + .map(|value| value == "firmware_binary") + .unwrap_or(false) + }), + "expected at least one firmware_binary protocol label" + ); +} + +#[test] +fn modbus_scenario_emits_protocol_hints() { + let (_, _, modbus) = scenario_paths(); + let pattern_file = modbus.join("patterns.pcre"); + let payloads = std::fs::read_to_string(modbus.join("payloads.hex")).expect("read payloads"); + + let output = run_precursor( + &[ + "-p", + pattern_file.to_str().expect("pattern path utf8"), + "-m", + "hex", + "-t", + "-d", + "-x", + "100", + "-P", + "--protocol-hints", + "--protocol-hints-limit", + "5", + "--similarity-mode", + "lzjd", + ], + payloads.as_str(), + ); + + let stderr = String::from_utf8_lossy(&output.stderr); + let start = stderr + .find('{') + .unwrap_or_else(|| panic!("expected protocol hint JSON in stderr, got: {}", stderr)); + let end = stderr + .rfind('}') + .unwrap_or_else(|| panic!("expected protocol hint JSON in stderr, got: {}", stderr)); + let hints: Value = serde_json::from_str(&stderr[start..=end]) + .unwrap_or_else(|err| panic!("unable to parse hint JSON: {} in {}", err, stderr)); + assert!(hints + .get("Candidates") + .and_then(Value::as_array) + .map(|candidates| !candidates.is_empty()) + .unwrap_or(false)); +}