feat: add optional Zeek alert enrichment

.env.example

@@ -127,3 +127,58 @@ LOG_LEVEL=INFO
 # timezone: Wazuh names its daily ossec-alerts-DD archives by the manager's
 # local calendar, and a mismatch can stop ingest fail-closed at rotation.
 # TZ=UTC
+
+# === Optional: Zeek conn.log enrichment index ===
+# The `zeek` profile builds a private local context index and exposes no port.
+# Automatic Core enrichment remains default-off; set this true only when the
+# profile is running and the index is populated.
+# conn.log must use Zeek's JSON format (`@load policy/tuning/json-logs`).
+# ZEEK_ENRICHMENT_ENABLED=false
+# HOST_ZEEK_LOG_DIR=/opt/zeek/logs
+# ZEEK_CONN_PATH=/var/log/zeek/current/conn.log
+# Optional application logs used only for an operator-triggered deep lookup.
+# ZEEK_DNS_PATH=/var/log/zeek/current/dns.log
+# ZEEK_HTTP_PATH=/var/log/zeek/current/http.log
+# ZEEK_SSL_PATH=/var/log/zeek/current/ssl.log
+# ZEEK_X509_PATH=/var/log/zeek/current/x509.log
+# ZEEK_FILES_PATH=/var/log/zeek/current/files.log
+# ZEEK_NOTICE_PATH=/var/log/zeek/current/notice.log
+# ZEEK_ARCHIVE_ROOT=/var/log/zeek
+# ZEEK_INDEX_PATH=/var/lib/triagewall/zeek-context.db
+# ZEEK_SOURCE_ID=zeek-local
+# Automatic enrichment retries only `no_match` within this bounded catch-up
+# window. Other outcomes remain immediate Core fallbacks.
+# ZEEK_CATCHUP_TIMEOUT_SECONDS=3
+# ZEEK_CATCHUP_RETRY_INTERVAL_SECONDS=0.5
+# ZEEK_POLL_INTERVAL=2
+# ZEEK_MAX_RECORDS_PER_POLL=1000
+# ZEEK_EOF_STABLE_OBSERVATIONS=2
+# ZEEK_RETENTION_DAYS=7
+# ZEEK_PRUNE_INTERVAL=60
+# ZEEK_PRUNE_BATCH_SIZE=1000
+# ZEEK_PRUNE_MAX_ROWS=10000
+
+# === Optional standalone TriageWall Lab UI ===
+# Lab is a separate authenticated service and is never started by the default
+# Core Compose command. Generate a private access key and configuration values:
+#   python scripts/generate_lab_api_key.py
+# Then set the operator name plus the two generated values and start only the
+# explicit Lab profile with: docker compose --profile lab up -d lab
+# TRIAGEWALL_LAB_OPERATOR=local-operator
+# TRIAGEWALL_LAB_API_KEY_HASH='pbkdf2_sha256$210000$...'
+# TRIAGEWALL_LAB_SESSION_SECRET='...'
+# LAB_BIND_ADDRESS=127.0.0.1
+# LAB_PORT=8085
+# TRIAGEWALL_LAB_COOKIE_SECURE=false
+# TRIAGEWALL_LAB_SESSION_TTL_SECONDS=28800
+# TRIAGEWALL_LAB_TRUSTED_HOSTS=localhost
+# Private literal IP (or loopback only when Ollama shares the worker network
+# namespace). Hostnames and public addresses are rejected by the worker.
+# TRIAGEWALL_LAB_OLLAMA_URL=http://192.168.1.10:11434
+# TRIAGEWALL_LAB_REQUEST_TIMEOUT_SECONDS=300
+# TRIAGEWALL_LAB_POLL_INTERVAL_SECONDS=2
+# TRIAGEWALL_LAB_MAX_PENDING_JOBS=4
+# TRIAGEWALL_LAB_MAX_RESULTS_PER_JOB=1000
+# TRIAGEWALL_LAB_STORAGE_QUOTA_BYTES=10737418240
+# TRIAGEWALL_LAB_RETENTION_DAYS=30
+# TRIAGEWALL_LAB_RETENTION_BATCH_SIZE=10

.github/workflows/regression.yml

@@ -39,26 +39,7 @@ jobs:
           --requirement triagewall/dashboard/requirements.txt
 
       - name: Run unit and security regressions
-        run: >-
-          python -m unittest
-          tests.test_api_v1
-          tests.test_asset_inventory
-          tests.test_config_api
-          tests.test_config_preview_activation
-          tests.test_config_runtime
-          tests.test_database
-          tests.test_db_startup
-          tests.test_dashboard_stats
-          tests.test_gold_gate
-          tests.test_ingest_checkpoint
-          tests.test_operator_config
-          tests.test_prefilter
-          tests.test_retention
-          tests.test_retention_cycle
-          tests.test_runtime_dependencies
-          tests.test_security_regressions
-          tests.test_timestamps
-          tests.test_wazuh_ingest
+        run: python -m unittest discover -s tests
 
       - name: Verify gold-set evidence is current
         # Deterministic Layer 1 only: no Ollama, no GPU, no production data.
@@ -75,9 +56,22 @@ jobs:
         run: python -m compileall -q triagewall tests scripts
 
       - name: Validate Docker Compose
+        env:
+          TRIAGEWALL_LAB_OPERATOR: ci-validation-only
+          TRIAGEWALL_LAB_API_KEY_HASH: ci-validation-only
+          TRIAGEWALL_LAB_SESSION_SECRET: ci-validation-only
+          TRIAGEWALL_LAB_OLLAMA_URL: http://127.0.0.1:11434
         run: |
           docker compose config --quiet
           docker compose -f docker-compose.yml -f docker-compose.wazuh.yml --profile wazuh config --quiet
+          docker compose --profile lab config --quiet
 
       - name: Build Docker services
-        run: docker compose -f docker-compose.yml -f docker-compose.wazuh.yml --profile wazuh build
+        env:
+          TRIAGEWALL_LAB_OPERATOR: ci-validation-only
+          TRIAGEWALL_LAB_API_KEY_HASH: ci-validation-only
+          TRIAGEWALL_LAB_SESSION_SECRET: ci-validation-only
+          TRIAGEWALL_LAB_OLLAMA_URL: http://127.0.0.1:11434
+        run: >-
+          docker compose -f docker-compose.yml -f docker-compose.wazuh.yml
+          --profile wazuh --profile lab build

.gitignore

@@ -2,6 +2,7 @@
 .env
 .env.bak*
 .env.local
+triagewall-lab-credentials.txt
 
 # Backups
 *.bak

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to TriageWall are documented in this file.
 
 ## Unreleased
 
+### Added
+
+- TriageWall Lab can now build, preview, and idempotently install a complete
+  experiment from one validated `test-package.json` file while preserving the
+  individual-artifact workflow under Advanced details.
+
 ## [v0.4](https://github.com/aaronphifer/triagewall/releases/tag/v0.4) - 2026-08-19
 
 ### Added

Dockerfile.lab

@@ -0,0 +1,16 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY . /app/
+
+RUN pip install --no-cache-dir -r triagewall/lab/requirements.txt \
+    && mkdir -p /var/lib/triagewall-lab \
+    && chown 65532:65532 /var/lib/triagewall-lab
+
+USER 65532:65532
+
+EXPOSE 8085
+
+CMD ["uvicorn", "triagewall.lab.app:app", "--host", "127.0.0.1", "--port", "8085"]
+

ROADMAP.md

@@ -198,9 +198,10 @@ fail for different reasons and need different review.
   validation result, and related context without scanning unbounded history.
 - [ ] **Investigation controls.** Add source, time, IP, subnet, and asset
   filters, then saved views and structured JSON/CSV export.
-- [ ] **Portable event-bundle v1.** Define and validate a sanitized,
-  integrity-protected contract that can reproduce a decision without direct
-  access to the production database or sensor logs.
+- [x] **Portable event-bundle v1 Phase 0 contract.** Define and validate a
+  sanitized, integrity-protected contract that can reproduce a decision
+  without direct access to the production database or sensor logs. Core export
+  remains a separate milestone below.
 - [ ] **Bounded asynchronous LLM queue.** Decouple checkpointed intake from
   model latency only after overload, retry, ordering, and recovery semantics
   are explicit.
@@ -333,10 +334,32 @@ riding on the dashboard feedback boundary.
 Measure candidate behaviour in an isolated Lab, then promote an approved
 revision into Core through an explicit audited boundary.
 
+The implementation draft is documented in
+[TriageWall Lab design](docs/lab-design.md). Its first controlled experiment
+compares Zeek-absent, connection-only, and deeper-evidence conditions to measure
+whether a prompt candidate cites supported Zeek facts and improves decisions.
+
 - [ ] Complete the replay provenance needed to reproduce a decision.
-- [ ] Specify and validate sanitized event-bundle v1.
+- [x] Specify and validate sanitized event-bundle v1 Phase 0 contract.
+- [x] Specify and validate candidate, experiment, paired-result, and sanitized
+  promotion-report v1 contracts.
+- [x] Define the Lab threat model and hostile-upload matrix, including
+  executable Phase 0 validator cases and explicit runtime graduation blockers.
+- [x] Add the initial balanced 15-case sanitized Zeek calibration corpus with
+  condition-specific human verdicts, contribution labels, and fact allowlists.
+- [x] Add the first private prompt-only Lab CLI: trusted experiment builder,
+  paired local-Ollama runner, strict output validation, immutable private
+  per-pair results, and deterministic Zeek evidence-use scoring.
+- [x] Add the first standalone authenticated Lab UI foundation: separate
+  loopback-default container/profile, immutable bundle/candidate/experiment
+  registry, complete-result verification, paired result views, and promotion
+  gate views without Core mounts or production authority.
+- [x] Add the isolated single-worker execution slice: exact-digest queueing,
+  cooperative cancellation, lease-expiry recovery, bounded quota/retention,
+  immutable complete runs, and deterministic aggregate promotion reports.
 - [ ] Build the isolated Lab runtime for replay, comparison, injection tests,
-  and gold-set evaluation without live Core access.
+  richer operational telemetry, audit history, and gold-set evaluation without
+  live Core access.
 - [ ] Compare baseline and candidate outcomes without collapsing distinct
   safety and performance signals into one score.
 - [ ] Require an authenticated operator action to promote a passing candidate;

docker-compose.yml

@@ -67,6 +67,11 @@ services:
       POSITION_PATH: ${POSITION_PATH:-/var/lib/triagewall/position.json}
       POLL_INTERVAL: ${POLL_INTERVAL:-10}
       TRIAGEWALL_CONFIG_RELOAD_INTERVAL_SECONDS: ${TRIAGEWALL_CONFIG_RELOAD_INTERVAL_SECONDS:-5}
+      ZEEK_ENRICHMENT_ENABLED: ${ZEEK_ENRICHMENT_ENABLED:-false}
+      ZEEK_INDEX_PATH: ${ZEEK_INDEX_PATH:-/var/lib/triagewall/zeek-context.db}
+      ZEEK_SOURCE_ID: ${ZEEK_SOURCE_ID:-zeek-local}
+      ZEEK_CATCHUP_TIMEOUT_SECONDS: ${ZEEK_CATCHUP_TIMEOUT_SECONDS:-3}
+      ZEEK_CATCHUP_RETRY_INTERVAL_SECONDS: ${ZEEK_CATCHUP_RETRY_INTERVAL_SECONDS:-0.5}
 
       OLLAMA_HOST: ${OLLAMA_HOST:-http://localhost:11434}
       OLLAMA_MODEL: ${OLLAMA_MODEL:-hf.co/gabriellarson/Foundation-Sec-8B-Instruct-GGUF:Q5_K_M}
@@ -84,6 +89,46 @@ services:
         condition: service_completed_successfully
     restart: unless-stopped
 
+  zeek-ingest:
+    profiles: ["zeek"]
+    build:
+      context: .
+      dockerfile: Dockerfile.ingest
+    command:
+      - python3
+      - triagewall/zeek_ingest.py
+    environment:
+      TZ: ${TZ:-UTC}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      PYTHONDONTWRITEBYTECODE: "1"
+      ZEEK_CONN_PATH: ${ZEEK_CONN_PATH:-/var/log/zeek/current/conn.log}
+      ZEEK_DNS_PATH: ${ZEEK_DNS_PATH:-/var/log/zeek/current/dns.log}
+      ZEEK_HTTP_PATH: ${ZEEK_HTTP_PATH:-/var/log/zeek/current/http.log}
+      ZEEK_SSL_PATH: ${ZEEK_SSL_PATH:-/var/log/zeek/current/ssl.log}
+      ZEEK_X509_PATH: ${ZEEK_X509_PATH:-/var/log/zeek/current/x509.log}
+      ZEEK_FILES_PATH: ${ZEEK_FILES_PATH:-/var/log/zeek/current/files.log}
+      ZEEK_NOTICE_PATH: ${ZEEK_NOTICE_PATH:-/var/log/zeek/current/notice.log}
+      ZEEK_ARCHIVE_ROOT: ${ZEEK_ARCHIVE_ROOT:-/var/log/zeek}
+      ZEEK_INDEX_PATH: ${ZEEK_INDEX_PATH:-/var/lib/triagewall/zeek-context.db}
+      ZEEK_SOURCE_ID: ${ZEEK_SOURCE_ID:-zeek-local}
+      ZEEK_POLL_INTERVAL: ${ZEEK_POLL_INTERVAL:-2}
+      ZEEK_MAX_RECORDS_PER_POLL: ${ZEEK_MAX_RECORDS_PER_POLL:-1000}
+      ZEEK_EOF_STABLE_OBSERVATIONS: ${ZEEK_EOF_STABLE_OBSERVATIONS:-2}
+      ZEEK_RETENTION_DAYS: ${ZEEK_RETENTION_DAYS:-7}
+      ZEEK_PRUNE_INTERVAL: ${ZEEK_PRUNE_INTERVAL:-60}
+      ZEEK_PRUNE_BATCH_SIZE: ${ZEEK_PRUNE_BATCH_SIZE:-1000}
+      ZEEK_PRUNE_MAX_ROWS: ${ZEEK_PRUNE_MAX_ROWS:-10000}
+    volumes:
+      - ${HOST_DATA_DIR:-./data}:/var/lib/triagewall
+      - ${HOST_ZEEK_LOG_DIR:-/opt/zeek/logs}:/var/log/zeek:ro
+    read_only: true
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    network_mode: none
+    restart: unless-stopped
+
   dashboard:
     build:
       context: .
@@ -109,6 +154,9 @@ services:
       TRIAGEWALL_API_REDACT_IPS: ${TRIAGEWALL_API_REDACT_IPS:-false}
       TRIAGEWALL_API_IP_HASH_SECRET: ${TRIAGEWALL_API_IP_HASH_SECRET:-}
       TRIAGEWALL_CONFIG_WRITES_ENABLED: ${TRIAGEWALL_CONFIG_WRITES_ENABLED:-false}
+      ZEEK_ENRICHMENT_ENABLED: ${ZEEK_ENRICHMENT_ENABLED:-false}
+      ZEEK_INDEX_PATH: ${ZEEK_INDEX_PATH:-/var/lib/triagewall/zeek-context.db}
+      ZEEK_SOURCE_ID: ${ZEEK_SOURCE_ID:-zeek-local}
       DASHBOARD_HOST: ${DASHBOARD_HOST:-0.0.0.0}
       DASHBOARD_PORT: ${DASHBOARD_PORT:-8084}
       TRUSTED_HOSTS: ${TRUSTED_HOSTS:-localhost}
@@ -145,3 +193,118 @@ services:
       - ALL
     security_opt:
       - no-new-privileges:true
+
+  lab-init:
+    profiles: ["lab"]
+    build:
+      context: .
+      dockerfile: Dockerfile.lab
+    user: "0:0"
+    command:
+      - chown
+      - "65532:65532"
+      - /var/lib/triagewall-lab
+    volumes:
+      - triagewall-lab-data:/var/lib/triagewall-lab
+    read_only: true
+    network_mode: none
+    cap_drop:
+      - ALL
+    pids_limit: 128
+    cap_add:
+      - CHOWN
+    security_opt:
+      - no-new-privileges:true
+    restart: "no"
+
+  lab:
+    profiles: ["lab"]
+    build:
+      context: .
+      dockerfile: Dockerfile.lab
+    command:
+      - uvicorn
+      - triagewall.lab.app:app
+      - --host
+      - "0.0.0.0"
+      - --port
+      - "8085"
+    environment:
+      TZ: ${TZ:-UTC}
+      PYTHONDONTWRITEBYTECODE: "1"
+      TRIAGEWALL_LAB_DATA_DIR: /var/lib/triagewall-lab
+      TRIAGEWALL_LAB_OPERATOR: ${TRIAGEWALL_LAB_OPERATOR:?set TRIAGEWALL_LAB_OPERATOR}
+      TRIAGEWALL_LAB_API_KEY_HASH: ${TRIAGEWALL_LAB_API_KEY_HASH:?set TRIAGEWALL_LAB_API_KEY_HASH}
+      TRIAGEWALL_LAB_SESSION_SECRET: ${TRIAGEWALL_LAB_SESSION_SECRET:?set TRIAGEWALL_LAB_SESSION_SECRET}
+      TRIAGEWALL_LAB_COOKIE_SECURE: ${TRIAGEWALL_LAB_COOKIE_SECURE:-false}
+      TRIAGEWALL_LAB_SESSION_TTL_SECONDS: ${TRIAGEWALL_LAB_SESSION_TTL_SECONDS:-28800}
+      TRIAGEWALL_LAB_TRUSTED_HOSTS: ${TRIAGEWALL_LAB_TRUSTED_HOSTS:-localhost}
+      TRIAGEWALL_LAB_STORAGE_QUOTA_BYTES: ${TRIAGEWALL_LAB_STORAGE_QUOTA_BYTES:-10737418240}
+      TRIAGEWALL_LAB_MAX_PENDING_JOBS: ${TRIAGEWALL_LAB_MAX_PENDING_JOBS:-4}
+      TRIAGEWALL_LAB_MAX_RESULTS_PER_JOB: ${TRIAGEWALL_LAB_MAX_RESULTS_PER_JOB:-1000}
+    volumes:
+      - triagewall-lab-data:/var/lib/triagewall-lab
+    ports:
+      - "${LAB_BIND_ADDRESS:-127.0.0.1}:${LAB_PORT:-8085}:8085"
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,nodev,size=32m,mode=1777
+    cap_drop:
+      - ALL
+    pids_limit: 128
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - triagewall-lab-ui
+    depends_on:
+      lab-init:
+        condition: service_completed_successfully
+      lab-worker:
+        condition: service_started
+    restart: unless-stopped
+
+  lab-worker:
+    profiles: ["lab"]
+    build:
+      context: .
+      dockerfile: Dockerfile.lab
+    command:
+      - python3
+      - -m
+      - triagewall.lab.worker
+    environment:
+      TZ: ${TZ:-UTC}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      PYTHONDONTWRITEBYTECODE: "1"
+      TRIAGEWALL_LAB_DATA_DIR: /var/lib/triagewall-lab
+      TRIAGEWALL_LAB_OLLAMA_URL: ${TRIAGEWALL_LAB_OLLAMA_URL:?set TRIAGEWALL_LAB_OLLAMA_URL to a private literal IP}
+      TRIAGEWALL_LAB_REQUEST_TIMEOUT_SECONDS: ${TRIAGEWALL_LAB_REQUEST_TIMEOUT_SECONDS:-300}
+      TRIAGEWALL_LAB_POLL_INTERVAL_SECONDS: ${TRIAGEWALL_LAB_POLL_INTERVAL_SECONDS:-2}
+      TRIAGEWALL_LAB_MAX_RESULTS_PER_JOB: ${TRIAGEWALL_LAB_MAX_RESULTS_PER_JOB:-1000}
+      TRIAGEWALL_LAB_STORAGE_QUOTA_BYTES: ${TRIAGEWALL_LAB_STORAGE_QUOTA_BYTES:-10737418240}
+      TRIAGEWALL_LAB_RETENTION_DAYS: ${TRIAGEWALL_LAB_RETENTION_DAYS:-30}
+      TRIAGEWALL_LAB_RETENTION_BATCH_SIZE: ${TRIAGEWALL_LAB_RETENTION_BATCH_SIZE:-10}
+    volumes:
+      - triagewall-lab-data:/var/lib/triagewall-lab
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,nodev,size=32m,mode=1777
+    cap_drop:
+      - ALL
+    pids_limit: 128
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - triagewall-lab-model
+    depends_on:
+      lab-init:
+        condition: service_completed_successfully
+    restart: unless-stopped
+
+volumes:
+  triagewall-lab-data:
+
+networks:
+  triagewall-lab-ui:
+    internal: true
+  triagewall-lab-model: