From 6d2fc61bc416b109282b83ea6b4ed3857487b674 Mon Sep 17 00:00:00 2001 From: Daniel Meyer Date: Sat, 5 Sep 2026 18:26:59 +0000 Subject: [PATCH 1/3] sync: bring personal Hyperlex product tip onto org main Graft product tree from scrimshawlife-ctrl/Hyperlex@b2cbfad6 (mutation detect v0.2, installable package, graft install, installer audit) onto org main after #8, while preserving Zero State proprietary LICENSE, LICENSE_POLICY, AGENTS.md, CLAUDE.md, intent/, and org .github SDLC. VERSION 0.4.2; license metadata kept proprietary (no MIT). --- .github/workflows/docs.yml | 68 ++++ .github/workflows/hermes-evals.yml | 41 +++ .specify/memory/constitution.md | 51 +++ CHANGELOG.md | 7 - README.md | 17 +- SKILL.md | 289 +++++++++++---- STATUS.md | 3 +- VERSION | 2 +- docs/README.md | 2 - docs/RELEASE_NOTES.md | 4 - docs/commands.md | 22 +- docs/index.md | 4 +- docs/install-package.md | 25 ++ docs/start/glossary.md | 13 +- docs/status.md | 3 +- docs/telemetry.md | 2 +- hyperlex.manifest.yaml | 73 ++-- install.sh | 106 ++---- mkdocs.yml | 2 +- pyproject.toml | 24 +- references/hermes-runtime-contract.md | 4 - references/source-and-upgrades.md | 89 ++--- schemas/mutation_trace.v0.1.schema.json | 94 +++++ scripts/hlx-mutation | 30 ++ scripts/hyperlex.py | 193 ---------- scripts/install_transaction.py | 340 ++++++++++-------- specs/000-hyperlex-spine/spec.md | 63 ++++ specs/001-mutation-grammar/checklist.md | 29 ++ specs/001-mutation-grammar/clarify.md | 26 ++ specs/001-mutation-grammar/dual-use-gate.md | 15 + specs/001-mutation-grammar/owasp-mapping.md | 27 ++ specs/001-mutation-grammar/plan.md | 36 ++ specs/001-mutation-grammar/spec.md | 171 +++++++++ specs/001-mutation-grammar/tasks.md | 15 + specs/001-mutation-grammar/threat-model.md | 36 ++ specs/001-mutation-grammar/ux-commands.md | 8 + specs/002-command-surface/spec.md | 55 +++ specs/002-command-surface/tasks.md | 11 + .../003-mutation-detect-v02/dual-use-gate.md | 18 + specs/003-mutation-detect-v02/plan.md | 49 +++ specs/003-mutation-detect-v02/spec.md | 168 +++++++++ specs/003-mutation-detect-v02/tasks.md | 13 + specs/README.md | 18 + specs/runtime-ready.md | 48 +++ src/hyperlex/__init__.py | 10 +- src/hyperlex/__main__.py | 4 +- src/hyperlex/_version.py | 18 + .../analysis/mutation_trace_attach.py | 22 ++ src/hyperlex/analysis/signal_report.py | 18 +- src/hyperlex/cli.py | 132 +++++++ src/hyperlex/command_router.py | 32 ++ src/hyperlex/data/SKILL.agent.md | 49 +++ src/hyperlex/entrypoint.py | 20 ++ src/hyperlex/init_skill.py | 108 ++++++ src/hyperlex/mutation/__init__.py | 24 ++ src/hyperlex/mutation/__main__.py | 32 ++ src/hyperlex/mutation/detect.py | 90 +++++ src/hyperlex/mutation/grammar.py | 198 ++++++++++ src/hyperlex/mutation/human.py | 45 +++ src/hyperlex/mutation/operators.py | 175 +++++++++ src/hyperlex/mutation/packet.py | 72 ++++ src/hyperlex/mutation/watch.py | 134 +++++++ .../schemas/mutation_trace.v0.1.schema.json | 82 +++++ tests/test_command_router.py | 10 + tests/test_hlx_mutation_alias.py | 28 ++ tests/test_hyperlex.py | 13 - tests/test_init_skill.py | 23 ++ tests/test_mutation_grammar.py | 121 +++++++ tests/test_mutation_v02.py | 224 ++++++++++++ tests/test_packaging_surface.py | 20 ++ tests_audit/PLAN.md | 11 +- tests_audit/test_hyperlex_install.py | 25 ++ tests_audit/test_install.py | 18 +- tests_audit/test_install_followup.py | 42 +-- tests_audit/test_transaction.py | 4 + 75 files changed, 3388 insertions(+), 730 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/hermes-evals.yml create mode 100644 .specify/memory/constitution.md create mode 100644 docs/install-package.md create mode 100644 schemas/mutation_trace.v0.1.schema.json create mode 100644 scripts/hlx-mutation mode change 100644 => 100755 scripts/install_transaction.py create mode 100644 specs/000-hyperlex-spine/spec.md create mode 100644 specs/001-mutation-grammar/checklist.md create mode 100644 specs/001-mutation-grammar/clarify.md create mode 100644 specs/001-mutation-grammar/dual-use-gate.md create mode 100644 specs/001-mutation-grammar/owasp-mapping.md create mode 100644 specs/001-mutation-grammar/plan.md create mode 100644 specs/001-mutation-grammar/spec.md create mode 100644 specs/001-mutation-grammar/tasks.md create mode 100644 specs/001-mutation-grammar/threat-model.md create mode 100644 specs/001-mutation-grammar/ux-commands.md create mode 100644 specs/002-command-surface/spec.md create mode 100644 specs/002-command-surface/tasks.md create mode 100644 specs/003-mutation-detect-v02/dual-use-gate.md create mode 100644 specs/003-mutation-detect-v02/plan.md create mode 100644 specs/003-mutation-detect-v02/spec.md create mode 100644 specs/003-mutation-detect-v02/tasks.md create mode 100644 specs/README.md create mode 100644 specs/runtime-ready.md create mode 100644 src/hyperlex/_version.py create mode 100644 src/hyperlex/analysis/mutation_trace_attach.py create mode 100644 src/hyperlex/command_router.py create mode 100644 src/hyperlex/data/SKILL.agent.md create mode 100644 src/hyperlex/entrypoint.py create mode 100644 src/hyperlex/init_skill.py create mode 100644 src/hyperlex/mutation/__init__.py create mode 100644 src/hyperlex/mutation/__main__.py create mode 100644 src/hyperlex/mutation/detect.py create mode 100644 src/hyperlex/mutation/grammar.py create mode 100644 src/hyperlex/mutation/human.py create mode 100644 src/hyperlex/mutation/operators.py create mode 100644 src/hyperlex/mutation/packet.py create mode 100644 src/hyperlex/mutation/watch.py create mode 100644 src/hyperlex/schemas/mutation_trace.v0.1.schema.json create mode 100644 tests/test_command_router.py create mode 100644 tests/test_hlx_mutation_alias.py create mode 100644 tests/test_init_skill.py create mode 100644 tests/test_mutation_grammar.py create mode 100644 tests/test_mutation_v02.py create mode 100644 tests/test_packaging_surface.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..a599dea --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,68 @@ +name: Docs (MkDocs) + +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + - "scripts/sync_mkdocs_pages.py" + - "README.md" + - "ARCHITECTURE.md" + - "DESIGN.md" + - "SPEC.md" + - "STATUS.md" + - "ROADMAP.md" + pull_request: + paths: + - "docs/**" + - "mkdocs.yml" + - "scripts/sync_mkdocs_pages.py" + - "STATUS.md" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install docs deps + run: pip install -U pip "mkdocs>=1.5" "mkdocs-material>=9.0" + - name: Sync root pages into docs/ + run: | + cp ARCHITECTURE.md docs/architecture.md + cp DESIGN.md docs/design.md + cp SPEC.md docs/spec.md + cp STATUS.md docs/status.md + python3 scripts/sync_mkdocs_pages.py + - name: Build site (strict) + run: mkdocs build --strict + - name: Upload artifact + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/hermes-evals.yml b/.github/workflows/hermes-evals.yml new file mode 100644 index 0000000..ef8dbe2 --- /dev/null +++ b/.github/workflows/hermes-evals.yml @@ -0,0 +1,41 @@ +name: Hyperlex Skill Validation + +on: + push: + branches: [main] + pull_request: + +jobs: + validate: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + env: + HYPERLEX_OFFLINE: "1" + HYPERLEX_NO_RATE_LIMIT: "1" + PYTHONPATH: src + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install test dependencies + run: python -m pip install -U pip pytest jsonschema + - name: Check skill + run: python3 scripts/hyperlex.py check + - name: Doctor + run: python3 scripts/hyperlex.py doctor + - name: Verify sources command + run: python3 scripts/hyperlex.py sources + - name: Run smoke + run: python3 scripts/hyperlex.py smoke + - name: Diagram from golden + run: python3 scripts/hyperlex.py diagram --from-golden --out-dir out/ci-diagrams --no-html + - name: Case study runner + run: python3 scripts/run_case_study.py --out-dir out/ci-case-study + - name: Run tests + run: python3 -m pytest -q diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md new file mode 100644 index 0000000..0ada91a --- /dev/null +++ b/.specify/memory/constitution.md @@ -0,0 +1,51 @@ +# Project Constitution: Hyperlex + +**Version**: 1.0.0 +**Ratified**: 2026-09-04 +**Last Amended**: 2026-09-04 +**Status**: SHADOW / ADVISORY until operator promotion +**Remote**: scrimshawlife-ctrl/Hyperlex +**Notion owner**: Hyperlex spine (01 Specs & Spines) + +This constitution is the evaluation gate for every later Spec Kit artifact (`specify`, `plan`, `tasks`, `implement`, `converge`). Existing `DESIGN.md`, `SPEC.md`, `SKILL.md`, and `docs/api-v1.md` remain runtime truth until an implement cycle lands. This file does not mutate API_V1. + +## Principles + +### I. Catch becoming-culture, not settled culture +Hyperlex exists to catch language while it is still becoming culture. Lineage, receipts, and settlement are first-class. Lexicon dumps without operator and mutation structure are incomplete. + +### II. Offline-first, fail-closed on scores, fail-open on ingest +Baseline analysis MUST run under `HYPERLEX_OFFLINE=1` / `source=mock` with no network. Ingest failures degrade. Missing settlements yield `NOT_COMPUTABLE`. Never crash the analyze path for optional enrichers. + +### III. Settled Brier only +Open analysis, Phase 5, mutation prediction, and mutation-grammar watch scores MUST keep `brier: null`. Numeric Brier exists only after operator settlement on an eligible forecast class. Mutation-grammar packets are never forecast-eligible. + +### IV. Provenance discipline +Every claim is `OBSERVED`, `INFERRED`, or `SPECULATIVE`. Lineage confidence is INFERRED. Next-form prediction is SPECULATIVE. Operator-stack detection on an attested surface may be OBSERVED for the surface and INFERRED for the stack. + +### V. No Abraxas hard import +Hyperlex MUST NOT import Abraxas. Wire shapes live under `hyperlex.compat.abraxas` as export-only. Hyperlex MUST NOT write Abraxas rune registry rows, p89 binds, or Forecast spines. + +### VI. Receipt integrity is frozen history +Lineage backprop, mutation traces, and grammar overlays MUST NOT rewrite historical receipt hashes or `provenance.integrity`. + +### VII. Detector over generator for adversarial grammar +Mutation *prediction* may emit civilian next surface forms of a slang atom (existing `hyperlex.mutation_prediction.v1`). Mutation *grammar* (this expansion) is a detector, tracer, and watcher. It MUST NOT generate restricted-intent wraps, jailbreak recipes, or reconstructable payloads when `restricted_intent_suspected` is true. Store hash + layer tags only in that case. + +### VIII. Human sovereignty on promotion +Schema landings, registry family adds, Chroma promotes, cron registration, and canon status changes require an operator gate. Specs do not self-promote. + +### IX. Library-first additive modules +New capability ships as a package module (`hyperlex.analysis.*` or `hyperlex.mutation.*`) with tests before CLI sugar. API_V1 stays frozen; additive symbols go on `API_EXTENDED`. + +### X. Dual-use bound +Cite safety literature. Do not reproduce restricted payloads. SUAS/SS may consume watch-language tags only. No operational evasion playbook in this repo. + +## Constraints +- Python ≥ 3.10, stdlib-first baseline. +- Atomic terms; do not density-stack multi-term bags as one seed by default. +- RUNE.HLX.* envelopes are advisory. No execute_production. +- Sacred-slang metrics (ritualCharge, paraphraseResistance, …) are overlays, not Brier inputs. + +## Amendment +Operator amends this file, bumps version, restamps Last Amended. Dependent specs must re-check principles I–X. diff --git a/CHANGELOG.md b/CHANGELOG.md index e0397f1..55f1646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,13 +15,6 @@ - CLI auto-loads `~/.hermes/.env` (and `~/.hyperlex/.env`); accepts official `CHROMA_API_KEY` / `CHROMA_TENANT` / `CHROMA_DATABASE` aliases - Cloud client only requires API key; tenant/database optional when Chroma can infer them -## 0.4.1 — Hermes Agent v0.21 skill index (2026-09-01) - -- `SKILL.md` matches the Hermes v0.21 (tag v2026.8.31) hardline: one-sentence - description ≤60 characters, human-first author, required frontmatter keys, - modern section order, CLI framed through the Hermes `terminal` tool -- CLI `check` asserts the skill-index description budget and required keys - ## 0.4.0 — Automatic backend pipeline (2026-08-05) - `run_pipeline` / CLI `pipeline`: ingest → analyze → receipt → forecasts → score log → Phase 5 risk diff --git a/README.md b/README.md index f7b4289..4d2189e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Docs CI Docs License: LicenseRef-Zero-State-Proprietary-1.0 - Version 0.4.1 + Version 0.4.0 Python Hermes skill Offline-first @@ -120,7 +120,7 @@ Phases **0–4** are production skill surface. **5.0–5.3** adds research simul | | | |--|--| -| **Version** | **0.4.1** | +| **Version** | **0.4.0** | | **Posture** | Hermes skill · Python package (≥3.10) · LicenseRef-Zero-State-Proprietary-1.0 | | **Primary store** | `~/.hyperlex/` | | **Public PyPI** | Not planned | @@ -347,8 +347,6 @@ from hyperlex.compat.abraxas import to_brier_ledger_entry, list_hlx_runes | [docs/phase5.md](./docs/phase5.md) | Research simulation | | [docs/slang-lineages.md](./docs/slang-lineages.md) | Lineage methodology | | [docs/hermes-skill.md](./docs/hermes-skill.md) | Skill model | -| [AGENTS.md](./AGENTS.md) | Multi-agent SDLC contract (`CLAUDE.md` is a thin pointer here) | -| [intent/](./intent/README.md) | Intent template for non-trivial work; then spec.md / plan | Examples: `examples/cron/` · `examples/receipts/golden/` · `examples/calibration/` · `examples/slang-families/` · `examples/case-studies/` @@ -356,7 +354,7 @@ Examples: `examples/cron/` · `examples/receipts/golden/` · `examples/calibrati ## Contributing -See [CONTRIBUTING.md](./CONTRIBUTING.md). Agent contract: [`AGENTS.md`](AGENTS.md) (`CLAUDE.md` is a thin pointer there). Non-trivial work starts with [`intent/`](intent/README.md). Keep the settlement rule and offline-first defaults intact. Prefer small, test-backed PRs. +See [CONTRIBUTING.md](./CONTRIBUTING.md). Keep the settlement rule and offline-first defaults intact. Prefer small, test-backed PRs. ```bash PYTHONPATH=src pytest -q @@ -370,12 +368,3 @@ bash install.sh --dry-run LicenseRef-Zero-State-Proprietary-1.0 — [LICENSE](./LICENSE). Built for operators who want memetic signal **with a receipt trail**, not a mood board. - - -## License - -Copyright © 2026 Zero State LLC. All Rights Reserved. Licensed under the Zero State Proprietary License v1.0. See [`LICENSE`](LICENSE). - -Third-party components remain subject to their respective licenses. - -Source, profile, and migration rules: `references/source-and-upgrades.md`. diff --git a/SKILL.md b/SKILL.md index ce47b6f..d0e24b0 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,10 +1,20 @@ --- name: hyperlex -description: Catch slang while it is still becoming culture. -version: 0.4.1 -author: Daniel Meyer (scrimshawlife-ctrl), Hermes Agent +description: > + Use when the user wants memetic emergence analysis, slang detection, + hyperstition / virality scoring, slang lineage matching, forecast extraction, + operator settlement, Brier calibration on cultural signals, or slang + mutation-operator detection on attested text. Triggers include slang, + memetics, hyperstition, virality, lineage, mutation trace, algospeak, + Brier, settle forecasts, score-series, betting slang, crypto-degen slang, + ai-native slang, brainrot, and receipt-backed cultural signal scans. + Not for general web research (use agent-reach), product audits (neon-genie), + cinematic work (kubrick), or jailbreak / wrap composition. +version: 0.4.2 +author: Applied Alchemy Labs / Hermes license: LicenseRef-Zero-State-Proprietary-1.0 platforms: [linux, macos] +dependencies: [] metadata: hermes: tags: @@ -17,96 +27,180 @@ metadata: - Brier - Receipts - Forecasting - related_skills: [] + - Mutation category: analysis + related_skills: [] openclaw: requires: bins: [python3] os: [darwin, linux] emoji: "🌀" +triggers: + - hyperlex + - memetic + - memetics + - slang + - slang lineage + - hyperstition + - virality + - neologism + - betting slang + - sharp money + - brainrot + - brier + - settle forecast + - score series + - cultural signal + - memetic receipt + - hyperlex wizard + - get started with hyperlex + - hyperlex onboarding + - mutation trace + - mutation grammar + - algospeak + - slang mutation --- # Hyperlex -Hyperlex is a Hermes skill for memetic emergence analysis: slang detection, lineage matching, virality and hyperstition scoring, integrity-hashed receipts, and Brier calibration after operator settlement. It does not invent numeric Brier on open analysis. Applied Alchemy Labs built the Python package under `src/hyperlex/`; the CLI is `scripts/hyperlex.py`. Baseline `mock` mode needs no network and no Abraxas import. +Standalone **Hermes skill** for memetic emergence analysis. -Hermes substitutes `${HERMES_SKILL_DIR}` with the installed skill directory. Call the bundled CLI through the Hermes `terminal` tool, not as a bare shell aside. +Hermes loads this directory and uses `SKILL.md` as the behavior contract. +The runtime is the bundled Python package under `src/hyperlex/` plus the CLI +at `scripts/hyperlex.py`. No Abraxas import, no required network for baseline +(`mock`) mode. -## When to Use +Resolve paths from the installed skill root. Set: -- Detect slang or neologisms and score virality, memetics, or hyperstition -- Match slang into historical lineage families with a transparent confidence breakdown -- Backfill YTD slang packs and backpropagate lineage onto historical receipts (report only; non-mutating) -- Run Phase 5 cultural-transmission simulation (always SPECULATIVE; `brier: null`) -- Propose advisory `LIVE_EMERGENCE_SCAN` cadence with `risk-schedule` (never auto-register Hermes cron) -- Emit integrity-hashed receipts and extract forecast probabilities -- Settle forecasts as an operator and recompute Brier series from the score log -- Scan betting-sharp, crypto-degen, ai-native, brainrot, kinship, or political-status families +```bash +export HERMES_SKILL_DIR="${HERMES_SKILL_DIR:-$HOME/.hermes/skills/hyperlex}" +``` -Don't use for general web research, product audits, cinematic continuity, or symbolic architecture mapping. +## When to Use + +- Detect slang / neologisms and score virality, memetics, hyperstition +- Match slang into historical **lineage families** with transparent confidence +- Detect slang **mutation operator stacks** on attested text (`mutation trace`) +- Next civilian surfaces of a slang atom (`mutation predict`) — SPECULATIVE, not Brier +- **Backfill** YTD slang packs and **backpropagate** lineage onto historical receipts (non-mutating) +- **Phase 5 simulate** cultural transmission, multi-agent memetics, hyperstition risk, phylogeny scaffold +- **Risk-schedule** advisory LIVE_EMERGENCE_SCAN cadence from risk tiers (never auto-registers cron) +- Emit integrity-hashed **receipts** for auditable runs +- Extract **forecasts** from analysis (probabilities only — no fake Brier) +- **Settle** forecasts as an operator and recompute Brier series from the score log +- Scan betting-sharp, crypto-degen, ai-native, brainrot, kinship, political-status families + +## When Not to Use + +- General multi-platform web research → agent-reach +- Product / opportunity intelligence → neon-genie +- Cinematic continuity / storyboards → kubrick +- Symbolic code architecture mapping → orchestra +- Jailbreak / wrap composition / model ASR boards — detector only; no generator path ## Prerequisites -- Python 3.10+ with `python3` on PATH -- Optional: `requests`, `jsonschema`, `crawl4ai` for richer ingest and validation +- Python 3.10+ +- `python3` on PATH +- Optional: `requests`, `jsonschema`, `crawl4ai` for richer ingest / validation - Optional: network for non-`mock` sources -- Offline force: `HYPERLEX_OFFLINE=1` - -## How to Run -After `skill_view` loads this skill, `${HERMES_SKILL_DIR}` is already substituted. Call `terminal`: +## Install +```bash +bash install.sh --dry-run +bash install.sh +# installs to ~/.hermes/skills/hyperlex by default +python3 "$HOME/.hermes/skills/hyperlex/scripts/hyperlex.py" check +python3 "$HOME/.hermes/skills/hyperlex/scripts/hyperlex.py" smoke ``` -terminal(command="bash install.sh --dry-run", timeout=60) -terminal(command="bash install.sh", timeout=120) + +## Commands (prefer simplified path) + +```bash +HLX="python3 $HERMES_SKILL_DIR/scripts/hyperlex.py" + +$HLX wizard --auto # week-one guided path (offline) +$HLX wizard # interactive (TTY) +$HLX commands # full simplified map (JSON) +$HLX sources # sources + routes +# AUTO backend — ingest → full results (receipt, forecasts, phase5 risk) +$HLX pipeline "rizz" --route offline +$HLX ingest "locked in" # same as pipeline (use --raw-only for signal only) +$HLX pipeline "sigma rizz locked in" # expands to atoms automatically +$HLX pending # open forecasts +$HLX settle --forecast-id --decision TRUE +$HLX score-series --mean-shift --verify-chain +$HLX scan --route offline --receipt --forecasts --append-log +$HLX risk-schedule --tier MODERATE --schedule-out /tmp/hlx-cron +$HLX doctor && $HLX smoke ``` -Default install path: `~/.hermes/skills/hyperlex`. Then prove the install: +**Ingest routing:** prefer `--route offline|live|glossary|social` over raw `--source`. +Aliases: `real`→glossary, `x`→x_search, `firecrawl`→crawl4ai. Offline env: `HYPERLEX_OFFLINE=1`. +Mutation (package CLI; prefer this over inventing a second skill): + +```bash +PYTHONPATH="$HERMES_SKILL_DIR/src" python3 -m hyperlex mutation trace "it's giving mid rizz" +PYTHONPATH="$HERMES_SKILL_DIR/src" python3 -m hyperlex mutation trace "rzz" --human +PYTHONPATH="$HERMES_SKILL_DIR/src" python3 -m hyperlex mutation watch +PYTHONPATH="$HERMES_SKILL_DIR/src" python3 -m hyperlex mutation predict rizz ``` -terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py check", timeout=60) -terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py smoke", timeout=120) -``` -Reuse `HLX="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py"` in later `terminal` commands. +`pipeline` / `analyze` attach `analysis.mutation_trace` when operators fire. +`trace --human` prints a civilian advisory card. `trace --watch-jsonl` appends instrumentation (not probabilities; never auto-fires tools). +Do not add a wizard step. Do not add `wrap` / `compose` / `asr` verbs. +Lane is SHADOW / advisory. + +Research / advanced: `simulate`, `vector-*`, `archive-export`, +`lineage-backfill`, `lineage-backprop`, `relay`, `signal`, `diagram`, `ledger-*`. + +Docs: `docs/operator-loop.md`, `docs/commands.md`. + +## Guided wizard -## Quick Reference +When the user is new to Hyperlex, asks to get started, or wants a guided +operator path: -| Action | `terminal` command | -|--------|-------------------| -| Guided week-one path | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py wizard --auto", timeout=180)` | -| Interactive wizard | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py wizard", timeout=300)` | -| Command map (JSON) | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py commands", timeout=30)` | -| Source and route catalog | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py sources", timeout=30)` | -| Offline pipeline (receipt + forecasts) | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py pipeline \"rizz\" --route offline", timeout=120)` | -| Signal only | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py ingest \"locked in\" --raw-only", timeout=120)` | -| Multi-term atoms | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py pipeline \"sigma rizz locked in\" --route offline", timeout=180)` | -| Open forecasts | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py pending", timeout=30)` | -| Operator settle | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py settle --forecast-id --decision TRUE", timeout=60)` | -| Brier series | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py score-series --mean-shift --verify-chain", timeout=60)` | -| Multi-query scan | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py scan --route offline --receipt --forecasts --append-log", timeout=300)` | -| Advisory schedule only | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py risk-schedule --tier MODERATE --schedule-out /tmp/hlx-cron", timeout=60)` | -| Health | `terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py doctor", timeout=120)` | +1. Ensure skill install (`bash install.sh` if missing). +2. Run `$HLX wizard --auto` (or `$HLX wizard --auto --query ""`). +3. Summarize steps: env → doctor → demo → first pipeline → calibration coach. +4. **Never invent Brier.** Open analysis keeps `brier: null`. +5. Show open forecasts from the wizard output / `$HLX pending`. +6. **Settlement requires operator authority** — ask for TRUE|FALSE|VOID, then: + `$HLX settle --forecast-id --decision …` +7. `$HLX score-series --mean-shift --verify-chain` +8. Optional advisory only: `$HLX risk-schedule --tier MODERATE --schedule-out /tmp/hlx-cron` + (never auto-register Hermes cron). -Ingest routing: prefer `--route offline|live|glossary|social` over raw `--source`. Aliases: `real`→glossary, `x`→x_search, `firecrawl`→crawl4ai. +Step IDs (stable): `env_intro`, `doctor`, `demo`, `first_pipeline`, +`calibration_coach`, `score_series_hint`, `handoff`. -Research commands (still available): `simulate`, `vector-*`, `archive-export`, `lineage-backfill`, `lineage-backprop`, `relay`, `signal`, `diagram`, `ledger-*`. Docs: `docs/operator-loop.md`, `docs/commands.md`. +### Operator calibration path -## Procedure +```text +run "" --route offline + → pending + → settle --forecast-id … --decision TRUE|FALSE + → score-series [--mean-shift] [--verify-chain] +``` -1. **Onboard.** If the user is new to Hyperlex, call `terminal` with `wizard --auto` (or `wizard --auto --query ""`). Done when the JSON lists step IDs `env_intro`, `doctor`, `demo`, `first_pipeline`, `calibration_coach`, `score_series_hint`, and `handoff`. -2. **Analyze.** Call `terminal` with `pipeline "" --route offline` (or `run` / `ingest`; use `--raw-only` for signal only). Multi-term bags expand to lexicon atoms automatically. Done when a receipt exists and `provenance.brier` is `null`. -3. **Calibrate.** Call `pending`, ask the operator for `TRUE|FALSE|VOID`, then `settle --forecast-id --decision …`, then `score-series --mean-shift --verify-chain`. Never invent Brier. Done when the series status is `SCORED` or `NOT_COMPUTABLE`. -4. **Scan (optional).** Call `scan --route offline --receipt --forecasts --append-log`. For cadence, call `risk-schedule` and write the advisory job under `--schedule-out`. Done when receipts land and no Hermes cron job was registered. -5. **Live ingest.** Use `--route live` (or glossary/social) only when the user allows network. Done when ingest metadata records the intended source and any offline fallback. -6. **Label claims.** Mark findings `OBSERVED`, `INFERRED`, or `SPECULATIVE`. Fail closed on missing outcomes. Done when every numeric score has a matching authority class. -7. **Research.** `simulate`, archive, and vector commands stay optional and SPECULATIVE. Phase 5 packets keep `brier: null`. +- Score log default: `~/.hyperlex/score_log.jsonl` +- Override: `HYPERLEX_SCORE_LOG`, `--log`, or `--repo-log` → `out/calibration/score_log.jsonl` +- **Never** emit numeric Brier without settlement. Empty series → `NOT_COMPUTABLE`. -Score log default: `~/.hyperlex/score_log.jsonl`. Override with `HYPERLEX_SCORE_LOG`, `--log`, or `--repo-log` → `out/calibration/score_log.jsonl`. +## Preferred sequence -**Authority.** Hyperlex may ingest, match lineage, write receipts and score-log events, compute Brier only from settled pairs, and export Abraxas-compatible ledger shapes (no Abraxas import). Hyperlex may not invent numeric Brier on open analysis, auto-settle without an authority marker, promote speculative hyperstition stages as hard truth, rewrite historical receipt integrity during lineage backprop, invent Brier from Phase 5, or mutate other systems. +1. **`wizard --auto`** then `commands` / `run` — week-one guided path, then map or one-shot. +2. **`run --route offline`** — one-shot analyze + receipt + forecasts + score log. +3. **`pending` → `settle` → `score-series`** — Brier only after operator settlement. +4. **Cron** — `risk-schedule` (advisory) + `scan --route offline` for multi-query. +5. **Live ingest** — only when network allowed: `--route live` (or glossary/social). +6. **Label claims** — `OBSERVED` / `INFERRED` / `SPECULATIVE`; fail closed on missing outcomes. +7. **Research** — `mutation trace` / `mutation predict` / `simulate` / archive / vector are optional and SPECULATIVE. Watch scores are not Brier. -**Library path** (when `src/` is on `PYTHONPATH`; the CLI inserts `src/` automatically): +## Public API (package) ```python from hyperlex import ( @@ -115,45 +209,88 @@ from hyperlex import ( settle_and_log, recompute_series, score_pair, score_series, NOT_COMPUTABLE, ) +from hyperlex.mutation import parse_mutation_trace result = detect_memetic_patterns(query="rizz", ingest_source="mock") +trace = parse_mutation_trace("it's giving mid rizz") forecasts = extract_forecasts(result) +# later, after operator review: # settle_and_log(forecast, outcome_value=1.0, settlement_decision="TRUE") # recompute_series() ``` +Ensure `src/` is on `PYTHONPATH` when importing outside the CLI +(CLI inserts `src/` automatically). + +## Authority boundaries + +Hyperlex **may**: + +- ingest and analyze signals +- match lineages with transparent score breakdowns +- detect mutation operator stacks on attested text +- extract forecasts and write receipts / score-log events +- compute Brier only from settled pairs +- export Abraxas-compatible ledger shapes (no Abraxas import) + +Hyperlex **may not**: + +- invent numeric Brier on open analysis (`provenance.brier` stays `null`) +- treat `watch_score` as Brier or as a tool-fire threshold +- auto-settle without authority marker +- promote speculative hyperstition stages as hard truth +- rewrite historical receipt integrity during lineage backprop (report only) +- invent Brier from Phase 5 simulation (always `brier: null`, SPECULATIVE) +- mutate Abraxas or other systems (export is optional and offline) +- generate restricted wraps, ASR boards, or call predict on restricted spans + ## Pitfalls -- Always invoke `scripts/hyperlex.py` through `${HERMES_SKILL_DIR}` so `src/` is first on `sys.path` and the script cannot shadow the package. -- `platforms` is `[linux, macos]`. `install.sh` is bash; Windows is not proven. -- Non-`mock` sources need network and may degrade to mock. Read ingest metadata before treating the source as live. -- Lineage confidence is INFERRED. Do not treat it as observed ground truth. -- `score-series --mean-shift` is advisory for future forecasts only. -- The score log is append-only. Recompute the series from the log; do not treat a stored series as sole truth. -- `risk-schedule` is advisory. Do not add a `metadata.hermes.blueprint` block and do not auto-register cron. +- `scripts/hyperlex.py` must not shadow the package: always run via the skill path so `src/` is first on `sys.path`. +- Non-`mock` sources need network and may degrade gracefully — check ingest metadata. +- Lineage confidence is **INFERRED**; do not treat it as observed ground truth. +- Mean-shift from `score-series --mean-shift` is **advisory** for future forecasts only. +- Score log is append-only; series is recomputed from the log, not stored as sole truth. +- Mutation predict is civilian next-forms only. Detector and generator stay separate modules. ## Verification -Call `terminal`: - -``` -terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py check", timeout=60) -terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py doctor", timeout=120) -terminal(command="python3 ${HERMES_SKILL_DIR}/scripts/hyperlex.py smoke", timeout=120) +```bash +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" check +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" doctor +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" ledger-stats +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" signal --input +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" feedback --signal-key hyperstition.stage +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" diagram --from-golden --out-dir out/diagrams +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" scan --config "$HERMES_SKILL_DIR/examples/cron/scan-queries.json" --source mock --receipt --forecasts +python3 "$HERMES_SKILL_DIR/scripts/hyperlex.py" smoke +PYTHONPATH="$HERMES_SKILL_DIR/src" python3 -m hyperlex mutation trace "it's giving mid rizz" ``` -Optional deeper proof: `ledger-stats`, `signal --input `, `feedback --signal-key hyperstition.stage`, `diagram --from-golden --out-dir out/diagrams`, and `scan --config ${HERMES_SKILL_DIR}/examples/cron/scan-queries.json --source mock --receipt --forecasts`. - Successful packaging: - `~/.hermes/skills/hyperlex/SKILL.md` exists - `check` returns `"ok": true` -- Frontmatter `description` is ≤60 characters and ends with `.` - `smoke` writes a receipt under `out/smoke/` - Open analysis has `"brier": null` +- Mutation trace packets have `"forecast_eligible": false` -Design references: `DESIGN.md`, `docs/brier-calibration.md`, `docs/slang-lineages.md`, `docs/phase5.md`, `docs/modules/simulation.md`, `schemas/`, `examples/slang-families/`, `data/backfill/2026/`, `references/hermes-runtime-contract.md`. +## Design references -Local stdlib-first CLI. Baseline (`mock`) needs no network. Real ingest may call public web APIs. Score log and receipts are local files under `~/.hyperlex/` or skill `out/`. +- `DESIGN.md` — principles (incl. 11 lineage, 12 Brier requires settlement) +- `docs/brier-calibration.md` — forecast → settlement → score +- `docs/slang-lineages.md` — family methodology +- `docs/phase5.md` / `docs/modules/simulation.md` — Phase 5 research simulation +- `specs/001-mutation-grammar/` — detector grammar + dual-use gate +- `specs/003-mutation-detect-v02/` — GAME_ENCODE / CODE_SWITCH / PHONETIC_WARP + watch jsonl + `--human` (SHADOW) +- `schemas/` — ingest, result, receipt, forecast, settlement, brier_series, lineage, mutation_trace +- `examples/slang-families/` — Mermaid + HTML family diagrams +- `data/backfill/2026/` — YTD slang packs +- `references/hermes-runtime-contract.md` — path / authority policy +- `references/source-and-upgrades.md` — install identity, two-rename limits, lock recovery + +## Security +Local stdlib-first CLI. Baseline (`mock`) needs no network. Real ingest may call public web APIs. Score log and receipts are local files under `~/.hyperlex/` or skill `out/`. +Mutation packets are untrusted structured output (hosts must not execute fields). Source, profile, and migration rules: `references/source-and-upgrades.md`. diff --git a/STATUS.md b/STATUS.md index f31b285..63ea392 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,6 +1,6 @@ # Hyperlex Skill Status -**Version:** 0.4.1 +**Version:** 0.4.0 **Posture:** Hermes skill (Python package repo) **Install:** `bash install.sh` → `~/.hermes/skills/hyperlex` **Track:** Phases 0–4 complete · Phase 5.0–5.3 · Pages static run history · Hallmark desk UI @@ -67,6 +67,7 @@ pipeline "rizz" | run "rizz" ~/.hyperlex/receipts/ ~/.hyperlex/receipt_ledger.jsonl ~/.hyperlex/score_log.jsonl +~/.hyperlex/mutation_watch.jsonl # mutation grammar instrumentation (not Brier) ~/.hyperlex/cache/ ~/.hyperlex/vector.db ~/.hyperlex/chroma/ diff --git a/VERSION b/VERSION index 267577d..2b7c5ae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.1 +0.4.2 diff --git a/docs/README.md b/docs/README.md index b2b74f8..d5716d1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,8 +25,6 @@ Hyperlex is a **Hermes skill** (Python package repo). Relevant Abraxas wire shap - [ARCHITECTURE.md](architecture.md) - [DESIGN.md](design.md) - [SPEC.md](spec.md) -- [AGENTS.md](../AGENTS.md) — multi-agent SDLC contract (`CLAUDE.md` is a thin pointer there) -- [intent/](../intent/README.md) — intent template for non-trivial work; then spec.md / plan ## Examples - `examples/slang-families/` — Mermaid family trees diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index bd4443c..7ed4297 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,9 +1,5 @@ # Release notes -## 0.4.1 -- Hermes Agent v0.21 SKILL.md contract: ≤60-character description, required frontmatter, `terminal` framing. -- CLI `check` asserts the skill-index description budget. - ## 0.4.0 - Automatic backend pipeline: `pipeline` / `run` / `ingest` → full results packet. - Ingest defaults to full results (`--raw-only` for signal-only). diff --git a/docs/commands.md b/docs/commands.md index 5a5e63a..99aabce 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -23,12 +23,12 @@ HLX="python3 ${HERMES_SKILL_DIR:-$HOME/.hermes/skills/hyperlex}/scripts/hyperlex ```text pipeline / run / ingest - │ - ▼ + | + v resolve route → expand atoms → for each atom: analyze → receipt → forecasts → score_log → phase5 risk - │ - ▼ + | + v results packet (brier always null until settle) ``` @@ -36,7 +36,7 @@ pipeline / run / ingest | Command | Purpose | |---------|---------| -| `pending` | Open (unsettled) forecasts from score log | +| `pending` | Open (unsettled) forecasts from the score log | | `settle --forecast-id ID --decision TRUE\|FALSE\|VOID` | Operator settlement | | `score-series --mean-shift --verify-chain` | Brier series from settled pairs | @@ -46,7 +46,7 @@ pipeline / run / ingest |---------|---------| | `sources` | Catalog + named routes | | `sources --route live` | Preview resolve | -| `ingest "" --route offline` | Ingest only (structured fingerprint) | +| `ingest "" --route offline` | Ingest only (structured + fingerprint) | | `analyze "" --route offline` | Analyze without auto-receipt | Prefer **`--route offline|live|glossary|social`** over raw adapter names. @@ -58,7 +58,11 @@ Aliases: `real`→glossary, `x`→x_search, `firecrawl`→crawl4ai, `live`→com |---------|---------| | `simulate --term T --mode scenario` | Phase 5 composed scenario | | `simulate --mode schedule --tier ELEVATED` | Risk→scan plan | -| `mutation-predict "rizz"` | Next surface forms (SPECULATIVE · brier null) | +| `mutation trace "it's giving mid rizz"` | Detect operator stacks (brier null · forecast_eligible false) | +| `mutation trace "…" --human` | Civilian advisory card (SHADOW; not a forecast) | +| `mutation watch` | Read mutation watch jsonl (instrumentation; never auto-fires) | +| `mutation predict "rizz"` | Next civilian surfaces (SPECULATIVE · brier null) | +| `mutation-predict "rizz"` | Deprecated alias of `mutation predict` | | `archive-export --history` | Sanitized Pages run history | ## Vector DB (sqlite · local chroma · cloud) @@ -74,7 +78,7 @@ Aliases: `real`→glossary, `x`→x_search, `firecrawl`→crawl4ai, `live`→com | `vector-export -o file.jsonl` | Dump embeddings JSONL | | `vector-import -i file.jsonl [--cloud]` | Load dump | -Flow: **seed local → search → sync to cloud**. Secrets: `~/.hermes/.env` (`CHROMA_API_KEY`, …). +Flow: **seed local → search → sync to cloud**. Secrets: `~/.hermes/.env` (`CHROMA_API_KEY`, …). Docs: [modules/vectordb.md](modules/vectordb.md). ## Maintenance @@ -97,6 +101,8 @@ Docs: [modules/vectordb.md](modules/vectordb.md). scan / risk-schedule (cron advisory) ``` +`analyze` / `pipeline` attach `analysis.mutation_trace` when operators fire. + ## Deferred (do not prioritize) - Multi-collection / remote embed models beyond hash default diff --git a/docs/index.md b/docs/index.md index c7b043a..a76fa45 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ hide: # Hyperlex
-v0.4.1 +v0.4.0 Hermes skill · Python package Settled Brier only Local-first · offline mock ready @@ -126,5 +126,5 @@ Full glossary: [start/glossary.md](start/glossary.md)

READ DEEPER. THINK WIDER.

-v0.4.1 · Hermes skill · settled Brier only · offline mock default for first success +v0.4.0 · Hermes skill · settled Brier only · offline mock default for first success

diff --git a/docs/install-package.md b/docs/install-package.md new file mode 100644 index 0000000..a54fbe0 --- /dev/null +++ b/docs/install-package.md @@ -0,0 +1,25 @@ +# Install like graft + +Graft: `npm i -g @nanonets/graft` then `graft init` writes a thin skill that calls `graft` on PATH. +Hyperlex is the same shape. + +```bash +pip install -e ".[dev]" # or pip install . +hyperlex init # Hermes + OpenClaw + Grok skill dirs +hyperlex init --dry-run +hyperlex init --target hermes --target grok +hyperlex uninstall-skill --target hermes +``` + +`init` writes only `SKILL.md` (marker ``) under: + +- `~/.hermes/skills/hyperlex/` +- `~/.openclaw/skills/hyperlex/` +- `~/.grok/skills/hyperlex/` +- optional `--target claude` → `~/.claude/skills/hyperlex/` + +It does **not** rsync `src/` or `scripts/hyperlex.py`. The agent runs `hyperlex …`. + +Reload the agent session after init. + +Legacy full-tree install remains `bash install.sh` for hosts that still want the fat skill CLI. diff --git a/docs/start/glossary.md b/docs/start/glossary.md index cdffb82..9c98727 100644 --- a/docs/start/glossary.md +++ b/docs/start/glossary.md @@ -49,13 +49,20 @@ Cosine-similar terms/receipts from the local vector index (sqlite or chroma). Similarity is **not** a probability of virality and **never** Brier. See [reading evidence](../demos/reading-evidence.md). +### Mutation trace +Detector-side operator stack on attested text (`AFFIX`, `SUBSTITUTE`, +`REGISTER_SHIFT`, `COMPOSE`, …). Attached as `analysis.mutation_trace`. +Always `brier: null` and `forecast_eligible: false`. +CLI: `mutation trace ""` (alias `mutation-trace`). +Does not call `predict_mutations`. Restricted flag redacts `surface_span`. + ### Mutation prediction Speculative **next surface forms** of a slang atom (compression, derivation, irony templates, family compounds). Attached as `analysis.mutation_prediction`. -Always `provenance: SPECULATIVE` and `brier: null` — ranking weights are not -calibrated probabilities. CLI: `mutation-predict ""`. +Always `provenance: SPECULATIVE` and `brier: null`. CLI: `mutation predict ""` +(alias `mutation-predict`). -### RUNE.HLX.\* +### RUNE.HLX.* Relay envelope naming for Hyperlex-shaped signals in host systems. Optional interop; not required for offline CLI use. diff --git a/docs/status.md b/docs/status.md index 5cc2072..caf29fc 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,6 +1,6 @@ # Hyperlex Skill Status -**Version:** 0.4.1 +**Version:** 0.4.0 **Posture:** Hermes skill (Python package repo) **Install:** `bash install.sh` → `~/.hermes/skills/hyperlex` **Track:** Phases 0–4 complete · Phase 5.0–5.3 · Pages static run history · Hallmark desk UI @@ -70,6 +70,7 @@ pipeline "rizz" | ingest "rizz" | run "rizz" # AUTO full results ~/.hyperlex/receipts/ ~/.hyperlex/receipt_ledger.jsonl ~/.hyperlex/score_log.jsonl +~/.hyperlex/mutation_watch.jsonl # mutation grammar instrumentation (not Brier) ~/.hyperlex/cache/ ~/.hyperlex/vector.db # local SQLite vector store (default) ~/.hyperlex/chroma/ # local Chroma persist (opt-in) diff --git a/docs/telemetry.md b/docs/telemetry.md index 0a9af7f..b8f758f 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -4,7 +4,7 @@ Operator desk on Pages — **publish-safe**, not a live dashboard. Real state lives in `~/.hyperlex/` (receipts, score log, vector DB).
-v0.4.1 +v0.4.0 Hermes skill · auto pipeline Atomic multi-term · settled Brier only Pages = static history diff --git a/hyperlex.manifest.yaml b/hyperlex.manifest.yaml index 48cb5c6..75a300c 100644 --- a/hyperlex.manifest.yaml +++ b/hyperlex.manifest.yaml @@ -1,16 +1,18 @@ name: hyperlex -version: 0.4.1 +version: 0.4.0 display_name: Hyperlex kind: hermes-openclaw-skill description: > Standalone memetic emergence engine for slang detection, hyperstition - tracking, virality analysis, lineage matching, and settled Brier calibration. + tracking, virality analysis, lineage matching, settled Brier calibration, + and slang mutation-operator detection on attested text. license: LicenseRef-Zero-State-Proprietary-1.0 homepage: https://github.com/scrimshawlife-ctrl/Hyperlex repository: https://github.com/scrimshawlife-ctrl/Hyperlex skill_entry: SKILL.md cli_entry: scripts/hyperlex.py +mutation_cli: scripts/hlx-mutation installer: install.sh package_root: src/hyperlex @@ -39,6 +41,8 @@ tags: - lineage - calibration - brier + - mutation + - algospeak - symbolic-analysis - hermes-skill @@ -50,61 +54,27 @@ commands: - sources - ingest - analyze - - extract-forecasts - - settle - - score-series - - verify-score-log - - emit-receipt - - list-receipts - - ledger-diff - - ledger-stats - - archive-export - - verify-receipt-ledger - - validate - - verify-receipt - - smoke - - scan - pipeline - run - - commands - pending - - risk-schedule + - settle + - score-series + - mutation + - mutation-trace + - mutation-predict + - commands + - scan - simulate - - relay - - feedback - - diagram - - signal + - smoke intents: - - check - - doctor - - sources - - ingest + - pipeline - analyze - - extract-forecasts - settle - - score-series - - verify-score-log - - emit-receipt - - list-receipts - - ledger-diff - - ledger-stats - - archive-export - - verify-receipt-ledger - - validate - - verify-receipt - - smoke - - scan - - pipeline - - run - - commands - - pending - - risk-schedule - - simulate - - relay - - feedback - - diagram - - signal + - mutation-trace + - mutation-predict + - slang-mutation + - algospeak schemas: - schemas/ingest.v1.schema.json @@ -114,11 +84,12 @@ schemas: - schemas/settlement.v1.schema.json - schemas/brier_series.v1.schema.json - schemas/lineage.v1.schema.json + - schemas/mutation_trace.v0.1.schema.json runtime_profiles: stdlib: dependencies: [] - guarantee: "mock ingest, analysis, lineage, calibration, receipts with Python 3.10+" + guarantee: "mock ingest, analysis, lineage, calibration, receipts, mutation trace with Python 3.10+" validation: dependencies: ["jsonschema>=4"] guarantee: "JSON Schema validation of artifacts" @@ -149,6 +120,8 @@ score_log: authority: - analyze emits forecasts only; provenance.brier is null until settlement + - mutation_trace is never forecast-eligible; watch_score is not Brier - settle requires operator (or declared) authority - VOID/CONFLICT settlements are not scored - score_series empty → NOT_COMPUTABLE + - detector-over-generator; no wrap composition diff --git a/install.sh b/install.sh index 22e89e5..0b0b407 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,6 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" HERMES_ROOT="${HERMES_HOME:-$HOME/.hermes}" DEFAULT_TARGET="${HERMES_ROOT}/skills/hyperlex" -BACKUP_ROOT="${HERMES_ROOT}/receipts/hyperlex-backups" OPENCLAW_TARGET="${HOME}/.openclaw/skills/hyperlex" VERSION="$(tr -d '[:space:]' < "${ROOT}/VERSION" 2>/dev/null || echo "0.0.0")" @@ -33,9 +32,9 @@ Usage: ./install.sh [options] Options: --dry-run Show actions without writing --target DIR Install to DIR (default: ${DEFAULT_TARGET}) - --rollback Restore most recent backup + --rollback Restore most recent target-keyed backup --openclaw Also install to ~/.openclaw/skills/hyperlex - --skip-smoke Skip post-install smoke check + --skip-smoke Skip staged check/smoke (marks UNVERIFIED) --allow-outside-home Permit --target outside \$HOME --version Print version and exit -h, --help Show this help @@ -116,6 +115,8 @@ validate_source() { "hyperlex.manifest.yaml" "VERSION" "scripts/hyperlex.py" + "scripts/hlx-mutation" + "scripts/install_transaction.py" "src/hyperlex/__init__.py" "src/hyperlex/calibration/scoring.py" "src/hyperlex/schemas/result.v1.schema.json" @@ -126,6 +127,7 @@ validate_source() { "schemas/settlement.v1.schema.json" "schemas/brier_series.v1.schema.json" "schemas/lineage.v1.schema.json" + "schemas/mutation_trace.v0.1.schema.json" ) local f for f in "${required[@]}"; do @@ -133,89 +135,25 @@ validate_source() { done python3 -c "import ast, pathlib; ast.parse(pathlib.Path(r'''${ROOT}/scripts/hyperlex.py''').read_text())" \ || die "scripts/hyperlex.py failed syntax check" + python3 -c "import ast, pathlib; ast.parse(pathlib.Path(r'''${ROOT}/scripts/hlx-mutation''').read_text())" \ + || die "scripts/hlx-mutation failed syntax check" + python3 -c "import ast, pathlib; ast.parse(pathlib.Path(r'''${ROOT}/scripts/install_transaction.py''').read_text())" \ + || die "scripts/install_transaction.py failed syntax check" log "Source validation OK" } -backup_existing() { - if [[ ! -d "$TARGET" ]]; then - return 0 - fi - local stamp dest - stamp="$(date -u +%Y%m%dT%H%M%SZ)" - dest="${BACKUP_ROOT}/${stamp}" - log "Backing up existing install to ${dest}" - if [[ $DRY_RUN -eq 1 ]]; then - printf '[dry-run] cp -a %q %q\n' "$TARGET" "$dest" - else - mkdir -p "${BACKUP_ROOT}" - cp -a "${TARGET}" "${dest}" - fi -} - -sync_tree() { - local dest="$1" - log "Installing to ${dest}" - if [[ $DRY_RUN -eq 1 ]]; then - printf '[dry-run] rsync/copy %q → %q\n' "$ROOT" "$dest" - return 0 - fi - mkdir -p "$dest" - if command -v rsync >/dev/null 2>&1; then - rsync -a --delete \ - --exclude '.git' \ - --exclude '.venv' \ - --exclude '__pycache__' \ - --exclude '.pytest_cache' \ - --exclude 'out/' \ - --exclude '*.pyc' \ - "${ROOT}/" "${dest}/" - else - tar -C "${ROOT}" \ - --exclude '.git' \ - --exclude '.venv' \ - --exclude '__pycache__' \ - --exclude '.pytest_cache' \ - --exclude 'out' \ - -cf - . | tar -C "${dest}" -xf - - fi - chmod +x "${dest}/install.sh" "${dest}/scripts/hyperlex.py" 2>/dev/null || true - # rsync --exclude '.git' also skips deleting a leftover dest .git; drop it so the - # skill install is never a nested half-repo from a prior manual checkout. - if [[ -e "${dest}/.git" ]]; then - rm -rf "${dest}/.git" - fi - mkdir -p "${dest}/.hermes" - cat > "${dest}/.hermes/install-receipt.json" </dev/null; then - warn "check failed — run: python3 ${dest}/scripts/hyperlex.py check" - return 1 - fi - if ! python3 "${dest}/scripts/hyperlex.py" smoke >/dev/null; then - warn "smoke failed — run: python3 ${dest}/scripts/hyperlex.py smoke" - return 1 - fi - log "Post-install check OK" + local check_args=() + [[ $SKIP_SMOKE -eq 1 ]] && check_args+=(--skip-checks) + python3 "${ROOT}/scripts/install_transaction.py" "$ROOT" "$dest" hyperlex "${check_args[@]}" } do_rollback() { validate_target if [[ $DRY_RUN -eq 1 ]]; then log "DRY RUN: would validate and restore a backup bound to ${TARGET}" + log "DRY RUN: locks are never auto-reclaimed; two-rename restore is not crash-atomic" return 0 fi local check_args=() @@ -232,23 +170,20 @@ validate_source validate_target if [[ $DRY_RUN -eq 1 ]]; then - log "DRY RUN: would install Hyperlex v${VERSION} → ${TARGET}" - [[ -d "$TARGET" ]] && log "DRY RUN: would backup existing install under ${BACKUP_ROOT}" + log "DRY RUN: would staged-validate then activate Hyperlex v${VERSION} → ${TARGET}" + log "DRY RUN: two-rename activation is not crash-atomic; locks are never auto-reclaimed" + [[ -d "$TARGET" ]] && log "DRY RUN: would publish a target-keyed backup under ${HERMES_ROOT}/backups/hyperlex/" [[ $INSTALL_OPENCLAW -eq 1 ]] && log "DRY RUN: would also install to ${OPENCLAW_TARGET}" exit 0 fi -CHECK_ARGS=() -[[ $SKIP_SMOKE -eq 1 ]] && CHECK_ARGS+=(--skip-checks) -python3 "${ROOT}/scripts/install_transaction.py" "$ROOT" "$TARGET" hyperlex "${CHECK_ARGS[@]}" +run_transaction "$TARGET" if [[ $INSTALL_OPENCLAW -eq 1 ]]; then - mkdir -p "$(dirname "$OPENCLAW_TARGET")" - # reuse TARGET temporarily for openclaw path _saved="$TARGET" TARGET="$OPENCLAW_TARGET" validate_target - python3 "${ROOT}/scripts/install_transaction.py" "$ROOT" "$TARGET" hyperlex "${CHECK_ARGS[@]}" + run_transaction "$TARGET" TARGET="$_saved" fi @@ -264,6 +199,7 @@ echo "" echo "Next:" echo " export HERMES_SKILL_DIR=\"${TARGET}\"" echo " python3 \"\$HERMES_SKILL_DIR/scripts/hyperlex.py\" check" -echo " python3 \"\$HERMES_SKILL_DIR/scripts/hyperlex.py\" analyze --query \"sharp steam\" --source mock --forecasts" +echo " python3 \"\$HERMES_SKILL_DIR/scripts/hyperlex.py\" pipeline \"rizz\" --route offline" +echo " python3 \"\$HERMES_SKILL_DIR/scripts/hlx-mutation\" trace \"it's giving mid rizz\"" echo " # Reload Hermes skills if the agent is already running" echo "" diff --git a/mkdocs.yml b/mkdocs.yml index 79e6c35..c3dad2a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -113,4 +113,4 @@ exclude_docs: | README.md extra: - version: 0.4.1 + version: 0.4.0 diff --git a/pyproject.toml b/pyproject.toml index 40567a5..2ddb7bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "hyperlex" -version = "0.4.1" -description = "Standalone memetic emergence engine for slang detection, hyperstition, virality, lineage, and settled Brier calibration." +dynamic = ["version"] +description = "Standalone memetic emergence engine for slang detection, hyperstition, virality, lineage, settled Brier calibration, and slang mutation-operator detection." authors = [ {name = "Applied Alchemy Labs", email = "scrimshawlife@gmail.com"} ] @@ -47,16 +47,24 @@ schema = [ ] [project.urls] -Homepage = "https://github.com/scrimshawlife-ctrl/Hyperlex" -Repository = "https://github.com/scrimshawlife-ctrl/Hyperlex" -Documentation = "https://github.com/scrimshawlife-ctrl/Hyperlex#readme" -Issues = "https://github.com/scrimshawlife-ctrl/Hyperlex/issues" +Homepage = "https://github.com/Zero-State-LLC/Hyperlex" +Repository = "https://github.com/Zero-State-LLC/Hyperlex" +Documentation = "https://github.com/Zero-State-LLC/Hyperlex#readme" +Issues = "https://github.com/Zero-State-LLC/Hyperlex/issues" [project.scripts] -hyperlex = "hyperlex.cli:main" +hyperlex = "hyperlex.entrypoint:main" +hlx = "hyperlex.entrypoint:main" + +[tool.setuptools.dynamic] +version = {file = "VERSION"} [tool.setuptools.packages.find] where = ["src"] [tool.setuptools.package-data] -hyperlex = ["schemas/*.json", "schemas/**/*.json"] +hyperlex = [ + "schemas/*.json", + "data/*.json", + "data/*.md", +] diff --git a/references/hermes-runtime-contract.md b/references/hermes-runtime-contract.md index 65294a7..bffb78a 100644 --- a/references/hermes-runtime-contract.md +++ b/references/hermes-runtime-contract.md @@ -13,10 +13,6 @@ Hermes discovers skills by scanning `$HERMES_HOME/skills/**/SKILL.md`. This skill is a **directory skill** (not a single markdown file). -Hermes Agent v0.21 indexes `description` in the system prompt at 57 characters -plus `...`. Keep the field to one sentence of ≤60 characters that ends with a -period. Use `${HERMES_SKILL_DIR}` in command examples; Hermes substitutes it. - ## Paths ```text diff --git a/references/source-and-upgrades.md b/references/source-and-upgrades.md index cb8dc00..98adf6e 100644 --- a/references/source-and-upgrades.md +++ b/references/source-and-upgrades.md @@ -1,58 +1,65 @@ # Source identity and upgrades -This distribution is `Zero-State-LLC/Hyperlex`, version `0.4.1`, based on commit -`5eaae2dee88d80b27415bf362a87924fb9c41c2e` before the local audit-fix commit. Record the actual installed commit -with `git rev-parse HEAD` before installation; do not use a version string alone -as a source identity. Root and hub packaging are distribution surfaces, not a -claim that organizational, personal, or legacy embedded variants are identical. -No personal version is deprecated by this change. - -Before switching sources, stop runtime writers, record current source/commit, -review the destination under `${HERMES_HOME:-$HOME/.hermes}`, compare contracts, -and retain a separate backup of outputs, sessions, and local customization. -Do not install two different contracts with the same skill name into one profile. -A successful compatibility check does not grant deployment/publication authority. +This distribution is `scrimshawlife-ctrl/Hyperlex`. Source identity is the +`VERSION` file plus `git rev-parse HEAD` recorded at install time. Do not use a +version string alone. Do not pin a commit SHA in this document; the installed +receipt stores the checkout's HEAD when the source is its own repository root. + +Before switching sources, stop runtime writers, record current source and +commit, review the destination under `${HERMES_HOME:-$HOME/.hermes}`, compare +contracts, and retain a separate backup of outputs, sessions, and local +customization. Do not install two different contracts with the same skill name +into one profile. A successful compatibility check does not grant +deployment or publication authority. The bundled LICENSE controls this distribution. No license grant is changed by -these fixes; earlier grants and third-party notices remain intact. - -The installer validates a fresh stage on the target filesystem before replacement, -then reads back the activated contract/version/provenance receipt. Checks use an -isolated temporary home. Legacy `out` contents (including an out symlink) survive. -Backups are unique, keyed by canonical destination under the active Hermes home's -`backups///`; `.install-provenance.json` records source, base -commit, dirty status, version, destination, and skipped checks. Stop writers during -upgrades. Two renames have an absent-target window: this is NOT crash-atomic. -On a reported recovery failure, retain the printed recovery directory and backup; -do not delete it or retry blindly. Inspect the exact target and restore from the -named backup only after validating it. No automatic source migration is implied. +these installer fixes; earlier grants and third-party notices remain intact. + +The installer validates a fresh stage on the target filesystem before +replacement, then reads back the activated contract, version, and provenance +receipt. Checks use an isolated temporary home. Legacy `out` contents +(including an `out` symlink) survive. Backups are unique, keyed by canonical +destination under the active Hermes home at `backups/hyperlex//`. +`.install-provenance.json` records source, base commit, dirty status, version, +destination, and skipped checks. + +Stop writers during upgrades. The activation path is a two-rename swap +(displace the live target, then publish the stage). Those two renames have an +absent-target window: this is NOT crash-atomic. On a reported recovery failure, +retain the printed recovery directory and backup; do not delete it or retry +blindly. Inspect the exact target and restore from the named backup only after +validating it. No automatic source migration is implied. ## Interrupted installs and stale locks SIGKILL or power loss can leave `..install-lock` beside the target. -Locks are never automatically reclaimed: age, an empty directory, or a reused PID -cannot prove that no installer is active. To recover: +Locks are never automatically reclaimed: age, an empty directory, or a reused +PID cannot prove that no installer is active. To recover: 1. Stop install launchers and runtime writers. Confirm no installer is active on - this target (including other sessions/hosts sharing the filesystem). + this target (including other sessions or hosts sharing the filesystem). 2. Inspect the exact target, sibling `.-stage-*` recovery directories - (especially `previous` and `failed-package`), and target-keyed backups. Retain - all recovery data until the original installation and outputs are accounted - for. Restore/validate a complete package first if activation was interrupted. + (especially `previous` and `failed-package`), and target-keyed backups. + Retain all recovery data until the original installation and outputs are + accounted for. Restore and validate a complete package first if activation + was interrupted. 3. Only then set `lock` to the exact path printed in the error and run `rmdir -- "$lock"`. This removes only an empty lock; never use recursive - deletion or remove a lock whose owner/activity is uncertain. Keep launchers - stopped through inspection and removal to avoid races, then retry installation. + deletion or remove a lock whose owner or activity is uncertain. Keep + launchers stopped through inspection and removal to avoid races, then retry + installation. -Backup copies are staged in `.backup-incomplete-*` outside the target's selectable -backup directory and published by rename after copying and writing the destination -record. Hard termination can leave these incomplete staging trees; never select -one for rollback. Inspect them manually after quiescing writers. Rollback skips -legacy partial entries without a valid matching destination record. +Backup copies are staged in `.backup-incomplete-*` outside the target's +selectable backup directory and published by rename after copying and writing +the destination record. Hard termination can leave these incomplete staging +trees; never select one for rollback. Inspect them manually after quiescing +writers. Rollback skips legacy partial entries without a valid matching +destination record. Git is optional. Archive sources, failed Git lookups, and unrelated enclosing worktrees record unknown Git fields as JSON `null`, never a false clean claim. -A source-root worktree or the tracked `skills/neon-genie` hub with matching root -contract/version supplies Git provenance. Receipts distinguish -`source_repository_root` from `source_subdirectory` (`.` or `skills/neon-genie`). -A dirty status lookup failure remains `null` even in a recognized worktree. +A source-root worktree supplies Git provenance. Receipts distinguish +`source_repository_root` from `source_subdirectory` (`.` for an own checkout). +A dirty-status lookup failure remains `null` even in a recognized worktree. +This repository is Hyperlex only; there is no neon-genie or sigil-forge hub +subtree. diff --git a/schemas/mutation_trace.v0.1.schema.json b/schemas/mutation_trace.v0.1.schema.json new file mode 100644 index 0000000..0f35b21 --- /dev/null +++ b/schemas/mutation_trace.v0.1.schema.json @@ -0,0 +1,94 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://hyperlex.local/schemas/mutation_trace.v0.1.schema.json", + "title": "hyperlex.mutation_trace.v0.1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "packet_id", + "operators", + "layers_touched", + "class", + "restricted_intent_suspected", + "forecast_eligible", + "brier" + ], + "properties": { + "schema": { "const": "hyperlex.mutation_trace.v0.1" }, + "packet_id": { "type": "string", "minLength": 8 }, + "observed_at": { "type": ["string", "null"] }, + "source": { + "type": ["string", "null"], + "enum": ["mock", "glossary", "x_search", "reddit", "web", "human", "analyze", "cli", null] + }, + "surface_span": { "type": ["string", "null"] }, + "recovered_lemma": { "type": ["string", "null"] }, + "canonical_gloss": { "type": ["string", "null"] }, + "operators": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "SUBSTITUTE", + "AFFIX", + "CLIP_BLEND", + "EGGCORN", + "PHONETIC_WARP", + "GAME_ENCODE", + "REGISTER_SHIFT", + "CODE_SWITCH", + "FRAME_WRAP", + "COMPOSE" + ] + } + }, + "layers_touched": { + "type": "array", + "items": { + "type": "string", + "enum": ["L0", "L1", "L2", "L3", "L4", "L5", "L6", "L7", "L8"] + } + }, + "register_shift": { + "type": "string", + "enum": ["none", "low", "med", "high"] + }, + "irony_flag": { "type": "boolean" }, + "dissociation_flag": { "type": "boolean" }, + "algospeak_flag": { "type": "boolean" }, + "machine_dialect": { "type": "boolean" }, + "affix_family": { "type": ["string", "null"] }, + "decode_confidence": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "lexicon_hit": { "type": "boolean" }, + "watch_score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "class": { "enum": ["OBSERVED", "INFERRED", "SPECULATIVE"] }, + "restricted_intent_suspected": { "type": "boolean" }, + "payload_ref": { "type": ["string", "null"] }, + "forecast_eligible": { "const": false }, + "brier": { "type": "null" }, + "provenance": { + "type": "array", + "items": { "type": "string" } + }, + "receipt_id": { "type": ["string", "null"] }, + "ritualCharge": { "type": ["number", "null"] }, + "inGroupKeying": { "type": ["number", "null"] }, + "paraphraseResistance": { "type": ["number", "null"] }, + "attractorStability": { "type": ["number", "null"] }, + "disseminationImperative": { "type": ["number", "null"] }, + "falseStabilizationRisk": { "type": ["number", "null"] } + }, + "allOf": [ + { + "if": { "properties": { "restricted_intent_suspected": { "const": true } } }, + "then": { + "required": ["payload_ref"], + "properties": { + "surface_span": { "type": "null" }, + "canonical_gloss": { "type": "null" } + } + } + } + ] +} diff --git a/scripts/hlx-mutation b/scripts/hlx-mutation new file mode 100644 index 0000000..cf93146 --- /dev/null +++ b/scripts/hlx-mutation @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Skill-tree alias → package mutation noun. + +Hermes: python3 "$HERMES_SKILL_DIR/scripts/hlx-mutation" trace "it's giving mid rizz" +""" +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +SCRIPTS = Path(__file__).resolve().parent + +# scripts/hyperlex.py shadows the package if it sits first on sys.path. +sys.path = [p for p in sys.path if Path(p).resolve() != SCRIPTS] +sys.path.insert(0, str(SRC)) +sys.modules.pop("hyperlex", None) + +from hyperlex.cli import main # noqa: E402 + + +def _argv(raw: list[str]) -> list[str]: + if raw and raw[0] == "mutation": + return raw + return ["mutation", *raw] + + +if __name__ == "__main__": + raise SystemExit(main(_argv(sys.argv[1:]))) diff --git a/scripts/hyperlex.py b/scripts/hyperlex.py index 991605b..a20689c 100755 --- a/scripts/hyperlex.py +++ b/scripts/hyperlex.py @@ -11,7 +11,6 @@ import argparse import hashlib import json -import re import sys import traceback from dataclasses import dataclass @@ -149,202 +148,10 @@ def _load_manifest() -> Tuple[Dict[str, Any], bool, str]: return {}, False, f"manifest parse failed: {exc}" -_SKILL_DESC_LIMIT = 60 -_SKILL_REQUIRED_KEYS = ("name", "description", "version", "author", "license", "platforms") - - -def _coerce_yaml_scalar(raw: str) -> Any: - text = raw.strip() - if text in ("[]",): - return [] - if text.startswith("[") and text.endswith("]"): - inner = text[1:-1].strip() - if not inner: - return [] - return [_coerce_yaml_scalar(part) for part in inner.split(",")] - if (text.startswith('"') and text.endswith('"')) or (text.startswith("'") and text.endswith("'")): - return text[1:-1] - return text - - -def _parse_frontmatter_stdlib(block: str) -> Dict[str, Any]: - """Parse the SKILL.md frontmatter subset without PyYAML.""" - data: Dict[str, Any] = {} - hermes: Dict[str, Any] = {} - in_hermes = False - in_tags = False - tags: List[str] = [] - for raw_line in block.splitlines(): - if not raw_line.strip() or raw_line.strip().startswith("#"): - continue - stripped = raw_line.strip() - indent = len(raw_line) - len(raw_line.lstrip(" ")) - if indent == 0: - in_hermes = False - in_tags = False - if stripped.startswith("metadata:"): - data.setdefault("metadata", {}) - continue - if ":" in stripped: - key, _, val = stripped.partition(":") - data[key.strip()] = _coerce_yaml_scalar(val) - continue - if stripped == "hermes:": - in_hermes = True - in_tags = False - data.setdefault("metadata", {}) - data["metadata"]["hermes"] = hermes - continue - if in_hermes and stripped == "tags:": - in_tags = True - hermes["tags"] = tags - continue - if in_hermes and in_tags and stripped.startswith("- "): - tags.append(stripped[2:].strip().strip('"').strip("'")) - continue - if in_hermes and stripped.startswith("related_skills:"): - in_tags = False - hermes["related_skills"] = _coerce_yaml_scalar(stripped.split(":", 1)[1]) - continue - if in_hermes and ":" in stripped and not stripped.startswith("- "): - in_tags = False - key, _, val = stripped.partition(":") - if val.strip(): - hermes[key.strip()] = _coerce_yaml_scalar(val) - if tags and "tags" not in hermes: - hermes["tags"] = tags - if hermes: - data.setdefault("metadata", {}) - data["metadata"]["hermes"] = hermes - return data - - -def _load_skill_frontmatter() -> Tuple[Dict[str, Any], str]: - path = ROOT / "SKILL.md" - if not path.exists(): - return {}, "SKILL.md missing" - raw = path.read_bytes() - if not raw.startswith(b"---"): - return {}, "SKILL.md must start at byte 0 with ---" - text = raw.decode("utf-8") - match = re.search(r"\n---\n", text[3:]) - if match is None: - return {}, "SKILL.md frontmatter must close with \\n---\\n" - block = text[3 : 3 + match.start()] - try: - import yaml # type: ignore - - data = yaml.safe_load(block) - if not isinstance(data, dict): - return {}, "SKILL.md frontmatter is not a mapping" - return data, "ok" - except Exception: - data = _parse_frontmatter_stdlib(block) - if not data.get("name"): - return {}, "SKILL.md frontmatter parse failed" - return data, "ok (stdlib yaml fallback)" - - -def _hermes_meta(frontmatter: Dict[str, Any]) -> Dict[str, Any]: - metadata = frontmatter.get("metadata") - if not isinstance(metadata, dict): - return {} - hermes = metadata.get("hermes") - return hermes if isinstance(hermes, dict) else {} - - -def _skill_frontmatter_checks(frontmatter: Dict[str, Any], parse_msg: str) -> List[_Check]: - checks: List[_Check] = [] - parsed = bool(frontmatter) - checks.append(_check(parsed, "skill_frontmatter", parse_msg, parse_msg)) - if not parsed: - return checks - for key in _SKILL_REQUIRED_KEYS: - present = key in frontmatter and frontmatter.get(key) not in (None, "") - checks.append( - _check( - present, - f"skill_frontmatter.{key}", - f"frontmatter includes {key}", - f"frontmatter missing {key}", - ) - ) - description = frontmatter.get("description") - desc = description.strip() if isinstance(description, str) else "" - if desc in (">", "|"): - checks.append( - _check( - False, - "skill_description_len", - "", - "description is a folded YAML block; Hermes indexes ≤60 chars", - ) - ) - checks.append(_check(False, "skill_description_period", "", "folded description has no period")) - else: - checks.append( - _check( - 0 < len(desc) <= _SKILL_DESC_LIMIT, - "skill_description_len", - f"description {len(desc)} chars (≤{_SKILL_DESC_LIMIT})", - f"description {len(desc)} chars exceeds {_SKILL_DESC_LIMIT}", - ) - ) - checks.append( - _check( - desc.endswith("."), - "skill_description_period", - "description ends with a period", - "description must end with a period", - ) - ) - author = frontmatter.get("author") - author_text = author.strip() if isinstance(author, str) else "" - checks.append( - _check( - bool(author_text) and author_text != "Hermes Agent", - "skill_author_human", - "author credits the human first", - "author must credit the human first, then Hermes Agent", - ) - ) - hermes = _hermes_meta(frontmatter) - tags = hermes.get("tags") - checks.append( - _check( - isinstance(tags, list) and all(isinstance(t, str) and t.strip() for t in tags), - "skill_frontmatter.tags", - "metadata.hermes.tags present", - "metadata.hermes.tags missing", - ) - ) - checks.append( - _check( - "related_skills" in hermes and isinstance(hermes.get("related_skills"), list), - "skill_frontmatter.related_skills", - "metadata.hermes.related_skills present", - "metadata.hermes.related_skills missing", - ) - ) - expected = _read_version() - version = str(frontmatter.get("version") or "").strip() - checks.append( - _check( - version == expected, - "skill_frontmatter.version_match", - f"SKILL.md version {version} matches VERSION", - f"SKILL.md version {version!r} != VERSION {expected!r}", - ) - ) - return checks - - def cmd_check(_args: argparse.Namespace) -> int: checks: List[_Check] = [] checks.append(_check((ROOT / "VERSION").exists(), "version_file", "VERSION exists", "VERSION missing")) checks.append(_check((ROOT / "SKILL.md").exists(), "skill_contract", "SKILL.md exists", "SKILL.md missing")) - frontmatter, fm_msg = _load_skill_frontmatter() - checks.extend(_skill_frontmatter_checks(frontmatter, fm_msg)) checks.append(_check((ROOT / "schemas/ingest.v1.schema.json").exists(), "schema_ingest", "ingest schema present", "ingest schema missing")) checks.append(_check((ROOT / "schemas/result.v1.schema.json").exists(), "schema_result", "result schema present", "result schema missing")) checks.append(_check((ROOT / "schemas/receipt.v1.schema.json").exists(), "schema_receipt", "receipt schema present", "receipt schema missing")) diff --git a/scripts/install_transaction.py b/scripts/install_transaction.py old mode 100644 new mode 100755 index c11124e..fd30b30 --- a/scripts/install_transaction.py +++ b/scripts/install_transaction.py @@ -1,8 +1,9 @@ -"""Staged, checked upgrades with target-keyed backups and recovery. +"""Staged Hyperlex upgrades with target-keyed backups and recovery. The two renames have an absent-target window; this is not crash-atomic. Stop runtime writers during upgrades. A retained recovery directory requires operator -inspection. No live profile is used by validation subprocesses. +inspection. Locks are never automatically reclaimed. Validation subprocesses +use an isolated temporary home. This helper is Hyperlex-only. """ from __future__ import annotations @@ -18,10 +19,9 @@ import uuid from pathlib import Path +KIND = "hyperlex" CHECKS = { - "neon-genie": [("validate_hermes_skill.py",), ("neon_genie.py", "do", "check")], - "sigil-forge": [("validate_hermes_skill.py",), ("sigil_forge.py", "check")], - "hyperlex": [("hyperlex.py", "check"), ("hyperlex.py", "smoke")], + KIND: [("hyperlex.py", "check"), ("hyperlex.py", "smoke")], } IGNORE = shutil.ignore_patterns( ".git", @@ -45,14 +45,34 @@ ) -def install(source: Path, target: Path, kind: str, skip_checks: bool = False) -> None: - # Check the lexical final component before resolve follows a dangling link. - if any(p.is_symlink() for p in (target.absolute(), *target.absolute().parents)): - raise ValueError("refusing symlink target") - source, target = source.resolve(), target.resolve() +def _refuse_symlink_target(target: Path, message: str) -> None: + # Inspect the lexical path before resolve() follows a dangling link. + if any(part.is_symlink() for part in (target.absolute(), *target.absolute().parents)): + raise ValueError(message) + + +def _hermes_home() -> Path: + raw = os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes") + return Path(raw).resolve() + + +def _git(source: Path, *args: str) -> str | None: + try: + result = subprocess.run( + check=False, + args=["git", "-C", str(source), *args], + capture_output=True, + text=True, + env={k: v for k, v in os.environ.items() if not k.startswith("GIT_")}, + ) + except OSError: + return None # Git is optional for archive/Python-only installs. + return result.stdout.strip() if result.returncode == 0 else None + + +def _assert_layout(source: Path, target: Path, home: Path, kind: str) -> Path: if source == target or source in target.parents or target in source.parents: raise ValueError("source and target must not overlap") - home = Path(os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes")).resolve() if target in (Path("/"), Path.home().resolve(), home, home / "skills"): raise ValueError("refusing installation root") if target.exists() and not target.is_dir(): @@ -61,6 +81,10 @@ def install(source: Path, target: Path, kind: str, skip_checks: bool = False) -> backups = home / "backups" / kind / key if target == backups or target in backups.parents or backups in target.parents: raise ValueError("backup and target must not overlap") + return backups + + +def _assert_no_payload_symlinks(source: Path) -> None: # Reject payload links rather than shipping aliases into private source state. for directory, dirs, files in os.walk(source, followlinks=False): ignored = IGNORE(directory, dirs + files) @@ -68,7 +92,9 @@ def install(source: Path, target: Path, kind: str, skip_checks: bool = False) -> for name in dirs + [f for f in files if f not in ignored]: if (Path(directory) / name).is_symlink(): raise ValueError(f"source payload symlink is not portable: {name}") - target.parent.mkdir(parents=True, exist_ok=True) + + +def _acquire_lock(target: Path) -> Path: lock = target.parent / ("." + target.name + ".install-lock") try: lock.mkdir() # exclusive; never remove another installer's lock @@ -80,156 +106,157 @@ def install(source: Path, target: Path, kind: str, skip_checks: bool = False) -> "Only after resolving recovery, use rmdir on this exact empty lock " "(never recursive removal), then retry. See references/source-and-upgrades.md." ) from exc + return lock + + +def _check_env(workspace: Path, stage: Path) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith("HYPERLEX_")} + check_home = workspace / "check-home" + check_home.mkdir() + env.update( + HOME=str(check_home), + HERMES_HOME=str(check_home / ".hermes"), + HERMES_SKILL_DIR=str(stage), + PYTHONDONTWRITEBYTECODE="1", + ) + return env + + +def _run_checks(stage: Path, kind: str, env: dict[str, str], skip_checks: bool) -> None: + if skip_checks: + return + for command in CHECKS[kind]: + subprocess.run( + [sys.executable, str(stage / "scripts" / command[0]), *command[1:]], + cwd=env["HOME"], + env=env, + check=True, + ) + + +def _preserve_legacy_out(target: Path, stage: Path) -> None: + if (stage / "out").exists(): + shutil.rmtree(stage / "out") # NEW staging data only + old_out = target / "out" + if old_out.is_symlink(): + (stage / "out").symlink_to(os.readlink(old_out), target_is_directory=True) + elif old_out.exists(): + shutil.copytree(old_out, stage / "out", symlinks=True) + + +def _checkout_meta(source: Path) -> tuple[Path | None, str | None]: + top = _git(source, "rev-parse", "--show-toplevel") + repo_root = Path(top).resolve() if top is not None else None + if repo_root == source: + return repo_root, "." + return None, None + + +def _write_receipt( + source: Path, + target: Path, + stage: Path, + kind: str, + skip_checks: bool, +) -> dict[str, object]: + repo_root, subtree = _checkout_meta(source) + own_checkout = repo_root is not None + dirty = _git(source, "status", "--porcelain") if own_checkout else None + receipt: dict[str, object] = { + "source": str(source), + "source_repository_root": str(repo_root) if own_checkout else None, + "source_subdirectory": subtree, + "repository": _git(source, "remote", "get-url", "origin") if own_checkout else None, + "source_commit": _git(source, "rev-parse", "HEAD") if own_checkout else None, + "source_dirty": bool(dirty) if dirty is not None else None, + "version": (source / "VERSION").read_text().strip(), + "destination": str(target), + "status": "UNVERIFIED" if skip_checks else "VALIDATED", + "checks_skipped": list(CHECKS[kind]) if skip_checks else [], + "validation": "staged runtime; activated contract/version read-back", + } + (stage / ".install-provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + return receipt + + +def _publish_backup(target: Path, backups: Path) -> Path | None: + if not target.exists(): + return None + backups.mkdir(parents=True, exist_ok=True) + backup = backups / uuid.uuid4().hex + # Incomplete copies live outside the selectable backup namespace. + # Publish only after the payload and destination record are complete. + with tempfile.TemporaryDirectory(prefix=".backup-incomplete-", dir=backups.parent) as tmp: + pending = Path(tmp) / "backup-payload" + shutil.copytree(target, pending, symlinks=True) + marker = pending / ".backup-target.json" + marker.unlink(missing_ok=True) + marker.write_text(json.dumps({"destination": str(target)}) + "\n") + os.replace(pending, backup) + return backup + + +def _activate( + target: Path, + stage: Path, + expected: dict[str, bytes], + workspace: Path, + backup: Path | None, +) -> None: + displaced = workspace / "previous" + try: + if target.exists(): + os.replace(target, displaced) + os.replace(stage, target) + for relative, content in expected.items(): + if (target / relative).read_bytes() != content: + raise OSError(f"activation read-back mismatch: {relative}") + except BaseException as original: + try: + # Infer completed renames from disk even if replace raised after + # its side effect. A still-staged package means target is not new. + if not stage.exists() and target.exists(): + os.replace(target, workspace / "failed-package") + if displaced.exists(): + os.replace(displaced, target) + except BaseException as recovery_error: + raise RuntimeError( + f"recovery required at {workspace}; backup {backup}; " + f"activation: {original}; restoration: {recovery_error}" + ) from recovery_error + raise + + +def install(source: Path, target: Path, kind: str, skip_checks: bool = False) -> None: + if kind not in CHECKS: + raise ValueError(f"unsupported skill kind: {kind}") + _refuse_symlink_target(target, "refusing symlink target") + source, target = source.resolve(), target.resolve() + home = _hermes_home() + backups = _assert_layout(source, target, home, kind) + _assert_no_payload_symlinks(source) + target.parent.mkdir(parents=True, exist_ok=True) + lock = _acquire_lock(target) workspace = None retain_recovery = False try: - workspace = Path( - tempfile.mkdtemp(prefix="." + kind + "-stage-", dir=target.parent) - ) + workspace = Path(tempfile.mkdtemp(prefix="." + kind + "-stage-", dir=target.parent)) stage = workspace / "package" shutil.copytree(source, stage, ignore=IGNORE) - check_home = workspace / "check-home" - check_home.mkdir() - env = { - k: v - for k, v in os.environ.items() - if not k.startswith(("HYPERLEX_", "SIGIL_FORGE_")) - } - env.update( - HOME=str(check_home), - HERMES_HOME=str(check_home / ".hermes"), - HERMES_SKILL_DIR=str(stage), - SIGIL_FORGE_STATE_DIR=str(check_home / "state"), - PYTHONDONTWRITEBYTECODE="1", - ) if not skip_checks: - for command in CHECKS[kind]: - subprocess.run( - [sys.executable, str(stage / "scripts" / command[0]), *command[1:]], - cwd=check_home, - env=env, - check=True, - ) - if (stage / "out").exists(): - shutil.rmtree(stage / "out") # NEW staging data only - old_out = target / "out" - if old_out.is_symlink(): - (stage / "out").symlink_to(os.readlink(old_out), target_is_directory=True) - elif old_out.exists(): - shutil.copytree(old_out, stage / "out", symlinks=True) - - def git(*args: str) -> str | None: - try: - r = subprocess.run( - check=False, - args=["git", "-C", str(source), *args], - capture_output=True, - text=True, - env={ - k: v for k, v in os.environ.items() if not k.startswith("GIT_") - }, - ) - except OSError: - return None # Git is optional for archive/Python-only installs. - return r.stdout.strip() if r.returncode == 0 else None - - # Git searches parents: accept the source root or the tracked Neon hub, - # not arbitrary archives nested in an unrelated worktree. - top = git("rev-parse", "--show-toplevel") - repo_root = Path(top).resolve() if top is not None else None - own_checkout = repo_root == source - subtree = "." if own_checkout else None - if ( - repo_root is not None - and kind == "neon-genie" - and source == repo_root / "skills/neon-genie" - ): - tracked = git( - "ls-files", - "--error-unmatch", - "--", - "SKILL.md", - "VERSION", - str(repo_root / "SKILL.md"), - str(repo_root / "VERSION"), - ) - try: - # Root/hub grants can legitimately differ; compare the contract - # apart from its license metadata, without changing either grant. - root_contract = (repo_root / "SKILL.md").read_text().splitlines() - hub_contract = (source / "SKILL.md").read_text().splitlines() - matching_root = [ - line for line in root_contract if not line.startswith("license:") - ] == [ - line for line in hub_contract if not line.startswith("license:") - ] and (repo_root / "VERSION").read_bytes() == ( - source / "VERSION" - ).read_bytes() - except (OSError, UnicodeError): - matching_root = False - if tracked is not None and matching_root: - own_checkout = True - subtree = "skills/neon-genie" - dirty = git("status", "--porcelain") if own_checkout else None - receipt = { - "source": str(source), - "source_repository_root": str(repo_root) if own_checkout else None, - "source_subdirectory": subtree, - "repository": git("remote", "get-url", "origin") if own_checkout else None, - "source_commit": git("rev-parse", "HEAD") if own_checkout else None, - "source_dirty": bool(dirty) if dirty is not None else None, - "version": (source / "VERSION").read_text().strip(), - "destination": str(target), - "status": "UNVERIFIED" if skip_checks else "VALIDATED", - "checks_skipped": list(CHECKS[kind]) if skip_checks else [], - "validation": "staged runtime; activated contract/version read-back", - } - receipt_text = json.dumps(receipt, indent=2) + "\n" - (stage / ".install-provenance.json").write_text(receipt_text) + _run_checks(stage, kind, _check_env(workspace, stage), skip_checks) + _preserve_legacy_out(target, stage) + receipt = _write_receipt(source, target, stage, kind, skip_checks) expected = { - p: (stage / p).read_bytes() - for p in ("SKILL.md", "VERSION", ".install-provenance.json") + path: (stage / path).read_bytes() + for path in ("SKILL.md", "VERSION", ".install-provenance.json") } - backup = None - if target.exists(): - backups.mkdir(parents=True, exist_ok=True) - backup = backups / uuid.uuid4().hex - # Incomplete copies live outside the selectable backup namespace. - # Publish only after the payload and destination record are complete. - with tempfile.TemporaryDirectory( - prefix=".backup-incomplete-", dir=backups.parent - ) as tmp: - pending = Path(tmp) / "backup-payload" - shutil.copytree(target, pending, symlinks=True) - marker = pending / ".backup-target.json" - marker.unlink(missing_ok=True) - marker.write_text(json.dumps({"destination": str(target)}) + "\n") - os.replace(pending, backup) - displaced = workspace / "previous" - # Detect a concurrent change of target identity since staging began. - if any(p.is_symlink() for p in (target.absolute(), *target.absolute().parents)): - raise ValueError("target became a symlink during staging") + backup = _publish_backup(target, backups) + _refuse_symlink_target(target, "target became a symlink during staging") try: - if target.exists(): - os.replace(target, displaced) - os.replace(stage, target) - for relative, content in expected.items(): - if (target / relative).read_bytes() != content: - raise OSError(f"activation read-back mismatch: {relative}") - except BaseException as original: - try: - # Infer completed renames from disk even if replace raised after - # its side effect. A still-staged package means target is not new. - if not stage.exists() and target.exists(): - os.replace(target, workspace / "failed-package") - if displaced.exists(): - os.replace(displaced, target) - except BaseException as recovery_error: - retain_recovery = True - raise RuntimeError( - f"recovery required at {workspace}; backup {backup}; " - f"activation: {original}; restoration: {recovery_error}" - ) from recovery_error + _activate(target, stage, expected, workspace, backup) + except RuntimeError: + retain_recovery = True raise print(f"Installed {kind}: {target} ({receipt['status']})") if backup: @@ -241,10 +268,11 @@ def git(*args: str) -> str | None: def rollback(target: Path, kind: str, skip_checks: bool = False) -> None: - if any(p.is_symlink() for p in (target.absolute(), *target.absolute().parents)): - raise ValueError("refusing symlink target") + if kind not in CHECKS: + raise ValueError(f"unsupported skill kind: {kind}") + _refuse_symlink_target(target, "refusing symlink target") target = target.resolve() - home = Path(os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes")).resolve() + home = _hermes_home() key = hashlib.sha256(str(target).encode()).hexdigest()[:20] backups = home / "backups" / kind / key candidates = sorted( diff --git a/specs/000-hyperlex-spine/spec.md b/specs/000-hyperlex-spine/spec.md new file mode 100644 index 0000000..2853e66 --- /dev/null +++ b/specs/000-hyperlex-spine/spec.md @@ -0,0 +1,63 @@ +# Spec 000 — Hyperlex Spine (as-built, SDD lock) + +**Feature**: Hyperlex system specification +**Date**: 2026-09-04 +**Status**: SPECIFY (retroactive lock of v0.4.0 behavior) +**Constitution**: `.specify/memory/constitution.md` v1.0.0 +**Runtime version described**: 0.4.0 (main @ 3e66637) + +## Why this spec exists +Hyperlex already ships. It did not start as Spec Kit. This document is the constitution-gated *what/why* lock so later features (including 001 mutation grammar) evaluate against a single spine instead of drifting off README + STATUS. + +## User / operator intent +An operator needs a local, receipt-backed engine that: +1. Ingests a query from mock or live sources. +2. Detects neologisms, lineage family, virality, memetics, hyperstition stage. +3. Emits integrity-hashed receipts. +4. Extracts forecasts with `brier: null`. +5. Settles those forecasts under human authority and only then scores Brier series. +6. Optionally simulates cultural transmission (Phase 5, SPECULATIVE). +7. Optionally predicts next civilian surface forms (`mutation_prediction`, SPECULATIVE). +8. Never depends on Abraxas at import time. + +## In scope (current surface) +- Intake adapters + cache + rate limit (`mock` required). +- `detect_memetic_patterns` analysis contract: `observed` / `inferred` / `speculative`. +- `match_lineage` + confidence threshold 0.42 + `score_breakdown`. +- Receipts + hash-chained ledger. +- Forecast extract → settle → score-series. +- Phase 5 simulation packets (`hyperlex.phase5_scenario.v1`), brier null. +- Lineage backfill packs + non-mutating backprop. +- `analysis.mutation_prediction` (`hyperlex.mutation_prediction.v1`). +- Compat export under `hyperlex.compat.abraxas`. +- Hermes skill contract (`SKILL.md`) + CLI (`scripts/hyperlex.py`, `python -m hyperlex`). + +## Out of scope +- Public PyPI. +- Hard Abraxas / Forecast / HollerSports writes. +- Auto-settle, auto-cron, auto-registry promotion. +- Adversarial wrap generation (see spec 001). +- Smash-merge with Abraxas-v2.0 doctrine repo. + +## Requirements +- R1. Offline analyze MUST succeed without network. +- R2. Open analysis MUST set `provenance.brier` / block `brier` to null. +- R3. Settlement MUST require an authority marker (TRUE|FALSE|VOID). +- R4. Backprop MUST NOT rewrite receipt integrity. +- R5. API_V1 symbols MUST remain unchanged by additive work. +- R6. Claims MUST be labeled OBSERVED / INFERRED / SPECULATIVE. +- R7. Fail-closed on missing settlements (`NOT_COMPUTABLE`). +- R8. Fail-open on optional LLM / live ingest. + +## Existing mutation operators (lineage docs) +Documented in `docs/slang-lineages.md`: +extra-grammatical formation, sense extension, irony inversion, platform compression, eggcorn / folk etymology, cross-family borrowing, hyperstition loop. + +Prediction vocabulary in `hyperlex.mutation_prediction.v1`: +`platform_compression`, `derivational`, `irony_inversion`, `compound_phrase`, `sense_extension`, `cross_family_borrowing`, `extra-grammatical`. + +Spec 001 extends this vocabulary for *detection* without replacing prediction. + +## Success (spine) +- `python3 scripts/hyperlex.py doctor` and `smoke` green offline. +- This spec plus constitution are the SDD entry points for Hyperlex work after 2026-09-04. diff --git a/specs/001-mutation-grammar/checklist.md b/specs/001-mutation-grammar/checklist.md new file mode 100644 index 0000000..84bdba4 --- /dev/null +++ b/specs/001-mutation-grammar/checklist.md @@ -0,0 +1,29 @@ +# Checklist — Spec 001 + +## Spec quality +- [x] Intent and non-goals explicit +- [x] Clarify decisions locked (C1–C10) +- [x] Enum frozen at ten ops +- [x] Irony split (register vs frame) +- [x] Hyperstition excluded from parse enum +- [x] Predict/detect module wall +- [x] Packet schema file with restricted redaction `allOf` +- [x] Watch score not described as a probability +- [x] Watcher pair specified +- [x] OWASP 2026 map with honest limitation +- [x] Threat model with abuse cases +- [x] Civilian-only examples +- [x] No ASR / no wrap recipes +- [x] Agency prohibition (LLM03) +- [x] Hosts told to treat output as untrusted (LLM10) + +## Implement readiness +- [x] T1–T11 in tasks.md (on main) +- [x] Pytest offline (`tests/test_mutation_grammar.py`) +- [x] SKILL.md When to Use updated +- [ ] Operator review recorded as constitution **promotion** (code merge ≠ promotion) + +## Constitution +- [x] I–X checked against this spec +- [x] VII detector-over-generator holds +- [x] III settled-Brier-only holds diff --git a/specs/001-mutation-grammar/clarify.md b/specs/001-mutation-grammar/clarify.md new file mode 100644 index 0000000..6b6ccac --- /dev/null +++ b/specs/001-mutation-grammar/clarify.md @@ -0,0 +1,26 @@ +# Clarify lock — Spec 001 + +**Date:** 2026-09-04 +**Status:** LOCKED for v0.1 +**Gate:** `/speckit.clarify` complete. Implement may not reopen these without an amendment row. + +## Decisions + +| ID | Question | Decision | Rationale | +|----|----------|----------|-----------| +| C1 | Detector enum size | Keep the ten ops. Do not add COMPOUND, HYPERSTITION, or L0-only ORTHOGRAPHIC in v0.1. | Compound is generation or COMPOSE+SUBSTITUTE. Hyperstition is Phase 5 actualization, not a rewrite rule. L0 stays inside PHONETIC_WARP / platform_compression until a dedicated normalizer exists. | +| C2 | Irony | Do **not** merge into one bucket. REGISTER_SHIFT = speech-act / affect. FRAME_WRAP = why-said / lore / dissociation. `irony_flag` is a feature, not an operator. | Style jailbreaks and “just the lore” wraps are different mechanisms. Merging smears watch_score. | +| C3 | Hyperstition | Stay off the parse enum. Remain Phase 5 / `forecast_hyperstition_risk`. | Different object: narrative actualization vs surface rewrite. | +| C4 | Predict vs detect | `mutation_prediction.v1` stays civilian next-forms of slang atoms. `mutation_trace.v0.1` never calls predict on a span with `restricted_intent_suspected`. | Constitution VII. | +| C5 | v0.1 parse layers | L2 AFFIX, L3 SUBSTITUTE (+ eggcorn fixtures), L5 REGISTER_SHIFT heuristic, COMPOSE if ≥2. | Highest cultural signal, lowest dual-use. | +| C6 | Restricted storage | If flag true: persist `payload_ref` (SHA-256 of normalized surface) only. Drop `surface_span` and `canonical_gloss` from receipt body. | Prevents the detector from becoming a wrap archive. | +| C7 | Brier / forecast | `brier` null. `forecast_eligible` false. Watch scores are instrumentation, not probabilities of harm or adoption. | Constitution III. | +| C8 | OWASP posture | Map operators to OWASP GenAI LLM Top 10 **2026** as *coverage of detection surface*, not as a claim that Hyperlex is a guardrail product. | Honest scope. | +| C9 | Examples | Civilian / linguistic only in-repo. No restricted how-to strings. Cite papers by id. | Dual-use + reviewability. | +| C10 | Agency | Mutation-trace output MUST NOT trigger tools, relays that execute, or cron. Advisory envelopes only. | OWASP LLM03 Excessive Agency. | + +## Deferred (explicit non-locks) +- Formal SAE informal-register feature as `register_shift` source. +- Live multi-model eval of register-skewed refusal (research fork, not this package). +- COMPOUND as its own enum value. +- GAME_ENCODE / CODE_SWITCH parsers. diff --git a/specs/001-mutation-grammar/dual-use-gate.md b/specs/001-mutation-grammar/dual-use-gate.md new file mode 100644 index 0000000..d5c4757 --- /dev/null +++ b/specs/001-mutation-grammar/dual-use-gate.md @@ -0,0 +1,15 @@ +# T11 Dual-use merge gate + +Answered 2026-09-04. Required before merge to main. + +| # | Question | Answer | +|---|----------|--------| +| 1 | Can a test fixture be reused as a restricted wrap? | No. Civilian only: `rizz`, `unalive`, `it's giving`, productive suffixes, eggcorns. No restricted how-to strings in-repo. | +| 2 | Does any path call `predict_mutations` after restricted flag? | No. `tests/test_mutation_grammar.py::test_predict_not_used_on_restricted` asserts `grammar.py` does not contain that name. Predict stays on slang-atom seeds. | +| 3 | Does watch_score write Brier? | No. Schema and packet force `brier: null`, `forecast_eligible: false`. | +| 4 | Can watch_score fire tools / cron? | No. C10 / constitution. Advisory envelopes only. | +| 5 | Restricted persistence | `restricted_intent_suspected` → drop `surface_span` and `canonical_gloss`; keep SHA-256 `payload_ref`. | +| 6 | Host output handling | Treat packets as untrusted (OWASP LLM10:2026). | +| 7 | Claim vs control | Maps to LLM01 linguistic injection *surface*. Does not close prompt injection. | + +If any answer flips to yes, block merge and delete the fixture or path that caused it. diff --git a/specs/001-mutation-grammar/owasp-mapping.md b/specs/001-mutation-grammar/owasp-mapping.md new file mode 100644 index 0000000..16c13a5 --- /dev/null +++ b/specs/001-mutation-grammar/owasp-mapping.md @@ -0,0 +1,27 @@ +# OWASP / ATLAS mapping — Spec 001 + +Hyperlex mutation grammar is a **sociolinguistic instrumentation layer**. It is not a WAF, not a guardrail product, and not a red-team harness. + +Canonical list referenced: **OWASP GenAI LLM Top 10 2026** (published 2026-08-04). Historical 2025 ids are noted where useful. MITRE ATLAS: AML.T0054 LLM Jailbreak. + +## Coverage matrix + +| Framework entry | Relevance to slang mutation | What 001 does | What 001 refuses | +|---|---|---|---| +| LLM01:2026 Prompt Injection | Register, dialect, game-encode, frame wraps are *linguistic* injection variants: they change how the model binds intent without a literal “ignore previous instructions.” Direct and indirect injection both travel on mutated surfaces. | Detect operator stacks on attested text. Tag REGISTER_SHIFT / FRAME_WRAP / GAME_ENCODE / CODE_SWITCH when parsers exist. | No payload generation. No bypass recipes. | +| LLM02:2026 Sensitive Information Disclosure | Cant and in-group slang can smuggle sensitive topics past keyword DLP. | SUBSTITUTE + recovered_lemma (civilian tables). Restricted flag redacts stored surface. | No attempt to recover secrets from model output. | +| LLM03:2026 Excessive Agency | A high watch_score must not become an autonomous action. | C10: advisory only. No tool fire from trace. | No agent loop. | +| LLM04:2026 Supply Chain | Machine-generated slang / poisoned slang tables could enter LINEAGE_REGISTRY or fixtures. | `machine_dialect` tag. Fixtures are reviewed civilian lists. | No unattended registry promote. | +| LLM05:2026 Data and Model Poisoning | Narrow harmful finetunes and slang-poisoned corpora are out of Hyperlex runtime. | Provenance labels on packets. | No training-set construction. | +| LLM06:2026 Unbounded Consumption | Language-game / run-on surfaces can inflate tokens. | Optional `layers_touched` includes L4 when present. No decoder bomb tests in-repo. | No stress-generation of long game-encoded strings. | +| LLM07:2026 Misinformation | Irony and frame wraps change apparent claim force. | `irony_flag`, FRAME_WRAP distinct from REGISTER_SHIFT. | No truth scoring of world claims. | +| LLM08:2026 Hidden Context Exposure | Not a slang problem per se. | None. | Out of scope. | +| LLM09:2026 Vector and Embedding Weaknesses | Informal-register subspaces and slang near-duplicates can evade embedding filters. | Future: register feature. v0.1 heuristic only. | No embedding inversion. | +| LLM10:2026 Improper Output Handling | If a host pipes `mutation_trace` or `mutation_prediction` into a tool, that is the host's LLM10. | Packet is data. Hosts must treat it as untrusted structured output. | Hyperlex does not execute trace fields. | +| ATLAS AML.T0054 Jailbreak | Style, encoding, persona, linguistic variation are documented jailbreak techniques. | Cite as threat *class*. Instrument surface operators. | No jailbreak success metric, no ASR leaderboard in this package. | + +## 2025 crosswalk (for readers on the prior list) +LLM01 Prompt Injection stays the primary map. 2025 LLM05 Improper Output Handling ≈ 2026 LLM10. 2025 LLM06 Excessive Agency ≈ 2026 LLM03. Do not mix year ids in receipts; store `owasp_llm_top10_year: 2026` if a host adds a mapping field later (non-normative on v0.1 packet). + +## Honest limitation +Keyword and embedding filters fail on productive morphology and register. This spec measures that failure *mode* on cultural text. It does not claim to close LLM01. diff --git a/specs/001-mutation-grammar/plan.md b/specs/001-mutation-grammar/plan.md new file mode 100644 index 0000000..960fd0c --- /dev/null +++ b/specs/001-mutation-grammar/plan.md @@ -0,0 +1,36 @@ +# Plan 001 — Mutation grammar + +**Spec**: `spec.md` + `clarify.md` +**Lane**: SHADOW +**Implement**: not in this commit + +## Approach +Sibling package `hyperlex.mutation`. Do not fold detection into `analysis/mutation.py`. That file stays `predict_mutations`. + +## Layout +``` +src/hyperlex/mutation/ + __init__.py + operators.py + packet.py + grammar.py + watch.py + lineage_graph.py # stub ok in v0.1 +schemas/mutation_trace.v0.1.schema.json # already added +tests/test_mutation_grammar.py +``` + +Wire after lineage inside `detect_memetic_patterns`. Receipt emit runs redaction helper first. + +## Order +1. Packet + schema tests (restricted `allOf`). +2. L2/L3/L5 parser + COMPOSE. +3. watch_score + pair counters. +4. Analyze attachment + CLI. +5. Docs / SKILL / STATUS. +6. Operator review. + +## Dual-use review questions (merge gate) +1. Can a test fixture be reused as a restricted wrap? If yes, delete it. +2. Does any path call `predict_mutations` after restricted flag? If yes, block merge. +3. Does watch_score get written into a Brier field? If yes, block merge. diff --git a/specs/001-mutation-grammar/spec.md b/specs/001-mutation-grammar/spec.md new file mode 100644 index 0000000..0fe3af1 --- /dev/null +++ b/specs/001-mutation-grammar/spec.md @@ -0,0 +1,171 @@ +# Spec 001 — Adversarial Slang Mutation Grammar (first-class Hyperlex) + +**Feature**: Detector-side mutation grammar +**Date**: 2026-09-04 +**Status**: SPECIFY + CLARIFY locked / SHADOW +**Depends on**: spec 000, constitution v1.0.0 +**Clarify**: `clarify.md` (C1–C10 locked) +**Companion docs**: `threat-model.md`, `owasp-mapping.md`, `plan.md`, `tasks.md`, `checklist.md` +**Schema**: `schemas/mutation_trace.v0.1.schema.json` +**Notion distillation**: https://www.notion.so/3d23e8ba2f5c81a88e52c26942bc1fad + +## Intent +Slang is a productive grammar. Hyperlex already labels historical branches and predicts civilian next surfaces. This spec makes **operator-stack detection** first-class so the engine can see when the *rule* moves faster than the *lexicon*. + +This is instrumentation for cultural language and for *linguistic* prompt-injection surfaces (OWASP LLM01:2026). It is not a guardrail product and not a jailbreak kit. + +## Problem +1. Lineage `branch_operator` describes finished history. +2. `mutation_prediction.v1` proposes future forms of a slang atom. +3. Neither traces stacked rewrites on an attested span. +4. Neither separates lexicon-hit rate from novel-rule rate (Goodhart). +5. Safety literature shows style, dialect, language-games, and phonetic/code-mix change refusal behavior. Treating that as a word list is how patches age out. + +## Goals +- G1 Parse attested civilian text into a typed operator stack. +- G2 Persist a receipt-safe packet with frozen integrity rules. +- G3 Expose watch metrics that cannot be satisfied by memorizing last quarter's tokens. +- G4 Keep prediction and detection in separate modules with a hard wall on restricted spans. +- G5 Be reviewable by LLM application-security practitioners: OWASP map, threat model, dual-use bound, epistemic labels. + +## Non-goals +- N1 Generate restricted-intent paraphrases, language-game encodings of disallowed requests, or “working wraps.” +- N2 Report attack success rate (ASR) against any model. +- N3 Close LLM01. Detection ≠ mitigation. +- N4 Auto-execute tools, crons, or Abraxas runes from a watch_score (LLM03). +- N5 Rewrite `mutation.py` into an adversarial composer. +- N6 Put hyperstition loop on the detector enum (C3). +- N7 Merge REGISTER_SHIFT and FRAME_WRAP (C2). +- N8 Forecast / Brier on grammar packets (C7). +- N9 SUAS operational evasion guidance. +- N10 Live red-team corpora in this repository. + +## Users +- Hyperlex operator studying emergence (primary). +- Abraxas ECO / Companion consuming SIGNAL REPORT mutation traces (advisory). +- Reviewers who need a clean dual-use story. + +## In scope (v0.1) +See clarify C5. Schema, enum, packet, L2/L3/L5 parser, COMPOSE aggregator, watch_score, watcher pair, analyze attachment, CLI `mutation-trace`, civilian fixtures, restricted redaction rule, docs. + +## Out of scope (v0.1) +GAME_ENCODE parser, CODE_SWITCH parser, FRAME_WRAP parser beyond a boolean `dissociation_flag` heuristic if cheap, SAE steering, multi-model evals, COMPOUND enum, L0 normalizer productization. + +## Operator taxonomy (normative) +Ten detector ops, locked (C1). + +| Op | Layer | Detect v0.1 | Generate (predict module) | Notes | +|----|-------|-------------|---------------------------|-------| +| SUBSTITUTE | L3 | family / algospeak table | no | cant/algospeak synonym | +| AFFIX | L2 | suffix table | derivational (civilian atom only) | productive morphology | +| CLIP_BLEND | L2 | no | platform_compression adjacent | later | +| EGGCORN | L3 | fixture list | no | folk reanalysis | +| PHONETIC_WARP | L1 | no | vowel-drop on slang atoms only | later for detect | +| GAME_ENCODE | L4 | no | **never** | detect-only when shipped | +| REGISTER_SHIFT | L5 | heuristic | irony templates on slang atoms | not FRAME_WRAP | +| CODE_SWITCH | L6 | no | **never** as generator | detect-only when shipped | +| FRAME_WRAP | L7 | flag only | **never** as generator | distinct from register | +| COMPOSE | ≥2 ops | yes | n/a | aggregator | + +Irony is a **feature** (`irony_flag`), not an op. Hyperstition stays in Phase 5. + +Each op is conceptually `(layer, polarity, arity, productivity, dual_use)`. Polarity DETECT vs GENERATE is enforced by module boundary, not by comment. + +## Packet (normative) +Schema id: `hyperlex.mutation_trace.v0.1` +File: `schemas/mutation_trace.v0.1.schema.json` + +Hard constants: +- `forecast_eligible` = false +- `brier` = null +- if `restricted_intent_suspected`: `payload_ref` required; `surface_span` and `canonical_gloss` null in persisted form + +`class` rules: threat-model epistemic section. + +Sacred overlays optional and non-Brier: ritualCharge, inGroupKeying, paraphraseResistance, attractorStability, disseminationImperative, falseStabilizationRisk. + +## Watch score (normative sketch) +Not a probability. INFERRED instrumentation. + +``` +watch = clip01( + 0.35 * decode_confidence + + 0.15 * min(n_ops, 4) / 4 + + 0.20 * register_w # none=0 low=0.33 med=0.66 high=1 + + 0.10 * irony_flag + + 0.15 * affix_productivity # 1 if suffix in productive family table + + 0.15 * (0 if lexicon_hit and n_ops==1 else 1) +) +``` + +Do not interpret watch_score as P(jailbreak) or P(adoption). + +## Watcher pair (Goodhart) +- **A** lexicon_hit_rate on known family terms + civilian algospeak table over a window. +- **B** novel_operator_rate: new affix-stem pairs, new eggcorn fixtures needed, unseen game-rule markers. + +A↑ B flat → overfitting yesterday's words. +B↑ A flat → grammar moving. That is the instrumentation analog of a moving 0day. It is not an exploit claim. + +## Functional requirements +- F1 Civilian affix or table hit → packet with ops + layers. +- F2 Offline parse in v0.1. +- F3 No numeric Brier on the block or packet. +- F4 Must not call `predict_mutations` on restricted-flagged spans. +- F5 Analyze MAY attach `analysis.mutation_trace`; omit if no ops. +- F6 Restricted redaction as C6. +- F7 Tests: benign fixtures only (`rizz`, `unalive`, `it's giving`, `-maxxing`, classic eggcorns). +- F8 Predict module unchanged in role. +- F9 Schema validates; restricted-true fixtures fail if surface persisted. +- F10 CLI prints JSON; exit 0 on empty operators with empty packet or omit. +- F11 No network in unit tests. +- F12 Hosts documented: treat JSON as untrusted output (LLM10). + +## Analyze attachment shape +```json +"analysis": { + "mutation_trace": { + "schema": "hyperlex.mutation_trace.v0.1", + "operators": ["REGISTER_SHIFT", "AFFIX", "COMPOSE"], + "layers_touched": ["L5", "L2"], + "brier": null, + "forecast_eligible": false, + "class": "INFERRED" + } +} +``` +Full packet fields allowed. Restricted redaction applies before receipt emit. + +## Acceptance examples (civilian) +Input: `it's giving mid rizz` +Expect: REGISTER_SHIFT and/or AFFIX/SUBSTITUTE family hit on `rizz`; COMPOSE if ≥2; `brier` null; `forecast_eligible` false. + +Input: `unalive` +Expect: SUBSTITUTE or algospeak_flag; lexicon_hit likely true. + +Input: empty / ordinary prose without slang +Expect: omit block or empty operators. + +## Risks and mitigations +| Risk | Mitigation | +|------|------------| +| Dual-use generator creep | C4, F4, N1, module split | +| Receipt cookbook | C6 | +| False OBSERVED on heuristics | epistemic rules | +| Watch Goodhart | pair A/B | +| Agency bleed | C10 | +| Scope cosplay as guardrail | owasp-mapping honest limitation | + +## Success +Clarify C1–C10 recorded. Schema in repo. Checklist pass. Operator review of constitution VII–X. Implement still a later cycle. + +## References (provenance, not payloads) +- OWASP GenAI LLM Top 10 2026 (LLM01 primary; LLM03 agency; LLM10 output handling) +- MITRE ATLAS AML.T0054 +- arXiv:2511.10519 linguistic style jailbreaks +- arXiv:2604.21152 dialect vs demographics / dialect jailbreak +- arXiv:2411.12762 language games +- arXiv:2505.14226 phonetic code-mix +- arXiv:2405.00718 dark jargon +- arXiv:2603.26236 informal-register SAE +- Hyperlex `docs/slang-lineages.md`, `src/hyperlex/analysis/mutation.py` diff --git a/specs/001-mutation-grammar/tasks.md b/specs/001-mutation-grammar/tasks.md new file mode 100644 index 0000000..e515819 --- /dev/null +++ b/specs/001-mutation-grammar/tasks.md @@ -0,0 +1,15 @@ +# Tasks 001 — Mutation grammar + +- [x] T1 Schema file +- [x] T2 Operator enum + maps +- [x] T3 Packet dataclass; forecast_eligible false; brier null +- [x] T4 Parser v0.1 +- [x] T5 watch_score + tests +- [x] T6 Redaction helper +- [x] T7 Fail-open attach via signal_report +- [x] T8 Package CLI `mutation trace|predict` +- [x] T9 Civilian fixtures +- [x] T10 Docs deltas +- [x] T11 Dual-use merge-gate — `dual-use-gate.md` + PR body + +Leftover (not merge-blocking): `scripts/hyperlex.py` noun alias. Package CLI is the Hermes path. diff --git a/specs/001-mutation-grammar/threat-model.md b/specs/001-mutation-grammar/threat-model.md new file mode 100644 index 0000000..778fe27 --- /dev/null +++ b/specs/001-mutation-grammar/threat-model.md @@ -0,0 +1,36 @@ +# Threat model — Spec 001 + +## Assets +- Lineage registry and family suffix tables. +- Receipt ledger integrity. +- Operator trust in OBSERVED / INFERRED / SPECULATIVE labels. +- Downstream hosts that might treat Hyperlex JSON as authoritative. + +## Actors +- Honest operator studying slang emergence. +- Careless host that pipes trace output into an agent tool (LLM03 / LLM10). +- Adversary who wants a wrap archive or a generator. +- Supply-chain actor who poisons fixtures or LLM enrich. + +## Trust boundaries +1. Raw ingest text — untrusted. +2. Parser — trusted to be deterministic and offline in v0.1; not trusted to judge harm. +3. Restricted-flag heuristic — INFERRED, fail-open toward redaction if unsure in later versions; v0.1 defaults false unless a closed civilian deny-lemma list hits after recovery. +4. Receipt store — trusted for integrity hashes; not a payload vault. +5. Abraxas / other hosts — untrusted from Hyperlex's view. Export only. No import. + +## Abuse cases (defensive statements) +- **A1 Wrap factory:** attacker asks predict/trace to mint restricted paraphrases. Control: F4, C4, C9. Predict only on slang atoms from family/seed path. +- **A2 Archive reconstruction:** receipts become a cookbook. Control: C6 hash-only. +- **A3 Metric gaming:** lexicon-only watcher looks “green.” Control: pair A/B. +- **A4 Agency bleed:** high watch_score auto-relays a scan against a live model. Control: C10. +- **A5 Fixture poison:** malicious “civilian” list. Control: human review, provenance on fixture packs. +- **A6 Label laundering:** SPECULATIVE stacks reported as OBSERVED. Control: class field rules below. + +## Epistemic rules for `class` +- OBSERVED: surface span literally present in input; operator is a closed-list affix or exact lexicon hit. +- INFERRED: register heuristic, decode_confidence, watch_score, recovered lemma via phonetics. +- SPECULATIVE: any link to predicted next-forms, Phase 5, hyperstition risk. + +## Residual risk +v0.1 heuristics will under-detect GAME_ENCODE and CODE_SWITCH. That is accepted. Shipping a weak game parser that emits false OBSERVED is worse than omitting the parser. diff --git a/specs/001-mutation-grammar/ux-commands.md b/specs/001-mutation-grammar/ux-commands.md new file mode 100644 index 0000000..bc052aa --- /dev/null +++ b/specs/001-mutation-grammar/ux-commands.md @@ -0,0 +1,8 @@ +# 001 addendum — command + agent UX + +See spec **002** for the normative command surface. + +v0.1 implement already has `python -m hyperlex.mutation`. +T8 on the original task list becomes: wire `$HLX mutation trace` + deprecated `mutation-trace` alias. Do not ship a third spelling as the primary. + +Hermes agent default: read `analysis.mutation_trace` off `pipeline` output. Only call the noun CLI when the user asked for operators in isolation or for next-forms. diff --git a/specs/002-command-surface/spec.md b/specs/002-command-surface/spec.md new file mode 100644 index 0000000..5bc4b6d --- /dev/null +++ b/specs/002-command-surface/spec.md @@ -0,0 +1,55 @@ +# Spec 002 — Hermes command surface (usability) + +**Date:** 2026-09-04 +**Status:** SPECIFY locked / implement SHADOW +**Depends on:** 000, 001, constitution v1.0.0 +**Problem:** Hyperlex already has too many top-level verbs. Mutation grammar must not add a second island (`mutation-predict` + `mutation-trace` + `python -m`). + +## Intent +One noun. Few verbs. Pipeline is the default. Agents learn four daily commands and a research namespace. Mutation detect/predict live under that namespace. Dual-use wall stays in the verb, not in a footnote. + +## Command architecture (normative) + +### Layers +1. **Daily** — `wizard`, `pipeline`/`run`/`ingest`, `analyze`, `pending`, `settle`, `score-series`, `doctor`/`check`/`smoke`. +2. **Research noun** — `mutation`, `simulate`, `vector`, `lineage`, `archive`. +3. **Meta** — `commands`, `sources`, `help`. + +Do not add new layer-1 verbs for mutation. + +### Mutation noun (normative verbs) +```text +$HLX mutation [text] + + trace detect operators on attested text (001 detector) + predict next civilian surfaces of a slang atom (existing) + watch print watcher A/B sketch for a window (003) +``` + +### Hermes invocation (v0.1 SHADOW) +Prefer package CLI after install: + +```bash +PYTHONPATH="$HERMES_SKILL_DIR/src" python3 -m hyperlex mutation trace "…" +python3 "$HERMES_SKILL_DIR/scripts/hlx-mutation" trace "…" +``` + +`$HLX mutation` on `scripts/hyperlex.py` remains T5. + +### Pipeline attach +`pipeline` / `analyze` / `run` attach `analysis.mutation_trace` when operators fire. Omit when empty. + +### Agent routing +| User says | Hermes runs | +|-----------|-------------| +| scan / analyze / pipeline | `$HLX pipeline` and read `mutation_trace` | +| operators in this sentence | mutation trace | +| next forms of rizz | mutation predict | +| jailbreak / wrap | refuse generator | +| settle / Brier | unchanged | + +## Non-goals +- New Hermes skill +- Auto cron from watch_score +- Wizard step for mutation +- Merging predict and trace into one flagged verb diff --git a/specs/002-command-surface/tasks.md b/specs/002-command-surface/tasks.md new file mode 100644 index 0000000..38378d2 --- /dev/null +++ b/specs/002-command-surface/tasks.md @@ -0,0 +1,11 @@ +# Tasks 002 — command surface + +- [x] T1 Specify noun+verb (`mutation trace|predict`) +- [x] T2 SKILL.md triggers + when-not +- [x] T3 Package CLI `python -m hyperlex mutation` / `hyperlex mutation` +- [x] T4 Thin skill alias `scripts/hlx-mutation` +- [ ] T5 Wire `scripts/hyperlex.py` `mutation` noun (fat CLI). Not blocking. Use package CLI or alias. +- [x] T6 Router table `docs/command-router.v1.json` + `hyperlex.command_router` +- [x] T7 Manifest commands/intents include mutation +- [x] T9 `hyperlex init` graft-style skill wire +- [x] T8 `--human` card (v0.2) — owned by `specs/003-mutation-detect-v02/` diff --git a/specs/003-mutation-detect-v02/dual-use-gate.md b/specs/003-mutation-detect-v02/dual-use-gate.md new file mode 100644 index 0000000..0005a24 --- /dev/null +++ b/specs/003-mutation-detect-v02/dual-use-gate.md @@ -0,0 +1,18 @@ +# Dual-use addendum — Spec 003 (detect-only L4 / L6) + +Answers 2026-09-05. Required before merge to main. Extends `specs/001-mutation-grammar/dual-use-gate.md`; does not reopen it. + +| # | Question | Answer | +|---|----------|--------| +| 1 | Can a v0.2 fixture be reused as a restricted wrap? | No. Civilian only: vowel-dropped slang (`rzz`, `brnrt`), leet of known slang (`r1zz`), bilingual particle + slang (`el rizz`), mixed-script slang (`rizz 리즈`), game-frame + slang (`rizz in fortnite`). No restricted how-to strings. | +| 2 | Does GAME_ENCODE generate encodings? | No. Detect-only. Closed leet-translate *into* `SUBSTITUTE_TERMS`. No encode API, no wrap verb, no language-game composer. | +| 3 | Does CODE_SWITCH generate mixed-language attacks? | No. Detect-only. Mixed Unicode scripts or closed particle + lexicon hit. No paraphrase generator. | +| 4 | Does any path call `predict_mutations` after restricted flag? | No. Detector still wall-tested against that name. Predict stays on slang-atom seeds. | +| 5 | Does watch_score write Brier? | No. Packet and watch jsonl force `brier: null`, `forecast_eligible: false`. | +| 6 | Can watch_score or watch jsonl fire tools / cron / runes? | No. Records set `auto_fire: false`. Writer is fail-open. Advisory only (C10 / constitution III, VII, X). | +| 7 | Restricted persistence | Unchanged: flag true → drop `surface_span` / `canonical_gloss`; keep SHA-256 `payload_ref`. | +| 8 | Host output handling | JSON packets **and** `--human` cards are untrusted structured output (OWASP LLM10:2026). | +| 9 | Claim vs control | Still maps to LLM01 *linguistic* injection surface. Does not close prompt injection. Does not report ASR. | +| 10 | New verbs | `trace` and `watch` only. No `wrap` / `compose` / `asr` verbs. COMPOSE remains a detector enum tag. | + +If any answer flips to a generator or agency yes, block merge and delete the fixture or path that caused it. diff --git a/specs/003-mutation-detect-v02/plan.md b/specs/003-mutation-detect-v02/plan.md new file mode 100644 index 0000000..6372344 --- /dev/null +++ b/specs/003-mutation-detect-v02/plan.md @@ -0,0 +1,49 @@ +# Plan 003 — Mutation detect v0.2 + +**Spec**: `spec.md` +**Lane**: SHADOW / advisory +**Constitution**: stays SHADOW (do not amend) + +## Approach +Keep `hyperlex.mutation` as the detector package. Do not fold v0.2 into `analysis/mutation.py` (that file stays `predict_mutations`). Invert `_vowel_drop` for PHONETIC_WARP detect; do not call predict from the parser. + +## Layout +``` +specs/003-mutation-detect-v02/ + spec.md + plan.md + tasks.md + dual-use-gate.md +src/hyperlex/mutation/ + detect.py # GAME_ENCODE / CODE_SWITCH / PHONETIC_WARP heuristics + grammar.py # wire detect after v0.1 ops, before COMPOSE + operators.py # closed marker / particle tables + watch.py # watch_score + jsonl writer/reader + human.py # --human card + __init__.py +src/hyperlex/cli.py # trace --human --watch-jsonl; mutation watch +tests/test_mutation_v02.py +``` + +`scripts/hlx-mutation` already forwards argv to the package CLI — no second parser. + +## Order +1. Spec artifacts (this folder). +2. Parsers + civilian fixture tests. +3. Watch jsonl + tests (fail-open, auto_fire false). +4. `--human` card + tests. +5. CLI + router + SKILL.md (SHADOW advisory). +6. Verify; new PR. + +## Dual-use review questions (merge gate) +1. Can a test fixture be reused as a restricted wrap? If yes, delete it. +2. Does any path generate a language-game or code-switched restricted paraphrase? If yes, block merge. +3. Does watch_score write Brier or fire a tool/cron/rune? If yes, block merge. +4. Did we add wrap / compose / asr verbs? If yes, block merge. + +## Non-touches +- `.specify/memory/constitution.md` +- Installer / `tests_audit/` +- Spec 001 T1–T11 redo +- Issue / PR #5 +- FRAME_WRAP expansion, SAE, COMPOUND diff --git a/specs/003-mutation-detect-v02/spec.md b/specs/003-mutation-detect-v02/spec.md new file mode 100644 index 0000000..a45a3ed --- /dev/null +++ b/specs/003-mutation-detect-v02/spec.md @@ -0,0 +1,168 @@ +# Spec 003 — Mutation detect v0.2 (SHADOW) + +**Feature**: Detect-only parsers + watch jsonl + civilian `--human` cards +**Date**: 2026-09-05 +**Status**: SPECIFY locked / implement SHADOW +**Depends on**: spec 000, 001 (C1–C10), 002 noun surface, constitution v1.0.0 +**Lane**: SHADOW / advisory +**Companion**: `plan.md`, `tasks.md`, `dual-use-gate.md` +**Packet schema**: still `hyperlex.mutation_trace.v0.1` (no bump) + +## Intent +v0.1 traces L2/L3/L5 + COMPOSE. v0.2 wires the three deferred **detector** ops that the enum already names: **GAME_ENCODE**, **CODE_SWITCH**, **PHONETIC_WARP**. It also lands append-only watch instrumentation and a civilian card render. + +This is still instrumentation. It is not a guardrail, not a generator, and not a forecast class. + +## Problem +1. Enum lists GAME_ENCODE / CODE_SWITCH / PHONETIC_WARP but `parse_mutation_trace` never fires them. +2. `watch_score` exists in-memory only — no append-only log for the Goodhart pair. +3. Operators reading JSON traces have no civilian card. + +## Goals +- G1 Detect language-game *markers* and leet-of-known-slang on attested civilian text (GAME_ENCODE / L4). +- G2 Detect mixed-script spans and closed bilingual particles next to slang (CODE_SWITCH / L6). +- G3 Detect vowel-drop / platform-compression of closed slang atoms (PHONETIC_WARP / L1). +- G4 Append watch records to jsonl (instrumentation, not probabilities). +- G5 Render `--human` advisory cards from a trace packet. +- G6 Keep packets `brier: null` and `forecast_eligible: false`. Never treat `watch_score` as Brier or as a tool-fire / cron / rune threshold. + +## Non-goals +- N1 Generate wraps, language-game encodings of restricted requests, code-switched jailbreaks, or ASR boards. +- N2 Add `wrap` / `compose` / `asr` **verbs**. COMPOSE remains a detector enum tag only. +- N3 FRAME_WRAP parser beyond the existing `dissociation_flag`. +- N4 SAE steering, multi-model evals, COMPOUND enum. +- N5 Constitution promotion off SHADOW. +- N6 Schema bump, installer rewrite, fat `scripts/hyperlex.py` mutation noun (002 T5), reopen #5. +- N7 Auto-execute tools, cron, or Abraxas runes from watch_score or watch jsonl. + +## Users +- Hyperlex operator reading stacks on attested slang (primary). +- Reviewers who need a dual-use story for L4/L6 detect. +- Downstream hosts that must treat cards and JSON as untrusted output (LLM10). + +## In scope (v0.2) +1. GAME_ENCODE parser (detect-only). +2. CODE_SWITCH parser (detect-only). +3. PHONETIC_WARP parser (detect-only; vowel-drop on slang atoms). +4. Mutation watch jsonl (append-only; fail-open). +5. `--human` cards (render from traces). +6. Smallest CLI: `trace --human` / `trace --watch-jsonl [path]` plus reserved `mutation watch` reader. +7. Civilian fixtures + dual-use addendum. + +## Out of scope +Installer audit, Spec 001 T1–T11 redo, package-CLI rewrite, #5, FRAME_WRAP expansion, generator polarity for any new op, constitution amendment. + +## Operator polarity (normative) + +| Op | Layer | Detect v0.2 | Generate | Notes | +|----|-------|-------------|----------|-------| +| GAME_ENCODE | L4 | yes | **never** | Closed civilian game markers and/or leet-decode of `SUBSTITUTE_TERMS`. | +| CODE_SWITCH | L6 | yes | **never** | Mixed Unicode scripts, or closed particle + slang lexicon hit. | +| PHONETIC_WARP | L1 | yes | no (predict already vowel-drops civilian atoms) | Inverse of predict `_vowel_drop` on closed slang atoms only. | +| COMPOSE | ≥2 ops | yes | n/a | Aggregator tag only. Not a CLI verb. | + +Irony stays a feature. FRAME_WRAP stays flag-only. Hyperstition stays Phase 5. + +## Detect heuristics (normative sketch) + +### PHONETIC_WARP +Build a closed map: vowel-drop(`SUBSTITUTE_TERMS` atom) → atom, using the same keep-first-and-last vowel rule as `analysis.mutation._vowel_drop`. Fire when a token equals a warped form and is **not** the full atom. Recovered lemma is the atom. Epistemic: recovered lemma INFERRED; operator OBSERVED when the warped token is literally present. + +Do not fire PHONETIC_WARP on the full lexicon form (`rizz` is SUBSTITUTE, not warp). + +### GAME_ENCODE +Fire when either: +1. A token contains a digit/symbol, leet-translates to a `SUBSTITUTE_TERMS` atom, and is not already that atom (`r1zz` → rizz), or +2. A closed civilian **game-frame** phrase is present (`in fortnite`, `in minecraft`, `pig latin`, …) **and** the span already has a slang lexicon hit or another detector op. + +Never emit a reconstructed restricted payload. Recovered lemma, if any, comes only from the civilian slang table. + +### CODE_SWITCH +Fire when either: +1. Alphabetic characters in the span use two or more Unicode scripts (LATIN + CYRILLIC/HANGUL/CJK/…), or +2. A closed bilingual particle (`que`, `el`, `très`, `c'est`, …) appears as a whole token **and** a slang lexicon hit is present in the same span. + +Do not treat common English tokens as particles. Do not generate mixed-language paraphrases. + +## Packet (unchanged hard constants) +- `schema` = `hyperlex.mutation_trace.v0.1` +- `forecast_eligible` = false +- `brier` = null +- Restricted redaction (001 C6) unchanged +- Provenance stamp may read `hyperlex.mutation.grammar.v0.2` + +`watch_score` remains the 001 formula. New ops change `n_ops` only. It is **not** a probability, **not** Brier, **not** a fire threshold. + +## Watch jsonl (normative) +Default path: `~/.hyperlex/mutation_watch.jsonl` +Override: `HYPERLEX_MUTATION_WATCH` or `--watch-jsonl PATH`. + +Each line is one JSON object: +- `schema`: `hyperlex.mutation_watch.v0.2` +- `logged_at`, `packet_id`, `operators`, `layers_touched`, `watch_score`, `n_ops`, `lexicon_hit` +- `brier`: null +- `forecast_eligible`: false +- `auto_fire`: false (constant) + +Writer is fail-open: I/O errors must not fail parse or CLI. Reader skips blank/corrupt lines. The log is instrumentation. Hosts MUST NOT wire `watch_score` or line count to tools, cron, or runes. + +`mutation watch` reads the log and prints a JSON summary (last N records + cheap A/B rates). It does not execute anything. + +## `--human` cards (normative) +Civilian text render of an existing packet. Must state: +- operators and layers +- that watch_score is instrumentation, not P(harm) and not a fire threshold +- Brier null / forecast-eligible no +- SHADOW / advisory lane +- restricted-redaction note when the flag is set (no surface reprint) + +Default CLI remains JSON. `--human` prints the card instead of JSON. + +## Functional requirements +- F1 Civilian fixtures fire the three new ops as specified. +- F2 v0.1 fixtures still fire L2/L3/L5 + COMPOSE; `it's giving mid rizz` does not gain PHONETIC_WARP. +- F3 Offline; no network in unit tests. +- F4 No path calls `predict_mutations` from the detector after restricted flag (001 F4 still holds). +- F5 Packets always `brier: null`, `forecast_eligible: false`. +- F6 Watch append never auto-fires tools; `auto_fire` is always false. +- F7 Restricted redaction unchanged. +- F8 `scripts/hlx-mutation` remains a thin pass-through (new flags work without a second parser). +- F9 No new layer-1 verbs. No wrap/compose/asr verbs. +- F10 Hosts treat JSON and cards as untrusted (LLM10). + +## Acceptance examples (civilian only) + +| Input | Expect | +|-------|--------| +| `rzz` | PHONETIC_WARP; recovered `rizz`; brier null | +| `it's giving rzz` | REGISTER_SHIFT + PHONETIC_WARP (+ COMPOSE); no GAME_ENCODE required | +| `r1zz` | GAME_ENCODE; recovered `rizz`; brier null | +| `el rizz` | CODE_SWITCH + SUBSTITUTE | +| `rizz 리즈` | CODE_SWITCH + SUBSTITUTE | +| `it's giving mid rizz` | v0.1 unchanged: REGISTER_SHIFT, SUBSTITUTE, COMPOSE; brier null; forecast_eligible false | +| empty / ordinary prose | empty operators or omit | + +## Risks and mitigations + +| Risk | Mitigation | +|------|------------| +| GAME_ENCODE / CODE_SWITCH used as a generator | Module polarity + dual-use gate + no encode/wrap verbs | +| Game-frame false OBSERVED on ordinary “in minecraft” | Marker requires slang context | +| Watch jsonl becomes a fire switch | `auto_fire: false`; tests forbid threshold-to-tool wiring | +| Card overclaims harm | Card copy forbids P(jailbreak) language | +| Fixture cookbook | Civilian slang only; restricted redaction | + +## Success +- Separate spec folder shipped. +- Three ops fire on fixtures. +- Watch jsonl appends and reads fail-open. +- `--human` cards render from traces. +- Constitution remains SHADOW. +- New PR, not a reopen of #5 or an installer redo. + +## References (provenance, not payloads) +- Spec 001 clarify C1, C4–C7, C9–C10; deferred GAME_ENCODE / CODE_SWITCH / PHONETIC_WARP +- OWASP GenAI LLM Top 10 2026 (LLM01 surface, LLM03 agency, LLM10 output) +- arXiv:2411.12762 language games +- arXiv:2505.14226 phonetic code-mix +- Hyperlex `src/hyperlex/analysis/mutation.py` `_vowel_drop` (predict polarity; detect inverts it) diff --git a/specs/003-mutation-detect-v02/tasks.md b/specs/003-mutation-detect-v02/tasks.md new file mode 100644 index 0000000..2236854 --- /dev/null +++ b/specs/003-mutation-detect-v02/tasks.md @@ -0,0 +1,13 @@ +# Tasks 003 — Mutation detect v0.2 + +- [x] T1 Spec artifacts (`spec.md`, `plan.md`, `tasks.md`, `dual-use-gate.md`) +- [x] T2 PHONETIC_WARP detect (vowel-drop invert of closed slang atoms) + fixtures +- [x] T3 GAME_ENCODE detect (leet-of-slang + game-frame with slang context) + fixtures +- [x] T4 CODE_SWITCH detect (mixed script / particle+slang) + fixtures +- [x] T5 Watch jsonl writer/reader (fail-open, `brier` null, `auto_fire` false) + tests +- [x] T6 `--human` card formatter + tests +- [x] T7 Package CLI: `trace --human` / `trace --watch-jsonl` / `mutation watch`; hlx-mutation pass-through +- [x] T8 SKILL.md + command-router research row (SHADOW advisory) +- [x] T9 Dual-use merge-gate answers recorded; v0.1 regression fixtures still green + +Leftover (not merge-blocking): fat `scripts/hyperlex.py` mutation noun (002 T5). FRAME_WRAP parser. diff --git a/specs/README.md b/specs/README.md new file mode 100644 index 0000000..75159b8 --- /dev/null +++ b/specs/README.md @@ -0,0 +1,18 @@ +# Hyperlex Spec Kit + +Constitution: `.specify/memory/constitution.md` (v1.0.0, **SHADOW** until operator promotion) + +| ID | Title | SDD status | Runtime on main | +|----|-------|------------|-----------------| +| 000 | Hyperlex spine (as-built v0.4.0) | SPECIFY locked | shipping | +| 001 | Mutation grammar detector | CLARIFY locked · implement on main · not CONVERGED | `hyperlex.mutation` + tests | +| 002 | Hermes command surface | SPECIFY locked · implement partial | package CLI + `hlx-mutation` + `hyperlex init` | +| 003 | Mutation detect v0.2 | SPECIFY locked · implement SHADOW | GAME_ENCODE / CODE_SWITCH / PHONETIC_WARP + watch jsonl + `--human` | + +001 extras: `clarify.md`, `threat-model.md`, `owasp-mapping.md`, `checklist.md`, `dual-use-gate.md`, `ux-commands.md`, `schemas/mutation_trace.v0.1.schema.json` + +003 extras: `dual-use-gate.md` (L4/L6 detect-only addendum). Packet schema stays `hyperlex.mutation_trace.v0.1`. Constitution stays SHADOW. + +Gate: `specs/runtime-ready.md` + +Merged PRs that landed the gate: #6 detector, #7–9 skill alias, #10 wheel, #11 graft-style init. diff --git a/specs/runtime-ready.md b/specs/runtime-ready.md new file mode 100644 index 0000000..5dbc6a6 --- /dev/null +++ b/specs/runtime-ready.md @@ -0,0 +1,48 @@ +# Runtime-ready gate — mutation grammar v0.1 + +**Date:** 2026-09-04 +**Lane:** SHADOW / advisory +**Code:** on `main` +**Constitution:** still SHADOW (not promoted) + +## Meaning of runtime-ready + +After `pip install` + `hyperlex init` (or legacy `bash install.sh`) and agent reload: + +1. Route slang measurement to Hyperlex (not ECO generate). +2. Run `hyperlex pipeline "…" --route offline` and read `analysis.mutation_trace` when operators fire. +3. Run isolated `hyperlex mutation trace|predict` as JSON. +4. Keep `brier: null` and `forecast_eligible: false` on grammar packets. +5. Refuse wrap / jailbreak composition. + +It does **not** mean GAME_ENCODE parsers, watch jsonl, `--human` cards, constitution promotion, or fat-CLI T5. + +## Bar vs tree + +| Item | On main | +|------|--------| +| constitution I–X | yes, SHADOW | +| 001 detector + tests + dual-use gate | yes | +| package CLI noun | yes | +| `scripts/hlx-mutation` | yes | +| `hyperlex init` skill wire | yes | +| manifest mutation intents | yes | +| fat `scripts/hyperlex.py mutation` | **no** (use package / alias) | +| result.v1 explicit attach field | **no** (additionalProperties) | +| 000 / 001 CONVERGE stamp | **no** | + +## Deferred (v0.2) — specified in `specs/003-mutation-detect-v02/` + +- GAME_ENCODE / CODE_SWITCH / PHONETIC_WARP parsers +- `mutation watch` + `~/.hyperlex/mutation_watch.jsonl` +- `--human` card helper +- Constitution promotion off SHADOW (still **not** in 003) + +## Integrate + +```bash +pip install -e ".[dev]" +hyperlex init --target hermes +hyperlex pipeline "it's giving mid rizz" --route offline +hyperlex mutation trace "it's giving mid rizz" +``` diff --git a/src/hyperlex/__init__.py b/src/hyperlex/__init__.py index f096b44..07ea0c5 100644 --- a/src/hyperlex/__init__.py +++ b/src/hyperlex/__init__.py @@ -14,10 +14,9 @@ def _read_version() -> str: - candidate = Path(__file__).resolve().parents[2] / "VERSION" - if candidate.exists(): - return candidate.read_text(encoding="utf-8").strip() or "0.1.0" - return "0.1.0" + from ._version import read_version + + return read_version() PKG_VERSION = _read_version() @@ -38,6 +37,7 @@ def _read_version() -> str: predict_mutations, ) from .analysis.backfill import apply_backfill, inventory_backfill, list_backfill_packs +from .mutation import parse_mutation_trace from .analysis.backprop import backpropagate_lineage from .analysis.terms import split_seed_terms, collect_lexicon, per_term_lineage from .pipeline import run_pipeline, run_one @@ -155,6 +155,7 @@ def _read_version() -> str: "write_diagram_bundle", "predict_virality", "predict_mutations", + "parse_mutation_trace", "export_analysis_archive", "export_run_history", "rebuild_archive_catalog", @@ -209,6 +210,7 @@ def _read_version() -> str: "compute_virality_score", "predict_virality", "predict_mutations", + "parse_mutation_trace", "simulate_hyperstition_loop", "detect_neologisms", "trace_semantic_variation", diff --git a/src/hyperlex/__main__.py b/src/hyperlex/__main__.py index d688fb5..f5b0736 100644 --- a/src/hyperlex/__main__.py +++ b/src/hyperlex/__main__.py @@ -1,6 +1,6 @@ -"""python -m hyperlex → package CLI.""" +"""python -m hyperlex → package CLI (includes init).""" -from .cli import main +from .entrypoint import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/hyperlex/_version.py b/src/hyperlex/_version.py new file mode 100644 index 0000000..3d54c63 --- /dev/null +++ b/src/hyperlex/_version.py @@ -0,0 +1,18 @@ +"""Version resolution that works in a wheel and a git checkout.""" +from __future__ import annotations + +from pathlib import Path + + +def read_version() -> str: + try: + from importlib.metadata import version as pkg_version + + return pkg_version("hyperlex") + except Exception: + pass + here = Path(__file__).resolve() + for candidate in (here.parents[2] / "VERSION", here.parent / "VERSION"): + if candidate.is_file(): + return candidate.read_text(encoding="utf-8").strip() or "0.4.0" + return "0.4.0" diff --git a/src/hyperlex/analysis/mutation_trace_attach.py b/src/hyperlex/analysis/mutation_trace_attach.py new file mode 100644 index 0000000..6b32214 --- /dev/null +++ b/src/hyperlex/analysis/mutation_trace_attach.py @@ -0,0 +1,22 @@ +"""Fail-open attach of mutation_trace onto an analysis dict.""" +from __future__ import annotations + +from typing import Any, Dict, Optional + + +def attach_mutation_trace( + analysis: Dict[str, Any], + *, + query: str = "", + observed: str = "", + ingest_source: Optional[str] = None, +) -> None: + try: + from ..mutation import parse_mutation_trace + + span = " ".join(x for x in [query, observed] if x).strip() or str(query or "") + mt = parse_mutation_trace(str(span), source=str(ingest_source or "analyze")) + if mt.get("operators"): + analysis["mutation_trace"] = mt + except Exception: + return diff --git a/src/hyperlex/analysis/signal_report.py b/src/hyperlex/analysis/signal_report.py index c5531ff..a7e3f7a 100644 --- a/src/hyperlex/analysis/signal_report.py +++ b/src/hyperlex/analysis/signal_report.py @@ -78,18 +78,15 @@ def build_typology_tags( if primary and primary not in ("one_off", ""): tags.append(primary) - # Secondary typology scores above a soft threshold for tid, sc in sorted(scores.items(), key=lambda kv: -float(kv[1] or 0)): if tid == primary: continue if float(sc or 0) >= 0.45 and tid not in tags: tags.append(str(tid)) - # Lineage family as soft tag if fam and fam not in tags: tags.append(f"lineage:{fam}") - # Semantic-variation drivers already used by Hyperlex for d in drivers: if d and d not in tags: tags.append(str(d)) @@ -104,7 +101,6 @@ def build_typology_tags( return tags[:8] -# Back-compat name used by older call sites / schema field symbolic_role def build_symbolic_roles(*args: Any, **kwargs: Any) -> List[str]: return build_typology_tags(*args, **kwargs) @@ -202,6 +198,15 @@ def attach_signal_report_fields( neos = analysis.get("neologisms") or [] mp = analysis.get("mutation_prediction") + if "mutation_trace" not in analysis: + from .mutation_trace_attach import attach_mutation_trace + attach_mutation_trace( + analysis, + query=str(observed or ""), + observed="", + ingest_source=str(ingest_source or "analyze"), + ) + analysis["compression_metrics"] = build_compression_metrics( virality=virality, memetic=memetic, @@ -209,8 +214,6 @@ def attach_signal_report_fields( n_neologisms=len(neos), hyper_stage=hyper.get("loop_stage"), ) - # Field name remains symbolic_role in schema for stability; - # values are typology-aligned tags. analysis["symbolic_role"] = build_typology_tags( memetic=memetic, lineage=lineage, @@ -252,7 +255,6 @@ def build_integrity_header( stage = str(hyper_stage or "").upper() elevated = stage in ("ACTUALIZING", "EMERGENT") risk = "elevated" if elevated else "baseline" - # Schema: provenance.seed.status enum PASS|PARTIAL|FAIL (result.v1) status = "PARTIAL" if elevated else "PASS" return { "status": status, @@ -260,7 +262,6 @@ def build_integrity_header( "provenance": f"ingest={ingest_source or 'unknown'};stage={stage or 'none'}", "entropy_class": "high" if elevated else "medium", "capability_lane": "advisory", - # Extra diagnostic fields (schema allows additionalProperties on seed) "method": "rule_based_lineage_vector", "ingest_source": ingest_source or "unknown", "hyperstition_stage": stage or "none", @@ -269,6 +270,5 @@ def build_integrity_header( } -# Back-compat alias used by detect_memetic_patterns def build_seed_header(*args: Any, **kwargs: Any) -> Dict[str, str]: return build_integrity_header(*args, **kwargs) diff --git a/src/hyperlex/cli.py b/src/hyperlex/cli.py index 1f5048f..f494ab3 100644 --- a/src/hyperlex/cli.py +++ b/src/hyperlex/cli.py @@ -122,12 +122,105 @@ def cmd_commands(_: argparse.Namespace) -> int: "scan --route offline --receipt --forecasts --append-log", "inbox list", ], + "research": [ + 'mutation trace ""', + 'mutation trace "" --human', + "mutation predict ", + "mutation watch", + ], "routes": ["offline", "mock", "default", "live", "glossary", "social"], "docs": "docs/operator-loop.md · docs/commands.md", }) return 0 +def cmd_mutation_trace(args: argparse.Namespace) -> int: + from hyperlex.mutation import append_watch, format_human_card, parse_mutation_trace + + parts = getattr(args, "query", None) or getattr(args, "text", None) or "" + if isinstance(parts, list): + seed = " ".join(str(x) for x in parts) + else: + seed = str(parts) + seed = seed.strip() + if not seed: + _emit({"ok": False, "error": "empty query", "brier": None, "forecast_eligible": False}) + return 2 + out = parse_mutation_trace( + seed, + source="cli", + restricted_intent_suspected=bool(getattr(args, "restricted", False)), + ) + watch_flag = getattr(args, "watch_jsonl", None) + if watch_flag is not None: + watch_path = watch_flag if watch_flag else None + append_watch(out, path=watch_path) + if getattr(args, "human", False): + print(format_human_card(out), end="") + return 0 + _emit({"ok": True, "command": "mutation-trace", **out}) + return 0 + + +def cmd_mutation_watch(args: argparse.Namespace) -> int: + from hyperlex.mutation import watch_summary + + path = getattr(args, "path", None) or None + limit = int(getattr(args, "limit", 20) or 20) + _emit(watch_summary(path, limit=limit)) + return 0 + + +def cmd_mutation_predict(args: argparse.Namespace) -> int: + from hyperlex.analysis import LINEAGE_REGISTRY, match_lineage, predict_mutations + + seed = (getattr(args, "query", None) or getattr(args, "term", None) or "") + if isinstance(seed, list): + seed = " ".join(str(x) for x in seed) + seed = str(seed).strip() + if not seed: + _emit({"ok": False, "error": "empty query", "brier": None}) + return 2 + family_id = (getattr(args, "family", None) or "").strip() or None + family_terms: List[str] = [] + family_operator = None + if not family_id: + lin = match_lineage(seed) + if lin: + family_id = lin.get("family_id") + family_operator = lin.get("branch_operator") + if family_id: + for entry in LINEAGE_REGISTRY: + if entry.get("family_id") == family_id: + family_terms = list(entry.get("terms") or []) + family_operator = family_operator or entry.get("branch_operator") + break + out = predict_mutations( + seed, + family_id=family_id, + family_terms=family_terms, + family_operator=family_operator, + ) + _emit({"ok": True, "command": "mutation-predict", **out}) + return 0 + + +def cmd_mutation(args: argparse.Namespace) -> int: + verb = getattr(args, "mutation_verb", None) + if verb == "trace": + return cmd_mutation_trace(args) + if verb == "predict": + return cmd_mutation_predict(args) + if verb == "watch": + return cmd_mutation_watch(args) + _emit({ + "ok": False, + "error": "usage: mutation trace|predict|watch", + "verbs": ["trace", "predict", "watch"], + }) + return 2 + + def cmd_relay(args: argparse.Namespace) -> int: from hyperlex import list_runes, relay_from_result, extract_forecasts, relay_forecasts @@ -367,6 +460,45 @@ def build_parser() -> argparse.ArgumentParser: cmds = sub.add_parser("commands", help="Simplified command map") cmds.set_defaults(func=cmd_commands) + mut = sub.add_parser("mutation", help="trace (detect) | predict (civilian next-forms) | watch") + mut_sub = mut.add_subparsers(dest="mutation_verb") + mut_tr = mut_sub.add_parser("trace", help="Detect operator stacks") + mut_tr.add_argument("query", nargs="*", default=[]) + mut_tr.add_argument("--restricted", action="store_true") + mut_tr.add_argument("--human", action="store_true", help="Civilian advisory card (not JSON)") + mut_tr.add_argument( + "--watch-jsonl", + nargs="?", + const="", + default=None, + metavar="PATH", + help="Append instrumentation record (default ~/.hyperlex/mutation_watch.jsonl)", + ) + mut_tr.set_defaults(func=cmd_mutation_trace) + mut_pr = mut_sub.add_parser("predict", help="Civilian next-forms") + mut_pr.add_argument("query", nargs="?", default="") + mut_pr.add_argument("--term", default="") + mut_pr.add_argument("--family", default="") + mut_pr.set_defaults(func=cmd_mutation_predict) + mut_w = mut_sub.add_parser("watch", help="Read watch jsonl (advisory; never auto-fires)") + mut_w.add_argument("--path", default="", help="Watch jsonl path") + mut_w.add_argument("--limit", type=int, default=20) + mut_w.set_defaults(func=cmd_mutation_watch) + mut.set_defaults(func=cmd_mutation) + + mt_alias = sub.add_parser("mutation-trace", help="DEPRECATED alias of mutation trace") + mt_alias.add_argument("query", nargs="*", default=[]) + mt_alias.add_argument("--restricted", action="store_true") + mt_alias.add_argument("--human", action="store_true") + mt_alias.add_argument("--watch-jsonl", nargs="?", const="", default=None, metavar="PATH") + mt_alias.set_defaults(func=cmd_mutation_trace) + + mp_alias = sub.add_parser("mutation-predict", help="DEPRECATED alias of mutation predict") + mp_alias.add_argument("query", nargs="?", default="") + mp_alias.add_argument("--term", default="") + mp_alias.add_argument("--family", default="") + mp_alias.set_defaults(func=cmd_mutation_predict) + a = sub.add_parser("analyze") a.add_argument("query_pos", nargs="?", default="") a.add_argument("--query", default="") diff --git a/src/hyperlex/command_router.py b/src/hyperlex/command_router.py new file mode 100644 index 0000000..3bebae4 --- /dev/null +++ b/src/hyperlex/command_router.py @@ -0,0 +1,32 @@ +"""Load the static Hermes command router table.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict + + +def router_path() -> Path: + packaged = Path(__file__).resolve().parent / "data" / "command-router.v1.json" + if packaged.is_file(): + return packaged + return Path(__file__).resolve().parents[2] / "docs" / "command-router.v1.json" + + +def load_router() -> Dict[str, Any]: + path = router_path() + if not path.is_file(): + return { + "daily": [], + "research": [], + "invoke": { + "pipeline": "hyperlex pipeline", + "mutation": "hyperlex mutation", + }, + "note": "router file missing", + } + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return {"daily": [], "research": [], "note": "router root must be object"} + data.pop("schema", None) + return data diff --git a/src/hyperlex/data/SKILL.agent.md b/src/hyperlex/data/SKILL.agent.md new file mode 100644 index 0000000..3fe447c --- /dev/null +++ b/src/hyperlex/data/SKILL.agent.md @@ -0,0 +1,49 @@ +--- +name: hyperlex +description: Use when the user wants slang, memetics, lineage, mutation trace, algospeak operators, Brier settlement, or cultural-signal receipts. Not for jailbreak or wrap composition. +version: 0.4.0 +metadata: + hermes: + tags: [Memetics, Slang, Mutation, Brier, Receipts] + category: analysis +--- + + +# Hyperlex + +Installed as a CLI (`pip install hyperlex` then `hyperlex init`). +Call the binary on PATH. Do not clone the repo into the skill tree. + +## When to Use + +- slang / lineage / hyperstition / virality +- mutation operators on attested text +- receipts, pending forecasts, settlement, Brier series + +## When Not to Use + +- jailbreak / wrap composition +- inventing a numeric Brier on open analysis + +## Commands + +```bash +hyperlex pipeline "" --route offline +hyperlex mutation trace "" +hyperlex mutation trace "" --human +hyperlex mutation watch +hyperlex mutation predict +hyperlex pending +hyperlex settle --forecast-id --decision TRUE|FALSE|VOID +hyperlex score-series +hyperlex commands +``` + +`pipeline` attaches `analysis.mutation_trace` when operators fire. +`--human` prints a SHADOW advisory card. `watch` reads jsonl instrumentation (never auto-fires). +Mutation packets keep `brier: null` and `forecast_eligible: false`. + +## Authority + +May analyze, detect, extract forecasts, write receipts. +May not invent Brier, auto-settle, generate restricted wraps, or treat watch_score as a tool trigger. diff --git a/src/hyperlex/entrypoint.py b/src/hyperlex/entrypoint.py new file mode 100644 index 0000000..fd230c5 --- /dev/null +++ b/src/hyperlex/entrypoint.py @@ -0,0 +1,20 @@ +"""Console-script router: init/uninstall-skill vs operator CLI.""" +from __future__ import annotations + +import sys +from typing import List, Optional + + +def main(argv: Optional[List[str]] = None) -> int: + raw = list(sys.argv[1:] if argv is None else argv) + if raw and raw[0] in {"init", "uninstall-skill"}: + from hyperlex.init_skill import dispatch + + return dispatch(raw) + from hyperlex.cli import main as cli_main + + return cli_main(raw) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/hyperlex/init_skill.py b/src/hyperlex/init_skill.py new file mode 100644 index 0000000..fd53341 --- /dev/null +++ b/src/hyperlex/init_skill.py @@ -0,0 +1,108 @@ +"""Wire Hyperlex into agent skill trees the way graft init does. + +pip install hyperlex +hyperlex init +# writes a thin SKILL.md that tells the agent to call `hyperlex` on PATH +""" +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path +from typing import Dict, List + + +TARGETS = { + "hermes": Path.home() / ".hermes" / "skills" / "hyperlex", + "openclaw": Path.home() / ".openclaw" / "skills" / "hyperlex", + "grok": Path.home() / ".grok" / "skills" / "hyperlex", + "claude": Path.home() / ".claude" / "skills" / "hyperlex", +} + +MARKER = "" + + +def skill_template() -> str: + packaged = Path(__file__).resolve().parent / "data" / "SKILL.agent.md" + if packaged.is_file(): + return packaged.read_text(encoding="utf-8") + return _FALLBACK_SKILL + + +_FALLBACK_SKILL = """--- +name: hyperlex +description: Use when the user wants slang, memetics, lineage, mutation trace, Brier settlement, or cultural-signal receipts. Not for jailbreak wraps. +version: 0.4.0 +--- + + +# Hyperlex + +CLI is on PATH after `pip install hyperlex`. Run the command. Do not copy the repo. + +```bash +hyperlex pipeline \"\" --route offline +hyperlex mutation trace \"\" +hyperlex mutation predict +hyperlex pending +hyperlex settle --forecast-id --decision TRUE +hyperlex commands +``` + +Never invent Brier. Mutation packets are forecast_eligible false. +""" + + +def _write_skill(dest: Path, *, dry_run: bool) -> str: + dest.mkdir(parents=True, exist_ok=True) + path = dest / "SKILL.md" + body = skill_template() + if not dry_run: + path.write_text(body, encoding="utf-8") + return str(path) + + +def run_init(names: List[str], *, dry_run: bool) -> Dict: + written = [] + for name in names: + dest = TARGETS[name] + written.append({"target": name, "path": _write_skill(dest, dry_run=dry_run)}) + return {"ok": True, "command": "init", "dry_run": dry_run, "written": written} + + +def run_uninstall(names: List[str], *, dry_run: bool) -> Dict: + removed = [] + for name in names: + dest = TARGETS[name] + skill = dest / "SKILL.md" + if skill.is_file() and MARKER in skill.read_text(encoding="utf-8", errors="ignore"): + if not dry_run: + skill.unlink() + if dest.exists() and not any(dest.iterdir()): + dest.rmdir() + removed.append(str(skill)) + return {"ok": True, "command": "uninstall-skill", "dry_run": dry_run, "removed": removed} + + +def dispatch(argv: List[str]) -> int: + p = argparse.ArgumentParser(prog="hyperlex") + sub = p.add_subparsers(dest="cmd", required=True) + ini = sub.add_parser("init") + ini.add_argument("--target", action="append", choices=sorted(TARGETS), dest="targets") + ini.add_argument("--all", action="store_true") + ini.add_argument("--dry-run", action="store_true") + un = sub.add_parser("uninstall-skill") + un.add_argument("--target", action="append", choices=sorted(TARGETS), dest="targets") + un.add_argument("--all", action="store_true") + un.add_argument("--dry-run", action="store_true") + args = p.parse_args(argv) + names = list(args.targets or []) + if args.all or not names: + names = ["hermes", "openclaw", "grok"] + if args.cmd == "init": + out = run_init(names, dry_run=bool(args.dry_run)) + else: + out = run_uninstall(names, dry_run=bool(args.dry_run)) + print(json.dumps(out, indent=2, sort_keys=True)) + return 0 diff --git a/src/hyperlex/mutation/__init__.py b/src/hyperlex/mutation/__init__.py new file mode 100644 index 0000000..61ca580 --- /dev/null +++ b/src/hyperlex/mutation/__init__.py @@ -0,0 +1,24 @@ +"""Detector-side slang mutation grammar (spec 001 + 003). + +Separate from ``hyperlex.analysis.mutation.predict_mutations``. +Never generates restricted wraps. Packets are never forecast-eligible. +""" +from .grammar import parse_mutation_trace +from .human import format_human_card +from .packet import MutationTracePacket, redact_packet, packet_id_for +from .watch import append_watch, read_watch_log, watch_score, watch_summary +from .operators import DETECTOR_OPS, LAYER_FOR_OP + +__all__ = [ + "parse_mutation_trace", + "format_human_card", + "MutationTracePacket", + "redact_packet", + "packet_id_for", + "watch_score", + "append_watch", + "read_watch_log", + "watch_summary", + "DETECTOR_OPS", + "LAYER_FOR_OP", +] diff --git a/src/hyperlex/mutation/__main__.py b/src/hyperlex/mutation/__main__.py new file mode 100644 index 0000000..b42829e --- /dev/null +++ b/src/hyperlex/mutation/__main__.py @@ -0,0 +1,32 @@ +"""CLI: python -m hyperlex.mutation [text] [--restricted]""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import List, Optional + +from .grammar import parse_mutation_trace + + +def main(argv: Optional[List[str]] = None) -> int: + p = argparse.ArgumentParser(prog="python -m hyperlex.mutation") + p.add_argument("text", nargs="+", help="attested civilian text") + p.add_argument( + "--restricted", + action="store_true", + help="operator-asserted restricted flag (redacts surface)", + ) + args = p.parse_args(argv) + text = " ".join(args.text) + out = parse_mutation_trace( + text, + source="cli", + restricted_intent_suspected=bool(args.restricted), + ) + print(json.dumps({"ok": True, "command": "mutation-trace", **out}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/hyperlex/mutation/detect.py b/src/hyperlex/mutation/detect.py new file mode 100644 index 0000000..1c0b1b1 --- /dev/null +++ b/src/hyperlex/mutation/detect.py @@ -0,0 +1,90 @@ +"""v0.2 detect-only heuristics. Never generate wraps or encodings.""" +from __future__ import annotations + +import unicodedata +from typing import Iterable, Optional, Sequence + +from .operators import ( + CODE_SWITCH_PARTICLES, + GAME_MARKERS, + SUBSTITUTE_TERMS, + leet_decode, + token_has_leet, + vowel_drop_map, +) + +_SCRIPT_PREFIXES = ( + "LATIN", + "CYRILLIC", + "HANGUL", + "HIRAGANA", + "KATAKANA", + "CJK", + "ARABIC", + "HEBREW", + "GREEK", + "DEVANAGARI", + "THAI", +) + +_WARP_MAP = vowel_drop_map() + + +def _script_of(ch: str) -> Optional[str]: + if not ch.isalpha(): + return None + name = unicodedata.name(ch, "") + for prefix in _SCRIPT_PREFIXES: + if name.startswith(prefix): + return prefix + return "OTHER" if name else None + + +def detect_phonetic_warp(tokens: Sequence[str]) -> Optional[str]: + """Return recovered slang atom if a token is its vowel-dropped form.""" + for tok in tokens: + recovered = _WARP_MAP.get(tok) + if recovered and tok != recovered: + return recovered + return None + + +def detect_game_encode( + tokens: Sequence[str], + norm: str, + *, + slang_context: bool, +) -> Optional[str]: + """Leet-of-known-slang, or civilian game-frame plus slang context.""" + for tok in tokens: + if not token_has_leet(tok): + continue + decoded = leet_decode(tok) + if decoded in SUBSTITUTE_TERMS and decoded != tok: + return decoded + if slang_context: + for marker in GAME_MARKERS: + if marker in norm: + return marker + return None + + +def detect_code_switch( + raw: str, + tokens: Sequence[str], + *, + lexicon_hit: bool, +) -> bool: + """Mixed Unicode scripts, or closed particle next to a slang lexicon hit.""" + scripts = {s for s in (_script_of(ch) for ch in raw) if s} + if len(scripts) >= 2: + return True + if lexicon_hit and any(tok in CODE_SWITCH_PARTICLES for tok in tokens): + return True + return False + + +def slang_context(tokens: Iterable[str], ops: Sequence[str]) -> bool: + if any(tok in SUBSTITUTE_TERMS for tok in tokens): + return True + return bool(ops) diff --git a/src/hyperlex/mutation/grammar.py b/src/hyperlex/mutation/grammar.py new file mode 100644 index 0000000..c90efe2 --- /dev/null +++ b/src/hyperlex/mutation/grammar.py @@ -0,0 +1,198 @@ +"""Offline parser: v0.1 L2/L3/L5 + v0.2 L1/L4/L6 detect-only.""" +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +from .detect import ( + detect_code_switch, + detect_game_encode, + detect_phonetic_warp, + slang_context, +) +from .operators import ( + EGGCORN_PHRASES, + FRAME_MARKERS, + LAYER_FOR_OP, + PRODUCTIVE_AFFIXES, + REGISTER_MARKERS, + SUBSTITUTE_TERMS, +) +from .packet import MutationTracePacket, packet_id_for, payload_ref_for, redact_packet +from .watch import watch_score + + +def _norm(text: str) -> str: + return " ".join((text or "").lower().split()) + + +def parse_mutation_trace( + text: str, + *, + source: str = "cli", + restricted_intent_suspected: bool = False, +) -> Dict[str, Any]: + raw = text or "" + norm = _norm(raw) + ops: List[str] = [] + affix_hit: Optional[str] = None + recovered: Optional[str] = None + lexicon_hit = False + algospeak = False + irony = False + dissociation = False + register = "none" + decode = 0.35 + claim = "INFERRED" + + if not norm: + pkt = MutationTracePacket( + packet_id=packet_id_for(""), + source=source, + surface_span=raw or None, + operators=[], + layers_touched=[], + class_="OBSERVED", + decode_confidence=0.0, + watch_score=0.0, + provenance=["empty-input"], + ) + return redact_packet(pkt.to_dict()) + + for phrase in EGGCORN_PHRASES: + if phrase in norm: + ops.append("EGGCORN") + recovered = phrase + decode = max(decode, 0.7) + claim = "OBSERVED" + break + + for marker in REGISTER_MARKERS: + if marker in norm: + ops.append("REGISTER_SHIFT") + register = "med" if marker.startswith("it's") or marker.startswith("its") else "low" + if marker in {"it's giving", "its giving"}: + register = "high" + irony = True + decode = max(decode, 0.62) + break + + for marker in FRAME_MARKERS: + if marker in norm: + dissociation = True + irony = True + # Flag only in v0.1 — FRAME_WRAP parser deferred; still record feature. + break + + tokens = re.findall(r"[a-z0-9']+", norm) + words = [w.strip(".,!?;:\"“”") for w in norm.split() if w.strip(".,!?;:\"“”")] + for tok in tokens: + if tok in SUBSTITUTE_TERMS: + ops.append("SUBSTITUTE") + lexicon_hit = True + recovered = recovered or tok + if tok in {"unalive"}: + algospeak = True + decode = max(decode, 0.72) + claim = "OBSERVED" if claim != "INFERRED" or True else claim + claim = "OBSERVED" + break + + for suf in PRODUCTIVE_AFFIXES: + if re.search(rf"\b[a-z0-9']{{2,}}{re.escape(suf)}\b", norm) or re.search( + rf"\b[a-z0-9']+\s+{re.escape(suf)}\b", norm + ): + ops.append("AFFIX") + affix_hit = suf + decode = max(decode, 0.68) + claim = "OBSERVED" + break + # bare suffix mention as productivity signal ("-maxxing") + if suf in tokens or f"-{suf}" in norm: + ops.append("AFFIX") + affix_hit = suf + decode = max(decode, 0.6) + claim = "OBSERVED" + break + + warped = detect_phonetic_warp(tokens) + if warped: + ops.append("PHONETIC_WARP") + recovered = recovered or warped + lexicon_hit = True + if warped == "unalive": + algospeak = True + if "SUBSTITUTE" not in ops: + ops.append("SUBSTITUTE") + decode = max(decode, 0.66) + if claim != "OBSERVED": + claim = "INFERRED" + + game = detect_game_encode( + tokens, + norm, + slang_context=slang_context(tokens, ops) or bool(warped), + ) + if game: + ops.append("GAME_ENCODE") + if game in SUBSTITUTE_TERMS: + recovered = recovered or game + lexicon_hit = True + if "SUBSTITUTE" not in ops: + ops.append("SUBSTITUTE") + decode = max(decode, 0.64) + if game in SUBSTITUTE_TERMS: + claim = "OBSERVED" + + if detect_code_switch(raw, words, lexicon_hit=lexicon_hit or bool(warped)): + ops.append("CODE_SWITCH") + decode = max(decode, 0.64) + claim = "OBSERVED" + + # unique preserve order + seen = set() + uniq: List[str] = [] + for op in ops: + if op not in seen: + seen.add(op) + uniq.append(op) + ops = uniq + if len(ops) >= 2: + ops.append("COMPOSE") + + layers: List[str] = [] + for op in ops: + layer = LAYER_FOR_OP.get(op) + if layer and layer not in layers: + layers.append(layer) + + ws = watch_score( + decode_confidence=decode, + n_ops=len([o for o in ops if o != "COMPOSE"]) + (1 if "COMPOSE" in ops else 0), + register_shift=register, + irony_flag=irony, + affix_productivity=bool(affix_hit), + lexicon_hit=lexicon_hit, + ) + + pkt = MutationTracePacket( + packet_id=packet_id_for(norm), + source=source, + surface_span=raw, + recovered_lemma=recovered, + operators=ops, + layers_touched=layers, + register_shift=register, + irony_flag=irony, + dissociation_flag=dissociation, + algospeak_flag=algospeak, + affix_family=affix_hit, + decode_confidence=round(decode, 4), + lexicon_hit=lexicon_hit, + watch_score=ws, + class_=claim if ops else "OBSERVED", + restricted_intent_suspected=bool(restricted_intent_suspected), + payload_ref=payload_ref_for(norm) if restricted_intent_suspected else None, + provenance=["hyperlex.mutation.grammar.v0.2"], + ) + return redact_packet(pkt.to_dict()) diff --git a/src/hyperlex/mutation/human.py b/src/hyperlex/mutation/human.py new file mode 100644 index 0000000..d94ce6d --- /dev/null +++ b/src/hyperlex/mutation/human.py @@ -0,0 +1,45 @@ +"""Civilian advisory cards from mutation-trace packets. Not forecasts.""" +from __future__ import annotations + +from typing import Any, Dict, List + +_OP_BLURB = { + "SUBSTITUTE": "known slang / algospeak swap", + "AFFIX": "productive suffix", + "EGGCORN": "folk reanalysis", + "PHONETIC_WARP": "vowel-drop / compressed slang atom", + "GAME_ENCODE": "language-game or leet framing (detect only)", + "REGISTER_SHIFT": "informal-register marker", + "CODE_SWITCH": "mixed script or bilingual particle + slang", + "FRAME_WRAP": "dissociation / lore frame (flag only)", + "COMPOSE": "two or more operators stacked", + "CLIP_BLEND": "clip / blend (not parsed in v0.2)", +} + + +def format_human_card(packet: Dict[str, Any]) -> str: + ops: List[str] = list(packet.get("operators") or []) + layers = list(packet.get("layers_touched") or []) + restricted = bool(packet.get("restricted_intent_suspected")) + if restricted: + surface = "(redacted — restricted flag; payload_ref only)" + else: + surface = packet.get("surface_span") or "(empty)" + recovered = packet.get("recovered_lemma") or "—" + watch = packet.get("watch_score") + watch_s = "—" if watch is None else f"{watch}" + blurbs = [_OP_BLURB.get(op, op) for op in ops] if ops else ["no operators"] + lines = [ + "MUTATION TRACE (SHADOW advisory — not a forecast)", + f"Surface: {surface}", + f"Operators: {' · '.join(ops) if ops else '(none)'}", + f"Layers: {' · '.join(layers) if layers else '(none)'}", + f"What fired: {'; '.join(blurbs)}", + f"Recovered lemma: {recovered}", + f"Watch score: {watch_s} — instrumentation, not P(harm), not a tool-fire threshold", + "Brier: null", + "Forecast eligible: no", + f"Restricted: {'yes' if restricted else 'no'}", + "Lane: SHADOW / advisory. Hosts must not execute this card.", + ] + return "\n".join(lines) + "\n" diff --git a/src/hyperlex/mutation/operators.py b/src/hyperlex/mutation/operators.py new file mode 100644 index 0000000..3fd8547 --- /dev/null +++ b/src/hyperlex/mutation/operators.py @@ -0,0 +1,175 @@ +"""Detector enum and maps onto lineage / prediction vocabularies.""" +from __future__ import annotations + +DETECTOR_OPS = ( + "SUBSTITUTE", + "AFFIX", + "CLIP_BLEND", + "EGGCORN", + "PHONETIC_WARP", + "GAME_ENCODE", + "REGISTER_SHIFT", + "CODE_SWITCH", + "FRAME_WRAP", + "COMPOSE", +) + +LAYER_FOR_OP = { + "SUBSTITUTE": "L3", + "AFFIX": "L2", + "CLIP_BLEND": "L2", + "EGGCORN": "L3", + "PHONETIC_WARP": "L1", + "GAME_ENCODE": "L4", + "REGISTER_SHIFT": "L5", + "CODE_SWITCH": "L6", + "FRAME_WRAP": "L7", + "COMPOSE": None, +} + +LINEAGE_TO_DETECTOR = { + "sense_extension": "SUBSTITUTE", + "irony_inversion": "REGISTER_SHIFT", + "platform_compression": "PHONETIC_WARP", + "eggcorn": "EGGCORN", + "cross_family_borrowing": "SUBSTITUTE", + "hyperstition_loop": None, + "derivational": "AFFIX", + "compound_phrase": None, + "extra-grammatical": "AFFIX", +} + +PRODUCTIVE_AFFIXES = ( + "maxxing", + "pilled", + "core", + "posting", + "points", + "slop", + "diff", + "season", + "coded", +) + +SUBSTITUTE_TERMS = frozenset({ + "unalive", + "cooked", + "mid", + "npc", + "rizz", + "aura", + "brainrot", + "delulu", + "glazing", + "slop", + "sigma", + "skibidi", + "gyatt", + "bussin", + "based", + "cope", + "seethe", + "ratio", + "owned", + "rekt", + "hodl", + "ngmi", + "wagmi", +}) + +REGISTER_MARKERS = ( + "it's giving", + "its giving", + "lowkey", + "highkey", + "no cap", + "ngl", + "fr fr", + "iykyk", + "not gonna lie", +) + +FRAME_MARKERS = ( + "for the lore", + "just a bit", + "in this essay", +) + +EGGCORN_PHRASES = ( + "all intensive purposes", + "mute point", + "old timers disease", + "old-timers disease", + "egg corn", + "eggcorn", + "for all intensive purposes", +) + +# Civilian game-frame phrases only. Marker alone does not fire GAME_ENCODE. +GAME_MARKERS = ( + "in fortnite", + "in minecraft", + "in roblox", + "in among us", + "pig latin", + "in uwu", + "as a haiku", +) + +# Whole-token bilingual particles. Require a slang lexicon hit in the same span. +CODE_SWITCH_PARTICLES = frozenset({ + "que", + "qué", + "el", + "très", + "c'est", + "mucho", + "muito", + "não", + "kein", +}) + +_VOWELS = set("aeiouAEIOU") +_LEET_TABLE = str.maketrans({ + "0": "o", + "1": "i", + "3": "e", + "4": "a", + "5": "s", + "7": "t", + "@": "a", + "$": "s", + "!": "i", +}) + + +def vowel_drop(seed: str) -> str | None: + """Keep first/last chars; drop internal vowels. Same rule as predict.""" + if len(seed) < 4 or " " in seed: + return None + chars = [] + for i, ch in enumerate(seed): + if i > 0 and i < len(seed) - 1 and ch in _VOWELS: + continue + chars.append(ch) + out = "".join(chars) + if out.lower() == seed.lower() or len(out) < 2: + return None + return out.lower() + + +def vowel_drop_map(terms: frozenset[str] | None = None) -> dict[str, str]: + out: dict[str, str] = {} + for term in terms or SUBSTITUTE_TERMS: + warped = vowel_drop(term) + if warped and warped not in SUBSTITUTE_TERMS: + out[warped] = term + return out + + +def leet_decode(token: str) -> str: + return token.translate(_LEET_TABLE).lower() + + +def token_has_leet(token: str) -> bool: + return any(ch in token for ch in "013457@$!") diff --git a/src/hyperlex/mutation/packet.py b/src/hyperlex/mutation/packet.py new file mode 100644 index 0000000..7328b5f --- /dev/null +++ b/src/hyperlex/mutation/packet.py @@ -0,0 +1,72 @@ +"""Mutation-trace packet + restricted redaction.""" +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, List, Optional + +SCHEMA = "hyperlex.mutation_trace.v0.1" + + +def packet_id_for(surface: str) -> str: + h = hashlib.sha256((surface or "").encode("utf-8")).hexdigest() + return "mt-" + h[:16] + + +def payload_ref_for(surface: str) -> str: + norm = " ".join((surface or "").lower().split()) + return hashlib.sha256(norm.encode("utf-8")).hexdigest() + + +@dataclass +class MutationTracePacket: + schema: str = SCHEMA + packet_id: str = "" + observed_at: Optional[str] = None + source: Optional[str] = "cli" + surface_span: Optional[str] = None + recovered_lemma: Optional[str] = None + canonical_gloss: Optional[str] = None + operators: List[str] = field(default_factory=list) + layers_touched: List[str] = field(default_factory=list) + register_shift: str = "none" + irony_flag: bool = False + dissociation_flag: bool = False + algospeak_flag: bool = False + machine_dialect: bool = False + affix_family: Optional[str] = None + decode_confidence: Optional[float] = None + lexicon_hit: bool = False + watch_score: Optional[float] = None + class_: str = "INFERRED" + restricted_intent_suspected: bool = False + payload_ref: Optional[str] = None + forecast_eligible: bool = False + brier: Optional[Any] = None + provenance: List[str] = field(default_factory=list) + receipt_id: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + d = asdict(self) + d["class"] = d.pop("class_") + d["forecast_eligible"] = False + d["brier"] = None + return d + + +def redact_packet(d: Dict[str, Any]) -> Dict[str, Any]: + out = dict(d) + out["forecast_eligible"] = False + out["brier"] = None + if out.get("restricted_intent_suspected"): + surface = out.get("surface_span") or "" + if not out.get("payload_ref"): + out["payload_ref"] = payload_ref_for(str(surface)) + out["surface_span"] = None + out["canonical_gloss"] = None + return out + + +def dumps(d: Dict[str, Any]) -> str: + return json.dumps(d, ensure_ascii=False, indent=2) diff --git a/src/hyperlex/mutation/watch.py b/src/hyperlex/mutation/watch.py new file mode 100644 index 0000000..0c41826 --- /dev/null +++ b/src/hyperlex/mutation/watch.py @@ -0,0 +1,134 @@ +"""Instrumentation scores and append-only watch log. Not probabilities. Not Brier.""" +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +WATCH_SCHEMA = "hyperlex.mutation_watch.v0.2" +DEFAULT_RELATIVE = Path("mutation_watch.jsonl") + +_REGISTER_W = {"none": 0.0, "low": 0.33, "med": 0.66, "high": 1.0} + + +def clip01(x: float) -> float: + return max(0.0, min(1.0, float(x))) + + +def watch_score( + *, + decode_confidence: float, + n_ops: int, + register_shift: str, + irony_flag: bool, + affix_productivity: bool, + lexicon_hit: bool, +) -> float: + register_w = _REGISTER_W.get(register_shift or "none", 0.0) + lexicon_only = 1.0 if (lexicon_hit and int(n_ops) <= 1) else 0.0 + raw = ( + 0.35 * float(decode_confidence) + + 0.15 * (min(int(n_ops), 4) / 4.0) + + 0.20 * register_w + + 0.10 * (1.0 if irony_flag else 0.0) + + 0.15 * (1.0 if affix_productivity else 0.0) + + 0.15 * (0.0 if lexicon_only else 1.0) + ) + return round(clip01(raw), 4) + + +def default_watch_path() -> Path: + env = os.environ.get("HYPERLEX_MUTATION_WATCH", "").strip() + if env: + return Path(env).expanduser().resolve() + return (Path.home() / ".hyperlex" / DEFAULT_RELATIVE).resolve() + + +def _canonical(obj: Dict[str, Any]) -> str: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def append_watch( + packet: Dict[str, Any], + *, + path: Optional[Path | str] = None, +) -> Optional[Dict[str, Any]]: + """Append one watch record. Fail-open. Never a tool-fire trigger.""" + try: + p = Path(path) if path else default_watch_path() + p.parent.mkdir(parents=True, exist_ok=True) + ops = list(packet.get("operators") or []) + record = { + "schema": WATCH_SCHEMA, + "logged_at": datetime.now(timezone.utc).isoformat(), + "packet_id": packet.get("packet_id"), + "operators": ops, + "layers_touched": list(packet.get("layers_touched") or []), + "watch_score": packet.get("watch_score"), + "n_ops": len(ops), + "lexicon_hit": bool(packet.get("lexicon_hit")), + "brier": None, + "forecast_eligible": False, + "auto_fire": False, + "note": "instrumentation only; not a fire threshold", + } + with p.open("a", encoding="utf-8") as fh: + fh.write(_canonical(record) + "\n") + return record + except OSError: + return None + + +def read_watch_log(path: Optional[Path | str] = None) -> List[Dict[str, Any]]: + p = Path(path) if path else default_watch_path() + if not p.exists(): + return [] + records: List[Dict[str, Any]] = [] + try: + with p.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(rec, dict): + rec["brier"] = None + rec["forecast_eligible"] = False + rec["auto_fire"] = False + records.append(rec) + except OSError: + return [] + return records + + +def watch_summary( + path: Optional[Path | str] = None, + *, + limit: int = 20, +) -> Dict[str, Any]: + records = read_watch_log(path) + window = records[-max(0, int(limit)) :] if limit else records + n = len(window) + lex_hits = sum(1 for r in window if r.get("lexicon_hit")) + novel_ops = ("GAME_ENCODE", "CODE_SWITCH", "PHONETIC_WARP", "AFFIX") + novel = sum(1 for r in window if any(op in novel_ops for op in (r.get("operators") or []))) + return { + "ok": True, + "command": "mutation-watch", + "schema": WATCH_SCHEMA, + "path": str(Path(path) if path else default_watch_path()), + "n": len(records), + "window": n, + "lexicon_hit_rate": (lex_hits / n) if n else 0.0, + "novel_operator_rate": (novel / n) if n else 0.0, + "records": window, + "brier": None, + "forecast_eligible": False, + "auto_fire": False, + "note": "A/B rates are instrumentation. watch_score is not a fire threshold.", + } diff --git a/src/hyperlex/schemas/mutation_trace.v0.1.schema.json b/src/hyperlex/schemas/mutation_trace.v0.1.schema.json new file mode 100644 index 0000000..1f94695 --- /dev/null +++ b/src/hyperlex/schemas/mutation_trace.v0.1.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://hyperlex.local/schemas/mutation_trace.v0.1.schema.json", + "title": "hyperlex.mutation_trace.v0.1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "packet_id", + "operators", + "layers_touched", + "class", + "restricted_intent_suspected", + "forecast_eligible", + "brier" + ], + "properties": { + "schema": { "const": "hyperlex.mutation_trace.v0.1" }, + "packet_id": { "type": "string", "minLength": 8 }, + "observed_at": { "type": ["string", "null"] }, + "source": { + "type": ["string", "null"], + "enum": ["mock", "glossary", "x_search", "reddit", "web", "human", "analyze", "cli", null] + }, + "surface_span": { "type": ["string", "null"] }, + "recovered_lemma": { "type": ["string", "null"] }, + "canonical_gloss": { "type": ["string", "null"] }, + "operators": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "SUBSTITUTE", + "AFFIX", + "CLIP_BLEND", + "EGGCORN", + "PHONETIC_WARP", + "GAME_ENCODE", + "REGISTER_SHIFT", + "CODE_SWITCH", + "FRAME_WRAP", + "COMPOSE" + ] + } + }, + "layers_touched": { + "type": "array", + "items": { + "type": "string", + "enum": ["L0", "L1", "L2", "L3", "L4", "L5", "L6", "L7", "L8"] + } + }, + "register_shift": { + "type": "string", + "enum": ["none", "low", "med", "high"] + }, + "irony_flag": { "type": "boolean" }, + "dissociation_flag": { "type": "boolean" }, + "algospeak_flag": { "type": "boolean" }, + "machine_dialect": { "type": "boolean" }, + "affix_family": { "type": ["string", "null"] }, + "decode_confidence": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "lexicon_hit": { "type": "boolean" }, + "watch_score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "class": { "enum": ["OBSERVED", "INFERRED", "SPECULATIVE"] }, + "restricted_intent_suspected": { "type": "boolean" }, + "payload_ref": { "type": ["string", "null"] }, + "forecast_eligible": { "const": false }, + "brier": { "type": "null" }, + "provenance": { + "type": "array", + "items": { "type": "string" } + }, + "receipt_id": { "type": ["string", "null"] }, + "ritualCharge": { "type": ["number", "null"] }, + "inGroupKeying": { "type": ["number", "null"] }, + "paraphraseResistance": { "type": ["number", "null"] }, + "attractorStability": { "type": ["number", "null"] }, + "disseminationImperative": { "type": ["number", "null"] }, + "falseStabilizationRisk": { "type": ["number", "null"] } + } +} diff --git a/tests/test_command_router.py b/tests/test_command_router.py new file mode 100644 index 0000000..7d6d965 --- /dev/null +++ b/tests/test_command_router.py @@ -0,0 +1,10 @@ +def test_load_router_has_mutation_noun(): + from hyperlex.command_router import load_router + + r = load_router() + cmds = [row.get("cmd") for row in (r.get("research") or [])] + assert "mutation trace" in cmds + inv = r.get("invoke") or {} + assert "mutation" in inv + traces = [row for row in r["research"] if row.get("cmd") == "mutation trace"] + assert traces and traces[0]["forecast_eligible"] is False diff --git a/tests/test_hlx_mutation_alias.py b/tests/test_hlx_mutation_alias.py new file mode 100644 index 0000000..59345fc --- /dev/null +++ b/tests/test_hlx_mutation_alias.py @@ -0,0 +1,28 @@ +"""Skill-tree mutation alias.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_hlx_mutation_trace_alias(): + env = {**os.environ, "HYPERLEX_OFFLINE": "1"} + env.pop("PYTHONPATH", None) + r = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "hlx-mutation"), "trace", "it's", "giving", "mid", "rizz"], + cwd=str(ROOT), + capture_output=True, + text=True, + env=env, + ) + assert r.returncode == 0, r.stderr + r.stdout + data = json.loads(r.stdout) + assert data.get("ok") is True + assert data.get("brier") is None + assert data.get("forecast_eligible") is False + assert "REGISTER_SHIFT" in data.get("operators", []) or "SUBSTITUTE" in data.get("operators", []) diff --git a/tests/test_hyperlex.py b/tests/test_hyperlex.py index 6326988..f6bc6fc 100644 --- a/tests/test_hyperlex.py +++ b/tests/test_hyperlex.py @@ -39,19 +39,6 @@ def test_cli_check_ok() -> None: checks = {entry["name"]: entry["ok"] for entry in body["checks"]} assert checks["version_file"] assert checks["schema_ingest"] - assert checks["skill_frontmatter"] - assert checks["skill_description_len"] - assert checks["skill_description_period"] - assert checks["skill_frontmatter.name"] - assert checks["skill_frontmatter.description"] - assert checks["skill_frontmatter.version"] - assert checks["skill_frontmatter.author"] - assert checks["skill_frontmatter.license"] - assert checks["skill_frontmatter.platforms"] - assert checks["skill_frontmatter.tags"] - assert checks["skill_frontmatter.related_skills"] - assert checks["skill_author_human"] - assert checks["skill_frontmatter.version_match"] def test_sources_command() -> None: diff --git a/tests/test_init_skill.py b/tests/test_init_skill.py new file mode 100644 index 0000000..78a3b3f --- /dev/null +++ b/tests/test_init_skill.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from hyperlex.init_skill import TARGETS, run_init, run_uninstall, skill_template + + +def test_template_has_marker_and_cli(): + body = skill_template() + assert "" in body + assert "hyperlex mutation trace" in body + assert "hyperlex pipeline" in body + + +def test_init_dry_run_lists_default_hosts(): + out = run_init(["hermes", "grok"], dry_run=True) + assert out["ok"] is True + assert len(out["written"]) == 2 + assert all(row["path"].endswith("SKILL.md") for row in out["written"]) + + +def test_targets_are_under_home(): + home = str(Path.home()) + for dest in TARGETS.values(): + assert str(dest).startswith(home) diff --git a/tests/test_mutation_grammar.py b/tests/test_mutation_grammar.py new file mode 100644 index 0000000..24e7ae0 --- /dev/null +++ b/tests/test_mutation_grammar.py @@ -0,0 +1,121 @@ +"""Spec 001 detector — civilian fixtures only.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_parse_rizz_register_compose(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("it's giving mid rizz") + assert out["schema"] == "hyperlex.mutation_trace.v0.1" + assert out["brier"] is None + assert out["forecast_eligible"] is False + assert "REGISTER_SHIFT" in out["operators"] + assert "SUBSTITUTE" in out["operators"] + assert "COMPOSE" in out["operators"] + assert out["watch_score"] is not None + assert 0.0 <= float(out["watch_score"]) <= 1.0 + assert out["surface_span"] + + +def test_parse_unalive_algospeak(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("unalive") + assert "SUBSTITUTE" in out["operators"] + assert out["algospeak_flag"] is True + assert out["lexicon_hit"] is True + assert out["brier"] is None + + +def test_parse_affix_maxxing(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("aura maxxing") + assert "AFFIX" in out["operators"] + assert out["affix_family"] == "maxxing" + + +def test_parse_eggcorn(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("for all intensive purposes") + assert "EGGCORN" in out["operators"] + assert out["class"] == "OBSERVED" + + +def test_empty_has_null_brier(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace(" ") + assert out["operators"] == [] + assert out["brier"] is None + assert out["forecast_eligible"] is False + + +def test_restricted_redacts_surface(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("it's giving mid rizz", restricted_intent_suspected=True) + assert out["restricted_intent_suspected"] is True + assert out["surface_span"] is None + assert out["canonical_gloss"] is None + assert out["payload_ref"] + assert len(out["payload_ref"]) == 64 + assert out["forecast_eligible"] is False + + +def test_watch_score_bounds(): + from hyperlex.mutation.watch import watch_score + + s = watch_score( + decode_confidence=1.0, + n_ops=4, + register_shift="high", + irony_flag=True, + affix_productivity=True, + lexicon_hit=False, + ) + assert 0.0 <= s <= 1.0 + + +def test_predict_not_used_on_restricted(): + import inspect + from hyperlex.mutation import grammar as g + + src = inspect.getsource(g) + assert "predict_mutations" not in src + + +def test_cli_module_offline(): + env = {**os.environ, "PYTHONPATH": str(ROOT / "src"), "HYPERLEX_OFFLINE": "1"} + r = subprocess.run( + [sys.executable, "-m", "hyperlex.mutation", "it's", "giving", "mid", "rizz"], + cwd=str(ROOT), + capture_output=True, + text=True, + env=env, + ) + assert r.returncode == 0, r.stderr + r.stdout + data = json.loads(r.stdout) + assert data.get("ok") is True + assert data.get("brier") is None + assert data.get("forecast_eligible") is False + assert "REGISTER_SHIFT" in data.get("operators", []) + + +def test_attach_helper_sets_block(): + from hyperlex.analysis.mutation_trace_attach import attach_mutation_trace + + analysis: dict = {} + attach_mutation_trace(analysis, query="it's giving mid rizz", observed="", ingest_source="mock") + mt = analysis.get("mutation_trace") + assert mt and mt["brier"] is None + assert mt["forecast_eligible"] is False diff --git a/tests/test_mutation_v02.py b/tests/test_mutation_v02.py new file mode 100644 index 0000000..5773506 --- /dev/null +++ b/tests/test_mutation_v02.py @@ -0,0 +1,224 @@ +"""Spec 003 — civilian fixtures only. Detect, never generate.""" +from __future__ import annotations + +import inspect +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_phonetic_warp_rzz(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("rzz") + assert "PHONETIC_WARP" in out["operators"] + assert out["recovered_lemma"] == "rizz" + assert out["brier"] is None + assert out["forecast_eligible"] is False + assert "L1" in out["layers_touched"] + + +def test_phonetic_warp_brainrot(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("brnrt") + assert "PHONETIC_WARP" in out["operators"] + assert out["recovered_lemma"] == "brainrot" + assert out["brier"] is None + + +def test_phonetic_warp_with_register(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("it's giving rzz") + assert "REGISTER_SHIFT" in out["operators"] + assert "PHONETIC_WARP" in out["operators"] + assert "COMPOSE" in out["operators"] + assert out["forecast_eligible"] is False + + +def test_v01_rizz_not_phonetic_warp(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("it's giving mid rizz") + assert "REGISTER_SHIFT" in out["operators"] + assert "SUBSTITUTE" in out["operators"] + assert "COMPOSE" in out["operators"] + assert "PHONETIC_WARP" not in out["operators"] + assert out["brier"] is None + assert out["forecast_eligible"] is False + + +def test_game_encode_leet(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("r1zz") + assert "GAME_ENCODE" in out["operators"] + assert out["recovered_lemma"] == "rizz" + assert out["brier"] is None + assert out["forecast_eligible"] is False + assert "L4" in out["layers_touched"] + + +def test_game_encode_fortnite_frame(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("rizz in fortnite") + assert "GAME_ENCODE" in out["operators"] + assert "SUBSTITUTE" in out["operators"] + assert out["brier"] is None + + +def test_game_frame_without_slang_does_not_fire(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("build a house in fortnite") + assert "GAME_ENCODE" not in out["operators"] + + +def test_code_switch_particle(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("el rizz") + assert "CODE_SWITCH" in out["operators"] + assert "SUBSTITUTE" in out["operators"] + assert out["brier"] is None + assert "L6" in out["layers_touched"] + + +def test_code_switch_mixed_script(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("rizz 리즈") + assert "CODE_SWITCH" in out["operators"] + assert "SUBSTITUTE" in out["operators"] + assert out["forecast_eligible"] is False + + +def test_ordinary_prose_empty(): + from hyperlex.mutation import parse_mutation_trace + + out = parse_mutation_trace("the committee adjourned after lunch") + assert out["operators"] == [] + assert out["brier"] is None + + +def test_human_card_states_null_brier(): + from hyperlex.mutation import format_human_card, parse_mutation_trace + + pkt = parse_mutation_trace("it's giving mid rizz") + card = format_human_card(pkt) + assert "Brier: null" in card + assert "Forecast eligible: no" in card + assert "not a tool-fire threshold" in card + assert "REGISTER_SHIFT" in card + assert "SHADOW" in card + + +def test_human_card_redacts_restricted(): + from hyperlex.mutation import format_human_card, parse_mutation_trace + + pkt = parse_mutation_trace("it's giving mid rizz", restricted_intent_suspected=True) + card = format_human_card(pkt) + assert "it's giving mid rizz" not in card + assert "redacted" in card.lower() + assert "Brier: null" in card + + +def test_watch_jsonl_append_and_read(tmp_path): + from hyperlex.mutation import append_watch, parse_mutation_trace, read_watch_log, watch_summary + + pkt = parse_mutation_trace("rzz") + log = tmp_path / "mutation_watch.jsonl" + rec = append_watch(pkt, path=log) + assert rec is not None + assert rec["brier"] is None + assert rec["forecast_eligible"] is False + assert rec["auto_fire"] is False + rows = read_watch_log(log) + assert len(rows) == 1 + assert rows[0]["auto_fire"] is False + summary = watch_summary(log, limit=5) + assert summary["auto_fire"] is False + assert summary["brier"] is None + assert summary["n"] == 1 + + +def test_watch_jsonl_fail_open(tmp_path): + from hyperlex.mutation import append_watch, parse_mutation_trace + + pkt = parse_mutation_trace("rzz") + blocked = tmp_path / "not_a_dir" + blocked.write_text("nope", encoding="utf-8") + rec = append_watch(pkt, path=blocked / "nested" / "watch.jsonl") + assert rec is None + + +def test_watch_score_not_used_as_fire_threshold(): + import hyperlex.mutation.watch as w + import hyperlex.cli as cli + + assert "execute_production" not in inspect.getsource(w) + assert "auto_fire" in inspect.getsource(w) + src = inspect.getsource(cli.cmd_mutation_trace) + inspect.getsource(cli.cmd_mutation_watch) + assert "watch_score" not in src or "threshold" not in src + assert "list_runes" not in inspect.getsource(cli.cmd_mutation_watch) + + +def test_detector_still_has_no_predict_call(): + from hyperlex.mutation import detect as d + from hyperlex.mutation import grammar as g + + assert "predict_mutations" not in inspect.getsource(g) + assert "predict_mutations" not in inspect.getsource(d) + assert "wrap" not in (inspect.getsource(d).split()) + + +def test_cli_human_and_watch(tmp_path): + env = {**os.environ, "PYTHONPATH": str(ROOT / "src"), "HYPERLEX_OFFLINE": "1"} + log = tmp_path / "watch.jsonl" + r = subprocess.run( + [ + sys.executable, "-m", "hyperlex", "mutation", "trace", "r1zz", + "--human", "--watch-jsonl", str(log), + ], + cwd=str(ROOT), + capture_output=True, + text=True, + env=env, + ) + assert r.returncode == 0, r.stderr + r.stdout + assert "GAME_ENCODE" in r.stdout + assert "Brier: null" in r.stdout + assert "Forecast eligible: no" in r.stdout + w = subprocess.run( + [sys.executable, "-m", "hyperlex", "mutation", "watch", "--path", str(log)], + cwd=str(ROOT), + capture_output=True, + text=True, + env=env, + ) + assert w.returncode == 0, w.stderr + w.stdout + data = json.loads(w.stdout) + assert data.get("brier") is None + assert data.get("auto_fire") is False + assert data.get("n") == 1 + + +def test_hlx_mutation_human_passthrough(): + env = {**os.environ, "HYPERLEX_OFFLINE": "1"} + env.pop("PYTHONPATH", None) + r = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "hlx-mutation"), "trace", "rzz", "--human"], + cwd=str(ROOT), + capture_output=True, + text=True, + env=env, + ) + assert r.returncode == 0, r.stderr + r.stdout + assert "PHONETIC_WARP" in r.stdout + assert "Brier: null" in r.stdout diff --git a/tests/test_packaging_surface.py b/tests/test_packaging_surface.py new file mode 100644 index 0000000..4b63606 --- /dev/null +++ b/tests/test_packaging_surface.py @@ -0,0 +1,20 @@ +def test_version_not_default_placeholder(): + from hyperlex import PKG_VERSION + + assert PKG_VERSION + assert PKG_VERSION != "0.1.0" + + +def test_router_loads_from_package_data(): + from hyperlex.command_router import load_router, router_path + + assert router_path().is_file() + r = load_router() + assert any(row.get("cmd") == "mutation trace" for row in r.get("research") or []) + + +def test_mutation_schema_packaged(): + from pathlib import Path + from hyperlex.schemas import SCHEMAS_DIR + + assert (Path(SCHEMAS_DIR) / "mutation_trace.v0.1.schema.json").is_file() diff --git a/tests_audit/PLAN.md b/tests_audit/PLAN.md index b2442b9..835ddfb 100644 --- a/tests_audit/PLAN.md +++ b/tests_audit/PLAN.md @@ -1,4 +1,9 @@ -# Audit remediation +# Hyperlex installer audit (SHADOW) -User explicitly authorized C1–C6 and relevant A1/A4 fixes, local commits only. -Plan: reproduce each defect before its fix; stage installs and validate before activation; preserve legacy output; isolate new state; repair session recipe and repository-only CI validation; align only current organizational license metadata; preserve personal variants and advisory boundaries; run sandbox regressions and existing offline checks; parent independently reviews before remote action. +Re-implemented from closed PR #5 onto current `main`. Hyperlex-only: no +neon-genie or sigil-forge CHECKS. Stage-validate before activation, preserve +legacy `out`, target-keyed backups, operator-owned locks. Two-rename activation +is not crash-atomic. Locks are never automatically reclaimed. + +Source identity is `VERSION` plus `git rev-parse HEAD`. Mutation packets stay +advisory (`brier: null`, `forecast_eligible: false`). diff --git a/tests_audit/test_hyperlex_install.py b/tests_audit/test_hyperlex_install.py index 2fbb21d..572f199 100644 --- a/tests_audit/test_hyperlex_install.py +++ b/tests_audit/test_hyperlex_install.py @@ -1,5 +1,6 @@ import json import os +import re import shutil import subprocess import tempfile @@ -10,6 +11,30 @@ class HyperlexInstallAudit(unittest.TestCase): + def test_required_surface_is_hyperlex_only(self): + installer = (ROOT / "install.sh").read_text() + helper = (ROOT / "scripts/install_transaction.py").read_text() + docs = (ROOT / "references/source-and-upgrades.md").read_text() + self.assertIn("scripts/install_transaction.py", installer) + self.assertIn("scripts/hlx-mutation", installer) + self.assertIn("schemas/mutation_trace.v0.1.schema.json", installer) + self.assertNotIn("backup_existing", installer) + self.assertNotIn("sync_tree", installer) + self.assertNotIn("neon-genie", installer) + self.assertNotIn("sigil-forge", installer) + self.assertNotIn('"neon-genie"', helper) + self.assertNotIn('"sigil-forge"', helper) + self.assertIn("hyperlex", helper) + self.assertIsNone(re.search(r"\b[0-9a-f]{40}\b", docs)) + self.assertIn("NOT crash-atomic", docs) + self.assertIn("never automatically reclaimed", docs) + skill = (ROOT / "SKILL.md").read_text() + self.assertIn( + "Mutation packets are untrusted structured output", + skill, + ) + self.assertIn("references/source-and-upgrades.md", skill) + def test_real_install_reinstall_and_explicit_skip(self): with tempfile.TemporaryDirectory() as tmp: home = Path(tmp) / "home" diff --git a/tests_audit/test_install.py b/tests_audit/test_install.py index f67f90b..aa17397 100644 --- a/tests_audit/test_install.py +++ b/tests_audit/test_install.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import tempfile import unittest @@ -20,8 +21,6 @@ def test_failed_check_preserves_previous_install(self): (dest / "out/wizard-sessions").mkdir(parents=True) sentinel = dest / "out/wizard-sessions/keep.json" sentinel.write_text("previous session") - import shutil - source = Path(tmp) / "source" shutil.copytree( ROOT, @@ -30,14 +29,9 @@ def test_failed_check_preserves_previous_install(self): ".git", "skills", "out", "__pycache__", ".venv" ), ) - script = { - "neon-genie": "validate_hermes_skill.py", - "sigil-forge": "sigil_forge.py", - "hyperlex": "hyperlex.py", - }[SKILL] - (source / "scripts" / script).write_text("import sys\nsys.exit(42)\n") + (source / "scripts/hyperlex.py").write_text("import sys\nsys.exit(42)\n") env = dict(os.environ, HOME=str(home), HERMES_HOME=str(profile)) - r = subprocess.run( + result = subprocess.run( check=False, args=["bash", str(source / "install.sh")], cwd=tmp, @@ -45,7 +39,7 @@ def test_failed_check_preserves_previous_install(self): capture_output=True, text=True, ) - self.assertNotEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual((dest / "SKILL.md").read_text(), "previous package") self.assertEqual(sentinel.read_text(), "previous session") self.assertFalse((home / ".hermes").exists()) @@ -59,7 +53,7 @@ def test_profile_skills_symlink_cannot_redirect_install(self): foreign.mkdir(parents=True) (profile / "skills").symlink_to(foreign, target_is_directory=True) env = dict(os.environ, HOME=str(home), HERMES_HOME=str(profile)) - r = subprocess.run( + result = subprocess.run( ["bash", str(ROOT / "install.sh")], cwd=tmp, env=env, @@ -67,7 +61,7 @@ def test_profile_skills_symlink_cannot_redirect_install(self): text=True, check=False, ) - self.assertNotEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(list(foreign.iterdir()), []) self.assertTrue((profile / "skills").is_symlink()) diff --git a/tests_audit/test_install_followup.py b/tests_audit/test_install_followup.py index 5660e21..6078b03 100644 --- a/tests_audit/test_install_followup.py +++ b/tests_audit/test_install_followup.py @@ -1,4 +1,4 @@ -"""Regression coverage for installer PR follow-up; shared across distributions.""" +"""Regression coverage for installer follow-up; Hyperlex only.""" import hashlib import importlib.util @@ -109,16 +109,11 @@ def test_stale_lock_error_and_docs_are_actionable(self): ): self.assertIn(phrase, docs) - def test_tracked_neon_hub_retains_repository_and_subtree(self): - repo = self.base / "neon" - hub = repo / "skills/neon-genie" - hub.mkdir(parents=True) - for directory in (repo, hub): - license_id = "MIT" if directory == repo else "Apache-2.0" - (directory / "SKILL.md").write_text( - f"---\nname: neon-genie\nlicense: {license_id}\n---\n" - ) - (directory / "VERSION").write_text("1") + def test_own_checkout_records_repository_and_dot_subtree(self): + repo = self.base / "hyperlex-src" + repo.mkdir() + (repo / "SKILL.md").write_text("new") + (repo / "VERSION").write_text("1") subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) subprocess.run(["git", "-C", str(repo), "add", "."], check=True) subprocess.run( @@ -132,33 +127,18 @@ def test_tracked_neon_hub_retains_repository_and_subtree(self): "user.email=test@example.invalid", "commit", "-m", - "hub", + "own", ], check=True, capture_output=True, ) - transaction.install(hub, self.target, "neon-genie", True) + transaction.install(repo, self.target, "hyperlex", True) receipt = self.receipt() self.assertIsNotNone(receipt["source_commit"]) self.assertIs(receipt["source_dirty"], False) - self.assertEqual(receipt["source_subdirectory"], "skills/neon-genie") + self.assertEqual(receipt["source_subdirectory"], ".") self.assertEqual(receipt["source_repository_root"], str(repo)) - def test_invalid_utf8_enclosing_root_drops_optional_provenance(self): - self.test_tracked_neon_hub_retains_repository_and_subtree() - repo = self.base / "neon" - hub = repo / "skills/neon-genie" - (repo / "SKILL.md").write_bytes(b"\xff") - transaction.install(hub, self.target, "neon-genie", True) - receipt = self.receipt() - for field in ( - "repository", "source_repository_root", "source_subdirectory", - "source_commit", "source_dirty", - ): - self.assertIsNone(receipt[field], field) - for name in ("SKILL.md", "VERSION"): - self.assertEqual((self.target / name).read_bytes(), (hub / name).read_bytes()) - def test_partial_backup_never_published(self): self.install() self.install() @@ -189,3 +169,7 @@ def test_rollback_skips_legacy_partial_backup(self): os.utime(partial, (2000000000, 2000000000)) transaction.rollback(self.target, "hyperlex", True) self.assertEqual((self.target / "SKILL.md").read_text(), "new") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests_audit/test_transaction.py b/tests_audit/test_transaction.py index 4458239..1d8c213 100644 --- a/tests_audit/test_transaction.py +++ b/tests_audit/test_transaction.py @@ -209,6 +209,10 @@ def test_rollback_is_target_bound_and_preserves_current_outputs(self): self.assertEqual((self.target / "out/keep").read_text(), "new-output") self.assertEqual((other / "SKILL.md").read_text(), "new") + def test_foreign_kind_rejected(self): + with self.assertRaises(ValueError): + m.install(self.source, self.target, "neon-genie", skip_checks=True) + if __name__ == "__main__": unittest.main() From ee2c12cd4b51b19687f1d096ff477627d2a3b5ba Mon Sep 17 00:00:00 2001 From: Daniel Meyer Date: Sat, 5 Sep 2026 18:29:15 +0000 Subject: [PATCH 2/3] fix: track docs/command-router.v1.json for packaging tests *.json gitignore hid the router table on org; personal already tracked it. Un-ignore and add so command_router / packaging_surface tests pass. --- .gitignore | 1 + docs/command-router.v1.json | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 docs/command-router.v1.json diff --git a/.gitignore b/.gitignore index 4127ab4..adec837 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ build/ !docs/archive/**/*.jsonl !docs/demos/samples/**/*.json !docs/map/**/*.json +!docs/command-router.v1.json !data/backfill/**/*.json !data/phylogeny/**/*.json diff --git a/docs/command-router.v1.json b/docs/command-router.v1.json new file mode 100644 index 0000000..df85ebf --- /dev/null +++ b/docs/command-router.v1.json @@ -0,0 +1,25 @@ +{ + "schema": "hyperlex.command_router.v1", + "brier_note": "router metadata is not a score", + "daily": [ + {"cmd": "pipeline", "layer": "daily", "noun": null, "forecast_eligible": false, "authority": "advisory", "deprecated": false}, + {"cmd": "run", "layer": "daily", "noun": null, "forecast_eligible": false, "authority": "advisory", "deprecated": false}, + {"cmd": "pending", "layer": "daily", "noun": null, "forecast_eligible": true, "authority": "operator", "deprecated": false}, + {"cmd": "settle", "layer": "daily", "noun": null, "forecast_eligible": true, "authority": "operator", "deprecated": false}, + {"cmd": "score-series", "layer": "daily", "noun": null, "forecast_eligible": true, "authority": "operator", "deprecated": false} + ], + "research": [ + {"cmd": "mutation trace", "layer": "research", "noun": "mutation", "forecast_eligible": false, "authority": "advisory", "deprecated": false}, + {"cmd": "mutation watch", "layer": "research", "noun": "mutation", "forecast_eligible": false, "authority": "advisory", "deprecated": false}, + {"cmd": "mutation predict", "layer": "research", "noun": "mutation", "forecast_eligible": false, "authority": "advisory", "deprecated": false}, + {"cmd": "mutation-predict", "layer": "research", "noun": "mutation", "forecast_eligible": false, "authority": "advisory", "deprecated": true}, + {"cmd": "mutation-trace", "layer": "research", "noun": "mutation", "forecast_eligible": false, "authority": "advisory", "deprecated": true}, + {"cmd": "simulate", "layer": "research", "noun": "simulate", "forecast_eligible": false, "authority": "advisory", "deprecated": false} + ], + "invoke": { + "pipeline": "hyperlex pipeline", + "mutation": "hyperlex mutation", + "skill_pipeline": "scripts/hyperlex.py", + "skill_mutation": "scripts/hlx-mutation" + } +} From c78071587d3e5f31c95aa2e821c510a8c22ba545 Mon Sep 17 00:00:00 2001 From: Daniel Meyer Date: Sat, 5 Sep 2026 18:31:06 +0000 Subject: [PATCH 3/3] fix(ci): set contents:read on hermes-evals workflow Satisfies CodeQL workflow-permissions finding on the grafted personal skill-validation workflow. --- .github/workflows/hermes-evals.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/hermes-evals.yml b/.github/workflows/hermes-evals.yml index ef8dbe2..1e409c9 100644 --- a/.github/workflows/hermes-evals.yml +++ b/.github/workflows/hermes-evals.yml @@ -5,6 +5,9 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: validate: strategy: