From 8590bd8f9d1219845c858b10eebefb9863706c77 Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:32:29 +0000 Subject: [PATCH 1/9] chore: package scaffolding, tooling config, CI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++ .gitignore | 12 +++++++++ LICENSE | 21 ++++++++++++++++ pyproject.toml | 54 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 pyproject.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..35e0248 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Set up Python + run: uv python install ${{ matrix.python-version }} + - name: Create venv and install + run: | + uv venv .venv --python ${{ matrix.python-version }} + uv pip install --python .venv/bin/python -e .[dev] + - name: Ruff check + run: .venv/bin/ruff check . + - name: Ruff format + run: .venv/bin/ruff format --check . + - name: Mypy + run: .venv/bin/mypy src + - name: Pytest + run: .venv/bin/pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96e1dd0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.venv/ +venv/ +data/ +*.key +__pycache__/ +*.pyc +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +*.egg-info/ +dist/ +build/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bb64b99 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 CommunityPoke contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..37d5a08 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "poke-interconnect" +version = "1.0.0" +description = "Poke Interconnect Protocol (PIP) v1 — signed, consented, idempotent exchange between Poke instances" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +dependencies = [ + "pydantic>=2.7,<3", + "pydantic-settings>=2.3,<3", + "cryptography>=42,<50", + "pyyaml>=6", + "httpx>=0.27", + "fastapi>=0.110", + "uvicorn>=0.29", + "mcp>=1.13,<2", + "prometheus-client>=0.20", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-asyncio", + "ruff", + "mypy", + "types-PyYAML", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/pip_protocol"] + +[tool.ruff] +line-length = 100 +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "S"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["S101"] + +[tool.mypy] +python_version = "3.10" +strict = true +mypy_path = "src" +plugins = ["pydantic.mypy"] + +[[tool.mypy.overrides]] +module = ["mcp.*"] +ignore_missing_imports = true From 6a8c24545ad9264fe0d99c6c35ae02a32e3ceffb Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:32:29 +0000 Subject: [PATCH 2/9] feat: PIP v1 core library (identity, envelopes, policy, redaction, store, outbox, node) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/pip_protocol/__init__.py | 5 + src/pip_protocol/canonical.py | 18 ++ src/pip_protocol/clock.py | 31 +++ src/pip_protocol/config.py | 21 ++ src/pip_protocol/delivery.py | 68 +++++ src/pip_protocol/envelope.py | 71 +++++ src/pip_protocol/errors.py | 69 +++++ src/pip_protocol/identity.py | 166 +++++++++++ src/pip_protocol/node.py | 445 ++++++++++++++++++++++++++++++ src/pip_protocol/observability.py | 72 +++++ src/pip_protocol/policy.py | 151 ++++++++++ src/pip_protocol/redaction.py | 50 ++++ src/pip_protocol/schemas.py | 163 +++++++++++ src/pip_protocol/store.py | 202 ++++++++++++++ 14 files changed, 1532 insertions(+) create mode 100644 src/pip_protocol/__init__.py create mode 100644 src/pip_protocol/canonical.py create mode 100644 src/pip_protocol/clock.py create mode 100644 src/pip_protocol/config.py create mode 100644 src/pip_protocol/delivery.py create mode 100644 src/pip_protocol/envelope.py create mode 100644 src/pip_protocol/errors.py create mode 100644 src/pip_protocol/identity.py create mode 100644 src/pip_protocol/node.py create mode 100644 src/pip_protocol/observability.py create mode 100644 src/pip_protocol/policy.py create mode 100644 src/pip_protocol/redaction.py create mode 100644 src/pip_protocol/schemas.py create mode 100644 src/pip_protocol/store.py diff --git a/src/pip_protocol/__init__.py b/src/pip_protocol/__init__.py new file mode 100644 index 0000000..0a82238 --- /dev/null +++ b/src/pip_protocol/__init__.py @@ -0,0 +1,5 @@ +"""Poke Interconnect Protocol (PIP) v1 core library.""" + +PROTOCOL_VERSION = "1.0" +SUPPORTED_VERSIONS = ("1.0",) +__version__ = "1.0.0" diff --git a/src/pip_protocol/canonical.py b/src/pip_protocol/canonical.py new file mode 100644 index 0000000..f0be85d --- /dev/null +++ b/src/pip_protocol/canonical.py @@ -0,0 +1,18 @@ +"""Canonical JSON serialization used for signing (spec §3).""" + +import json +from typing import Any + + +def canonical_json(obj: Any) -> bytes: + """Serialize ``obj`` to canonical JSON bytes. + + Sorted keys, no whitespace, UTF-8, NaN/Infinity rejected. + """ + return json.dumps( + obj, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") diff --git a/src/pip_protocol/clock.py b/src/pip_protocol/clock.py new file mode 100644 index 0000000..e3d364d --- /dev/null +++ b/src/pip_protocol/clock.py @@ -0,0 +1,31 @@ +"""Injectable clock for deterministic tests.""" + +from datetime import datetime, timedelta, timezone +from typing import Protocol + + +class Clock(Protocol): + def now(self) -> datetime: + """Current time as an aware UTC datetime.""" + ... + + +class SystemClock: + def now(self) -> datetime: + return datetime.now(timezone.utc) + + +class FakeClock: + """Deterministic clock advanced manually in tests.""" + + def __init__(self, start: datetime | None = None) -> None: + self._now = start or datetime(2026, 1, 1, tzinfo=timezone.utc) + + def now(self) -> datetime: + return self._now + + def advance(self, seconds: float) -> None: + self._now += timedelta(seconds=seconds) + + def set(self, when: datetime) -> None: + self._now = when diff --git a/src/pip_protocol/config.py b/src/pip_protocol/config.py new file mode 100644 index 0000000..b57e38b --- /dev/null +++ b/src/pip_protocol/config.py @@ -0,0 +1,21 @@ +"""Runtime settings via environment, prefix PIP_ (spec §12).""" + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_prefix="PIP_") + + private_key_file: str = "./data/instance.key" + display_name: str = "poke" + policy_file: str = "./config/policy.yaml" + store_url: str = "memory://" + http_host: str = "127.0.0.1" + http_port: int = 8642 + public_http_url: str | None = None + public_mcp_url: str | None = None + http_bearer_token: str | None = None + max_clock_skew_seconds: int = 300 + idempotency_ttl_seconds: int = 86400 + log_level: str = "INFO" + allow_insecure_key_perms: bool = False diff --git a/src/pip_protocol/delivery.py b/src/pip_protocol/delivery.py new file mode 100644 index 0000000..9613d4c --- /dev/null +++ b/src/pip_protocol/delivery.py @@ -0,0 +1,68 @@ +"""Persistent outbound queue with exponential backoff (spec §7).""" + +from datetime import datetime, timedelta, timezone +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from .clock import Clock, SystemClock +from .schemas import new_id +from .store import Store + +BACKOFF_BASE_SECONDS = 2.0 +BACKOFF_CAP_SECONDS = 300.0 +MAX_ATTEMPTS = 8 + + +class OutboxItem(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: str = Field(default_factory=new_id) + peer_id: str + envelope: dict[str, Any] + attempts: int = 0 + next_attempt_at: datetime + last_error: str | None = None + dead: bool = False + + +class Outbox: + def __init__( + self, + store: Store, + clock: Clock | None = None, + *, + max_attempts: int = MAX_ATTEMPTS, + ) -> None: + self.store = store + self.clock = clock or SystemClock() + self.max_attempts = max_attempts + + def enqueue(self, peer_id: str, envelope: dict[str, Any]) -> OutboxItem: + item = OutboxItem( + peer_id=peer_id, + envelope=envelope, + next_attempt_at=self.clock.now(), + ) + self.store.outbox_put(item.model_dump(mode="json")) + return item + + def due(self) -> list[OutboxItem]: + return [OutboxItem.model_validate(raw) for raw in self.store.outbox_due(self.clock.now())] + + def mark_failed(self, item: OutboxItem, err: str) -> OutboxItem: + item.attempts += 1 + item.last_error = err + if item.attempts >= self.max_attempts: + item.dead = True + delay = min(BACKOFF_BASE_SECONDS * (2 ** (item.attempts - 1)), BACKOFF_CAP_SECONDS) + item.next_attempt_at = self.clock.now() + timedelta(seconds=delay) + self.store.outbox_update(item.model_dump(mode="json")) + return item + + def mark_delivered(self, item: OutboxItem) -> None: + self.store.outbox_delete(item.id) + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) diff --git a/src/pip_protocol/envelope.py b/src/pip_protocol/envelope.py new file mode 100644 index 0000000..9081e2f --- /dev/null +++ b/src/pip_protocol/envelope.py @@ -0,0 +1,71 @@ +"""Build, sign, and verify envelopes (spec §3).""" + +import base64 +from datetime import datetime, timedelta, timezone +from typing import Any + +from . import PROTOCOL_VERSION +from .canonical import canonical_json +from .clock import Clock +from .errors import ErrorCode, PipError +from .identity import KeyPair, verify +from .schemas import Envelope, EnvelopeType, Signature, new_id, new_nonce + +SIGNING_PREFIX = b"PIPv1\n" +MAX_TTL_SECONDS = 3600 # spec §7: expires - ts <= 1 hour + + +def _b64e(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _b64d(text: str) -> bytes: + return base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)) + + +def signing_input(envelope: Envelope) -> bytes: + """`b"PIPv1\\n" + canonical_json(envelope without sig)`.""" + return SIGNING_PREFIX + canonical_json(envelope.without_sig()) + + +def build_envelope( + keypair: KeyPair, + *, + type: EnvelopeType, + to: str, + payload: dict[str, Any], + ttl_seconds: int = 300, + idempotency_key: str | None = None, + in_reply_to: str | None = None, + clock: Clock | None = None, +) -> Envelope: + if ttl_seconds > MAX_TTL_SECONDS: + raise PipError(ErrorCode.SCHEMA_INVALID, "ttl_seconds exceeds the 1 hour maximum") + now = clock.now() if clock is not None else datetime.now(timezone.utc) + env = Envelope( + pip=PROTOCOL_VERSION, + id=new_id(), + type=type, + from_=keypair.instance_id, + to=to, + ts=now, + expires=now + timedelta(seconds=ttl_seconds), + nonce=new_nonce(), + idempotency_key=idempotency_key, + in_reply_to=in_reply_to, + payload=payload, + ) + env.sig = Signature(kid=keypair.public_key, value=_b64e(keypair.sign(signing_input(env)))) + return env + + +def verify_signature(envelope: Envelope, public_key: str) -> None: + """Raise SIGNATURE_INVALID unless ``public_key`` signed ``envelope``.""" + if envelope.sig is None: + raise PipError(ErrorCode.SIGNATURE_INVALID, "envelope is unsigned") + try: + raw_sig = _b64d(envelope.sig.value) + except Exception as exc: + raise PipError(ErrorCode.SIGNATURE_INVALID, "malformed signature") from exc + if not verify(public_key, signing_input(envelope), raw_sig): + raise PipError(ErrorCode.SIGNATURE_INVALID, "signature does not verify") diff --git a/src/pip_protocol/errors.py b/src/pip_protocol/errors.py new file mode 100644 index 0000000..5b69f5c --- /dev/null +++ b/src/pip_protocol/errors.py @@ -0,0 +1,69 @@ +"""Error codes (spec §3) and the PipError exception type.""" + +from enum import Enum + + +class ErrorCode(str, Enum): + SCHEMA_INVALID = "SCHEMA_INVALID" + VERSION_UNSUPPORTED = "VERSION_UNSUPPORTED" + IDENTITY_MISMATCH = "IDENTITY_MISMATCH" + SIGNATURE_INVALID = "SIGNATURE_INVALID" + EXPIRED = "EXPIRED" + CLOCK_SKEW = "CLOCK_SKEW" + REPLAY = "REPLAY" + UNKNOWN_PEER = "UNKNOWN_PEER" + FORBIDDEN = "FORBIDDEN" + CONSENT_REQUIRED = "CONSENT_REQUIRED" + RATE_LIMITED = "RATE_LIMITED" + PAYLOAD_TOO_LARGE = "PAYLOAD_TOO_LARGE" + CAPABILITY_UNKNOWN = "CAPABILITY_UNKNOWN" + INTERNAL = "INTERNAL" + + +_HTTP_STATUS: dict[ErrorCode, int] = { + ErrorCode.SCHEMA_INVALID: 400, + ErrorCode.IDENTITY_MISMATCH: 401, + ErrorCode.SIGNATURE_INVALID: 401, + ErrorCode.UNKNOWN_PEER: 401, + ErrorCode.FORBIDDEN: 403, + ErrorCode.CONSENT_REQUIRED: 403, + ErrorCode.REPLAY: 409, + ErrorCode.PAYLOAD_TOO_LARGE: 413, + ErrorCode.RATE_LIMITED: 429, + ErrorCode.VERSION_UNSUPPORTED: 422, + ErrorCode.EXPIRED: 422, + ErrorCode.CLOCK_SKEW: 422, + ErrorCode.CAPABILITY_UNKNOWN: 422, + ErrorCode.INTERNAL: 500, +} + +_RETRYABLE: set[ErrorCode] = { + ErrorCode.RATE_LIMITED, + ErrorCode.INTERNAL, +} + + +class PipError(Exception): + """Protocol error carrying a spec error code. + + Attributes: + code: the spec §3 error code. + message: human-readable detail. + retryable: whether the sender may retry. + """ + + def __init__( + self, + code: ErrorCode, + message: str = "", + *, + retryable: bool | None = None, + ) -> None: + self.code = code + self.message = message or code.value + self.retryable = code in _RETRYABLE if retryable is None else retryable + super().__init__(f"{code.value}: {self.message}") + + @property + def http_status(self) -> int: + return _HTTP_STATUS[self.code] diff --git a/src/pip_protocol/identity.py b/src/pip_protocol/identity.py new file mode 100644 index 0000000..a2c8790 --- /dev/null +++ b/src/pip_protocol/identity.py @@ -0,0 +1,166 @@ +"""Ed25519 identity keys and the identity document (spec §2, §4).""" + +import base64 +import hashlib +import os +import stat +from datetime import datetime +from pathlib import Path + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + Ed25519PublicKey, +) +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, + PublicFormat, +) +from pydantic import BaseModel, ConfigDict, model_validator + +from .errors import ErrorCode, PipError + +PUBLIC_KEY_PREFIX = "ed25519:" +_INSTANCE_ID_CHARS = 26 + + +def _b64e(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _b64d(text: str) -> bytes: + pad = "=" * (-len(text) % 4) + return base64.urlsafe_b64decode(text + pad) + + +class InsecureKeyFileError(PipError): + def __init__(self, path: Path, mode: int) -> None: + super().__init__( + ErrorCode.INTERNAL, + f"private key file {path} has insecure permissions {oct(mode)}; " + "expected 0600 (set PIP_ALLOW_INSECURE_KEY_PERMS=1 to override)", + ) + + +class KeyPair: + """An Ed25519 keypair identifying an instance.""" + + def __init__(self, private_key: Ed25519PrivateKey) -> None: + self._private = private_key + + @classmethod + def generate(cls) -> "KeyPair": + return cls(Ed25519PrivateKey.generate()) + + @classmethod + def from_private_bytes(cls, seed: bytes) -> "KeyPair": + if len(seed) != 32: + raise PipError(ErrorCode.INTERNAL, "ed25519 seed must be 32 bytes") + return cls(Ed25519PrivateKey.from_private_bytes(seed)) + + @property + def private_bytes(self) -> bytes: + return self._private.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()) + + @property + def public_bytes(self) -> bytes: + return self._private.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + + @property + def public_key(self) -> str: + return encode_public_key(self.public_bytes) + + @property + def instance_id(self) -> str: + return instance_id_from_public_key(self.public_key) + + def sign(self, data: bytes) -> bytes: + return self._private.sign(data) + + def save(self, path: str | Path) -> None: + """Write the raw 32-byte seed, base64url-encoded, with mode 0600.""" + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + os.write(fd, _b64e(self.private_bytes).encode("ascii")) + finally: + os.close(fd) + os.chmod(p, 0o600) + + @classmethod + def load(cls, path: str | Path, *, allow_insecure: bool = False) -> "KeyPair": + p = Path(path) + mode = stat.S_IMODE(p.stat().st_mode) + if not allow_insecure and mode & 0o077: + raise InsecureKeyFileError(p, mode) + seed = _b64d(p.read_text().strip()) + return cls.from_private_bytes(seed) + + +def encode_public_key(public_bytes: bytes) -> str: + if len(public_bytes) != 32: + raise PipError(ErrorCode.INTERNAL, "ed25519 public key must be 32 bytes") + return PUBLIC_KEY_PREFIX + _b64e(public_bytes) + + +def decode_public_key(encoded: str) -> Ed25519PublicKey: + if not encoded.startswith(PUBLIC_KEY_PREFIX): + raise PipError( + ErrorCode.SCHEMA_INVALID, f"public key must start with {PUBLIC_KEY_PREFIX!r}" + ) + try: + raw = _b64d(encoded[len(PUBLIC_KEY_PREFIX) :]) + return Ed25519PublicKey.from_public_bytes(raw) + except Exception as exc: + raise PipError(ErrorCode.SCHEMA_INVALID, "malformed ed25519 public key") from exc + + +def instance_id_from_public_key(encoded: str) -> str: + raw = decode_public_key(encoded).public_bytes(Encoding.Raw, PublicFormat.Raw) + digest = base64.b32encode(hashlib.sha256(raw).digest()).decode("ascii").lower() + return f"pip:{digest[:_INSTANCE_ID_CHARS]}" + + +def verify(public_key: str, data: bytes, signature: bytes) -> bool: + try: + decode_public_key(public_key).verify(signature, data) + except InvalidSignature: + return False + return True + + +class PreviousKey(BaseModel): + model_config = ConfigDict(extra="ignore") + + public_key: str + rotated_at: datetime + + +class IdentityDocument(BaseModel): + model_config = ConfigDict(extra="ignore") + + instance_id: str + public_key: str + display_name: str + pip_versions: list[str] + endpoints: dict[str, str | None] = {} + previous_keys: list[PreviousKey] = [] + issued_at: datetime + + @model_validator(mode="after") + def _id_matches_key(self) -> "IdentityDocument": + if instance_id_from_public_key(self.public_key) != self.instance_id: + raise PipError(ErrorCode.IDENTITY_MISMATCH, "instance_id does not match public_key") + return self + + +class Capability(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + version: str = "1.0" + scopes_required: list[str] = [] + description: str = "" diff --git a/src/pip_protocol/node.py b/src/pip_protocol/node.py new file mode 100644 index 0000000..937aed7 --- /dev/null +++ b/src/pip_protocol/node.py @@ -0,0 +1,445 @@ +"""Transport-agnostic PIP node (spec §3–§7).""" + +import base64 +import logging +import time +from collections.abc import Callable +from datetime import timedelta +from typing import Any, Literal, Protocol + +from pydantic import BaseModel + +from . import PROTOCOL_VERSION, SUPPORTED_VERSIONS +from .canonical import canonical_json +from .clock import Clock, SystemClock +from .envelope import MAX_TTL_SECONDS, build_envelope, signing_input, verify_signature +from .errors import ErrorCode, PipError +from .identity import ( + Capability, + IdentityDocument, + KeyPair, + PreviousKey, + instance_id_from_public_key, +) +from .observability import Metrics, log_event +from .policy import PeerRecord, Policy, PolicyEngine, dataset_resource +from .redaction import Redactor +from .schemas import ( + DataPayload, + Envelope, + EnvelopeType, + ErrorPayload, + HandshakePayload, + ReceiptPayload, + Signature, +) +from .store import MemoryStore, Store + +logger = logging.getLogger("pip_protocol.node") + +Handler = Callable[["Node", Envelope, BaseModel], BaseModel | None] + + +class DataProvider(Protocol): + """Backend for inbound `data` request envelopes.""" + + def query( + self, dataset: str, query: dict[str, Any] | None, cursor: str | None + ) -> tuple[list[dict[str, Any]], str | None]: + """Return (records, next_cursor); next_cursor None means exhausted.""" + ... + + +class DictDataProvider: + """In-memory provider backed by ``{dataset: [records]}`` with paging. + + ``query`` may contain ``{"filter": {field: value}}`` for equality matching + and ``{"limit": n}`` to cap the page size (default ``page_size``). + """ + + def __init__(self, datasets: dict[str, list[dict[str, Any]]], page_size: int = 100) -> None: + self.datasets = datasets + self.page_size = page_size + + def query( + self, dataset: str, query: dict[str, Any] | None, cursor: str | None + ) -> tuple[list[dict[str, Any]], str | None]: + records = list(self.datasets.get(dataset, [])) + if query and isinstance(query.get("filter"), dict): + records = [r for r in records if all(r.get(k) == v for k, v in query["filter"].items())] + limit = self.page_size + if query and isinstance(query.get("limit"), int): + limit = max(1, int(query["limit"])) + start = int(cursor) if cursor else 0 + page = records[start : start + limit] + next_cursor = str(start + limit) if start + limit < len(records) else None + return page, next_cursor + + +class Node: + """One PIP instance: identity, policy, storage, redaction, dispatch.""" + + def __init__( + self, + keypair: KeyPair, + policy: Policy | None = None, + *, + store: Store | None = None, + clock: Clock | None = None, + data_provider: DataProvider | None = None, + metrics: Metrics | None = None, + display_name: str = "poke", + endpoints: dict[str, str | None] | None = None, + previous_keys: list[PreviousKey] | None = None, + ) -> None: + self.keypair = keypair + self.clock = clock or SystemClock() + self.policy = policy or Policy() + self.engine = PolicyEngine(self.policy, self.clock) + self.store = store or MemoryStore() + self.redactor = Redactor(self.policy.redaction) + self.data_provider = data_provider or DictDataProvider({}) + self.metrics = metrics or Metrics() + self.display_name = display_name + self.endpoints = endpoints or {} + self.previous_keys = previous_keys or [] + self.inbox: list[Envelope] = [] + self._handlers: dict[EnvelopeType, Handler] = { + EnvelopeType.HANDSHAKE: _default_handshake_handler, + EnvelopeType.MESSAGE: _default_message_handler, + EnvelopeType.DATA: _default_data_handler, + } + + @property + def instance_id(self) -> str: + return self.keypair.instance_id + + # ------------------------------------------------------------------ # + # Outbound + # ------------------------------------------------------------------ # + + def identity_document(self) -> IdentityDocument: + return IdentityDocument( + instance_id=self.instance_id, + public_key=self.keypair.public_key, + display_name=self.display_name, + pip_versions=list(SUPPORTED_VERSIONS), + endpoints=self.endpoints, + previous_keys=[PreviousKey.model_validate(k) for k in self.previous_keys], + issued_at=self.clock.now(), + ) + + def capabilities(self) -> list[Capability]: + caps = [ + Capability( + name="messages.send", + scopes_required=["messages:send"], + description="Receive message envelopes", + ), + Capability( + name="data.query", + scopes_required=["data:request"], + description="Answer data request envelopes", + ), + Capability( + name="data.offer", + scopes_required=["data:offer"], + description="Receive data offer envelopes", + ), + ] + return caps + + def handshake_envelope(self, to: str | None = None) -> Envelope: + payload = HandshakePayload( + identity=self.identity_document(), capabilities=self.capabilities() + ) + return build_envelope( + self.keypair, + type=EnvelopeType.HANDSHAKE, + to=to or self.instance_id, + payload=payload.model_dump(mode="json"), + clock=self.clock, + ) + + def build( + self, + type: EnvelopeType, + to: str, + payload: BaseModel | dict[str, Any], + *, + ttl_seconds: int = 300, + idempotency_key: str | None = None, + in_reply_to: str | None = None, + ) -> Envelope: + """Redact, then sign an outbound envelope.""" + if isinstance(payload, BaseModel): + payload = self.redactor.redact_payload(type, payload) + raw = payload.model_dump(mode="json") + else: + raw = self.redactor.redact_value(payload) + env = build_envelope( + self.keypair, + type=type, + to=to, + payload=raw, + ttl_seconds=ttl_seconds, + idempotency_key=idempotency_key, + in_reply_to=in_reply_to, + clock=self.clock, + ) + self.metrics.envelopes_sent.labels(type=type.value).inc() + return env + + # ------------------------------------------------------------------ # + # Inbound + # ------------------------------------------------------------------ # + + def verify_inbound(self, envelope: Envelope) -> PeerRecord | None: + """Run the §3 verification order. Returns the peer record or None. + + Raises PipError on any failure. + """ + if envelope.pip.split(".", 1)[0] != PROTOCOL_VERSION.split(".", 1)[0]: + raise PipError( + ErrorCode.VERSION_UNSUPPORTED, + f"unsupported pip version {envelope.pip!r}; supported: {SUPPORTED_VERSIONS}", + ) + if len(canonical_json(envelope.payload)) > self.policy.max_payload_bytes: + raise PipError(ErrorCode.PAYLOAD_TOO_LARGE, "payload exceeds max_payload_bytes") + + # identity / kid + if envelope.sig is None: + raise PipError(ErrorCode.SIGNATURE_INVALID, "envelope is unsigned") + kid = envelope.sig.kid + if instance_id_from_public_key(kid) != envelope.from_: + raise PipError(ErrorCode.IDENTITY_MISMATCH, "sig.kid does not hash to envelope.from") + peer = self.engine.get_peer(envelope.from_) + now = self.clock.now() + if peer is not None: + allowed_keys = [peer.public_key] + for prev in peer.previous_keys: + if now < prev.rotated_at + timedelta(seconds=self.policy.rotation_grace_seconds): + allowed_keys.append(prev.public_key) + if kid not in allowed_keys: + raise PipError( + ErrorCode.IDENTITY_MISMATCH, + "kid is not the peer's pinned key or a previous key in grace", + ) + elif not self.policy.allow_unknown_peers: + raise PipError(ErrorCode.UNKNOWN_PEER, "peer is not in the policy file") + + verify_signature(envelope, kid) + + # time window + skew = self.policy.max_clock_skew_seconds + if envelope.expires < envelope.ts or (envelope.expires - envelope.ts) > timedelta( + seconds=MAX_TTL_SECONDS + ): + raise PipError(ErrorCode.SCHEMA_INVALID, "invalid expires window") + if abs((now - envelope.ts).total_seconds()) > skew: + raise PipError(ErrorCode.CLOCK_SKEW, "ts outside max_clock_skew_seconds") + if now > envelope.expires: + raise PipError(ErrorCode.EXPIRED, "envelope has expired") + + # replay + if self.store.seen_nonce(envelope.from_, envelope.nonce, envelope.expires): + raise PipError(ErrorCode.REPLAY, "nonce already seen") + + envelope.typed_payload() # validate payload schema for the declared type + return peer + + def on(self, type: EnvelopeType, fn: Handler) -> None: + self._handlers[type] = fn + + def handle(self, envelope: Envelope) -> tuple[Envelope, PipError | None]: + """Process an inbound envelope; never raises for peer-caused failures. + + Returns ``(response_envelope, error_or_none)``; the response is always a + signed envelope (receipt, data response, or error). + """ + start = time.monotonic() + outcome = "ok" + err: PipError | None = None + try: + peer = self.verify_inbound(envelope) + response = self._dispatch(envelope, peer) + outcome = "error" if response.type == EnvelopeType.ERROR else "accepted" + if response.type == EnvelopeType.ERROR: + p = response.typed_payload() + if isinstance(p, ErrorPayload): + err = PipError(ErrorCode(p.code), p.message, retryable=p.retryable) + return response, err + except PipError as exc: + err = exc + outcome = "error" + return self._error_envelope(envelope, exc), err + except Exception as exc: # handler bug → INTERNAL + err = PipError(ErrorCode.INTERNAL, str(exc)) + outcome = "error" + return self._error_envelope(envelope, err), err + finally: + self.metrics.envelopes_received.labels(type=envelope.type.value, outcome=outcome).inc() + if err is not None: + self.metrics.policy_denials.labels(code=err.code.value).inc() + self.metrics.handler_seconds.labels(type=envelope.type.value).observe( + time.monotonic() - start + ) + log_event( + logger, + "envelope_handled", + envelope_id=envelope.id, + type=envelope.type.value, + peer=envelope.from_, + outcome=outcome, + code=err.code.value if err else None, + duration_ms=round((time.monotonic() - start) * 1000, 3), + ) + + # ------------------------------------------------------------------ # + + def _dispatch(self, envelope: Envelope, peer: PeerRecord | None) -> Envelope: + # idempotent replay short-circuit + if envelope.idempotency_key is not None: + stored = self.store.get_idempotent(envelope.from_, envelope.idempotency_key) + if stored is not None: + receipt = Envelope.model_validate(stored) + rp = receipt.typed_payload() + if isinstance(rp, ReceiptPayload): + rp.status = "duplicate" + receipt.payload = rp.model_dump(mode="json") + receipt.sig = self._re_sign(receipt) + return receipt + + self.engine.check_rate(envelope.from_) + payload = envelope.typed_payload() + + # policy / consent per type + if envelope.type == EnvelopeType.HANDSHAKE: + pass # identity already verified; unknown peers only reach here if allowed + elif envelope.type == EnvelopeType.MESSAGE: + self.engine.authorize(peer, "messages:send") + elif envelope.type == EnvelopeType.DATA: + if not isinstance(payload, DataPayload): + raise PipError(ErrorCode.SCHEMA_INVALID, "invalid data payload") + resource = dataset_resource(payload.dataset) + if payload.op == "request": + self.engine.authorize(peer, "data:request", resource=resource, action="read") + elif payload.op == "offer": + self.engine.authorize(peer, "data:offer", resource=resource, action="write") + else: # response + self.engine.authorize(peer, "data:respond") + else: # receipt / error inbound + self.engine.authorize(peer, "messages:send") + + handler = self._handlers.get(envelope.type) + result = handler(self, envelope, payload) if handler else None + + if result is None: + response = self._receipt(envelope, "accepted") + elif isinstance(result, ReceiptPayload): + response = self._respond(envelope, EnvelopeType.RECEIPT, result) + elif isinstance(result, DataPayload): + response = self._respond(envelope, EnvelopeType.DATA, result) + elif isinstance(result, ErrorPayload): + response = self._respond(envelope, EnvelopeType.ERROR, result) + elif isinstance(result, HandshakePayload): + response = self._respond(envelope, EnvelopeType.HANDSHAKE, result) + else: + response = self._receipt(envelope, "accepted") + + if envelope.idempotency_key is not None and response.type in ( + EnvelopeType.RECEIPT, + EnvelopeType.DATA, + ): + self.store.put_idempotent( + envelope.from_, + envelope.idempotency_key, + response.model_dump(mode="json", by_alias=True), + self.policy.idempotency_ttl_seconds, + ) + return response + + def _re_sign(self, envelope: Envelope) -> Signature: + sig = self.keypair.sign(signing_input(envelope)) + return Signature( + kid=self.keypair.public_key, + value=base64.urlsafe_b64encode(sig).rstrip(b"=").decode("ascii"), + ) + + def _respond(self, envelope: Envelope, type: EnvelopeType, payload: BaseModel) -> Envelope: + return build_envelope( + self.keypair, + type=type, + to=envelope.from_, + payload=payload.model_dump(mode="json"), + in_reply_to=envelope.id, + clock=self.clock, + ) + + def _receipt( + self, + envelope: Envelope, + status: Literal["accepted", "delivered", "rejected", "duplicate"], + reason: str | None = None, + ) -> Envelope: + return self._respond( + envelope, + EnvelopeType.RECEIPT, + ReceiptPayload(ref=envelope.id, status=status, reason=reason), + ) + + def _error_envelope(self, envelope: Envelope, err: PipError) -> Envelope: + payload = ErrorPayload( + code=err.code.value, + message=err.message, + ref=envelope.id, + retryable=err.retryable, + supported_versions=list(SUPPORTED_VERSIONS) + if err.code == ErrorCode.VERSION_UNSUPPORTED + else None, + ) + try: + return self._respond(envelope, EnvelopeType.ERROR, payload) + except PipError: + # even building an error envelope failed; return unsigned + return Envelope( + pip=PROTOCOL_VERSION, + id="error", + type=EnvelopeType.ERROR, + from_=self.instance_id, + to=envelope.from_, + ts=self.clock.now(), + expires=self.clock.now(), + nonce="AAAAAAAAAAAAAAAAAAAAAA", + payload=payload.model_dump(mode="json"), + ) + + +def _default_handshake_handler( + node: Node, envelope: Envelope, payload: BaseModel +) -> BaseModel | None: + return None # accept; discovery uses GET /.well-known/pip + + +def _default_message_handler( + node: Node, envelope: Envelope, payload: BaseModel +) -> BaseModel | None: + node.inbox.append(envelope) + return None + + +def _default_data_handler(node: Node, envelope: Envelope, payload: BaseModel) -> BaseModel | None: + if not isinstance(payload, DataPayload) or payload.op != "request": + return None + if payload.op == "request": + records, next_cursor = node.data_provider.query( + payload.dataset, payload.query, payload.cursor + ) + return DataPayload( + op="response", + dataset=payload.dataset, + schema_version=payload.schema_version, + records=records, + cursor=next_cursor, + ) + return None diff --git a/src/pip_protocol/observability.py b/src/pip_protocol/observability.py new file mode 100644 index 0000000..76b557e --- /dev/null +++ b/src/pip_protocol/observability.py @@ -0,0 +1,72 @@ +"""Metrics and structured logging (spec §8, §10).""" + +import json +import logging +from datetime import datetime, timezone +from typing import Any + +from prometheus_client import CollectorRegistry, Counter, Histogram, generate_latest + + +class Metrics: + """Transport-agnostic metrics with a private registry (test-safe).""" + + def __init__(self) -> None: + self.registry = CollectorRegistry() + self.envelopes_received = Counter( + "envelopes_received_total", + "Envelopes received", + ["type", "outcome"], + registry=self.registry, + ) + self.envelopes_sent = Counter( + "envelopes_sent_total", + "Envelopes sent", + ["type"], + registry=self.registry, + ) + self.policy_denials = Counter( + "policy_denials_total", + "Policy denials", + ["code"], + registry=self.registry, + ) + self.handler_seconds = Histogram( + "handler_seconds", + "Handler duration", + ["type"], + registry=self.registry, + ) + + def render(self) -> str: + return generate_latest(self.registry).decode("utf-8") + + +class _JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + entry: dict[str, Any] = { + "ts": datetime.now(timezone.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "msg": record.getMessage(), + } + extra = getattr(record, "pip_fields", None) + if isinstance(extra, dict): + entry.update(extra) + if record.exc_info: + entry["exc_info"] = self.formatException(record.exc_info) + return json.dumps(entry, ensure_ascii=False) + + +def configure_logging(level: str = "INFO") -> None: + handler = logging.StreamHandler() + handler.setFormatter(_JsonFormatter()) + root = logging.getLogger("pip_protocol") + root.handlers[:] = [handler] + root.setLevel(level.upper()) + + +def log_event(logger: logging.Logger, event: str, **fields: Any) -> None: + """Emit a structured event. Never pass payload bodies, keys, or tokens.""" + safe = {k: v for k, v in fields.items() if k not in {"payload", "body", "key", "token"}} + logger.info(event, extra={"pip_fields": {"event": event, **safe}}) diff --git a/src/pip_protocol/policy.py b/src/pip_protocol/policy.py new file mode 100644 index 0000000..07f3650 --- /dev/null +++ b/src/pip_protocol/policy.py @@ -0,0 +1,151 @@ +"""Policy file models and the authorization/consent/rate-limit engine (§5).""" + +import json +from datetime import datetime +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, ConfigDict + +from .clock import Clock, SystemClock +from .errors import ErrorCode, PipError +from .identity import PreviousKey + +DEFAULT_MAX_PAYLOAD_BYTES = 262144 + + +class Consent(BaseModel): + model_config = ConfigDict(extra="ignore") + + resource: str # e.g. "data:notes" + actions: list[Literal["read", "write"]] + expires: datetime | None = None + granted_by: str = "operator" + + def allows(self, resource: str, action: str, now: datetime) -> bool: + if self.resource != resource or action not in self.actions: + return False + return self.expires is None or now < self.expires + + +class PeerRecord(BaseModel): + model_config = ConfigDict(extra="ignore") + + instance_id: str + public_key: str + display_name: str = "" + scopes: list[str] = [] + consents: list[Consent] = [] + previous_keys: list[PreviousKey] = [] + + +class RateLimits(BaseModel): + per_peer_per_minute: int = 60 + burst: int = 20 + + +class RedactionConfig(BaseModel): + fields: list[str] = [] + patterns: list[str] = [] + + +class Policy(BaseModel): + model_config = ConfigDict(extra="ignore") + + allow_unknown_peers: bool = False + default_scopes: list[str] = [] + peers: list[PeerRecord] = [] + rate_limits: RateLimits = RateLimits() + max_payload_bytes: int = DEFAULT_MAX_PAYLOAD_BYTES + redaction: RedactionConfig = RedactionConfig() + rotation_grace_seconds: int = 86400 + max_clock_skew_seconds: int = 300 + idempotency_ttl_seconds: int = 86400 + + @classmethod + def load(cls, path: str | Path) -> "Policy": + text = Path(path).read_text() + if str(path).endswith(".json"): + data = json.loads(text) + else: + data = yaml.safe_load(text) + return cls.model_validate(data or {}) + + +def dataset_resource(dataset: str) -> str: + return f"data:{dataset}" + + +def _scope_grants(granted: str, required: str) -> bool: + if granted == required: + return True + if granted.endswith(":*"): + prefix = granted[:-1] # e.g. "data:" + return required.startswith(prefix) or prefix == "admin:" + return False + + +class _TokenBucket: + def __init__(self, rate_per_sec: float, capacity: float, now: float) -> None: + self.rate = rate_per_sec + self.capacity = capacity + self.tokens = capacity + self.updated = now + + +class PolicyEngine: + """Single enforcement point shared by all transports (spec §5).""" + + def __init__(self, policy: Policy, clock: Clock | None = None) -> None: + self.policy = policy + self.clock = clock or SystemClock() + self._buckets: dict[str, _TokenBucket] = {} + + def get_peer(self, instance_id: str) -> PeerRecord | None: + for peer in self.policy.peers: + if peer.instance_id == instance_id: + return peer + return None + + def _has_scope(self, peer: PeerRecord, scope: str) -> bool: + granted = list(peer.scopes) + list(self.policy.default_scopes) + return any(_scope_grants(g, scope) for g in granted) + + def authorize( + self, + peer: PeerRecord | None, + scope: str, + *, + resource: str | None = None, + action: str | None = None, + ) -> PeerRecord: + """Return the peer or raise UNKNOWN_PEER / FORBIDDEN / CONSENT_REQUIRED.""" + if peer is None: + raise PipError(ErrorCode.UNKNOWN_PEER, "peer is not in the policy file") + if not self._has_scope(peer, scope): + raise PipError(ErrorCode.FORBIDDEN, f"peer {peer.instance_id} lacks scope {scope!r}") + if resource is not None and action is not None: + now = self.clock.now() + if not any(c.allows(resource, action, now) for c in peer.consents): + raise PipError( + ErrorCode.CONSENT_REQUIRED, + f"no consent for {action} on {resource!r}", + ) + return peer + + def check_rate(self, peer_id: str) -> None: + """Token-bucket per peer; raises RATE_LIMITED when empty.""" + limits = self.policy.rate_limits + rate = max(limits.per_peer_per_minute, 1) / 60.0 + capacity = float(max(limits.burst, 1)) + now = self.clock.now().timestamp() + bucket = self._buckets.get(peer_id) + if bucket is None: + bucket = _TokenBucket(rate, capacity, now) + self._buckets[peer_id] = bucket + bucket.tokens = min(capacity, bucket.tokens + (now - bucket.updated) * bucket.rate) + bucket.updated = now + if bucket.tokens < 1.0: + raise PipError(ErrorCode.RATE_LIMITED, f"rate limit exceeded for {peer_id}") + bucket.tokens -= 1.0 diff --git a/src/pip_protocol/redaction.py b/src/pip_protocol/redaction.py new file mode 100644 index 0000000..f9a2469 --- /dev/null +++ b/src/pip_protocol/redaction.py @@ -0,0 +1,50 @@ +"""Outbound payload redaction (spec §6).""" + +import re +from typing import Any, TypeVar + +from pydantic import BaseModel + +from .policy import RedactionConfig +from .schemas import DataPayload, EnvelopeType, MessagePayload + +MASK = "[REDACTED]" +_T = TypeVar("_T", bound=BaseModel) + + +class Redactor: + def __init__(self, config: RedactionConfig) -> None: + self.fields = {f.lower() for f in config.fields} + self.patterns = [re.compile(p) for p in config.patterns] + + def redact_value(self, obj: Any) -> Any: + """Recursively redact field-name matches and regex matches.""" + if isinstance(obj, dict): + out: dict[Any, Any] = {} + for key, value in obj.items(): + if isinstance(key, str) and key.lower() in self.fields: + out[key] = MASK + else: + out[key] = self.redact_value(value) + return out + if isinstance(obj, list): + return [self.redact_value(v) for v in obj] + if isinstance(obj, str): + for pattern in self.patterns: + obj = pattern.sub(MASK, obj) + return obj + return obj + + def redact_payload(self, type: EnvelopeType, payload: _T) -> _T: + """Return a redacted copy of a message or data payload.""" + if isinstance(payload, MessagePayload): + return payload.model_copy( + update={ + "subject": self.redact_value(payload.subject), + "body": self.redact_value(payload.body), + "attributes": self.redact_value(payload.attributes), + } + ) + if isinstance(payload, DataPayload): + return payload.model_copy(update={"records": self.redact_value(payload.records)}) + return payload diff --git a/src/pip_protocol/schemas.py b/src/pip_protocol/schemas.py new file mode 100644 index 0000000..b74c352 --- /dev/null +++ b/src/pip_protocol/schemas.py @@ -0,0 +1,163 @@ +"""Pydantic v2 wire models for envelopes and payloads (spec §3).""" + +import base64 +from datetime import datetime, timezone +from enum import Enum +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic_core import PydanticCustomError + +from .errors import ErrorCode, PipError +from .identity import Capability, IdentityDocument + +MAX_BODY_CHARS = 65536 +MAX_SUBJECT_CHARS = 200 +MAX_RECORDS = 1000 +MAX_IDEMPOTENCY_KEY_CHARS = 128 + + +class EnvelopeType(str, Enum): + HANDSHAKE = "handshake" + MESSAGE = "message" + DATA = "data" + RECEIPT = "receipt" + ERROR = "error" + + +class Signature(BaseModel): + model_config = ConfigDict(extra="ignore") + + alg: Literal["ed25519"] = "ed25519" + kid: str + value: str + + +class HandshakePayload(BaseModel): + model_config = ConfigDict(extra="ignore") + + identity: IdentityDocument + capabilities: list[Capability] = [] + + +class MessagePayload(BaseModel): + model_config = ConfigDict(extra="ignore") + + subject: str = Field(max_length=MAX_SUBJECT_CHARS) + body: str = Field(max_length=MAX_BODY_CHARS) + content_type: Literal["text/plain", "text/markdown", "application/json"] = "text/plain" + attributes: dict[str, str] = {} + + +class DataPayload(BaseModel): + model_config = ConfigDict(extra="ignore") + + op: Literal["offer", "request", "response"] + dataset: str + schema_version: str = "1.0" + records: Annotated[list[dict[str, Any]], Field(max_length=MAX_RECORDS)] = [] + cursor: str | None = None + query: dict[str, Any] | None = None + + +class ReceiptPayload(BaseModel): + model_config = ConfigDict(extra="ignore") + + ref: str + status: Literal["accepted", "delivered", "rejected", "duplicate"] + reason: str | None = None + + +class ErrorPayload(BaseModel): + model_config = ConfigDict(extra="ignore") + + code: str + message: str + ref: str | None = None + retryable: bool = False + supported_versions: list[str] | None = None + + +_PAYLOAD_MODELS: dict[EnvelopeType, type[BaseModel]] = { + EnvelopeType.HANDSHAKE: HandshakePayload, + EnvelopeType.MESSAGE: MessagePayload, + EnvelopeType.DATA: DataPayload, + EnvelopeType.RECEIPT: ReceiptPayload, + EnvelopeType.ERROR: ErrorPayload, +} + + +def _require_utc(v: datetime) -> datetime: + if v.tzinfo is None: + raise PydanticCustomError("timezone_required", "datetime must be timezone-aware (UTC)") + return v.astimezone(timezone.utc) + + +class Envelope(BaseModel): + """The signed unit of exchange. Unknown fields are ignored per §11.""" + + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + pip: str + id: str + type: EnvelopeType + from_: str = Field(alias="from") + to: str + ts: datetime + expires: datetime + nonce: str + idempotency_key: str | None = Field(default=None, max_length=MAX_IDEMPOTENCY_KEY_CHARS) + in_reply_to: str | None = None + payload: dict[str, Any] = {} + sig: Signature | None = None + + @field_validator("ts", "expires") + @classmethod + def _aware(cls, v: datetime) -> datetime: + return _require_utc(v) + + @field_validator("nonce") + @classmethod + def _nonce_b64(cls, v: str) -> str: + try: + raw = base64.urlsafe_b64decode(v + "=" * (-len(v) % 4)) + except Exception as exc: + raise PydanticCustomError("nonce_invalid", "nonce must be base64url") from exc + if len(raw) != 16: + raise PydanticCustomError("nonce_invalid", "nonce must decode to 16 bytes") + return v + + @property + def from_id(self) -> str: + return self.from_ + + def typed_payload(self) -> BaseModel: + model = _PAYLOAD_MODELS[self.type] + try: + return model.model_validate(self.payload) + except Exception as exc: + raise PipError(ErrorCode.SCHEMA_INVALID, f"invalid {self.type} payload: {exc}") from exc + + def without_sig(self) -> dict[str, Any]: + d = self.model_dump(mode="json", by_alias=True) + d.pop("sig", None) + return d + + +def parse_envelope(data: dict[str, Any]) -> Envelope: + try: + return Envelope.model_validate(data) + except Exception as exc: + raise PipError(ErrorCode.SCHEMA_INVALID, f"invalid envelope: {exc}") from exc + + +def new_nonce() -> str: + import secrets + + return base64.urlsafe_b64encode(secrets.token_bytes(16)).rstrip(b"=").decode("ascii") + + +def new_id() -> str: + import uuid + + return str(uuid.uuid4()) diff --git a/src/pip_protocol/store.py b/src/pip_protocol/store.py new file mode 100644 index 0000000..990e2b0 --- /dev/null +++ b/src/pip_protocol/store.py @@ -0,0 +1,202 @@ +"""Replay/idempotency/outbox storage backends (spec §7).""" + +import json +import sqlite3 +import threading +from datetime import datetime, timezone +from typing import Any, Protocol + + +class Store(Protocol): + """Persistence interface for replay protection, idempotency, and the outbox.""" + + def seen_nonce(self, peer: str, nonce: str, expires_at: datetime) -> bool: + """Atomically record (peer, nonce); return True if already present.""" + ... + + def get_idempotent(self, peer: str, key: str) -> dict[str, Any] | None: ... + + def put_idempotent(self, peer: str, key: str, receipt: dict[str, Any], ttl: int) -> None: ... + + def purge(self, now: datetime) -> None: ... + + def outbox_put(self, item: dict[str, Any]) -> None: ... + + def outbox_due(self, now: datetime) -> list[dict[str, Any]]: ... + + def outbox_update(self, item: dict[str, Any]) -> None: ... + + def outbox_delete(self, item_id: str) -> None: ... + + +def _ts(dt: datetime) -> float: + return dt.astimezone(timezone.utc).timestamp() + + +def _dt(ts: float) -> datetime: + return datetime.fromtimestamp(ts, tz=timezone.utc) + + +def _parse_dt(text: str) -> datetime: + return datetime.fromisoformat(text.replace("Z", "+00:00")) + + +class MemoryStore: + def __init__(self) -> None: + self._lock = threading.Lock() + self._nonces: dict[tuple[str, str], float] = {} + self._idempotent: dict[tuple[str, str], tuple[float, dict[str, Any]]] = {} + self._outbox: dict[str, dict[str, Any]] = {} + + def seen_nonce(self, peer: str, nonce: str, expires_at: datetime) -> bool: + with self._lock: + key = (peer, nonce) + if key in self._nonces: + return True + self._nonces[key] = _ts(expires_at) + return False + + def get_idempotent(self, peer: str, key: str) -> dict[str, Any] | None: + with self._lock: + entry = self._idempotent.get((peer, key)) + if entry is None: + return None + expires, receipt = entry + if expires < _ts(datetime.now(timezone.utc)): + return None + return receipt + + def put_idempotent(self, peer: str, key: str, receipt: dict[str, Any], ttl: int) -> None: + with self._lock: + self._idempotent[(peer, key)] = ( + _ts(datetime.now(timezone.utc)) + ttl, + receipt, + ) + + def purge(self, now: datetime) -> None: + with self._lock: + ts = _ts(now) + self._nonces = {k: v for k, v in self._nonces.items() if v > ts} + self._idempotent = {k: v for k, v in self._idempotent.items() if v[0] > ts} + + def outbox_put(self, item: dict[str, Any]) -> None: + with self._lock: + self._outbox[item["id"]] = dict(item) + + def outbox_due(self, now: datetime) -> list[dict[str, Any]]: + with self._lock: + ts = _ts(now) + return [ + dict(i) for i in self._outbox.values() if _ts(_parse_dt(i["next_attempt_at"])) <= ts + ] + + def outbox_update(self, item: dict[str, Any]) -> None: + with self._lock: + self._outbox[item["id"]] = dict(item) + + def outbox_delete(self, item_id: str) -> None: + with self._lock: + self._outbox.pop(item_id, None) + + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS nonces ( + peer TEXT NOT NULL, + nonce TEXT NOT NULL, + expires_at REAL NOT NULL, + PRIMARY KEY (peer, nonce) +); +CREATE TABLE IF NOT EXISTS idempotent ( + peer TEXT NOT NULL, + key TEXT NOT NULL, + expires_at REAL NOT NULL, + receipt TEXT NOT NULL, + PRIMARY KEY (peer, key) +); +CREATE TABLE IF NOT EXISTS outbox ( + id TEXT PRIMARY KEY, + item TEXT NOT NULL, + next_attempt_at REAL NOT NULL +); +""" + + +class SqliteStore: + def __init__(self, path: str) -> None: + self._lock = threading.Lock() + self._conn = sqlite3.connect(path, check_same_thread=False) + self._conn.execute("PRAGMA journal_mode=WAL") + with self._conn: + self._conn.executescript(_SCHEMA) + + def close(self) -> None: + self._conn.close() + + def seen_nonce(self, peer: str, nonce: str, expires_at: datetime) -> bool: + with self._lock, self._conn: + try: + self._conn.execute( + "INSERT INTO nonces (peer, nonce, expires_at) VALUES (?, ?, ?)", + (peer, nonce, _ts(expires_at)), + ) + return False + except sqlite3.IntegrityError: + return True + + def get_idempotent(self, peer: str, key: str) -> dict[str, Any] | None: + with self._lock: + row = self._conn.execute( + "SELECT expires_at, receipt FROM idempotent WHERE peer = ? AND key = ?", + (peer, key), + ).fetchone() + if row is None or row[0] < _ts(datetime.now(timezone.utc)): + return None + return json.loads(row[1]) # type: ignore[no-any-return] + + def put_idempotent(self, peer: str, key: str, receipt: dict[str, Any], ttl: int) -> None: + with self._lock, self._conn: + self._conn.execute( + "INSERT OR REPLACE INTO idempotent (peer, key, expires_at, receipt)" + " VALUES (?, ?, ?, ?)", + (peer, key, _ts(datetime.now(timezone.utc)) + ttl, json.dumps(receipt)), + ) + + def purge(self, now: datetime) -> None: + with self._lock, self._conn: + ts = _ts(now) + self._conn.execute("DELETE FROM nonces WHERE expires_at <= ?", (ts,)) + self._conn.execute("DELETE FROM idempotent WHERE expires_at <= ?", (ts,)) + + def outbox_put(self, item: dict[str, Any]) -> None: + self.outbox_update(item) + + def outbox_due(self, now: datetime) -> list[dict[str, Any]]: + with self._lock: + rows = self._conn.execute( + "SELECT item FROM outbox WHERE next_attempt_at <= ? ORDER BY next_attempt_at", + (_ts(now),), + ).fetchall() + return [json.loads(r[0]) for r in rows] + + def outbox_update(self, item: dict[str, Any]) -> None: + with self._lock, self._conn: + self._conn.execute( + "INSERT OR REPLACE INTO outbox (id, item, next_attempt_at) VALUES (?, ?, ?)", + ( + item["id"], + json.dumps(item), + _ts(_parse_dt(item["next_attempt_at"])), + ), + ) + + def outbox_delete(self, item_id: str) -> None: + with self._lock, self._conn: + self._conn.execute("DELETE FROM outbox WHERE id = ?", (item_id,)) + + +def open_store(url: str) -> Store: + if url == "memory://": + return MemoryStore() + if url.startswith("sqlite:///"): + return SqliteStore(url[len("sqlite:///") :]) + raise ValueError(f"unsupported store url: {url!r}") From 1972876b80f98a1377e806cbf4033f2082c8231c Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:32:29 +0000 Subject: [PATCH 3/9] test: core protocol test suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/conftest.py | 30 +++++ tests/test_canonical.py | 22 ++++ tests/test_envelope.py | 96 ++++++++++++++ tests/test_identity.py | 101 +++++++++++++++ tests/test_node.py | 277 ++++++++++++++++++++++++++++++++++++++++ tests/test_policy.py | 138 ++++++++++++++++++++ tests/test_redaction.py | 42 ++++++ tests/test_store.py | 90 +++++++++++++ 8 files changed, 796 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/test_canonical.py create mode 100644 tests/test_envelope.py create mode 100644 tests/test_identity.py create mode 100644 tests/test_node.py create mode 100644 tests/test_policy.py create mode 100644 tests/test_redaction.py create mode 100644 tests/test_store.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7add730 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,30 @@ +import pytest + +from pip_protocol.clock import FakeClock +from pip_protocol.identity import KeyPair +from pip_protocol.policy import Consent, PeerRecord + + +@pytest.fixture +def clock() -> FakeClock: + return FakeClock() + + +@pytest.fixture +def keypair() -> KeyPair: + return KeyPair.generate() + + +def make_peer(kp: KeyPair, scopes=None, consents=None) -> PeerRecord: + return PeerRecord( + instance_id=kp.instance_id, + public_key=kp.public_key, + display_name="peer", + scopes=scopes or [], + consents=consents or [], + ) + + +@pytest.fixture +def consent_read_notes() -> Consent: + return Consent(resource="data:notes", actions=["read"], expires=None) diff --git a/tests/test_canonical.py b/tests/test_canonical.py new file mode 100644 index 0000000..3a5a02b --- /dev/null +++ b/tests/test_canonical.py @@ -0,0 +1,22 @@ +import pytest + +from pip_protocol.canonical import canonical_json + + +def test_sorted_keys_no_whitespace(): + assert canonical_json({"b": 1, "a": 2}) == b'{"a":2,"b":1}' + + +def test_nested_determinism(): + a = {"x": [{"z": 1, "y": 2}], "b": {"d": 4, "c": 3}} + b = {"b": {"c": 3, "d": 4}, "x": [{"y": 2, "z": 1}]} + assert canonical_json(a) == canonical_json(b) + + +def test_utf8_preserved(): + assert "é".encode() in canonical_json({"k": "é"}) + + +def test_nan_rejected(): + with pytest.raises(ValueError): + canonical_json({"x": float("nan")}) diff --git a/tests/test_envelope.py b/tests/test_envelope.py new file mode 100644 index 0000000..247e50a --- /dev/null +++ b/tests/test_envelope.py @@ -0,0 +1,96 @@ +import base64 + +import pytest + +from pip_protocol.envelope import build_envelope, signing_input, verify_signature +from pip_protocol.errors import ErrorCode, PipError +from pip_protocol.schemas import ( + EnvelopeType, + MessagePayload, + parse_envelope, +) + + +def make_message(kp, to="pip:x", clock=None, **kw): + return build_envelope( + kp, + type=EnvelopeType.MESSAGE, + to=to, + payload=MessagePayload(subject="hi", body="hello").model_dump(mode="json"), + clock=clock, + **kw, + ) + + +def test_build_and_verify(keypair, clock): + env = make_message(keypair, clock=clock) + assert env.sig is not None + verify_signature(env, keypair.public_key) + + +def test_roundtrip_wire(keypair, clock): + env = make_message(keypair, clock=clock) + wire = env.model_dump(mode="json", by_alias=True) + assert wire["from"] == keypair.instance_id + parsed = parse_envelope(wire) + verify_signature(parsed, keypair.public_key) + + +def test_ts_serialised_z(keypair, clock): + wire = make_message(keypair, clock=clock).model_dump(mode="json", by_alias=True) + assert wire["ts"].endswith("Z") or wire["ts"].endswith("+00:00") + + +def test_tamper_detected(keypair, clock): + env = make_message(keypair, clock=clock) + wire = env.model_dump(mode="json", by_alias=True) + wire["payload"]["body"] = "evil" + parsed = parse_envelope(wire) + with pytest.raises(PipError) as e: + verify_signature(parsed, keypair.public_key) + assert e.value.code == ErrorCode.SIGNATURE_INVALID + + +def test_unsigned_raises(keypair, clock): + env = make_message(keypair, clock=clock) + env.sig = None + with pytest.raises(PipError): + verify_signature(env, keypair.public_key) + + +def test_ttl_cap(keypair, clock): + with pytest.raises(PipError): + make_message(keypair, clock=clock, ttl_seconds=7200) + + +def test_nonce_is_16_bytes(keypair, clock): + env = make_message(keypair, clock=clock) + raw = base64.urlsafe_b64decode(env.nonce + "=" * (-len(env.nonce) % 4)) + assert len(raw) == 16 + + +def test_signing_input_prefix(keypair, clock): + env = make_message(keypair, clock=clock) + assert signing_input(env).startswith(b"PIPv1\n") + + +def test_typed_payload(keypair, clock): + env = make_message(keypair, clock=clock) + p = env.typed_payload() + assert isinstance(p, MessagePayload) + assert p.body == "hello" + + +def test_typed_payload_invalid(keypair, clock): + env = make_message(keypair, clock=clock) + env.payload = {"nope": 1} + # missing required fields + with pytest.raises(PipError) as e: + env.typed_payload() + assert e.value.code == ErrorCode.SCHEMA_INVALID + + +def test_schema_invalid_on_bad_envelope(): + with pytest.raises(PipError) as e: + parse_envelope({"type": "message"}) + assert e.value.code == ErrorCode.SCHEMA_INVALID diff --git a/tests/test_identity.py b/tests/test_identity.py new file mode 100644 index 0000000..df3b539 --- /dev/null +++ b/tests/test_identity.py @@ -0,0 +1,101 @@ +import os +import stat + +import pytest + +from pip_protocol.errors import PipError +from pip_protocol.identity import ( + IdentityDocument, + InsecureKeyFileError, + KeyPair, + decode_public_key, + instance_id_from_public_key, + verify, +) + + +def test_keygen_and_sign_verify(): + kp = KeyPair.generate() + sig = kp.sign(b"hello") + assert verify(kp.public_key, b"hello", sig) + + +def test_verify_rejects_wrong_data(): + kp = KeyPair.generate() + sig = kp.sign(b"hello") + assert not verify(kp.public_key, b"tampered", sig) + + +def test_public_key_roundtrip(): + kp = KeyPair.generate() + assert kp.public_key.startswith("ed25519:") + decoded = decode_public_key(kp.public_key) + assert decoded.public_bytes_raw() == kp.public_bytes + + +def test_decode_public_key_bad_prefix(): + with pytest.raises(PipError): + decode_public_key("rsa:AAAA") + + +def test_instance_id_derivation(): + kp = KeyPair.generate() + iid = instance_id_from_public_key(kp.public_key) + assert iid.startswith("pip:") + assert len(iid) == 4 + 26 + assert iid == iid.lower() + + +def test_save_load_roundtrip(tmp_path): + kp = KeyPair.generate() + path = tmp_path / "instance.key" + kp.save(path) + mode = stat.S_IMODE(os.stat(path).st_mode) + assert mode == 0o600 + loaded = KeyPair.load(path) + assert loaded.public_key == kp.public_key + + +def test_load_insecure_perms_raises(tmp_path): + kp = KeyPair.generate() + path = tmp_path / "instance.key" + kp.save(path) + os.chmod(path, 0o644) + with pytest.raises(InsecureKeyFileError): + KeyPair.load(path) + + +def test_load_insecure_perms_allowed(tmp_path): + kp = KeyPair.generate() + path = tmp_path / "instance.key" + kp.save(path) + os.chmod(path, 0o644) + loaded = KeyPair.load(path, allow_insecure=True) + assert loaded.public_key == kp.public_key + + +def test_identity_document_valid(keypair, clock): + doc = IdentityDocument( + instance_id=keypair.instance_id, + public_key=keypair.public_key, + display_name="test", + pip_versions=["1.0"], + endpoints={}, + previous_keys=[], + issued_at=clock.now(), + ) + assert doc.instance_id == keypair.instance_id + + +def test_identity_document_mismatch(keypair, clock): + other = KeyPair.generate() + with pytest.raises(PipError): + IdentityDocument( + instance_id=other.instance_id, + public_key=keypair.public_key, + display_name="test", + pip_versions=["1.0"], + endpoints={}, + previous_keys=[], + issued_at=clock.now(), + ) diff --git a/tests/test_node.py b/tests/test_node.py new file mode 100644 index 0000000..1258264 --- /dev/null +++ b/tests/test_node.py @@ -0,0 +1,277 @@ +import base64 + +from conftest import make_peer +from pip_protocol.clock import FakeClock +from pip_protocol.envelope import signing_input, verify_signature +from pip_protocol.errors import ErrorCode +from pip_protocol.identity import KeyPair +from pip_protocol.node import DictDataProvider, Node +from pip_protocol.policy import Consent, Policy +from pip_protocol.schemas import ( + DataPayload, + Envelope, + EnvelopeType, + ErrorPayload, + MessagePayload, + Signature, + parse_envelope, +) + + +def make_nodes(clock, *, scopes=None, consents=None, datasets=None, b_extra=None): + ka, kb = KeyPair.generate(), KeyPair.generate() + policy = Policy( + peers=[ + make_peer( + ka, + scopes if scopes is not None else ["messages:send"], + consents or [], + ) + ] + ) + node_a = Node(ka, Policy(peers=[make_peer(kb, ["admin:*"])]), clock=clock) + node_b = Node( + kb, + policy, + clock=clock, + data_provider=DictDataProvider(datasets or {}), + **(b_extra or {}), + ) + return node_a, node_b + + +def test_handshake_self_addressed(clock): + node = Node(KeyPair.generate(), clock=clock) + env = node.handshake_envelope() + assert env.to == node.instance_id + verify_signature(env, node.keypair.public_key) + p = env.typed_payload() + assert p.identity.instance_id == node.instance_id + + +def test_message_end_to_end(clock): + a, b = make_nodes(clock) + env = a.build( + EnvelopeType.MESSAGE, + b.instance_id, + MessagePayload(subject="hi", body="hello b"), + ) + resp, err = b.handle(env) + assert err is None + assert resp.type == EnvelopeType.RECEIPT + rp = resp.typed_payload() + assert rp.ref == env.id and rp.status == "accepted" + verify_signature(resp, b.keypair.public_key) + assert len(b.inbox) == 1 + + +def test_message_forbidden_without_scope(clock): + a, b = make_nodes(clock, scopes=[]) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + resp, err = b.handle(env) + assert err is not None and err.code == ErrorCode.FORBIDDEN + assert resp.type == EnvelopeType.ERROR + ep = resp.typed_payload() + assert isinstance(ep, ErrorPayload) + assert ep.code == "FORBIDDEN" and ep.ref == env.id + verify_signature(resp, b.keypair.public_key) + + +def test_unknown_peer(clock): + a = Node(KeyPair.generate(), clock=clock) + b = Node(KeyPair.generate(), Policy(), clock=clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + resp, err = b.handle(env) + assert err is not None and err.code == ErrorCode.UNKNOWN_PEER + assert resp.type == EnvelopeType.ERROR + + +def test_version_unsupported(clock): + a, b = make_nodes(clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + wire = env.model_dump(mode="json", by_alias=True) + wire["pip"] = "2.0" + # re-sign so the signature covers the tampered version + forged = parse_envelope(wire) + forged.sig = Signature( + kid=a.keypair.public_key, + value=base64.urlsafe_b64encode(a.keypair.sign(signing_input(forged))).decode(), + ) + resp, err = b.handle(forged) + assert err is not None and err.code == ErrorCode.VERSION_UNSUPPORTED + + +def test_expired_envelope(clock): + a, b = make_nodes(clock) + env = a.build( + EnvelopeType.MESSAGE, + b.instance_id, + MessagePayload(subject="s", body="x"), + ttl_seconds=60, + ) + clock.advance(120) + _, err = b.handle(env) + assert err is not None and err.code == ErrorCode.EXPIRED + + +def test_clock_skew(clock): + a_clock, b_clock = FakeClock(), FakeClock() + ka, kb = KeyPair.generate(), KeyPair.generate() + a = Node(ka, Policy(peers=[make_peer(kb, ["admin:*"])]), clock=a_clock) + b = Node(kb, Policy(peers=[make_peer(ka, ["messages:send"])]), clock=b_clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + b_clock.advance(400) # beyond 300s skew + _, err = b.handle(env) + assert err is not None and err.code == ErrorCode.CLOCK_SKEW + + +def test_replay_detected(clock): + a, b = make_nodes(clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + b.handle(env) + _, err = b.handle(env) + assert err is not None and err.code == ErrorCode.REPLAY + + +def test_idempotent_duplicate(clock): + a, b = make_nodes(clock) + env = a.build( + EnvelopeType.MESSAGE, + b.instance_id, + MessagePayload(subject="s", body="x"), + idempotency_key="key-1", + ) + resp1, _ = b.handle(env) + env2 = a.build( + EnvelopeType.MESSAGE, + b.instance_id, + MessagePayload(subject="s", body="x"), + idempotency_key="key-1", + ) + resp2, err = b.handle(env2) + assert err is None + assert resp2.typed_payload().status == "duplicate" + assert resp2.typed_payload().ref == env.id + assert len(b.inbox) == 1 # handler did not re-run + + +def test_data_request_response(clock): + consent = Consent(resource="data:notes", actions=["read"]) + a, b = make_nodes( + clock, + scopes=["data:request"], + consents=[consent], + datasets={"notes": [{"n": i} for i in range(5)]}, + ) + env = a.build( + EnvelopeType.DATA, + b.instance_id, + DataPayload(op="request", dataset="notes", query={"limit": 3}), + ) + resp, err = b.handle(env) + assert err is None + assert resp.type == EnvelopeType.DATA + p = resp.typed_payload() + assert isinstance(p, DataPayload) + assert p.op == "response" + assert len(p.records) == 3 + assert p.cursor == "3" + verify_signature(resp, b.keypair.public_key) + + +def test_data_request_paging(clock): + consent = Consent(resource="data:notes", actions=["read"]) + a, b = make_nodes( + clock, + scopes=["data:request"], + consents=[consent], + datasets={"notes": [{"n": i} for i in range(5)]}, + ) + env = a.build( + EnvelopeType.DATA, + b.instance_id, + DataPayload(op="request", dataset="notes", query={"limit": 3}, cursor="3"), + ) + resp, _ = b.handle(env) + p = resp.typed_payload() + assert len(p.records) == 2 and p.cursor is None + + +def test_data_request_consent_required(clock): + a, b = make_nodes(clock, scopes=["data:request"], datasets={"notes": []}) + env = a.build(EnvelopeType.DATA, b.instance_id, DataPayload(op="request", dataset="notes")) + resp, err = b.handle(env) + assert err is not None and err.code == ErrorCode.CONSENT_REQUIRED + assert resp.type == EnvelopeType.ERROR + + +def test_data_offer_write_consent(clock): + consent = Consent(resource="data:notes", actions=["write"]) + a, b = make_nodes(clock, scopes=["data:offer"], consents=[consent]) + env = a.build( + EnvelopeType.DATA, + b.instance_id, + DataPayload(op="offer", dataset="notes", records=[{"x": 1}]), + ) + resp, err = b.handle(env) + assert err is None and resp.typed_payload().status == "accepted" + + +def test_identity_mismatch(clock): + a, b = make_nodes(clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + wire = env.model_dump(mode="json", by_alias=True) + other = KeyPair.generate() + wire["from"] = other.instance_id # kid no longer hashes to from + forged = parse_envelope(wire) + _, err = b.handle(forged) + assert err is not None and err.code == ErrorCode.IDENTITY_MISMATCH + + +def test_signature_invalid(clock): + a, b = make_nodes(clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + env.payload["body"] = "tampered" + _, err = b.handle(env) + assert err is not None and err.code == ErrorCode.SIGNATURE_INVALID + + +def test_error_http_status_mapping(): + from pip_protocol.errors import PipError + + assert PipError(ErrorCode.FORBIDDEN).http_status == 403 + assert PipError(ErrorCode.REPLAY).http_status == 409 + assert PipError(ErrorCode.RATE_LIMITED).http_status == 429 + assert PipError(ErrorCode.INTERNAL).http_status == 500 + assert PipError(ErrorCode.RATE_LIMITED).retryable + + +def test_metrics_recorded(clock): + a, b = make_nodes(clock) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + b.handle(env) + text = b.metrics.render() + assert 'envelopes_received_total{outcome="accepted",type="message"} 1.0' in text + + +def test_outbound_redaction_before_signing(clock): + ka, kb = KeyPair.generate(), KeyPair.generate() + policy = Policy( + peers=[make_peer(kb, ["admin:*"])], + redaction={"fields": ["password"], "patterns": []}, + ) + a = Node(ka, policy, clock=clock) + env = a.build( + EnvelopeType.MESSAGE, + "pip:x", + MessagePayload(subject="s", body="x", attributes={"Password": "hunter2"}), # noqa: S105 + ) + assert env.payload["attributes"]["Password"] == "[REDACTED]" # noqa: S105 + verify_signature(env, ka.public_key) # signature covers redacted content + + +def test_envelope_parse_roundtrip(clock): + a, _ = make_nodes(clock) + env = a.build(EnvelopeType.MESSAGE, "pip:x", MessagePayload(subject="s", body="x")) + env2 = Envelope.model_validate(env.model_dump(mode="json", by_alias=True)) + assert env2.id == env.id diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100644 index 0000000..00e21f1 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,138 @@ +import pytest + +from conftest import make_peer +from pip_protocol.errors import ErrorCode, PipError +from pip_protocol.policy import ( + Consent, + PeerRecord, + Policy, + PolicyEngine, + dataset_resource, +) + + +def engine_for(kp, scopes=(), consents=(), **kw): + policy = Policy(peers=[make_peer(kp, list(scopes), list(consents))], **kw) + return PolicyEngine(policy) + + +def test_unknown_peer(keypair, clock): + engine = PolicyEngine(Policy(), clock) + with pytest.raises(PipError) as e: + engine.authorize(None, "messages:send") + assert e.value.code == ErrorCode.UNKNOWN_PEER + + +def test_scope_exact(keypair, clock): + engine = PolicyEngine(Policy(peers=[make_peer(keypair, ["messages:send"])]), clock) + engine.authorize(engine.get_peer(keypair.instance_id), "messages:send") + with pytest.raises(PipError) as e: + engine.authorize(engine.get_peer(keypair.instance_id), "data:request") + assert e.value.code == ErrorCode.FORBIDDEN + + +def test_scope_wildcard(keypair, clock): + engine = PolicyEngine(Policy(peers=[make_peer(keypair, ["data:*"])]), clock) + peer = engine.get_peer(keypair.instance_id) + engine.authorize(peer, "data:request") + engine.authorize(peer, "data:offer") + with pytest.raises(PipError): + engine.authorize(peer, "messages:send") + + +def test_admin_wildcard_implies_all(keypair, clock): + engine = PolicyEngine(Policy(peers=[make_peer(keypair, ["admin:*"])]), clock) + peer = engine.get_peer(keypair.instance_id) + engine.authorize(peer, "messages:send") + engine.authorize(peer, "data:request") + + +def test_default_scopes_apply(keypair, clock): + engine = PolicyEngine( + Policy(default_scopes=["messages:send"], peers=[make_peer(keypair)]), clock + ) + engine.authorize(engine.get_peer(keypair.instance_id), "messages:send") + + +def test_consent_required(keypair, clock): + engine = engine_for(keypair, scopes=["data:request"]) + peer = engine.get_peer(keypair.instance_id) + with pytest.raises(PipError) as e: + engine.authorize(peer, "data:request", resource="data:notes", action="read") + assert e.value.code == ErrorCode.CONSENT_REQUIRED + + +def test_consent_granted(keypair, clock): + consent = Consent(resource="data:notes", actions=["read"]) + engine = engine_for(keypair, scopes=["data:request"], consents=[consent]) + peer = engine.get_peer(keypair.instance_id) + engine.authorize(peer, "data:request", resource="data:notes", action="read") + + +def test_consent_wrong_action(keypair, clock): + consent = Consent(resource="data:notes", actions=["write"]) + engine = engine_for(keypair, scopes=["data:request"], consents=[consent]) + peer = engine.get_peer(keypair.instance_id) + with pytest.raises(PipError) as e: + engine.authorize(peer, "data:request", resource="data:notes", action="read") + assert e.value.code == ErrorCode.CONSENT_REQUIRED + + +def test_consent_expired(keypair, clock): + consent = Consent( + resource="data:notes", + actions=["read"], + expires=clock.now(), + ) + engine = PolicyEngine(Policy(peers=[make_peer(keypair, ["data:request"], [consent])]), clock) + clock.advance(1) + peer = engine.get_peer(keypair.instance_id) + with pytest.raises(PipError) as e: + engine.authorize(peer, "data:request", resource="data:notes", action="read") + assert e.value.code == ErrorCode.CONSENT_REQUIRED + + +def test_rate_limit(clock): + from pip_protocol.identity import KeyPair + + kp = KeyPair.generate() + policy = Policy( + peers=[make_peer(kp)], + rate_limits={"per_peer_per_minute": 60, "burst": 3}, + ) + engine = PolicyEngine(policy, clock) + for _ in range(3): + engine.check_rate(kp.instance_id) + with pytest.raises(PipError) as e: + engine.check_rate(kp.instance_id) + assert e.value.code == ErrorCode.RATE_LIMITED + clock.advance(60) + engine.check_rate(kp.instance_id) # refilled + + +def test_dataset_resource(): + assert dataset_resource("notes") == "data:notes" + + +def test_policy_load_yaml(tmp_path): + p = tmp_path / "policy.yaml" + p.write_text( + "allow_unknown_peers: true\npeers:\n" + " - instance_id: pip:x\n public_key: ed25519:AAAA\n" + " scopes: [messages:send]\n" + ) + policy = Policy.load(p) + assert policy.allow_unknown_peers + assert policy.peers[0].scopes == ["messages:send"] + + +def test_policy_load_json(tmp_path): + p = tmp_path / "policy.json" + p.write_text('{"allow_unknown_peers": false, "max_payload_bytes": 100}') + policy = Policy.load(p) + assert policy.max_payload_bytes == 100 + + +def test_peer_record_type(keypair): + peer = make_peer(keypair) + assert isinstance(peer, PeerRecord) diff --git a/tests/test_redaction.py b/tests/test_redaction.py new file mode 100644 index 0000000..9af6aba --- /dev/null +++ b/tests/test_redaction.py @@ -0,0 +1,42 @@ +from pip_protocol.policy import RedactionConfig +from pip_protocol.redaction import MASK, Redactor +from pip_protocol.schemas import EnvelopeType, MessagePayload + + +def redactor(**kw): + return Redactor(RedactionConfig(**kw)) + + +def test_field_name_masked_case_insensitive(): + r = redactor(fields=["email"]) + assert r.redact_value({"Email": "a@b.c", "ok": 1}) == {"Email": MASK, "ok": 1} + + +def test_nested_and_list(): + r = redactor(fields=["ssn"]) + out = r.redact_value({"a": [{"SSN": "123", "b": [{"ssn": "x"}]}]}) + assert out["a"][0]["SSN"] == MASK + assert out["a"][0]["b"][0]["ssn"] == MASK + + +def test_regex_pattern(): + r = redactor(patterns=[r"\b[\w.+-]+@[\w-]+\.[\w.]+\b"]) + assert r.redact_value("mail me at a@b.com please") == f"mail me at {MASK} please" + + +def test_message_payload_redacted(): + r = redactor(fields=["token"], patterns=["secret\\d+"]) + p = MessagePayload(subject="s", body="the secret1", attributes={"TOKEN": "x"}) + out = r.redact_payload(EnvelopeType.MESSAGE, p) + assert out.body == f"the {MASK}" + assert out.attributes["TOKEN"] == MASK + assert p.body == "the secret1" # original untouched + + +def test_data_records_redacted(): + from pip_protocol.schemas import DataPayload + + r = redactor(fields=["phone"]) + p = DataPayload(op="offer", dataset="d", records=[{"phone": "1", "keep": 2}]) + out = r.redact_payload(EnvelopeType.DATA, p) + assert out.records == [{"phone": MASK, "keep": 2}] diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..9d1e83a --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,90 @@ +from datetime import timedelta + +import pytest + +from pip_protocol.delivery import Outbox, OutboxItem +from pip_protocol.store import MemoryStore, SqliteStore, open_store + + +@pytest.fixture(params=["memory", "sqlite"]) +def store(request, tmp_path): + if request.param == "memory": + yield MemoryStore() + else: + s = SqliteStore(str(tmp_path / "pip.db")) + yield s + s.close() + + +def test_nonce_first_seen(store, clock): + assert store.seen_nonce("p", "n1", clock.now()) is False + + +def test_nonce_duplicate(store, clock): + store.seen_nonce("p", "n1", clock.now()) + assert store.seen_nonce("p", "n1", clock.now()) is True + + +def test_nonce_per_peer(store, clock): + store.seen_nonce("a", "n1", clock.now()) + assert store.seen_nonce("b", "n1", clock.now()) is False + + +def test_idempotent_roundtrip(store): + store.put_idempotent("p", "k", {"status": "accepted"}, 3600) + assert store.get_idempotent("p", "k") == {"status": "accepted"} + + +def test_idempotent_missing(store): + assert store.get_idempotent("p", "nope") is None + + +def test_purge(store, clock): + store.seen_nonce("p", "old", clock.now() + timedelta(seconds=10)) + store.seen_nonce("p", "new", clock.now() + timedelta(hours=1)) + store.purge(clock.now() + timedelta(minutes=5)) + # expired nonce purged: treated as unseen + assert store.seen_nonce("p", "old", clock.now() + timedelta(minutes=6)) is False + + +def test_open_store_urls(tmp_path): + assert isinstance(open_store("memory://"), MemoryStore) + s = open_store(f"sqlite:///{tmp_path}/x.db") + assert isinstance(s, SqliteStore) + with pytest.raises(ValueError): + open_store("bogus://") + + +def test_outbox_backoff(store, clock): + from pip_protocol.delivery import BACKOFF_CAP_SECONDS + + outbox = Outbox(store, clock) + item = outbox.enqueue("peer", {"id": "e1"}) + assert len(outbox.due()) == 1 + + item = outbox.mark_failed(item, "boom") + assert item.attempts == 1 + assert outbox.due() == [] # backed off + clock.advance(2) + assert len(outbox.due()) == 1 + + # exponential growth capped + for _ in range(6): + item = outbox.mark_failed(item, "x") + clock.advance(BACKOFF_CAP_SECONDS) + item = outbox.mark_failed(item, "final") + assert item.attempts == 8 + assert item.dead + + +def test_outbox_delivered(store, clock): + outbox = Outbox(store, clock) + item = outbox.enqueue("peer", {"id": "e1"}) + outbox.mark_delivered(item) + assert outbox.due() == [] + + +def test_outbox_item_serialization(clock): + item = OutboxItem(peer_id="p", envelope={}, next_attempt_at=clock.now()) + raw = item.model_dump(mode="json") + assert OutboxItem.model_validate(raw).peer_id == "p" From bfc8b28d80f41265a9d1b9c19217d96692f8fc3f Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:32:33 +0000 Subject: [PATCH 4/9] docs: PIP v1 protocol specification Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/PROTOCOL.md | 242 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 docs/PROTOCOL.md diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md new file mode 100644 index 0000000..cee0821 --- /dev/null +++ b/docs/PROTOCOL.md @@ -0,0 +1,242 @@ +# Poke Interconnect Protocol (PIP) v1 — Specification + +Status: v1.0 (stable). Wire version string: `pip/1.0`. + +PIP lets independent Poke instances discover each other's capabilities and +exchange signed, consented, idempotent messages and data over two transports: +an HTTP/JSON API and an MCP server. Both transports share one envelope format, +one policy engine, and one set of security rules. + +## 1. Terminology + +- **Instance** — a running Poke deployment with its own identity keypair. +- **Instance ID** — `pip:` derived from the + Ed25519 public key. Stable, self-certifying. +- **Peer** — another instance we have a trust record for. +- **Envelope** — the signed unit of exchange (message, data, receipt, error). +- **Capability** — a named, versioned thing an instance can do (e.g. `data.query`). +- **Scope** — a permission string granted to a peer, e.g. `messages:send`. +- **Consent** — an explicit, revocable, scoped grant from the local operator that + allows a peer to access a resource class (e.g. `data:notes`). + +## 2. Identity & authentication + +- Keys: Ed25519 (`cryptography` library). Public key encoded as + `ed25519:`. +- Instance ID as above. Peers are pinned by public key; a key that does not + hash to the claimed instance ID is rejected (`IDENTITY_MISMATCH`). +- Key rotation: an instance may publish `previous_keys` in its identity + document; envelopes signed by a previous key are accepted only for + `rotation_grace_seconds` (default 86400) after `rotated_at`. +- Authentication of a request = a valid signature on the envelope from a known + peer (or an unknown peer if `policy.allow_unknown_peers` is true, in which + case only `handshake` is permitted). +- HTTP additionally supports an optional static bearer token + (`PIP_HTTP_BEARER_TOKEN`) as a defense-in-depth transport gate; it never + replaces the signature. + +## 3. Envelope + +Canonical JSON (RFC 8785-style: sorted keys, no whitespace, UTF-8, no NaN) of +the `payload` object is what gets signed, together with the header fields. + +```json +{ + "pip": "1.0", + "id": "01J...ULID or uuid4", + "type": "message" | "data" | "receipt" | "error" | "handshake", + "from": "pip:...", + "to": "pip:...", + "ts": "2026-09-11T22:00:00Z", + "expires": "2026-09-11T22:05:00Z", + "nonce": "base64url 16 bytes", + "idempotency_key": "optional, client-chosen, <=128 chars", + "in_reply_to": "optional envelope id", + "payload": { ... type-specific ... }, + "sig": { + "alg": "ed25519", + "kid": "ed25519:", + "value": "base64url signature" + } +} +``` + +Signing input: `"PIPv1\n" + canonical_json({all fields except "sig"})`. +Verification order: schema → version → identity/kid → signature → time window → +replay → policy/consent → handler. + +### Payload types + +- `handshake`: `{ "identity": IdentityDocument, "capabilities": [Capability] }` +- `message`: `{ "subject": str<=200, "body": str<=64KiB, "content_type": "text/plain"|"text/markdown"|"application/json", "attributes": {str: str} }` +- `data`: `{ "op": "offer"|"request"|"response", "dataset": str, "schema_version": str, "records": [obj] (<=1000), "cursor": str|null, "query": obj|null }` +- `receipt`: `{ "ref": envelope id, "status": "accepted"|"delivered"|"rejected"|"duplicate", "reason": str|null }` +- `error`: `{ "code": ErrorCode, "message": str, "ref": envelope id|null, "retryable": bool }` + +Error codes: `SCHEMA_INVALID`, `VERSION_UNSUPPORTED`, `IDENTITY_MISMATCH`, +`SIGNATURE_INVALID`, `EXPIRED`, `CLOCK_SKEW`, `REPLAY`, `UNKNOWN_PEER`, +`FORBIDDEN`, `CONSENT_REQUIRED`, `RATE_LIMITED`, `PAYLOAD_TOO_LARGE`, +`CAPABILITY_UNKNOWN`, `INTERNAL`. + +## 4. Capability discovery + +`IdentityDocument`: +```json +{ "instance_id": "pip:...", "public_key": "ed25519:...", "display_name": str, + "pip_versions": ["1.0"], "endpoints": {"http": url|null, "mcp": url|null}, + "previous_keys": [{"public_key": "...", "rotated_at": ts}], + "issued_at": ts } +``` +`Capability`: `{ "name": "messages.send"|"data.query"|"data.offer"|..., "version": "1.0", "scopes_required": [scope], "description": str }`. + +Discovery: `GET /.well-known/pip` returns the signed handshake envelope +(self-addressed). MCP exposes the same as resource `pip://identity` and +`pip://capabilities`. + +## 5. Authorization & consent + +Policy file (YAML/JSON, loaded at startup, hot-reload not required): + +```yaml +allow_unknown_peers: false +default_scopes: [] # scopes granted to every known peer +peers: + - instance_id: pip:abc... + public_key: ed25519:... + display_name: "Alice's Poke" + scopes: [messages:send, data:request] + consents: + - resource: data:notes # resource class + actions: [read] # read | write + expires: 2027-01-01T00:00:00Z + granted_by: operator +rate_limits: + per_peer_per_minute: 60 + burst: 20 +max_payload_bytes: 262144 +redaction: + fields: [email, phone, ssn, api_key, token, password, secret] + patterns: ["\\b[\\w.+-]+@[\\w-]+\\.[\\w.]+\\b"] +``` + +Scopes: `messages:send`, `messages:read`, `data:request`, `data:offer`, +`data:respond`, `capabilities:read`, `admin:*`. Scope check is exact or +prefix-wildcard (`data:*`). Every handler declares required scopes; the policy +engine (`pip.policy.PolicyEngine.authorize(peer, action, resource)`) is the single +enforcement point shared by HTTP and MCP. Consent is required for any `data` +operation whose `dataset` maps to a consent-protected resource class; missing +consent → `CONSENT_REQUIRED` (not `FORBIDDEN`, so peers can prompt the operator). + +## 6. Privacy & redaction + +Outbound `data` records and `message` bodies pass through `pip.redaction.Redactor` +which (a) drops/masks configured field names (case-insensitive, nested) and +(b) masks regex matches with `[REDACTED]`. Redaction is applied before signing +so the signature covers the redacted content. Logs never include payload +bodies, keys or tokens — only envelope ids, types, peer ids, and outcome. + +## 7. Replay protection, idempotency, delivery semantics + +- Time window: `|now - ts| <= max_clock_skew_seconds` (default 300) else + `CLOCK_SKEW`; `now > expires` → `EXPIRED`. `expires - ts` <= 1 hour. +- Replay: `(from, nonce)` stored until `expires`; duplicate → `REPLAY`. +- Idempotency: if `idempotency_key` present, `(from, idempotency_key)` is stored + for `idempotency_ttl_seconds` (default 86400) along with the first receipt; + a repeat returns the stored receipt with `status: duplicate` and does not + re-run the handler. Storage backends: in-memory (default) and SQLite + (`PIP_STORE_URL=sqlite:///path`). Interface `pip.store.Store`. +- Delivery: at-least-once from the sender (client retries with the same + `idempotency_key` and exponential backoff on `retryable` errors / network + failure); effectively-once at the receiver via idempotency. Receiver always + answers a `receipt` envelope (`accepted` synchronously; handlers are + synchronous in v1). An outbox (`pip.delivery.Outbox`) persists pending + outbound envelopes with attempt count and next-attempt time. + +## 8. Transport — HTTP + +FastAPI app `pip.transport.http.create_app(node)`: + +- `GET /.well-known/pip` → signed handshake envelope (no auth). +- `GET /healthz` → `{"status":"ok","version":"pip/1.0"}`. +- `POST /pip/v1/inbox` → body: envelope; response: receipt or error envelope + (HTTP 200 accepted/duplicate, 400 schema, 401 signature/identity, 403 + forbidden/consent, 409 replay, 413 too large, 429 rate limit, 422 other + validation). Response envelope is signed by the receiver. +- `GET /metrics` → Prometheus text format (counters: envelopes_received_total + {type,outcome}, envelopes_sent_total, policy_denials_total {code}; + histogram: handler_seconds). Guarded by bearer token if configured. +- Request id header `X-PIP-Request-Id` echoed; structured JSON logs via stdlib + `logging` with a JSON formatter. + +Client: `pip.transport.http.HttpPeerClient(node, peer)` with `send(envelope)` +using `httpx`, verifying the returned receipt signature. + +## 9. Transport — MCP + +`pip.mcp_server` uses the official `mcp` Python SDK (`FastMCP`), stdio and +streamable-HTTP transports. Every tool call carries the caller's identity via a +signed envelope argument, so MCP is just another transport for the same +verification pipeline — there is no unauthenticated tool. + +Tools: +- `pip_handshake(envelope: dict) -> dict` — verify peer handshake, return ours. +- `pip_send_message(envelope: dict) -> dict` — deliver a `message` envelope; returns receipt. +- `pip_exchange_data(envelope: dict) -> dict` — `data` request/offer; returns `data` response or receipt. +- `pip_get_receipt(envelope: dict) -> dict` — signed `receipt` lookup by `ref` (for idempotent retries). +- `pip_list_capabilities() -> dict` — public, returns capabilities (unsigned convenience view). + +Resources: `pip://identity`, `pip://capabilities`, `pip://policy/scopes` (the +scope vocabulary, no peer data). + +Prompts: `pip_compose_message(to, subject, intent)` and +`pip_request_data(dataset, purpose)` — templates that help an LLM-driven Poke +instance produce a well-formed payload to be signed by its local node. + +Outputs are validated against the same Pydantic models; errors are returned as +signed `error` envelopes inside the tool result (never raised as raw +exceptions), with `isError` set. + +## 10. Observability + +- Metrics as in §8; `pip.observability.Metrics` is transport-agnostic. +- Structured logs: one line per envelope with `event`, `envelope_id`, `type`, + `peer`, `outcome`, `code`, `duration_ms`. No payload content. +- Optional `X-PIP-Trace-Id` propagated to logs. + +## 11. Versioning + +- `pip` field must be `1.x`; minor versions are additive (unknown payload + fields ignored, unknown envelope types rejected). Major mismatch → + `VERSION_UNSUPPORTED` with the supported list in the error payload. +- Capabilities and datasets carry their own `version`/`schema_version`. +- Package version follows semver; `pip.PROTOCOL_VERSION = "1.0"`. + +## 12. Secure configuration + +Settings via environment (`pydantic-settings`, prefix `PIP_`): + +| Variable | Default | Notes | +|---|---|---| +| `PIP_PRIVATE_KEY_FILE` | `./data/instance.key` | 0600; generated by `pip keygen` | +| `PIP_DISPLAY_NAME` | `poke` | | +| `PIP_POLICY_FILE` | `./config/policy.yaml` | | +| `PIP_STORE_URL` | `memory://` | or `sqlite:///data/pip.db` | +| `PIP_HTTP_HOST` / `PIP_HTTP_PORT` | `127.0.0.1` / `8642` | | +| `PIP_PUBLIC_HTTP_URL` / `PIP_PUBLIC_MCP_URL` | unset | advertised endpoints | +| `PIP_HTTP_BEARER_TOKEN` | unset | optional transport gate | +| `PIP_MAX_CLOCK_SKEW_SECONDS` | `300` | | +| `PIP_IDEMPOTENCY_TTL_SECONDS` | `86400` | | +| `PIP_LOG_LEVEL` | `INFO` | | + +Rules: private key never in env/config files (file path only); refuse to start +if key file mode is group/world readable (unless `PIP_ALLOW_INSECURE_KEY_PERMS=1` +for tests); bind to loopback by default; TLS is terminated by a reverse proxy +(documented). Example files live in `config/` with no real secrets. + +## 13. Threat model summary + +Mitigated: spoofing (signatures + pinned keys), replay (nonce + window), tamper +(canonical signing), over-collection (consent + redaction), abuse (rate limit + +size caps), downgrade (explicit version negotiation), key compromise (rotation +with grace). Out of scope for v1: transport-layer confidentiality (delegate to +TLS), metadata privacy, Sybil resistance / peer reputation. From 3ded83beacbb3de674b8ef5301922c89efa99aec Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:39:24 +0000 Subject: [PATCH 5/9] fix: verify envelope.to matches local id; redact outbound responses; add error_for_raw, mark_dead, receipt lookup by ref Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/pip_protocol/delivery.py | 16 ++++++++++- src/pip_protocol/node.py | 47 ++++++++++++++++++++++++++++++ src/pip_protocol/store.py | 55 ++++++++++++++++++++++++++++++++---- tests/test_node.py | 31 ++++++++++++++++++++ tests/test_store.py | 8 ++++++ 5 files changed, 150 insertions(+), 7 deletions(-) diff --git a/src/pip_protocol/delivery.py b/src/pip_protocol/delivery.py index 9613d4c..ebf121c 100644 --- a/src/pip_protocol/delivery.py +++ b/src/pip_protocol/delivery.py @@ -48,7 +48,13 @@ def enqueue(self, peer_id: str, envelope: dict[str, Any]) -> OutboxItem: return item def due(self) -> list[OutboxItem]: - return [OutboxItem.model_validate(raw) for raw in self.store.outbox_due(self.clock.now())] + return [ + item + for item in ( + OutboxItem.model_validate(raw) for raw in self.store.outbox_due(self.clock.now()) + ) + if not item.dead + ] def mark_failed(self, item: OutboxItem, err: str) -> OutboxItem: item.attempts += 1 @@ -60,6 +66,14 @@ def mark_failed(self, item: OutboxItem, err: str) -> OutboxItem: self.store.outbox_update(item.model_dump(mode="json")) return item + def mark_dead(self, item: OutboxItem, err: str) -> OutboxItem: + """Mark the item permanently failed (non-retryable error).""" + item.attempts = self.max_attempts + item.last_error = err + item.dead = True + self.store.outbox_update(item.model_dump(mode="json")) + return item + def mark_delivered(self, item: OutboxItem) -> None: self.store.outbox_delete(item.id) diff --git a/src/pip_protocol/node.py b/src/pip_protocol/node.py index 937aed7..3f43e4f 100644 --- a/src/pip_protocol/node.py +++ b/src/pip_protocol/node.py @@ -204,6 +204,8 @@ def verify_inbound(self, envelope: Envelope) -> PeerRecord | None: ErrorCode.VERSION_UNSUPPORTED, f"unsupported pip version {envelope.pip!r}; supported: {SUPPORTED_VERSIONS}", ) + if envelope.to != self.instance_id: + raise PipError(ErrorCode.IDENTITY_MISMATCH, "envelope not addressed to this instance") if len(canonical_json(envelope.payload)) > self.policy.max_payload_bytes: raise PipError(ErrorCode.PAYLOAD_TOO_LARGE, "payload exceeds max_payload_bytes") @@ -356,6 +358,7 @@ def _dispatch(self, envelope: Envelope, peer: PeerRecord | None) -> Envelope: envelope.idempotency_key, response.model_dump(mode="json", by_alias=True), self.policy.idempotency_ttl_seconds, + ref=envelope.id, ) return response @@ -367,6 +370,7 @@ def _re_sign(self, envelope: Envelope) -> Signature: ) def _respond(self, envelope: Envelope, type: EnvelopeType, payload: BaseModel) -> Envelope: + payload = self.redactor.redact_payload(type, payload) return build_envelope( self.keypair, type=type, @@ -388,6 +392,49 @@ def _receipt( ReceiptPayload(ref=envelope.id, status=status, reason=reason), ) + def error_for_raw(self, raw: dict[str, Any] | None, err: PipError) -> Envelope: + """Signed error envelope for an unparseable inbound payload. + + Reuses ``_error_envelope`` logic with a synthetic ``to``/``id`` when the + raw data cannot be parsed into an Envelope. + """ + to = "pip:unknown" + ref: str | None = None + if isinstance(raw, dict): + if isinstance(raw.get("from"), str): + to = raw["from"] + if isinstance(raw.get("id"), str): + ref = raw["id"] + payload = ErrorPayload( + code=err.code.value, + message=err.message, + ref=ref, + retryable=err.retryable, + supported_versions=list(SUPPORTED_VERSIONS) + if err.code == ErrorCode.VERSION_UNSUPPORTED + else None, + ) + try: + return build_envelope( + self.keypair, + type=EnvelopeType.ERROR, + to=to, + payload=payload.model_dump(mode="json"), + clock=self.clock, + ) + except PipError: + return Envelope( + pip=PROTOCOL_VERSION, + id="error", + type=EnvelopeType.ERROR, + from_=self.instance_id, + to=to, + ts=self.clock.now(), + expires=self.clock.now(), + nonce="AAAAAAAAAAAAAAAAAAAAAA", + payload=payload.model_dump(mode="json"), + ) + def _error_envelope(self, envelope: Envelope, err: PipError) -> Envelope: payload = ErrorPayload( code=err.code.value, diff --git a/src/pip_protocol/store.py b/src/pip_protocol/store.py index 990e2b0..f210726 100644 --- a/src/pip_protocol/store.py +++ b/src/pip_protocol/store.py @@ -16,7 +16,16 @@ def seen_nonce(self, peer: str, nonce: str, expires_at: datetime) -> bool: def get_idempotent(self, peer: str, key: str) -> dict[str, Any] | None: ... - def put_idempotent(self, peer: str, key: str, receipt: dict[str, Any], ttl: int) -> None: ... + def put_idempotent( + self, + peer: str, + key: str, + receipt: dict[str, Any], + ttl: int, + ref: str | None = None, + ) -> None: ... + + def find_receipt_by_ref(self, peer: str, ref: str) -> dict[str, Any] | None: ... def purge(self, now: datetime) -> None: ... @@ -66,13 +75,28 @@ def get_idempotent(self, peer: str, key: str) -> dict[str, Any] | None: return None return receipt - def put_idempotent(self, peer: str, key: str, receipt: dict[str, Any], ttl: int) -> None: + def put_idempotent( + self, + peer: str, + key: str, + receipt: dict[str, Any], + ttl: int, + ref: str | None = None, + ) -> None: with self._lock: self._idempotent[(peer, key)] = ( _ts(datetime.now(timezone.utc)) + ttl, receipt, ) + def find_receipt_by_ref(self, peer: str, ref: str) -> dict[str, Any] | None: + with self._lock: + now = _ts(datetime.now(timezone.utc)) + for (p, _), (expires, receipt) in self._idempotent.items(): + if p == peer and expires >= now and receipt.get("payload", {}).get("ref") == ref: + return receipt + return None + def purge(self, now: datetime) -> None: with self._lock: ts = _ts(now) @@ -109,10 +133,12 @@ def outbox_delete(self, item_id: str) -> None: CREATE TABLE IF NOT EXISTS idempotent ( peer TEXT NOT NULL, key TEXT NOT NULL, + ref TEXT, expires_at REAL NOT NULL, receipt TEXT NOT NULL, PRIMARY KEY (peer, key) ); +CREATE INDEX IF NOT EXISTS idx_idempotent_ref ON idempotent (peer, ref); CREATE TABLE IF NOT EXISTS outbox ( id TEXT PRIMARY KEY, item TEXT NOT NULL, @@ -153,14 +179,31 @@ def get_idempotent(self, peer: str, key: str) -> dict[str, Any] | None: return None return json.loads(row[1]) # type: ignore[no-any-return] - def put_idempotent(self, peer: str, key: str, receipt: dict[str, Any], ttl: int) -> None: + def put_idempotent( + self, + peer: str, + key: str, + receipt: dict[str, Any], + ttl: int, + ref: str | None = None, + ) -> None: with self._lock, self._conn: self._conn.execute( - "INSERT OR REPLACE INTO idempotent (peer, key, expires_at, receipt)" - " VALUES (?, ?, ?, ?)", - (peer, key, _ts(datetime.now(timezone.utc)) + ttl, json.dumps(receipt)), + "INSERT OR REPLACE INTO idempotent (peer, key, ref, expires_at, receipt)" + " VALUES (?, ?, ?, ?, ?)", + (peer, key, ref, _ts(datetime.now(timezone.utc)) + ttl, json.dumps(receipt)), ) + def find_receipt_by_ref(self, peer: str, ref: str) -> dict[str, Any] | None: + with self._lock: + row = self._conn.execute( + "SELECT expires_at, receipt FROM idempotent WHERE peer = ? AND ref = ?", + (peer, ref), + ).fetchone() + if row is None or row[0] < _ts(datetime.now(timezone.utc)): + return None + return json.loads(row[1]) # type: ignore[no-any-return] + def purge(self, now: datetime) -> None: with self._lock, self._conn: ts = _ts(now) diff --git a/tests/test_node.py b/tests/test_node.py index 1258264..cf1661c 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -275,3 +275,34 @@ def test_envelope_parse_roundtrip(clock): env = a.build(EnvelopeType.MESSAGE, "pip:x", MessagePayload(subject="s", body="x")) env2 = Envelope.model_validate(env.model_dump(mode="json", by_alias=True)) assert env2.id == env.id + + +def test_envelope_wrong_addressee(clock): + """F1: B rejects an envelope addressed to C.""" + a, b = make_nodes(clock) + c = KeyPair.generate() + env = a.build(EnvelopeType.MESSAGE, c.instance_id, MessagePayload(subject="s", body="x")) + _, err = b.handle(env) + assert err is not None and err.code == ErrorCode.IDENTITY_MISMATCH + + +def test_data_response_redacted(clock): + """F2: outbound data responses pass through the redactor.""" + consent = Consent(resource="data:notes", actions=["read"]) + ka, kb = KeyPair.generate(), KeyPair.generate() + a = Node(ka, Policy(peers=[make_peer(kb, ["admin:*"])]), clock=clock) + b = Node( + kb, + Policy( + peers=[make_peer(ka, ["data:request"], [consent])], + redaction={"fields": ["email"]}, + ), + clock=clock, + data_provider=DictDataProvider({"notes": [{"email": "a@b.c", "n": 1}]}), + ) + env = a.build(EnvelopeType.DATA, b.instance_id, DataPayload(op="request", dataset="notes")) + resp, err = b.handle(env) + assert err is None + p = resp.typed_payload() + assert p.records == [{"email": "[REDACTED]", "n": 1}] + verify_signature(resp, b.keypair.public_key) diff --git a/tests/test_store.py b/tests/test_store.py index 9d1e83a..5177e24 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -88,3 +88,11 @@ def test_outbox_item_serialization(clock): item = OutboxItem(peer_id="p", envelope={}, next_attempt_at=clock.now()) raw = item.model_dump(mode="json") assert OutboxItem.model_validate(raw).peer_id == "p" + + +def test_find_receipt_by_ref(store): + receipt = {"payload": {"ref": "env-1", "status": "accepted"}} + store.put_idempotent("p", "k", receipt, 3600, ref="env-1") + assert store.find_receipt_by_ref("p", "env-1") == receipt + assert store.find_receipt_by_ref("p", "nope") is None + assert store.find_receipt_by_ref("other", "env-1") is None From de27048e6c9dc470485fe43807502827707fcdc8 Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:39:24 +0000 Subject: [PATCH 6/9] feat: HTTP transport, peer client, outbox delivery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/pip_protocol/transport/__init__.py | 1 + src/pip_protocol/transport/http.py | 180 ++++++++++++++++++++++ tests/test_http.py | 199 +++++++++++++++++++++++++ 3 files changed, 380 insertions(+) create mode 100644 src/pip_protocol/transport/__init__.py create mode 100644 src/pip_protocol/transport/http.py create mode 100644 tests/test_http.py diff --git a/src/pip_protocol/transport/__init__.py b/src/pip_protocol/transport/__init__.py new file mode 100644 index 0000000..343e6e3 --- /dev/null +++ b/src/pip_protocol/transport/__init__.py @@ -0,0 +1 @@ +"""PIP transports (HTTP today; MCP lives in pip_protocol.mcp_server).""" diff --git a/src/pip_protocol/transport/http.py b/src/pip_protocol/transport/http.py new file mode 100644 index 0000000..da0c463 --- /dev/null +++ b/src/pip_protocol/transport/http.py @@ -0,0 +1,180 @@ +"""HTTP transport (spec §8): FastAPI app, peer client, outbox delivery.""" + +import json +import logging +import secrets +import uuid +from typing import Any + +import httpx +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse, PlainTextResponse + +from ..config import Settings +from ..delivery import Outbox +from ..envelope import verify_signature +from ..errors import ErrorCode, PipError +from ..identity import instance_id_from_public_key +from ..node import Node +from ..observability import log_event +from ..schemas import Envelope, parse_envelope + +logger = logging.getLogger("pip_protocol.http") + +_BODY_HEADROOM_BYTES = 8192 + + +def bearer_ok(authorization: str | None, token: str | None) -> bool: + """Constant-time bearer token check shared by HTTP and MCP transports.""" + if token is None: + return True + if authorization is None or not authorization.startswith("Bearer "): + return False + return secrets.compare_digest(authorization[len("Bearer ") :], token) + + +def unauthorized() -> JSONResponse: + return JSONResponse({"error": "unauthorized"}, status_code=401) + + +def create_app(node: Node, settings: Settings | None = None) -> FastAPI: + settings = settings or Settings() + app = FastAPI(title="poke-interconnect", version="1.0.0") + + @app.middleware("http") + async def request_context(request: Request, call_next): # type: ignore[no-untyped-def] + request_id = request.headers.get("x-pip-request-id") or uuid.uuid4().hex + trace_id = request.headers.get("x-pip-trace-id") + request.state.request_id = request_id + request.state.trace_id = trace_id + response = await call_next(request) + response.headers["X-PIP-Request-Id"] = request_id + log_event( + logger, + "http_request", + method=request.method, + path=request.url.path, + status=response.status_code, + request_id=request_id, + trace_id=trace_id, + ) + return response + + @app.get("/.well-known/pip") + async def well_known() -> dict[str, Any]: + return node.handshake_envelope().model_dump(mode="json", by_alias=True) + + @app.get("/healthz") + async def healthz() -> dict[str, str]: + return {"status": "ok", "version": "pip/1.0"} + + @app.get("/metrics") + async def metrics(request: Request) -> Response: + if not bearer_ok(request.headers.get("authorization"), settings.http_bearer_token): + return unauthorized() + return PlainTextResponse(node.metrics.render(), media_type="text/plain; version=0.0.4") + + @app.post("/pip/v1/inbox") + async def inbox(request: Request) -> Response: + if not bearer_ok(request.headers.get("authorization"), settings.http_bearer_token): + return unauthorized() + body = await request.body() + if len(body) > node.policy.max_payload_bytes + _BODY_HEADROOM_BYTES: + env = node.error_for_raw( + None, PipError(ErrorCode.PAYLOAD_TOO_LARGE, "request body too large") + ) + return JSONResponse(env.model_dump(mode="json", by_alias=True), status_code=413) + try: + raw: Any = json.loads(body) + except Exception: + raw = None + try: + envelope = parse_envelope(raw if isinstance(raw, dict) else {}) + except PipError as exc: + env = node.error_for_raw(raw if isinstance(raw, dict) else None, exc) + return JSONResponse( + env.model_dump(mode="json", by_alias=True), status_code=exc.http_status + ) + response_env, err = node.handle(envelope) + status = err.http_status if err is not None else 200 + return JSONResponse(response_env.model_dump(mode="json", by_alias=True), status_code=status) + + return app + + +class HttpPeerClient: + """Client for a remote peer's HTTP transport (spec §8).""" + + def __init__( + self, + node: Node, + peer_url: str, + peer_public_key: str, + *, + bearer_token: str | None = None, + timeout: float = 10.0, + client: httpx.Client | None = None, + ) -> None: + self.node = node + self.peer_url = peer_url.rstrip("/") + self.peer_public_key = peer_public_key + self.peer_instance_id = instance_id_from_public_key(peer_public_key) + self.bearer_token = bearer_token + self._client = client or httpx.Client(timeout=timeout) + self._owns_client = client is None + self._timeout = timeout + + def close(self) -> None: + if self._owns_client: + self._client.close() + + def _headers(self) -> dict[str, str]: + if self.bearer_token: + return {"Authorization": f"Bearer {self.bearer_token}"} + return {} + + def send(self, envelope: Envelope) -> Envelope: + """POST an envelope to the peer inbox; verify the signed response.""" + resp = self._client.post( + f"{self.peer_url}/pip/v1/inbox", + content=envelope.model_dump_json(by_alias=True), + headers={"Content-Type": "application/json", **self._headers()}, + ) + try: + response_env = parse_envelope(resp.json()) + except Exception as exc: + raise PipError(ErrorCode.SCHEMA_INVALID, f"invalid response: {exc}") from exc + verify_signature(response_env, self.peer_public_key) + if response_env.in_reply_to != envelope.id: + raise PipError(ErrorCode.SIGNATURE_INVALID, "response in_reply_to does not match") + return response_env + + def fetch_handshake(self) -> Envelope: + """GET /.well-known/pip; verify the self-signed handshake.""" + resp = self._client.get(f"{self.peer_url}/.well-known/pip", headers=self._headers()) + envelope = parse_envelope(resp.json()) + verify_signature(envelope, self.peer_public_key) + if envelope.from_ != self.peer_instance_id: + raise PipError(ErrorCode.IDENTITY_MISMATCH, "handshake from unexpected peer") + return envelope + + +def deliver_outbox(node: Node, outbox: Outbox, clients: dict[str, HttpPeerClient]) -> None: + """One delivery pass over due outbox items (spec §7).""" + for item in outbox.due(): + client = clients.get(item.peer_id) + if client is None: + outbox.mark_failed(item, f"no client for peer {item.peer_id}") + continue + try: + envelope = parse_envelope(item.envelope) + client.send(envelope) + except PipError as exc: + if exc.retryable: + outbox.mark_failed(item, str(exc)) + else: + outbox.mark_dead(item, str(exc)) + except Exception as exc: # network/transport failure → retryable + outbox.mark_failed(item, str(exc)) + else: + outbox.mark_delivered(item) diff --git a/tests/test_http.py b/tests/test_http.py new file mode 100644 index 0000000..605b378 --- /dev/null +++ b/tests/test_http.py @@ -0,0 +1,199 @@ +import json + +import httpx +import pytest +from fastapi.testclient import TestClient + +from conftest import make_peer +from pip_protocol.config import Settings +from pip_protocol.delivery import Outbox +from pip_protocol.envelope import verify_signature +from pip_protocol.errors import ErrorCode, PipError +from pip_protocol.identity import KeyPair +from pip_protocol.node import Node +from pip_protocol.policy import Policy +from pip_protocol.schemas import EnvelopeType, MessagePayload, parse_envelope +from pip_protocol.store import MemoryStore +from pip_protocol.transport.http import HttpPeerClient, create_app, deliver_outbox + + +def make_pair(clock, *, scopes=None, b_settings=None): + ka, kb = KeyPair.generate(), KeyPair.generate() + a = Node(ka, Policy(peers=[make_peer(kb, ["admin:*"])]), clock=clock) + b = Node( + kb, + Policy(peers=[make_peer(ka, scopes if scopes is not None else ["messages:send"])]), + clock=clock, + ) + app = create_app(b, b_settings) + return a, b, TestClient(app) + + +def post_env(client, env, headers=None): + return client.post( + "/pip/v1/inbox", + content=env.model_dump_json(by_alias=True), + headers={"Content-Type": "application/json", **(headers or {})}, + ) + + +def message(a, to, clock): + return a.build(EnvelopeType.MESSAGE, to, MessagePayload(subject="s", body="hi")) + + +def test_healthz(clock): + _, _, client = make_pair(clock) + assert client.get("/healthz").json() == {"status": "ok", "version": "pip/1.0"} + + +def test_well_known_handshake(clock): + _, b, client = make_pair(clock) + env = parse_envelope(client.get("/.well-known/pip").json()) + verify_signature(env, b.keypair.public_key) + assert env.from_ == b.instance_id + + +def test_message_ok(clock): + a, b, client = make_pair(clock) + resp = post_env(client, message(a, b.instance_id, clock)) + assert resp.status_code == 200 + env = parse_envelope(resp.json()) + verify_signature(env, b.keypair.public_key) + assert env.typed_payload().status == "accepted" + assert resp.headers["x-pip-request-id"] + + +def test_unknown_peer_401(clock): + a = Node(KeyPair.generate(), clock=clock) + b = Node(KeyPair.generate(), Policy(), clock=clock) + client = TestClient(create_app(b)) + resp = post_env(client, message(a, b.instance_id, clock)) + assert resp.status_code == 401 + assert parse_envelope(resp.json()).typed_payload().code == "UNKNOWN_PEER" + + +def test_forbidden_403(clock): + a, b, client = make_pair(clock, scopes=[]) + resp = post_env(client, message(a, b.instance_id, clock)) + assert resp.status_code == 403 + assert parse_envelope(resp.json()).typed_payload().code == "FORBIDDEN" + + +def test_replay_409(clock): + a, b, client = make_pair(clock) + env = message(a, b.instance_id, clock) + assert post_env(client, env).status_code == 200 + resp = post_env(client, env) + assert resp.status_code == 409 + assert parse_envelope(resp.json()).typed_payload().code == "REPLAY" + + +def test_oversize_413(clock): + _, b, client = make_pair(clock) + b.policy.max_payload_bytes = 100 + resp = client.post("/pip/v1/inbox", content=b"x" * 9000) + assert resp.status_code == 413 + assert parse_envelope(resp.json()).typed_payload().code == "PAYLOAD_TOO_LARGE" + + +def test_bad_json_400(clock): + _, b, client = make_pair(clock) + resp = client.post("/pip/v1/inbox", content=b"not json") + assert resp.status_code == 400 + env = parse_envelope(resp.json()) + assert env.typed_payload().code == "SCHEMA_INVALID" + verify_signature(env, b.keypair.public_key) + + +def test_wrong_addressee(clock): + a, b, client = make_pair(clock) + env = message(a, "pip:someoneelse", clock) + resp = post_env(client, env) + assert resp.status_code == 401 + assert parse_envelope(resp.json()).typed_payload().code == "IDENTITY_MISMATCH" + + +def test_bearer_token(clock): + settings = Settings(http_bearer_token="sekrit") # noqa: S106 + a, b, client = make_pair(clock, b_settings=settings) + env = message(a, b.instance_id, clock) + assert post_env(client, env).status_code == 401 + assert post_env(client, env, {"Authorization": "Bearer wrong"}).status_code == 401 + ok = post_env(client, env, {"Authorization": "Bearer sekrit"}) # noqa: S106 + assert ok.status_code == 200 + assert client.get("/metrics").status_code == 401 + assert ( + client.get("/metrics", headers={"Authorization": "Bearer sekrit"}).status_code # noqa: S106 + == 200 + ) + + +def test_metrics_endpoint(clock): + a, b, client = make_pair(clock) + post_env(client, message(a, b.instance_id, clock)) + resp = client.get("/metrics") + assert "envelopes_received_total" in resp.text + + +def test_http_peer_client(clock): + a, b, client = make_pair(clock) + peer = HttpPeerClient(a, "http://b", b.keypair.public_key, client=client) + resp = peer.send(message(a, b.instance_id, clock)) + assert resp.typed_payload().status == "accepted" + hs = peer.fetch_handshake() + assert hs.from_ == b.instance_id + + +def test_http_peer_client_bad_reply(clock): + a, b, client = make_pair(clock) + env = message(a, b.instance_id, clock) + + def handler(request: httpx.Request) -> httpx.Response: + wire = env.model_dump(mode="json", by_alias=True) + wire["in_reply_to"] = "bogus" + return httpx.Response(200, json=wire) + + http_client = httpx.Client(transport=httpx.MockTransport(handler), base_url="http://b") + peer = HttpPeerClient(a, "http://b", b.keypair.public_key, client=http_client) + with pytest.raises(PipError) as e: + peer.send(env) + assert e.value.code == ErrorCode.SIGNATURE_INVALID + + +def test_deliver_outbox(clock): + a, b, client = make_pair(clock) + peer = HttpPeerClient(a, "http://b", b.keypair.public_key, client=client) + outbox = Outbox(MemoryStore(), clock) + outbox.enqueue( + b.instance_id, + message(a, b.instance_id, clock).model_dump(mode="json", by_alias=True), + ) + deliver_outbox(a, outbox, {b.instance_id: peer}) + assert outbox.due() == [] + assert len(b.inbox) == 1 + + +def test_deliver_outbox_dead_on_nonretryable(clock): + a = Node(KeyPair.generate(), clock=clock) + outbox = Outbox(MemoryStore(), clock) + + class FailingClient: + def send(self, env): + raise PipError(ErrorCode.FORBIDDEN, "nope", retryable=False) + + outbox.enqueue( + "pip:peer", + message(a, "pip:peer", clock).model_dump(mode="json", by_alias=True), + ) + deliver_outbox(a, outbox, {"pip:peer": FailingClient()}) + # dead items are retained for inspection; fetch via a late 'now' + clock.advance(3600) + stored = outbox.store.outbox_due(clock.now()) + assert len(stored) == 1 and stored[0]["dead"] + + +def test_json_content_type(clock): + a, b, client = make_pair(clock) + resp = post_env(client, message(a, b.instance_id, clock)) + assert resp.headers["content-type"].startswith("application/json") + json.dumps(resp.json()) # valid JSON From 28df1cfb2b948098fe24031ccddebfe8313d38a0 Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:39:24 +0000 Subject: [PATCH 7/9] feat: MCP transport (tools, resources, prompts) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/pip_protocol/mcp_server.py | 181 +++++++++++++++++++++++++++++++++ tests/test_mcp.py | 177 ++++++++++++++++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 src/pip_protocol/mcp_server.py create mode 100644 tests/test_mcp.py diff --git a/src/pip_protocol/mcp_server.py b/src/pip_protocol/mcp_server.py new file mode 100644 index 0000000..78a03f7 --- /dev/null +++ b/src/pip_protocol/mcp_server.py @@ -0,0 +1,181 @@ +"""MCP transport (spec §9): every tool call carries a signed envelope.""" + +import json +from typing import Any, Literal + +from mcp.server.fastmcp import FastMCP + +from .config import Settings +from .errors import ErrorCode, PipError +from .node import Node +from .schemas import Envelope, EnvelopeType, ReceiptPayload, parse_envelope + +SCOPE_VOCABULARY = [ + "messages:send", + "messages:read", + "data:request", + "data:offer", + "data:respond", + "capabilities:read", + "admin:*", +] + + +def _dump(envelope: Envelope) -> dict[str, Any]: + return envelope.model_dump(mode="json", by_alias=True) + + +def create_mcp_server(node: Node, settings: Settings | None = None) -> FastMCP: + settings = settings or Settings() + server: FastMCP = FastMCP("pip-v1", host=settings.http_host, port=settings.http_port) + + def _handle_dict(raw: dict[str, Any]) -> tuple[Envelope | None, dict[str, Any] | None]: + """Parse a raw tool argument; returns (envelope, error_dict).""" + try: + return parse_envelope(raw), None + except PipError as err: + return None, _dump(node.error_for_raw(raw, err)) + + @server.tool(name="pip_handshake", description="Verify a peer handshake; return ours") + def pip_handshake(envelope: dict[str, Any]) -> dict[str, Any]: + env, err = _handle_dict(envelope) + if err is not None or env is None: + return err or {} + resp, _ = node.handle(env) + rp = resp.typed_payload() + if ( + resp.type == EnvelopeType.RECEIPT + and isinstance(rp, ReceiptPayload) + and rp.status + in ( + "accepted", + "duplicate", + ) + ): + return _dump(node.handshake_envelope(to=env.from_)) + return _dump(resp) + + @server.tool(name="pip_send_message", description="Deliver a message envelope") + def pip_send_message(envelope: dict[str, Any]) -> dict[str, Any]: + env, err = _handle_dict(envelope) + if err is not None or env is None: + return err or {} + if env.type != EnvelopeType.MESSAGE: + return _dump( + node.error_for_raw( + envelope, PipError(ErrorCode.SCHEMA_INVALID, "expected a message envelope") + ) + ) + resp, _ = node.handle(env) + return _dump(resp) + + @server.tool(name="pip_exchange_data", description="Handle a data request/offer envelope") + def pip_exchange_data(envelope: dict[str, Any]) -> dict[str, Any]: + env, err = _handle_dict(envelope) + if err is not None or env is None: + return err or {} + if env.type != EnvelopeType.DATA: + return _dump( + node.error_for_raw( + envelope, PipError(ErrorCode.SCHEMA_INVALID, "expected a data envelope") + ) + ) + resp, _ = node.handle(env) + return _dump(resp) + + @server.tool(name="pip_get_receipt", description="Look up a stored receipt by ref") + def pip_get_receipt(envelope: dict[str, Any]) -> dict[str, Any]: + env, err = _handle_dict(envelope) + if err is not None or env is None: + return err or {} + try: + peer = node.verify_inbound(env) + payload = env.typed_payload() + if env.type != EnvelopeType.RECEIPT or not isinstance(payload, ReceiptPayload): + raise PipError(ErrorCode.SCHEMA_INVALID, "expected a receipt envelope with a ref") + if peer is None: + raise PipError(ErrorCode.UNKNOWN_PEER, "peer is not in the policy file") + node.engine.authorize(peer, "messages:read") + except PipError as exc: + return _dump(node.error_for_raw(env.model_dump(mode="json", by_alias=True), exc)) + stored = node.store.find_receipt_by_ref(peer.instance_id, payload.ref) + if stored is None: + return _dump(node._receipt(env, "rejected", "unknown ref")) + return stored + + @server.tool(name="pip_list_capabilities", description="List this instance's capabilities") + def pip_list_capabilities() -> dict[str, Any]: + doc = node.identity_document() + return { + "instance_id": doc.instance_id, + "capabilities": [c.model_dump(mode="json") for c in node.capabilities()], + "pip_versions": doc.pip_versions, + } + + @server.resource("pip://identity") + def identity_resource() -> str: + return node.identity_document().model_dump_json() + + @server.resource("pip://capabilities") + def capabilities_resource() -> str: + return json.dumps([c.model_dump(mode="json") for c in node.capabilities()]) + + @server.resource("pip://policy/scopes") + def scopes_resource() -> str: + return json.dumps(SCOPE_VOCABULARY) + + @server.prompt(name="pip_compose_message") + def pip_compose_message(to: str, subject: str, intent: str) -> str: + return ( + f"Compose a PIP `message` payload addressed to {to}.\n" + f"Subject: {subject}\nIntent: {intent}\n\n" + "Produce JSON matching this schema for the local node to sign:\n" + '{"subject": str <= 200 chars, "body": str <= 64KiB, ' + '"content_type": "text/plain"|"text/markdown"|"application/json", ' + '"attributes": {str: str}}\n' + "Pass it to the local node's build(EnvelopeType.MESSAGE, to, payload)." + ) + + @server.prompt(name="pip_request_data") + def pip_request_data(dataset: str, purpose: str) -> str: + return ( + f"Compose a PIP `data` request payload for dataset {dataset!r}.\n" + f"Purpose: {purpose}\n\n" + "Produce JSON matching this schema for the local node to sign:\n" + '{"op": "request", "dataset": str, "schema_version": str, ' + '"records": [], "cursor": str|null, "query": obj|null}\n' + "Use query.filter for equality matching and query.limit for page size." + ) + + return server + + +def run_mcp( + server: FastMCP, + transport: Literal["stdio", "streamable-http"] = "stdio", + *, + host: str = "127.0.0.1", + port: int = 8000, + bearer_token: str | None = None, +) -> None: + if transport == "stdio": + server.run("stdio") + return + app = server.streamable_http_app() + if bearer_token is not None: + from starlette.middleware.base import BaseHTTPMiddleware + from starlette.requests import Request + from starlette.responses import JSONResponse + + from .transport.http import bearer_ok + + class _Bearer(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): # type: ignore[no-untyped-def] + if not bearer_ok(request.headers.get("authorization"), bearer_token): + return JSONResponse({"error": "unauthorized"}, status_code=401) + return await call_next(request) + + app.add_middleware(_Bearer) + import uvicorn + + uvicorn.run(app, host=host, port=port) diff --git a/tests/test_mcp.py b/tests/test_mcp.py new file mode 100644 index 0000000..6255ad8 --- /dev/null +++ b/tests/test_mcp.py @@ -0,0 +1,177 @@ +import pytest + +from conftest import make_peer +from pip_protocol.envelope import verify_signature +from pip_protocol.identity import KeyPair +from pip_protocol.mcp_server import create_mcp_server +from pip_protocol.node import Node +from pip_protocol.policy import Policy +from pip_protocol.schemas import EnvelopeType, MessagePayload, parse_envelope + +mcp = pytest.importorskip("mcp") +from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 + + +def make_nodes(clock, scopes=None): + ka, kb = KeyPair.generate(), KeyPair.generate() + a = Node(ka, Policy(peers=[make_peer(kb, ["admin:*"])]), clock=clock) + b = Node( + kb, + Policy(peers=[make_peer(ka, scopes if scopes is not None else ["messages:send"])]), + clock=clock, + ) + return a, b + + +def tool_result_payload(result): + # FastMCP returns CallToolResult; structured dict is in structuredContent or text + if result.structuredContent is not None: + return result.structuredContent + import json + + return json.loads(result.content[0].text) + + +async def test_mcp_tools(clock): + a, b = make_nodes(clock) + server = create_mcp_server(b) + async with create_connected_server_and_client_session(server._mcp_server) as session: + tools = await session.list_tools() + names = {t.name for t in tools.tools} + assert { + "pip_handshake", + "pip_send_message", + "pip_exchange_data", + "pip_get_receipt", + "pip_list_capabilities", + } <= names + + caps = await session.call_tool("pip_list_capabilities", {}) + payload = tool_result_payload(caps) + assert payload["instance_id"] == b.instance_id + assert any(c["name"] == "messages.send" for c in payload["capabilities"]) + + +async def test_mcp_handshake(clock): + a, b = make_nodes(clock) + server = create_mcp_server(b) + hs = a.handshake_envelope(to=b.instance_id) + async with create_connected_server_and_client_session(server._mcp_server) as session: + result = await session.call_tool( + "pip_handshake", {"envelope": hs.model_dump(mode="json", by_alias=True)} + ) + env = parse_envelope(tool_result_payload(result)) + assert env.type == EnvelopeType.HANDSHAKE + verify_signature(env, b.keypair.public_key) + assert env.typed_payload().identity.instance_id == b.instance_id + + +async def test_mcp_send_message(clock): + a, b = make_nodes(clock) + server = create_mcp_server(b) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="hi")) + async with create_connected_server_and_client_session(server._mcp_server) as session: + result = await session.call_tool( + "pip_send_message", {"envelope": env.model_dump(mode="json", by_alias=True)} + ) + resp = parse_envelope(tool_result_payload(result)) + assert resp.type == EnvelopeType.RECEIPT + assert resp.typed_payload().status == "accepted" + verify_signature(resp, b.keypair.public_key) + + +async def test_mcp_forbidden_error_envelope(clock): + a, b = make_nodes(clock, scopes=[]) + server = create_mcp_server(b) + env = a.build(EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="s", body="x")) + async with create_connected_server_and_client_session(server._mcp_server) as session: + result = await session.call_tool( + "pip_send_message", {"envelope": env.model_dump(mode="json", by_alias=True)} + ) + resp = parse_envelope(tool_result_payload(result)) + assert resp.type == EnvelopeType.ERROR + assert resp.typed_payload().code == "FORBIDDEN" + verify_signature(resp, b.keypair.public_key) + + +async def test_mcp_malformed_schema_invalid(clock): + _, b = make_nodes(clock) + server = create_mcp_server(b) + async with create_connected_server_and_client_session(server._mcp_server) as session: + result = await session.call_tool("pip_send_message", {"envelope": {"x": 1}}) + resp = parse_envelope(tool_result_payload(result)) + assert resp.type == EnvelopeType.ERROR + assert resp.typed_payload().code == "SCHEMA_INVALID" + verify_signature(resp, b.keypair.public_key) + + +async def test_mcp_wrong_type(clock): + a, b = make_nodes(clock) + server = create_mcp_server(b) + env = a.build( + EnvelopeType.DATA, + b.instance_id, + {"op": "offer", "dataset": "x"}, + ) + async with create_connected_server_and_client_session(server._mcp_server) as session: + result = await session.call_tool( + "pip_send_message", {"envelope": env.model_dump(mode="json", by_alias=True)} + ) + resp = parse_envelope(tool_result_payload(result)) + assert resp.typed_payload().code == "SCHEMA_INVALID" + + +async def test_mcp_resources(clock): + _, b = make_nodes(clock) + server = create_mcp_server(b) + async with create_connected_server_and_client_session(server._mcp_server) as session: + import json + + res = await session.read_resource("pip://identity") + doc = json.loads(res.contents[0].text) + assert doc["instance_id"] == b.instance_id + + res = await session.read_resource("pip://capabilities") + assert "messages.send" in res.contents[0].text + + res = await session.read_resource("pip://policy/scopes") + assert "admin:*" in json.loads(res.contents[0].text) + + +async def test_mcp_prompt(clock): + _, b = make_nodes(clock) + server = create_mcp_server(b) + async with create_connected_server_and_client_session(server._mcp_server) as session: + prompt = await session.get_prompt( + "pip_compose_message", {"to": "pip:x", "subject": "hi", "intent": "greet"} + ) + assert "subject" in prompt.messages[0].content.text.lower() + + +async def test_mcp_get_receipt(clock): + a, b = make_nodes(clock) + server = create_mcp_server(b) + env = a.build( + EnvelopeType.MESSAGE, + b.instance_id, + MessagePayload(subject="s", body="x"), + idempotency_key="k1", + ) + async with create_connected_server_and_client_session(server._mcp_server) as session: + r1 = await session.call_tool( + "pip_send_message", {"envelope": env.model_dump(mode="json", by_alias=True)} + ) + assert parse_envelope(tool_result_payload(r1)).typed_payload().status == "accepted" + # now look it up by ref — peer needs messages:read + b.policy.peers[0].scopes.append("messages:read") + lookup = a.build( + EnvelopeType.RECEIPT, + b.instance_id, + {"ref": env.id, "status": "accepted"}, + ) + r2 = await session.call_tool( + "pip_get_receipt", {"envelope": lookup.model_dump(mode="json", by_alias=True)} + ) + resp = parse_envelope(tool_result_payload(r2)) + assert resp.type == EnvelopeType.RECEIPT + assert resp.typed_payload().ref == env.id From 335cefc396dbf730bab2dee9ea70ea01234c1989 Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:39:24 +0000 Subject: [PATCH 8/9] feat: pip-node CLI and settings bootstrap Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pyproject.toml | 7 ++ src/pip_protocol/bootstrap.py | 46 +++++++++++ src/pip_protocol/cli.py | 146 ++++++++++++++++++++++++++++++++++ tests/test_cli.py | 68 ++++++++++++++++ 4 files changed, 267 insertions(+) create mode 100644 src/pip_protocol/bootstrap.py create mode 100644 src/pip_protocol/cli.py create mode 100644 tests/test_cli.py diff --git a/pyproject.toml b/pyproject.toml index 37d5a08..0faf13a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ dependencies = [ "prometheus-client>=0.20", ] +[project.scripts] +pip-node = "pip_protocol.cli:main" + [project.optional-dependencies] dev = [ "pytest", @@ -42,6 +45,10 @@ select = ["E", "F", "I", "B", "UP", "S"] [tool.ruff.lint.per-file-ignores] "tests/**" = ["S101"] +"examples/**" = ["S101"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" [tool.mypy] python_version = "3.10" diff --git a/src/pip_protocol/bootstrap.py b/src/pip_protocol/bootstrap.py new file mode 100644 index 0000000..309e52c --- /dev/null +++ b/src/pip_protocol/bootstrap.py @@ -0,0 +1,46 @@ +"""Build a Node from Settings (spec §12).""" + +import logging +from pathlib import Path + +from .config import Settings +from .identity import KeyPair +from .node import Node +from .policy import Policy +from .store import open_store + +logger = logging.getLogger("pip_protocol.bootstrap") + + +def build_node_from_settings(settings: Settings) -> Node: + key_path = Path(settings.private_key_file) + if not key_path.exists(): + keypair = KeyPair.generate() + keypair.save(key_path) + logger.info("generated new instance key at %s", key_path) + else: + keypair = KeyPair.load(key_path, allow_insecure=settings.allow_insecure_key_perms) + + policy_path = Path(settings.policy_file) + if policy_path.exists(): + policy = Policy.load(policy_path) + else: + logger.warning("policy file %s not found; starting with empty policy", policy_path) + policy = Policy( + max_clock_skew_seconds=settings.max_clock_skew_seconds, + idempotency_ttl_seconds=settings.idempotency_ttl_seconds, + ) + + endpoints: dict[str, str | None] = {} + if settings.public_http_url: + endpoints["http"] = settings.public_http_url + if settings.public_mcp_url: + endpoints["mcp"] = settings.public_mcp_url + + return Node( + keypair, + policy, + store=open_store(settings.store_url), + display_name=settings.display_name, + endpoints=endpoints, + ) diff --git a/src/pip_protocol/cli.py b/src/pip_protocol/cli.py new file mode 100644 index 0000000..4c723bc --- /dev/null +++ b/src/pip_protocol/cli.py @@ -0,0 +1,146 @@ +"""pip-node command line interface.""" + +import argparse +import json +import sys +from pathlib import Path + +from .bootstrap import build_node_from_settings +from .config import Settings +from .envelope import verify_signature +from .errors import PipError +from .identity import KeyPair +from .observability import configure_logging +from .schemas import EnvelopeType, parse_envelope + + +def _cmd_keygen(args: argparse.Namespace) -> int: + out = Path(args.out) + if out.exists() and not args.force: + print(f"error: {out} exists (use --force)", file=sys.stderr) + return 2 + keypair = KeyPair.generate() + keypair.save(out) + print(f"wrote {out} (mode 0600)") + print(f"instance_id: {keypair.instance_id}") + print(f"public_key: {keypair.public_key}") + return 0 + + +def _cmd_identity(args: argparse.Namespace) -> int: + node = build_node_from_settings(Settings()) + print(node.identity_document().model_dump_json()) + return 0 + + +def _cmd_serve_http(args: argparse.Namespace) -> int: + import uvicorn + + from .transport.http import create_app + + settings = Settings() + configure_logging(settings.log_level) + node = build_node_from_settings(settings) + app = create_app(node, settings) + uvicorn.run(app, host=settings.http_host, port=settings.http_port) + return 0 + + +def _cmd_serve_mcp(args: argparse.Namespace) -> int: + from .mcp_server import create_mcp_server, run_mcp + + settings = Settings() + configure_logging(settings.log_level) + node = build_node_from_settings(settings) + server = create_mcp_server(node, settings) + run_mcp( + server, + args.transport, + host=settings.http_host, + port=settings.http_port, + bearer_token=settings.http_bearer_token, + ) + return 0 + + +def _cmd_sign(args: argparse.Namespace) -> int: + node = build_node_from_settings(Settings()) + payload = json.loads(args.payload_json) + env = node.build( + EnvelopeType(args.type), + args.to, + payload, + idempotency_key=args.idempotency_key, + ) + print(env.model_dump_json(by_alias=True)) + return 0 + + +def _cmd_send(args: argparse.Namespace) -> int: + from .transport.http import HttpPeerClient + + settings = Settings() + node = build_node_from_settings(settings) + envelope = parse_envelope(json.loads(Path(args.envelope_file).read_text())) + client = HttpPeerClient( + node, + args.url, + args.peer_key, + bearer_token=settings.http_bearer_token, + ) + try: + response = client.send(envelope) + except PipError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(response.model_dump_json(by_alias=True)) + verify_signature(response, args.peer_key) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="pip-node", description="PIP v1 node CLI") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("keygen", help="generate an instance key (mode 0600)") + p.add_argument("--out", default="./data/instance.key") + p.add_argument("--force", action="store_true") + p.set_defaults(func=_cmd_keygen) + + p = sub.add_parser("identity", help="print the identity document") + p.set_defaults(func=_cmd_identity) + + p = sub.add_parser("serve-http", help="run the HTTP transport") + p.set_defaults(func=_cmd_serve_http) + + p = sub.add_parser("serve-mcp", help="run the MCP transport") + p.add_argument("--transport", choices=["stdio", "streamable-http"], default="stdio") + p.set_defaults(func=_cmd_serve_mcp) + + p = sub.add_parser("sign", help="sign a payload into an envelope (prints JSON)") + p.add_argument("--type", required=True, choices=[t.value for t in EnvelopeType]) + p.add_argument("--to", required=True) + p.add_argument("--payload-json", required=True) + p.add_argument("--idempotency-key", default=None) + p.set_defaults(func=_cmd_sign) + + p = sub.add_parser("send", help="send a signed envelope to a peer") + p.add_argument("--url", required=True) + p.add_argument("--peer-key", required=True) + p.add_argument("--envelope-file", required=True) + p.set_defaults(func=_cmd_send) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return int(args.func(args)) + except PipError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..933f6ae --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,68 @@ +import json +import os + +from pip_protocol.cli import main +from pip_protocol.identity import KeyPair + + +def test_keygen(tmp_path, capsys): + out = tmp_path / "k.key" + assert main(["keygen", "--out", str(out)]) == 0 + captured = capsys.readouterr().out + assert "instance_id: pip:" in captured + assert "public_key: ed25519:" in captured + # private key material never printed + assert KeyPair.load(out).private_bytes.hex() not in captured + + +def test_keygen_refuses_overwrite(tmp_path, capsys): + out = tmp_path / "k.key" + main(["keygen", "--out", str(out)]) + assert main(["keygen", "--out", str(out)]) == 2 + assert main(["keygen", "--out", str(out), "--force"]) == 0 + + +def test_identity(tmp_path, capsys, monkeypatch): + out = tmp_path / "k.key" + main(["keygen", "--out", str(out)]) + monkeypatch.setenv("PIP_PRIVATE_KEY_FILE", str(out)) + assert main(["identity"]) == 0 + doc = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert doc["instance_id"] == KeyPair.load(out).instance_id + + +def test_sign(tmp_path, capsys, monkeypatch): + out = tmp_path / "k.key" + main(["keygen", "--out", str(out)]) + capsys.readouterr() + monkeypatch.setenv("PIP_PRIVATE_KEY_FILE", str(out)) + monkeypatch.setenv("PIP_POLICY_FILE", str(tmp_path / "nonexistent.yaml")) + payload = json.dumps({"subject": "s", "body": "hi"}) + rc = main( + [ + "sign", + "--type", + "message", + "--to", + "pip:example", + "--payload-json", + payload, + "--idempotency-key", + "k1", + ] + ) + assert rc == 0 + env = json.loads(capsys.readouterr().out) + assert env["type"] == "message" + assert env["to"] == "pip:example" + assert env["idempotency_key"] == "k1" + assert env["sig"]["kid"].startswith("ed25519:") + + +def test_env_prefix_settings(monkeypatch): + monkeypatch.setenv("PIP_DISPLAY_NAME", "tester") + from pip_protocol.config import Settings + + s = Settings() + assert s.display_name == "tester" + assert "PIP_PRIVATE_KEY_FILE" not in os.environ or True From c1b92d45fc7f89104a8f2d4514bfbb08a79adb1a Mon Sep 17 00:00:00 2001 From: commdevin Date: Fri, 11 Sep 2026 22:39:24 +0000 Subject: [PATCH 9/9] docs: README, security notes, examples, config templates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- CHANGELOG.md | 17 ++++ README.md | 139 ++++++++++++++++++++++++++++++++- config/.env.example | 15 ++++ config/mcp-client.example.json | 18 +++++ config/policy.example.yaml | 21 +++++ docs/SECURITY.md | 31 ++++++++ examples/two_nodes_demo.py | 76 ++++++++++++++++++ tests/test_demo.py | 11 +++ 8 files changed, 326 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 config/.env.example create mode 100644 config/mcp-client.example.json create mode 100644 config/policy.example.yaml create mode 100644 docs/SECURITY.md create mode 100644 examples/two_nodes_demo.py create mode 100644 tests/test_demo.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..531a706 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +## 1.0.0 + +Initial release of the Poke Interconnect Protocol (PIP) v1. + +- Core: Ed25519 identity, canonical-JSON signed envelopes, schema-validated + payloads (handshake, message, data, receipt, error), policy engine with + scopes/consent/rate limits, outbound redaction, replay protection, + idempotency, persistent outbox with backoff, in-memory and SQLite stores. +- Transports: FastAPI HTTP (`/.well-known/pip`, `/healthz`, `/pip/v1/inbox`, + `/metrics`, optional bearer gate, `HttpPeerClient`) and MCP via `FastMCP` + (5 tools, 3 resources, 2 prompts, stdio + streamable-http). +- CLI `pip-node`: `keygen`, `identity`, `serve-http`, `serve-mcp`, `sign`, + `send`. +- Config via `PIP_*` environment (pydantic-settings); Prometheus metrics and + JSON structured logs. diff --git a/README.md b/README.md index a4bbabb..a340be4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,137 @@ -# hivemind-public -Public Poke Interconnect Protocol (PIP v1) and Poke-compatible MCP server for autonomous communication and data exchange between Poke instances +# poke-interconnect — Poke Interconnect Protocol (PIP) v1 + +PIP lets independent Poke instances discover each other's capabilities and +exchange **signed, consented, idempotent** messages and data. One envelope +format, one policy engine, and one security model are shared by two +transports: an HTTP/JSON API and an MCP server. + +The full normative spec lives in [`docs/PROTOCOL.md`](docs/PROTOCOL.md). +Install name: `poke-interconnect`; import name: `pip_protocol`; CLI: `pip-node`. + +## Architecture + +``` + Peer A Peer B + ┌──────────────────┐ ┌──────────────────┐ + │ pip-node / SDK │ │ pip-node / SDK │ + │ ┌────────────┐ │ │ ┌────────────┐ │ + │ │ Node │ │ envelope │ │ Node │ │ + │ │ sign/verify│ ├─────────►│ │ verify → │ │ + │ │ policy │ │ receipt │ │ policy → │ │ + │ │ redaction │ │◄─────────┤ │ handler │ │ + │ └─────┬──────┘ │ │ └─────┬──────┘ │ + │ HTTP / MCP │ │ HTTP / MCP │ + └──────────────────┘ └──────────────────┘ + Ed25519 identity · pip: +``` + +## Quickstart + +```bash +uv venv .venv && uv pip install -e .[dev] # or: pip install poke-interconnect + +# 1. generate an instance key (mode 0600) +pip-node keygen --out ./data/instance.key + +# 2. configure peers and consent +cp config/policy.example.yaml config/policy.yaml # edit peers/scopes/consents +cp config/.env.example .env # optional + +# 3. serve +PIP_PRIVATE_KEY_FILE=./data/instance.key PIP_POLICY_FILE=./config/policy.yaml \ + pip-node serve-http # HTTP on 127.0.0.1:8642 +pip-node serve-mcp --transport stdio # MCP over stdio +pip-node serve-mcp --transport streamable-http # MCP over HTTP +``` + +Other CLI commands: `pip-node identity`, `pip-node sign`, `pip-node send` +(see `pip-node --help`). Try the in-process demo: +`python examples/two_nodes_demo.py`. + +### MCP client config + +See `config/mcp-client.example.json`: + +```json +{ "mcpServers": { "poke": { + "command": "pip-node", + "args": ["serve-mcp", "--transport", "stdio"], + "env": { "PIP_PRIVATE_KEY_FILE": "./data/instance.key" } } } } +``` + +## HTTP API (§8) + +| Endpoint | Auth | Description | +|---|---|---| +| `GET /.well-known/pip` | none | signed handshake envelope (identity + capabilities) | +| `GET /healthz` | none | `{"status":"ok","version":"pip/1.0"}` | +| `POST /pip/v1/inbox` | signature (+bearer) | envelope → signed `receipt`/`data`/`error` | +| `GET /metrics` | bearer if set | Prometheus text format | + +`PIP_HTTP_BEARER_TOKEN` is an optional defense-in-depth gate; it never +replaces the envelope signature. `X-PIP-Request-Id` is echoed/generated, +`X-PIP-Trace-Id` propagated to logs. + +## MCP (§9) + +Tools: `pip_handshake`, `pip_send_message`, `pip_exchange_data`, +`pip_get_receipt`, `pip_list_capabilities` (public). Every tool takes a signed +`envelope` dict; failures return a **signed `error` envelope** in the result +(never a raw exception). + +Resources: `pip://identity`, `pip://capabilities`, `pip://policy/scopes`. +Prompts: `pip_compose_message(to, subject, intent)`, +`pip_request_data(dataset, purpose)`. + +## Configuration (§12) + +| Variable | Default | Notes | +|---|---|---| +| `PIP_PRIVATE_KEY_FILE` | `./data/instance.key` | mode 0600; key material never in env | +| `PIP_DISPLAY_NAME` | `poke` | | +| `PIP_POLICY_FILE` | `./config/policy.yaml` | YAML or JSON | +| `PIP_STORE_URL` | `memory://` | or `sqlite:///data/pip.db` | +| `PIP_HTTP_HOST` / `PIP_HTTP_PORT` | `127.0.0.1` / `8642` | loopback by default | +| `PIP_PUBLIC_HTTP_URL` / `PIP_PUBLIC_MCP_URL` | unset | advertised endpoints | +| `PIP_HTTP_BEARER_TOKEN` | unset | optional transport gate | +| `PIP_MAX_CLOCK_SKEW_SECONDS` | `300` | | +| `PIP_IDEMPOTENCY_TTL_SECONDS` | `86400` | | +| `PIP_LOG_LEVEL` | `INFO` | JSON structured logs | + +## Security model (summary) + +Ed25519 signatures over canonical JSON (`"PIPv1\n" + canonical_json(env-sig)`); +peers pinned by public key with rotation grace; nonce + time-window replay +protection; idempotency keys for effectively-once delivery; scope + consent +authorization; outbound redaction before signing; size caps and per-peer token +bucket rate limits. See §13 threat model and `docs/SECURITY.md`. + +## Delivery semantics + +At-least-once from the sender (client retries with the same +`idempotency_key`), effectively-once at the receiver. `pip.delivery.Outbox` +persists pending envelopes with exponential backoff (base 2s, cap 300s, 8 +attempts → dead). + +## Observability + +Prometheus counters `envelopes_received_total{type,outcome}`, +`envelopes_sent_total`, `policy_denials_total{code}`; histogram +`handler_seconds`. Structured JSON logs, one line per envelope/request — never +payload bodies, keys, or tokens. + +## Development + +```bash +uv pip install -e .[dev] +ruff check . && ruff format --check . && mypy src && pytest -q +``` + +## Limitations (v1) + +Transport confidentiality is delegated to TLS (terminate at a reverse proxy); +no metadata privacy; no Sybil resistance / peer reputation. + +## License + +MIT — see `LICENSE`. diff --git a/config/.env.example b/config/.env.example new file mode 100644 index 0000000..d2afc40 --- /dev/null +++ b/config/.env.example @@ -0,0 +1,15 @@ +# Example PIP environment configuration (spec §12). No real secrets here. +PIP_PRIVATE_KEY_FILE=./data/instance.key +PIP_DISPLAY_NAME=poke +PIP_POLICY_FILE=./config/policy.yaml +PIP_STORE_URL=memory:// +# PIP_STORE_URL=sqlite:///data/pip.db +PIP_HTTP_HOST=127.0.0.1 +PIP_HTTP_PORT=8642 +# PIP_PUBLIC_HTTP_URL=https://poke.example.com +# PIP_PUBLIC_MCP_URL= +# PIP_HTTP_BEARER_TOKEN= +PIP_MAX_CLOCK_SKEW_SECONDS=300 +PIP_IDEMPOTENCY_TTL_SECONDS=86400 +PIP_LOG_LEVEL=INFO +# PIP_ALLOW_INSECURE_KEY_PERMS=1 # tests only diff --git a/config/mcp-client.example.json b/config/mcp-client.example.json new file mode 100644 index 0000000..4e4732a --- /dev/null +++ b/config/mcp-client.example.json @@ -0,0 +1,18 @@ +{ + "mcpServers": { + "poke-stdio": { + "command": "pip-node", + "args": ["serve-mcp", "--transport", "stdio"], + "env": { + "PIP_PRIVATE_KEY_FILE": "./data/instance.key", + "PIP_POLICY_FILE": "./config/policy.yaml", + "PIP_STORE_URL": "memory://", + "PIP_DISPLAY_NAME": "poke" + } + }, + "poke-http": { + "url": "http://127.0.0.1:8642/mcp", + "comment": "streamable-http variant: run `pip-node serve-mcp --transport streamable-http` and set Authorization header if PIP_HTTP_BEARER_TOKEN is configured" + } + } +} diff --git a/config/policy.example.yaml b/config/policy.example.yaml new file mode 100644 index 0000000..08637aa --- /dev/null +++ b/config/policy.example.yaml @@ -0,0 +1,21 @@ +# Example PIP policy file (spec §5). Copy to config/policy.yaml and edit. +# Contains NO real keys or secrets. +allow_unknown_peers: false +default_scopes: [] +peers: + - instance_id: pip:examplepeer00000000000000 + public_key: ed25519:REPLACE_WITH_PEER_PUBLIC_KEY + display_name: "Alice's Poke" + scopes: [messages:send, data:request] + consents: + - resource: data:notes + actions: [read] + expires: 2027-01-01T00:00:00Z + granted_by: operator +rate_limits: + per_peer_per_minute: 60 + burst: 20 +max_payload_bytes: 262144 +redaction: + fields: [email, phone, ssn, api_key, token, password, secret] + patterns: ["\\b[\\w.+-]+@[\\w-]+\\.[\\w.]+\\b"] diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..7b2a126 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,31 @@ +# Security + +## Reporting a vulnerability + +Please report security issues privately to the CommunityPoke maintainers — +open a GitHub security advisory on this repository rather than a public issue. + +## Key handling + +- Instance keys are Ed25519. The private key lives **only** in a file + (`PIP_PRIVATE_KEY_FILE`, mode `0600`); it is never placed in environment + variables or config files, and never logged. +- The loader refuses group/world-readable key files. Set + `PIP_ALLOW_INSECURE_KEY_PERMS=1` for tests only. +- Rotate keys by publishing `previous_keys` in your identity document; peers + accept the old key for `rotation_grace_seconds` (default 86400). + +## Deployment + +- Bind to loopback by default (`PIP_HTTP_HOST=127.0.0.1`). PIP v1 delegates + transport confidentiality to TLS — terminate TLS at a reverse proxy in + front of `serve-http` / `serve-mcp --transport streamable-http`. +- `PIP_HTTP_BEARER_TOKEN` is an optional defense-in-depth gate; the envelope + signature remains the real authentication. +- Logs and metrics never include payload bodies, keys, or tokens. + +## What PIP does and does not cover + +See `docs/PROTOCOL.md` §13. Mitigated: spoofing, replay, tamper, +over-collection, abuse, downgrade, key compromise. Out of scope for v1: +transport-layer confidentiality, metadata privacy, Sybil resistance. diff --git a/examples/two_nodes_demo.py b/examples/two_nodes_demo.py new file mode 100644 index 0000000..c8a12fb --- /dev/null +++ b/examples/two_nodes_demo.py @@ -0,0 +1,76 @@ +"""In-process demo: two PIP nodes exchange a message and a data request. + +Run: python examples/two_nodes_demo.py +""" + +from pip_protocol.identity import KeyPair +from pip_protocol.node import DictDataProvider, Node +from pip_protocol.policy import Consent, PeerRecord, Policy +from pip_protocol.schemas import DataPayload, EnvelopeType, MessagePayload + + +def peer(kp, scopes, consents=None): + return PeerRecord( + instance_id=kp.instance_id, + public_key=kp.public_key, + scopes=scopes, + consents=consents or [], + ) + + +def main() -> None: + ka, kb = KeyPair.generate(), KeyPair.generate() + a = Node(ka, Policy(peers=[peer(kb, ["admin:*"])]), display_name="alice") + b = Node( + kb, + Policy( + peers=[ + peer( + ka, + ["messages:send", "data:request"], + [Consent(resource="data:notes", actions=["read"])], + ) + ] + ), + data_provider=DictDataProvider({"notes": [{"n": i} for i in range(3)]}), + display_name="bob", + ) + print(f"A={a.instance_id} B={b.instance_id}") + + # direct Node API + env = a.build( + EnvelopeType.MESSAGE, b.instance_id, MessagePayload(subject="hi", body="hello bob") + ) + receipt, err = b.handle(env) + assert err is None + print("message receipt:", receipt.typed_payload().status, "| inbox:", len(b.inbox)) + + req = a.build(EnvelopeType.DATA, b.instance_id, DataPayload(op="request", dataset="notes")) + resp, err = b.handle(req) + assert err is None + print("data records:", resp.typed_payload().records) + + # over the HTTP transport (in-process ASGI) + try: + from fastapi.testclient import TestClient + + from pip_protocol.transport.http import HttpPeerClient, create_app + + peer_client = HttpPeerClient( + a, "http://bob", kb.public_key, client=TestClient(create_app(b)) + ) + env2 = a.build( + EnvelopeType.MESSAGE, + b.instance_id, + MessagePayload(subject="via http", body="hi over http"), + ) + receipt2 = peer_client.send(env2) + print("http receipt:", receipt2.typed_payload().status) + hs = peer_client.fetch_handshake() + print("handshake from:", hs.typed_payload().identity.display_name) + except ImportError: + print("(install httpx+fastapi for the HTTP part of the demo)") + + +if __name__ == "__main__": + main() diff --git a/tests/test_demo.py b/tests/test_demo.py new file mode 100644 index 0000000..ce428ac --- /dev/null +++ b/tests/test_demo.py @@ -0,0 +1,11 @@ +def test_two_nodes_demo_runs(): + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "examples")) + try: + import two_nodes_demo + + two_nodes_demo.main() + finally: + sys.path.pop(0)