From 21a5be5dae7e9b69fd002f17d2790521e0d13194 Mon Sep 17 00:00:00 2001 From: Alex Soffronow Pagonidis <237136924+alex-clickhouse@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:46:57 +0000 Subject: [PATCH 01/10] Sources: rules on the JSON data of generic hosted records A generic hosted source sends each record's data as a JSON object that the provider defines, and its subscription carries allow/deny rules on JSON Pointer paths into that data. The channel gateway applies the rules before it stores a record, and the inbox guardrail must apply the same rules again, so both sides drop the same records. nerve/sources/source_json.py holds what the channel contract and the guardrail share: the checks of a source JSON object, its canonical form (RFC 8785), and JSON Pointers. DataRule in filters.py evaluates a rule on a record's metadata with the contract's value semantics, and the contract's rule vectors pin them. The runner's drop log names the path and the value of the rule that dropped a record. Co-Authored-By: Claude Opus 5.5 --- nerve/sources/filters.py | 132 ++++- nerve/sources/runner.py | 5 +- nerve/sources/source_json.py | 253 ++++++++ .../channel_contract/source_rules.json | 548 ++++++++++++++++++ tests/test_source_filters.py | 84 ++- 5 files changed, 1002 insertions(+), 20 deletions(-) create mode 100644 nerve/sources/source_json.py create mode 100644 tests/fixtures/channel_contract/source_rules.json diff --git a/nerve/sources/filters.py b/nerve/sources/filters.py index 4c2380526..35ae15651 100644 --- a/nerve/sources/filters.py +++ b/nerve/sources/filters.py @@ -29,6 +29,12 @@ (``ClickHouse/*``). List-valued metadata (e.g. Gmail ``labels``) matches if *any* element matches. Non-string scalars (e.g. Telegram ``chat_id``) are coerced to ``str`` before matching. + +A generic hosted source (Linear, for example) uses :class:`DataRule`: the +record's metadata is the JSON data that the provider sent, and each rule +names a value in it with a JSON Pointer. The channel gateway applies the same +rules before it stores a record, so these semantics are the channel +contract's, and the two sides drop the same records. """ from __future__ import annotations @@ -36,8 +42,11 @@ import dataclasses import fnmatch import logging +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + +from nerve.sources.source_json import SourceJSONError, canonical_number, pointer_tokens, resolve_pointer if TYPE_CHECKING: from nerve.sources.models import SourceRecord @@ -50,7 +59,7 @@ def _norm(value: object) -> str: return str(value).strip().lower() -def matches_any(value: object, patterns: list[str]) -> bool: +def matches_any(value: object, patterns: Iterable[str]) -> bool: """Case-insensitive shell-glob match of *value* against any pattern. The guardrail rules, for a source that applies them itself (the GitHub @@ -101,34 +110,127 @@ def _values(self, record: SourceRecord) -> list[str]: return [str(v) for v in raw if v is not None] return [str(raw)] + def value(self, record: SourceRecord) -> object: + """The metadata value that the rule judges, for a log line.""" + return (record.metadata or {}).get(self.field) + def passes(self, record: SourceRecord) -> bool: """Return True if *record* satisfies this rule (should be kept).""" if record.record_type in self.exempt_record_types: return True - values = self._values(record) + return _passes(self._values(record), self.allow, self.deny) + - # deny wins — drop on any deny match. - if self.deny and any(matches_any(v, self.deny) for v in values): +def _passes(values: list[str], allow: Sequence[str], deny: Sequence[str]) -> bool: + """The decision of one rule on its values, with the semantics at the top of this module.""" + # deny wins — drop on any deny match. + if deny and any(matches_any(v, deny) for v in values): + return False + + # allow gate — when set, require a match. An absent field fails closed. + if allow: + if not values: return False + return any(matches_any(v, allow) for v in values) + + return True + - # allow gate — when set, require a match. An absent field fails closed. - if self.allow: - if not values: - return False - return any(matches_any(v, self.allow) for v in values) +@dataclass(frozen=True) +class DataRule: + """Allow/deny matching on a value in the data of a generic hosted record. + + ``path`` is a JSON Pointer (RFC 6901) into the record's metadata, which + holds the provider's data, such as ``/actor/kind``. The values at the + path are: + + * a string, as it is; + * a number, as its canonical JSON text: ``2`` for ``2.0``, ``1.5``; + * ``true``, ``false`` and ``null``, as those words; + * each element of an array that is one of the above. + + An object, an array or object in an array, an empty array, and a path + that names nothing in the data give no value. ``allow`` and ``deny`` + match those values as :class:`FieldRule` matches: a deny match drops the + record, and a non-empty allow list drops a record without a match, also + one without a value. A path that is not a valid JSON Pointer names + nothing. + """ + + path: str + allow: tuple[str, ...] = () + deny: tuple[str, ...] = () + + @property + def field(self) -> str: + """The path, as the runner's drop log and the drop hook name a rule.""" + return self.path - return True + @property + def active(self) -> bool: + return bool(self.allow or self.deny) + + def _resolved(self, record: SourceRecord) -> tuple[bool, Any]: + tokens = pointer_tokens(self.path) + if tokens is None: + return False, None + return resolve_pointer(record.metadata or {}, tokens) + + def values(self, record: SourceRecord) -> list[str]: + """The values that the rule judges in *record*.""" + found, value = self._resolved(record) + if not found: + return [] + values: list[str] = [] + for element in value if isinstance(value, (list, tuple)) else [value]: + if isinstance(element, str): + values.append(element) + elif element is None: + values.append("null") + elif isinstance(element, bool): + values.append("true" if element else "false") + elif isinstance(element, (int, float)): + try: + values.append(canonical_number(element)) + except SourceJSONError: # a number that no JSON data can hold + continue + return values + + def value(self, record: SourceRecord) -> object: + """The value at the path, for a log line; ``None`` when there is none.""" + return self._resolved(record)[1] + + def passes(self, record: SourceRecord) -> bool: + """Return True if *record* satisfies this rule (should be kept).""" + return _passes(self.values(record), self.allow, self.deny) + + +def data_rules(rules: Iterable[Mapping[str, Any]]) -> list[DataRule]: + """The :class:`DataRule` of each rule of a generic source's settings. + + Each rule is a mapping with a ``path`` and an ``allow`` or ``deny`` list, + as the config loads it (``sync.linear.rules``, for example). + """ + return [ + DataRule( + path=str(rule.get("path", "")), + allow=tuple(rule.get("allow") or ()), + deny=tuple(rule.get("deny") or ()), + ) + for rule in rules + ] @dataclass class InboxFilter: - """A set of :class:`FieldRule` applied to source records (logical AND). + """A set of rules applied to source records (logical AND). - An inactive filter (no rules, or all rules empty) is a pass-through — + Each rule is a :class:`FieldRule` or a :class:`DataRule`. An inactive + filter (no rules, or all rules empty) is a pass-through — :meth:`partition` returns every record as kept with nothing dropped. """ - rules: list[FieldRule] = dataclasses.field(default_factory=list) + rules: list[FieldRule | DataRule] = dataclasses.field(default_factory=list) @property def active(self) -> bool: @@ -139,7 +241,7 @@ def passes(self, record: SourceRecord) -> bool: """Return True if *record* passes all rules (should be kept).""" return all(r.passes(record) for r in self.rules) - def rejects(self, record: SourceRecord) -> FieldRule | None: + def rejects(self, record: SourceRecord) -> FieldRule | DataRule | None: """The first rule that drops *record*, or None if it passes. Dropped records are never persisted, so this is the only way to tell diff --git a/nerve/sources/runner.py b/nerve/sources/runner.py index 39cd338ea..5deab526a 100644 --- a/nerve/sources/runner.py +++ b/nerve/sources/runner.py @@ -411,11 +411,10 @@ async def _ingest_batch(self, cursor: str | None) -> _BatchResult: # itself is gone after this point. Capped to keep logs sane. for record in dropped[:_MAX_DROP_LOG_LINES]: rule = self.inbox_filter.rejects(record) - field = rule.field if rule else "?" logger.info( "Source %s: dropped %s on %s=%r — %s", - self.source.source_name, record.id, field, - (record.metadata or {}).get(field), record.summary, + self.source.source_name, record.id, rule.field if rule else "?", + rule.value(record) if rule else None, record.summary, ) records = kept dropped_count = len(dropped) diff --git a/nerve/sources/source_json.py b/nerve/sources/source_json.py new file mode 100644 index 000000000..f4f5f2746 --- /dev/null +++ b/nerve/sources/source_json.py @@ -0,0 +1,253 @@ +"""The JSON values of generic hosted sources. + +A generic source of the hosted channel gateway, such as Linear, sends the data +of each record as a JSON object that the provider defines, and Nerve sends +the settings of a subscription to it as a JSON object too. The channel +contract calls both a *source JSON object*: a JSON object without a repeated +member name, of valid Unicode (no unpaired surrogate), with at most +:data:`MAX_SOURCE_JSON_DEPTH` levels of objects and arrays (the object itself +is the first level), and with numbers that IEEE 754 binary64 can hold. Its +byte bounds count the UTF-8 bytes of its canonical form (RFC 8785, the JSON +Canonicalization Scheme), which does not depend on how a sender encodes it. + +The rules of a subscription keep or drop records by the value at a JSON +Pointer (RFC 6901) into their data. + +This module holds what the channel contract (which checks these values on +the wire) and the inbox guardrail (which applies the rules again) share. It +works on the values that :func:`json.loads` returns, and it imports nothing +from Nerve. +""" + +from __future__ import annotations + +import json +import math +import re +import unicodedata +from decimal import Decimal +from typing import Any + +# The levels of objects and arrays in a source JSON object, the object itself +# as the first. A JSON Pointer into one has at most this many tokens. +MAX_SOURCE_JSON_DEPTH = 8 +# The UTF-8 bytes of a JSON Pointer in a subscription rule. +MAX_POINTER_BYTES = 256 +# Plain decimal notation of a canonical number: from 1e-6 up to but not +# including 1e21 (ECMAScript Number.prototype.toString). +_PLAIN_EXPONENT_MIN = -6 +_PLAIN_EXPONENT_MAX = 21 +# A "~" that does not start the escape "~0" or "~1". +_BAD_ESCAPE = re.compile(r"~(?![01])") +# An array index in a JSON Pointer: decimal, without a leading zero. +_ARRAY_INDEX = re.compile(r"0|[1-9][0-9]*") + + +class SourceJSONError(ValueError): + """A value that breaks a rule of a source JSON object. + + ``limit`` is true when the value is above a bound (its depth or its + size), and false when it is not valid. + """ + + def __init__(self, detail: str, *, limit: bool = False) -> None: + super().__init__(detail) + self.limit = limit + + +def check_source_json_object(value: Any, maximum_bytes: int) -> None: + """Check that *value* is a source JSON object within *maximum_bytes*. + + The bound counts the UTF-8 bytes of the canonical form. Raises + :class:`SourceJSONError`. A repeated member name cannot be seen here, so + the parser refuses it (see ``nerve.channels.hosted.contract.wire``). + """ + if not isinstance(value, dict): + raise SourceJSONError("is not a JSON object") + _check(value, 1) + if len(canonical_json(value).encode("utf-8")) > maximum_bytes: + raise SourceJSONError("exceeds its byte limit", limit=True) + + +def _check(value: Any, depth: int) -> None: + if isinstance(value, (dict, list, tuple)): + if depth > MAX_SOURCE_JSON_DEPTH: + raise SourceJSONError("exceeds its depth limit", limit=True) + if isinstance(value, dict): + for name, member in value.items(): + if not isinstance(name, str): + raise SourceJSONError("has a member name that is not a string") + _check_text(name) + _check(member, depth + 1) + else: + for element in value: + _check(element, depth + 1) + elif isinstance(value, str): + _check_text(value) + elif isinstance(value, bool) or value is None: + pass + elif isinstance(value, (int, float)): + _binary64(value) + else: + raise SourceJSONError(f"holds a {type(value).__name__}, which is not a JSON value") + + +def _check_text(text: str) -> None: + try: + text.encode("utf-8") + except UnicodeEncodeError: + raise SourceJSONError("has a string with an unpaired surrogate") from None + + +def _binary64(number: int | float) -> float: + """*number* as binary64, or :class:`SourceJSONError` when it does not fit. + + A number below the smallest binary64 value is zero, as the parsers of + both sides read it. + """ + try: + value = float(number) + except OverflowError: + raise SourceJSONError("has a number outside the binary64 range") from None + if not math.isfinite(value): + raise SourceJSONError("has a number outside the binary64 range") + return value + + +# ---------------------------------------------------------------------- # +# Canonical form (RFC 8785) # +# ---------------------------------------------------------------------- # + + +def canonical_json(value: Any) -> str: + """The RFC 8785 canonical form of a value that passes the checks. + + Object members sorted by the UTF-16 code units of their names, no white + space, the shortest ECMAScript form of each number, and the minimal + escapes of each string: only the quotation mark, the reverse solidus and + the characters below U+0020 are escaped. + """ + parts: list[str] = [] + _write(parts, value) + return "".join(parts) + + +def _write(parts: list[str], value: Any) -> None: + if value is None: + parts.append("null") + elif value is True: + parts.append("true") + elif value is False: + parts.append("false") + elif isinstance(value, (int, float)): + parts.append(canonical_number(value)) + elif isinstance(value, str): + # The escapes of json.dumps without ensure_ascii are those of RFC 8785 + # section 3.2.2.2, with lowercase hexadecimal digits. + parts.append(json.dumps(value, ensure_ascii=False)) + elif isinstance(value, dict): + parts.append("{") + names = sorted(value, key=lambda name: name.encode("utf-16-be", "surrogatepass")) + for index, name in enumerate(names): + if index: + parts.append(",") + parts.append(json.dumps(name, ensure_ascii=False)) + parts.append(":") + _write(parts, value[name]) + parts.append("}") + else: + parts.append("[") + for index, element in enumerate(value): + if index: + parts.append(",") + _write(parts, element) + parts.append("]") + + +def canonical_number(number: int | float) -> str: + """The ECMAScript ``Number.prototype.toString`` form of a binary64 number. + + RFC 8785 section 3.2.2.3: the fewest digits that read back as the same + number, plain decimal notation from 1e-6 up to but not including 1e21, + exponent notation otherwise, and ``0`` for negative zero. An integer is + read as binary64 first, so ``12345678901234567890`` is + ``12345678901234567000`` and ``2.0`` is ``2``. + """ + value = _binary64(number) + if value == 0: + return "0" + sign = "-" if value < 0 else "" + # repr gives the shortest digits that read back as the same binary64. + _, digit_tuple, exponent = Decimal(repr(abs(value))).normalize().as_tuple() + digits = "".join(map(str, digit_tuple)) + count = len(digits) + point = int(exponent) + count # the decimal point is after this many digits + if count <= point <= _PLAIN_EXPONENT_MAX: + return sign + digits + "0" * (point - count) + if 0 < point <= _PLAIN_EXPONENT_MAX: + return sign + digits[:point] + "." + digits[point:] + if _PLAIN_EXPONENT_MIN < point <= 0: + return sign + "0." + "0" * -point + digits + power = point - 1 + mantissa = digits if count == 1 else f"{digits[0]}.{digits[1:]}" + return f"{sign}{mantissa}e{'+' if power > 0 else '-'}{abs(power)}" + + +# ---------------------------------------------------------------------- # +# JSON Pointer (RFC 6901) # +# ---------------------------------------------------------------------- # + + +def pointer_tokens(path: str) -> list[str] | None: + """The reference tokens of *path*, unescaped, or ``None`` when it is not valid. + + A valid path starts with ``/``, has 1 to :data:`MAX_SOURCE_JSON_DEPTH` + tokens and at most :data:`MAX_POINTER_BYTES` bytes of UTF-8 without + control characters, and writes ``~`` only in the escapes ``~0`` (``~``) + and ``~1`` (``/``). + """ + if not isinstance(path, str) or not path.startswith("/"): + return None + try: + size = len(path.encode("utf-8")) + except UnicodeEncodeError: + return None + if size > MAX_POINTER_BYTES or any(unicodedata.category(c) == "Cc" for c in path): + return None + tokens = path[1:].split("/") + if len(tokens) > MAX_SOURCE_JSON_DEPTH or any(_BAD_ESCAPE.search(t) for t in tokens): + return None + # RFC 6901 section 4: ~1 becomes / before ~0 becomes ~. + return [token.replace("~1", "/").replace("~0", "~") for token in tokens] + + +def resolve_pointer(value: Any, tokens: list[str]) -> tuple[bool, Any]: + """Whether the *tokens* name a value in *value*, and that value. + + A token names the member of an object, or the element of an array at the + index that it writes in decimal without a leading zero. + """ + for token in tokens: + if isinstance(value, dict): + if token not in value: + return False, None + value = value[token] + elif isinstance(value, (list, tuple)): + if _ARRAY_INDEX.fullmatch(token) is None or int(token) >= len(value): + return False, None + value = value[int(token)] + else: + return False, None + return True, value + + +__all__ = [ + "MAX_POINTER_BYTES", + "MAX_SOURCE_JSON_DEPTH", + "SourceJSONError", + "canonical_json", + "canonical_number", + "check_source_json_object", + "pointer_tokens", + "resolve_pointer", +] diff --git a/tests/fixtures/channel_contract/source_rules.json b/tests/fixtures/channel_contract/source_rules.json new file mode 100644 index 000000000..4c8925c87 --- /dev/null +++ b/tests/fixtures/channel_contract/source_rules.json @@ -0,0 +1,548 @@ +{ + "data": { + "event": "assigned", + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "name": "Alice Example", + "kind": "user", + "credential_user": false + }, + "issue": { + "identifier": "ENG-12", + "team": { + "key": "ENG" + }, + "priority": 2, + "estimate": 1.5, + "size": 2.0, + "labels": [ + "bug", + "Frontend", + 7, + true, + null, + [ + "nested" + ], + { + "name": "object" + } + ], + "watchers": [], + "assignee": null + }, + "changes": [ + { + "field": "assignee", + "from": null, + "to": "8d1e3f5a-7b9c-4d2e-a6f8-1c3e5a7b9d20" + } + ], + "a/b": "slash", + "m~n": "tilde", + "": "empty name" + }, + "cases": [ + { + "name": "no rules keep every record", + "rules": [], + "keep": true + }, + { + "name": "a deny pattern that matches drops", + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot", + "user" + ] + } + ], + "keep": false + }, + { + "name": "a deny list without a match keeps", + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot", + "integration" + ] + } + ], + "keep": true + }, + { + "name": "an allow pattern matches without case", + "rules": [ + { + "path": "/issue/team/key", + "allow": [ + "eng*" + ] + } + ], + "keep": true + }, + { + "name": "an upper-case allow pattern matches", + "rules": [ + { + "path": "/issue/team/key", + "allow": [ + "ENG*" + ] + } + ], + "keep": true + }, + { + "name": "an allow list without a match drops", + "rules": [ + { + "path": "/issue/team/key", + "allow": [ + "OPS*" + ] + } + ], + "keep": false + }, + { + "name": "a question mark matches one character", + "rules": [ + { + "path": "/issue/identifier", + "allow": [ + "eng-1?" + ] + } + ], + "keep": true + }, + { + "name": "a deny rule on a missing path keeps", + "rules": [ + { + "path": "/issue/project/name", + "deny": [ + "*" + ] + } + ], + "keep": true + }, + { + "name": "an allow rule on a missing path drops", + "rules": [ + { + "path": "/issue/project/name", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "an integer matches its canonical text", + "rules": [ + { + "path": "/issue/priority", + "allow": [ + "1", + "2" + ] + } + ], + "keep": true + }, + { + "name": "a fraction matches its canonical text", + "rules": [ + { + "path": "/issue/estimate", + "allow": [ + "1.5" + ] + } + ], + "keep": true + }, + { + "name": "a number written as 2.0 matches 2", + "rules": [ + { + "path": "/issue/size", + "allow": [ + "2" + ] + } + ], + "keep": true + }, + { + "name": "a number written as 2.0 does not match 2.0", + "rules": [ + { + "path": "/issue/size", + "allow": [ + "2.0" + ] + } + ], + "keep": false + }, + { + "name": "false matches false", + "rules": [ + { + "path": "/actor/credential_user", + "allow": [ + "false" + ] + } + ], + "keep": true + }, + { + "name": "a deny of true keeps false", + "rules": [ + { + "path": "/actor/credential_user", + "deny": [ + "true" + ] + } + ], + "keep": true + }, + { + "name": "null matches null", + "rules": [ + { + "path": "/issue/assignee", + "deny": [ + "null" + ] + } + ], + "keep": false + }, + { + "name": "any element of an array matches", + "rules": [ + { + "path": "/issue/labels", + "deny": [ + "frontend" + ] + } + ], + "keep": false + }, + { + "name": "a scalar element of an array matches its canonical text", + "rules": [ + { + "path": "/issue/labels", + "allow": [ + "7" + ] + } + ], + "keep": true + }, + { + "name": "true as an element of an array matches", + "rules": [ + { + "path": "/issue/labels", + "allow": [ + "true" + ] + } + ], + "keep": true + }, + { + "name": "null as an element of an array matches", + "rules": [ + { + "path": "/issue/labels", + "allow": [ + "null" + ] + } + ], + "keep": true + }, + { + "name": "an array in an array gives no value", + "rules": [ + { + "path": "/issue/labels", + "allow": [ + "nested" + ] + } + ], + "keep": false + }, + { + "name": "an object in an array gives no value", + "rules": [ + { + "path": "/issue/labels", + "allow": [ + "object" + ] + } + ], + "keep": false + }, + { + "name": "an array index selects one element", + "rules": [ + { + "path": "/issue/labels/0", + "allow": [ + "bug" + ] + } + ], + "keep": true + }, + { + "name": "an index with a leading zero names nothing", + "rules": [ + { + "path": "/issue/labels/01", + "allow": [ + "bug" + ] + } + ], + "keep": false + }, + { + "name": "an index past the end names nothing", + "rules": [ + { + "path": "/issue/labels/7", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "the index - names nothing", + "rules": [ + { + "path": "/issue/labels/-", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "an object gives no value to an allow rule", + "rules": [ + { + "path": "/issue/team", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "an object gives no value to a deny rule", + "rules": [ + { + "path": "/issue/team", + "deny": [ + "*" + ] + } + ], + "keep": true + }, + { + "name": "an empty array gives no value to an allow rule", + "rules": [ + { + "path": "/issue/watchers", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "an empty array gives no value to a deny rule", + "rules": [ + { + "path": "/issue/watchers", + "deny": [ + "*" + ] + } + ], + "keep": true + }, + { + "name": "an array of objects gives no value", + "rules": [ + { + "path": "/changes", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "a path through an array of objects", + "rules": [ + { + "path": "/changes/0/field", + "allow": [ + "assignee" + ] + } + ], + "keep": true + }, + { + "name": "a path through a string names nothing", + "rules": [ + { + "path": "/event/name", + "allow": [ + "*" + ] + } + ], + "keep": false + }, + { + "name": "~1 names a slash", + "rules": [ + { + "path": "/a~1b", + "allow": [ + "slash" + ] + } + ], + "keep": true + }, + { + "name": "~0 names a tilde", + "rules": [ + { + "path": "/m~0n", + "allow": [ + "tilde" + ] + } + ], + "keep": true + }, + { + "name": "the path / names the empty member name", + "rules": [ + { + "path": "/", + "allow": [ + "empty name" + ] + } + ], + "keep": true + }, + { + "name": "a star matches every string", + "rules": [ + { + "path": "/event", + "allow": [ + "*" + ] + } + ], + "keep": true + }, + { + "name": "every rule must keep the record", + "rules": [ + { + "path": "/issue/team/key", + "allow": [ + "ENG" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "user" + ] + } + ], + "keep": false + }, + { + "name": "two allow rules on one path both apply", + "rules": [ + { + "path": "/issue/team/key", + "allow": [ + "ENG" + ] + }, + { + "path": "/issue/team/key", + "allow": [ + "OPS" + ] + } + ], + "keep": false + }, + { + "name": "every rule keeps the record", + "rules": [ + { + "path": "/issue/team/key", + "allow": [ + "ENG" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/event", + "allow": [ + "assigned", + "unassigned" + ] + } + ], + "keep": true + } + ] +} diff --git a/tests/test_source_filters.py b/tests/test_source_filters.py index eb4c4763f..9a080d054 100644 --- a/tests/test_source_filters.py +++ b/tests/test_source_filters.py @@ -2,16 +2,20 @@ Covers the pure matching logic of :class:`FieldRule` / :class:`InboxFilter` plus an end-to-end runner test proving dropped records never reach the inbox -while the source cursor still advances. +while the source cursor still advances, and :class:`DataRule`, the rules of a +generic hosted source, against the channel contract's rule vectors. """ from __future__ import annotations +import json +from pathlib import Path + import pytest from nerve.config import GitHubSyncConfig from nerve.sources.base import Source -from nerve.sources.filters import FieldRule, InboxFilter +from nerve.sources.filters import DataRule, FieldRule, InboxFilter, data_rules from nerve.sources.github import github_inbox_filter from nerve.sources.models import FetchResult, SourceRecord from nerve.sources.runner import SourceRunner @@ -358,3 +362,79 @@ def test_the_repo_rule_still_applies_to_a_pull_request_record(settings): rejected_by = flt.rejects(_github("github_pr_activity")) assert rejected_by is not None and rejected_by.field == "repo_name" + + +# --------------------------------------------------------------------------- +# DataRule: the rules of a generic hosted source, on JSON Pointers into the +# record's data (its metadata), with the semantics of the channel contract +# --------------------------------------------------------------------------- + +# A byte-exact copy of the contract's rule vectors (testdata/source_rules.json); +# test_channel_contract_generic pins its bytes. +RULE_VECTORS = json.loads( + (Path(__file__).parent / "fixtures" / "channel_contract" / "source_rules.json").read_text(encoding="utf-8") +) + + +def _data_record(data: dict) -> SourceRecord: + return SourceRecord( + id="note:1", source="example", record_type="example_note", summary="[NOTE-7] Plan", + content="", timestamp="2026-11-10T08:00:00Z", metadata=data, + ) + + +@pytest.mark.parametrize("case", RULE_VECTORS["cases"], ids=lambda case: case["name"]) +def test_the_rule_vectors_shared_with_the_gateway(case): + record = _data_record(RULE_VECTORS["data"]) + + assert InboxFilter(rules=data_rules(case["rules"])).passes(record) is case["keep"] + + +def test_the_values_at_a_path(): + record = _data_record(RULE_VECTORS["data"]) + + assert DataRule("/issue/labels", deny=("x",)).values(record) == ["bug", "Frontend", "7", "true", "null"] + assert DataRule("/issue/size", deny=("x",)).values(record) == ["2"] + assert DataRule("/issue/team", deny=("x",)).values(record) == [] + assert DataRule("/issue/missing", deny=("x",)).values(record) == [] + assert DataRule("not a pointer", deny=("x",)).values(record) == [] + + +def test_a_data_rule_without_patterns_is_inactive_and_an_invalid_path_names_nothing(): + record = _data_record(RULE_VECTORS["data"]) + + assert DataRule("/actor/kind").active is False + assert InboxFilter(rules=[DataRule("/actor/kind")]).active is False + assert DataRule("actor/kind", deny=("user",)).passes(record) is True + assert DataRule("actor/kind", allow=("user",)).passes(record) is False + + +def test_rejects_names_the_path_and_the_value_of_the_data_rule(): + record = _data_record(RULE_VECTORS["data"]) + flt = InboxFilter(rules=data_rules([ + {"path": "/issue/team/key", "allow": ["ENG*"]}, + {"path": "/actor/kind", "deny": ["user"]}, + ])) + + rule = flt.rejects(record) + + assert (rule.field, rule.value(record)) == ("/actor/kind", "user") + + +class _DataSource(_FakeSource): + source_name = "example" + + +@pytest.mark.asyncio +async def test_runner_drops_on_a_data_rule_and_logs_its_path_and_value(db, caplog): + kept = _data_record({"actor": {"kind": "user"}}) + bot = SourceRecord(**{**kept.__dict__, "id": "note:2", "metadata": {"actor": {"kind": "bot"}}}) + flt = InboxFilter(rules=data_rules([{"path": "/actor/kind", "deny": ["bot"]}])) + runner = SourceRunner(source=_DataSource([kept, bot]), db=db, inbox_filter=flt) + + with caplog.at_level("INFO", logger="nerve.sources.runner"): + result = await runner.run() + + assert (result.records_ingested, result.records_dropped) == (1, 1) + assert await db.get_source_message("example", "note:2") is None + assert "dropped note:2 on /actor/kind='bot'" in caplog.text From 0af5e9fcbb5216bdecca7d97d9580b7b7a432d4c Mon Sep 17 00:00:00 2001 From: Alex Soffronow Pagonidis <237136924+alex-clickhouse@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:46:57 +0000 Subject: [PATCH 02/10] Channel contract: generic source records and subscriptions Every provider outside the closed vocabularies (github, github_repos, email) is a generic source. Its records carry a schema and data, a JSON object that the provider defines, in place of metadata, and a subscription to it carries provider settings, the record schemas that Nerve reads, and rules on the data. Nerve must decode such records and send such subscriptions exactly as the gateway checks them, or the gateway closes the stream. The mirror keeps the data and the settings as parsed JSON with every member, refuses what the gateway refuses (repeated member names, NaN, lone surrogates, numbers outside binary64, more than 8 levels, data or settings above their canonical byte bounds), checks record types and schemas, and treats a subscription as generic only when it names a schema. A member with its zero value counts as absent, except settings and data. Conversation kind item names a tracked item. Byte-exact copies of the contract's synthetic vectors pin the canonical form, the bounds, the record and subscription verdicts, the generic subscriptions frame, and the canonical data in the digest projection. Co-Authored-By: Claude Opus 5.5 --- nerve/channels/hosted/contract/__init__.py | 2 + nerve/channels/hosted/contract/model.py | 199 ++- nerve/channels/hosted/contract/wire.py | 49 +- .../event_digest_generic.json | 7 + .../channel_contract/example_note.json | 43 + .../example_note.v1.schema.json | 47 + .../channel_contract/source_json.json | 186 +++ .../channel_contract/source_records.json | 491 +++++++ .../source_subscriptions.json | 1248 +++++++++++++++++ .../subscriptions_generic.json | 42 + tests/test_channel_contract_generic.py | 343 +++++ 11 files changed, 2649 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/channel_contract/event_digest_generic.json create mode 100644 tests/fixtures/channel_contract/example_note.json create mode 100644 tests/fixtures/channel_contract/example_note.v1.schema.json create mode 100644 tests/fixtures/channel_contract/source_json.json create mode 100644 tests/fixtures/channel_contract/source_records.json create mode 100644 tests/fixtures/channel_contract/source_subscriptions.json create mode 100644 tests/fixtures/channel_contract/subscriptions_generic.json create mode 100644 tests/test_channel_contract_generic.py diff --git a/nerve/channels/hosted/contract/__init__.py b/nerve/channels/hosted/contract/__init__.py index b5a36e569..c4c01b940 100644 --- a/nerve/channels/hosted/contract/__init__.py +++ b/nerve/channels/hosted/contract/__init__.py @@ -36,6 +36,7 @@ RepoWatch, SourceFilters, SourceRecordPayload, + SourceRule, SourceSubscription, StreamLimits, Subscriptions, @@ -79,6 +80,7 @@ "RepoWatch", "SourceFilters", "SourceRecordPayload", + "SourceRule", "SourceSubscription", "StreamLimits", "Subscriptions", diff --git a/nerve/channels/hosted/contract/model.py b/nerve/channels/hosted/contract/model.py index 73d856847..e5fe4c405 100644 --- a/nerve/channels/hosted/contract/model.py +++ b/nerve/channels/hosted/contract/model.py @@ -24,8 +24,10 @@ omit_empty, parse_json, reject, + source_json, struct, ) +from nerve.sources.source_json import SourceJSONError, check_source_json_object, pointer_tokens _record = dataclasses.dataclass(frozen=True) @@ -81,6 +83,29 @@ # A provider name is a lowercase token of at most this many bytes. MAX_PROVIDER_BYTES = 64 _PROVIDER_CHARACTERS = frozenset("abcdefghijklmnopqrstuvwxyz0123456789_-.") +# The provider of GitHub notifications and pull request activity. +GITHUB_PROVIDER = "github" +# The providers with a closed vocabulary: their records carry metadata from a +# closed set of keys, and their subscriptions have members of their own. +# Every other provider is a generic source: its records carry a schema and +# data that the provider defines, and its subscriptions carry settings, +# record schemas, and rules (see SourceSubscription). +CLOSED_PROVIDERS = frozenset({GITHUB_PROVIDER, REPO_WATCH_PROVIDER, EMAIL_PROVIDER}) +# The conversation kind of a tracked item, such as an issue. +CONVERSATION_KIND_ITEM = "item" +# Bounds of generic sources. The data of a record and the settings of a +# subscription count the UTF-8 bytes of their canonical form. +MAX_SOURCE_DATA_BYTES = 32 * 1024 +MAX_SOURCE_SETTINGS_BYTES = 8 * 1024 +MAX_RECORD_TYPE_BYTES = 64 +MAX_RECORD_SCHEMA_MAJOR_DIGITS = 6 +MAX_SUBSCRIPTION_SCHEMAS = 16 +MAX_SOURCE_RULES = 16 +MAX_SOURCE_RULE_PATTERNS = 64 +# The name of a generic record type after its provider and "_". +_RECORD_TYPE_NAME = re.compile(r"[a-z0-9_]+") +# The major version of a record schema: decimal, without a leading zero. +_MAJOR_VERSION = re.compile(rf"[1-9][0-9]{{0,{MAX_RECORD_SCHEMA_MAJOR_DIGITS - 1}}}") # How Nerve answers the turns of a connection's events (ConnectionBehaviour). ANSWER_POST = "post" ANSWER_TOOL = "tool" @@ -155,6 +180,54 @@ def valid_provider(provider: str) -> bool: ) +def generic_provider(provider: str) -> bool: + """Whether *provider* is a generic source (not in :data:`CLOSED_PROVIDERS`).""" + return provider not in CLOSED_PROVIDERS + + +def valid_record_type(provider: str, record_type: str) -> bool: + """Whether a generic *provider* can produce *record_type*. + + That is the provider, ``_``, and a name of lowercase ASCII letters, + digits and ``_``, at most :data:`MAX_RECORD_TYPE_BYTES` in total, such as + ``linear_issue_event``. + """ + prefix = f"{provider}_" + return ( + record_type.startswith(prefix) + and byte_length(record_type) <= MAX_RECORD_TYPE_BYTES + and _RECORD_TYPE_NAME.fullmatch(record_type[len(prefix):]) is not None + ) + + +def schema_record_type(provider: str, schema: str) -> str: + """The record type that a record schema of a generic *provider* names. + + A schema is the provider, ``.``, the record type's name, ``/``, and its + major version (up to :data:`MAX_RECORD_SCHEMA_MAJOR_DIGITS` digits, + without a leading zero): ``linear.issue_event/1`` names the record type + ``linear_issue_event``. Anything else is malformed. + """ + prefix = f"{provider}." + name, slash, major = schema[len(prefix):].partition("/") + record_type = f"{provider}_{name}" + if not ( + schema.startswith(prefix) and slash and generic_provider(provider) + and valid_record_type(provider, record_type) and _MAJOR_VERSION.fullmatch(major) + ): + malformed(f"record schema is not a schema of provider {provider!r}") + return record_type + + +def _validate_source_json(name: str, value: dict[str, Any], maximum_bytes: int) -> None: + """Check a source JSON object (the data of a record, the settings of a subscription).""" + try: + check_source_json_object(value, maximum_bytes) + except SourceJSONError as error: + reason = RejectionReason.LIMIT_EXCEEDED if error.limit else RejectionReason.MALFORMED_FRAME + reject(reason, f"{name} {error}") + + # ---------------------------------------------------------------------- # # References and content # # ---------------------------------------------------------------------- # @@ -363,22 +436,45 @@ class SourceRecordPayload: ``id`` is the source's own key for the record. A changed record keeps its ID, and Nerve stores it in place of the earlier copy. + + A record of a provider in :data:`CLOSED_PROVIDERS` carries ``metadata`` + from the closed vocabulary of its record type, and no ``schema`` or + ``data``. A record of a generic source carries ``schema`` and ``data`` + and no metadata: ``data`` is a JSON object that the provider defines and + that the schema describes. Nerve keeps every member of it, also the ones + that it does not know. """ id: str = "" record_type: str = "" + schema: str = omit_empty() summary: str = "" content: str = omit_empty() timestamp: datetime = ZERO_TIME metadata: tuple[MetadataEntry, ...] = omit_empty(()) + data: dict[str, Any] | None = source_json() - def validate(self) -> None: + def validate(self, provider: str) -> None: + """Check the record of a source of *provider*, as the gateway does.""" if not self.id or not self.record_type or not self.summary: malformed("source record has no ID, record type, or summary") if max(byte_length(self.id), byte_length(self.record_type)) > MAX_IDENTIFIER_BYTES: reject(RejectionReason.LIMIT_EXCEEDED, "source record ID or type exceeds the identifier limit") if max(byte_length(self.summary), byte_length(self.content)) > MAX_TEXT_BYTES: reject(RejectionReason.LIMIT_EXCEEDED, "source record text exceeds the text limit") + if not generic_provider(provider): + if self.schema or self.data is not None: + malformed(f"a {provider} source record carries a schema or data") + return + if not valid_record_type(provider, self.record_type): + reject(RejectionReason.KIND_UNSUPPORTED, f"record type is not valid for provider {provider!r}") + if schema_record_type(provider, self.schema) != self.record_type: + malformed("source record schema names another record type") + if self.metadata: + malformed("a generic source record carries metadata") + if self.data is None: + malformed("a generic source record has no data") + _validate_source_json("source record data", self.data, MAX_SOURCE_DATA_BYTES) @_record @@ -426,7 +522,7 @@ def validate(self) -> None: if self.kind == EVENT_KIND_SOURCE_RECORD: if self.record is None: malformed("source record event has no record") - self.record.validate() + self.record.validate(self.provider) # ---------------------------------------------------------------------- # @@ -858,6 +954,43 @@ class RepoWatch: deny_authors: tuple[str, ...] = () +@_record +class SourceRule: + """Keeps or drops the records of a generic source by a value of their data. + + ``path`` is a JSON Pointer into the data, such as ``/actor/kind``. A rule + has exactly one non-empty list of glob patterns, ``allow`` or ``deny``, + which match as the filter patterns do. A deny rule drops a record when + one of the values at its path matches; an allow rule drops a record when + none does, also when the record has no value there. The gateway applies + the rules before it stores a record, and Nerve's inbox guardrail applies + them again (:class:`nerve.sources.filters.DataRule`, which also defines + the values at a path). + """ + + path: str = "" + allow: tuple[str, ...] = omit_empty(()) + deny: tuple[str, ...] = omit_empty(()) + + def validate(self) -> None: + """Check the rule as the gateway does. + + The path is a bounded JSON Pointer (see + :func:`nerve.sources.source_json.pointer_tokens`), and the rule has + one list of 1 to :data:`MAX_SOURCE_RULE_PATTERNS` patterns that a + filter list takes. + """ + if pointer_tokens(self.path) is None: + malformed("subscription rule path is not a bounded JSON Pointer") + if bool(self.allow) == bool(self.deny): + malformed("a subscription rule needs exactly one of allow and deny") + patterns = self.deny or self.allow + if len(patterns) > MAX_SOURCE_RULE_PATTERNS: + reject(RejectionReason.LIMIT_EXCEEDED, "a subscription rule has too many patterns") + if not all(valid_pattern(pattern) for pattern in patterns): + malformed("a subscription rule pattern is empty, too long, or has a control character") + + @_record class SourceSubscription: """What the agent takes from one provider's hosted source. @@ -874,6 +1007,16 @@ class SourceSubscription: is switched on, so every other frame is unchanged. The ``email`` subscription carries only its provider and empty filters. + + A subscription of a generic source names its record ``schemas``: the + versions of the data that Nerve reads, at least one. For each record + type, the gateway sends the highest major version that ``schemas`` names + and that it produces. It then carries ``settings``, a JSON object that + the provider defines (absent settings are the provider's defaults), and + ``rules`` on the data of its records (see :class:`SourceRule`), and none + of the members above other than empty filters. A subscription of a + provider outside :data:`CLOSED_PROVIDERS` without schemas is not generic, + and carries no settings or rules either. """ provider: str = "" @@ -881,6 +1024,9 @@ class SourceSubscription: pr_watch: bool = omit_empty(False) inbox: str = omit_empty("") repos: tuple[RepoWatch, ...] = omit_empty(()) + settings: dict[str, Any] | None = source_json() + schemas: tuple[str, ...] = omit_empty(()) + rules: tuple[SourceRule, ...] = omit_empty(()) @_record @@ -900,8 +1046,9 @@ def validate(self) -> None: The version, the number of sources, each provider name and that no provider appears twice, each inbox, the bounds of every filter list, the watched repositories, which only ``github_repos`` names (see - :func:`_validate_repo_watches`), and that an ``email`` subscription - carries nothing but its provider. + :func:`_validate_repo_watches`), that an ``email`` subscription + carries nothing but its provider, and the members of a generic + source (see :func:`_validate_generic_members`). """ if self.version != SUBSCRIPTIONS_VERSION: reject(RejectionReason.VERSION_UNSUPPORTED, f"unknown subscriptions version {self.version!r}") @@ -930,24 +1077,66 @@ def validate(self) -> None: any(source.filters.pattern_lists().values()) or source.pr_watch or source.inbox ): malformed(f"an {EMAIL_PROVIDER} subscription carries filters, pr_watch or an inbox") + _validate_generic_members(source) def without_filters(self) -> Subscriptions: """The same sources with no filter patterns. The gateway then keeps every record of those sources, and Nerve's own inbox filter is the only one that drops records. A watched - repository's ``deny_authors`` are filter patterns too. + repository's ``deny_authors`` and the rules of a generic source are + filter patterns too. """ return dataclasses.replace(self, sources=tuple( dataclasses.replace( source, filters=SourceFilters(), repos=tuple(dataclasses.replace(repo, deny_authors=()) for repo in source.repos), + rules=(), ) for source in self.sources )) +def _validate_generic_members(source: SourceSubscription) -> None: + """Check ``settings``, ``schemas`` and ``rules`` as the gateway does. + + Their counts are bounded (:data:`MAX_SUBSCRIPTION_SCHEMAS`, + :data:`MAX_SOURCE_RULES`). A subscription is generic when its provider + is outside :data:`CLOSED_PROVIDERS` and it names at least one schema. + Then it has no filter pattern, ``pr_watch``, ``inbox`` or ``repos``; its + settings are a source JSON object within + :data:`MAX_SOURCE_SETTINGS_BYTES`; its schemas are distinct schemas of its + provider (see :func:`schema_record_type`); and its rules are valid. Any + other subscription names none of the three. A member with its zero value + counts as absent, except the settings, which count whenever they are + present. + """ + if len(source.schemas) > MAX_SUBSCRIPTION_SCHEMAS: + reject(RejectionReason.LIMIT_EXCEEDED, "a subscription names too many record schemas") + if len(source.rules) > MAX_SOURCE_RULES: + reject(RejectionReason.LIMIT_EXCEEDED, "a subscription has too many rules") + if not (generic_provider(source.provider) and source.schemas): + if source.settings is not None or source.schemas or source.rules: + malformed( + "a subscription of a closed provider, or without record schemas, names " + "settings, record schemas, or rules" + ) + return + if any(source.filters.pattern_lists().values()) or source.pr_watch or source.inbox or source.repos: + malformed("a generic subscription names filters, a pull request watch, an inbox, or repositories") + if source.settings is not None: + _validate_source_json("subscription settings", source.settings, MAX_SOURCE_SETTINGS_BYTES) + seen: set[str] = set() + for schema in source.schemas: + schema_record_type(source.provider, schema) + if schema in seen: + malformed("subscription schemas repeat a schema") + seen.add(schema) + for rule in source.rules: + rule.validate() + + def _validate_patterns(patterns: tuple[str, ...], what: str) -> None: """The bounds of one pattern list: its length and each pattern.""" if len(patterns) > MAX_SUBSCRIPTION_PATTERNS: diff --git a/nerve/channels/hosted/contract/wire.py b/nerve/channels/hosted/contract/wire.py index 24f7674f2..18174ff98 100644 --- a/nerve/channels/hosted/contract/wire.py +++ b/nerve/channels/hosted/contract/wire.py @@ -6,6 +6,14 @@ members without a change in Nerve. A missing or null member takes the field's default. Encoding leaves out a field marked with :func:`omit_empty` when it holds its zero value. + +A field marked with :func:`source_json` holds a JSON object that a provider +defines, such as the data of a generic source record. Decoding keeps it as +the parsed object, with every member, and tells an absent member (``None``) +from a present one: a present ``null`` or another JSON type is malformed. + +The parser refuses what the gateway's parser refuses: a JSON object that +repeats a member name, and ``NaN`` or ``Infinity``. """ from __future__ import annotations @@ -63,6 +71,15 @@ def struct(factory: type) -> Any: return dataclasses.field(default_factory=factory) +def source_json() -> Any: + """A JSON object that a provider defines: ``None`` when the member is absent. + + It is left out of the JSON when it is ``None``. Any other value, an empty + object too, is sent. + """ + return dataclasses.field(default=None, metadata={"source_json": True}) + + def format_time(value: datetime) -> str: """An RFC 3339 UTC timestamp, as the contract sends it.""" utc = value.astimezone(timezone.utc) @@ -73,15 +90,30 @@ def format_time(value: datetime) -> str: def parse_json(text: str) -> Any: - """Parse one JSON value, or raise :class:`Rejected`.""" + """Parse one JSON value, or raise :class:`Rejected`. + + An object that repeats a member name, and the literals ``NaN``, + ``Infinity`` and ``-Infinity``, are malformed, as for the gateway. + """ try: - return json.loads(text) + return json.loads(text, object_pairs_hook=_unique_members, parse_constant=_not_json) except RecursionError: reject(RejectionReason.LIMIT_EXCEEDED, "JSON nesting exceeds the parser's depth") except ValueError as error: malformed(f"decode frame: {error}") +def _unique_members(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + members = dict(pairs) + if len(members) != len(pairs): + malformed("decode frame: a JSON object repeats a member name") + return members + + +def _not_json(literal: str) -> typing.NoReturn: + malformed(f"decode frame: {literal} is not a JSON value") + + _HINTS: dict[type, dict[str, Any]] = {} @@ -96,9 +128,15 @@ def decode_record(cls: type, raw: Any, path: str = "") -> Any: hints = _HINTS[cls] = typing.get_type_hints(cls) values: dict[str, Any] = {} for field in dataclasses.fields(cls): + member_path = f"{path}.{field.name}" if path else field.name + if field.metadata.get("source_json"): + if field.name in raw: + if not isinstance(raw[field.name], dict): + malformed(f"decode frame: {member_path} is not a JSON object") + values[field.name] = raw[field.name] + continue value = raw.get(field.name) if value is not None: - member_path = f"{path}.{field.name}" if path else field.name values[field.name] = _decode_value(hints[field.name], value, member_path) return cls(**values) @@ -149,6 +187,10 @@ def encode_record(record: Any) -> dict[str, Any]: value = getattr(record, field.name) if field.metadata.get("omitempty") and value in (None, False, 0, "", ()): continue + if field.metadata.get("source_json"): + if value is not None: + out[field.name] = value + continue out[field.name] = _encode_value(value) return out @@ -183,5 +225,6 @@ def dumps(record: Any) -> str: "omit_empty", "parse_json", "reject", + "source_json", "struct", ] diff --git a/tests/fixtures/channel_contract/event_digest_generic.json b/tests/fixtures/channel_contract/event_digest_generic.json new file mode 100644 index 000000000..4c1a4e282 --- /dev/null +++ b/tests/fixtures/channel_contract/event_digest_generic.json @@ -0,0 +1,7 @@ +{ + "schema": "nerve.channel-event-digest/2", + "algorithm": "sha256", + "event_fixture": "events/example_note.json", + "normalized_base64": "AAAAHG5lcnZlLmNoYW5uZWwtZXZlbnQtZGlnZXN0LzJvHTosC05PepwdLluKP3wEsoxekX1KTDuPYQqeLUtsFzyafhUrTU9gjhpdfJsvSmgAAAAHZXhhbXBsZQAAAAAAAAAAAAAAXWV4YW1wbGU6M2M5YTdlMTUtMmI0ZC00ZjYwLThlMWEtNWQ3YzliMmY0YTY4OmNvbW1lbnRlZDo5ZTJmNmE0Yi0xYzNkLTRlNWYtYTdiOC1jOWQwZTFmMmEzYjQ6MQAAADBjb21tZW50ZWQ6OWUyZjZhNGItMWMzZC00ZTVmLWE3YjgtYzlkMGUxZjJhM2I0OjEAAAANc291cmNlX3JlY29yZAAAACQ1ZjBjMWI5ZS0zYzJhLTRkN2UtOWIxZi04YTZkMmU0YzdiMzAAAAAEaXRlbQAAAAZOT1RFLTcAAAAAAQAAACQwYjdkNGMyYS02ZTFmLTRhM2ItOWM4ZC0yZjVlN2ExYjNjNDkAAAAFaHVtYW4AAAANQWxpY2UgRXhhbXBsZQAAAAAAavLPAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAACxjb21tZW50OjllMmY2YTRiLTFjM2QtNGU1Zi1hN2I4LWM5ZDBlMWYyYTNiNAAAAAxleGFtcGxlX25vdGUAAAAyW05PVEUtN10gQWxpY2UgRXhhbXBsZSBjb21tZW50ZWQ6IFBsYW4gdGhlIHJlbGVhc2UAAACDKipBbGljZSBFeGFtcGxlKiogY29tbWVudGVkIG9uIFtOT1RFLTddKGh0dHBzOi8vbm90ZXMuZXhhbXBsZS50ZXN0L05PVEUtNykgUGxhbiB0aGUgcmVsZWFzZToKCj4gU2hpcCBpdCBhZnRlciB0aGUgZnJlZXplLgo+IFRoYW5rcyEAAAAAavLPAAAAAAAAAAAAAAAADmV4YW1wbGUubm90ZS8xAAABl3siYWN0b3IiOnsiaWQiOiIwYjdkNGMyYS02ZTFmLTRhM2ItOWM4ZC0yZjVlN2ExYjNjNDkiLCJraW5kIjoidXNlciIsIm5hbWUiOiJBbGljZSBFeGFtcGxlIn0sImNvbW1lbnQiOnsiYm9keSI6IlNoaXAgaXQgYWZ0ZXIgdGhlIGZyZWV6ZS5cblRoYW5rcyEiLCJpZCI6IjllMmY2YTRiLTFjM2QtNGU1Zi1hN2I4LWM5ZDBlMWYyYTNiNCJ9LCJldmVudCI6ImNvbW1lbnRlZCIsIm5vdGUiOnsiaWQiOiI1ZjBjMWI5ZS0zYzJhLTRkN2UtOWIxZi04YTZkMmU0YzdiMzAiLCJrZXkiOiJOT1RFLTciLCJsYWJlbHMiOlsicGxhbm5pbmciLCJRNCJdLCJwcmlvcml0eSI6MiwidGl0bGUiOiJQbGFuIHRoZSByZWxlYXNlIMOpdMOpIiwidXJsIjoiaHR0cHM6Ly9ub3Rlcy5leGFtcGxlLnRlc3QvTk9URS03In0sInNjb3JlIjoyLjV9", + "digest": "sha256:f9a5241f1ca1662471fbfab48ff081c482dbac8e1be4bb3cd8af81bc656cdc41" +} diff --git a/tests/fixtures/channel_contract/example_note.json b/tests/fixtures/channel_contract/example_note.json new file mode 100644 index 000000000..6d3cfa554 --- /dev/null +++ b/tests/fixtures/channel_contract/example_note.json @@ -0,0 +1,43 @@ +{ + "event_id": "example:3c9a7e15-2b4d-4f60-8e1a-5d7c9b2f4a68:commented:9e2f6a4b-1c3d-4e5f-a7b8-c9d0e1f2a3b4:1", + "message_revision_id": "commented:9e2f6a4b-1c3d-4e5f-a7b8-c9d0e1f2a3b4:1", + "tenant_id": "6f1d3a2c-0b4e-4f7a-9c1d-2e5b8a3f7c04", + "agent_id": "b28c5e91-7d4a-4c3b-8f61-0a9e2d4b6c17", + "connection_id": "3c9a7e15-2b4d-4f60-8e1a-5d7c9b2f4a68", + "provider": "example", + "kind": "source_record", + "conversation": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "kind": "item", + "display_name": "NOTE-7" + }, + "author": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "human", + "display_name": "Alice Example" + }, + "admission": { + "purpose": "observe", + "observe": { + "provenance": { + "provider": "example", + "issuer": "", + "installation_id": "" + } + } + }, + "delivery": { + "inbox_id": "1020", + "received_at": "2026-11-10T08:00:12.5Z" + }, + "occurred_at": "2026-11-10T08:00:00Z", + "record": { + "id": "comment:9e2f6a4b-1c3d-4e5f-a7b8-c9d0e1f2a3b4", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] Alice Example commented: Plan the release", + "content": "**Alice Example** commented on [NOTE-7](https://notes.example.test/NOTE-7) Plan the release:\n\n> Ship it after the freeze.\n> Thanks!", + "timestamp": "2026-11-10T08:00:00Z", + "data": {"note":{"title":"Plan the release \u00e9t\u00e9","key":"NOTE-7","id":"5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30","priority":2,"url":"https://notes.example.test/NOTE-7","labels":["planning","Q4"]},"event":"commented","actor":{"kind":"user","name":"Alice Example","id":"0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49"},"comment":{"id":"9e2f6a4b-1c3d-4e5f-a7b8-c9d0e1f2a3b4","body":"Ship it after the freeze.\nThanks!"},"score":2.50} + } +} diff --git a/tests/fixtures/channel_contract/example_note.v1.schema.json b/tests/fixtures/channel_contract/example_note.v1.schema.json new file mode 100644 index 000000000..cfc7dba64 --- /dev/null +++ b/tests/fixtures/channel_contract/example_note.v1.schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "example.note/1", + "description": "Test-only data of the record type example_note of the provider example. No gateway produces it.", + "type": "object", + "additionalProperties": false, + "required": ["event", "note", "actor"], + "properties": { + "event": {"type": "string", "enum": ["created", "commented"]}, + "note": { + "type": "object", + "additionalProperties": false, + "required": ["id", "key", "title"], + "properties": { + "id": {"type": "string"}, + "key": {"type": "string", "maxLength": 32}, + "title": {"type": "string", "maxLength": 256}, + "url": {"type": "string"}, + "priority": {"type": "integer", "minimum": 0, "maximum": 4}, + "labels": {"type": "array", "maxItems": 16, "items": {"type": "string"}} + } + }, + "actor": {"$ref": "#/$defs/person"}, + "comment": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["id", "body"], + "properties": { + "id": {"type": "string"}, + "body": {"type": "string", "maxLength": 4000} + } + }, + "score": {"type": "number"} + }, + "$defs": { + "person": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind"], + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "kind": {"type": "string", "enum": ["user", "bot"]} + } + } + } +} diff --git a/tests/fixtures/channel_contract/source_json.json b/tests/fixtures/channel_contract/source_json.json new file mode 100644 index 000000000..b851280f3 --- /dev/null +++ b/tests/fixtures/channel_contract/source_json.json @@ -0,0 +1,186 @@ +{ + "canonical": [ + { + "name": "white space and member order", + "json": "{ \"b\" : 2 , \"a\" : [ 1 , { \"d\" : true , \"c\" : null } ] }", + "canonical": "{\"a\":[1,{\"c\":null,\"d\":true}],\"b\":2}" + }, + { + "name": "the example of RFC 8785 section 3.2.2", + "json": "{\n \"numbers\": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001],\n \"string\": \"\\u20ac$\\u000F\\u000aA'\\u0042\\u0022\\u005c\\\\\\\"\\/\",\n \"literals\": [null, true, false]\n}", + "canonical": "{\"literals\":[null,true,false],\"numbers\":[333333333.3333333,1e+30,4.5,0.002,1e-27],\"string\":\"\u20ac$\\u000f\\nA'B\\\"\\\\\\\\\\\"/\"}" + }, + { + "name": "member names sorted by UTF-16 code units (RFC 8785 section 3.2.3)", + "json": "{\"\\u20ac\":\"Euro Sign\",\"\\r\":\"Carriage Return\",\"\\ufb33\":\"Hebrew Letter Dalet With Dagesh\",\"1\":\"One\",\"\\ud83d\\ude00\":\"Emoji: Grinning Face\",\"\\u0080\":\"Control\",\"\\u00f6\":\"Latin Small Letter O With Diaeresis\"}", + "canonical": "{\"\\r\":\"Carriage Return\",\"1\":\"One\",\"\u0080\":\"Control\",\"\u00f6\":\"Latin Small Letter O With Diaeresis\",\"\u20ac\":\"Euro Sign\",\"\ud83d\ude00\":\"Emoji: Grinning Face\",\"\ufb33\":\"Hebrew Letter Dalet With Dagesh\"}" + }, + { + "name": "characters that need no escape stay as they are", + "json": "{\"a\\/b\":\"<>&\\u007f\\u2028\\u2029\\/\\u00e9\"}", + "canonical": "{\"a/b\":\"<>&\u007f\u2028\u2029/\u00e9\"}" + }, + { + "name": "control characters use the short escapes or lowercase hexadecimal", + "json": "{\"a\":\"\\u0000\\u0001\\u001F\\u0008\\u000c\\u000A\\u000d\\u0009\"}", + "canonical": "{\"a\":\"\\u0000\\u0001\\u001f\\b\\f\\n\\r\\t\"}" + }, + { + "name": "numbers in their shortest ECMAScript form", + "json": "[0, -0, -0.0, 1.0, 100e-2, 1E2, 1e21, 1e-7, 0.000001, 1e-6, 0.1, 1.5e300, 123456789012345678901234567890, 12345678901234567890, 9007199254740993, -9007199254740992, 5e-324, -5e-324, 1e-400, -1e-400, 1.7976931348623157e308, -1.7976931348623157e308, 295147905179352830000, 9.999999999999997e+22, 1e23, 1.0000000000000001e+23, 999999999999999700000, 999999999999999900000, 999999999999999999999, 9.999999999999997e-7, 333333333.3333332, 333333333.33333325, 333333333.3333333, 333333333.3333334, 333333333.33333343, -0.0000033333333333333333, 1424953923781206.2]", + "canonical": "[0,0,0,1,1,100,1e+21,1e-7,0.000001,0.000001,0.1,1.5e+300,1.2345678901234568e+29,12345678901234567000,9007199254740992,-9007199254740992,5e-324,-5e-324,0,0,1.7976931348623157e+308,-1.7976931348623157e+308,295147905179352830000,9.999999999999997e+22,1e+23,1.0000000000000001e+23,999999999999999700000,999999999999999900000,1e+21,9.999999999999997e-7,333333333.3333332,333333333.33333325,333333333.3333333,333333333.3333334,333333333.33333343,-0.0000033333333333333333,1424953923781206.2]" + }, + { + "name": "empty containers and strings", + "json": "{\"c\":\"\",\"b\":[],\"a\":{}}", + "canonical": "{\"a\":{},\"b\":[],\"c\":\"\"}" + }, + { + "name": "eight levels of objects and arrays", + "json": "{\"a\":[[[[[[[1]]]]]]]}", + "canonical": "{\"a\":[[[[[[[1]]]]]]]}" + } + ], + "rejected": [ + { + "name": "an array", + "json": "[]", + "reason": "malformed_frame" + }, + { + "name": "a string", + "json": "\"text\"", + "reason": "malformed_frame" + }, + { + "name": "null", + "json": "null", + "reason": "malformed_frame" + }, + { + "name": "invalid JSON", + "json": "{\"a\":}", + "reason": "malformed_frame" + }, + { + "name": "trailing data", + "json": "{} {}", + "reason": "malformed_frame" + }, + { + "name": "a repeated member name", + "json": "{\"a\":1,\"a\":1}", + "reason": "malformed_frame" + }, + { + "name": "a repeated member name in a nested object", + "json": "{\"a\":[{\"b\":1,\"b\":2}]}", + "reason": "malformed_frame" + }, + { + "name": "an unpaired high surrogate escape", + "json": "{\"a\":\"\\ud800\"}", + "reason": "malformed_frame" + }, + { + "name": "an unpaired low surrogate escape", + "json": "{\"a\":\"\\udc00x\"}", + "reason": "malformed_frame" + }, + { + "name": "a number outside binary64", + "json": "{\"a\":-1e309}", + "reason": "malformed_frame" + }, + { + "name": "NaN", + "json": "{\"a\":NaN}", + "reason": "malformed_frame" + }, + { + "name": "Infinity", + "json": "{\"a\":Infinity}", + "reason": "malformed_frame" + }, + { + "name": "a leading zero", + "json": "{\"a\":01}", + "reason": "malformed_frame" + }, + { + "name": "nine levels of arrays", + "json": "{\"a\":[[[[[[[[1]]]]]]]]}", + "reason": "limit_exceeded" + }, + { + "name": "nine levels of objects", + "json": "{\"a\":{\"b\":{\"c\":{\"d\":{\"e\":{\"f\":{\"g\":{\"h\":{}}}}}}}}}", + "reason": "limit_exceeded" + } + ], + "bounds": [ + { + "name": "the largest data", + "object": "data", + "prefix": "{\"a\":\"", + "repeat": "x", + "count": 32760, + "suffix": "\"}", + "reason": "" + }, + { + "name": "data one byte too large", + "object": "data", + "prefix": "{\"a\":\"", + "repeat": "x", + "count": 32761, + "suffix": "\"}", + "reason": "limit_exceeded" + }, + { + "name": "white space does not count", + "object": "data", + "prefix": "{ \"a\" : \"", + "repeat": "x", + "count": 32760, + "suffix": "\" }", + "reason": "" + }, + { + "name": "an escape counts as the character that it writes", + "object": "data", + "prefix": "{\"a\":\"", + "repeat": "\\u0078", + "count": 32760, + "suffix": "\"}", + "reason": "" + }, + { + "name": "a two-byte character counts two bytes", + "object": "data", + "prefix": "{\"a\":\"x", + "repeat": "\u00e9", + "count": 16380, + "suffix": "\"}", + "reason": "limit_exceeded" + }, + { + "name": "the largest settings", + "object": "settings", + "prefix": "{\"a\":\"", + "repeat": "x", + "count": 8184, + "suffix": "\"}", + "reason": "" + }, + { + "name": "settings one byte too large", + "object": "settings", + "prefix": "{\"a\":\"", + "repeat": "x", + "count": 8185, + "suffix": "\"}", + "reason": "limit_exceeded" + } + ] +} diff --git a/tests/fixtures/channel_contract/source_records.json b/tests/fixtures/channel_contract/source_records.json new file mode 100644 index 000000000..000b84d2f --- /dev/null +++ b/tests/fixtures/channel_contract/source_records.json @@ -0,0 +1,491 @@ +{ + "records": [ + { + "name": "a generic record", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "" + }, + { + "name": "the largest major version", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/999999", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "" + }, + { + "name": "a record type of another provider", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "other_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "kind_unsupported" + }, + { + "name": "a record type without the provider", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "kind_unsupported" + }, + { + "name": "an upper-case record type", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_Note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "kind_unsupported" + }, + { + "name": "a record type of more than 64 bytes", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn", + "schema": "example.nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "kind_unsupported" + }, + { + "name": "no schema", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "malformed_frame" + }, + { + "name": "a schema of another record type", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.task/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "malformed_frame" + }, + { + "name": "a schema of another provider", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "other.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "malformed_frame" + }, + { + "name": "a schema without a major version", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "malformed_frame" + }, + { + "name": "empty metadata", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + }, + "metadata": [] + }, + "reason": "" + }, + { + "name": "metadata", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + }, + "metadata": [ + { + "key": "reason", + "value": "mention" + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "no data", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z" + }, + "reason": "malformed_frame" + }, + { + "name": "null data", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": null + }, + "reason": "malformed_frame" + }, + { + "name": "data that is not an object", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": [ + { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "data of nine levels", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "a": { + "b": { + "c": { + "d": { + "e": { + "f": { + "g": { + "h": {} + } + } + } + } + } + } + } + } + }, + "reason": "limit_exceeded" + }, + { + "name": "an empty summary", + "provider": "example", + "record": { + "id": "note:5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "record_type": "example_note", + "schema": "example.note/1", + "summary": "", + "timestamp": "2026-11-10T08:00:00Z", + "data": { + "event": "created", + "note": { + "id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", + "key": "NOTE-7", + "title": "Plan" + }, + "actor": { + "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49", + "kind": "user" + } + } + }, + "reason": "malformed_frame" + }, + { + "name": "a github record", + "provider": "github", + "record": { + "id": "9000000001", + "record_type": "github_notification", + "summary": "[owner/repo] Fix", + "timestamp": "2026-11-10T08:00:00Z", + "metadata": [ + { + "key": "reason", + "value": "mention" + } + ] + }, + "reason": "" + }, + { + "name": "a github record with a schema", + "provider": "github", + "record": { + "id": "9000000001", + "record_type": "github_notification", + "summary": "[owner/repo] Fix", + "timestamp": "2026-11-10T08:00:00Z", + "metadata": [ + { + "key": "reason", + "value": "mention" + } + ], + "schema": "github.notification/1" + }, + "reason": "malformed_frame" + }, + { + "name": "a github record with data", + "provider": "github", + "record": { + "id": "9000000001", + "record_type": "github_notification", + "summary": "[owner/repo] Fix", + "timestamp": "2026-11-10T08:00:00Z", + "metadata": [ + { + "key": "reason", + "value": "mention" + } + ], + "data": {} + }, + "reason": "malformed_frame" + }, + { + "name": "a github record with an empty schema", + "provider": "github", + "record": { + "id": "9000000001", + "record_type": "github_notification", + "summary": "[owner/repo] Fix", + "timestamp": "2026-11-10T08:00:00Z", + "metadata": [ + { + "key": "reason", + "value": "mention" + } + ], + "schema": "" + }, + "reason": "" + }, + { + "name": "a github record with null data", + "provider": "github", + "record": { + "id": "9000000001", + "record_type": "github_notification", + "summary": "[owner/repo] Fix", + "timestamp": "2026-11-10T08:00:00Z", + "metadata": [ + { + "key": "reason", + "value": "mention" + } + ], + "data": null + }, + "reason": "malformed_frame" + } + ] +} diff --git a/tests/fixtures/channel_contract/source_subscriptions.json b/tests/fixtures/channel_contract/source_subscriptions.json new file mode 100644 index 000000000..644e634c8 --- /dev/null +++ b/tests/fixtures/channel_contract/source_subscriptions.json @@ -0,0 +1,1248 @@ +{ + "sources": [ + { + "name": "the fewest members", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ] + }, + "reason": "" + }, + { + "name": "an empty filters object, as a sender encodes one", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "filters": {} + }, + "reason": "" + }, + { + "name": "settings, two major versions, and rules", + "source": { + "provider": "example", + "schemas": [ + "example.note/1", + "example.note/2", + "example.task/1" + ], + "settings": { + "events": [ + "created" + ], + "inbox": false, + "nested": { + "list": [ + 1, + 2.5, + null + ] + } + }, + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/note/key", + "allow": [ + "NOTE-*" + ] + } + ] + }, + "reason": "" + }, + { + "name": "empty settings", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "settings": {} + }, + "reason": "" + }, + { + "name": "the most schemas", + "source": { + "provider": "example", + "schemas": [ + "example.note/1", + "example.note/2", + "example.note/3", + "example.note/4", + "example.note/5", + "example.note/6", + "example.note/7", + "example.note/8", + "example.note/9", + "example.note/10", + "example.note/11", + "example.note/12", + "example.note/13", + "example.note/14", + "example.note/15", + "example.note/16" + ] + }, + "reason": "" + }, + { + "name": "the most rules", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "" + }, + { + "name": "the most patterns", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p10", + "p11", + "p12", + "p13", + "p14", + "p15", + "p16", + "p17", + "p18", + "p19", + "p20", + "p21", + "p22", + "p23", + "p24", + "p25", + "p26", + "p27", + "p28", + "p29", + "p30", + "p31", + "p32", + "p33", + "p34", + "p35", + "p36", + "p37", + "p38", + "p39", + "p40", + "p41", + "p42", + "p43", + "p44", + "p45", + "p46", + "p47", + "p48", + "p49", + "p50", + "p51", + "p52", + "p53", + "p54", + "p55", + "p56", + "p57", + "p58", + "p59", + "p60", + "p61", + "p62", + "p63" + ] + } + ] + }, + "reason": "" + }, + { + "name": "the deepest path", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/a/b/c/d/e/f/g/h", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "" + }, + { + "name": "zero values of the GitHub members on a generic source", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "filters": { + "deny_actors": [] + }, + "pr_watch": false, + "inbox": "", + "repos": [] + }, + "reason": "" + }, + { + "name": "empty schemas and rules on github", + "source": { + "provider": "github", + "schemas": [], + "rules": [] + }, + "reason": "" + }, + { + "name": "null schemas on github", + "source": { + "provider": "github", + "schemas": null + }, + "reason": "" + }, + { + "name": "settings on github", + "source": { + "provider": "github", + "settings": {} + }, + "reason": "malformed_frame" + }, + { + "name": "schemas on github", + "source": { + "provider": "github", + "schemas": [ + "github.notification/1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "rules on github", + "source": { + "provider": "github", + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "settings on github_repos", + "source": { + "provider": "github_repos", + "repos": [ + { + "name": "acme/widgets", + "types": [ + "issue" + ] + } + ], + "settings": {} + }, + "reason": "malformed_frame" + }, + { + "name": "a filter pattern on a generic provider", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "filters": { + "deny_actors": [ + "bot" + ] + } + }, + "reason": "malformed_frame" + }, + { + "name": "a pull request watch on a generic provider", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "pr_watch": true + }, + "reason": "malformed_frame" + }, + { + "name": "an inbox on a generic provider", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "inbox": "all" + }, + "reason": "malformed_frame" + }, + { + "name": "repositories on a generic provider", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "repos": [ + { + "name": "acme/widgets", + "types": [ + "issue" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a provider without schemas, which is not generic", + "source": { + "provider": "example" + }, + "reason": "" + }, + { + "name": "an empty schema list, which is no schema", + "source": { + "provider": "example", + "schemas": [] + }, + "reason": "" + }, + { + "name": "an email subscription as older Nerve releases send it", + "source": { + "provider": "email", + "filters": {} + }, + "reason": "" + }, + { + "name": "GitHub members on a provider without schemas", + "source": { + "provider": "gitlab", + "filters": { + "deny_actors": [ + "bot" + ] + }, + "pr_watch": true, + "inbox": "all", + "repos": [ + { + "name": "acme/widgets", + "types": [ + "issue" + ] + } + ] + }, + "reason": "" + }, + { + "name": "invalid repositories on a provider without schemas", + "source": { + "provider": "gitlab", + "repos": [ + { + "name": "group/sub/project", + "types": [ + "issue" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "settings without schemas", + "source": { + "provider": "example", + "settings": { + "inbox": true + } + }, + "reason": "malformed_frame" + }, + { + "name": "empty settings without schemas", + "source": { + "provider": "example", + "settings": {} + }, + "reason": "malformed_frame" + }, + { + "name": "rules without schemas", + "source": { + "provider": "example", + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "too many rules without schemas", + "source": { + "provider": "example", + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "limit_exceeded" + }, + { + "name": "too many schemas", + "source": { + "provider": "example", + "schemas": [ + "example.note/1", + "example.note/2", + "example.note/3", + "example.note/4", + "example.note/5", + "example.note/6", + "example.note/7", + "example.note/8", + "example.note/9", + "example.note/10", + "example.note/11", + "example.note/12", + "example.note/13", + "example.note/14", + "example.note/15", + "example.note/16", + "example.note/17" + ] + }, + "reason": "limit_exceeded" + }, + { + "name": "a repeated schema", + "source": { + "provider": "example", + "schemas": [ + "example.note/1", + "example.note/1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a schema of another provider", + "source": { + "provider": "example", + "schemas": [ + "other.note/1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a schema without a major version", + "source": { + "provider": "example", + "schemas": [ + "example.note" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "major version 0", + "source": { + "provider": "example", + "schemas": [ + "example.note/0" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a major version with a leading zero", + "source": { + "provider": "example", + "schemas": [ + "example.note/01" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a major version of seven digits", + "source": { + "provider": "example", + "schemas": [ + "example.note/1000000" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a minor version", + "source": { + "provider": "example", + "schemas": [ + "example.note/1.0" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "an upper-case record type", + "source": { + "provider": "example", + "schemas": [ + "example.Note/1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a record type with a hyphen", + "source": { + "provider": "example", + "schemas": [ + "example.big-note/1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "an empty record type", + "source": { + "provider": "example", + "schemas": [ + "example./1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a record type of more than 64 bytes", + "source": { + "provider": "example", + "schemas": [ + "example.nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn/1" + ] + }, + "reason": "malformed_frame" + }, + { + "name": "settings that are not an object", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "settings": [ + 1 + ] + }, + "reason": "malformed_frame" + }, + { + "name": "null settings", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "settings": null + }, + "reason": "malformed_frame" + }, + { + "name": "settings of nine levels", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "settings": { + "a": [ + [ + [ + [ + [ + [ + [ + [ + 1 + ] + ] + ] + ] + ] + ] + ] + ] + } + }, + "reason": "limit_exceeded" + }, + { + "name": "too many rules", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "limit_exceeded" + }, + { + "name": "too many patterns", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p10", + "p11", + "p12", + "p13", + "p14", + "p15", + "p16", + "p17", + "p18", + "p19", + "p20", + "p21", + "p22", + "p23", + "p24", + "p25", + "p26", + "p27", + "p28", + "p29", + "p30", + "p31", + "p32", + "p33", + "p34", + "p35", + "p36", + "p37", + "p38", + "p39", + "p40", + "p41", + "p42", + "p43", + "p44", + "p45", + "p46", + "p47", + "p48", + "p49", + "p50", + "p51", + "p52", + "p53", + "p54", + "p55", + "p56", + "p57", + "p58", + "p59", + "p60", + "p61", + "p62", + "p63", + "p64" + ] + } + ] + }, + "reason": "limit_exceeded" + }, + { + "name": "a rule with allow and deny", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "allow": [ + "a" + ], + "deny": [ + "b" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a rule without a list", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind" + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a rule with an empty list", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "allow": [] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a pattern with a control character", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot\u0007" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a pattern of 257 bytes", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a path without a leading slash", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "actor/kind", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "an empty path", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a path with the escape ~2", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/a~2b", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a path that ends with ~", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/a~", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a path of nine tokens", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/a/b/c/d/e/f/g/h/i", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a path with a control character", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/a\u0001", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + }, + { + "name": "a path of 257 bytes", + "source": { + "provider": "example", + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp", + "deny": [ + "bot" + ] + } + ] + }, + "reason": "malformed_frame" + } + ] +} diff --git a/tests/fixtures/channel_contract/subscriptions_generic.json b/tests/fixtures/channel_contract/subscriptions_generic.json new file mode 100644 index 000000000..c7e7dc627 --- /dev/null +++ b/tests/fixtures/channel_contract/subscriptions_generic.json @@ -0,0 +1,42 @@ +{ + "version": "1", + "kind": "subscriptions", + "request_id": "n-26", + "payload": { + "subscriptions": { + "version": "1", + "sources": [ + { + "provider": "github", + "filters": { + "deny_actors": [ + "*-bot" + ] + } + }, + { + "provider": "example", + "filters": {}, + "settings": {"events":["created","commented"],"inbox":true}, + "schemas": [ + "example.note/1" + ], + "rules": [ + { + "path": "/actor/kind", + "deny": [ + "bot" + ] + }, + { + "path": "/note/key", + "allow": [ + "NOTE-*" + ] + } + ] + } + ] + } + } +} diff --git a/tests/test_channel_contract_generic.py b/tests/test_channel_contract_generic.py new file mode 100644 index 000000000..bc3435f01 --- /dev/null +++ b/tests/test_channel_contract_generic.py @@ -0,0 +1,343 @@ +"""The generic sources of the channel contract, against the contract's vectors. + +A provider outside the closed vocabularies (github, github_repos, email) is a +generic source: its records carry a ``schema`` and ``data``, a JSON object +that the provider defines, and a subscription to it carries ``settings``, +``schemas`` and ``rules``. ``fixtures/channel_contract`` holds byte-exact +copies of the contract's synthetic vectors (testdata/source_json.json, +source_records.json, source_subscriptions.json, event_digest_generic.json, +events/example_note.json, frames/nerve/subscriptions_generic.json, and the +test-only schema sources/example/example_note.v1.schema.json). The rule +vectors (source_rules.json) run against the inbox guardrail in +test_source_filters. +""" + +from __future__ import annotations + +import base64 +import copy +import hashlib +import json +import struct +import uuid +from pathlib import Path +from typing import Any + +import jsonschema +import pytest + +from nerve.channels.hosted.contract import ( + Envelope, + Payload, + Rejected, + SourceFilters, + SourceRule, + SourceSubscription, + Subscriptions, + decode_envelope, + encode_envelope, +) +from nerve.channels.hosted.contract.model import ( + CLOSED_PROVIDERS, + CONVERSATION_KIND_ITEM, + MAX_SOURCE_DATA_BYTES, + MAX_SOURCE_SETTINGS_BYTES, + schema_record_type, +) +from nerve.channels.hosted.contract.wire import decode_record, dumps, parse_json +from nerve.sources.source_json import canonical_json + +FIXTURES = Path(__file__).parent / "fixtures" / "channel_contract" + +# The SHA-256 of the contract's vectors, which the copies must match byte for +# byte (the contract's testdata, under the names above). +VECTOR_SHA256 = { + "source_json.json": "43651bdc7631b88573030d048e7ae9b94f50eb60cacf4d67c19e1a5d5c3f8c2f", + "source_records.json": "67029d8b3301200215176cd1b0b7fc59f46e62cc7b0be5a6e8c39fe8441d349c", + "source_rules.json": "e49acca284612c0d270b896d6aa5f5a8c452082a7ca31c536e6b0e4bf414341d", + "source_subscriptions.json": "c68335d8634f3a4a801242dadc8b63a17a3d1a60603f66a158154554068f8802", + "event_digest_generic.json": "9caaa75dc67b90068c921cf4aa3d6e3284f2d248af73042eddf17d06ae0a4ef4", + "example_note.json": "44b4b47e907276316740bd4d20ad502140c6bf144252bb0ddac1349189f34f77", + "example_note.v1.schema.json": "3b3b93c1561607e2d7017036db3bc18e257e1e8338dddfba2923834a6e1f5b90", + "subscriptions_generic.json": "7188e0efd42a679c0710c4ee28d55e8d4bac238146883b9b4096ecf524fe8804", +} + +# Vectors that Nerve's own check refuses although the gateway takes them. +# Nerve checks the subscriptions that it builds, and it never builds these. +STRICTER_THAN_THE_GATEWAY = { + # A subscription names repositories only for github_repos: Nerve's check + # predates generic sources. + "GitHub members on a provider without schemas": "malformed_frame", +} + + +def vectors(name: str) -> Any: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def reason(check) -> str: + """The rejection reason of *check*, or ``""`` when it passes.""" + try: + check() + except Rejected as error: + return error.reason + return "" + + +def page_text(event_text: str) -> str: + """An inbox page with one event, written as text so that the event keeps its bytes.""" + return ( + '{"version":"1","kind":"inbox_read_result","correlation_id":"n-1",' + '"payload":{"inbox_read_result":{"outcome":"succeeded","events":[' + event_text + "]}}}" + ) + + +def record_event(provider: str, record: Any) -> dict[str, Any]: + """A delivered source record event of *provider* that carries *record*.""" + return { + "event_id": f"{provider}:3c9a7e15-2b4d-4f60-8e1a-5d7c9b2f4a68:created:1", + "connection_id": "3c9a7e15-2b4d-4f60-8e1a-5d7c9b2f4a68", + "provider": provider, + "kind": "source_record", + "conversation": {"id": "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", "kind": "item"}, + "admission": {"purpose": "observe"}, + "delivery": {"inbox_id": "1020"}, + "occurred_at": "2026-11-10T08:00:00Z", + "record": record, + } + + +def data_page(data_text: str) -> str: + """A page whose one generic record has *data_text* as its data, byte for byte.""" + event = record_event("example", { + "id": "note:1", "record_type": "example_note", "schema": "example.note/1", + "summary": "[NOTE-7] New note: Plan", "timestamp": "2026-11-10T08:00:00Z", "data": {}, + }) + text = json.dumps(event) + return page_text(text.replace('"data": {}', '"data": ' + data_text)) + + +def settings_check(settings_text: str): + """A check of *settings_text* as the settings of a generic subscription that Nerve sends.""" + def check() -> None: + source = decode_record(SourceSubscription, parse_json( + '{"provider":"example","schemas":["example.note/1"],"settings":' + settings_text + "}", + )) + Subscriptions(version="1", sources=(source,)).validate() + return check + + +class TestVectorCopies: + @pytest.mark.parametrize("name", sorted(VECTOR_SHA256)) + def test_the_copy_of_the_contract_vector_has_its_exact_bytes(self, name): + assert hashlib.sha256((FIXTURES / name).read_bytes()).hexdigest() == VECTOR_SHA256[name] + + +class TestSourceJSON: + @pytest.mark.parametrize("case", vectors("source_json.json")["canonical"], ids=lambda case: case["name"]) + def test_the_canonical_form_is_the_contracts(self, case): + assert canonical_json(parse_json(case["json"])) == case["canonical"] + + @pytest.mark.parametrize("case", vectors("source_json.json")["rejected"], ids=lambda case: case["name"]) + def test_a_value_that_breaks_a_rule_is_refused_as_data_and_as_settings(self, case): + assert reason(lambda: decode_envelope(data_page(case["json"]))) == case["reason"] + assert reason(settings_check(case["json"])) == case["reason"] + + @pytest.mark.parametrize("case", vectors("source_json.json")["bounds"], ids=lambda case: case["name"]) + def test_the_bounds_count_the_canonical_bytes(self, case): + text = case["prefix"] + case["repeat"] * case["count"] + case["suffix"] + check = ( + (lambda: decode_envelope(data_page(text))) if case["object"] == "data" + else settings_check(text) + ) + + assert reason(check) == case["reason"] + + def test_the_bounds_are_the_contracts(self): + assert (MAX_SOURCE_DATA_BYTES, MAX_SOURCE_SETTINGS_BYTES) == (32768, 8192) + + +class TestParser: + """The frame parser refuses what the gateway's parser refuses.""" + + def test_a_repeated_member_name_anywhere_in_a_frame_is_malformed(self): + envelope = '{"version":"1","kind":"heartbeat","payload":{"heartbeat":{"sequence":1,"sequence":2}}}' + + assert reason(lambda: decode_envelope(envelope)) == "malformed_frame" + + @pytest.mark.parametrize("literal", ["NaN", "Infinity", "-Infinity"]) + def test_non_finite_literals_are_malformed(self, literal): + envelope = '{"version":"1","kind":"heartbeat","payload":{"heartbeat":{"sequence":' + literal + "}}}" + + assert reason(lambda: decode_envelope(envelope)) == "malformed_frame" + + +class TestDigestVector: + """Nerve computes no event digest, but the data enters it in the canonical form of + :func:`canonical_json`: the generic digest vector ends with the record's schema and + that form, each as a 4-byte big-endian length and the UTF-8 bytes.""" + + def test_the_projection_ends_with_the_schema_and_the_canonical_data(self): + vector = vectors("event_digest_generic.json") + event = vectors("example_note.json") + projection = base64.b64decode(vector["normalized_base64"]) + # The fixture's data is unsorted and writes escapes and 2.50. + data = parse_json(json.dumps(event["record"]["data"])) + + tail = b"".join( + struct.pack(">I", len(text.encode("utf-8"))) + text.encode("utf-8") + for text in (event["record"]["schema"], canonical_json(data)) + ) + + assert vector["event_fixture"] == "events/example_note.json" + assert "sha256:" + hashlib.sha256(projection).hexdigest() == vector["digest"] + assert projection.endswith(tail) + assert '"score":2.5}' in canonical_json(data) and "été" in canonical_json(data) + + +def example_note_page() -> str: + """The contract's generic record event, with its record exactly as written.""" + return page_text((FIXTURES / "example_note.json").read_text(encoding="utf-8")) + + +class TestGenericRecords: + def test_the_contract_sample_decodes_with_its_schema_data_and_item(self): + event = decode_envelope(example_note_page()).body.events[0] + + assert (event.provider, event.kind, event.admission.purpose) == ("example", "source_record", "observe") + assert event.conversation.kind == CONVERSATION_KIND_ITEM == "item" + assert (event.conversation.id, event.conversation.display_name) == ( + "5f0c1b9e-3c2a-4d7e-9b1f-8a6d2e4c7b30", "NOTE-7", + ) + assert event.connection_id == uuid.UUID("3c9a7e15-2b4d-4f60-8e1a-5d7c9b2f4a68") + record = event.record + assert (record.record_type, record.schema, record.metadata) == ("example_note", "example.note/1", ()) + assert record.data["note"]["title"] == "Plan the release été" + assert record.data["score"] == 2.5 + assert record.data["actor"] == {"kind": "user", "name": "Alice Example", "id": "0b7d4c2a-6e1f-4a3b-9c8d-2f5e7a1b3c49"} + + def test_members_of_the_data_that_nerve_does_not_know_are_kept(self): + event = vectors("example_note.json") + event["record"]["data"]["timestamp"] = "not a time: protocol names mean nothing here" + event["record"]["data"]["note"]["tenant_id"] = {"nested": [1, {"deep": True}]} + + data = decode_envelope(page_text(json.dumps(event))).body.events[0].record.data + + assert data["timestamp"] == "not a time: protocol names mean nothing here" + assert data["note"]["tenant_id"] == {"nested": [1, {"deep": True}]} + + @pytest.mark.parametrize("case", vectors("source_records.json")["records"], ids=lambda case: case["name"]) + def test_a_record_gets_the_contracts_verdict(self, case): + page = page_text(json.dumps(record_event(case["provider"], case["record"]))) + + assert reason(lambda: decode_envelope(page)) == case["reason"] + + @pytest.mark.parametrize("provider", sorted(CLOSED_PROVIDERS)) + def test_a_record_of_a_closed_provider_carries_no_schema_or_data(self, provider): + record = {"id": "1", "record_type": f"{provider}_message", "summary": "s", + "timestamp": "2026-11-10T08:00:00Z", "data": {}} + + assert reason(lambda: decode_envelope(page_text(json.dumps(record_event(provider, record))))) == ( + "malformed_frame" + ) + + def test_the_test_schema_takes_the_sample_and_refuses_an_unknown_member(self): + """The producer writes strictly to the schema file; Nerve reads tolerantly.""" + schema = vectors("example_note.v1.schema.json") + data = vectors("example_note.json")["record"]["data"] + validator = jsonschema.Draft202012Validator(schema) + + validator.validate(data) + with pytest.raises(jsonschema.ValidationError): + validator.validate({**data, "unreviewed": "value"}) + + def test_the_schema_names_its_record_type(self): + assert schema_record_type("linear", "linear.issue_event/1") == "linear_issue_event" + assert reason(lambda: schema_record_type("linear", "linear.issue_event/0")) == "malformed_frame" + assert reason(lambda: schema_record_type("github", "github.notification/1")) == "malformed_frame" + + +class TestGenericSubscriptions: + @pytest.mark.parametrize( + "case", vectors("source_subscriptions.json")["sources"], ids=lambda case: case["name"], + ) + def test_a_subscription_gets_the_contracts_verdict(self, case): + def check() -> None: + source = decode_record(SourceSubscription, parse_json(json.dumps(case["source"]))) + Subscriptions(version="1", sources=(source,)).validate() + + expected = STRICTER_THAN_THE_GATEWAY.get(case["name"], case["reason"]) + assert reason(check) == expected + + def test_the_stricter_cases_are_vectors(self): + names = {case["name"] for case in vectors("source_subscriptions.json")["sources"]} + + assert set(STRICTER_THAN_THE_GATEWAY) <= names + + def test_nerve_sends_the_contracts_generic_frame(self): + sample = vectors("subscriptions_generic.json") + body = Subscriptions(version="1", sources=( + SourceSubscription(provider="github", filters=SourceFilters(deny_actors=("*-bot",))), + SourceSubscription( + provider="example", + settings={"events": ["created", "commented"], "inbox": True}, + schemas=("example.note/1",), + rules=( + SourceRule(path="/actor/kind", deny=("bot",)), + SourceRule(path="/note/key", allow=("NOTE-*",)), + ), + ), + )) + envelope = Envelope(version="1", kind="subscriptions", request_id="n-26", payload=Payload(subscriptions=body)) + + body.validate() + assert json.loads(encode_envelope(envelope)) == sample + + def test_empty_settings_are_sent_and_absent_ones_are_not(self): + def sent(settings: dict[str, Any] | None) -> dict[str, Any]: + return json.loads(dumps(SourceSubscription( + provider="linear", settings=settings, schemas=("linear.issue_event/1",), + ))) + + assert sent(None) == {"provider": "linear", "filters": {}, "schemas": ["linear.issue_event/1"]} + assert sent({})["settings"] == {} + + def test_without_filters_drops_the_rules_and_keeps_the_settings_and_schemas(self): + narrow = Subscriptions(version="1", sources=(SourceSubscription( + provider="example", settings={"inbox": False}, schemas=("example.note/1",), + rules=(SourceRule(path="/actor/kind", deny=("bot",)),), + ),)) + + wide = narrow.without_filters() + + assert wide.sources == (SourceSubscription( + provider="example", settings={"inbox": False}, schemas=("example.note/1",), + ),) + wide.validate() + + def test_the_email_subscription_is_not_generic(self): + Subscriptions(version="1", sources=(SourceSubscription(provider="email"),)).validate() + with pytest.raises(Rejected) as error: + Subscriptions(version="1", sources=( + SourceSubscription(provider="email", schemas=("email.message/1",)), + )).validate() + assert error.value.reason == "malformed_frame" + + def test_settings_that_nerve_builds_are_checked_like_received_data(self): + deep: dict[str, Any] = {} + node = deep + for _ in range(8): + node["a"] = {} + node = node["a"] + for settings, expected in [ + ({"events": ["assigned"], "inbox": False}, ""), + ({"score": float("nan")}, "malformed_frame"), + ({"big": 10 ** 400}, "malformed_frame"), + ({"text": "\ud800"}, "malformed_frame"), + (deep, "limit_exceeded"), + ({"a": "x" * MAX_SOURCE_SETTINGS_BYTES}, "limit_exceeded"), + ]: + subscriptions = Subscriptions(version="1", sources=(SourceSubscription( + provider="linear", settings=copy.deepcopy(settings), schemas=("linear.issue_event/1",), + ),)) + assert reason(subscriptions.validate) == expected, settings From 3a991ba24286d799b2de697006d32b800fa73b94 Mon Sep 17 00:00:00 2001 From: Alex Soffronow Pagonidis <237136924+alex-clickhouse@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:47:12 +0000 Subject: [PATCH 03/10] Sources: the hosted Linear source on the generic contract The channel gateway polls Linear for each agent with the agent's Linear connector in the MCP catalog (its credential, project scope and GitHub filter) and streams the records as a generic source. Nerve needs a setting that subscribes to them and a source that drains them into the inbox. sync.linear has enabled, mode (hosted only; poll refuses to load with a message that it is not available), events, inbox, include_bots, drop_own_actions, rules, schedule, condense and batch_size. While it is on, Nerve subscribes to provider linear with the settings that differ from the provider's defaults, the schema linear.issue_event/1, and the rules. HostedSource drains the buffer of any generic provider and keeps the record data as the metadata, with every member; HostedGitHubSource builds on it and keeps its own records. The registry runs the Linear drain with the rules as the guardrail, and nerve doctor reports the source. Rules that the gateway would refuse refuse to load. Docs: docs/sources.md "Linear (hosted)" and "Rules of generic hosted sources", docs/config.md (keys and the restart row), the example config, and a line in the TOOLS.md templates. Co-Authored-By: Claude Opus 5.5 --- config.example.yaml | 16 + docs/config.md | 33 +- docs/sources.md | 147 +++++++- nerve/channels/hosted/__init__.py | 5 +- nerve/channels/hosted/runtime.py | 7 +- nerve/channels/hosted/sources.py | 106 ++++-- nerve/cli.py | 42 +++ nerve/config.py | 176 +++++++++ nerve/sources/github_hosted.py | 95 +---- nerve/sources/hosted.py | 154 ++++++++ nerve/sources/registry.py | 27 ++ nerve/templates/personal/TOOLS.md | 3 +- nerve/templates/worker/TOOLS.md | 3 +- tests/test_config_reload.py | 1 + tests/test_hosted_linear_source.py | 562 +++++++++++++++++++++++++++++ 15 files changed, 1259 insertions(+), 118 deletions(-) create mode 100644 nerve/sources/hosted.py create mode 100644 tests/test_hosted_linear_source.py diff --git a/config.example.yaml b/config.example.yaml index fb8f3e6b7..ba8a5cb52 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -371,6 +371,22 @@ sync: # close of the PRs it reported # deny_authors: [] # case-insensitive globs on the author login batch_size: 50 + # Linear issue events of the agent's Linear connector in the MCP catalog. + # The channel gateway polls Linear with the connector's credential and + # project scope and streams the records (needs channels.hosted and a + # gateway that knows generic sources). Each record names the catalog + # server whose Linear tools act on it. Switching it on needs a restart. + # See docs/sources.md "Linear (hosted)". + # linear: + # enabled: false + # mode: hosted # the only mode; poll refuses to load + # events: [assigned, commented, mentioned] # default: every event + # inbox: true # also the agent's own Linear notifications + # include_bots: false # also events of bots and integrations + # drop_own_actions: true # drop the credential user's own events + # rules: # JSON Pointers into the record data; + # - path: /issue/team/key # applied by the gateway and again here + # allow: ["ENG*"] # Memory (memU) memory: diff --git a/docs/config.md b/docs/config.md index 8992bbabd..a9d33155c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -242,7 +242,7 @@ A reload is always explicit. Two things cause one: | Cron jobs (`config/cron/*.yaml`) | ✅ | | Custom cron gate plugins (new, edited *and* deleted `.py` files) | ✅ the registry is rebuilt from the directory | | Cron file locations (`cron.jobs_file`, `system_file`, `gate_plugins_dir`) | ✅ re-read from the new config | -| Cron sources: `sync.telegram`, `.gmail`, `.github`, `.github_events`, `.github_repos`, `.message_ttl_days`, and each source's `schedule` | ✅ runners are rebuilt and rescheduled, all or nothing: a `schedule` the scheduler will not take refuses the source reload with the running sources on their old triggers, and reports the error rather than `ok`. In hosted mode the GitHub filters, `pr_watch`, `inbox` and the repository watch (`sync.github_repos`) are published to the gateway again. **`sync.codex` and `sync.github.mode` are not among these**; see the restart table | +| Cron sources: `sync.telegram`, `.gmail`, `.github`, `.github_events`, `.github_repos`, `.linear`, `.message_ttl_days`, and each source's `schedule` | ✅ runners are rebuilt and rescheduled, all or nothing: a `schedule` the scheduler will not take refuses the source reload with the running sources on their old triggers, and reports the error rather than `ok`. In hosted mode the GitHub filters, `pr_watch`, `inbox` and the repository watch (`sync.github_repos`) are published to the gateway again, and so are the Linear settings and rules. **`sync.codex` and `sync.github.mode` are not among these**; see the restart table | | `cron.delivery_debounce_seconds` | ✅ read for each delivery | | MCP servers (`mcp_servers`) | ✅ new sessions get the new set | | Skills (`skills/`, and the platform skills in `skills/` in the config directory) | ✅ re-scanned | @@ -298,6 +298,7 @@ reload cannot inspect, and are documented here only. | `mcp_gateway_url` | the catalog reader is built at startup. See [MCP servers in external mode](#mcp-servers-in-external-mode) | | `slack.mode`, `channels.hosted.*` | the hosted channels, the email channel included, their stream endpoint, and their token verifier are built at startup. A reload reads the file that `gateway_jwks_file` names again, but not a changed path. Changing `mode` to `socket` reports a Slack reload error until a restart, because the hosted channel still holds the `slack` name. Changing it to `hosted` stops the Socket Mode connection at once, and the stream endpoint opens after a restart | | `sync.github.mode` | the GitHub sources (`github` and the repository watch, `github_repos`), and in hosted mode the stream endpoint and the consumers of the gateway's records, are chosen at startup. Until a restart the sources keep the mode they started with, and so does the `github_pr_activity` gate. The filters, `pr_watch`, `inbox` and `sync.github_repos` do follow a reload: in hosted mode the reload publishes them to the gateway again. To go back to `poll`, set `enabled: false` (and `sync.github_repos.enabled: false`) and reload first (see [sources.md](sources.md#hosted-github-source)) | +| Turning `sync.linear.enabled` **on** | the consumer of the gateway's Linear records is built at startup, only while the source is on. Turning it **off** is hot: the reload tells the gateway that Nerve takes no Linear records. The events, the other settings and the rules follow a reload | | `auth.jwt_secret` | pinned at startup for every consumer, web gateway and MCP endpoint alike. A reload that changes or removes it is reported and changes nothing live: removing the key must not reopen the instance, and rotating it must not swap the key under live sessions half-way. The next restart applies it (with the key removed, the secret generated into `nerve.db` takes over) | | `workflows.enabled`, `workflows.review_loop.enabled` | each service is created at startup and only when its flag is on. Turning one **off** does not stop the service already running, and turning it **on** creates nothing for a reload to reach | | `workflows.poll_interval_seconds`, `workflows.review_loop.reconcile_interval_seconds` | both loops were handed their interval when they started. Everything else under `workflows.*` is read per use (see the table above) | @@ -1457,13 +1458,15 @@ notifications, and with `sync.github_repos` the repositories it watches; the `channels.hosted` settings above apply unchanged, and they are enough on their own: Slack need not be hosted too. See [Hosted GitHub source](sources.md#hosted-github-source) and -[Hosted repository watch](sources.md#hosted-repository-watch). They also +[Hosted repository watch](sources.md#hosted-repository-watch). With +`sync.linear.enabled` the gateway also polls Linear with the agent's Linear +connector (see [Linear](sources.md#linear-hosted)). They also carry the mail that members send to the agent: the usable `channels.hosted` settings give the email channel, which `email.enabled` switches on (see [Email](#email)). -When Slack is hosted as well, its switch also holds GitHub and member mail -back. Nerve reads the events of every hosted provider from one ordered +When Slack is hosted as well, its switch also holds GitHub, Linear and member +mail back. Nerve reads the events of every hosted provider from one ordered inbox, and it reads nothing while hosted Slack is switched off (`slack.enabled: false`): the GitHub records wait at the gateway until Slack is switched on again, and past the gateway's limit of stored records the @@ -2008,6 +2011,28 @@ reload: | `sync.github_repos.updates` | bool | `false` | Also report a new head commit, ready for review, and the merge or close of each pull request the watch reported | | `sync.github_repos.deny_authors` | list | `[]` | Case-insensitive globs on the author's login, with the guardrail rules; what they match is never reported, releases included | +**Linear-specific** (issue events of the agent's Linear connector in the MCP +catalog; see [sources.md](sources.md#linear-hosted)). The channel gateway +polls Linear with the connector's credential and scope and streams the +records, so the source needs the `channels.hosted` settings (see +[Hosted mode](#hosted-mode)) and a gateway that knows generic sources. The +settings and the rules go to the gateway in the subscription, and the rules +are applied here again. Everything follows a reload, except that switching +the source on needs a restart: + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `sync.linear.enabled` | bool | `false` | Take the Linear records from the channel gateway. Switching it on takes effect at the next restart; switching it off follows a reload | +| `sync.linear.mode` | string | `hosted` | `hosted`, the only mode. `poll` and any other value refuse to load | +| `sync.linear.events` | list | every event | What is reported: `issue_created`, `assigned`, `unassigned`, `delegated`, `state_changed`, `priority_changed`, `moved_in`, `commented`, `mentioned`. Case does not matter; an unknown event or an empty list refuses to load. With the default, an event that a later gateway adds is reported too | +| `sync.linear.inbox` | bool | `true` | Also report the Linear notifications of the agent's Linear user. Only a credential that belongs to the agent (an agent API key, the agent's sign-in) has them | +| `sync.linear.include_bots` | bool | `false` | Also report the events of bots and integrations | +| `sync.linear.drop_own_actions` | bool | `true` | Drop the events of the credential's Linear user. With a tenant API key they are kept and marked with `actor.credential_user: true` | +| `sync.linear.rules` | list | `[]` | At most 16 rules, each a `path` (a JSON Pointer into the record data, such as `/issue/team/key`) and exactly one non-empty `allow` or `deny` list of globs. A rule the gateway would refuse refuses to load. See [sources.md](sources.md#rules-of-generic-hosted-sources) | +| `sync.linear.schedule` | cron | `*/15 * * * *` | Fallback cadence of the drain, which also runs on every delivery | +| `sync.linear.batch_size` | int | `50` | Max records per drain | +| `sync.linear.condense` | bool | `false` | LLM-condense long records via `memory.fast_model` | + `cron.delivery_debounce_seconds` (int, default `60`) spaces the job runs that a hosted delivery starts; a negative value reads as `0`, and a value above `600` waits `600` seconds (`nerve config validate` warns). See diff --git a/docs/sources.md b/docs/sources.md index 6b50d6083..5460fa259 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -428,6 +428,91 @@ machine for them. platform that parks the machine does not wake it for that run. The mode follows `sync.github.mode`, which is restart-only. +### Linear (hosted) +- **Adapter:** `nerve/sources/hosted.py` (`HostedSource`). It drains the records that the channel gateway delivered. This process makes no Linear calls and needs no Linear key. + +With `sync.linear.enabled: true`, the shared channel gateway (the one of +[hosted Slack](config.md#hosted-mode) and the +[hosted GitHub source](#hosted-github-source), set up with the same +`channels.hosted` settings) polls Linear for the agent and streams the records +to Nerve. It uses the agent's Linear connector in the MCP catalog: its +credential, its project scope, and its GitHub filter. A record holds only what +the connector would show the agent. The gateway keeps the records until Nerve +reads them, also while Nerve is not running. + +- **The connector is the switch.** The gateway polls Linear for an agent only + while the agent has a Linear connector in the MCP catalog and Nerve + subscribes, which it does while `enabled` is true. An agent with more than + one Linear connector gets the records of each one. +- **What arrives:** `linear_issue_event` records, one for each event: + `issue_created`, `assigned`, `unassigned`, `delegated`, `state_changed`, + `priority_changed`, `moved_in` (moved into a project of the scope), + `commented` and `mentioned`. The summary is one line, such as + `[ENG-12] Alice assigned you: Fix the login page`, and the content is + Markdown for the agent. With a credential that belongs to the agent (an + agent API key, or the agent's sign-in), the Linear notifications of the + agent's Linear user come too. +- **The data of a record is its metadata.** The gateway sends the data of a + record as a JSON object (record schema `linear.issue_event/1`), and Nerve + stores it as the record's metadata with every member, also the members that + this release does not know. The members: + - `event`, and `via`: `watch` for an issue of the scope, `inbox` for a + notification; + - `server`: the `id` and `name` of the Linear server in the MCP catalog; + - `issue`: `id`, `identifier`, `title`, `url`, `team` (`key`), `project` + (`id`, `name`), `state` (`name`, `type`), `priority`, `assignee`, + `delegate`; + - `comment`: `id`, `url`, `body` (at most 4,000 characters), `parent_id`; + - `actor`: `id`, `name`, `kind` (`user`, `app`, `bot`, `integration` or + `external`), and `credential_user`; + - `changes`: a list of `{field, from, to}`. +- **Act on a record with the Linear tools of the server that it names** + (`server`). That MCP catalog server reaches the same issues, with the same + scope. +- **The settings go to the gateway.** `events`, `inbox`, `include_bots` and + `drop_own_actions` go in the subscription as the provider settings. Only the + values that differ from the defaults are sent, so with the default `events` + an event that a later gateway adds arrives too. `inbox` has an effect only + with a credential that belongs to the agent. `drop_own_actions` drops the + events of the credential's Linear user. A tenant API key belongs to a + person, so the events of that person are kept and marked with + `actor.credential_user: true`, whatever the setting is. To drop them, add the + rule `{path: /actor/credential_user, deny: ["true"]}`. +- **The rules go to the gateway too.** The gateway applies `rules` before it + stores a record, and the guardrail here applies them again; see + [Rules of generic hosted sources](#rules-of-generic-hosted-sources). A rule + that the gateway would refuse refuses to load. +- **The gateway must know generic sources.** A gateway that predates them + closes a stream whose subscriptions carry `settings`, `schemas` or `rules`, + hosted Slack traffic included. Update the gateway before you switch Linear on. +- **Same runner model as hosted GitHub.** The records go to a local buffer, + and the `source:linear` runner drains it as soon as records are delivered. + Its scheduled run is a fallback that is not the instance's next scheduled + task while nothing waits. Consumer cursors, `messages` gates, `condense` and + the Sources page work as for the other sources; the Sources page shows each + record with a Linear card. +- **A switched-off hosted Slack holds Linear back**, as it holds hosted GitHub + back (see [Hosted GitHub source](#hosted-github-source)). +- **Hosted only.** `mode` is `hosted`, also when it is not set. `mode: poll` + refuses to load: this release has no Linear source that polls from this + process. +- **Switching it on needs a restart.** The consumer of the gateway's Linear + records is built at startup, so `enabled: true` takes effect at the next + restart. Switching it off follows a reload: Nerve tells the gateway that it + takes no Linear records. The other settings follow a reload. + +```yaml +sync: + linear: + enabled: true + events: [assigned, commented, mentioned] # default: every event + rules: + - path: /issue/team/key + allow: ["ENG*"] # applied by the gateway and again here +``` + +`nerve doctor` reports the source and checks the hosted settings. + ### Telegram account sync - **Adapter:** `nerve/sources/telegram.py` — uses Telethon (user account API). This is the pull sync for your own Telegram account, and is separate from the @@ -668,6 +753,18 @@ sync: deny_authors: [] # Globs on the author's login; never reported batch_size: 50 condense: false + + linear: + enabled: false # The channel gateway polls Linear with the agent's Linear connector + mode: hosted # The only mode; poll refuses to load + events: [issue_created, assigned, unassigned, delegated, state_changed, priority_changed, moved_in, commented, mentioned] + inbox: true # Also the notifications of the agent's Linear user (agent credentials only) + include_bots: false # Also the events of bots and integrations + drop_own_actions: true # Drop the events of the credential's Linear user (not with a tenant API key) + rules: [] # Allow/deny globs on JSON Pointers into the data. See "Rules of generic hosted sources" + schedule: "*/15 * * * *" # Fallback cadence of the drain + batch_size: 50 + condense: false ``` ### Config Fields @@ -961,6 +1058,45 @@ Rules a hook can rely on: |-------|------|---------|-------------| | `github.drop_hook` | list | `[]` | argv of a command that gets each dropped record as a JSON line on stdin. Passed as is: no shell, nothing stripped. Empty = off | +### Rules of generic hosted sources + +A generic hosted source, such as [Linear](#linear-hosted), stores the data of +each record (a JSON object) as the record's metadata. Its rules +(`sync.linear.rules`) name a value in that data with a JSON Pointer (RFC 6901) +and match it with an `allow` or a `deny` list of glob patterns. The channel +gateway applies the same rules before it stores a record, so the two sides +drop the same records, and the guardrail here (`DataRule` in +`nerve/sources/filters.py`) applies them again. + +```yaml +sync: + linear: + rules: + - path: /issue/team/key + allow: ["ENG*", "OPS"] + - path: /actor/kind + deny: [bot, integration] +``` + +- **The path** starts with `/`, has 1 to 8 parts and at most 256 bytes, and + writes `~` as `~0` and `/` as `~1`. A part names the member of an object, or + the element of an array at its index (`/changes/0/field`). +- **The values at the path:** a string as it is; a number as its shortest + JSON text (`2` for `2.0`, `1.5`); `true`, `false` and `null` as those words; + and each such element of an array. An object, an array or an object in an + array, an empty array, and a path that names nothing give no value. +- **Each rule has exactly one non-empty list**, `allow` or `deny`, of at most + 64 patterns of at most 256 bytes without control characters. Patterns match + as the other guardrail patterns do: case-insensitive shell globs on the + stripped value. A `deny` rule drops a record when one of its values matches. + An `allow` rule drops a record when none of its values matches, also a record + without a value at its path. A record stays only when every rule keeps it. +- **At most 16 rules.** A rule outside these bounds, or with unknown keys, + refuses to load, so the gateway never closes the stream over one. + +The runner's drop log names the path of the rule that dropped a record, and the +drop hook gets it as `dropped_on`. + ### Extending to other sources Adding a guardrail to another source is a config field plus one registry line. In @@ -1173,7 +1309,10 @@ web/src/components/Sources/ ├── index.ts ← Registry: source name → renderer type mapping ├── MarkdownRenderer.tsx ← Default: ReactMarkdown with prose styles ├── EmailRenderer.tsx ← Gmail: sandboxed iframe for HTML + text toggle - └── GitHubRenderer.tsx ← GitHub: repo/PR header card + markdown body + ├── GitHubRenderer.tsx ← GitHub: repo/PR header card + markdown body + ├── LinearRenderer.tsx ← Linear: issue card + markdown body + the record data + ├── DataRenderer.tsx ← Another generic hosted source: markdown body + the record data + └── RecordData.tsx ← The record data (nested JSON metadata) as an expandable JSON view ``` ### Steps @@ -1212,7 +1351,9 @@ export function MyRenderer({ content, metadata, summary }: Props) { | `nerve/sources/models.py` | `SourceRecord`, `FetchResult`, `IngestResult` dataclasses | | `nerve/sources/base.py` | `Source` abstract base class | | `nerve/sources/runner.py` | `SourceRunner` — pure ingestion pipeline (fetch → preprocess → guardrail → persist → condense → drop hook → advance) | -| `nerve/sources/filters.py` | `InboxFilter` / `FieldRule` — declarative allow/deny guardrails applied before persist | +| `nerve/sources/filters.py` | `InboxFilter` / `FieldRule` / `DataRule` — declarative allow/deny guardrails applied before persist | +| `nerve/sources/source_json.py` | The JSON data of generic hosted sources: checks, canonical form (RFC 8785), JSON Pointers | +| `nerve/sources/hosted.py` | `HostedSource`: drain the records of a generic hosted source (Linear) that the channel gateway delivered | | `nerve/sources/processor.py` | Legacy agent prompt building (unused by runner, may be used by tools) | | `nerve/sources/registry.py` | Config → `list[SourceRunner]` factory | | `nerve/sources/telegram.py` | Telegram adapter (Telethon) | @@ -1222,7 +1363,7 @@ export function MyRenderer({ content, metadata, summary }: Props) { | `nerve/sources/github_events.py` | GitHub user events adapter (gh CLI) | | `nerve/sources/github_repos.py` | GitHub repository watch adapter (gh CLI), record builder, its guardrail | | `nerve/agent/tools.py` | Consumer tools: `list_sources`, `poll_source`, `read_source` | -| `web/src/components/Sources/` | Source message renderers (email, GitHub, default markdown) | +| `web/src/components/Sources/` | Source message renderers (email, GitHub, Linear, record data, default markdown) | ### Cursor Design diff --git a/nerve/channels/hosted/__init__.py b/nerve/channels/hosted/__init__.py index fdbaadbbd..095bb3ff5 100644 --- a/nerve/channels/hosted/__init__.py +++ b/nerve/channels/hosted/__init__.py @@ -6,6 +6,7 @@ gateway's inbox over them. Hosted sources use the same streams: the gateway polls a source (GitHub, with -``sync.github.mode: hosted``) under the filters that Nerve publishes, and -Nerve reads the records like any other event (see :mod:`.sources`). +``sync.github.mode: hosted``; Linear, with ``sync.linear.enabled``) under the +filters that Nerve publishes, and Nerve reads the records like any other +event (see :mod:`.sources`). """ diff --git a/nerve/channels/hosted/runtime.py b/nerve/channels/hosted/runtime.py index 91c4fbe4e..09dee094b 100644 --- a/nerve/channels/hosted/runtime.py +++ b/nerve/channels/hosted/runtime.py @@ -74,9 +74,10 @@ def hosted_channel_providers(config: NerveConfig) -> list[str]: def hosted_providers(config: NerveConfig) -> list[str]: """Every provider whose traffic comes from the gateway: chat, then sources. - A hosted source that is switched off is included too: the gateway is - told that the agent subscribes to nothing, and the records already on - their way are kept for when it is switched on. + A hosted GitHub source that is switched off is included too: the gateway + is told that the agent subscribes to nothing, and the records already on + their way are kept for when it is switched on. Linear is included while + it is switched on (see :func:`~nerve.channels.hosted.sources.hosted_sources`). """ return hosted_channel_providers(config) + hosted_sources(config) diff --git a/nerve/channels/hosted/sources.py b/nerve/channels/hosted/sources.py index 20724ecc2..399261f62 100644 --- a/nerve/channels/hosted/sources.py +++ b/nerve/channels/hosted/sources.py @@ -4,16 +4,25 @@ It applies the filters that Nerve publishes before it stores a record, keeps the records in the agent's inbox until Nerve reads them, also while Nerve is not running, and Nerve reads them over the same streams that carry hosted -channel traffic. GitHub is the hosted source, as a whole: with -``sync.github.mode: hosted`` both the notifications (provider ``github``) and -the repository watch (provider ``github_repos``) come from the gateway. +channel traffic. The hosted sources: + +- GitHub, as a whole: with ``sync.github.mode: hosted`` both the + notifications (provider ``github``) and the repository watch (provider + ``github_repos``) come from the gateway. Their records carry metadata from + the contract's closed vocabulary. +- Linear (provider ``linear``), with ``sync.linear.enabled``: a generic + source. Its records carry ``data``, a JSON object that the provider + defines (record schema ``linear.issue_event/1``), and its subscription + carries provider ``settings``, the record ``schemas`` that Nerve reads, + and ``rules`` on the data. :class:`HostedSourceConsumer` takes a provider's records from the inbox reader and writes each one to the local ``channel_observations`` buffer under the provider's name. The provider's source runner drains that buffer into the -inbox with the guardrail, drop hook, and condensing of the polling source -(:mod:`nerve.sources.github_hosted`). A record is acknowledged only after its -buffer row is written. +inbox with the guardrail and condensing of every source, and for GitHub the +drop hook of the polling source (:mod:`nerve.sources.github_hosted`, +:mod:`nerve.sources.hosted`). A record is acknowledged only after its buffer +row is written. :func:`subscriptions_for` builds the ``subscriptions`` frame from the same settings that build the inbox filter, so the gateway and Nerve drop the same @@ -36,6 +45,7 @@ Event, RepoWatch, SourceFilters, + SourceRule, SourceSubscription, Subscriptions, ) @@ -51,9 +61,10 @@ ) from nerve.channels.hosted.contract.wire import format_time from nerve.channels.hosted.intake import Disposition +from nerve.config import LINEAR_EVENTS if TYPE_CHECKING: - from nerve.config import GitHubReposSyncConfig, GitHubSyncConfig, NerveConfig + from nerve.config import GitHubReposSyncConfig, GitHubSyncConfig, LinearSyncConfig, NerveConfig from nerve.db import Database logger = logging.getLogger(__name__) @@ -61,6 +72,10 @@ GITHUB = "github" # The repository watch: the repositories that sync.github_repos names. GITHUB_REPOS = REPO_WATCH_PROVIDER +# Linear issue events of the agent's Linear connector. +LINEAR = "linear" +# The record schemas of Linear records that Nerve reads. +LINEAR_SCHEMAS = ("linear.issue_event/1",) # Row target of a provider's buffer. The gateway keeps at most this many # source records for an agent, and the runner drains the buffer after every @@ -72,9 +87,14 @@ def hosted_sources(config: NerveConfig) -> list[str]: """The source providers whose records come from the gateway. GitHub is hosted as a whole: ``sync.github.mode: hosted`` makes the - repository watch (``github_repos``) hosted too. + repository watch (``github_repos``) hosted too, switched on or not. + Linear is hosted only (``sync.linear.mode``), and is a hosted source + while ``sync.linear.enabled`` is on when the process starts. """ - return [GITHUB, GITHUB_REPOS] if config.sync.github.mode == "hosted" else [] + providers = [GITHUB, GITHUB_REPOS] if config.sync.github.mode == "hosted" else [] + if config.sync.linear.enabled: + providers.append(LINEAR) + return providers # ---------------------------------------------------------------------- # @@ -99,6 +119,8 @@ def subscriptions_for(config: NerveConfig, providers: Iterable[str]) -> Subscrip watch = github_repos_subscription(config.sync.github_repos) if watch is not None: sources.append(watch) + if LINEAR in providers and config.sync.linear.enabled: + sources.append(linear_subscription(config.sync.linear)) return Subscriptions(version=SUBSCRIPTIONS_VERSION, sources=tuple(sources)) @@ -191,6 +213,40 @@ def github_repos_subscription(settings: GitHubReposSyncConfig) -> SourceSubscrip ) +def linear_subscription(settings: LinearSyncConfig) -> SourceSubscription: + """The Linear subscription: provider settings, record schemas, and rules. + + The settings carry only the values that differ from the provider's + defaults (every event, the inbox on, bot events off, own actions + dropped), so that absent settings mean the defaults and an event kind + that a later gateway adds reaches an agent that keeps the default. The + config refuses to load rules that the gateway would refuse, so they go + as they are, and the inbox guardrail applies them again. + """ + provider_settings: dict[str, Any] = {} + if set(settings.events) != set(LINEAR_EVENTS): + provider_settings["events"] = [event for event in LINEAR_EVENTS if event in settings.events] + if not settings.inbox: + provider_settings["inbox"] = False + if settings.include_bots: + provider_settings["include_bots"] = True + if not settings.drop_own_actions: + provider_settings["drop_own_actions"] = False + return SourceSubscription( + provider=LINEAR, + settings=provider_settings or None, + schemas=LINEAR_SCHEMAS, + rules=tuple( + SourceRule( + path=rule["path"], + allow=tuple(rule.get("allow") or ()), + deny=tuple(rule.get("deny") or ()), + ) + for rule in settings.rules + ), + ) + + def _patterns(values: object) -> list[str]: """The non-blank patterns of a setting, stripped as the inbox filter strips them.""" if not isinstance(values, (list, tuple)): @@ -234,25 +290,30 @@ def buffered_record(event: Event) -> dict[str, Any]: ``record.metadata`` keeps the wire entries (``key`` with ``value`` or ``values``); the source that drains the buffer knows which keys hold - lists and which hold flags. + lists and which hold flags. A record of a generic source adds its + ``schema`` and its ``data``, with every member. """ record = event.record timestamp = record.timestamp if record.timestamp != ZERO_TIME else event.occurred_at + fields: dict[str, Any] = { + "id": record.id, + "record_type": record.record_type, + "summary": record.summary, + "content": record.content, + "timestamp": format_time(timestamp) if timestamp != ZERO_TIME else "", + "metadata": [ + {"key": entry.key, "value": entry.value, "values": list(entry.values)} + for entry in record.metadata + ], + } + if record.data is not None: + fields["schema"] = record.schema + fields["data"] = record.data return { "event_id": event.event_id, "connection_id": str(event.connection_id), "conversation_id": event.conversation.id, - "record": { - "id": record.id, - "record_type": record.record_type, - "summary": record.summary, - "content": record.content, - "timestamp": format_time(timestamp) if timestamp != ZERO_TIME else "", - "metadata": [ - {"key": entry.key, "value": entry.value, "values": list(entry.values)} - for entry in record.metadata - ], - }, + "record": fields, } @@ -349,11 +410,14 @@ async def stop(self) -> None: "BUFFER_MAX_ROWS", "GITHUB", "GITHUB_REPOS", + "LINEAR", + "LINEAR_SCHEMAS", "HostedSourceConsumer", "buffered_record", "chat_subscription", "github_repos_subscription", "github_subscription", "hosted_sources", + "linear_subscription", "subscriptions_for", ] diff --git a/nerve/cli.py b/nerve/cli.py index e6a30c2a3..28a23c7db 100644 --- a/nerve/cli.py +++ b/nerve/cli.py @@ -1771,6 +1771,47 @@ def _github_repos_lines(config, warnings: list[str]) -> list[str]: return [line] +def _linear_lines(config, errors: list[str], warnings: list[str]) -> list[str]: + """Doctor lines for the Linear source (``sync.linear``). + + Linear records come only from the channel gateway, so the source needs + usable ``channels.hosted`` settings and a key file that can be read. A + switched-off hosted Slack holds the records back too, as for hosted + GitHub. + """ + linear = config.sync.linear + if not linear.enabled: + return ["[--] Linear source off (sync.linear.enabled is false)"] + from nerve.channels.hosted.auth import KeyFileError, read_key_file + from nerve.config import LINEAR_EVENTS + + hosted = config.channels.hosted + problems = hosted.problems() + if not problems: + try: + read_key_file(hosted.gateway_jwks_file) + except KeyFileError as error: + problems = [f"channels.hosted.gateway_jwks_file: {error}"] + if problems: + errors.extend(f"[ERR] Linear is on but {problem}" for problem in problems) + return [] + if config.slack.mode == "hosted" and not config.slack.enabled: + warnings.append( + "[WARN] Slack is hosted but switched off (slack.enabled is false): Nerve " + "reads nothing from the channel gateway until it is switched on, so the " + "Linear records wait there too" + ) + events = "all" if len(linear.events) == len(LINEAR_EVENTS) else ", ".join(linear.events) + count = len(linear.rules) + return [ + "[OK] Linear records come from the channel gateway (the agent's Linear " + f"connector): events {events}; inbox {'on' if linear.inbox else 'off'}; bots " + f"{'included' if linear.include_bots else 'left out'}; own actions " + f"{'dropped' if linear.drop_own_actions else 'kept'}; " + f"{count} {'rule' if count == 1 else 'rules'}" + ] + + def _source_suffix(config, path: str) -> str: """`` (environment)`` / `` (config-repo)`` after a configured connection line when the value did not come from a file on this machine; ``""`` for @@ -2262,6 +2303,7 @@ def doctor_report(config, config_source: str = "", check_api: bool = False) -> s lines.append("[--] Slack disabled") lines.extend(_hosted_github_lines(config, errors, warnings)) lines.extend(_github_repos_lines(config, warnings)) + lines.extend(_linear_lines(config, errors, warnings)) lines.extend(_email_lines(config, errors, warnings)) # Check SSL diff --git a/nerve/config.py b/nerve/config.py index aa5f81b17..b9142f8d8 100644 --- a/nerve/config.py +++ b/nerve/config.py @@ -10,6 +10,7 @@ import logging import os import re +import unicodedata import uuid from dataclasses import dataclass, field, replace from pathlib import Path @@ -23,6 +24,7 @@ from nerve.coerce import coerced as _coerced from nerve.coerce import lenient_int as _lenient_int from nerve.inference_principal import direct_client_headers +from nerve.sources.source_json import pointer_tokens import yaml @@ -2300,6 +2302,178 @@ def from_dict(cls, d: dict) -> GitHubReposSyncConfig: ) +# How Linear issue events reach the inbox: the channel gateway polls Linear +# with the agent's Linear connector and streams the records here. "poll", a +# Linear source that this process runs, does not exist. +LINEAR_SYNC_MODES: tuple[str, ...] = ("hosted",) +LINEAR_POLL_MODE = "poll" +# The Linear issue events that the channel gateway reports +# (LinearSyncConfig.events): the names of the records' data.event. +LINEAR_EVENTS: tuple[str, ...] = ( + "issue_created", "assigned", "unassigned", "delegated", "state_changed", + "priority_changed", "moved_in", "commented", "mentioned", +) +# The rules of a generic hosted source (LinearSyncConfig.rules): at most this +# many rules, each with at most this many patterns of at most this many +# UTF-8 bytes. The channel gateway's bounds too. +SOURCE_RULES_MAX = 16 +SOURCE_RULE_PATTERNS_MAX = 64 +SOURCE_RULE_PATTERN_MAX_BYTES = 256 +_SOURCE_RULE_KEYS = frozenset({"path", "allow", "deny"}) + + +def _linear_mode(value: object) -> str: + """``sync.linear.mode``: ``hosted``, also when unset; anything else refuses to load.""" + mode = _setting_str(value, "hosted").lower() + if mode == LINEAR_POLL_MODE: + raise ValueError( + "sync.linear.mode 'poll' is not available: Linear records come only from the " + "hosted channel gateway, so the mode is 'hosted'" + ) + if mode not in LINEAR_SYNC_MODES: + raise ValueError(f"sync.linear.mode must be one of {LINEAR_SYNC_MODES}, got {mode!r}") + return mode + + +def _linear_events(value: object) -> list[str]: + """``sync.linear.events``, checked against :data:`LINEAR_EVENTS`. + + Unset (missing, null or blank) is every event. Case does not matter, an + event named twice counts once, and they are kept in the order of + :data:`LINEAR_EVENTS`. An unknown event, or an empty list, refuses to + load: switch the source off with ``enabled: false`` instead. + """ + if value is None or (isinstance(value, str) and not value.strip()): + return list(LINEAR_EVENTS) + chosen: set[str] = set() + for entry in value if isinstance(value, (list, tuple)) else [value]: + name = str(entry or "").strip().lower() + if name not in LINEAR_EVENTS: + raise ValueError(f"sync.linear.events must be one or more of {LINEAR_EVENTS}, got {name!r}") + chosen.add(name) + if not chosen: + raise ValueError(f"sync.linear.events must be one or more of {LINEAR_EVENTS}, got none") + return [event for event in LINEAR_EVENTS if event in chosen] + + +def _source_rules(value: object, setting: str) -> list[dict[str, Any]]: + """The rules of a generic hosted source, such as ``sync.linear.rules``. + + Each rule is a mapping with a ``path``, a JSON Pointer into the record's + data such as ``/actor/kind``, and exactly one non-empty ``allow`` or + ``deny`` list of glob patterns. Patterns are stripped and blank ones + dropped; a single value is a list of one. A rule that the channel gateway + would refuse refuses to load, so the gateway never closes the stream over + one. The loaded rule has its path and its one list. + """ + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"{setting} must be a list of rules, each with a path and an allow or deny list") + if len(value) > SOURCE_RULES_MAX: + raise ValueError(f"{setting} must have at most {SOURCE_RULES_MAX} rules, got {len(value)}") + rules: list[dict[str, Any]] = [] + for index, entry in enumerate(value): + where = f"{setting}[{index}]" + if not isinstance(entry, dict): + raise ValueError(f"{where} must be a mapping with a path and an allow or deny list") + unknown = sorted(str(key) for key in set(entry) - _SOURCE_RULE_KEYS) + if unknown: + raise ValueError(f"{where} has unknown keys {unknown}; a rule has path, and allow or deny") + path = entry.get("path") + if not isinstance(path, str) or pointer_tokens(path) is None: + raise ValueError( + f"{where}.path must be a JSON Pointer into the record data, such as /actor/kind " + "(it starts with '/', has 1 to 8 parts and at most 256 bytes without control " + f"characters, and writes '~' as '~0' and '/' as '~1'), got {path!r}" + ) + lists: dict[str, list[str]] = {} + for key in ("allow", "deny"): + patterns = _source_rule_patterns(entry.get(key), f"{where}.{key}") + if patterns: + lists[key] = patterns + if len(lists) != 1: + raise ValueError(f"{where} needs exactly one non-empty list: allow or deny") + rules.append({"path": path, **lists}) + return rules + + +def _source_rule_patterns(value: object, where: str) -> list[str]: + """The patterns of one rule list, stripped, without the blank ones (none when unset).""" + if value is None: + return [] + entries = value if isinstance(value, (list, tuple)) else [value] + patterns = [text for text in (str(entry).strip() for entry in entries if entry is not None) if text] + if len(patterns) > SOURCE_RULE_PATTERNS_MAX: + raise ValueError(f"{where} must have at most {SOURCE_RULE_PATTERNS_MAX} patterns, got {len(patterns)}") + for pattern in patterns: + if ( + len(pattern.encode("utf-8", "replace")) > SOURCE_RULE_PATTERN_MAX_BYTES + or any(unicodedata.category(character) in ("Cc", "Cs") for character in pattern) + ): + raise ValueError( + f"{where} patterns must have at most {SOURCE_RULE_PATTERN_MAX_BYTES} bytes and no " + f"control characters, got {pattern[:40]!r}" + ) + return patterns + + +@dataclass +class LinearSyncConfig: + """Config for the Linear source: issue events of the agent's Linear connector. + + The channel gateway polls Linear for the agent with the credential of the + agent's Linear connector in the MCP catalog, and only within the + connector's project scope, then streams the records to Nerve (``mode: + hosted``, the only mode). Each record names the catalog server, whose + Linear tools act on it. The settings below go to the gateway as the + provider settings of the subscription; the rules go too, and the inbox + guardrail applies them again. All of it follows a reload, but switching + the source on needs a restart. See docs/sources.md "Linear". + """ + enabled: bool = False + # "hosted" only, also when unset. "poll" and any other value refuse to + # load. Restart-only. + mode: str = "hosted" + # The events to report (LINEAR_EVENTS); every one by default. An unknown + # event or an empty list refuses to load. + events: list[str] = field(default_factory=lambda: list(LINEAR_EVENTS)) + # Also report the Linear notifications of the agent's own Linear user + # (mentions, assignments, comments on issues it subscribes to). Only a + # credential of the agent's own (an agent API key, the agent's sign-in) + # has them; a credential of the tenant ignores it. + inbox: bool = True + # Also report the events of bots and integrations. + include_bots: bool = False + # Drop the events of the credential's own Linear user. A tenant API key + # belongs to a person, so its events are kept and marked + # (actor.credential_user) whatever this says. + drop_own_actions: bool = True + # Allow/deny glob lists on JSON Pointer paths into the record data, such + # as {"path": "/issue/team/key", "allow": ["ENG*"]}. At most + # SOURCE_RULES_MAX rules; a rule the gateway would refuse refuses to load. + rules: list[dict[str, Any]] = field(default_factory=list) + schedule: str = "*/15 * * * *" + condense: bool = False + batch_size: int = 50 + + @classmethod + @_coerced + def from_dict(cls, d: dict) -> LinearSyncConfig: + return cls( + enabled=d.get("enabled", False), + mode=_linear_mode(d.get("mode")), + events=_linear_events(d.get("events")), + inbox=d.get("inbox", True), + include_bots=d.get("include_bots", False), + drop_own_actions=d.get("drop_own_actions", True), + rules=_source_rules(d.get("rules"), "sync.linear.rules"), + schedule=d.get("schedule", "*/15 * * * *"), + condense=d.get("condense", False), + batch_size=d.get("batch_size", 50), + ) + + @dataclass class CodexOriginConfig: """A single Codex thread sync origin. @@ -2402,6 +2576,7 @@ class SyncConfig: github: GitHubSyncConfig = field(default_factory=GitHubSyncConfig) github_events: GitHubEventsSyncConfig = field(default_factory=GitHubEventsSyncConfig) github_repos: GitHubReposSyncConfig = field(default_factory=GitHubReposSyncConfig) + linear: LinearSyncConfig = field(default_factory=LinearSyncConfig) codex: CodexSyncConfig = field(default_factory=CodexSyncConfig) message_ttl_days: int = 7 # How long to keep source messages in the inbox consumer_cursor_ttl_days: int = 2 # Consumer cursors expire after N days of inactivity @@ -2416,6 +2591,7 @@ def from_dict(cls, d: dict) -> SyncConfig: github=GitHubSyncConfig.from_dict(d.get("github", {})), github_events=GitHubEventsSyncConfig.from_dict(d.get("github_events", {})), github_repos=GitHubReposSyncConfig.from_dict(d.get("github_repos", {})), + linear=LinearSyncConfig.from_dict(d.get("linear", {})), codex=CodexSyncConfig.from_dict(d.get("codex", {})), message_ttl_days=d.get("message_ttl_days", 7), consumer_cursor_ttl_days=d.get("consumer_cursor_ttl_days", 2), diff --git a/nerve/sources/github_hosted.py b/nerve/sources/github_hosted.py index 178cef892..61d7f1d33 100644 --- a/nerve/sources/github_hosted.py +++ b/nerve/sources/github_hosted.py @@ -31,17 +31,14 @@ from __future__ import annotations -import logging from typing import TYPE_CHECKING, Any -from nerve.sources.base import Source -from nerve.sources.models import FetchResult, SourceRecord +from nerve.sources.hosted import HostedSource, record_fields +from nerve.sources.models import SourceRecord if TYPE_CHECKING: from nerve.db import Database -logger = logging.getLogger(__name__) - SOURCE_NAME = "github" # The buffer channel that the hosted source consumer writes for this source. BUFFER_CHANNEL = "github" @@ -53,7 +50,7 @@ _FLAG_KEYS = frozenset({"unread"}) -class HostedGitHubSource(Source): +class HostedGitHubSource(HostedSource): """A GitHub source over the hosted records buffer of its provider. *provider* is ``github`` (the default) or ``github_repos``: the buffer @@ -61,77 +58,14 @@ class HostedGitHubSource(Source): """ source_name = SOURCE_NAME - push_fed = True def __init__(self, db: Database, provider: str = SOURCE_NAME): if provider not in PROVIDERS: raise ValueError(f"no hosted GitHub source for provider {provider!r}") - self._db = db - self.source_name = provider - self._buffer = provider - - async def fetch(self, cursor: str | None, limit: int = 100) -> FetchResult: - """Read buffered records past *cursor*, oldest first. - - A failed read raises, so the runner reports it and its health checks - count it. - """ - rows = await self._db.read_channel_observations( - self._buffer, after_id=self._position(cursor), limit=limit, - ) - if not rows: - return FetchResult(records=[], next_cursor=cursor, has_more=False) - - records: list[SourceRecord] = [] - for row_id, payload in rows: - record = ( - record_from_buffer(payload, source=self.source_name) - if isinstance(payload, dict) else None - ) - if record is None: - logger.warning( - "Hosted %s source: skipping unreadable buffer row %s", self.source_name, row_id, - ) - continue - records.append(record) - # The cursor tracks what was scanned, so an unreadable row is passed - # and does not stop the drain. - return FetchResult( - records=records, - next_cursor=str(rows[-1][0]), - has_more=len(rows) >= limit, - ) - - async def has_undrained(self, cursor: str | None) -> bool: - """Whether a record waits in the buffer past *cursor*. - - Normally none does: the cron service drains the buffer as records are - delivered. One row is enough to know; a failed read answers no, so a - forecast never fails for it. - """ - try: - rows = await self._db.read_channel_observations( - self._buffer, after_id=self._position(cursor, quiet=True), limit=1, - ) - except Exception: - logger.debug("Hosted %s source: checking the buffer failed", self.source_name, exc_info=True) - return False - return bool(rows) - - def _position(self, cursor: str | None, *, quiet: bool = False) -> int: - """The buffer row ID that *cursor* names, or 0.""" - if not cursor: - return 0 - try: - return int(cursor) - except ValueError: - if not quiet: - logger.info( - "Hosted %s source: cursor %.80r is not a buffer position (a cursor of " - "the polling source); draining from the start of the buffer", - self.source_name, cursor, - ) - return 0 + super().__init__(db, provider) + + def record(self, payload: dict[str, Any]) -> SourceRecord | None: + return record_from_buffer(payload, source=self.source_name) def record_from_buffer(payload: dict[str, Any], source: str = SOURCE_NAME) -> SourceRecord | None: @@ -139,21 +73,16 @@ def record_from_buffer(payload: dict[str, Any], source: str = SOURCE_NAME) -> So *source* is the provider whose buffer held it, and the record's source. """ - record = payload.get("record") - if not isinstance(record, dict): - return None - record_id = record.get("id") - record_type = record.get("record_type") - summary = record.get("summary") - if not all(isinstance(value, str) and value for value in (record_id, record_type, summary)): + record = record_fields(payload) + if record is None: return None content = record.get("content") timestamp = record.get("timestamp") return SourceRecord( - id=record_id, + id=record["id"], source=source, - record_type=record_type, - summary=summary, + record_type=record["record_type"], + summary=record["summary"], content=content if isinstance(content, str) else "", timestamp=timestamp if isinstance(timestamp, str) else "", metadata=_metadata(record.get("metadata")), diff --git a/nerve/sources/hosted.py b/nerve/sources/hosted.py new file mode 100644 index 000000000..d13fae975 --- /dev/null +++ b/nerve/sources/hosted.py @@ -0,0 +1,154 @@ +"""Hosted sources: drain the records that the channel gateway delivered. + +In hosted mode the channel gateway polls a provider for the agent and +streams the records to Nerve. The hosted source consumer writes every record +to the ``channel_observations`` buffer under the provider's name (see +:mod:`nerve.channels.hosted.sources`), and a :class:`HostedSource` turns the +buffered payloads back into :class:`SourceRecord` objects. It makes no +provider calls. + +The source keeps the provider's name, so it runs as ``source:`` +with the guardrail, condensing, health checks, consumer cursors and +``messages`` gates of every source. The cursor is the buffer's row ID, as for +the chat channel drains. + +:class:`HostedSource` drains a generic source, such as Linear: the record's +``data``, the JSON object that the provider defines, becomes the record's +metadata, with every member. GitHub has a closed vocabulary of its own +(:class:`~nerve.sources.github_hosted.HostedGitHubSource`). + +Buffered payload of a generic record (one row per record):: + + {"event_id": ..., "connection_id": ..., "conversation_id": ..., + "record": {"id", "record_type", "schema", "summary", "content", + "timestamp": RFC 3339, "metadata": [], "data": {...}}} +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from nerve.sources.base import Source +from nerve.sources.models import FetchResult, SourceRecord + +if TYPE_CHECKING: + from nerve.db import Database + +logger = logging.getLogger(__name__) + + +class HostedSource(Source): + """A source over the hosted records buffer of one provider. + + *provider* is the buffer channel that the hosted source consumer writes, + and the source name. + """ + + push_fed = True + + def __init__(self, db: Database, provider: str): + self._db = db + self.source_name = provider + self._buffer = provider + + def record(self, payload: dict[str, Any]) -> SourceRecord | None: + """The inbox record of one buffered payload, or ``None`` when it is unusable.""" + return record_from_buffer(payload, source=self.source_name) + + async def fetch(self, cursor: str | None, limit: int = 100) -> FetchResult: + """Read buffered records past *cursor*, oldest first. + + A failed read raises, so the runner reports it and its health checks + count it. + """ + rows = await self._db.read_channel_observations( + self._buffer, after_id=self._position(cursor), limit=limit, + ) + if not rows: + return FetchResult(records=[], next_cursor=cursor, has_more=False) + + records: list[SourceRecord] = [] + for row_id, payload in rows: + record = self.record(payload) if isinstance(payload, dict) else None + if record is None: + logger.warning( + "Hosted %s source: skipping unreadable buffer row %s", self.source_name, row_id, + ) + continue + records.append(record) + # The cursor tracks what was scanned, so an unreadable row is passed + # and does not stop the drain. + return FetchResult( + records=records, + next_cursor=str(rows[-1][0]), + has_more=len(rows) >= limit, + ) + + async def has_undrained(self, cursor: str | None) -> bool: + """Whether a record waits in the buffer past *cursor*. + + Normally none does: the cron service drains the buffer as records are + delivered. One row is enough to know; a failed read answers no, so a + forecast never fails for it. + """ + try: + rows = await self._db.read_channel_observations( + self._buffer, after_id=self._position(cursor, quiet=True), limit=1, + ) + except Exception: + logger.debug("Hosted %s source: checking the buffer failed", self.source_name, exc_info=True) + return False + return bool(rows) + + def _position(self, cursor: str | None, *, quiet: bool = False) -> int: + """The buffer row ID that *cursor* names, or 0.""" + if not cursor: + return 0 + try: + return int(cursor) + except ValueError: + if not quiet: + logger.info( + "Hosted %s source: cursor %.80r is not a buffer position (a cursor of " + "the polling source); draining from the start of the buffer", + self.source_name, cursor, + ) + return 0 + + +def record_fields(payload: dict[str, Any]) -> dict[str, Any] | None: + """The record of a buffered payload, or ``None`` without its ID, type and summary.""" + record = payload.get("record") + if not isinstance(record, dict): + return None + if not all(isinstance(record.get(key), str) and record[key] for key in ("id", "record_type", "summary")): + return None + return record + + +def record_from_buffer(payload: dict[str, Any], source: str) -> SourceRecord | None: + """The inbox record of one buffered generic payload, or ``None`` when it is unusable. + + *source* is the provider whose buffer held it, and the record's source. + The record's ``data`` becomes its metadata; a record without data has + none. + """ + record = record_fields(payload) + if record is None: + return None + content = record.get("content") + timestamp = record.get("timestamp") + data = record.get("data") + return SourceRecord( + id=record["id"], + source=source, + record_type=record["record_type"], + summary=record["summary"], + content=content if isinstance(content, str) else "", + timestamp=timestamp if isinstance(timestamp, str) else "", + metadata=dict(data) if isinstance(data, dict) else {}, + ) + + +__all__ = ["HostedSource", "record_fields", "record_from_buffer"] diff --git a/nerve/sources/registry.py b/nerve/sources/registry.py index 55275e9d4..278add899 100644 --- a/nerve/sources/registry.py +++ b/nerve/sources/registry.py @@ -277,6 +277,33 @@ def build_source_runners( gh_repos.deny_authors or [], ) + # Linear: hosted only. The channel gateway polls Linear with the agent's + # Linear connector and streams the records; the runner drains them with + # the rules of sync.linear as the guardrail, which the gateway applied + # before it stored them. + linear = config.sync.linear + if linear.enabled: + from nerve.channels.hosted.sources import LINEAR + from nerve.sources.filters import InboxFilter, data_rules + from nerve.sources.hosted import HostedSource + + runners.append(SourceRunner( + source=HostedSource(db, LINEAR), + db=db, + batch_size=linear.batch_size, + condense=linear.condense, + condense_model=condense_model, + condense_client_factory=condense_factory, + ttl_days=ttl_days, + inbox_filter=InboxFilter(rules=data_rules(linear.rules)), + )) + logger.info( + "Registered source: linear (hosted) (batch=%d, events=%s, inbox=%s, " + "include_bots=%s, drop_own_actions=%s, rules=%d)", + linear.batch_size, linear.events, linear.inbox, linear.include_bots, + linear.drop_own_actions, len(linear.rules), + ) + # Channel source drains. Not a poll: the channel transport already # delivered and buffered these events. This only moves them into the # inbox. What to watch is a property of the channel, so the config lives at diff --git a/nerve/templates/personal/TOOLS.md b/nerve/templates/personal/TOOLS.md index 06305e83f..a1b1ad643 100644 --- a/nerve/templates/personal/TOOLS.md +++ b/nerve/templates/personal/TOOLS.md @@ -47,7 +47,8 @@ Call tools as `mcp__nerve__`. The system prompt lists what is live; this i - `send_file`: deliver a workspace file as an attachment or download card - `send_channel_message`: post to a Slack conversation you name; offered only with Slack enabled and `slack.allow_outbound: true`, limited to `slack.allow_channels` -**Sync sources (inbox)**: `telegram`, `gmail:`, `imap: