feat(tsp-js): merged-spec (AAC) vectors, re-packed byte for byte

packages/tsp-js/CHANGELOG.md

@@ -14,6 +14,25 @@ For history before this file, see `git log` on `packages/tsp-js`.
   accept answers the invite we have outstanding (§7.2.2). `transition` sees
   only the state, so until now every client had to compare the digests itself
   or adopt an accept for an invite it never sent.
+- `@openvtc/vti-tsp-js/unsafe-testing`, a **test-only** subpath for
+  byte-reproducible packing: `__unsafeDeterministicPack`, `…PackInvite`,
+  `…PackAccept`, `…PackCancel`, `…PackNested` and `…PackRouted` take an
+  `__unsafeIkmE` (RFC 9180 DeriveKeyPair input) for the HPKE-Base ephemeral and
+  can write the NULL VID in the ESSR sender field. A fixed ephemeral key breaks
+  confidentiality; the subpath exists so the Appendix A vectors can be
+  reproduced and is not part of the documented API. The main entry point's
+  packers are unchanged and never take either knob. With it, all six HPKE-Base
+  vectors (`direct-hpke-base`, `control-rfi-direct`, `control-rfa-direct`,
+  `control-rfd`, `nested-direct`, `routed`) re-pack byte for byte.
+
+### Changed
+
+- The Appendix A test vectors are the merged specification's
+  ([tswg-tsp-specification@f5b8668](https://github.com/trustoverip/tswg-tsp-specification/commit/f5b8668952aabe8e541b535fcbdf589484ffc4f4)),
+  which carry `YTSP-AAC` — the marker this package packs — in place of the
+  pre-merge `YTSP-ABA` set. Every message and the control vectors' digests
+  changed; all still open and verify. A pre-merge `ABA` message stays pinned in
+  the tests: reading it is unchanged. Test-only; no library behaviour changed.
 
 ### Fixed
 

packages/tsp-js/README.md

@@ -108,15 +108,23 @@ Byte-compatibility is proven in three ways, because a round trip proves none of
 it — encoder and decoder agree with each other whatever they both get wrong,
 which is exactly the failure mode Rev 3's one-character changes produce:
 
-- **The specification's own Appendix A vectors** run as a test suite
-  (`tests/interop.spec-vectors.mjs`), fixed and external and produced by the
-  ToIP reference implementation. Every published vector is either exercised or
-  named as uncovered, so the list cannot quietly shrink.
+- **The specification's own Appendix A vectors** (`YTSP-AAC`, as merged at
+  [tswg-tsp-specification@f5b8668](https://github.com/trustoverip/tswg-tsp-specification/commit/f5b8668952aabe8e541b535fcbdf589484ffc4f4))
+  run as a test suite, fixed and external and produced by the ToIP reference
+  implementation. Every HPKE-Base vector both opens
+  (`tests/interop.spec-vectors.mjs`, `tests/control.spec-vectors.mjs`) and
+  **re-packs byte for byte** from its published `ikmE`
+  (`tests/interop.spec-vectors-repack.mjs`). Every published vector is either
+  exercised or named as uncovered, so the list cannot quietly shrink.
 - **Both directions against `affinidi-tsp`** — its Rev 2 vector unpacks here
   (`tests/interop.rust-vector.mjs`), and a message packed here unpacks there,
   thread digest included.
 - **Pinned bytes** for the deterministic parts of what we emit, since the sealed
-  message itself is not reproducible (HPKE draws a fresh ephemeral key).
+  message itself is not reproducible (HPKE draws a fresh ephemeral key). The
+  vector re-pack fixes that key through a test-only subpath,
+  `@openvtc/vti-tsp-js/unsafe-testing`, which is deliberately not part of the
+  API above: a fixed ephemeral key destroys confidentiality, and nothing but a
+  test reproducing a published vector has a reason to import it.
 
 The HPKE implementation is pinned three ways on every CI run: the official CFRG
 RFC 9180 `mode_auth` vector asserted in-tree (`tests/crypto.cfrg-vector.mjs` —

packages/tsp-js/package.json

@@ -13,6 +13,10 @@
     "./hpke": {
       "types": "./dist/crypto/hpke.d.ts",
       "import": "./dist/crypto/hpke.js"
+    },
+    "./unsafe-testing": {
+      "types": "./dist/unsafe-testing.d.ts",
+      "import": "./dist/unsafe-testing.js"
     }
   },
   "files": [

packages/tsp-js/src/cesr/wire.ts

@@ -58,10 +58,11 @@ export const LONG_COUNT_REV3 = DASH;
 /** TSP version `(major, minor)` this package **packs** — Rev 3, `YTSP-AAC`.
  *
  *  MAJOR.MINOR, two components rather than three: MINOR occupies the whole
- *  12-bit count. The published §9.1 text reads the three characters as MAJOR,
- *  MINOR, PATCH and gives `YTSP-ABA`; affinidi-tsp deliberately does not follow
+ *  12-bit count. Pre-merge drafts of §9.1 read the three characters as MAJOR,
+ *  MINOR, PATCH and gave `YTSP-ABA`; affinidi-tsp deliberately does not follow
  *  that reading, and neither do we — see that crate's `TSP_VERSION` for the
- *  argument. Nothing about interoperating depends on the choice: only MAJOR
+ *  argument. The merged specification's Appendix A vectors carry `YTSP-AAC`,
+ *  the marker packed here. Nothing about interoperating depends on the choice: only MAJOR
  *  gates processability, it is the same character either way, and no
  *  implementation refuses a message on MINOR. */
 export const TSP_VERSION = { major: 0, minor: 2 } as const;

packages/tsp-js/src/crypto/hpke-noble.ts

@@ -115,6 +115,27 @@ function dh(sk: Uint8Array, pk: Uint8Array): Uint8Array {
 // public `hpke.ts` wrappers deliberately do not forward it.
 type UnsafeFixedEphemeral = { __unsafeFixedEphemeralSk?: Uint8Array };
 
+/**
+ * §7.1.3 DeriveKeyPair for DHKEM(X25519, HKDF-SHA256): `ikm` → `(skE, pkE)`.
+ *
+ * Exported for test-vector verification only. RFC 9180 and TSP Appendix A
+ * publish the ephemeral as `ikmE`, the input to this function, so a vector's
+ * `enc` can only be reproduced by running it. X25519 needs no rejection
+ * sampling: every 32-byte string is a valid scalar (clamped at use).
+ *
+ * Nothing on a production path calls this. A random ephemeral is drawn from
+ * `x25519.utils.randomSecretKey()` directly, and deriving one from caller
+ * input is exactly the reuse the `__unsafe…` hooks warn about.
+ */
+export function deriveKeyPair(ikm: Uint8Array): { sk: Uint8Array; pk: Uint8Array } {
+  if (ikm.length < NX25519) {
+    throw new Error(`tsp: DeriveKeyPair input must be at least ${NX25519} bytes`);
+  }
+  const dkpPrk = labeledExtract(KEM_SUITE_ID, EMPTY, "dkp_prk", ikm);
+  const sk = labeledExpand(KEM_SUITE_ID, dkpPrk, "sk", EMPTY, NX25519);
+  return { sk, pk: x25519.getPublicKey(sk) };
+}
+
 /** §4.1 Encap (base mode). Exported for test-vector verification. */
 export function encap(recipientPk: Uint8Array, unsafe?: UnsafeFixedEphemeral): {
   sharedSecret: Uint8Array;

packages/tsp-js/src/rev3/direct.ts

@@ -25,6 +25,7 @@
 
 import * as wire from "../cesr/wire.js";
 import * as hpke from "../crypto/hpke.js";
+import * as noble from "../crypto/hpke-noble.js";
 import * as sign from "../crypto/sign.js";
 import { decodeEnvelope, encodeFields, finalizeFrame } from "./envelope.js";
 import {
@@ -87,6 +88,35 @@ export interface UnpackedMessage {
   threadDigest: Uint8Array;
 }
 
+/**
+ * Test-only knobs that make a pack byte-reproducible. **Never use outside
+ * tests.**
+ *
+ * Internal to the package: reached only through the `./unsafe-testing` subpath,
+ * and the public wrappers in `message/` deliberately do not forward it — the
+ * same arrangement as the `__unsafeFixedEphemeralSk` hook in `hpke-noble.ts`.
+ *
+ * Fixing the HPKE ephemeral key makes every message packed with it to the same
+ * recipient share one (key, base_nonce) pair. Under ChaCha20Poly1305 that leaks
+ * the XOR of the plaintexts and the Poly1305 one-time key: confidentiality and
+ * integrity both go. It exists so that the specification's Appendix A vectors,
+ * which publish their ephemeral as `ikmE`, can be reproduced byte for byte.
+ */
+export interface UnsafeDeterministicPack {
+  /** RFC 9180 §7.1.3 `DeriveKeyPair` input for the HPKE-Base ephemeral — the
+   *  `ikmE` Appendix A prints. At least 32 bytes. */
+  __unsafeIkmE: Uint8Array;
+  /** Write the NULL VID `4BAA` in the ESSR sender field instead of the sender's
+   *  VID. §9.2 permits it under HPKE-Base, and the published vectors use it;
+   *  this package's own stance is to always write the VID (see `payload.ts`),
+   *  so it is offered only here, for reproducing those vectors. */
+  nullPayloadSender?: boolean;
+}
+
+/** The ESSR sender field's content for this pack. */
+const payloadSender = (senderVid: string, unsafe?: UnsafeDeterministicPack): string =>
+  unsafe?.nullPayloadSender === true ? "" : senderVid;
+
 /** Encode the signature attachment: `-C23 -K22 B0 sig(64)`. */
 function encodeSignatureFrame(signature: Uint8Array, out: number[]): void {
   wire.encodeCount(wire.TSP_ATTACH_GRP, ATTACH_GROUP_QUADLETS, out);
@@ -129,10 +159,16 @@ async function sealFrame(
   fields: Uint8Array,
   frame: Uint8Array,
   keys: PackKeys,
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<Uint8Array> {
   // `aad` binds the ciphertext to the version and both VIDs; `info` is the
   // fixed protocol code.
-  const sealed = await hpke.sealBase(frame, fields, keys.receiverEncryptionKey, wire.TSP_INFO);
+  const sealed =
+    unsafe === undefined
+      ? await hpke.sealBase(frame, fields, keys.receiverEncryptionKey, wire.TSP_INFO)
+      : await noble.sealBase(frame, fields, keys.receiverEncryptionKey, wire.TSP_INFO, {
+          __unsafeFixedEphemeralSk: noble.deriveKeyPair(unsafe.__unsafeIkmE).sk,
+        });
 
   // Ciphertext field: `enc ‖ ct`, with the AEAD tag inside `ct`.
   const ciphertext = new Uint8Array(sealed.enc.length + sealed.ciphertext.length);
@@ -158,10 +194,11 @@ export async function packWithHops(
   senderVid: string,
   receiverVid: string,
   keys: PackKeys,
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<PackedMessage> {
   const fields = encodeFields(senderVid, receiverVid);
-  const { frame, threadDigest } = encodePayloadFrame(body, kind, hops, senderVid);
-  return { bytes: await sealFrame(fields, frame, keys), threadDigest };
+  const { frame, threadDigest } = encodePayloadFrame(body, kind, hops, payloadSender(senderVid, unsafe));
+  return { bytes: await sealFrame(fields, frame, keys, unsafe), threadDigest };
 }
 
 /**
@@ -178,10 +215,11 @@ export async function packControl(
   senderVid: string,
   receiverVid: string,
   keys: PackKeys,
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<PackedMessage> {
   const fields = encodeFields(senderVid, receiverVid);
-  const { frame, threadDigest } = encodeControlFrame(control, senderVid, fields);
-  return { bytes: await sealFrame(fields, frame, keys), threadDigest };
+  const { frame, threadDigest } = encodeControlFrame(control, payloadSender(senderVid, unsafe), fields);
+  return { bytes: await sealFrame(fields, frame, keys, unsafe), threadDigest };
 }
 
 /**
@@ -196,6 +234,7 @@ export function packInvite(
   receiverVid: string,
   keys: PackKeys,
   opts: { route?: string[]; nonce?: Uint8Array } = {},
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<PackedMessage> {
   return packControl(
     {
@@ -206,6 +245,7 @@ export function packInvite(
     senderVid,
     receiverVid,
     keys,
+    unsafe,
   );
 }
 
@@ -221,12 +261,14 @@ export function packAccept(
   senderVid: string,
   receiverVid: string,
   keys: PackKeys,
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<PackedMessage> {
   return packControl(
     { controlType: "accept", inReplyTo: inviteDigest, route: [] },
     senderVid,
     receiverVid,
     keys,
+    unsafe,
   );
 }
 
@@ -239,12 +281,14 @@ export function packCancel(
   senderVid: string,
   receiverVid: string,
   keys: PackKeys,
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<PackedMessage> {
   return packControl(
     { controlType: "cancel", inReplyTo: relationshipDigest, route: [] },
     senderVid,
     receiverVid,
     keys,
+    unsafe,
   );
 }
 
@@ -254,8 +298,9 @@ export function pack(
   senderVid: string,
   receiverVid: string,
   keys: PackKeys,
+  unsafe?: UnsafeDeterministicPack,
 ): Promise<PackedMessage> {
-  return packWithHops(body, "direct", [], senderVid, receiverVid, keys);
+  return packWithHops(body, "direct", [], senderVid, receiverVid, keys, unsafe);
 }
 
 /** Unpack a Rev 3 message. */

packages/tsp-js/src/revision.ts

@@ -19,9 +19,10 @@
 // Rev 2 is the only MINOR we match exactly. Everything else at MAJOR 0 is read
 // as Rev 3, because §9.1 makes MAJOR the field that gates processability and
 // MINOR one that no implementation may refuse a message on — the ToIP reference
-// discards MINOR entirely. Upstream Rev 3 ships `YTSP-ABA`, which is MINOR 64
-// under this (MAJOR.MINOR) reading and MINOR 1 / PATCH 0 under the published
-// three-component one; affinidi-tsp emits `AAC` = 2. Both must parse as Rev 3,
+// discards MINOR entirely. Pre-merge drafts of Rev 3 shipped `YTSP-ABA`, which
+// is MINOR 64 under this (MAJOR.MINOR) reading and MINOR 1 / PATCH 0 under the
+// three-component one; the merged specification's Appendix A vectors, like
+// affinidi-tsp and this package, carry `AAC` = 2. Both must parse as Rev 3,
 // and so must whatever the resolution of that argument turns out to be, so
 // enumerating known-good MINORs would be the wrong shape. See
 // `KNOWN_MINORS` for what the list is actually for.

packages/tsp-js/src/unsafe-testing.ts

@@ -0,0 +1,122 @@
+// @openvtc/vti-tsp-js/unsafe-testing — deterministic packing, for reproducing
+// published test vectors. **NOT FOR PRODUCTION USE.**
+//
+// Every function here packs exactly what its namesake in the main entry point
+// packs, except that the HPKE-Base ephemeral key is derived from a caller-fixed
+// `ikmE` (RFC 9180 §7.1.3 DeriveKeyPair) instead of drawn at random, and the
+// ESSR sender field may be written as the NULL VID. Ed25519 signatures are
+// already deterministic, so with those two fixed — and the invite nonce, which
+// `packInvite` already takes — the whole message is reproducible byte for byte.
+//
+// ── Why this is unsafe ──
+//
+// A fixed ephemeral key repeats the HPKE (key, base_nonce) pair for every
+// message sealed with it to the same recipient. Under ChaCha20Poly1305 that
+// leaks the XOR of the plaintexts and the Poly1305 one-time key, so both
+// confidentiality and integrity are gone. Anyone who learns `ikmE` can also
+// derive the ephemeral secret and open the message outright. The only safe
+// `ikmE` is one already published next to a test vector.
+//
+// It is a separate subpath, absent from the main entry point and from the
+// documented API, so that it cannot be reached by accident: importing it is a
+// statement that the caller is a test. Every export carries the `__unsafe`
+// prefix the package already uses for its fixed-ephemeral HPKE hook.
+
+import { MAX_HOPS } from "./cesr/wire.js";
+import {
+  pack,
+  packAccept,
+  packCancel,
+  packInvite,
+  packWithHops,
+  type PackKeys,
+  type PackedMessage,
+  type UnsafeDeterministicPack,
+} from "./rev3/direct.js";
+
+export type { PackKeys, PackedMessage, UnsafeDeterministicPack };
+
+/** `pack` (direct, `XSCS`) with a fixed ephemeral. Test vectors only. */
+export async function __unsafeDeterministicPack(
+  body: Uint8Array,
+  senderVid: string,
+  receiverVid: string,
+  keys: PackKeys,
+  unsafe: UnsafeDeterministicPack,
+): Promise<PackedMessage> {
+  return pack(body, senderVid, receiverVid, keys, requireIkm(unsafe));
+}
+
+/** `packInvite` (`XRFI`) with a fixed ephemeral and a caller nonce. Test vectors
+ *  only. The nonce is required here: a random one would defeat the point. */
+export async function __unsafeDeterministicPackInvite(
+  senderVid: string,
+  receiverVid: string,
+  keys: PackKeys,
+  opts: { route?: string[]; nonce: Uint8Array },
+  unsafe: UnsafeDeterministicPack,
+): Promise<PackedMessage> {
+  return packInvite(senderVid, receiverVid, keys, opts, requireIkm(unsafe));
+}
+
+/** `packAccept` (`XRFA`) with a fixed ephemeral. Test vectors only. */
+export async function __unsafeDeterministicPackAccept(
+  inviteDigest: Uint8Array,
+  senderVid: string,
+  receiverVid: string,
+  keys: PackKeys,
+  unsafe: UnsafeDeterministicPack,
+): Promise<PackedMessage> {
+  return packAccept(inviteDigest, senderVid, receiverVid, keys, requireIkm(unsafe));
+}
+
+/** `packCancel` (`XRFD`) with a fixed ephemeral. Test vectors only. */
+export async function __unsafeDeterministicPackCancel(
+  relationshipDigest: Uint8Array,
+  senderVid: string,
+  receiverVid: string,
+  keys: PackKeys,
+  unsafe: UnsafeDeterministicPack,
+): Promise<PackedMessage> {
+  return packCancel(relationshipDigest, senderVid, receiverVid, keys, requireIkm(unsafe));
+}
+
+/** `packNested` (`XHOP`, empty hop list) with a fixed ephemeral. Test vectors
+ *  only. */
+export async function __unsafeDeterministicPackNested(
+  innerBytes: Uint8Array,
+  senderVid: string,
+  intermediaryVid: string,
+  keys: PackKeys,
+  unsafe: UnsafeDeterministicPack,
+): Promise<PackedMessage> {
+  return packWithHops(innerBytes, "nested", [], senderVid, intermediaryVid, keys, requireIkm(unsafe));
+}
+
+/** `packRouted` (`XHOP`) with a fixed ephemeral. Test vectors only. Same route
+ *  bounds as `packRouted`. */
+export async function __unsafeDeterministicPackRouted(
+  inner: Uint8Array,
+  remainingRoute: string[],
+  senderVid: string,
+  firstHopVid: string,
+  keys: PackKeys,
+  unsafe: UnsafeDeterministicPack,
+): Promise<PackedMessage> {
+  if (remainingRoute.length === 0) {
+    throw new Error("tsp: a routed message requires at least one onward hop");
+  }
+  if (remainingRoute.length > MAX_HOPS) {
+    throw new Error(`tsp: route has ${remainingRoute.length} hops, exceeds max ${MAX_HOPS}`);
+  }
+  return packWithHops(inner, "routed", remainingRoute, senderVid, firstHopVid, keys, requireIkm(unsafe));
+}
+
+/** An absent `ikmE` would silently fall back to a random ephemeral and produce
+ *  bytes that match nothing — refuse it by name instead. */
+function requireIkm(unsafe: UnsafeDeterministicPack): UnsafeDeterministicPack {
+  if (!(unsafe?.__unsafeIkmE instanceof Uint8Array)) {
+    throw new Error("tsp: unsafe-testing packers need __unsafeIkmE (a Uint8Array)");
+  }
+  return unsafe;
+}