Live transport: crossbook connects to venues, and verifies against them

.github/workflows/ci.yml

@@ -85,6 +85,77 @@ jobs:
           ctest --test-dir build --build-config Release --output-on-failure
           -R "determinism|equivalence|checksum"
 
+  # The end-to-end claim, checked offline on every platform.
+  #
+  # `tests/fixtures/kraken_btcusd_l2.cbcap` is a verbatim recording of Kraken's v2 book
+  # channel. Replaying it must rebuild the book and match all 301 checksums the
+  # exchange published, with no network access at all. This is what turns the
+  # match rate in the README from a claim into a regression test — and it is only
+  # possible because crypto market data can be redistributed, which is why public
+  # equities order book projects ship without runnable data.
+  replay:
+    name: replay-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install OpenSSL (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install openssl@3
+          echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV"
+
+      - name: Configure
+        run: >
+          cmake -S . -B build-tools
+          -DCMAKE_BUILD_TYPE=Release
+          -DCROSSBOOK_BUILD_TESTS=ON
+          -DCROSSBOOK_BUILD_TOOLS=ON
+          -DCROSSBOOK_WERROR=ON
+
+      - name: Build
+        run: cmake --build build-tools --config Release --parallel
+
+      # The library-level assertion: every checksum, the depth contract, and the
+      # exact state hash.
+      - name: Fixture replay suite
+        run: >
+          ctest --test-dir build-tools --build-config Release --output-on-failure
+          -R "fixture|trim"
+
+      # The same claim through the actual binary, whose exit status is non-zero
+      # on any divergence. Proves the shipped tool works, not just the library.
+      - name: crossbook_verify --replay (Unix)
+        if: runner.os != 'Windows'
+        run: ./build-tools/tools/crossbook_verify --replay tests/fixtures/kraken_btcusd_l2.cbcap
+
+      - name: crossbook_verify --replay (Windows)
+        if: runner.os == 'Windows'
+        run: .\build-tools\tools\Release\crossbook_verify.exe --replay tests\fixtures\kraken_btcusd_l2.cbcap
+
+  # Connecting to a real venue, on demand only.
+  #
+  # Deliberately not on push: a green build must never depend on an exchange
+  # being reachable, and a test that fails because Kraken had a slow minute
+  # teaches people to ignore red. Run it by hand when the transport changes.
+  live:
+    name: live (manual)
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: Configure
+        run: >
+          cmake -S . -B build-live -DCMAKE_BUILD_TYPE=Release -DCROSSBOOK_BUILD_TOOLS=ON
+      - name: Build
+        run: cmake --build build-live --parallel
+      - name: Verify against Kraken for one minute
+        run: ./build-live/tools/crossbook_verify --venue kraken --symbol BTC/USD --seconds 60 --quiet
+
   sanitizers:
     name: sanitizers (asan+ubsan)
     runs-on: ubuntu-latest
@@ -135,6 +206,10 @@ jobs:
         run: ./build-fuzz/fuzz/fuzz_sequence -max_total_time=45 -print_final_stats=1
       - name: Fuzz JSON scanner and venue decoders
         run: ./build-fuzz/fuzz/fuzz_decode -max_total_time=90 -print_final_stats=1
+      # The frame reader is the only parser here fed bytes that have not been
+      # framed by anything we control, so it gets the longest budget.
+      - name: Fuzz websocket frame reader
+        run: ./build-fuzz/fuzz/fuzz_ws_frame -max_total_time=90 -print_final_stats=1
       - name: Upload crash artefacts
         if: failure()
         uses: actions/upload-artifact@v4

.gitignore

@@ -38,10 +38,13 @@ leak-*
 timeout-*
 oom-*
 
-# Captures — feed recordings are large and regenerable. Small committed
-# fixtures live in tests/fixtures/ and are added with `git add -f`.
+# Captures — feed recordings are large and regenerable, so they are ignored by
+# default. The exception is tests/fixtures/, which holds the small committed
+# captures CI replays offline; those are the evidence behind the match rate in
+# the README, so they are un-ignored explicitly rather than force-added by hand.
 captures/
 *.cbcap
+!tests/fixtures/*.cbcap
 
 # Benchmark output
 bench-results/

CMakeLists.txt

@@ -23,6 +23,8 @@ else()
 endif()
 
 option(CROSSBOOK_BUILD_TESTS "Build the test suite" ${CROSSBOOK_IS_TOP_LEVEL})
+option(CROSSBOOK_BUILD_TOOLS "Build the network tools (needs a TLS backend)"
+       ${CROSSBOOK_IS_TOP_LEVEL})
 option(CROSSBOOK_BUILD_BENCH "Build the benchmarks" OFF)
 option(CROSSBOOK_BUILD_FUZZ "Build the fuzz targets (clang only)" OFF)
 option(CROSSBOOK_WERROR "Treat warnings as errors" ${CROSSBOOK_IS_TOP_LEVEL})
@@ -90,6 +92,39 @@ if(CROSSBOOK_BUILD_BENCH)
     add_subdirectory(bench)
 endif()
 
+# ---------------------------------------------------------------------------
+# Transport and tools
+#
+# Kept behind an option and in a separate target because this is the only part
+# of the project with a platform dependency. Windows uses Schannel, which ships
+# with the OS; everywhere else uses OpenSSL. Nothing here is reachable from the
+# header-only library, so consuming `crossbook::crossbook` still costs nothing.
+#
+# When there is no TLS backend the tools are skipped with a message rather than
+# built without TLS. Talking to an exchange in plaintext is not a degraded mode,
+# it is a different and much worse program.
+# ---------------------------------------------------------------------------
+if(CROSSBOOK_BUILD_TOOLS)
+    if(WIN32)
+        set(CROSSBOOK_HAVE_TLS ON)
+    else()
+        find_package(OpenSSL)
+        if(OpenSSL_FOUND)
+            set(CROSSBOOK_HAVE_TLS ON)
+        else()
+            set(CROSSBOOK_HAVE_TLS OFF)
+            message(STATUS
+                "crossbook: OpenSSL not found - skipping the network tools. "
+                "Install libssl-dev (Debian/Ubuntu) or openssl (Homebrew) to build them.")
+        endif()
+    endif()
+
+    if(CROSSBOOK_HAVE_TLS)
+        add_subdirectory(src/net)
+        add_subdirectory(tools)
+    endif()
+endif()
+
 if(CROSSBOOK_BUILD_FUZZ)
     add_subdirectory(fuzz)
 endif()

README.md

@@ -82,6 +82,83 @@ Divergences are never summarised away. Every mismatch is
 [recorded with a cause](include/crossbook/divergence.hpp), because a match rate
 without an enumerated remainder isn't evidence.
 
+## The measurement
+
+`crossbook_verify` connects to Kraken, rebuilds the book, and recomputes the
+exchange's CRC32 over local state on every update. No API key, no account.
+
+```bash
+cmake --preset release && cmake --build build/release
+./build/release/tools/crossbook_verify --venue kraken --symbol BTC/USD --seconds 180
+```
+
+A three-minute run on BTC/USD, 2026-08-01:
+
+```text
+frames               2939
+  applied            2759
+checksums verified   2759
+checksum mismatches  0
+match rate           100.000000%  (2759 of 2759)
+state hash           7648057f6909c67a
+```
+
+The exit status is the point: any mismatch, any decode failure, any resync and it
+exits non-zero.
+
+**And you can check this without taking my word for it.** A recorded minute of
+that feed is committed at [`tests/fixtures/kraken_btcusd_l2.cbcap`](tests/fixtures/) — 72 KB
+of verbatim Kraken bytes — and replays offline, deterministically, on every
+platform:
+
+```bash
+./build/release/tools/crossbook_verify --replay tests/fixtures/kraken_btcusd_l2.cbcap
+# 301 of 301 checksums matched, state hash 080281c2dd87183f
+```
+
+CI runs exactly that on Linux, macOS and Windows on every push, and asserts the
+state hash is bit-identical across all three. That is why the number above is a
+regression test rather than an anecdote. In equities the equivalent data is
+licensed and cannot be redistributed, which is why every public ITCH order book
+repository ships without runnable data and asks to be believed.
+
+Recording your own is one command, and works for Binance too:
+
+```bash
+./build/release/tools/crossbook_capture --venue kraken --symbol ETH/USD \
+    --seconds 60 --out eth.cbcap
+./build/release/tools/crossbook_verify --replay eth.cbcap
+```
+
+`crossbook_capture` deliberately does not decode anything. Recording and
+interpreting are separate jobs, and keeping them separate is what makes a
+capture evidence rather than output: change the book implementation and the
+capture is still the bytes the exchange sent, so the new implementation can be
+held to them.
+
+### Live verification found a real bug
+
+Worth stating plainly, because it is the reason the verifier exists.
+
+The first live run reported **98.66%** — 4 of 298 updates mismatched — and the
+book held 20 bid levels for a subscription that asked for 10.
+
+The cause is a gap in the depth-limited contract that unit tests do not reach.
+Kraken reports cancellations, so a reader that handles those looks correct. It
+never reports that a level fell out of the top ten because a *better* level
+arrived — from the venue's side there is nothing to say. Those orphaned levels
+sit below the checksummed depth doing no harm, until enough removals near the
+touch promote one back into view, and then the checksum fails on an update that
+was itself perfectly fine. The divergence is minutes away from its cause.
+
+The fix is [`BasicL2Book::trim`](include/crossbook/book.hpp), and the reason it
+is trustworthy is the same reason the bug was found: replaying the committed
+capture with trimming disabled still fails, and
+[a test asserts that it does](tests/test_fixture_replay.cpp).
+
+No amount of testing the book against itself would have surfaced this. The
+exchange's checksum did, in sixty seconds.
+
 ## Performance
 
 Measured, with the methodology stated, because a number without one is noise.
@@ -246,14 +323,39 @@ for (std::string_view frame : frames_from_your_transport) {
 - [x] Feed handler with resnapshot recovery and staleness detection
 - [x] HDR histogram with coordinated-omission correction
 - [x] Open-loop replay harness measuring against the schedule
-- [x] 139 test cases / 841 assertions, `-Werror`, ASan + UBSan, four fuzz targets
-- [ ] Websocket transport — **not built.** Bring your own frames.
+- [x] Websocket transport: RFC 6455 framing, TLS via Schannel and OpenSSL
+- [x] Depth-limited book trimming — found by live verification, not by a test
+- [x] Capture and byte-exact offline replay, with a recorded capture committed
+- [x] 196 test cases, `-Werror`, ASan + UBSan, five fuzz targets
+- [ ] Automatic Binance REST snapshot reconciliation in the tool (v0.3)
 - [ ] L3 / order-by-order books (v0.4)
 - [ ] Cross-venue consolidated book and `executable_size` (v0.4)
 
-The library decodes, verifies, and recovers; it does not open sockets. Feed it
-frames from whatever transport you like — that boundary keeps the correctness
-core testable offline and free of a TLS dependency.
+The library still decodes, verifies, and recovers without opening a socket: the
+transport is a separate, optional target, and consuming `crossbook::crossbook`
+pulls in no TLS stack. `-DCROSSBOOK_BUILD_TOOLS=OFF` drops it entirely. That
+boundary is what keeps the correctness core testable offline — which is also how
+the whole stack gets tested, since CI verifies a recorded capture rather than a
+live venue.
+
+### Dependencies, and the deliberate lack of them
+
+The library has none: standard library only. The JSON reader, the RFC 6455
+codec, SHA-1 and base64 are written here rather than pulled in, and two of those
+are load-bearing rather than stylistic.
+
+The JSON reader returns the **untouched wire token** for every value, because
+Kraken's checksum is computed over the digits as the venue spelled them — a
+parser that hands back a `double` has already destroyed the information needed
+to verify the book. And SHA-1 is here so that `Sec-WebSocket-Accept` is actually
+*verified* rather than assumed; that check is what proves the peer parsed the
+upgrade request rather than merely answering 101, and it is the step most
+hand-rolled clients skip.
+
+TLS is the one thing that cannot reasonably be written here, so each platform's
+own is used: Schannel on Windows, which ships with the OS, and OpenSSL
+elsewhere. `cmake --build` therefore produces a working client on a stock
+Windows machine with nothing installed.
 
 ## What this is not
 

fuzz/CMakeLists.txt

@@ -17,3 +17,4 @@ add_fuzz_target(fuzz_parse_fixed fuzz_parse_fixed.cpp)
 add_fuzz_target(fuzz_book fuzz_book.cpp)
 add_fuzz_target(fuzz_sequence fuzz_sequence.cpp)
 add_fuzz_target(fuzz_decode fuzz_decode.cpp)
+add_fuzz_target(fuzz_ws_frame fuzz_ws_frame.cpp)