From 03d2ffe693782cf2799032dcb0c3534374d20b08 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:32:21 +0800 Subject: [PATCH 01/60] feat(apple): add realtime protocol client to SitrepKit Add a Swift mirror of proto/realtime/ (envelope + 14 message types), SpaceState with the SPEC.md section 6.4 deterministic folding rules, a RealtimeResumeGate implementing the section 6.2/6.3 resume/delta/snapshot gating as a transport-agnostic state machine, and a RealtimeClient actor that drives one WebSocket connection through hello -> subscribe -> resume, interest-lease renewal, ping/pong, and reconnect with backoff. Viewer-role only, matching SitrepApp's use as an observer rather than a task source. --- .../Realtime/RealtimeAuthorization.swift | 46 ++ .../SitrepKit/Realtime/RealtimeBodies.swift | 775 ++++++++++++++++++ .../SitrepKit/Realtime/RealtimeClient.swift | 429 ++++++++++ .../SitrepKit/Realtime/RealtimeCommon.swift | 483 +++++++++++ .../SitrepKit/Realtime/RealtimeEndpoint.swift | 27 + .../SitrepKit/Realtime/RealtimeEnvelope.swift | 163 ++++ .../Realtime/RealtimeResumeGate.swift | 119 +++ .../SitrepKit/Realtime/SpaceState.swift | 144 ++++ 8 files changed, 2186 insertions(+) create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEnvelope.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift new file mode 100644 index 0000000..f573676 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift @@ -0,0 +1,46 @@ +import Foundation + +/// A narrow mirror of SPEC.md §10.1's authorization matrix, scoped to what a +/// *client* (source or viewer) may legally send — the rules a conformant +/// server enforces against inbound client traffic. SitrepApp only ever acts +/// as a viewer, so `RealtimeClient` only ever needs the viewer column, but +/// this stays general so the two role-tagged invalid fixtures +/// (`fixtures/invalid/role-client-hello-accept.json`, +/// `fixtures/invalid/role-viewer-command-origin-server.json`) can be checked +/// exactly as `proto/realtime/tools/validate.js` checks them, and so a +/// future source-role implementation (e.g. the menu bar app, stage 5) can +/// reuse it instead of re-deriving the matrix. +/// +/// This is a client-side sanity net, not a substitute for the server's own +/// enforcement: the server is the authority per §10.1; this only prevents +/// *this* implementation from ever constructing or accepting a frame the +/// matrix forbids. +public enum RealtimeAuthorization { + /// Whether a client authenticated with `role` may send a frame of this + /// shape. Only encodes the rules exercised by the two role-tagged + /// invalid fixtures (hello stage/role, command origin) — see SPEC.md + /// §10.1 for the full matrix, most of which server-side implementations + /// need and a viewer-only client does not. + public static func clientMaySend(_ frame: RealtimeFrame, as role: RTDeviceRole) -> Bool { + switch frame { + case .hello(let e): + // §9.1: a client may only ever send stage "offer"; "accept" + // belongs exclusively to the server. + if case .accept = e.body { return false } + return true + case .command(let e): + // §8: a client-sent command with origin "server" is always + // unauthorized, regardless of role or action. + return e.body.origin != .server + case .resume, .subscribe, .unsubscribe, .interestRenew: + return role == .viewer + case .taskEvent, .messageEvent, .metricFrame: + return role == .source + case .snapshot, .delta, .configEvent: + // Server-only in both directions; no client may send these. + return false + case .ack, .error: + return true + } + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift new file mode 100644 index 0000000..07f6934 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift @@ -0,0 +1,775 @@ +import Foundation + +// One type per messages/*.schema.json body. Ordered to match SPEC.md §4's +// table. Each body's Decodable init enforces exactly the constraints its +// schema file expresses (required fields via non-optional properties, +// everything else via explicit checks); unrecognized extra keys are ignored +// for free by Codable, matching the body-level leniency in SPEC.md §15. + +// MARK: - hello + +/// `hello{body.stage: "offer"}` — sent by the connecting device, in both +/// directions of the realtime protocol (only source/viewer send offers). +public struct HelloOffer: Codable, Sendable, Equatable { + public var deviceID: String + public var role: RTDeviceRole + public var protocolVersions: [Int] + public var capabilities: [String] + + public init(deviceID: String, role: RTDeviceRole, protocolVersions: [Int], capabilities: [String] = []) { + self.deviceID = deviceID + self.role = role + self.protocolVersions = protocolVersions + self.capabilities = capabilities + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + role = try c.decode(RTDeviceRole.self, forKey: .role) + protocolVersions = try RealtimeValidation.uniqueItems( + RealtimeValidation.nonEmpty(c.decode([Int].self, forKey: .protocolVersions), field: "protocol_versions"), + field: "protocol_versions") + capabilities = try c.decodeIfPresent([String].self, forKey: .capabilities) ?? [] + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(role, forKey: .role) + try c.encode(protocolVersions, forKey: .protocolVersions) + try c.encode(capabilities, forKey: .capabilities) + } + + private enum CodingKeys: String, CodingKey { + case deviceID = "device_id", role + case protocolVersions = "protocol_versions", capabilities + } +} + +/// `hello{body.stage: "accept"}` — server-only. +public struct HelloAccept: Codable, Sendable, Equatable { + public var protocolVersion: Int + public var sessionID: String + public var heartbeatIntervalMs: Int + public var capabilities: [String] + + public init(protocolVersion: Int, sessionID: String, heartbeatIntervalMs: Int, capabilities: [String] = []) { + self.protocolVersion = protocolVersion + self.sessionID = sessionID + self.heartbeatIntervalMs = heartbeatIntervalMs + self.capabilities = capabilities + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + protocolVersion = try RealtimeValidation.minimum( + c.decode(Int.self, forKey: .protocolVersion), 1, field: "protocol_version") + sessionID = try RealtimeValidation.length(c.decode(String.self, forKey: .sessionID), 1...64, field: "session_id") + heartbeatIntervalMs = try RealtimeValidation.range( + c.decode(Int.self, forKey: .heartbeatIntervalMs), 1000...300_000, field: "heartbeat_interval_ms") + capabilities = try c.decodeIfPresent([String].self, forKey: .capabilities) ?? [] + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(protocolVersion, forKey: .protocolVersion) + try c.encode(sessionID, forKey: .sessionID) + try c.encode(heartbeatIntervalMs, forKey: .heartbeatIntervalMs) + try c.encode(capabilities, forKey: .capabilities) + } + + private enum CodingKeys: String, CodingKey { + case protocolVersion = "protocol_version", sessionID = "session_id" + case heartbeatIntervalMs = "heartbeat_interval_ms", capabilities + } +} + +/// `hello.body`: `oneOf(offer, accept)` discriminated by `stage`. +public enum HelloBody: RealtimeBody { + case offer(HelloOffer) + case accept(HelloAccept) + + public static let messageType = "hello" + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + let stage = try c.decode(String.self, forKey: .stage) + switch stage { + case "offer": self = .offer(try HelloOffer(from: decoder)) + case "accept": self = .accept(try HelloAccept(from: decoder)) + default: throw RealtimeDecodingError.malformed("hello.stage must be 'offer' or 'accept': \(stage)") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .offer(let o): + try c.encode("offer", forKey: .stage) + try o.encode(to: encoder) + case .accept(let a): + try c.encode("accept", forKey: .stage) + try a.encode(to: encoder) + } + } + + private enum CodingKeys: String, CodingKey { case stage } +} + +// MARK: - resume + +public struct ResumeBody: RealtimeBody { + public static let messageType = "resume" + + /// The space_revision this viewer last fully applied. 0 means "no prior + /// state, send me a snapshot" (§6.3). + public var lastRevision: Int + + public init(lastRevision: Int) { self.lastRevision = lastRevision } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + lastRevision = try RealtimeValidation.minimum( + c.decode(Int.self, forKey: .lastRevision), 0, field: "last_revision") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(lastRevision, forKey: .lastRevision) + } + + private enum CodingKeys: String, CodingKey { case lastRevision = "last_revision" } +} + +// MARK: - snapshot + +public struct SnapshotBody: RealtimeBody { + public static let messageType = "snapshot" + + public var revision: Int + public var part: Int + public var final: Bool + public var tasks: [RTTaskState] + public var metrics: [RTMetricSample] + public var messages: [RTMessageRecord] + public var automations: [RTAutomationState] + + public init(revision: Int, part: Int, final: Bool, tasks: [RTTaskState], metrics: [RTMetricSample], + messages: [RTMessageRecord], automations: [RTAutomationState]) { + self.revision = revision + self.part = part + self.final = final + self.tasks = tasks + self.metrics = metrics + self.messages = messages + self.automations = automations + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + revision = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .revision), 0, field: "revision") + part = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .part), 1, field: "part") + final = try c.decode(Bool.self, forKey: .final) + tasks = try c.decode([RTTaskState].self, forKey: .tasks) + metrics = try c.decode([RTMetricSample].self, forKey: .metrics) + messages = try c.decode([RTMessageRecord].self, forKey: .messages) + automations = try c.decode([RTAutomationState].self, forKey: .automations) + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(revision, forKey: .revision) + try c.encode(part, forKey: .part) + try c.encode(final, forKey: .final) + try c.encode(tasks, forKey: .tasks) + try c.encode(metrics, forKey: .metrics) + try c.encode(messages, forKey: .messages) + try c.encode(automations, forKey: .automations) + } + + private enum CodingKeys: String, CodingKey { case revision, part, final, tasks, metrics, messages, automations } +} + +// MARK: - delta + +/// One entry of `delta.body.events`: `oneOf` tagged by `event_type`. An +/// `event_type` this build does not recognize is kept as `.unknown` rather +/// than failing the whole delta — a future minor version could add a new +/// reliable event kind, and dropping just that one entry (while the +/// revision still advances by the full `events.length`, per §6.2's exact +/// arithmetic) is safer than losing revision continuity over it. This is a +/// defensive interpretation beyond what SPEC.md §15 states explicitly (which +/// only names *envelope* `type`, not nested `event_type`) — flagged in the +/// handoff as a place a protocol owner may want to rule on explicitly. +public enum DeltaEvent: Sendable, Equatable { + case taskEvent(TaskEventBody) + case messageEvent(MessageEventBody) + case configEvent(ConfigEventBody) + case unknown(eventType: String) +} + +extension DeltaEvent: Codable { + private enum CodingKeys: String, CodingKey { case eventType = "event_type", event } + + public init(from decoder: Decoder) throws { + let dyn = try decoder.container(keyedBy: AnyKey.self) + let present = Set(dyn.allKeys.map(\.stringValue)) + guard present == ["event_type", "event"] else { + throw RealtimeDecodingError.malformed("delta event must carry exactly event_type/event: \(present)") + } + let c = try decoder.container(keyedBy: CodingKeys.self) + let type = try c.decode(String.self, forKey: .eventType) + switch type { + case "task.event": self = .taskEvent(try c.decode(TaskEventBody.self, forKey: .event)) + case "message.event": self = .messageEvent(try c.decode(MessageEventBody.self, forKey: .event)) + case "config.event": self = .configEvent(try c.decode(ConfigEventBody.self, forKey: .event)) + default: self = .unknown(eventType: type) + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .taskEvent(let b): + try c.encode("task.event", forKey: .eventType) + try c.encode(b, forKey: .event) + case .messageEvent(let b): + try c.encode("message.event", forKey: .eventType) + try c.encode(b, forKey: .event) + case .configEvent(let b): + try c.encode("config.event", forKey: .eventType) + try c.encode(b, forKey: .event) + case .unknown(let type): + // Never constructed from an outbound encode in this client (we + // never originate delta), but keep encode total. + try c.encode(type, forKey: .eventType) + try c.encode([String: String](), forKey: .event) + } + } +} + +public struct DeltaBody: RealtimeBody { + public static let messageType = "delta" + + public var fromRevision: Int + public var toRevision: Int + public var events: [DeltaEvent] + + public init(fromRevision: Int, toRevision: Int, events: [DeltaEvent]) { + self.fromRevision = fromRevision + self.toRevision = toRevision + self.events = events + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + fromRevision = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .fromRevision), 0, field: "from_revision") + toRevision = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .toRevision), 0, field: "to_revision") + events = try c.decode([DeltaEvent].self, forKey: .events) + // §6.2: to_revision - from_revision MUST equal events.length exactly. + guard toRevision - fromRevision == events.count else { + throw RealtimeDecodingError.malformed( + "delta arithmetic violated: to_revision(\(toRevision)) - from_revision(\(fromRevision)) != events.length(\(events.count))") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(fromRevision, forKey: .fromRevision) + try c.encode(toRevision, forKey: .toRevision) + try c.encode(events, forKey: .events) + } + + private enum CodingKeys: String, CodingKey { + case fromRevision = "from_revision", toRevision = "to_revision", events + } +} + +// MARK: - ack + +public struct DeviceSeqPair: Codable, Sendable, Equatable { + public var deviceID: String + public var deviceSeq: Int + + public init(deviceID: String, deviceSeq: Int) { + self.deviceID = deviceID + self.deviceSeq = deviceSeq + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + deviceSeq = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .deviceSeq), 1, field: "device_seq") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(deviceSeq, forKey: .deviceSeq) + } + + private enum CodingKeys: String, CodingKey { case deviceID = "device_id", deviceSeq = "device_seq" } +} + +public struct LeaseInfo: Codable, Sendable, Equatable { + /// Absolute deadline; the viewer must `interest.renew` before this to + /// keep the lease alive (SPEC.md §7). + public var expiresAt: Int + + public init(expiresAt: Int) { self.expiresAt = expiresAt } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + expiresAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .expiresAt), field: "lease.expires_at") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(expiresAt, forKey: .expiresAt) + } + + private enum CodingKeys: String, CodingKey { case expiresAt = "expires_at" } +} + +public struct AckBody: RealtimeBody { + public static let messageType = "ack" + + public var acked: [DeviceSeqPair]? + public var inReplyTo: String? + public var lease: LeaseInfo? + + public init(acked: [DeviceSeqPair]? = nil, inReplyTo: String? = nil, lease: LeaseInfo? = nil) { + self.acked = acked + self.inReplyTo = inReplyTo + self.lease = lease + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + acked = try c.decodeIfPresent([DeviceSeqPair].self, forKey: .acked) + if let acked { try _ = RealtimeValidation.nonEmpty(acked, field: "acked") } + inReplyTo = try c.decodeIfPresent(String.self, forKey: .inReplyTo) + lease = try c.decodeIfPresent(LeaseInfo.self, forKey: .lease) + // anyOf(acked, in_reply_to) — see fixtures/invalid/ack-neither-acked-nor-in-reply-to.json. + guard acked != nil || inReplyTo != nil else { + throw RealtimeDecodingError.malformed("ack.body must carry 'acked' and/or 'in_reply_to'") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encodeIfPresent(acked, forKey: .acked) + try c.encodeIfPresent(inReplyTo, forKey: .inReplyTo) + try c.encodeIfPresent(lease, forKey: .lease) + } + + private enum CodingKeys: String, CodingKey { case acked, inReplyTo = "in_reply_to", lease } +} + +// MARK: - task.event + +public struct TaskEventBody: RealtimeBody { + public static let messageType = "task.event" + + public var deviceID: String + public var deviceSeq: Int + public var taskID: String + public var kind: RTTaskKind + public var occurredAt: Int + public var title: String? + public var percent: Int? + public var step: String? + public var message: String? + public var display: RTDisplayHints? + + public init(deviceID: String, deviceSeq: Int, taskID: String, kind: RTTaskKind, occurredAt: Int, + title: String? = nil, percent: Int? = nil, step: String? = nil, message: String? = nil, + display: RTDisplayHints? = nil) { + self.deviceID = deviceID + self.deviceSeq = deviceSeq + self.taskID = taskID + self.kind = kind + self.occurredAt = occurredAt + self.title = title + self.percent = percent + self.step = step + self.message = message + self.display = display + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + deviceSeq = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .deviceSeq), 1, field: "device_seq") + taskID = try RealtimeValidation.length(c.decode(String.self, forKey: .taskID), 1...256, field: "task_id") + kind = try c.decode(RTTaskKind.self, forKey: .kind) + occurredAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .occurredAt), field: "occurred_at") + title = try c.decodeIfPresent(String.self, forKey: .title) + if let title { try _ = RealtimeValidation.maxLength(title, 2048, field: "title") } + percent = try c.decodeIfPresent(Int.self, forKey: .percent) + if let percent { try _ = RealtimeValidation.range(percent, 0...100, field: "percent") } + step = try c.decodeIfPresent(String.self, forKey: .step) + if let step { try _ = RealtimeValidation.maxLength(step, 2048, field: "step") } + message = try c.decodeIfPresent(String.self, forKey: .message) + if let message { try _ = RealtimeValidation.maxLength(message, 2048, field: "message") } + display = try c.decodeIfPresent(RTDisplayHints.self, forKey: .display) + // if kind == progress then percent required. + if kind == .progress && percent == nil { + throw RealtimeDecodingError.malformed("task.event.body.percent required when kind == 'progress'") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(deviceSeq, forKey: .deviceSeq) + try c.encode(taskID, forKey: .taskID) + try c.encode(kind, forKey: .kind) + try c.encode(occurredAt, forKey: .occurredAt) + try c.encodeIfPresent(title, forKey: .title) + try c.encodeIfPresent(percent, forKey: .percent) + try c.encodeIfPresent(step, forKey: .step) + try c.encodeIfPresent(message, forKey: .message) + try c.encodeIfPresent(display, forKey: .display) + } + + private enum CodingKeys: String, CodingKey { + case deviceID = "device_id", deviceSeq = "device_seq", taskID = "task_id", kind + case occurredAt = "occurred_at", title, percent, step, message, display + } +} + +// MARK: - message.event + +public struct MessageEventBody: RealtimeBody { + public static let messageType = "message.event" + + public var deviceID: String + public var deviceSeq: Int + public var messageID: String + public var level: RTMessageLevel + public var text: String + public var occurredAt: Int + public var automationID: String? + + public init(deviceID: String, deviceSeq: Int, messageID: String, level: RTMessageLevel, text: String, + occurredAt: Int, automationID: String? = nil) { + self.deviceID = deviceID + self.deviceSeq = deviceSeq + self.messageID = messageID + self.level = level + self.text = text + self.occurredAt = occurredAt + self.automationID = automationID + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + deviceSeq = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .deviceSeq), 1, field: "device_seq") + messageID = try RealtimeValidation.length(c.decode(String.self, forKey: .messageID), 1...128, field: "message_id") + level = try c.decode(RTMessageLevel.self, forKey: .level) + text = try RealtimeValidation.maxLength(c.decode(String.self, forKey: .text), 2048, field: "text") + occurredAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .occurredAt), field: "occurred_at") + automationID = try c.decodeIfPresent(String.self, forKey: .automationID) + if let automationID { try _ = RealtimeValidation.maxLength(automationID, 128, field: "automation_id") } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(deviceSeq, forKey: .deviceSeq) + try c.encode(messageID, forKey: .messageID) + try c.encode(level, forKey: .level) + try c.encode(text, forKey: .text) + try c.encode(occurredAt, forKey: .occurredAt) + try c.encodeIfPresent(automationID, forKey: .automationID) + } + + private enum CodingKeys: String, CodingKey { + case deviceID = "device_id", deviceSeq = "device_seq", messageID = "message_id" + case level, text, occurredAt = "occurred_at", automationID = "automation_id" + } +} + +// MARK: - metric.frame + +public struct MetricFrameBody: RealtimeBody { + public static let messageType = "metric.frame" + + public var deviceID: String + public var metrics: [RTMetricSample] + + public init(deviceID: String, metrics: [RTMetricSample]) { + self.deviceID = deviceID + self.metrics = metrics + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + let decoded = try c.decode([RTMetricSample].self, forKey: .metrics) + metrics = try RealtimeValidation.maxItems( + RealtimeValidation.nonEmpty(decoded, field: "metrics"), 64, field: "metrics") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(metrics, forKey: .metrics) + } + + private enum CodingKeys: String, CodingKey { case deviceID = "device_id", metrics } +} + +// MARK: - config.event + +public struct ConfigEventBody: RealtimeBody { + public static let messageType = "config.event" + + public enum Kind: String, Codable, Sendable, Equatable { + case upserted = "automation.upserted" + case removed = "automation.removed" + } + + public var kind: Kind + public var automationID: String + public var automation: RTAutomationState? + public var occurredAt: Int + + public init(kind: Kind, automationID: String, automation: RTAutomationState? = nil, occurredAt: Int) { + self.kind = kind + self.automationID = automationID + self.automation = automation + self.occurredAt = occurredAt + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + kind = try c.decode(Kind.self, forKey: .kind) + automationID = try RealtimeValidation.length(c.decode(String.self, forKey: .automationID), 1...128, field: "automation_id") + automation = try c.decodeIfPresent(RTAutomationState.self, forKey: .automation) + occurredAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .occurredAt), field: "occurred_at") + if kind == .upserted && automation == nil { + throw RealtimeDecodingError.malformed("config.event.body.automation required when kind == 'automation.upserted'") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(kind, forKey: .kind) + try c.encode(automationID, forKey: .automationID) + try c.encodeIfPresent(automation, forKey: .automation) + try c.encode(occurredAt, forKey: .occurredAt) + } + + private enum CodingKeys: String, CodingKey { + case kind, automationID = "automation_id", automation, occurredAt = "occurred_at" + } +} + +// MARK: - subscribe / unsubscribe / interest.renew + +/// Shared body shape for `subscribe`, sent as a viewer declares (or wholly +/// replaces) its device's interest lease. +public struct SubscribeBody: RealtimeBody { + public static let messageType = "subscribe" + + public var topics: [RTTopic] + + public init(topics: [RTTopic] = []) { self.topics = topics } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + let decoded = try c.decodeIfPresent([RTTopic].self, forKey: .topics) ?? [] + topics = try RealtimeValidation.uniqueItems(decoded, field: "topics") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(topics, forKey: .topics) + } + + private enum CodingKeys: String, CodingKey { case topics } +} + +public struct UnsubscribeBody: RealtimeBody { + public static let messageType = "unsubscribe" + + public init() {} + public init(from decoder: Decoder) throws { _ = try decoder.container(keyedBy: AnyKey.self) } + public func encode(to encoder: Encoder) throws { _ = encoder.container(keyedBy: AnyKey.self) } +} + +/// Same shape as `SubscribeBody` (interest.renew.schema.json literally +/// `$ref`s subscribe's body) but a distinct Swift type since `messageType` +/// differs and each schema file is versioned independently. +public struct InterestRenewBody: RealtimeBody { + public static let messageType = "interest.renew" + + public var topics: [RTTopic] + + public init(topics: [RTTopic] = []) { self.topics = topics } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + let decoded = try c.decodeIfPresent([RTTopic].self, forKey: .topics) ?? [] + topics = try RealtimeValidation.uniqueItems(decoded, field: "topics") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(topics, forKey: .topics) + } + + private enum CodingKeys: String, CodingKey { case topics } +} + +// MARK: - command + +public struct CommandBody: RealtimeBody { + public static let messageType = "command" + + public var commandID: String + public var origin: RTCommandOrigin + public var issuedByDeviceID: String? + public var action: RTCommandAction + public var taskID: String? + public var automationID: String? + public var targetDeviceID: String? + public var ttlMs: Int + public var params: [String: JSONValue]? + + public init(commandID: String, origin: RTCommandOrigin, issuedByDeviceID: String? = nil, + action: RTCommandAction, taskID: String? = nil, automationID: String? = nil, + targetDeviceID: String? = nil, ttlMs: Int, params: [String: JSONValue]? = nil) { + self.commandID = commandID + self.origin = origin + self.issuedByDeviceID = issuedByDeviceID + self.action = action + self.taskID = taskID + self.automationID = automationID + self.targetDeviceID = targetDeviceID + self.ttlMs = ttlMs + self.params = params + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + commandID = try RealtimeValidation.length(c.decode(String.self, forKey: .commandID), 1...64, field: "command_id") + origin = try c.decode(RTCommandOrigin.self, forKey: .origin) + issuedByDeviceID = try c.decodeIfPresent(String.self, forKey: .issuedByDeviceID) + action = try c.decode(RTCommandAction.self, forKey: .action) + taskID = try c.decodeIfPresent(String.self, forKey: .taskID) + if let taskID { try _ = RealtimeValidation.length(taskID, 1...256, field: "task_id") } + automationID = try c.decodeIfPresent(String.self, forKey: .automationID) + if let automationID { try _ = RealtimeValidation.length(automationID, 1...128, field: "automation_id") } + targetDeviceID = try c.decodeIfPresent(String.self, forKey: .targetDeviceID) + ttlMs = try RealtimeValidation.range(c.decode(Int.self, forKey: .ttlMs), 1...86_400_000, field: "ttl_ms") + params = try c.decodeIfPresent([String: JSONValue].self, forKey: .params) + + // §8 per-action required/forbidden field matrix. + switch action { + case .pause, .resume, .stop: + guard origin == .viewer else { + throw RealtimeDecodingError.malformed("command.action \(action) requires origin 'viewer'") + } + guard issuedByDeviceID != nil else { + throw RealtimeDecodingError.malformed("command.action \(action) requires issued_by_device_id") + } + guard taskID != nil else { + throw RealtimeDecodingError.malformed("command.action \(action) requires task_id") + } + guard automationID == nil else { + throw RealtimeDecodingError.malformed("command.action \(action) forbids automation_id") + } + case .runNow: + guard origin == .viewer else { + throw RealtimeDecodingError.malformed("command.action run_now requires origin 'viewer'") + } + guard issuedByDeviceID != nil else { + throw RealtimeDecodingError.malformed("command.action run_now requires issued_by_device_id") + } + guard automationID != nil else { + throw RealtimeDecodingError.malformed("command.action run_now requires automation_id") + } + guard taskID == nil else { + throw RealtimeDecodingError.malformed("command.action run_now forbids task_id") + } + case .throttle, .resumeRate: + guard origin == .server else { + throw RealtimeDecodingError.malformed("command.action \(action) requires origin 'server'") + } + guard taskID == nil && automationID == nil else { + throw RealtimeDecodingError.malformed("command.action \(action) forbids task_id and automation_id") + } + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(commandID, forKey: .commandID) + try c.encode(origin, forKey: .origin) + try c.encodeIfPresent(issuedByDeviceID, forKey: .issuedByDeviceID) + try c.encode(action, forKey: .action) + try c.encodeIfPresent(taskID, forKey: .taskID) + try c.encodeIfPresent(automationID, forKey: .automationID) + try c.encodeIfPresent(targetDeviceID, forKey: .targetDeviceID) + try c.encode(ttlMs, forKey: .ttlMs) + try c.encodeIfPresent(params, forKey: .params) + } + + private enum CodingKeys: String, CodingKey { + case commandID = "command_id", origin, issuedByDeviceID = "issued_by_device_id", action + case taskID = "task_id", automationID = "automation_id", targetDeviceID = "target_device_id" + case ttlMs = "ttl_ms", params + } +} + +// MARK: - error + +public struct ErrorBody: RealtimeBody { + public static let messageType = "error" + + public var code: RTErrorCode + public var message: String + public var inReplyTo: String? + public var retryable: Bool + public var fatal: Bool + + public init(code: RTErrorCode, message: String, inReplyTo: String? = nil, retryable: Bool, fatal: Bool) { + self.code = code + self.message = message + self.inReplyTo = inReplyTo + self.retryable = retryable + self.fatal = fatal + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + code = try c.decode(RTErrorCode.self, forKey: .code) + message = try RealtimeValidation.maxLength(c.decode(String.self, forKey: .message), 500, field: "message") + inReplyTo = try c.decodeIfPresent(String.self, forKey: .inReplyTo) + retryable = try c.decode(Bool.self, forKey: .retryable) + fatal = try c.decode(Bool.self, forKey: .fatal) + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(code, forKey: .code) + try c.encode(message, forKey: .message) + try c.encodeIfPresent(inReplyTo, forKey: .inReplyTo) + try c.encode(retryable, forKey: .retryable) + try c.encode(fatal, forKey: .fatal) + } + + private enum CodingKeys: String, CodingKey { + case code, message, inReplyTo = "in_reply_to", retryable, fatal + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift new file mode 100644 index 0000000..72c8d18 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift @@ -0,0 +1,429 @@ +import Foundation + +/// Errors `RealtimeClient` raises itself (as opposed to a `RTErrorCode` the +/// server sent, which arrives wrapped in `.serverError`). +public enum RealtimeClientError: Error, Sendable { + case timeout + case protocolViolation(String) + case serverError(ErrorBody) +} + +/// A viewer-role client for the Sitrep realtime protocol +/// (`proto/realtime/SPEC.md`). Owns one WebSocket connection at a time and +/// drives the mandatory `hello → subscribe → resume` gate, snapshot chunk +/// reassembly, delta application, interest-lease renewal, and reconnect +/// with backoff. SitrepApp is a viewer only (it observes a Mac Agent's +/// tasks; it never executes or reports them), so this client never sends +/// `task.event`/`message.event`/`metric.frame` uplink — only `hello`, +/// `subscribe`, `unsubscribe`, `resume`, `interest.renew`, and viewer +/// `command`s. +/// +/// All connection state lives inside this actor; callers observe it through +/// the three `AsyncStream`s (`phases`, `states`, `notices`) rather than by +/// reading actor-isolated properties directly. +public actor RealtimeClient { + /// Connection lifecycle, per the brief: idle/connecting/handshaking/ + /// subscribed/live/failed. + public enum Phase: Sendable, Equatable { + case idle + case connecting + case handshaking + case subscribed + case live + case failed(String) + } + + /// One-off, non-state notifications a UI layer may want to surface. + public enum Notice: Sendable, Equatable { + /// This device's credential/connection was superseded by another + /// connection (SPEC.md §9.4). SitrepApp never intentionally opens a + /// second concurrent connection, so this always means "surprising" + /// per §9.4's client guidance — surface a one-time, non-disruptive + /// banner ("this session may be in use elsewhere"), don't reconnect + /// in a tight loop. + case superseded + case serverError(RTErrorCode, message: String) + case revisionGap(from: Int) + /// WS has failed for `fallbackAfterFailures` consecutive backoff + /// cycles; the caller should fall back to low-frequency HTTP + /// polling until `.recovered`. + case fellBackToPolling(reason: String) + /// WS reached `.live` again after a fallback; the caller should stop + /// the HTTP polling fallback. + case recovered + } + + public struct Configuration: Sendable { + /// The realtime WebSocket endpoint (see `APIClient.realtimeURL`). + public var url: URL + /// Presented once at connection establishment (SPEC.md §10), as an + /// `Authorization: Bearer` header — never repeated inside an + /// envelope afterward. + public var token: String? + public var deviceID: String + public var protocolVersions: [Int] + public var topics: [RTTopic] + + public init(url: URL, token: String?, deviceID: String, protocolVersions: [Int] = [1], topics: [RTTopic] = []) { + self.url = url + self.token = token + self.deviceID = deviceID + self.protocolVersions = protocolVersions + self.topics = topics + } + } + + /// Consecutive failed connection attempts (full backoff cycles) before + /// signalling `.fellBackToPolling`. + public var fallbackAfterFailures = 3 + + private let configuration: Configuration + + private var runTask: Task? + private var socket: URLSessionWebSocketTask? + private var leaseRenewalTask: Task? + private var watchdogTask: Task? + + /// All resume/delta/snapshot gating and folding decisions are delegated + /// to this shared, transport-agnostic state machine (SPEC.md §6.2/§6.3) + /// — see `RealtimeResumeGateScenarioTests` for the same logic driven + /// directly by scenario fixtures. + private var gate: RealtimeResumeGate + private var phase: Phase = .idle { + didSet { phaseContinuation.yield(phase) } + } + + private var consecutiveFailures = 0 + private var firedFallback = false + private var lastInboundAt: Int = 0 + + private let phaseContinuation: AsyncStream.Continuation + public nonisolated let phases: AsyncStream + private let noticeContinuation: AsyncStream.Continuation + public nonisolated let notices: AsyncStream + private let stateContinuation: AsyncStream.Continuation + public nonisolated let states: AsyncStream + + public init(configuration: Configuration, initialState: SpaceState = SpaceState()) { + self.configuration = configuration + self.gate = RealtimeResumeGate(state: initialState) + var pc: AsyncStream.Continuation! + self.phases = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { pc = $0 } + self.phaseContinuation = pc + var nc: AsyncStream.Continuation! + self.notices = AsyncStream(bufferingPolicy: .bufferingNewest(8)) { nc = $0 } + self.noticeContinuation = nc + var sc: AsyncStream.Continuation! + self.states = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { sc = $0 } + self.stateContinuation = sc + } + + public var currentState: SpaceState { gate.state } + public var currentPhase: Phase { phase } + + // MARK: - Lifecycle + + /// Idempotent: does nothing if already running. Call when the app enters + /// the foreground. + public func start() { + guard runTask == nil else { return } + consecutiveFailures = 0 + firedFallback = false + runTask = Task { [weak self] in await self?.runLoop() } + } + + /// Tears down the connection and stops reconnecting. The interest lease + /// naturally expires on the server side (SPEC.md §7) — this deliberately + /// does not send `unsubscribe`, matching the brief: backgrounding closes + /// the connection and lets the lease lapse rather than releasing it + /// explicitly. Call when the app leaves the foreground. + public func stop() { + runTask?.cancel() + runTask = nil + teardownConnection() + phase = .idle + } + + private func runLoop() async { + while !Task.isCancelled { + do { + try await connectOnce() + } catch is CancellationError { + break + } catch { + consecutiveFailures += 1 + phase = .failed(String(describing: error)) + if consecutiveFailures >= fallbackAfterFailures && !firedFallback { + firedFallback = true + noticeContinuation.yield(.fellBackToPolling(reason: String(describing: error))) + } + } + guard !Task.isCancelled else { break } + if consecutiveFailures > 0 { + try? await Task.sleep(for: .seconds(backoffDelay(attempt: consecutiveFailures))) + } + } + phase = .idle + } + + private func backoffDelay(attempt: Int) -> Double { + let capped = min(attempt, 6) + let base = min(30.0, pow(2.0, Double(capped))) + return base + Double.random(in: 0...(base * 0.3)) + } + + // MARK: - One connection's lifetime + + private func connectOnce() async throws { + phase = .connecting + var request = URLRequest(url: configuration.url) + if let token = configuration.token, !token.isEmpty { + request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + } + let socket = URLSession.shared.webSocketTask(with: request) + self.socket = socket + socket.resume() + defer { teardownConnection() } + + phase = .handshaking + let accept = try await performHandshake(on: socket) + startHeartbeat(on: socket, intervalMs: accept.heartbeatIntervalMs) + + let subscribeID = newEnvelopeID() + try await sendFrame( + .subscribe(Envelope(id: subscribeID, ts: nowMs(), body: SubscribeBody(topics: configuration.topics))), + on: socket) + let ackFrame = try await receiveFrame(on: socket) + guard case .ack(let ackEnvelope) = ackFrame, let lease = ackEnvelope.body.lease else { + if case .error(let e) = ackFrame { throw RealtimeClientError.serverError(e.body) } + throw RealtimeClientError.protocolViolation("expected subscribe ack with lease, got \(ackFrame)") + } + phase = .subscribed + scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket) + + let lastRevision = gate.state.revision + gate.beganResume(lastRevision: lastRevision) + try await sendFrame( + .resume(Envelope(id: newEnvelopeID(), ts: nowMs(), body: ResumeBody(lastRevision: lastRevision))), + on: socket) + + while !Task.isCancelled { + let frame = try await receiveFrame(on: socket) + try await dispatch(frame, on: socket) + } + } + + private func teardownConnection() { + socket?.cancel(with: .normalClosure, reason: nil) + socket = nil + leaseRenewalTask?.cancel() + leaseRenewalTask = nil + watchdogTask?.cancel() + watchdogTask = nil + } + + // MARK: - Handshake (§9) + + private func performHandshake(on socket: URLSessionWebSocketTask) async throws -> HelloAccept { + let offer = HelloOffer(deviceID: configuration.deviceID, role: .viewer, protocolVersions: configuration.protocolVersions) + try await sendFrame(.hello(Envelope(id: newEnvelopeID(), ts: nowMs(), body: .offer(offer))), on: socket) + // §9.1: the client MUST NOT send anything else until the accept + // arrives; we simply don't send anything else here. Recommended 10s + // timeout on the client side. + let frame = try await withTimeout(seconds: 10) { try await self.receiveFrame(on: socket) } + switch frame { + case .hello(let env): + guard case .accept(let accept) = env.body else { + throw RealtimeClientError.protocolViolation("client received a hello offer, expected accept") + } + return accept + case .error(let env): + throw RealtimeClientError.serverError(env.body) + default: + throw RealtimeClientError.protocolViolation("expected hello accept, got \(frame)") + } + } + + // MARK: - Frame dispatch (post-handshake, post-subscribe) + + private func dispatch(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask) async throws { + switch frame { + case .snapshot(let env): + switch gate.receiveSnapshotChunk(env.body) { + case .applied: + markDeltaEligible() + stateContinuation.yield(gate.state) + case .snapshotChunkBuffered, .discarded: + break + case .gapDetected: + break // not reachable for a snapshot chunk + case .malformedSequence(let reason): + throw RealtimeClientError.protocolViolation("snapshot chunk out of sequence: \(reason)") + } + case .delta(let env): + switch gate.receiveDelta(env.body) { + case .applied: + markDeltaEligible() + stateContinuation.yield(gate.state) + case .discarded: + break + case .gapDetected(let resendLastRevision): + noticeContinuation.yield(.revisionGap(from: resendLastRevision)) + try await sendFrame( + .resume(Envelope(id: newEnvelopeID(), ts: nowMs(), body: ResumeBody(lastRevision: resendLastRevision))), + on: socket) + case .snapshotChunkBuffered, .malformedSequence: + break // not reachable for a delta + } + case .metricFrame(let env): + gate.receiveMetricFrame(env.body) + stateContinuation.yield(gate.state) + case .ack(let env): + if let lease = env.body.lease { + scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket) + } + case .error(let env): + try await handleError(env.body, on: socket) + case .command, .hello, .subscribe, .unsubscribe, .interestRenew, .taskEvent, .messageEvent, .configEvent, .resume: + // A conformant server never sends any of these to a viewer + // (§10.1: server-only or source-only in the other direction). + // Ignore defensively rather than tearing down the connection. + break + } + } + + private func markDeltaEligible() { + phase = .live + if consecutiveFailures > 0 { consecutiveFailures = 0 } + if firedFallback { + firedFallback = false + noticeContinuation.yield(.recovered) + } + } + + private func handleError(_ body: ErrorBody, on socket: URLSessionWebSocketTask) async throws { + // §6.3: revision_unavailable answering our outstanding resume ⇒ + // retry with last_revision: 0, staying in the "awaiting reply" state. + if gate.receiveErrorAnswersOutstandingResume(body) { + try await sendFrame( + .resume(Envelope(id: newEnvelopeID(), ts: nowMs(), body: ResumeBody(lastRevision: 0))), + on: socket) + return + } + + if body.code == .superseded { + noticeContinuation.yield(.superseded) + } else { + noticeContinuation.yield(.serverError(body.code, message: body.message)) + } + + if body.fatal { + throw RealtimeClientError.serverError(body) + } + } + + // MARK: - Interest lease (§7) + + /// Renews at 1/3 of the window remaining before `expiresAt`, as + /// instructed. Rescheduled every time we learn a fresh `expiresAt` (the + /// initial subscribe ack, or any subsequent renew ack) — see `dispatch`. + private func scheduleLeaseRenewal(expiresAt: Int, on socket: URLSessionWebSocketTask) { + leaseRenewalTask?.cancel() + let window = max(expiresAt - nowMs(), 0) + let fireAfterMs = window - window / 3 + leaseRenewalTask = Task { [weak self] in + try? await Task.sleep(for: .milliseconds(max(fireAfterMs, 0))) + guard !Task.isCancelled else { return } + try? await self?.sendFrame( + .interestRenew(Envelope( + id: self?.newEnvelopeID() ?? UUID().uuidString, ts: self?.nowMs() ?? 0, + body: InterestRenewBody(topics: self?.configuration.topics ?? []))), + on: socket) + } + } + + // MARK: - Heartbeat (§9.3) + + private func startHeartbeat(on socket: URLSessionWebSocketTask, intervalMs: Int) { + watchdogTask?.cancel() + lastInboundAt = nowMs() + watchdogTask = Task { [weak self] in + while !Task.isCancelled { + try? await Task.sleep(for: .milliseconds(intervalMs)) + guard !Task.isCancelled, let self else { return } + try? await self.sendPing(on: socket) + let idleFor = await self.millisecondsSinceLastInbound() + if idleFor > intervalMs * 2 { + // RECOMMENDED in §9.3: no pong (or any traffic) within + // 2x the interval ⇒ treat the connection as dead. + socket.cancel(with: .abnormalClosure, reason: nil) + return + } + } + } + } + + private func sendPing(on socket: URLSessionWebSocketTask) async throws { + try await socket.send(.string("ping")) + } + + private func millisecondsSinceLastInbound() -> Int { nowMs() - lastInboundAt } + + // MARK: - Transport helpers + + private func sendFrame(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask) async throws { + let data = try frame.encoded() + guard let text = String(data: data, encoding: .utf8) else { + throw RealtimeClientError.protocolViolation("failed to encode outbound frame as UTF-8") + } + try await socket.send(.string(text)) + } + + /// Reads the next real envelope off the socket, transparently answering + /// `ping` with `pong` and dropping stray `pong`/unrecognized-type frames + /// (§9.3, §15) without surfacing them to `dispatch`. + private func receiveFrame(on socket: URLSessionWebSocketTask) async throws -> RealtimeFrame { + while true { + let message = try await socket.receive() + lastInboundAt = nowMs() + let data: Data + switch message { + case .string(let text): + if text == "ping" { + try? await socket.send(.string("pong")) + continue + } + if text == "pong" { + continue + } + guard let d = text.data(using: .utf8) else { continue } + data = d + case .data(let d): + data = d + @unknown default: + continue + } + do { + return try RealtimeFrame.decode(data) + } catch RealtimeDecodingError.unknownType { + continue // §15: ignore, do not treat as malformed + } + } + } + + private func withTimeout(seconds: Double, operation: @escaping @Sendable () async throws -> T) async throws -> T { + try await withThrowingTaskGroup(of: T.self) { group in + group.addTask { try await operation() } + group.addTask { + try await Task.sleep(for: .seconds(seconds)) + throw RealtimeClientError.timeout + } + guard let result = try await group.next() else { throw RealtimeClientError.timeout } + group.cancelAll() + return result + } + } + + private func nowMs() -> Int { Int(Date().timeIntervalSince1970 * 1000) } + private func newEnvelopeID() -> String { UUID().uuidString } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift new file mode 100644 index 0000000..605f7dc --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift @@ -0,0 +1,483 @@ +import Foundation + +// Swift mirror of proto/realtime/. This is a derived artifact: field names, +// enum values, and validation rules MUST match the frozen schemas under +// proto/realtime/ byte-for-byte in semantics. Any deviation is a bug, not a +// style choice. Do not "clean up" a name here without checking the schema +// first. +// +// Strictness split (SPEC.md §3, §15): +// - The envelope's top level (type/id/ts/body) is permanently strict: an +// unknown top-level field is always `malformed`. `Envelope` enforces this +// explicitly (see RealtimeEnvelope.swift). +// - Fields *inside* `body` are lenient at runtime: a receiver MUST ignore +// unrecognized body fields. Swift's `Decodable` already does this for any +// JSON key not named in a type's `CodingKeys`, so the body structs below +// get that leniency for free and only need to enforce the fields this +// version of the protocol actually defines (required-ness, bounds, enums). + +/// Every error a fixture in `proto/realtime/fixtures/invalid/` is expected to +/// produce when decoded by a conformant receiver. +public enum RealtimeDecodingError: Error, Sendable, Equatable, CustomStringConvertible { + /// The envelope or body violates a constraint this protocol version + /// defines (§13 `malformed`): an unknown top-level envelope field, a + /// missing required field, a value outside its schema bound, or a + /// same-message cross-field rule (e.g. `command` action/field + /// exclusivity, `ack` requiring `acked` or `in_reply_to`). + case malformed(String) + /// `envelope.type` is not one of the 14 known message types. Per §15 + /// this is NOT malformed — a receiver MUST ignore it (forward + /// compatibility with a future additive message type). + case unknownType(String) + + public var description: String { + switch self { + case .malformed(let reason): "realtime malformed: \(reason)" + case .unknownType(let type): "realtime unknown type (ignored per SPEC.md §15): \(type)" + } + } +} + +/// Schema-bound checks mirroring `common.schema.json` `$defs`. Every check +/// here corresponds to a named constraint in that file so a reviewer can +/// diff the two side by side. +enum RealtimeValidation { + /// `$defs/unix_ms_timestamp`: 13-digit millisecond bound. This is the + /// mechanical guard against the historical seconds/milliseconds bug — + /// see `fixtures/invalid/message-event-timestamp-in-seconds.json`. + static func unixMs(_ value: Int, field: String) throws -> Int { + guard value >= 1_000_000_000_000 && value <= 9_999_999_999_999 else { + throw RealtimeDecodingError.malformed("\(field) is not a Unix ms timestamp (13 digits): \(value)") + } + return value + } + + static func maxLength(_ s: String, _ n: Int, field: String) throws -> String { + guard s.count <= n else { + throw RealtimeDecodingError.malformed("\(field) exceeds max length \(n): \(s.count) chars") + } + return s + } + + static func length(_ s: String, _ range: ClosedRange, field: String) throws -> String { + guard range.contains(s.count) else { + throw RealtimeDecodingError.malformed("\(field) length \(s.count) outside \(range)") + } + return s + } + + static func minimum(_ v: Int, _ lower: Int, field: String) throws -> Int { + guard v >= lower else { + throw RealtimeDecodingError.malformed("\(field) \(v) below minimum \(lower)") + } + return v + } + + static func range(_ v: Int, _ r: ClosedRange, field: String) throws -> Int { + guard r.contains(v) else { + throw RealtimeDecodingError.malformed("\(field) \(v) outside \(r)") + } + return v + } + + static func nonEmpty(_ arr: [T], field: String) throws -> [T] { + guard !arr.isEmpty else { + throw RealtimeDecodingError.malformed("\(field) must not be empty") + } + return arr + } + + static func maxItems(_ arr: [T], _ n: Int, field: String) throws -> [T] { + guard arr.count <= n else { + throw RealtimeDecodingError.malformed("\(field) exceeds max items \(n): \(arr.count)") + } + return arr + } + + static func uniqueItems(_ arr: [T], field: String) throws -> [T] { + guard Set(arr).count == arr.count else { + throw RealtimeDecodingError.malformed("\(field) contains duplicate items") + } + return arr + } + + static func pattern(_ s: String, _ pattern: String, field: String) throws -> String { + guard s.range(of: pattern, options: .regularExpression) != nil, + s.range(of: pattern, options: .regularExpression) == s.startIndex..: Sendable, Equatable { + public var id: String + public var ts: Int + public var body: Body + + public init(id: String, ts: Int, body: Body) { + self.id = id + self.ts = ts + self.body = body + } +} + +extension Envelope: Codable { + private enum CodingKeys: String, CodingKey { case type, id, ts, body } + + public init(from decoder: Decoder) throws { + // Strict top-level check: exactly {type, id, ts, body}, no more, no + // less. A fixed CodingKeys-keyed container can't see keys it doesn't + // name, so this uses a dynamic-key container to enumerate what's + // actually present in the JSON object. + let dyn = try decoder.container(keyedBy: AnyKey.self) + let present = Set(dyn.allKeys.map(\.stringValue)) + let allowed: Set = ["type", "id", "ts", "body"] + guard present == allowed else { + let extra = present.subtracting(allowed) + let missing = allowed.subtracting(present) + throw RealtimeDecodingError.malformed( + "envelope must carry exactly type/id/ts/body" + + (extra.isEmpty ? "" : "; unexpected field(s): \(extra.sorted().joined(separator: ", "))") + + (missing.isEmpty ? "" : "; missing field(s): \(missing.sorted().joined(separator: ", "))") + ) + } + + let c = try decoder.container(keyedBy: CodingKeys.self) + let type = try c.decode(String.self, forKey: .type) + guard type == Body.messageType else { + throw RealtimeDecodingError.malformed("expected envelope.type '\(Body.messageType)', got '\(type)'") + } + id = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .id), "^[A-Za-z0-9_-]{1,64}$", field: "id") + ts = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .ts), field: "ts") + body = try c.decode(Body.self, forKey: .body) + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(Body.messageType, forKey: .type) + try c.encode(id, forKey: .id) + try c.encode(ts, forKey: .ts) + try c.encode(body, forKey: .body) + } +} + +/// One of the 14 message types (SPEC.md §4), fully decoded. `RealtimeFrame` +/// is the unit `RealtimeClient` sends/receives; a value round-trips through +/// `decode(_:)`/`encoded()` byte-for-byte in semantics (field order may +/// differ, JSON does not guarantee it). +public enum RealtimeFrame: Sendable, Equatable { + case hello(Envelope) + case resume(Envelope) + case snapshot(Envelope) + case delta(Envelope) + case ack(Envelope) + case taskEvent(Envelope) + case messageEvent(Envelope) + case metricFrame(Envelope) + case configEvent(Envelope) + case subscribe(Envelope) + case unsubscribe(Envelope) + case interestRenew(Envelope) + case command(Envelope) + case error(Envelope) + + private struct TypePeek: Decodable { let type: String } + + /// Decodes one JSON frame (one WebSocket text frame carries exactly one + /// envelope, SPEC.md §11). Throws `RealtimeDecodingError.unknownType` + /// for a `type` outside the 14 known ones (§15: a receiver MUST ignore + /// this, not treat it as malformed) and `.malformed` for every other + /// schema/sequence violation. + public static func decode(_ data: Data) throws -> RealtimeFrame { + let decoder = JSONDecoder() + let peek: TypePeek + do { + peek = try decoder.decode(TypePeek.self, from: data) + } catch { + throw RealtimeDecodingError.malformed("envelope missing/invalid 'type': \(error)") + } + switch peek.type { + case HelloBody.messageType: return .hello(try decoder.decode(Envelope.self, from: data)) + case ResumeBody.messageType: return .resume(try decoder.decode(Envelope.self, from: data)) + case SnapshotBody.messageType: return .snapshot(try decoder.decode(Envelope.self, from: data)) + case DeltaBody.messageType: return .delta(try decoder.decode(Envelope.self, from: data)) + case AckBody.messageType: return .ack(try decoder.decode(Envelope.self, from: data)) + case TaskEventBody.messageType: return .taskEvent(try decoder.decode(Envelope.self, from: data)) + case MessageEventBody.messageType: return .messageEvent(try decoder.decode(Envelope.self, from: data)) + case MetricFrameBody.messageType: return .metricFrame(try decoder.decode(Envelope.self, from: data)) + case ConfigEventBody.messageType: return .configEvent(try decoder.decode(Envelope.self, from: data)) + case SubscribeBody.messageType: return .subscribe(try decoder.decode(Envelope.self, from: data)) + case UnsubscribeBody.messageType: return .unsubscribe(try decoder.decode(Envelope.self, from: data)) + case InterestRenewBody.messageType: return .interestRenew(try decoder.decode(Envelope.self, from: data)) + case CommandBody.messageType: return .command(try decoder.decode(Envelope.self, from: data)) + case ErrorBody.messageType: return .error(try decoder.decode(Envelope.self, from: data)) + default: + throw RealtimeDecodingError.unknownType(peek.type) + } + } + + public func encoded() throws -> Data { + let encoder = JSONEncoder() + switch self { + case .hello(let e): return try encoder.encode(e) + case .resume(let e): return try encoder.encode(e) + case .snapshot(let e): return try encoder.encode(e) + case .delta(let e): return try encoder.encode(e) + case .ack(let e): return try encoder.encode(e) + case .taskEvent(let e): return try encoder.encode(e) + case .messageEvent(let e): return try encoder.encode(e) + case .metricFrame(let e): return try encoder.encode(e) + case .configEvent(let e): return try encoder.encode(e) + case .subscribe(let e): return try encoder.encode(e) + case .unsubscribe(let e): return try encoder.encode(e) + case .interestRenew(let e): return try encoder.encode(e) + case .command(let e): return try encoder.encode(e) + case .error(let e): return try encoder.encode(e) + } + } + + /// The envelope `id` shared by every case, for logging/`in_reply_to` + /// correlation. + public var envelopeID: String { + switch self { + case .hello(let e): e.id + case .resume(let e): e.id + case .snapshot(let e): e.id + case .delta(let e): e.id + case .ack(let e): e.id + case .taskEvent(let e): e.id + case .messageEvent(let e): e.id + case .metricFrame(let e): e.id + case .configEvent(let e): e.id + case .subscribe(let e): e.id + case .unsubscribe(let e): e.id + case .interestRenew(let e): e.id + case .command(let e): e.id + case .error(let e): e.id + } + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift new file mode 100644 index 0000000..0b0af14 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift @@ -0,0 +1,119 @@ +import Foundation + +/// The pure decision logic behind SPEC.md §6.2/§6.3's resume/delta/snapshot +/// gating and §6.4's folding, extracted from `RealtimeClient` so it has +/// exactly one implementation shared by the live client and by tests that +/// drive it directly with decoded fixtures — no WebSocket transport +/// required. `RealtimeClient` is the only thing that owns a socket; this +/// type only ever sees already-decoded `RealtimeFrame` bodies and reports +/// what happened, never performs I/O itself. +public struct RealtimeResumeGate: Sendable, Equatable { + public private(set) var state: SpaceState + /// True once this connection has received a snapshot-or-delta reply to + /// its most recent `resume` (SPEC.md §6.2/§6.3) — i.e. it may now accept + /// live deltas via the ordinary `from_revision` comparison. + public private(set) var deltaEligible = false + /// Non-nil while a resume reply (initial, or a gap-triggered re-resume) + /// is outstanding; holds the `last_revision` that was requested, so a + /// racing stray delta (one that doesn't match) can be told apart from + /// the actual reply. + public private(set) var awaitingResumeReply: Int? + private var pendingSnapshotChunks: [SnapshotBody] = [] + + public init(state: SpaceState = SpaceState()) { + self.state = state + } + + public enum Outcome: Sendable, Equatable { + /// The frame was folded into `state` (a snapshot final chunk, a + /// delta that matched, or the resume reply itself). + case applied + /// Silently ignored per SPEC.md §6.3: a pre-reply delta, a stale + /// `from_revision < C` delta, or a stray that doesn't match an + /// outstanding resume request. + case discarded + /// `from_revision > C` on an already-eligible connection: the + /// caller MUST send a fresh `resume{last_revision: resendLastRevision}`. + case gapDetected(resendLastRevision: Int) + /// A non-final snapshot chunk was buffered; nothing applied yet. + case snapshotChunkBuffered + /// The peer violated §6.2's chunk sequencing (SPEC.md: "a viewer + /// that receives any non-ping/pong envelope between chunks MUST + /// treat it as malformed"; the same applies to a genuinely + /// out-of-sequence chunk). The caller should close and reconnect. + case malformedSequence(String) + } + + /// Call once, immediately after sending `resume{last_revision: N}` — + /// including the very first resume on a fresh connection and any + /// gap-triggered re-resume. + public mutating func beganResume(lastRevision: Int) { + awaitingResumeReply = lastRevision + deltaEligible = false + pendingSnapshotChunks = [] + } + + public mutating func receiveSnapshotChunk(_ chunk: SnapshotBody) -> Outcome { + if let first = pendingSnapshotChunks.first { + guard chunk.revision == first.revision, chunk.part == pendingSnapshotChunks.count + 1 else { + return .malformedSequence( + "expected revision \(first.revision) part \(pendingSnapshotChunks.count + 1), got revision \(chunk.revision) part \(chunk.part)") + } + } else { + guard chunk.part == 1 else { + return .malformedSequence("snapshot chunk sequence must start at part 1, got \(chunk.part)") + } + } + pendingSnapshotChunks.append(chunk) + guard chunk.final else { return .snapshotChunkBuffered } + + let aggregated = SnapshotBody( + revision: chunk.revision, part: 1, final: true, + tasks: pendingSnapshotChunks.flatMap(\.tasks), + metrics: pendingSnapshotChunks.flatMap(\.metrics), + messages: pendingSnapshotChunks.flatMap(\.messages), + automations: pendingSnapshotChunks.flatMap(\.automations)) + pendingSnapshotChunks = [] + state.apply(snapshot: aggregated) + awaitingResumeReply = nil + deltaEligible = true + return .applied + } + + public mutating func receiveDelta(_ body: DeltaBody) -> Outcome { + if let requested = awaitingResumeReply { + guard body.fromRevision == requested else { return .discarded } + state.apply(delta: body) + awaitingResumeReply = nil + deltaEligible = true + return .applied + } + guard deltaEligible else { return .discarded } + + if body.fromRevision < state.revision { + return .discarded + } else if body.fromRevision == state.revision { + state.apply(delta: body) + return .applied + } else { + let resendFrom = state.revision + awaitingResumeReply = resendFrom + deltaEligible = false + return .gapDetected(resendLastRevision: resendFrom) + } + } + + public mutating func receiveMetricFrame(_ body: MetricFrameBody) { + state.apply(metricFrame: body) + } + + /// `true` if this error is the reply to our outstanding resume + /// (`revision_unavailable`, SPEC.md §6.3's decision table): the caller + /// MUST retry with `resume{last_revision: 0}`. The gate stays in the + /// "awaiting reply" state (now for revision 0) either way. + public mutating func receiveErrorAnswersOutstandingResume(_ body: ErrorBody) -> Bool { + guard awaitingResumeReply != nil, body.code == .revisionUnavailable else { return false } + awaitingResumeReply = 0 + return true + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift new file mode 100644 index 0000000..fcaffa8 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift @@ -0,0 +1,144 @@ +import Foundation + +/// Unified realtime read model for one space: `tasks`/`messages`/ +/// `automations`/`metrics` plus the local `revision` (`C` in SPEC.md §6.3). +/// +/// Two convergent paths build this exact same shape (§6.4): `apply(snapshot:)` +/// replaces it wholesale from a resume reply's folded state, and +/// `apply(delta:)` folds reliable events into it one at a time. Both MUST +/// reach identical `tasks`/`automations` state for the same event history — +/// that equivalence is what `SpaceStateFoldingTests` asserts. +/// +/// This is a plain value type with no transport/connection knowledge; +/// `RealtimeClient` owns the connection state machine and mutates one of +/// these as frames arrive. +public struct SpaceState: Sendable, Equatable { + /// Local current revision (`C`). 0 before the first resume reply. + public private(set) var revision: Int + + public private(set) var tasks: [String: RTTaskState] = [:] + public private(set) var automations: [String: RTAutomationState] = [:] + /// Oldest-first. Capped locally (`messageWindow`) so a very long-lived + /// connection's delta-appended history doesn't grow unbounded — the + /// server's own snapshot window (SPEC.md §6.4 recommends N=200) bounds + /// what a fresh snapshot carries, but nothing bounds how many + /// `message.event`s a live connection can accumulate over days, so the + /// client enforces its own cap defensively. This is a client-local + /// memory bound, not a protocol requirement. + public private(set) var messages: [RTMessageRecord] = [] + /// Latest known sample per metric_id. `metric.frame` is best-effort and + /// out-of-band from `revision` (§12); ordering is enforced purely by + /// `ts`, independent of arrival or revision order. + public private(set) var metrics: [String: RTMetricSample] = [:] + + public var messageWindow: Int = 200 + + public init(revision: Int = 0) { + self.revision = revision + } + + /// Replaces all four collections wholesale from an aggregated (already + /// chunk-reassembled) snapshot, and sets `revision` to the snapshot's + /// revision. This is always safe to call — a snapshot is the + /// authoritative folded state at its revision, never a partial update. + public mutating func apply(snapshot: SnapshotBody) { + revision = snapshot.revision + tasks = Dictionary(uniqueKeysWithValues: snapshot.tasks.map { ($0.taskID, $0) }) + automations = Dictionary(uniqueKeysWithValues: snapshot.automations.map { ($0.automationID, $0) }) + messages = Array(snapshot.messages.suffix(messageWindow)) + metrics = Dictionary(uniqueKeysWithValues: snapshot.metrics.map { ($0.metricID, $0) }) + } + + /// Folds one delta's events into state and advances `revision` to + /// `delta.toRevision`. The caller (`RealtimeClient`) is responsible for + /// the §6.3 gating decision (apply only when `fromRevision == revision`) + /// — this method unconditionally applies, so tests and the client share + /// one folding implementation without duplicating the gate. + public mutating func apply(delta: DeltaBody) { + for event in delta.events { + switch event { + case .taskEvent(let e): foldTaskEvent(e) + case .messageEvent(let e): foldMessageEvent(e) + case .configEvent(let e): foldConfigEvent(e) + case .unknown: break // §15-style forward-compat: ignore what we don't recognize. + } + } + revision = delta.toRevision + } + + /// `metric.frame` is out-of-band from `revision` (§12): apply + /// independent of delta/snapshot state, discarding any sample whose + /// `ts` does not strictly advance past the last-applied `ts` for that + /// `metric_id`. + public mutating func apply(metricFrame: MetricFrameBody) { + for sample in metricFrame.metrics { + if let existing = metrics[sample.metricID], sample.ts <= existing.ts { continue } + metrics[sample.metricID] = sample + } + } + + // MARK: - §6.4 deterministic folding + + private mutating func foldTaskEvent(_ e: TaskEventBody) { + var state = tasks[e.taskID] ?? RTTaskState( + taskID: e.taskID, deviceID: e.deviceID, title: nil, state: .running, + percent: nil, step: nil, message: nil, updatedAt: e.occurredAt, display: nil) + + switch e.kind { + case .started, .progress, .step: + state.state = .running + case .done: + state.state = .done + case .failed: + state.state = .failed + } + + if let title = e.title, !title.isEmpty { + state.title = title + } + + switch e.kind { + case .started, .progress, .step: + if let percent = e.percent { state.percent = percent } + if let step = e.step { state.step = step } + case .done: + state.percent = 100 + state.step = nil + case .failed: + // percent keeps its last running value; step cleared. + state.step = nil + } + + switch e.kind { + case .done, .failed: + if let message = e.message { state.message = message } + case .started, .progress, .step: + break // message stays absent while running (never set here). + } + + if let display = e.display { state.display = display } + state.updatedAt = e.occurredAt + + tasks[e.taskID] = state + } + + private mutating func foldMessageEvent(_ e: MessageEventBody) { + let record = RTMessageRecord( + messageID: e.messageID, deviceID: e.deviceID, level: e.level, text: e.text, occurredAt: e.occurredAt) + messages.append(record) + if messages.count > messageWindow { + messages.removeFirst(messages.count - messageWindow) + } + } + + private mutating func foldConfigEvent(_ e: ConfigEventBody) { + switch e.kind { + case .upserted: + if let automation = e.automation { + automations[e.automationID] = automation + } + case .removed: + automations.removeValue(forKey: e.automationID) + } + } +} From 6124df863a514f95abc28914b44ed97d546701c7 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:32:28 +0800 Subject: [PATCH 02/60] test(apple): cover realtime state convergence Add fixture-driven SitrepKit tests against proto/realtime/fixtures/ (loaded directly from the frozen protocol directory, never copied): every valid fixture and every scenario message decodes and round-trips; every invalid fixture fails to decode or is rejected by the client authorization matrix. RealtimeResumeGateScenarioTests drives the resume/delta/snapshot gate with the four viewer-relevant scenario directories in file order. SpaceStateFoldingTests asserts delta-by-delta replay and a direct snapshot converge on identical task/automation state per SPEC.md section 6.4. --- .../Realtime/FixtureLoader.swift | 43 ++++ .../RealtimeMessageFixtureTests.swift | 111 +++++++++ .../RealtimeResumeGateScenarioTests.swift | 234 ++++++++++++++++++ .../Realtime/SpaceStateFoldingTests.swift | 151 +++++++++++ 4 files changed, 539 insertions(+) create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift new file mode 100644 index 0000000..fb70a52 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift @@ -0,0 +1,43 @@ +import Foundation + +/// Loads fixtures directly from `proto/realtime/fixtures/` (the frozen +/// protocol repository, never copied or duplicated into `apple/`) using +/// this test file's own `#filePath` to locate the repo root. This keeps the +/// Swift test suite testing the SAME fixtures the spec's own +/// `tools/validate.js` checks, with zero drift risk from a stale copy. +enum FixtureLoader { + /// `.../apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift` + /// → walk up to the repo root. + static var repoRoot: URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // FixtureLoader.swift -> Realtime/ + .deletingLastPathComponent() // Realtime/ -> SitrepKitTests/ + .deletingLastPathComponent() // SitrepKitTests/ -> Tests/ + .deletingLastPathComponent() // Tests/ -> SitrepKit/ + .deletingLastPathComponent() // SitrepKit/ -> apple/ + .deletingLastPathComponent() // apple/ -> repo root + } + + static var fixturesRoot: URL { + repoRoot.appendingPathComponent("proto/realtime/fixtures") + } + + static func data(_ relativePath: String) throws -> Data { + try Data(contentsOf: fixturesRoot.appendingPathComponent(relativePath)) + } + + static func names(in subdirectory: String) throws -> [String] { + let dir = fixturesRoot.appendingPathComponent(subdirectory) + return try FileManager.default.contentsOfDirectory(atPath: dir.path) + .filter { $0.hasSuffix(".json") } + .sorted() + } + + /// Ordered `.json` files in a scenario directory (excludes `README.md`), + /// sorted by filename — the scenarios are numbered `01-...json`, + /// `02-...json`, so lexicographic order is message order. + static func scenarioFiles(_ scenario: String) throws -> [(name: String, data: Data)] { + let names = try names(in: "scenarios/\(scenario)") + return try names.map { ($0, try data("scenarios/\(scenario)/\($0)")) } + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift new file mode 100644 index 0000000..dea3803 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift @@ -0,0 +1,111 @@ +import XCTest +@testable import SitrepKit + +/// Fixture-driven coverage of the Swift protocol layer against +/// `proto/realtime/fixtures/`, mirroring what `tools/validate.js` checks for +/// the TS/Go implementations: every valid fixture decodes and round-trips; +/// every invalid fixture is rejected, either by decoding (schema-equivalent +/// validation) or by the client authorization matrix for the two +/// role-tagged fixtures. +final class RealtimeMessageFixtureTests: XCTestCase { + // MARK: - valid/ + + func testAllValidFixturesDecodeAndRoundTrip() throws { + let names = try FixtureLoader.names(in: "valid") + XCTAssertFalse(names.isEmpty, "expected at least one valid fixture") + for name in names { + try assertDecodesAndRoundTrips(fixture: "valid/\(name)") + } + } + + // MARK: - scenarios/**/*.json (every message in every scenario is individually valid) + + func testAllScenarioMessagesDecodeAndRoundTrip() throws { + let scenarioDirs = try scenarioDirectoryNames() + XCTAssertFalse(scenarioDirs.isEmpty) + for dir in scenarioDirs { + for file in try FixtureLoader.scenarioFiles(dir) { + try assertDecodesAndRoundTrips(fixture: "scenarios/\(dir)/\(file.name)", data: file.data) + } + } + } + + private func assertDecodesAndRoundTrips(fixture: String, data: Data? = nil) throws { + let bytes = try data ?? FixtureLoader.data(fixture) + let frame: RealtimeFrame + do { + frame = try RealtimeFrame.decode(bytes) + } catch { + XCTFail("\(fixture): expected successful decode, got \(error)") + return + } + let reencoded = try frame.encoded() + let redecoded = try RealtimeFrame.decode(reencoded) + XCTAssertEqual(frame, redecoded, "\(fixture): round-trip changed semantics") + } + + // MARK: - invalid/ + + /// Fixtures whose invalidity is a role/authorization violation rather + /// than a schema violation: the frame body decodes fine on its own (it's + /// well-formed JSON matching its schema), but the sender's role forbids + /// sending it (SPEC.md §10.1). These use the + /// `{"sender_role": ..., "frame": {...}}` wrapper described in §16. + private static let roleTaggedInvalidFixtures: Set = [ + "role-client-hello-accept.json", + "role-viewer-command-origin-server.json", + ] + + func testInvalidFixturesFailDecodeOrAreUnauthorized() throws { + let names = try FixtureLoader.names(in: "invalid") + XCTAssertFalse(names.isEmpty) + for name in names { + let bytes = try FixtureLoader.data("invalid/\(name)") + if Self.roleTaggedInvalidFixtures.contains(name) { + try assertRejectedByAuthorization(fixture: name, wrapperData: bytes) + } else { + assertFailsToDecode(fixture: name, data: bytes) + } + } + } + + private func assertFailsToDecode(fixture: String, data: Data) { + XCTAssertThrowsError(try RealtimeFrame.decode(data), "invalid/\(fixture) should fail to decode") { error in + guard error is RealtimeDecodingError || error is DecodingError else { + XCTFail("invalid/\(fixture): unexpected error type \(error)") + return + } + } + } + + private func assertRejectedByAuthorization(fixture: String, wrapperData: Data) throws { + guard let object = try JSONSerialization.jsonObject(with: wrapperData) as? [String: Any], + let senderRoleRaw = object["sender_role"] as? String, + let senderRole = RTDeviceRole(rawValue: senderRoleRaw), + let frameObject = object["frame"] + else { + XCTFail("invalid/\(fixture): expected {sender_role, frame} wrapper") + return + } + let frameData = try JSONSerialization.data(withJSONObject: frameObject) + // The wrapped frame is schema-valid on its own — decoding it must + // succeed; the rejection is purely a role/authorization matter. + let frame = try RealtimeFrame.decode(frameData) + XCTAssertFalse( + RealtimeAuthorization.clientMaySend(frame, as: senderRole), + "invalid/\(fixture): expected role '\(senderRole)' to be disallowed from sending this frame") + } + + // MARK: - helpers + + private func scenarioDirectoryNames() throws -> [String] { + let dir = FixtureLoader.fixturesRoot.appendingPathComponent("scenarios") + return try FileManager.default.contentsOfDirectory(atPath: dir.path) + .filter { name in + var isDir: ObjCBool = false + FileManager.default.fileExists(atPath: dir.appendingPathComponent(name).path, isDirectory: &isDir) + return isDir.boolValue + } + .sorted() + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift new file mode 100644 index 0000000..96624c6 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift @@ -0,0 +1,234 @@ +import XCTest +@testable import SitrepKit + +/// Drives `RealtimeResumeGate` — the same gating logic `RealtimeClient` uses +/// — directly with the scenario fixtures under `proto/realtime/fixtures/scenarios/`, +/// feeding each directory's messages in filename order (as `tools/validate.js` +/// does) and asserting the terminal state matches each scenario's own +/// README. This exercises the viewer-relevant scenarios end to end without +/// a real WebSocket transport. +final class RealtimeResumeGateScenarioTests: XCTestCase { + /// Decodes one scenario fixture file, expecting envelope `type` to be + /// one of the given cases (a lightweight helper since scenario fixtures + /// are plain envelopes, not the `{sender_role, frame}` wrapper). + private func decode(_ data: Data) throws -> RealtimeFrame { + try RealtimeFrame.decode(data) + } + + // MARK: - client-resume-delta + + /// 126 -> (catch-up delta, 126->128) -> (live delta, 128->129). + func testClientResumeDelta() throws { + let files = try FixtureLoader.scenarioFiles("client-resume-delta") + var gate = RealtimeResumeGate(state: SpaceState(revision: 126)) + + // 01 hello offer / 02 hello accept — connection-setup, not gate concerns. + // 03 subscribe / 04 ack — lease, not gate concerns. + // 05 resume{last_revision: 126} + let resumeFrame = try decode(files[4].data) + guard case .resume(let resumeEnv) = resumeFrame else { return XCTFail("expected resume") } + XCTAssertEqual(resumeEnv.body.lastRevision, 126) + gate.beganResume(lastRevision: resumeEnv.body.lastRevision) + XCTAssertFalse(gate.deltaEligible, "must not be delta-eligible before the resume reply") + + // 06 catch-up delta 126 -> 128 (the resume reply itself) + guard case .delta(let catchupEnv) = try decode(files[5].data) else { return XCTFail("expected delta") } + XCTAssertEqual(gate.receiveDelta(catchupEnv.body), .applied) + XCTAssertTrue(gate.deltaEligible, "connection becomes delta-eligible on its first snapshot-or-delta reply") + XCTAssertEqual(gate.state.revision, 128) + XCTAssertEqual(gate.state.tasks["run-2f9a"]?.percent, 40) + XCTAssertEqual(gate.state.messages.count, 1) + + // 07 live delta 128 -> 129 + guard case .delta(let liveEnv) = try decode(files[6].data) else { return XCTFail("expected delta") } + XCTAssertEqual(gate.receiveDelta(liveEnv.body), .applied) + XCTAssertEqual(gate.state.revision, 129) + XCTAssertEqual(gate.state.tasks["run-2f9a"]?.percent, 60) + + XCTAssertEqual(files.count, 7) + } + + /// A delta that arrives before the resume reply must be discarded + /// (SPEC.md §6.3's defense-in-depth rule), never applied and never + /// advancing C. + func testClientResumeDeltaDiscardsPreReplyDelta() throws { + let files = try FixtureLoader.scenarioFiles("client-resume-delta") + var gate = RealtimeResumeGate(state: SpaceState(revision: 126)) + gate.beganResume(lastRevision: 126) + + // Simulate a (non-conformant-server) delta racing ahead of the + // reply: feed the LIVE delta (from step 07, from_revision 128) + // before the catch-up reply (step 06). + guard case .delta(let liveEnv) = try decode(files[6].data) else { return XCTFail() } + XCTAssertEqual(gate.receiveDelta(liveEnv.body), .discarded) + XCTAssertEqual(gate.state.revision, 126, "a pre-reply delta must not advance C") + XCTAssertFalse(gate.deltaEligible) + + // The real reply still applies normally afterward. + guard case .delta(let catchupEnv) = try decode(files[5].data) else { return XCTFail() } + XCTAssertEqual(gate.receiveDelta(catchupEnv.body), .applied) + XCTAssertEqual(gate.state.revision, 128) + } + + // MARK: - client-revision-gap-snapshot + + /// last_revision 40, server can't serve incrementally -> two-chunk + /// snapshot at revision 150. + func testClientRevisionGapSnapshot() throws { + let files = try FixtureLoader.scenarioFiles("client-revision-gap-snapshot") + XCTAssertEqual(files.count, 7) + var gate = RealtimeResumeGate(state: SpaceState(revision: 999)) // arbitrary stale prior state + + guard case .resume(let resumeEnv) = try decode(files[4].data) else { return XCTFail("expected resume") } + XCTAssertEqual(resumeEnv.body.lastRevision, 40) + gate.beganResume(lastRevision: resumeEnv.body.lastRevision) + + guard case .snapshot(let part1) = try decode(files[5].data) else { return XCTFail("expected snapshot part 1") } + XCTAssertEqual(part1.body.part, 1) + XCTAssertFalse(part1.body.final) + XCTAssertEqual(gate.receiveSnapshotChunk(part1.body), .snapshotChunkBuffered) + XCTAssertFalse(gate.deltaEligible, "nothing applies until the final chunk") + XCTAssertEqual(gate.state.revision, 999, "prior stale state must not change before the final chunk") + + guard case .snapshot(let part2) = try decode(files[6].data) else { return XCTFail("expected snapshot part 2") } + XCTAssertEqual(part2.body.part, 2) + XCTAssertTrue(part2.body.final) + XCTAssertEqual(gate.receiveSnapshotChunk(part2.body), .applied) + + XCTAssertTrue(gate.deltaEligible) + XCTAssertEqual(gate.state.revision, 150) + // Concatenated across both chunks. + XCTAssertEqual(gate.state.tasks.count, 1) + XCTAssertEqual(gate.state.automations.count, 1) + XCTAssertEqual(gate.state.metrics.count, 1) + XCTAssertEqual(gate.state.messages.count, 1) + + // A live delta with from_revision == 150 is now applicable. + var followUp = gate + let liveDelta = DeltaBody( + fromRevision: 150, toRevision: 151, + events: [.taskEvent(TaskEventBody( + deviceID: "mac-quintin-01", deviceSeq: 200, taskID: "run-2f9a", kind: .done, occurredAt: 1_752_483_100_000))]) + XCTAssertEqual(followUp.receiveDelta(liveDelta), .applied) + XCTAssertEqual(followUp.state.revision, 151) + } + + /// A non-conformant server interleaving something other than the next + /// chunk (or ping/pong, which never reaches the gate at all — see + /// `RealtimeClient.receiveFrame`) between snapshot chunks must be + /// treated as a malformed sequence. + func testSnapshotChunkOutOfSequenceIsMalformed() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let part1 = SnapshotBody(revision: 10, part: 1, final: false, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(part1), .snapshotChunkBuffered) + + // Wrong revision. + let wrongRevision = SnapshotBody(revision: 11, part: 2, final: true, tasks: [], metrics: [], messages: [], automations: []) + guard case .malformedSequence = gate.receiveSnapshotChunk(wrongRevision) else { + return XCTFail("expected malformed sequence for a revision change mid-snapshot") + } + } + + // MARK: - gap detection -> re-resume, and revision_unavailable -> retry with 0 + + func testGapDetectionTriggersReResume() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let empty = DeltaBody(fromRevision: 0, toRevision: 0, events: []) + XCTAssertEqual(gate.receiveDelta(empty), .applied) + XCTAssertTrue(gate.deltaEligible) + + // A delta arrives with from_revision > C: a gap. + let gapped = DeltaBody( + fromRevision: 5, toRevision: 6, + events: [.taskEvent(TaskEventBody(deviceID: "d1", deviceSeq: 1, taskID: "t1", kind: .started, occurredAt: 1_752_000_000_000))]) + guard case .gapDetected(let resendFrom) = gate.receiveDelta(gapped) else { + return XCTFail("expected a gap") + } + XCTAssertEqual(resendFrom, 0, "must resend resume with the last revision actually applied") + XCTAssertEqual(gate.state.revision, 0, "the gapped delta itself is never applied") + XCTAssertFalse(gate.deltaEligible, "not delta-eligible again until the fresh resume's reply") + + // Simulate the caller re-issuing resume and the server replying. + gate.beganResume(lastRevision: resendFrom) + let reply = DeltaBody(fromRevision: 0, toRevision: 1, + events: [.taskEvent(TaskEventBody(deviceID: "d1", deviceSeq: 1, taskID: "t1", kind: .started, occurredAt: 1_752_000_000_000))]) + XCTAssertEqual(gate.receiveDelta(reply), .applied) + XCTAssertEqual(gate.state.revision, 1) + } + + /// A stray live delta racing an outstanding re-resume (from_revision + /// doesn't match what was requested) must be discarded, not misapplied + /// as if it were the reply and not re-triggering yet another resume. + func testRacingDeltaDuringOutstandingResumeIsDiscarded() { + var gate = RealtimeResumeGate(state: SpaceState(revision: 5)) + gate.beganResume(lastRevision: 5) + let racing = DeltaBody( + fromRevision: 7, toRevision: 8, + events: [.taskEvent(TaskEventBody(deviceID: "d1", deviceSeq: 9, taskID: "t1", kind: .progress, occurredAt: 1_752_000_000_000, percent: 10))]) + XCTAssertEqual(gate.receiveDelta(racing), .discarded) + XCTAssertEqual(gate.state.revision, 5) + XCTAssertNotNil(gate.awaitingResumeReply, "still waiting for the true reply") + } + + func testRevisionUnavailableRetriesWithZero() { + var gate = RealtimeResumeGate(state: SpaceState(revision: 200)) + gate.beganResume(lastRevision: 200) + let error = ErrorBody(code: .revisionUnavailable, message: "gone", retryable: true, fatal: false) + XCTAssertTrue(gate.receiveErrorAnswersOutstandingResume(error)) + XCTAssertEqual(gate.awaitingResumeReply, 0) + + let snapshot = SnapshotBody(revision: 5, part: 1, final: true, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(snapshot), .applied) + XCTAssertEqual(gate.state.revision, 5) + } + + // MARK: - interest-lease-expiry (viewer-visible parts: subscribe/renew acks and the broadcast commands) + + func testInterestLeaseExpiryScenarioShapes() throws { + let files = try FixtureLoader.scenarioFiles("interest-lease-expiry") + XCTAssertEqual(files.count, 6) + + guard case .ack(let ack1) = try decode(files[1].data), let lease1 = ack1.body.lease else { + return XCTFail("expected subscribe ack with lease") + } + XCTAssertEqual(lease1.expiresAt - ack1.ts, 45000) // within [30000, 60000] of envelope ts + + guard case .command(let throttle) = try decode(files[2].data) else { return XCTFail("expected throttle command") } + XCTAssertEqual(throttle.body.origin, .server) + XCTAssertEqual(throttle.body.action, .throttle) + XCTAssertNil(throttle.body.targetDeviceID, "broadcasts to every source device in the space") + // A viewer must never be allowed to originate this itself. + XCTAssertFalse(RealtimeAuthorization.clientMaySend(.command(throttle), as: .viewer)) + + guard case .ack(let ack2) = try decode(files[4].data), let lease2 = ack2.body.lease else { + return XCTFail("expected second viewer's subscribe ack with lease") + } + XCTAssertTrue((30_000...60_000).contains(lease2.expiresAt - ack2.ts)) + + guard case .command(let resumeRate) = try decode(files[5].data) else { return XCTFail("expected resume_rate command") } + XCTAssertEqual(resumeRate.body.origin, .server) + XCTAssertEqual(resumeRate.body.action, .resumeRate) + } + + // MARK: - duplicate-connection-supersede + + func testDuplicateConnectionSupersedeScenarioShapes() throws { + let files = try FixtureLoader.scenarioFiles("duplicate-connection-supersede") + XCTAssertEqual(files.count, 5) + + guard case .hello(let offer1) = try decode(files[0].data), case .offer(let o1) = offer1.body else { + return XCTFail("expected hello offer") + } + guard case .hello(let offer2) = try decode(files[2].data), case .offer(let o2) = offer2.body else { + return XCTFail("expected second hello offer") + } + XCTAssertEqual(o1.deviceID, o2.deviceID, "same device_id opens a second connection") + + guard case .error(let superseded) = try decode(files[4].data) else { return XCTFail("expected superseded error") } + XCTAssertEqual(superseded.body.code, .superseded) + XCTAssertTrue(superseded.body.fatal) + XCTAssertFalse(superseded.body.retryable, "not retryable on the dying connection itself") + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift new file mode 100644 index 0000000..053c41c --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift @@ -0,0 +1,151 @@ +import XCTest +@testable import SitrepKit + +/// SPEC.md §6.4: two viewers — one that replayed every delta, one that took +/// a snapshot — MUST converge on identical task and automation state for +/// the same event history. These tests build one event history, fold it +/// two ways (delta-by-delta vs. a directly-applied snapshot equivalent to +/// what a server would have computed), and assert the two `SpaceState` +/// values agree on `tasks`/`automations`. +final class SpaceStateFoldingTests: XCTestCase { + private func taskEvent( + seq: Int, kind: RTTaskKind, occurredAt: Int, title: String? = nil, percent: Int? = nil, + step: String? = nil, message: String? = nil, display: RTDisplayHints? = nil + ) -> TaskEventBody { + TaskEventBody( + deviceID: "mac-1", deviceSeq: seq, taskID: "run-1", kind: kind, occurredAt: occurredAt, + title: title, percent: percent, step: step, message: message, display: display) + } + + private func applyAsDeltaStream(_ events: [DeltaEvent]) -> SpaceState { + var state = SpaceState() + for (index, event) in events.enumerated() { + state.apply(delta: DeltaBody(fromRevision: index, toRevision: index + 1, events: [event])) + } + return state + } + + /// started -> progress(40, "uploading") -> progress(80, no title change) -> done("all good"). + func testTaskFoldingConvergesDeltaVsSnapshot() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Nightly backup", display: RTDisplayHints(icon: "moon"))), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 40, step: "uploading")), + .taskEvent(taskEvent(seq: 3, kind: .progress, occurredAt: 3000, title: "", percent: 80)), // empty title must NOT overwrite + .taskEvent(taskEvent(seq: 4, kind: .done, occurredAt: 4000, message: "all good")), + ] + + let viaDeltas = applyAsDeltaStream(events) + + // A snapshot is the server's already-folded state at this revision; + // construct it by hand per the exact §6.4 rules and apply it + // wholesale, simulating "the other viewer took a snapshot". + let expectedFolded = RTTaskState( + taskID: "run-1", deviceID: "mac-1", title: "Nightly backup", state: .done, + percent: 100, step: nil, message: "all good", updatedAt: 4000, + display: RTDisplayHints(icon: "moon")) + var viaSnapshot = SpaceState() + viaSnapshot.apply(snapshot: SnapshotBody( + revision: 4, part: 1, final: true, tasks: [expectedFolded], metrics: [], messages: [], automations: [])) + + XCTAssertEqual(viaDeltas.tasks, viaSnapshot.tasks) + XCTAssertEqual(viaDeltas.tasks["run-1"], expectedFolded) + } + + func testTaskFoldingFailedKeepsLastPercentAndClearsStep() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 55, step: "step 2")), + .taskEvent(taskEvent(seq: 3, kind: .failed, occurredAt: 3000, message: "boom")), + ] + let state = applyAsDeltaStream(events) + let task = try! XCTUnwrap(state.tasks["run-1"]) + XCTAssertEqual(task.state, .failed) + XCTAssertEqual(task.percent, 55, "percent keeps its last running value on failure") + XCTAssertNil(task.step, "step is cleared on done/failed") + XCTAssertEqual(task.message, "boom") + } + + func testTaskFoldingMessageStaysAbsentWhileRunning() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 10)), + ] + let state = applyAsDeltaStream(events) + XCTAssertNil(state.tasks["run-1"]?.message) + } + + func testTaskFoldingDisplayReplacesWholesaleNotMerged() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job", display: RTDisplayHints(icon: "a", tint: "blue"))), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 10, display: RTDisplayHints(tint: "red"))), + ] + let state = applyAsDeltaStream(events) + // The second event's display object wins WHOLESALE: `icon` is gone, + // not preserved from the first event. + XCTAssertEqual(state.tasks["run-1"]?.display, RTDisplayHints(tint: "red")) + } + + // MARK: - automation folding (config.event) + + func testAutomationUpsertThenRemoveConvergesDeltaVsSnapshot() { + let automationV1 = RTAutomationState( + automationID: "auto-1", name: "Disk watch", executorKind: "script", + scheduleEverySeconds: 300, state: .active, lastRunAt: nil) + let automationV2 = RTAutomationState( + automationID: "auto-1", name: "Disk watch (paused)", executorKind: "script", + scheduleEverySeconds: 600, state: .paused, lastRunAt: 5000) + + let events: [DeltaEvent] = [ + .configEvent(ConfigEventBody(kind: .upserted, automationID: "auto-1", automation: automationV1, occurredAt: 1000)), + .configEvent(ConfigEventBody(kind: .upserted, automationID: "auto-1", automation: automationV2, occurredAt: 2000)), + ] + let viaDeltas = applyAsDeltaStream(events) + XCTAssertEqual(viaDeltas.automations["auto-1"], automationV2, "upsert replaces wholesale, no per-field merge") + + let removed = events + [.configEvent(ConfigEventBody(kind: .removed, automationID: "auto-1", occurredAt: 3000))] + let viaDeltasRemoved = applyAsDeltaStream(removed) + XCTAssertNil(viaDeltasRemoved.automations["auto-1"]) + + var viaSnapshot = SpaceState() + viaSnapshot.apply(snapshot: SnapshotBody( + revision: 2, part: 1, final: true, tasks: [], metrics: [], messages: [], automations: [automationV2])) + XCTAssertEqual(viaDeltas.automations, viaSnapshot.automations) + } + + // MARK: - message.event append-only history + + func testMessageEventsAppendInRevisionOrder() { + let events: [DeltaEvent] = [ + .messageEvent(MessageEventBody(deviceID: "mac-1", deviceSeq: 1, messageID: "m1", level: .info, text: "first", occurredAt: 1000)), + .messageEvent(MessageEventBody(deviceID: "mac-1", deviceSeq: 2, messageID: "m2", level: .warn, text: "second", occurredAt: 2000)), + ] + let state = applyAsDeltaStream(events) + XCTAssertEqual(state.messages.map(\.text), ["first", "second"]) + } + + // MARK: - metric.frame ts-based discard + + func testMetricFrameDiscardsStaleOrDuplicateTimestamps() { + var state = SpaceState() + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.5", ts: 2000), + ])) + // Same ts again: discarded (ts <= last-applied ts). + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.9", ts: 2000), + ])) + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.5") + + // Earlier ts: discarded. + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.1", ts: 1000), + ])) + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.5") + + // Later ts: applied. + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.7", ts: 3000), + ])) + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.7") + } +} From 7e960b1f6b3418aef747042ff1510e4f6adb5b19 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:40:00 +0800 Subject: [PATCH 03/60] feat(apple): synchronize foreground state over websocket Wire SitrepApp's AppModel to SitrepKit's RealtimeClient: tasks, metrics, messages, and automations now update live from SpaceState deltas while the app is foreground, replacing the always-on 3s HTTP poll. The poll survives only as a low-frequency (>=30s) fallback that engages after repeated WebSocket backoff failures and disengages once the realtime connection recovers. scenePhase drives connection lifecycle deterministically: .active (re)connects and revives anything silently killed while suspended, .background closes the connection and lets the interest lease lapse. Add RealtimeUIBridge in SitrepKit adapting the realtime wire model to the existing poll-era UI types (TaskState/MetricState/EventLogEntry/ AutomationInfo) so NowView/DashboardView/HistoryView need no changes. Add public initializers to AutomationInfo/Executor/Schedule to support that construction from outside SitrepKit. MainTabView gains a subtle sync-status strip (reusing the presence-pill dot+caption language, not a new visual system) that only appears while not live, and a one-shot, tap-or-6s-dismiss banner for SPEC.md section 9.4 `superseded`. --- apple/SitrepApp/Sitrep/MainTabView.swift | 91 +++++++++- apple/SitrepApp/Sitrep/SitrepApp.swift | 164 ++++++++++++++++-- .../SitrepKit/Sources/SitrepKit/Models.swift | 16 ++ .../SitrepKit/Realtime/RealtimeUIBridge.swift | 70 ++++++++ 4 files changed, 323 insertions(+), 18 deletions(-) create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift diff --git a/apple/SitrepApp/Sitrep/MainTabView.swift b/apple/SitrepApp/Sitrep/MainTabView.swift index 3cf4513..5db7f4b 100644 --- a/apple/SitrepApp/Sitrep/MainTabView.swift +++ b/apple/SitrepApp/Sitrep/MainTabView.swift @@ -15,9 +15,16 @@ struct MainTabView: View { var body: some View { configuredTabs .safeAreaInset(edge: .top, spacing: 0) { - // Honesty first: a monitoring tool must SAY when it can't see. - if let err = model.lastError { - ErrorBanner(text: err) + VStack(spacing: 0) { + // Honesty first: a monitoring tool must SAY when it can't see. + if let err = model.lastError { + ErrorBanner(text: err) + } else if let status = SyncStatusStrip.label(for: model.connectionPhase) { + SyncStatusStrip(color: status.color, text: status.text) + } + if model.supersededNotice { + SupersededBanner(isPresented: $model.supersededNotice) + } } } } @@ -66,12 +73,20 @@ struct MainTabView: View { .onReceive(NotificationCenter.default.publisher(for: .sitrepOpenEvents)) { _ in selection = 2 } - // Coming back to foreground: refresh immediately and revive the - // poll loop if anything killed it while suspended. + // Coming back to foreground: refresh immediately, (re)connect the + // realtime channel, and revive the HTTP fallback poll if anything + // killed it while suspended. Leaving the foreground closes the + // realtime connection — the interest lease simply lapses server-side + // rather than being explicitly released. .onChange(of: scenePhase) { _, phase in - if phase == .active { + switch phase { + case .active: model.ensurePolling() Task { await model.refresh() } + case .background: + model.leaveBackground() + default: + break } } } @@ -94,6 +109,70 @@ struct ErrorBanner: View { } } +/// A subtle, transient strip about THIS PHONE's realtime connection to the +/// server — distinct from `PresencePill`/`PresenceRow`, which report +/// whether the Mac/Agent is reporting in. Reuses the same dot-plus-caption +/// language rather than a new visual system; shows nothing once the +/// connection is `.live` (the common case), so it stays out of the way. +struct SyncStatusStrip: View { + let color: Color + let text: String + + /// nil means "healthy, show nothing" — only `.live` and `.idle` (not yet + /// started) count as nothing-to-say; every other phase is transient and + /// worth a one-line, low-emphasis mention. + static func label(for phase: RealtimeClient.Phase) -> (color: Color, text: String)? { + switch phase { + case .idle, .live: nil + case .connecting, .handshaking: (.gray, "正在连接实时同步…") + case .subscribed: (.gray, "正在同步…") + case .failed: (.orange, "实时同步中断，已切换到低频刷新") + } + } + + var body: some View { + HStack(spacing: 6) { + Circle().fill(color).frame(width: 6, height: 6) + Text(text).font(.caption2.weight(.medium)).foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 4) + .background(.background.secondary) + .transition(.move(edge: .top).combined(with: .opacity)) + .animation(.default, value: text) + } +} + +/// One-shot, non-disruptive notice for SPEC.md §9.4 `superseded`: this +/// device's realtime connection was replaced by another connection using +/// the same credential — most often the same phone's own connection racing +/// itself across a fast background/foreground cycle, but possibly a sign +/// the credential is in use elsewhere. Tap to dismiss; not repeated. +struct SupersededBanner: View { + @Binding var isPresented: Bool + + var body: some View { + Button { + withAnimation { isPresented = false } + } label: { + Label("这台设备的实时连接被另一个连接取代，可能是该凭据在别处也在使用", systemImage: "info.circle") + .font(.caption.weight(.medium)) + .lineLimit(2) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 12) + .padding(.vertical, 7) + } + .buttonStyle(.plain) + .background(.blue.opacity(0.12)) + .foregroundStyle(.blue) + .transition(.move(edge: .top).combined(with: .opacity)) + .task { + try? await Task.sleep(for: .seconds(6)) + withAnimation { isPresented = false } + } + } +} + /// "Is my Mac alive" — the first question a monitoring tool must answer. /// Green: agent heartbeat or ingest within a minute. Orange: minutes quiet. /// Red: nothing for 10+ minutes. diff --git a/apple/SitrepApp/Sitrep/SitrepApp.swift b/apple/SitrepApp/Sitrep/SitrepApp.swift index 1c5a062..de94e2f 100644 --- a/apple/SitrepApp/Sitrep/SitrepApp.swift +++ b/apple/SitrepApp/Sitrep/SitrepApp.swift @@ -81,7 +81,29 @@ final class AppModel { /// computed against this, not against server timestamps. var lastSyncAt: Date? - private var pollTask: Task? + /// The realtime WebSocket connection's lifecycle (SitrepKit's + /// `RealtimeClient`). Drives the subtle sync-status indicator; `.live` + /// means tasks/metrics/events/automations are updating from `delta`s in + /// real time rather than from the HTTP fallback poll below. + var connectionPhase: RealtimeClient.Phase = .idle + /// One-shot: this device's realtime connection was superseded by + /// another connection using the same credential (SPEC.md §9.4). Surfaced + /// once, non-disruptively, then cleared by the view that shows it. + var supersededNotice = false + + private var realtimeClient: RealtimeClient? + private var realtimeObservers: [Task] = [] + /// Low-frequency (>=30s) HTTP snapshot polling, used ONLY while the + /// realtime connection is down for multiple backoff cycles — see + /// `RealtimeClient.Notice.fellBackToPolling`/`.recovered`. This replaces + /// what used to be an always-on 3s poll loop: normal sync now comes from + /// `delta`s over the WebSocket. + private var fallbackPollTask: Task? + /// Whether the fallback poll is *supposed* to be running (independent of + /// whether `fallbackPollTask` is currently non-nil) — lets `ensurePolling()` + /// revive it defensively if something silently killed the task while + /// the realtime connection was still down. + private var fallbackActive = false var lastError: String? // Keychain-backed (survives reinstalls); UserDefaults is a legacy @@ -109,6 +131,13 @@ final class AppModel { WidgetCenter.shared.reloadAllTimelines() let client = self.client Task { await PushRegistrar.shared.configure(client: client) } + // Credentials changed under an existing connection (e.g. re-paired) + // — reconnect with the new ones rather than keep talking under the + // old identity. + if realtimeClient != nil { + stopRealtime() + enterForeground() + } } func start() async { @@ -121,7 +150,117 @@ final class AppModel { } await PushRegistrar.shared.configure(client: client) await PushRegistrar.shared.startObserving() - ensurePolling() + // Seed the UI with one HTTP snapshot immediately; the realtime + // connection (started below) then takes over as the live source. + await refresh() + enterForeground() + } + + // MARK: - Realtime sync (foreground WebSocket + HTTP fallback) + + /// Call when the app becomes foreground-active. Idempotent. + func enterForeground() { + guard !needsOnboarding else { return } + startRealtime() + } + + /// Call when the app leaves the foreground. The interest lease simply + /// lapses server-side (SPEC.md §7) rather than being released with an + /// explicit `unsubscribe` — closing the connection is enough. + func leaveBackground() { + stopRealtime() + stopFallbackPolling() + } + + private func startRealtime() { + guard let restClient = client, let url = restClient.realtimeURL, !deviceID.isEmpty else { return } + if let existing = realtimeClient { + // Idempotent on the actor side even if already running — this + // doubles as the revival path if something silently killed the + // connection's internal loop without going through `stop()`. + Task { await existing.start() } + return + } + let configuration = RealtimeClient.Configuration( + url: url, token: token.isEmpty ? nil : token, deviceID: deviceID) + let rt = RealtimeClient(configuration: configuration) + realtimeClient = rt + realtimeObservers = [ + Task { [weak self] in + for await phase in rt.phases { + self?.connectionPhase = phase + } + }, + Task { [weak self] in + for await state in rt.states { + self?.applyRealtimeState(state) + } + }, + Task { [weak self] in + for await notice in rt.notices { + self?.handleRealtimeNotice(notice) + } + }, + ] + Task { await rt.start() } + } + + private func stopRealtime() { + guard let rt = realtimeClient else { return } + realtimeClient = nil + for observer in realtimeObservers { observer.cancel() } + realtimeObservers = [] + connectionPhase = .idle + Task { await rt.stop() } + } + + private func applyRealtimeState(_ state: SpaceState) { + tasks = state.uiTasks + let sortedMetrics = state.uiMetrics + // Widgets refresh on a slow OS budget; while the app is foreground + // we piggyback any change onto them so the two surfaces never + // disagree (same rule `refresh()` applies for the HTTP path). + if sortedMetrics != metrics { + WidgetCenter.shared.reloadAllTimelines() + } + metrics = sortedMetrics + events = state.uiEvents + automations = state.uiAutomations + lastError = nil + lastSyncAt = .now + adoptOrphanedTasks() + } + + private func handleRealtimeNotice(_ notice: RealtimeClient.Notice) { + switch notice { + case .superseded: + supersededNotice = true + case .serverError(let code, let message): + lastError = "\(code.rawValue): \(message)" + case .revisionGap: + break // transparent to the user; the client re-resumes on its own + case .fellBackToPolling: + startFallbackPolling() + case .recovered: + stopFallbackPolling() + } + } + + private func startFallbackPolling() { + fallbackActive = true + guard fallbackPollTask == nil else { return } + fallbackPollTask = Task { [weak self] in + while !Task.isCancelled { + await self?.refresh() + try? await Task.sleep(for: .seconds(30)) + } + } + } + + private func stopFallbackPolling() { + fallbackActive = false + fallbackPollTask?.cancel() + fallbackPollTask = nil } /// v2 intentionally breaks the old space/auth model. Keychain values @@ -154,17 +293,16 @@ final class AppModel { } } - /// The poll loop is owned, observable, and restartable — whatever kills - /// it (suspension edge cases, cancellation), the next foreground pass - /// revives it. + /// Called on every `scenePhase == .active` transition (see + /// `MainTabView`). Both the realtime connection and the HTTP fallback + /// poll are owned, observable, and restartable — whatever kills them + /// (suspension edge cases, cancellation), the next foreground pass + /// revives them. This is the same self-healing shape the old always-on + /// 3s poll loop used; it now applies to `RealtimeClient.start()` + /// (idempotent even if already running) and to the fallback poll task. func ensurePolling() { - if let pollTask, !pollTask.isCancelled { return } - pollTask = Task { [weak self] in - while !Task.isCancelled { - await self?.refresh() - try? await Task.sleep(for: .seconds(3)) - } - } + enterForeground() + if fallbackActive { startFallbackPolling() } } /// Local fallback for remote push-to-start: if the server has running @@ -207,6 +345,8 @@ final class AppModel { /// Revoke this phone's device record, clear credentials, return to /// onboarding. Other devices in the space are untouched. func disconnect() async { + stopRealtime() + stopFallbackPolling() if !deviceID.isEmpty { try? await client?.revokeDevice(id: deviceID) } diff --git a/apple/SitrepKit/Sources/SitrepKit/Models.swift b/apple/SitrepKit/Sources/SitrepKit/Models.swift index a7beff5..9db7d2f 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Models.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Models.swift @@ -111,14 +111,30 @@ public struct AutomationInfo: Codable, Identifiable, Sendable, Equatable { public var state: State public var lastRun: Date? + public init(id: String, name: String, executor: Executor, schedule: Schedule, state: State, lastRun: Date? = nil) { + self.id = id + self.name = name + self.executor = executor + self.schedule = schedule + self.state = state + self.lastRun = lastRun + } + public struct Executor: Codable, Sendable, Equatable { public var kind: String + + public init(kind: String) { self.kind = kind } } public struct Schedule: Codable, Sendable, Equatable { public var kind: String public var everySeconds: Int + public init(kind: String, everySeconds: Int) { + self.kind = kind + self.everySeconds = everySeconds + } + enum CodingKeys: String, CodingKey { case kind case everySeconds = "every_seconds" diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift new file mode 100644 index 0000000..0ddabb2 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift @@ -0,0 +1,70 @@ +import Foundation + +// Adapters from the realtime wire model (RTTaskState/RTMetricSample/...) to +// the existing poll-era UI models (TaskState/MetricState/EventLogEntry/ +// AutomationInfo), so SitrepApp's views keep reading the same types whether +// their data came from an HTTP snapshot or from `SpaceState`. This mirrors +// the existing `SnapshotMetric.state`/`SnapshotMessage.state` adapters in +// Models.swift, which do the equivalent job for the REST `/v2/snapshot` +// response shape. + +private extension Date { + init(msSinceEpoch ms: Int) { self.init(timeIntervalSince1970: Double(ms) / 1000) } +} + +public extension RTTaskState { + var uiState: TaskState { + let status: TaskStatus + switch state { + case .running: status = .running + case .done: status = .done + case .failed: status = .failed + } + var t = TaskState( + sourceID: taskID, title: title ?? "", status: status, percent: percent, step: step, + updatedAt: Date(msSinceEpoch: updatedAt)) + t.icon = display?.icon + t.tint = display?.tint + t.template = display?.template + return t + } +} + +public extension RTMetricSample { + var uiState: MetricState { + var m = MetricState(key: metricID, value: value, label: label, updatedAt: Date(msSinceEpoch: ts)) + m.icon = display?.icon + m.tint = display?.tint + m.template = display?.template + m.target = target + m.min = min + m.max = max + m.alertAbove = alertAbove + m.alertBelow = alertBelow + return m + } +} + +public extension RTMessageRecord { + var uiEvent: EventLogEntry { + EventLogEntry(serverID: messageID, text: text, level: level.rawValue, ts: Date(msSinceEpoch: occurredAt), source: nil) + } +} + +public extension RTAutomationState { + var uiInfo: AutomationInfo { + AutomationInfo( + id: automationID, name: name, + executor: .init(kind: executorKind), + schedule: .init(kind: "interval", everySeconds: scheduleEverySeconds), + state: state == .active ? .active : .paused, + lastRun: lastRunAt.map { Date(msSinceEpoch: $0) }) + } +} + +public extension SpaceState { + var uiTasks: [TaskState] { tasks.values.map(\.uiState).sorted { $0.updatedAt > $1.updatedAt } } + var uiMetrics: [MetricState] { metrics.values.map(\.uiState).sorted { $0.key < $1.key } } + var uiEvents: [EventLogEntry] { messages.map(\.uiEvent) } + var uiAutomations: [AutomationInfo] { automations.values.map(\.uiInfo).sorted { $0.name < $1.name } } +} From 88e79eaee3ecf48d4ffa43f103598ba4ed6114d9 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:40:54 +0800 Subject: [PATCH 04/60] feat(daemon): implement realtime protocol wire types Add internal/realtime/wire, a Go mirror of proto/realtime's frozen JSON Schemas: the strict envelope (type/id/ts/body), one struct per message body, and validation matching the schema's required fields, enums, ranges, and conditional requirements. Round-trips every fixture under proto/realtime/fixtures/valid and fixtures/scenarios through decode -> re-encode and asserts semantic equality with the original, and asserts every fixture under fixtures/invalid is rejected (including the sender_role-wrapped authorization-matrix cases via a new Authorize function mirroring SPEC.md section 10.1). proto/realtime/ itself is untouched; this package only depends on it as a read-only source of truth. --- daemon/internal/realtime/wire/authz.go | 77 ++++ daemon/internal/realtime/wire/bodies.go | 337 +++++++++++++++ daemon/internal/realtime/wire/envelope.go | 133 ++++++ daemon/internal/realtime/wire/validate.go | 470 +++++++++++++++++++++ daemon/internal/realtime/wire/wire_test.go | 202 +++++++++ 5 files changed, 1219 insertions(+) create mode 100644 daemon/internal/realtime/wire/authz.go create mode 100644 daemon/internal/realtime/wire/bodies.go create mode 100644 daemon/internal/realtime/wire/envelope.go create mode 100644 daemon/internal/realtime/wire/validate.go create mode 100644 daemon/internal/realtime/wire/wire_test.go diff --git a/daemon/internal/realtime/wire/authz.go b/daemon/internal/realtime/wire/authz.go new file mode 100644 index 0000000..bca590f --- /dev/null +++ b/daemon/internal/realtime/wire/authz.go @@ -0,0 +1,77 @@ +package wire + +import "fmt" + +// Role is the realtime-protocol role of a connecting device (SPEC.md +// section 9.2) — coarser than and distinct from the HTTP-layer roles used +// for pairing/REST. +type Role string + +const ( + RoleSource Role = "source" + RoleViewer Role = "viewer" +) + +// Authorize checks a client-originated envelope against the section 10.1 +// authorization matrix, plus the two rules that apply regardless of the +// declared role: +// - stage:"accept" belongs exclusively to the server (section 9.1); any +// client-sent accept is a handshake violation, answered with +// error{code: hello_required} rather than the ordinary +// error{code: unauthorized} the matrix would otherwise suggest. +// - command{origin: "server"} can never be sent by a client, regardless +// of the sender's role or the action named (section 8). +// +// body must be the already-decoded, already-Validate()'d body value +// returned by DecodeBody for the same envelope. Authorize does not itself +// re-run schema validation. +func Authorize(role Role, envType string, body any) error { + switch envType { + case TypeHello: + hb, ok := body.(HelloBody) + if !ok { + return fmt.Errorf("authorize: expected HelloBody, got %T", body) + } + if hb.Accept != nil { + return fmt.Errorf("%s: stage \"accept\" may only be sent by the server", ErrHelloRequired) + } + // stage "offer" is permitted from both source and viewer. + return nil + case TypeResume, TypeSubscribe, TypeUnsubscribe, TypeInterestRenew: + if role != RoleViewer { + return fmt.Errorf("%s: %q may only be sent by a viewer", ErrUnauthorized, envType) + } + return nil + case TypeSnapshot, TypeDelta, TypeConfigEvent: + return fmt.Errorf("%s: %q is server-only, a client MUST NOT send it", ErrUnauthorized, envType) + case TypeTaskEvent, TypeMessageEvent, TypeMetricFrame: + if role != RoleSource { + return fmt.Errorf("%s: %q may only be sent by a source", ErrUnauthorized, envType) + } + return nil + case TypeAck: + if role != RoleSource { + return fmt.Errorf("%s: only a source may send ack (optional command-delivery ack)", ErrUnauthorized) + } + return nil + case TypeCommand: + cb, ok := body.(CommandBody) + if !ok { + return fmt.Errorf("authorize: expected CommandBody, got %T", body) + } + // This rule applies regardless of role: a client can never + // originate a server-side command, whatever action it names. + if cb.Origin == "server" { + return fmt.Errorf("%s: a client may never send command with origin \"server\"", ErrUnauthorized) + } + if role != RoleViewer { + return fmt.Errorf("%s: command may only be sent by a viewer (with origin viewer)", ErrUnauthorized) + } + return nil + case TypeError: + return nil // either role may send error + default: + // Unrecognized type: SPEC.md section 15 says ignore, not reject. + return nil + } +} diff --git a/daemon/internal/realtime/wire/bodies.go b/daemon/internal/realtime/wire/bodies.go new file mode 100644 index 0000000..d9c47cc --- /dev/null +++ b/daemon/internal/realtime/wire/bodies.go @@ -0,0 +1,337 @@ +package wire + +import ( + "encoding/json" + "fmt" +) + +// DisplayHints mirrors common.schema.json#/$defs/display_hints. +type DisplayHints struct { + Icon string `json:"icon,omitempty"` + Tint string `json:"tint,omitempty"` + Template string `json:"template,omitempty"` +} + +// TaskState mirrors common.schema.json#/$defs/task_state. +type TaskState struct { + TaskID string `json:"task_id"` + DeviceID string `json:"device_id"` + Title string `json:"title,omitempty"` + State string `json:"state"` // running|done|failed + Percent *int `json:"percent,omitempty"` + Step string `json:"step,omitempty"` + Message string `json:"message,omitempty"` + UpdatedAt int64 `json:"updated_at"` + Display *DisplayHints `json:"display,omitempty"` +} + +// MetricSample mirrors common.schema.json#/$defs/metric_sample. +type MetricSample struct { + MetricID string `json:"metric_id"` + Value string `json:"value"` + Label string `json:"label,omitempty"` + TS int64 `json:"ts"` + Display *DisplayHints `json:"display,omitempty"` + Target string `json:"target,omitempty"` + Min string `json:"min,omitempty"` + Max string `json:"max,omitempty"` + AlertAbove string `json:"alert_above,omitempty"` + AlertBelow string `json:"alert_below,omitempty"` +} + +// Schedule mirrors the schedule object nested in automation_state. +type Schedule struct { + Kind string `json:"kind"` // always "interval" in v1 + EverySeconds int `json:"every_seconds"` +} + +// AutomationState mirrors common.schema.json#/$defs/automation_state. +type AutomationState struct { + AutomationID string `json:"automation_id"` + Name string `json:"name"` + ExecutorKind string `json:"executor_kind"` // script|agent|hybrid + Schedule Schedule `json:"schedule"` + State string `json:"state"` // active|paused + LastRunAt int64 `json:"last_run_at,omitempty"` +} + +// MessageRecord mirrors common.schema.json#/$defs/message_record. +type MessageRecord struct { + MessageID string `json:"message_id"` + DeviceID string `json:"device_id"` + Level string `json:"level"` + Text string `json:"text"` + OccurredAt int64 `json:"occurred_at"` +} + +// ---- hello (messages/hello.schema.json) ---- + +// HelloOffer is sent by the connecting device, client -> server. +// +// Capabilities is a pointer so an explicitly-present-but-empty +// "capabilities": [] round-trips distinctly from an entirely absent field +// (both are valid on the wire; schema default is [] when omitted). +type HelloOffer struct { + Stage string `json:"stage"` // const "offer" + DeviceID string `json:"device_id"` + Role string `json:"role"` // source|viewer + ProtocolVersions []int `json:"protocol_versions"` + Capabilities *[]string `json:"capabilities,omitempty"` +} + +// HelloAccept is sent by the server once it has negotiated a version. +type HelloAccept struct { + Stage string `json:"stage"` // const "accept" + ProtocolVersion int `json:"protocol_version"` + SessionID string `json:"session_id"` + Capabilities *[]string `json:"capabilities,omitempty"` + HeartbeatIntervalMS int `json:"heartbeat_interval_ms"` +} + +// HelloBody is the oneOf{offer, accept} discriminated union keyed by `stage`. +// Exactly one of Offer/Accept is non-nil after decode. +type HelloBody struct { + Offer *HelloOffer + Accept *HelloAccept +} + +func (h HelloBody) MarshalJSON() ([]byte, error) { + switch { + case h.Offer != nil: + return json.Marshal(h.Offer) + case h.Accept != nil: + return json.Marshal(h.Accept) + default: + return nil, fmt.Errorf("hello body: neither offer nor accept set") + } +} + +func (h *HelloBody) UnmarshalJSON(data []byte) error { + var disc struct { + Stage string `json:"stage"` + } + if err := json.Unmarshal(data, &disc); err != nil { + return err + } + switch disc.Stage { + case "offer": + var o HelloOffer + if err := json.Unmarshal(data, &o); err != nil { + return err + } + h.Offer = &o + case "accept": + var a HelloAccept + if err := json.Unmarshal(data, &a); err != nil { + return err + } + h.Accept = &a + default: + return fmt.Errorf("hello body: unknown stage %q", disc.Stage) + } + return nil +} + +// ---- resume (messages/resume.schema.json) ---- + +type ResumeBody struct { + LastRevision int64 `json:"last_revision"` +} + +// ---- snapshot (messages/snapshot.schema.json) ---- + +type SnapshotBody struct { + Revision int64 `json:"revision"` + Part int `json:"part"` + Final bool `json:"final"` + Tasks []TaskState `json:"tasks"` + Metrics []MetricSample `json:"metrics"` + Messages []MessageRecord `json:"messages"` + Automations []AutomationState `json:"automations"` +} + +// ---- reliable events ---- + +// TaskEventBody mirrors messages/task.event.schema.json#/$defs/body. +type TaskEventBody struct { + DeviceID string `json:"device_id"` + DeviceSeq int64 `json:"device_seq"` + TaskID string `json:"task_id"` + Kind string `json:"kind"` // started|progress|step|done|failed + OccurredAt int64 `json:"occurred_at"` + Title string `json:"title,omitempty"` + Percent *int `json:"percent,omitempty"` + Step string `json:"step,omitempty"` + Message string `json:"message,omitempty"` + Display *DisplayHints `json:"display,omitempty"` +} + +// MessageEventBody mirrors messages/message.event.schema.json#/$defs/body. +type MessageEventBody struct { + DeviceID string `json:"device_id"` + DeviceSeq int64 `json:"device_seq"` + MessageID string `json:"message_id"` + Level string `json:"level"` // info|warn|error + Text string `json:"text"` + OccurredAt int64 `json:"occurred_at"` + AutomationID string `json:"automation_id,omitempty"` +} + +// ConfigEventBody mirrors messages/config.event.schema.json#/$defs/body. +// Server-minted only: no device_id/device_seq (SPEC.md section 5.5). +type ConfigEventBody struct { + Kind string `json:"kind"` // automation.upserted|automation.removed + AutomationID string `json:"automation_id"` + Automation *AutomationState `json:"automation,omitempty"` + OccurredAt int64 `json:"occurred_at"` +} + +// ---- metric.frame (messages/metric.frame.schema.json) ---- + +type MetricFrameBody struct { + DeviceID string `json:"device_id"` + Metrics []MetricSample `json:"metrics"` +} + +// ---- delta (messages/delta.schema.json) ---- + +// DeltaEventItem is one oneOf{task.event, message.event, config.event} entry +// of delta.body.events, tagged by event_type. +type DeltaEventItem struct { + EventType string + TaskEvent *TaskEventBody + MessageEvent *MessageEventBody + ConfigEvent *ConfigEventBody +} + +func (d DeltaEventItem) MarshalJSON() ([]byte, error) { + var wrapper struct { + EventType string `json:"event_type"` + Event any `json:"event"` + } + wrapper.EventType = d.EventType + switch d.EventType { + case TypeTaskEvent: + wrapper.Event = d.TaskEvent + case TypeMessageEvent: + wrapper.Event = d.MessageEvent + case TypeConfigEvent: + wrapper.Event = d.ConfigEvent + default: + return nil, fmt.Errorf("delta event: unknown event_type %q", d.EventType) + } + return json.Marshal(wrapper) +} + +func (d *DeltaEventItem) UnmarshalJSON(data []byte) error { + var wrapper struct { + EventType string `json:"event_type"` + Event json.RawMessage `json:"event"` + } + if err := json.Unmarshal(data, &wrapper); err != nil { + return err + } + d.EventType = wrapper.EventType + switch wrapper.EventType { + case TypeTaskEvent: + var b TaskEventBody + if err := json.Unmarshal(wrapper.Event, &b); err != nil { + return err + } + d.TaskEvent = &b + case TypeMessageEvent: + var b MessageEventBody + if err := json.Unmarshal(wrapper.Event, &b); err != nil { + return err + } + d.MessageEvent = &b + case TypeConfigEvent: + var b ConfigEventBody + if err := json.Unmarshal(wrapper.Event, &b); err != nil { + return err + } + d.ConfigEvent = &b + default: + return fmt.Errorf("delta event: unknown event_type %q", wrapper.EventType) + } + return nil +} + +type DeltaBody struct { + FromRevision int64 `json:"from_revision"` + ToRevision int64 `json:"to_revision"` + Events []DeltaEventItem `json:"events"` +} + +// ---- ack (messages/ack.schema.json) ---- + +type AckedPair struct { + DeviceID string `json:"device_id"` + DeviceSeq int64 `json:"device_seq"` +} + +type Lease struct { + ExpiresAt int64 `json:"expires_at"` +} + +type AckBody struct { + Acked []AckedPair `json:"acked,omitempty"` + InReplyTo string `json:"in_reply_to,omitempty"` + Lease *Lease `json:"lease,omitempty"` +} + +// ---- subscribe / interest.renew (messages/subscribe.schema.json) ---- + +// SubscribeBody is shared verbatim by subscribe and interest.renew +// (interest.renew.schema.json $refs subscribe's body). +type SubscribeBody struct { + Topics []string `json:"topics,omitempty"` +} + +// UnsubscribeBody is intentionally empty (messages/unsubscribe.schema.json). +type UnsubscribeBody struct{} + +// ---- command (messages/command.schema.json) ---- + +type CommandBody struct { + CommandID string `json:"command_id"` + Origin string `json:"origin"` // viewer|server + IssuedByDeviceID string `json:"issued_by_device_id,omitempty"` + Action string `json:"action"` + TaskID string `json:"task_id,omitempty"` + AutomationID string `json:"automation_id,omitempty"` + TargetDeviceID string `json:"target_device_id,omitempty"` + TTLMs int64 `json:"ttl_ms"` + // Params is a pointer for the same reason as HelloOffer.Capabilities: + // an explicit "params": {} must round-trip distinctly from an absent + // field, even though both mean "no extension keys" (protocol v1 defines + // none) per messages/command.schema.json. + Params *map[string]any `json:"params,omitempty"` +} + +// ---- error (messages/error.schema.json) ---- + +type ErrorBody struct { + Code string `json:"code"` + Message string `json:"message"` + InReplyTo string `json:"in_reply_to,omitempty"` + Retryable *bool `json:"retryable"` + Fatal *bool `json:"fatal"` +} + +// Error codes, SPEC.md section 13. +const ( + ErrVersionUnsupported = "version_unsupported" + ErrHelloRequired = "hello_required" + ErrUnauthenticated = "unauthenticated" + ErrUnauthorized = "unauthorized" + ErrMalformed = "malformed" + ErrRateLimited = "rate_limited" + ErrFrameTooLarge = "frame_too_large" + ErrBatchTooLarge = "batch_too_large" + ErrRevisionUnavailable = "revision_unavailable" + ErrCommandExpired = "command_expired" + ErrSequenceGap = "sequence_gap" + ErrSuperseded = "superseded" + ErrInternal = "internal_error" +) diff --git a/daemon/internal/realtime/wire/envelope.go b/daemon/internal/realtime/wire/envelope.go new file mode 100644 index 0000000..8f16ee9 --- /dev/null +++ b/daemon/internal/realtime/wire/envelope.go @@ -0,0 +1,133 @@ +// Package wire implements the Go representation of the frozen Sitrep +// realtime protocol (proto/realtime/SPEC.md, v1.0.0). It mirrors the JSON +// Schemas under proto/realtime/ field-for-field: this package MUST NOT +// diverge from that spec, which is the sole source of truth. When in doubt, +// re-read proto/realtime/SPEC.md and the relevant messages/*.schema.json +// before changing anything here. +// +// The envelope's top level is permanently strict (SPEC.md section 3): every +// frame carries exactly type/id/ts/body and nothing else. The tolerance for +// unrecognized fields described in section 15 applies only inside `body`, +// which is why body structs are decoded with the standard (lenient) +// encoding/json behavior while the envelope itself is decoded with +// DisallowUnknownFields. +package wire + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" +) + +// MessageType enumerates the complete, closed set of 14 message types for +// protocol version 1 (SPEC.md section 4). No implementation may invent an +// additional type without a version negotiation both peers agree to. +const ( + TypeHello = "hello" + TypeResume = "resume" + TypeSnapshot = "snapshot" + TypeDelta = "delta" + TypeAck = "ack" + TypeTaskEvent = "task.event" + TypeMessageEvent = "message.event" + TypeMetricFrame = "metric.frame" + TypeConfigEvent = "config.event" + TypeSubscribe = "subscribe" + TypeUnsubscribe = "unsubscribe" + TypeInterestRenew = "interest.renew" + TypeCommand = "command" + TypeError = "error" +) + +// knownTypes is used by callers that must decide, per SPEC.md section 15, +// whether an unrecognized type should be silently ignored (not an error) +// rather than rejected as malformed. +var knownTypes = map[string]bool{ + TypeHello: true, TypeResume: true, TypeSnapshot: true, TypeDelta: true, + TypeAck: true, TypeTaskEvent: true, TypeMessageEvent: true, + TypeMetricFrame: true, TypeConfigEvent: true, TypeSubscribe: true, + TypeUnsubscribe: true, TypeInterestRenew: true, TypeCommand: true, + TypeError: true, +} + +// KnownType reports whether typ is one of the 14 registered message types. +func KnownType(typ string) bool { return knownTypes[typ] } + +// Envelope is the generic frame shape shared by every message +// (envelope.schema.json). Body is kept as raw JSON so a receiver can dispatch +// on Type before committing to a concrete body shape. +type Envelope struct { + Type string `json:"type"` + ID string `json:"id"` + TS int64 `json:"ts"` + Body json.RawMessage `json:"body"` +} + +// envelopeIDPattern / deviceIDPattern mirror common.schema.json's $defs. +var ( + envelopeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + deviceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) +) + +// ValidEnvelopeID reports whether id satisfies common.schema.json#/$defs/envelope_id. +func ValidEnvelopeID(id string) bool { return envelopeIDPattern.MatchString(id) } + +// ValidDeviceID reports whether id satisfies common.schema.json#/$defs/device_id. +func ValidDeviceID(id string) bool { return deviceIDPattern.MatchString(id) } + +// MinUnixMS / MaxUnixMS mirror common.schema.json#/$defs/unix_ms_timestamp's +// bound: any value in this range cannot plausibly be Unix seconds (10 digits, +// under 1e10) and is therefore guaranteed to be milliseconds. This is a +// deliberate mechanical guard against the historical seconds/milliseconds +// confusion called out in SPEC.md section 3.1. +const ( + MinUnixMS int64 = 1_000_000_000_000 + MaxUnixMS int64 = 9_999_999_999_999 +) + +// ValidUnixMS reports whether ts is a plausible Unix-milliseconds timestamp. +func ValidUnixMS(ts int64) bool { return ts >= MinUnixMS && ts <= MaxUnixMS } + +// DecodeEnvelope parses one frame's top level. It enforces the permanently +// strict envelope shape: exactly type/id/ts/body, no other top-level field +// (SPEC.md section 3) — an envelope carrying, e.g., a smuggled "token" field +// is rejected here with the same effect as error{code: malformed}. +func DecodeEnvelope(data []byte) (Envelope, error) { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + var env Envelope + if err := dec.Decode(&env); err != nil { + return Envelope{}, fmt.Errorf("malformed envelope: %w", err) + } + if env.Type == "" { + return Envelope{}, fmt.Errorf("malformed envelope: missing type") + } + if env.ID == "" { + return Envelope{}, fmt.Errorf("malformed envelope: missing id") + } + if !ValidEnvelopeID(env.ID) { + return Envelope{}, fmt.Errorf("malformed envelope: id %q does not match envelope_id pattern", env.ID) + } + if !ValidUnixMS(env.TS) { + return Envelope{}, fmt.Errorf("malformed envelope: ts %d is not a plausible unix-ms timestamp", env.TS) + } + if env.Body == nil { + return Envelope{}, fmt.Errorf("malformed envelope: missing body") + } + return env, nil +} + +// NewEnvelope marshals body and wraps it in an Envelope ready for encoding. +func NewEnvelope(typ, id string, ts int64, body any) (Envelope, error) { + raw, err := json.Marshal(body) + if err != nil { + return Envelope{}, err + } + return Envelope{Type: typ, ID: id, TS: ts, Body: raw}, nil +} + +// Encode serializes the envelope to its wire form. The envelope struct's own +// field set is exactly {type,id,ts,body}, so plain json.Marshal already +// satisfies the strict top level. +func (e Envelope) Encode() ([]byte, error) { return json.Marshal(e) } diff --git a/daemon/internal/realtime/wire/validate.go b/daemon/internal/realtime/wire/validate.go new file mode 100644 index 0000000..148038e --- /dev/null +++ b/daemon/internal/realtime/wire/validate.go @@ -0,0 +1,470 @@ +package wire + +import ( + "encoding/json" + "fmt" +) + +// Field length caps from common.schema.json. +const ( + freeTextMax = 2048 // $defs/free_text + labelTextMax = 256 // $defs/label_text + metricValueMax = 256 // metric_sample.value +) + +// DecodeBody decodes env.Body into the concrete Go type for env.Type and +// validates it against the semantic rules this package mirrors from the +// JSON Schemas (required fields, enums, ranges, conditional requirements). +// It returns an error for any envelope whose type is not one of the 14 +// registered types; per SPEC.md section 15 a caller that only needs +// forward-compatible tolerance should check KnownType(env.Type) before +// calling DecodeBody and ignore (not reject) an unrecognized type. +func DecodeBody(env Envelope) (any, error) { + switch env.Type { + case TypeHello: + var b HelloBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeResume: + var b ResumeBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeSnapshot: + var b SnapshotBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeDelta: + var b DeltaBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeAck: + var b AckBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeTaskEvent: + var b TaskEventBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeMessageEvent: + var b MessageEventBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeMetricFrame: + var b MetricFrameBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeConfigEvent: + var b ConfigEventBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeSubscribe, TypeInterestRenew: + var b SubscribeBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeUnsubscribe: + var b UnsubscribeBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + return b, nil + case TypeCommand: + var b CommandBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeError: + var b ErrorBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + default: + return nil, fmt.Errorf("unknown message type %q", env.Type) + } +} + +func (h HelloBody) Validate() error { + switch { + case h.Offer != nil: + o := h.Offer + if o.DeviceID == "" || !ValidDeviceID(o.DeviceID) { + return fmt.Errorf("hello offer: invalid device_id %q", o.DeviceID) + } + if o.Role != "source" && o.Role != "viewer" { + return fmt.Errorf("hello offer: invalid role %q", o.Role) + } + if len(o.ProtocolVersions) == 0 { + return fmt.Errorf("hello offer: protocol_versions must not be empty") + } + for _, v := range o.ProtocolVersions { + if v < 1 { + return fmt.Errorf("hello offer: protocol_versions entries must be >= 1") + } + } + return nil + case h.Accept != nil: + a := h.Accept + if a.ProtocolVersion < 1 { + return fmt.Errorf("hello accept: invalid protocol_version %d", a.ProtocolVersion) + } + if a.SessionID == "" || len(a.SessionID) > 64 { + return fmt.Errorf("hello accept: invalid session_id") + } + if a.HeartbeatIntervalMS < 1000 || a.HeartbeatIntervalMS > 300000 { + return fmt.Errorf("hello accept: heartbeat_interval_ms out of range") + } + return nil + default: + return fmt.Errorf("hello body: neither offer nor accept present") + } +} + +func (r ResumeBody) Validate() error { + if r.LastRevision < 0 { + return fmt.Errorf("resume: last_revision must be >= 0, got %d", r.LastRevision) + } + return nil +} + +func (s SnapshotBody) Validate() error { + if s.Revision < 0 { + return fmt.Errorf("snapshot: revision must be >= 0") + } + if s.Part < 1 { + return fmt.Errorf("snapshot: part must be >= 1") + } + for i, t := range s.Tasks { + if err := t.Validate(); err != nil { + return fmt.Errorf("snapshot: tasks[%d]: %w", i, err) + } + } + for i, m := range s.Metrics { + if err := m.Validate(); err != nil { + return fmt.Errorf("snapshot: metrics[%d]: %w", i, err) + } + } + return nil +} + +func (t TaskState) Validate() error { + if t.State != "running" && t.State != "done" && t.State != "failed" { + return fmt.Errorf("task_state: invalid state %q", t.State) + } + if !ValidUnixMS(t.UpdatedAt) { + return fmt.Errorf("task_state: updated_at is not a plausible unix-ms timestamp") + } + if len(t.Title) > freeTextMax || len(t.Step) > freeTextMax || len(t.Message) > freeTextMax { + return fmt.Errorf("task_state: free-text field exceeds %d chars", freeTextMax) + } + return nil +} + +func (m MetricSample) Validate() error { + if len(m.Value) > metricValueMax { + return fmt.Errorf("metric_sample: value exceeds %d chars", metricValueMax) + } + if len(m.Label) > labelTextMax { + return fmt.Errorf("metric_sample: label exceeds %d chars", labelTextMax) + } + if !ValidUnixMS(m.TS) { + return fmt.Errorf("metric_sample: ts is not a plausible unix-ms timestamp") + } + return nil +} + +func (b DeltaBody) Validate() error { + if b.FromRevision < 0 || b.ToRevision < 0 { + return fmt.Errorf("delta: revisions must be >= 0") + } + if b.ToRevision-b.FromRevision != int64(len(b.Events)) { + return fmt.Errorf("delta: to_revision - from_revision (%d) must equal events.length (%d)", + b.ToRevision-b.FromRevision, len(b.Events)) + } + for i, e := range b.Events { + switch e.EventType { + case TypeTaskEvent: + if e.TaskEvent == nil { + return fmt.Errorf("delta: events[%d]: missing task.event body", i) + } + if err := e.TaskEvent.Validate(); err != nil { + return fmt.Errorf("delta: events[%d]: %w", i, err) + } + case TypeMessageEvent: + if e.MessageEvent == nil { + return fmt.Errorf("delta: events[%d]: missing message.event body", i) + } + if err := e.MessageEvent.Validate(); err != nil { + return fmt.Errorf("delta: events[%d]: %w", i, err) + } + case TypeConfigEvent: + if e.ConfigEvent == nil { + return fmt.Errorf("delta: events[%d]: missing config.event body", i) + } + if err := e.ConfigEvent.Validate(); err != nil { + return fmt.Errorf("delta: events[%d]: %w", i, err) + } + default: + return fmt.Errorf("delta: events[%d]: unknown event_type %q", i, e.EventType) + } + } + return nil +} + +func (a AckBody) Validate() error { + if len(a.Acked) == 0 && a.InReplyTo == "" { + return fmt.Errorf("ack: at least one of acked or in_reply_to is required") + } + if a.InReplyTo != "" && !ValidEnvelopeID(a.InReplyTo) { + return fmt.Errorf("ack: in_reply_to is not a valid envelope id") + } + for i, p := range a.Acked { + if !ValidDeviceID(p.DeviceID) { + return fmt.Errorf("ack: acked[%d]: invalid device_id", i) + } + if p.DeviceSeq < 1 { + return fmt.Errorf("ack: acked[%d]: device_seq must be >= 1", i) + } + } + if a.Lease != nil && !ValidUnixMS(a.Lease.ExpiresAt) { + return fmt.Errorf("ack: lease.expires_at is not a plausible unix-ms timestamp") + } + return nil +} + +func (t TaskEventBody) Validate() error { + if !ValidDeviceID(t.DeviceID) { + return fmt.Errorf("task.event: invalid device_id %q", t.DeviceID) + } + if t.DeviceSeq < 1 { + return fmt.Errorf("task.event: device_seq must be >= 1 (missing or invalid)") + } + if t.TaskID == "" { + return fmt.Errorf("task.event: missing task_id") + } + switch t.Kind { + case "started", "progress", "step", "done", "failed": + default: + return fmt.Errorf("task.event: invalid kind %q", t.Kind) + } + if !ValidUnixMS(t.OccurredAt) { + return fmt.Errorf("task.event: occurred_at is not a plausible unix-ms timestamp") + } + if t.Kind == "progress" && t.Percent == nil { + return fmt.Errorf("task.event: percent is required when kind is progress") + } + if t.Percent != nil && (*t.Percent < 0 || *t.Percent > 100) { + return fmt.Errorf("task.event: percent out of range") + } + if len(t.Title) > freeTextMax || len(t.Step) > freeTextMax || len(t.Message) > freeTextMax { + return fmt.Errorf("task.event: free-text field exceeds %d chars", freeTextMax) + } + return nil +} + +func (m MessageEventBody) Validate() error { + if !ValidDeviceID(m.DeviceID) { + return fmt.Errorf("message.event: invalid device_id %q", m.DeviceID) + } + if m.DeviceSeq < 1 { + return fmt.Errorf("message.event: device_seq must be >= 1 (missing or invalid)") + } + if m.MessageID == "" { + return fmt.Errorf("message.event: missing message_id") + } + switch m.Level { + case "info", "warn", "error": + default: + return fmt.Errorf("message.event: invalid level %q", m.Level) + } + if len(m.Text) > freeTextMax { + return fmt.Errorf("message.event: text exceeds %d chars", freeTextMax) + } + if !ValidUnixMS(m.OccurredAt) { + return fmt.Errorf("message.event: occurred_at is not a plausible unix-ms timestamp") + } + return nil +} + +func (c ConfigEventBody) Validate() error { + switch c.Kind { + case "automation.upserted", "automation.removed": + default: + return fmt.Errorf("config.event: invalid kind %q", c.Kind) + } + if c.AutomationID == "" { + return fmt.Errorf("config.event: missing automation_id") + } + if c.Kind == "automation.upserted" && c.Automation == nil { + return fmt.Errorf("config.event: automation is required when kind is automation.upserted") + } + if !ValidUnixMS(c.OccurredAt) { + return fmt.Errorf("config.event: occurred_at is not a plausible unix-ms timestamp") + } + return nil +} + +func (m MetricFrameBody) Validate() error { + if !ValidDeviceID(m.DeviceID) { + return fmt.Errorf("metric.frame: invalid device_id %q", m.DeviceID) + } + if len(m.Metrics) == 0 { + return fmt.Errorf("metric.frame: metrics must not be empty") + } + if len(m.Metrics) > 64 { + return fmt.Errorf("metric.frame: metrics exceeds 64-item cap") + } + seen := make(map[string]bool, len(m.Metrics)) + for i, s := range m.Metrics { + if err := s.Validate(); err != nil { + return fmt.Errorf("metric.frame: metrics[%d]: %w", i, err) + } + if seen[s.MetricID] { + return fmt.Errorf("metric.frame: duplicate metric_id %q in one frame", s.MetricID) + } + seen[s.MetricID] = true + } + return nil +} + +var validTopics = map[string]bool{"task": true, "metric": true, "message": true} + +func (s SubscribeBody) Validate() error { + seen := make(map[string]bool, len(s.Topics)) + for _, t := range s.Topics { + if !validTopics[t] { + return fmt.Errorf("subscribe: unknown topic %q", t) + } + if seen[t] { + return fmt.Errorf("subscribe: duplicate topic %q", t) + } + seen[t] = true + } + return nil +} + +func (c CommandBody) Validate() error { + if c.CommandID == "" || len(c.CommandID) > 64 { + return fmt.Errorf("command: invalid command_id") + } + if c.Origin != "viewer" && c.Origin != "server" { + return fmt.Errorf("command: invalid origin %q", c.Origin) + } + if c.TTLMs < 1 || c.TTLMs > 86400000 { + return fmt.Errorf("command: ttl_ms out of range") + } + switch c.Action { + case "pause", "resume", "stop": + if c.Origin != "viewer" { + return fmt.Errorf("command: action %q requires origin viewer", c.Action) + } + if c.IssuedByDeviceID == "" { + return fmt.Errorf("command: action %q requires issued_by_device_id", c.Action) + } + if c.TaskID == "" { + return fmt.Errorf("command: action %q requires task_id", c.Action) + } + if c.AutomationID != "" { + return fmt.Errorf("command: action %q must not carry automation_id", c.Action) + } + case "run_now": + if c.Origin != "viewer" { + return fmt.Errorf("command: action run_now requires origin viewer") + } + if c.IssuedByDeviceID == "" { + return fmt.Errorf("command: action run_now requires issued_by_device_id") + } + if c.AutomationID == "" { + return fmt.Errorf("command: action run_now requires automation_id") + } + if c.TaskID != "" { + return fmt.Errorf("command: action run_now must not carry task_id") + } + case "throttle", "resume_rate": + if c.Origin != "server" { + return fmt.Errorf("command: action %q requires origin server", c.Action) + } + if c.TaskID != "" || c.AutomationID != "" { + return fmt.Errorf("command: action %q must not carry task_id or automation_id", c.Action) + } + default: + return fmt.Errorf("command: invalid action %q", c.Action) + } + return nil +} + +func (e ErrorBody) Validate() error { + if e.Code == "" { + return fmt.Errorf("error: missing code") + } + switch e.Code { + case ErrVersionUnsupported, ErrHelloRequired, ErrUnauthenticated, ErrUnauthorized, + ErrMalformed, ErrRateLimited, ErrFrameTooLarge, ErrBatchTooLarge, + ErrRevisionUnavailable, ErrCommandExpired, ErrSequenceGap, ErrSuperseded, ErrInternal: + default: + return fmt.Errorf("error: unknown code %q", e.Code) + } + if len(e.Message) > 500 { + return fmt.Errorf("error: message exceeds 500 chars") + } + if e.Retryable == nil || e.Fatal == nil { + return fmt.Errorf("error: retryable and fatal are both required") + } + return nil +} diff --git a/daemon/internal/realtime/wire/wire_test.go b/daemon/internal/realtime/wire/wire_test.go new file mode 100644 index 0000000..0f4b0b1 --- /dev/null +++ b/daemon/internal/realtime/wire/wire_test.go @@ -0,0 +1,202 @@ +package wire + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" +) + +// fixturesDir locates proto/realtime/fixtures relative to this source file, +// independent of the test runner's working directory. +func fixturesDir(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + dir := filepath.Dir(file) // .../daemon/internal/realtime/wire + root := filepath.Join(dir, "..", "..", "..", "..") + fixtures := filepath.Join(root, "proto", "realtime", "fixtures") + if _, err := os.Stat(fixtures); err != nil { + t.Fatalf("fixtures dir not found at %s: %v", fixtures, err) + } + return fixtures +} + +func listJSONFiles(t *testing.T, dir string) []string { + t.Helper() + var out []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(path) == ".json" { + out = append(out, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + sort.Strings(out) + return out +} + +// normalize decodes JSON into a generic interface{} tree so two +// differently-formatted (but semantically identical) documents compare +// equal with reflect.DeepEqual. +func normalize(t *testing.T, data []byte) any { + t.Helper() + var v any + if err := json.Unmarshal(data, &v); err != nil { + t.Fatalf("normalize: %v", err) + } + return v +} + +// TestValidFixturesRoundTrip decodes every bare-envelope fixture under +// fixtures/valid and fixtures/scenarios, re-encodes it through the Go +// types, and asserts the result is semantically identical to the original. +// This is the "Go types match the schema byte for byte" check called for by +// the realtime daemon work order: if a field were missing, renamed, or +// mistyped in bodies.go, this test would lose or corrupt it on round trip. +func TestValidFixturesRoundTrip(t *testing.T) { + fixtures := fixturesDir(t) + dirs := []string{ + filepath.Join(fixtures, "valid"), + filepath.Join(fixtures, "scenarios"), + } + tested := 0 + for _, dir := range dirs { + for _, path := range listJSONFiles(t, dir) { + if filepath.Base(path) == "README.md" { + continue + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + t.Run(relPath(fixtures, path), func(t *testing.T) { + env, err := DecodeEnvelope(data) + if err != nil { + t.Fatalf("DecodeEnvelope: %v", err) + } + body, err := DecodeBody(env) + if err != nil { + t.Fatalf("DecodeBody: %v", err) + } + reEncodedBody, err := json.Marshal(body) + if err != nil { + t.Fatalf("re-marshal body: %v", err) + } + roundTripped := Envelope{Type: env.Type, ID: env.ID, TS: env.TS, Body: reEncodedBody} + out, err := roundTripped.Encode() + if err != nil { + t.Fatalf("Encode: %v", err) + } + want := normalize(t, data) + got := normalize(t, out) + if !reflect.DeepEqual(want, got) { + t.Fatalf("round trip mismatch:\n original: %s\n produced: %s", data, out) + } + }) + tested++ + } + } + if tested == 0 { + t.Fatal("no valid fixtures were exercised") + } + t.Logf("round-tripped %d fixtures", tested) +} + +func relPath(base, path string) string { + rel, err := filepath.Rel(base, path) + if err != nil { + return path + } + return rel +} + +// roleFixture is the {"sender_role": ..., "frame": {...}} wrapper format +// used by a handful of invalid fixtures to test the section 10.1 +// authorization matrix rather than pure schema shape (SPEC.md section 16). +type roleFixture struct { + SenderRole string `json:"sender_role"` + Frame json.RawMessage `json:"frame"` +} + +func isRoleFixture(data []byte) (roleFixture, bool) { + var rf roleFixture + if err := json.Unmarshal(data, &rf); err != nil { + return roleFixture{}, false + } + return rf, rf.SenderRole != "" && rf.Frame != nil +} + +// TestInvalidFixturesRejected asserts every fixture under fixtures/invalid +// is rejected by this package, either at envelope decode, body decode/ +// validate, or (for the sender_role-wrapped fixtures) the authorization +// matrix. +func TestInvalidFixturesRejected(t *testing.T) { + fixtures := fixturesDir(t) + dir := filepath.Join(fixtures, "invalid") + tested := 0 + for _, path := range listJSONFiles(t, dir) { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + t.Run(relPath(fixtures, path), func(t *testing.T) { + if rf, ok := isRoleFixture(data); ok { + env, err := DecodeEnvelope(rf.Frame) + if err != nil { + return // rejected at envelope level: acceptable + } + body, err := DecodeBody(env) + if err != nil { + return // rejected at body validation: acceptable + } + if err := Authorize(Role(rf.SenderRole), env.Type, body); err == nil { + t.Fatalf("expected rejection for role %q sending %q, got none", rf.SenderRole, env.Type) + } + return + } + env, err := DecodeEnvelope(data) + if err != nil { + return // rejected at envelope level: acceptable + } + if _, err := DecodeBody(env); err == nil { + t.Fatalf("expected DecodeBody to reject fixture, got none") + } + }) + tested++ + } + if tested == 0 { + t.Fatal("no invalid fixtures were exercised") + } + t.Logf("checked %d invalid fixtures", tested) +} + +// TestUnixMSBoundary pins the seconds/milliseconds guard from +// common.schema.json#/$defs/unix_ms_timestamp (SPEC.md section 3.1). +func TestUnixMSBoundary(t *testing.T) { + cases := []struct { + ts int64 + want bool + }{ + {1000000000000, true}, // exact minimum + {9999999999999, true}, // exact maximum + {999999999999, false}, // one below minimum + {10000000000000, false}, // one above maximum + {1752480006, false}, // a plausible *seconds* value must be rejected + } + for _, c := range cases { + if got := ValidUnixMS(c.ts); got != c.want { + t.Errorf("ValidUnixMS(%d) = %v, want %v", c.ts, got, c.want) + } + } +} From b938c73e7d64078c02f9379e0207c9682cb926a9 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:41:23 +0800 Subject: [PATCH 05/60] feat(daemon): add crash-safe outbox for reliable realtime events Add internal/realtime/outbox, a SQLite-backed (modernc.org/sqlite, pure Go, no cgo) local queue for task.event/message.event delivery. It owns the per-(device, space) device_seq counter from SPEC.md section 5.1: allocating the next sequence number and durably persisting the event it belongs to happen in one transaction, so a crash between the two is impossible to observe as a gap or a double-allocation. Enqueue/Ack/ Pending give a source device exactly what section 5.3/5.4 need: keep every unacked event until its ack arrives, and replay outstanding ones oldest-first after a reconnect or a process restart. Tests cover monotonic and concurrent-safe allocation, the crash-safety property (an aborted transaction must not skip a sequence number), ack-then-delete, oldest-first ordering, and full survival across a simulated process restart (close and reopen the same database file). --- daemon/go.mod | 15 ++ daemon/go.sum | 49 ++++ daemon/internal/realtime/outbox/outbox.go | 205 +++++++++++++++ .../internal/realtime/outbox/outbox_test.go | 248 ++++++++++++++++++ 4 files changed, 517 insertions(+) create mode 100644 daemon/go.sum create mode 100644 daemon/internal/realtime/outbox/outbox.go create mode 100644 daemon/internal/realtime/outbox/outbox_test.go diff --git a/daemon/go.mod b/daemon/go.mod index b1a7e47..b654100 100644 --- a/daemon/go.mod +++ b/daemon/go.mod @@ -1,3 +1,18 @@ module github.com/QuintinShaw/sitrep/daemon go 1.26 + +require modernc.org/sqlite v1.39.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/sys v0.34.0 // indirect + modernc.org/libc v1.66.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/daemon/go.sum b/daemon/go.sum new file mode 100644 index 0000000..c46fb2d --- /dev/null +++ b/daemon/go.sum @@ -0,0 +1,49 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= +modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= +modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= +modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/daemon/internal/realtime/outbox/outbox.go b/daemon/internal/realtime/outbox/outbox.go new file mode 100644 index 0000000..5f169b7 --- /dev/null +++ b/daemon/internal/realtime/outbox/outbox.go @@ -0,0 +1,205 @@ +// Package outbox is the local, crash-safe queue of reliable realtime events +// (task.event / message.event, SPEC.md section 5) awaiting acknowledgement +// from the server. It also owns the per-(device, space) device_seq counter +// (SPEC.md section 5.1): allocating the next sequence number and durably +// enqueueing the event it belongs to happen in a single SQLite transaction, +// so a crash between "I picked seq N" and "event N is safely on disk" is +// impossible — either both happened, or neither did, and the next call sees +// the same N again. +// +// Storage is modernc.org/sqlite, a pure-Go (no cgo) SQLite driver, chosen so +// the daemon keeps its zero-cgo build. The database file lives alongside +// the existing ~/.config/sitrep/config.json (internal/config.Path). +package outbox + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "time" + + _ "modernc.org/sqlite" +) + +const schema = ` +CREATE TABLE IF NOT EXISTS space_seq_counters ( + space TEXT PRIMARY KEY, + next_seq INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS outbox ( + space TEXT NOT NULL, + device_seq INTEGER NOT NULL, + kind TEXT NOT NULL, + body TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (space, device_seq) +); +` + +// Item is one persisted, not-yet-acknowledged reliable event. +type Item struct { + Space string + DeviceSeq int64 + Kind string // "task.event" | "message.event" + Body json.RawMessage + CreatedAt int64 // unix ms +} + +// Store is the outbox's SQLite-backed handle. It is safe for concurrent use +// from multiple goroutines: every method serializes through the underlying +// *sql.DB, and every mutating operation is one transaction. +type Store struct { + db *sql.DB +} + +// Open creates (if needed) and opens the outbox database at path. Callers +// should keep one *Store per daemon process for a given path. +func Open(path string) (*Store, error) { + // _txlock=immediate: acquire the write lock at BEGIN rather than at + // first write, so two overlapping transactions serialize instead of + // racing to upgrade a deferred lock (SQLITE_BUSY under concurrent + // Enqueue calls from multiple goroutines). + dsn := path + "?_pragma=busy_timeout(5000)&_txlock=immediate" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("outbox: open %s: %w", path, err) + } + // SQLite allows only one writer at a time; a single connection avoids + // spurious SQLITE_BUSY errors from the driver opening a second one. + db.SetMaxOpenConns(1) + if _, err := db.Exec(schema); err != nil { + db.Close() + return nil, fmt.Errorf("outbox: init schema: %w", err) + } + return &Store{db: db}, nil +} + +// Close releases the underlying database handle. +func (s *Store) Close() error { return s.db.Close() } + +// BuildBody produces the JSON body for a reliable event once its device_seq +// is known — Enqueue calls it inside the allocating transaction so the +// caller never has to allocate a seq itself ahead of time. +type BuildBody func(seq int64) (json.RawMessage, error) + +// Enqueue allocates the next device_seq for space and durably stores the +// event BuildBody produces for it, atomically: SPEC.md section 5.1 requires +// device_seq to be assigned once, never reused, and strictly increasing per +// (device, space); doing the allocation and the insert in one transaction +// means a process crash between them can never happen — the transaction +// either commits both effects or neither, so a restarted daemon reusing this +// same database file always sees a counter that agrees with what is (or +// isn't) sitting in the outbox. +func (s *Store) Enqueue(ctx context.Context, space, kind string, build BuildBody) (Item, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Item{}, fmt.Errorf("outbox: begin: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + + seq, err := nextSeqTx(tx, space) + if err != nil { + return Item{}, err + } + body, err := build(seq) + if err != nil { + return Item{}, fmt.Errorf("outbox: build body for seq %d: %w", seq, err) + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, + `INSERT INTO outbox (space, device_seq, kind, body, created_at) VALUES (?, ?, ?, ?, ?)`, + space, seq, kind, string(body), now, + ); err != nil { + return Item{}, fmt.Errorf("outbox: insert seq %d: %w", seq, err) + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO space_seq_counters (space, next_seq) VALUES (?, ?) + ON CONFLICT(space) DO UPDATE SET next_seq = excluded.next_seq`, + space, seq+1, + ); err != nil { + return Item{}, fmt.Errorf("outbox: advance counter: %w", err) + } + if err := tx.Commit(); err != nil { + return Item{}, fmt.Errorf("outbox: commit: %w", err) + } + return Item{Space: space, DeviceSeq: seq, Kind: kind, Body: body, CreatedAt: now}, nil +} + +// nextSeqTx reads and does NOT yet persist the next device_seq for space; +// the caller is expected to write space_seq_counters within the same +// transaction once it has successfully built and inserted the event that +// consumes this value (see Enqueue). Device_seq starts at 1 (SPEC.md +// section 5.1). +func nextSeqTx(tx *sql.Tx, space string) (int64, error) { + var next int64 + err := tx.QueryRow(`SELECT next_seq FROM space_seq_counters WHERE space = ?`, space).Scan(&next) + switch { + case err == sql.ErrNoRows: + return 1, nil + case err != nil: + return 0, fmt.Errorf("outbox: read counter: %w", err) + default: + return next, nil + } +} + +// NextSeq reports the device_seq the next Enqueue(space, ...) call will +// allocate, without allocating it. Intended for tests and diagnostics. +func (s *Store) NextSeq(ctx context.Context, space string) (int64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() //nolint:errcheck + return nextSeqTx(tx, space) +} + +// Ack deletes exactly the (space, device_seq) pair, per SPEC.md section 5.3 +// ("A source device MUST keep every sent task.event/message.event in a +// local resend queue until it receives an ack that covers its device_seq"). +// Acking a seq that is not present (already acked, or never enqueued here) +// is a no-op, not an error — acks are idempotent (SPEC.md section 5.2). +func (s *Store) Ack(ctx context.Context, space string, seq int64) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ? AND device_seq = ?`, space, seq) + if err != nil { + return fmt.Errorf("outbox: ack seq %d: %w", seq, err) + } + return nil +} + +// Pending returns every outstanding (unacknowledged) item for space, +// oldest-first by device_seq — the order SPEC.md section 5.3/5.4 requires +// for reconnect replay ("resend every still-unacked event, oldest first, +// immediately after hello/hello-accept completes on a new connection"). +// Because the outbox is durable, this also serves a process restart: the +// queue is exactly as it was left, so a freshly started daemon replays from +// disk with no special-cased recovery path. +func (s *Store) Pending(ctx context.Context, space string) ([]Item, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT space, device_seq, kind, body, created_at FROM outbox WHERE space = ? ORDER BY device_seq ASC`, + space) + if err != nil { + return nil, fmt.Errorf("outbox: query pending: %w", err) + } + defer rows.Close() + var items []Item + for rows.Next() { + var it Item + var body string + if err := rows.Scan(&it.Space, &it.DeviceSeq, &it.Kind, &body, &it.CreatedAt); err != nil { + return nil, fmt.Errorf("outbox: scan pending: %w", err) + } + it.Body = json.RawMessage(body) + items = append(items, it) + } + return items, rows.Err() +} + +// Count returns the number of unacknowledged items across every space, +// mainly for observability/tests. +func (s *Store) Count(ctx context.Context) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&n) + return n, err +} diff --git a/daemon/internal/realtime/outbox/outbox_test.go b/daemon/internal/realtime/outbox/outbox_test.go new file mode 100644 index 0000000..9cf9697 --- /dev/null +++ b/daemon/internal/realtime/outbox/outbox_test.go @@ -0,0 +1,248 @@ +package outbox + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "sync" + "testing" +) + +func open(t *testing.T) *Store { + t.Helper() + path := filepath.Join(t.TempDir(), "outbox.db") + s, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func bodyFor(seq int64) BuildBody { + return func(s int64) (json.RawMessage, error) { + return json.RawMessage(fmt.Sprintf(`{"device_seq":%d}`, s)), nil + } +} + +func TestEnqueueAllocatesMonotonicSeq(t *testing.T) { + s := open(t) + ctx := context.Background() + for want := int64(1); want <= 5; want++ { + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(want)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if item.DeviceSeq != want { + t.Fatalf("Enqueue seq = %d, want %d", item.DeviceSeq, want) + } + } + // A second space starts its own counter at 1 (device_seq is scoped to + // one (device, space) pair, SPEC.md section 5.1). + item, err := s.Enqueue(ctx, "space-b", "task.event", bodyFor(1)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if item.DeviceSeq != 1 { + t.Fatalf("space-b first seq = %d, want 1", item.DeviceSeq) + } +} + +func TestEnqueueConcurrentIsMonotonicAndUnique(t *testing.T) { + s := open(t) + ctx := context.Background() + const n = 50 + seqs := make([]int64, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if err != nil { + t.Errorf("Enqueue: %v", err) + return + } + seqs[i] = item.DeviceSeq + }(i) + } + wg.Wait() + seen := make(map[int64]bool, n) + for _, seq := range seqs { + if seq < 1 { + t.Fatalf("got invalid seq %d", seq) + } + if seen[seq] { + t.Fatalf("seq %d allocated twice", seq) + } + seen[seq] = true + } + if len(seen) != n { + t.Fatalf("got %d unique seqs, want %d", len(seen), n) + } +} + +// TestEnqueueFailureLeavesNoGap asserts the crash-safety property required +// by the work order: allocating a device_seq and durably enqueueing the +// event it belongs to happen in ONE transaction. If building the event body +// fails after the seq was chosen but before the transaction commits, the +// whole transaction MUST roll back — including the counter advance — so the +// next successful Enqueue reuses the same seq rather than skipping it. This +// stands in for an actual process crash between "seq chosen" and "event +// durably stored": from the database's perspective the two are +// indistinguishable, and both must leave the counter exactly where it was. +func TestEnqueueFailureLeavesNoGap(t *testing.T) { + s := open(t) + ctx := context.Background() + + boom := errors.New("boom") + _, err := s.Enqueue(ctx, "space-a", "task.event", func(seq int64) (json.RawMessage, error) { + if seq != 1 { + t.Fatalf("expected first attempted seq to be 1, got %d", seq) + } + return nil, boom + }) + if !errors.Is(err, boom) { + t.Fatalf("Enqueue error = %v, want wrapping %v", err, boom) + } + + next, err := s.NextSeq(ctx, "space-a") + if err != nil { + t.Fatalf("NextSeq: %v", err) + } + if next != 1 { + t.Fatalf("NextSeq after failed build = %d, want 1 (no gap left by the aborted transaction)", next) + } + + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(1)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if item.DeviceSeq != 1 { + t.Fatalf("seq after retry = %d, want 1 (reused, not skipped)", item.DeviceSeq) + } + + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("Pending returned %d items, want exactly 1 (the failed build must not have left a row)", len(pending)) + } +} + +func TestEnqueueAckDeletes(t *testing.T) { + s := open(t) + ctx := context.Background() + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(1)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil || len(pending) != 1 { + t.Fatalf("Pending = %v, %v; want 1 item", pending, err) + } + if err := s.Ack(ctx, "space-a", item.DeviceSeq); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err = s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 0 { + t.Fatalf("Pending after ack = %v, want empty", pending) + } + // Acking an already-acked (or never-enqueued) seq is a no-op, not an + // error (SPEC.md section 5.2: acking a duplicate is always safe). + if err := s.Ack(ctx, "space-a", item.DeviceSeq); err != nil { + t.Fatalf("double Ack: %v", err) + } + if err := s.Ack(ctx, "space-a", 999); err != nil { + t.Fatalf("Ack of unknown seq: %v", err) + } +} + +func TestPendingOrderedOldestFirst(t *testing.T) { + s := open(t) + ctx := context.Background() + for i := int64(1); i <= 5; i++ { + if _, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(i)); err != nil { + t.Fatalf("Enqueue: %v", err) + } + } + // Ack the middle one out of order to make sure ordering survives gaps. + if err := s.Ack(ctx, "space-a", 3); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + want := []int64{1, 2, 4, 5} + if len(pending) != len(want) { + t.Fatalf("Pending = %d items, want %d", len(pending), len(want)) + } + for i, w := range want { + if pending[i].DeviceSeq != w { + t.Fatalf("Pending[%d].DeviceSeq = %d, want %d (order: %v)", i, pending[i].DeviceSeq, w, pending) + } + } +} + +// TestSurvivesRestart simulates a daemon process restart: closing the Store +// and reopening the same database file must preserve both the pending +// queue (for replay) and the device_seq counter (so a restarted source +// never reuses or skips a sequence number). +func TestSurvivesRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + ctx := context.Background() + + s1, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + if _, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(1)); err != nil { + t.Fatalf("Enqueue: %v", err) + } + if _, err := s1.Enqueue(ctx, "space-a", "message.event", bodyFor(2)); err != nil { + t.Fatalf("Enqueue: %v", err) + } + // This one gets acked before the "restart" — it must NOT reappear. + item3, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if err := s1.Ack(ctx, "space-a", item3.DeviceSeq); err != nil { + t.Fatalf("Ack: %v", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + s2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + pending, err := s2.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending after restart: %v", err) + } + if len(pending) != 2 { + t.Fatalf("Pending after restart = %d items, want 2 (got %v)", len(pending), pending) + } + if pending[0].DeviceSeq != 1 || pending[1].DeviceSeq != 2 { + t.Fatalf("Pending after restart out of order: %v", pending) + } + + next, err := s2.Enqueue(ctx, "space-a", "task.event", bodyFor(4)) + if err != nil { + t.Fatalf("Enqueue after restart: %v", err) + } + if next.DeviceSeq != 4 { + t.Fatalf("seq after restart = %d, want 4 (counter must survive restart)", next.DeviceSeq) + } +} From 941828ad6a4a830fa402f12dbedbfd33a94f83be Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:41:50 +0800 Subject: [PATCH 06/60] feat(daemon): batch ephemeral metric frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the metric batcher for internal/realtime/client: merges multiple pending updates to the same metric_id into a single latest sample before emission (SPEC.md section 12), and gates emission at a bounded cadence — no metric_id is re-included more than once per 500ms (the protocol's 2 Hz per-metric ceiling), and the whole batcher only flushes at its own current interval, switchable between a normal and a slower "throttled" cadence for the command{throttle}/{resume_rate} pair from SPEC.md section 7. A sample offered while nothing is due simply overwrites the pending entry for its metric_id rather than queueing, matching metric.frame's best-effort, never-persisted, never-retried nature (section 12): only the latest value survives to be sent. It is a pure, clock-driven data structure with no timer of its own, so every merge/rate-limit/throttle-switch behavior is tested by calling Flush with an explicit instant — no sleeping real durations. --- daemon/internal/realtime/client/metrics.go | 117 ++++++++++++++++++ .../internal/realtime/client/metrics_test.go | 117 ++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 daemon/internal/realtime/client/metrics.go create mode 100644 daemon/internal/realtime/client/metrics_test.go diff --git a/daemon/internal/realtime/client/metrics.go b/daemon/internal/realtime/client/metrics.go new file mode 100644 index 0000000..b36529c --- /dev/null +++ b/daemon/internal/realtime/client/metrics.go @@ -0,0 +1,117 @@ +package client + +import ( + "sync" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// metricBatcher merges pending metric samples per metric_id (SPEC.md +// section 12: "A sender MUST merge multiple pending updates to the same +// metric_id into a single sample ... before emitting a frame") and gates +// emission at a bounded cadence: no metric_id may be re-included more than +// once per 500ms (the protocol's 2 Hz per-metric ceiling), and Flush is +// itself only "due" at the batcher's current interval (normal or, while +// throttled, the slower one), which — kept at or above 100ms by the +// client's poll loop — also keeps the connection under the 10 frames/sec +// cap (section 11). +// +// It is a pure, clock-driven data structure with no goroutine or timer of +// its own: the caller decides when to call Flush. This makes it directly +// unit-testable (inject instants, no sleeping) and keeps the "when do we +// actually write to the socket" decision in the Client, which knows whether +// a connection currently exists. +// +// metric.frame is never persisted and never acknowledged (SPEC.md section +// 12): a sample that arrives while disconnected simply overwrites the +// pending entry for its metric_id, same as any other merge — nothing is +// queued, and only the latest value survives to be sent after reconnect. +type metricBatcher struct { + mu sync.Mutex + + pending map[string]wire.MetricSample + + normalInterval time.Duration + throttledInterval time.Duration + throttled bool + + lastFlushAt time.Time // wall-clock time of the last non-empty Flush + lastSentAt map[string]time.Time // per metric_id: wall-clock time last included in a sent frame +} + +func newMetricBatcher(normal, throttled time.Duration) *metricBatcher { + return &metricBatcher{ + pending: make(map[string]wire.MetricSample), + lastSentAt: make(map[string]time.Time), + normalInterval: normal, + throttledInterval: throttled, + } +} + +// Offer merges one sample in: last value wins per metric_id. +func (b *metricBatcher) Offer(s wire.MetricSample) { + b.mu.Lock() + defer b.mu.Unlock() + b.pending[s.MetricID] = s +} + +// SetThrottled switches the batcher between its normal and throttled +// cadence (SPEC.md section 7: command{throttle}/{resume_rate}). +func (b *metricBatcher) SetThrottled(throttled bool) { + b.mu.Lock() + defer b.mu.Unlock() + b.throttled = throttled +} + +// Throttled reports the batcher's current cadence state, mainly for +// observability and tests. +func (b *metricBatcher) Throttled() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.throttled +} + +func (b *metricBatcher) interval() time.Duration { + if b.throttled { + return b.throttledInterval + } + return b.normalInterval +} + +// Flush returns the next metric.frame's worth of samples if the batcher's +// current interval has elapsed since the last emission AND at least one +// pending sample clears its own per-metric_id 500ms gate. Samples that +// don't clear the gate are held back — merged with whatever arrives next — +// rather than dropped; a call that finds nothing eligible returns +// (Body{}, false) and does not disturb any bookkeeping. +func (b *metricBatcher) Flush(now time.Time) ([]wire.MetricSample, bool) { + const perMetricGate = 500 * time.Millisecond + + b.mu.Lock() + defer b.mu.Unlock() + + if !b.lastFlushAt.IsZero() && now.Sub(b.lastFlushAt) < b.interval() { + return nil, false + } + if len(b.pending) == 0 { + return nil, false + } + + var out []wire.MetricSample + for id, sample := range b.pending { + if last, ok := b.lastSentAt[id]; ok && now.Sub(last) < perMetricGate { + continue // held back for a later cycle, not dropped + } + out = append(out, sample) + } + if len(out) == 0 { + return nil, false + } + for _, s := range out { + delete(b.pending, s.MetricID) + b.lastSentAt[s.MetricID] = now + } + b.lastFlushAt = now + return out, true +} diff --git a/daemon/internal/realtime/client/metrics_test.go b/daemon/internal/realtime/client/metrics_test.go new file mode 100644 index 0000000..14de641 --- /dev/null +++ b/daemon/internal/realtime/client/metrics_test.go @@ -0,0 +1,117 @@ +package client + +import ( + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +func sample(id, value string) wire.MetricSample { + return wire.MetricSample{MetricID: id, Value: value, TS: wire.MinUnixMS + 1000} +} + +func TestMetricBatcherMergesMultipleSamplesIntoOneFrame(t *testing.T) { + b := newMetricBatcher(100*time.Millisecond, 30*time.Second) + base := time.Unix(1700000000, 0) + + // Three updates to the same metric_id before any flush: only the + // latest must survive (SPEC.md section 12). + b.Offer(sample("cpu.load", "0.1")) + b.Offer(sample("cpu.load", "0.2")) + b.Offer(sample("cpu.load", "0.3")) + b.Offer(sample("mem.used_gb", "18.4")) + + out, ok := b.Flush(base) + if !ok { + t.Fatal("Flush: expected a due frame on first call") + } + if len(out) != 2 { + t.Fatalf("Flush returned %d samples, want 2 (one per metric_id)", len(out)) + } + byID := map[string]wire.MetricSample{} + for _, s := range out { + byID[s.MetricID] = s + } + if byID["cpu.load"].Value != "0.3" { + t.Fatalf("cpu.load = %q, want latest value 0.3", byID["cpu.load"].Value) + } +} + +func TestMetricBatcherRateLimitsPerMetric(t *testing.T) { + b := newMetricBatcher(10*time.Millisecond, 30*time.Second) + base := time.Unix(1700000000, 0) + + b.Offer(sample("cpu.load", "0.1")) + out, ok := b.Flush(base) + if !ok || len(out) != 1 { + t.Fatalf("first flush = %v, %v; want 1 sample", out, ok) + } + + // Immediately offer a new value and try to flush again well within the + // per-metric 500ms gate (even though the batcher's own interval has + // elapsed) — the sample must be held back, not dropped. + b.Offer(sample("cpu.load", "0.9")) + out, ok = b.Flush(base.Add(50 * time.Millisecond)) + if ok { + t.Fatalf("expected no due frame within the 500ms per-metric gate, got %v", out) + } + + // After the gate clears, the latest pending value (0.9) must appear. + out, ok = b.Flush(base.Add(600 * time.Millisecond)) + if !ok || len(out) != 1 || out[0].Value != "0.9" { + t.Fatalf("flush after gate = %v, %v; want [0.9]", out, ok) + } +} + +func TestMetricBatcherOverallIntervalGate(t *testing.T) { + b := newMetricBatcher(200*time.Millisecond, 30*time.Second) + base := time.Unix(1700000000, 0) + + b.Offer(sample("cpu.load", "0.1")) + if _, ok := b.Flush(base); !ok { + t.Fatal("expected first flush to be due") + } + b.Offer(sample("mem.used_gb", "1")) + if _, ok := b.Flush(base.Add(50 * time.Millisecond)); ok { + t.Fatal("expected flush before the batcher interval elapsed to be a no-op") + } + if out, ok := b.Flush(base.Add(250 * time.Millisecond)); !ok || len(out) != 1 { + t.Fatalf("flush after interval elapsed = %v, %v; want 1 sample", out, ok) + } +} + +func TestMetricBatcherThrottleSwitchesCadence(t *testing.T) { + b := newMetricBatcher(100*time.Millisecond, 10*time.Second) + base := time.Unix(1700000000, 0) + + b.Offer(sample("cpu.load", "0.1")) + if _, ok := b.Flush(base); !ok { + t.Fatal("expected first flush to be due") + } + + b.SetThrottled(true) + b.Offer(sample("cpu.load", "0.2")) + // Well past the *normal* interval but nowhere near the throttled one. + if _, ok := b.Flush(base.Add(500 * time.Millisecond)); ok { + t.Fatal("expected throttled batcher to withhold the frame") + } + if out, ok := b.Flush(base.Add(11 * time.Second)); !ok || len(out) != 1 { + t.Fatalf("flush past throttled interval = %v, %v; want 1 sample", out, ok) + } + + b.SetThrottled(false) + b.Offer(sample("cpu.load", "0.3")) + // +700ms past the last flush at 11s: clears both the batcher's now-fast + // interval and the per-metric 500ms gate. + if _, ok := b.Flush(base.Add(11700 * time.Millisecond)); !ok { + t.Fatal("expected resume_rate to restore the normal (fast) cadence") + } +} + +func TestMetricBatcherEmptyIsNeverDue(t *testing.T) { + b := newMetricBatcher(time.Millisecond, time.Second) + if out, ok := b.Flush(time.Now()); ok { + t.Fatalf("expected no frame with nothing pending, got %v", out) + } +} From 8846b817a092caa620b6c1b935444ec4b301c9e2 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:42:30 +0800 Subject: [PATCH 07/60] fix(apple): target /v3 realtime endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per protocol-owner ruling, the cross-implementation route convention for the realtime WebSocket channel is /v3/realtime, matching the server implementation — not the /v2/realtime path this client previously assumed from the REST surface's prefix. Update the derived URL and the doc comment accordingly. --- .../SitrepKit/Realtime/RealtimeEndpoint.swift | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift index 114cc99..7d8ced2 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift @@ -5,14 +5,12 @@ public extension APIClient { /// this client's REST `baseURL` by swapping the scheme and appending the /// realtime path. /// - /// NOTE for the protocol/server owner: `proto/realtime/SPEC.md` is - /// deliberately transport-independent and does not pin a concrete HTTP - /// path — this repository currently has no server-side realtime route - /// to confirm against (checked: no `server/src` file references - /// WebSockets or a realtime endpoint as of this change). `/v2/realtime` - /// is this client's assumption, chosen to match the existing `/v2/...` - /// REST surface; it is not yet a cross-implementation contract. Update - /// this in lockstep once the server lands its route. + /// `proto/realtime/SPEC.md` is deliberately transport-independent and + /// does not pin a concrete HTTP path; the cross-implementation route + /// convention is `/v3/realtime`, per protocol-owner ruling (the server + /// implementation targets the same path). Note this is intentionally + /// NOT `/v2/...`: the v2 prefix is the REST snapshot surface, while the + /// realtime channel lives under its own v3 route. var realtimeURL: URL? { guard var components = URLComponents(url: baseURL, resolvingAgainstBaseURL: false) else { return nil } switch components.scheme { @@ -21,7 +19,7 @@ public extension APIClient { default: break } let base = components.path.hasSuffix("/") ? String(components.path.dropLast()) : components.path - components.path = base + "/v2/realtime" + components.path = base + "/v3/realtime" return components.url } } From 26859ff553bf24c1704ec36f90f0b8413a754c2f Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:42:43 +0800 Subject: [PATCH 08/60] feat(daemon): add reliable realtime uplink client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add internal/realtime/client, the reconnecting WebSocket source connection (github.com/coder/websocket, pure Go): the mandatory hello{offer}/hello{accept} sequence with role "source", exponential backoff with jitter on any drop (doubling from a 1s base, capped at 60s, +/-20%; stretched to the cap instead of reconnecting aggressively after a superseded error, since that means our own newer connection already won or the credential is compromised), the ping/pong heartbeat with silent-peer detection, and command handling: TTL validation against the local clock with the +/-30s skew allowance from SPEC.md section 8, command_id dedup, and dispatch of throttle/resume_rate into the metric batcher (other actions forward to an optional callback for future process-control wiring). Every reliable send (SendTaskEvent/SendMessageEvent) goes through the outbox first and is replayed oldest-first, in a fresh envelope but with its original device_seq, immediately after each reconnect's hello completes (SPEC.md section 5.4) — on top of the outbox's own already-tested crash safety, this is where that guarantee actually reaches the wire. Add internal/realtime/rttest, a minimal scriptable mock WebSocket server used only by this package's tests to drive hello/ack/drop/ command scenarios without a real server dependency; it intentionally does not persist state or model space_revision and must never be described as a self-hosted server implementation. Backoff is a pure, unexported-state-free function tested without any sleeping; the reconnect/replay/throttle/command behaviors are tested end-to-end against rttest. --- daemon/go.mod | 5 +- daemon/go.sum | 2 + daemon/internal/realtime/client/backoff.go | 45 ++ .../internal/realtime/client/backoff_test.go | 85 +++ daemon/internal/realtime/client/client.go | 595 ++++++++++++++++++ .../internal/realtime/client/client_test.go | 363 +++++++++++ daemon/internal/realtime/client/config.go | 149 +++++ daemon/internal/realtime/rttest/server.go | 187 ++++++ daemon/internal/realtime/rttest/util.go | 19 + 9 files changed, 1449 insertions(+), 1 deletion(-) create mode 100644 daemon/internal/realtime/client/backoff.go create mode 100644 daemon/internal/realtime/client/backoff_test.go create mode 100644 daemon/internal/realtime/client/client.go create mode 100644 daemon/internal/realtime/client/client_test.go create mode 100644 daemon/internal/realtime/client/config.go create mode 100644 daemon/internal/realtime/rttest/server.go create mode 100644 daemon/internal/realtime/rttest/util.go diff --git a/daemon/go.mod b/daemon/go.mod index b654100..003711f 100644 --- a/daemon/go.mod +++ b/daemon/go.mod @@ -2,7 +2,10 @@ module github.com/QuintinShaw/sitrep/daemon go 1.26 -require modernc.org/sqlite v1.39.0 +require ( + github.com/coder/websocket v1.8.15 + modernc.org/sqlite v1.39.0 +) require ( github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/daemon/go.sum b/daemon/go.sum index c46fb2d..56481fb 100644 --- a/daemon/go.sum +++ b/daemon/go.sum @@ -1,3 +1,5 @@ +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= diff --git a/daemon/internal/realtime/client/backoff.go b/daemon/internal/realtime/client/backoff.go new file mode 100644 index 0000000..5bc6e83 --- /dev/null +++ b/daemon/internal/realtime/client/backoff.go @@ -0,0 +1,45 @@ +package client + +import "time" + +// Backoff computes the delay before reconnect attempt N (0-based: attempt 0 +// is the wait before the first retry after the first connection failure). +// The base delay doubles per attempt and is capped at max, then jittered by +// +/- jitterFrac (proportionally) using randFn, which must return a value +// in [0,1). This is a pure function — no sleeping, no shared state — so it +// can be tested by simply calling it with a fixed randFn and asserting on +// the numbers it returns. +func Backoff(attempt int, base, max time.Duration, jitterFrac float64, randFn func() float64) time.Duration { + if attempt < 0 { + attempt = 0 + } + if base <= 0 { + base = time.Second + } + if max < base { + max = base + } + d := base + for i := 0; i < attempt && d < max; i++ { + d *= 2 + if d > max { + d = max + } + } + if d > max { + d = max + } + if jitterFrac <= 0 || randFn == nil { + return d + } + r := randFn() + if r < 0 { + r = 0 + } + if r > 1 { + r = 1 + } + // Map r in [0,1) to a factor in [1-jitterFrac, 1+jitterFrac]. + factor := 1 + (r*2-1)*jitterFrac + return time.Duration(float64(d) * factor) +} diff --git a/daemon/internal/realtime/client/backoff_test.go b/daemon/internal/realtime/client/backoff_test.go new file mode 100644 index 0000000..45db2a5 --- /dev/null +++ b/daemon/internal/realtime/client/backoff_test.go @@ -0,0 +1,85 @@ +package client + +import ( + "testing" + "time" +) + +func fixedRand(v float64) func() float64 { return func() float64 { return v } } + +func TestBackoffDoublesAndCaps(t *testing.T) { + base := time.Second + max := 60 * time.Second + mid := fixedRand(0.5) // r=0.5 => factor=1 => no jitter distortion + + cases := []struct { + attempt int + want time.Duration + }{ + {0, 1 * time.Second}, + {1, 2 * time.Second}, + {2, 4 * time.Second}, + {3, 8 * time.Second}, + {4, 16 * time.Second}, + {5, 32 * time.Second}, + {6, 60 * time.Second}, // would be 64s, capped + {7, 60 * time.Second}, + {100, 60 * time.Second}, + } + for _, c := range cases { + got := Backoff(c.attempt, base, max, 0.2, mid) + if got != c.want { + t.Errorf("Backoff(%d) = %v, want %v", c.attempt, got, c.want) + } + } +} + +func TestBackoffJitterBounds(t *testing.T) { + base := time.Second + max := 60 * time.Second + attempt := 3 // nominal 8s + nominal := 8 * time.Second + jitter := 0.2 + + for _, r := range []float64{0, 0.25, 0.5, 0.75, 1} { + got := Backoff(attempt, base, max, jitter, fixedRand(r)) + lo := time.Duration(float64(nominal) * (1 - jitter)) + hi := time.Duration(float64(nominal) * (1 + jitter)) + if got < lo || got > hi { + t.Errorf("Backoff with r=%v = %v, want within [%v, %v]", r, got, lo, hi) + } + } +} + +func TestBackoffZeroJitterIsDeterministic(t *testing.T) { + got1 := Backoff(2, time.Second, 60*time.Second, 0, fixedRand(0.99)) + got2 := Backoff(2, time.Second, 60*time.Second, 0, fixedRand(0.01)) + if got1 != got2 || got1 != 4*time.Second { + t.Fatalf("zero jitter should ignore randFn entirely: got %v and %v, want 4s both", got1, got2) + } +} + +func TestBackoffNegativeAttemptClampsToZero(t *testing.T) { + got := Backoff(-5, time.Second, 60*time.Second, 0, fixedRand(0.5)) + if got != time.Second { + t.Fatalf("Backoff(-5) = %v, want base (1s)", got) + } +} + +// TestBackoffMonotonicNonDecreasing pins the "1s起, 2倍, 上限60s" shape +// across a full attempt sweep without ever sleeping — the whole point of +// keeping Backoff a pure function. +func TestBackoffMonotonicNonDecreasing(t *testing.T) { + base, max := time.Second, 60*time.Second + prev := time.Duration(0) + for attempt := 0; attempt < 10; attempt++ { + got := Backoff(attempt, base, max, 0, nil) + if got < prev { + t.Fatalf("attempt %d: backoff %v < previous %v (must be non-decreasing)", attempt, got, prev) + } + if got > max { + t.Fatalf("attempt %d: backoff %v exceeds cap %v", attempt, got, max) + } + prev = got + } +} diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go new file mode 100644 index 0000000..398436d --- /dev/null +++ b/daemon/internal/realtime/client/client.go @@ -0,0 +1,595 @@ +package client + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +const sourceRole = "source" + +var errClientClosing = errors.New("realtime client: closing") + +// Client is a reconnecting realtime-protocol source connection. Construct +// with New; it starts connecting immediately in the background. Call +// SendTaskEvent/SendMessageEvent for reliable events and SendMetric for +// best-effort metric samples; call Close to shut down. +type Client struct { + cfg Config + + mu sync.Mutex + conn *websocket.Conn + connected bool + attempt int + + metrics *metricBatcher + + seenMu sync.Mutex + seenOrder []string + seen map[string]bool + + supersededPenalty atomic.Bool + + closing chan struct{} + closed chan struct{} + once sync.Once +} + +// New starts a Client connecting in the background. +func New(cfg Config) *Client { + cfg.applyDefaults() + c := &Client{ + cfg: cfg, + metrics: newMetricBatcher(cfg.MetricFlushInterval, cfg.MetricThrottledInterval), + seen: make(map[string]bool), + closing: make(chan struct{}), + closed: make(chan struct{}), + } + go c.run() + return c +} + +// Close stops the client and waits for its background goroutine to exit. +func (c *Client) Close() { + c.once.Do(func() { close(c.closing) }) + <-c.closed +} + +// Connected reports whether a hello-completed connection is currently up. +func (c *Client) Connected() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.connected +} + +// MetricsThrottled reports whether the metric batcher is currently in its +// throttled cadence (SPEC.md section 7's command{throttle}/{resume_rate}). +func (c *Client) MetricsThrottled() bool { return c.metrics.Throttled() } + +func newEnvelopeID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return fmt.Sprintf("rt%d", time.Now().UnixNano()) + } + return "rt" + hex.EncodeToString(b) +} + +func (c *Client) now() time.Time { return c.cfg.Clock.Now() } +func (c *Client) nowMS() int64 { return c.now().UnixMilli() } + +// ---- public send API ---- + +// TaskEvent is the caller-facing shape for SendTaskEvent; see +// wire.TaskEventBody for field semantics. +type TaskEvent struct { + TaskID string + Kind string // started|progress|step|done|failed + OccurredAt time.Time + Title string + Percent *int + Step string + Message string + Display *wire.DisplayHints +} + +// SendTaskEvent durably enqueues a task lifecycle/progress event (assigning +// its device_seq) and attempts immediate delivery if connected. The event +// survives in the outbox until acknowledged, across any number of +// reconnects or process restarts. +func (c *Client) SendTaskEvent(ev TaskEvent) error { + item, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeTaskEvent, func(seq int64) (json.RawMessage, error) { + body := wire.TaskEventBody{ + DeviceID: c.cfg.DeviceID, + DeviceSeq: seq, + TaskID: ev.TaskID, + Kind: ev.Kind, + OccurredAt: ev.OccurredAt.UnixMilli(), + Title: ev.Title, + Percent: ev.Percent, + Step: ev.Step, + Message: ev.Message, + Display: ev.Display, + } + if err := body.Validate(); err != nil { + return nil, err + } + return json.Marshal(body) + }) + if err != nil { + return err + } + c.trySendNow(item) + return nil +} + +// MessageEvent is the caller-facing shape for SendMessageEvent. If +// MessageID is empty it defaults to ":". +type MessageEvent struct { + MessageID string + Level string // info|warn|error + Text string + OccurredAt time.Time + AutomationID string +} + +// SendMessageEvent durably enqueues a message event; see SendTaskEvent. +func (c *Client) SendMessageEvent(ev MessageEvent) error { + item, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + id := ev.MessageID + if id == "" { + id = fmt.Sprintf("%s:%d", c.cfg.DeviceID, seq) + } + body := wire.MessageEventBody{ + DeviceID: c.cfg.DeviceID, + DeviceSeq: seq, + MessageID: id, + Level: ev.Level, + Text: ev.Text, + OccurredAt: ev.OccurredAt.UnixMilli(), + AutomationID: ev.AutomationID, + } + if err := body.Validate(); err != nil { + return nil, err + } + return json.Marshal(body) + }) + if err != nil { + return err + } + c.trySendNow(item) + return nil +} + +// SendMetric offers one best-effort metric sample. It is merged with any +// other pending sample for the same MetricID and flushed on the client's +// own cadence (SPEC.md section 12); it is never persisted, never +// retransmitted, and silently superseded by the next sample if lost. +func (c *Client) SendMetric(s wire.MetricSample) { + c.metrics.Offer(s) +} + +// trySendNow best-effort writes one outbox item immediately if a +// connection is currently up. On any failure it does nothing further: the +// resend loop and the next reconnect's replay will pick it up. +func (c *Client) trySendNow(item outbox.Item) { + c.mu.Lock() + conn := c.conn + connected := c.connected + c.mu.Unlock() + if !connected || conn == nil { + return + } + env, err := wire.NewEnvelope(item.Kind, newEnvelopeID(), c.nowMS(), json.RawMessage(item.Body)) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + data, err := env.Encode() + if err != nil { + return + } + _ = conn.Write(ctx, websocket.MessageText, data) +} + +// ---- connection lifecycle ---- + +func (c *Client) run() { + defer close(c.closed) + for { + select { + case <-c.closing: + return + default: + } + + err := c.connectAndServe() + if errors.Is(err, errClientClosing) { + return + } + c.cfg.Logf("realtime: connection ended: %v", err) + + c.mu.Lock() + attempt := c.attempt + c.attempt++ + c.mu.Unlock() + + delay := Backoff(attempt, c.cfg.BackoffBase, c.cfg.BackoffMax, c.cfg.BackoffJitter, c.cfg.Rand) + if c.supersededPenalty.Swap(false) { + // Superseded means our own newer connection won, or our + // credential is compromised — either way, reconnecting + // aggressively is the wrong instinct; stretch the wait to the + // backoff ceiling instead of a reconnect storm. + delay = c.cfg.BackoffMax + } + select { + case <-time.After(delay): + case <-c.closing: + return + } + } +} + +func (c *Client) connectAndServe() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dialCtx, dialCancel := context.WithTimeout(ctx, c.cfg.HelloTimeout) + header := http.Header{} + if c.cfg.Token != "" { + header.Set("Authorization", "Bearer "+c.cfg.Token) + } + conn, _, err := websocket.Dial(dialCtx, c.cfg.URL, &websocket.DialOptions{HTTPHeader: header}) + dialCancel() + if err != nil { + return fmt.Errorf("dial: %w", err) + } + defer conn.CloseNow() + + if err := c.sendHelloOffer(ctx, conn); err != nil { + return fmt.Errorf("hello offer: %w", err) + } + accept, err := c.awaitHelloAccept(ctx, conn) + if err != nil { + return fmt.Errorf("hello accept: %w", err) + } + heartbeatInterval := time.Duration(accept.HeartbeatIntervalMS) * time.Millisecond + + c.mu.Lock() + c.conn = conn + c.connected = true + c.attempt = 0 + c.mu.Unlock() + defer func() { + c.mu.Lock() + c.connected = false + c.conn = nil + c.mu.Unlock() + }() + + c.replayPending(ctx, conn) + + frames := make(chan frameMsg, 8) + pings := make(chan struct{}, 8) + pongs := make(chan struct{}, 8) + go c.readLoop(ctx, conn, frames, pings, pongs) + + resendTicker := time.NewTicker(c.cfg.ResendInterval) + defer resendTicker.Stop() + metricsTicker := time.NewTicker(c.cfg.MetricFlushInterval) + defer metricsTicker.Stop() + heartbeatTicker := time.NewTicker(heartbeatInterval) + defer heartbeatTicker.Stop() + + lastPong := c.now() + heartbeatTimeout := time.Duration(c.cfg.HeartbeatTimeoutFactor) * heartbeatInterval + + for { + select { + case fm := <-frames: + if fm.err != nil { + return fm.err + } + if err := c.handleFrame(ctx, conn, fm.env); err != nil { + return err + } + case <-pings: + _ = conn.Write(ctx, websocket.MessageText, []byte("pong")) + case <-pongs: + lastPong = c.now() + case <-resendTicker.C: + c.replayPending(ctx, conn) + case <-metricsTicker.C: + c.flushMetrics(ctx, conn) + case <-heartbeatTicker.C: + if err := conn.Write(ctx, websocket.MessageText, []byte("ping")); err != nil { + return fmt.Errorf("heartbeat write: %w", err) + } + if c.now().Sub(lastPong) > heartbeatTimeout { + return fmt.Errorf("heartbeat timeout: no pong within %v", heartbeatTimeout) + } + case <-c.closing: + _ = conn.Close(websocket.StatusNormalClosure, "client closing") + return errClientClosing + } + } +} + +func (c *Client) sendHelloOffer(ctx context.Context, conn *websocket.Conn) error { + offer := wire.HelloOffer{ + Stage: "offer", + DeviceID: c.cfg.DeviceID, + Role: sourceRole, + ProtocolVersions: c.cfg.ProtocolVersions, + } + env, err := wire.NewEnvelope(wire.TypeHello, newEnvelopeID(), c.nowMS(), wire.HelloBody{Offer: &offer}) + if err != nil { + return err + } + data, err := env.Encode() + if err != nil { + return err + } + return conn.Write(ctx, websocket.MessageText, data) +} + +func (c *Client) awaitHelloAccept(ctx context.Context, conn *websocket.Conn) (wire.HelloAccept, error) { + ctx, cancel := context.WithTimeout(ctx, c.cfg.HelloTimeout) + defer cancel() + _, data, err := conn.Read(ctx) + if err != nil { + return wire.HelloAccept{}, err + } + env, err := wire.DecodeEnvelope(data) + if err != nil { + return wire.HelloAccept{}, err + } + if env.Type == wire.TypeError { + body, derr := wire.DecodeBody(env) + if derr == nil { + eb := body.(wire.ErrorBody) + return wire.HelloAccept{}, fmt.Errorf("server rejected hello: %s: %s", eb.Code, eb.Message) + } + return wire.HelloAccept{}, fmt.Errorf("server rejected hello with malformed error body") + } + if env.Type != wire.TypeHello { + return wire.HelloAccept{}, fmt.Errorf("expected hello, got %q", env.Type) + } + body, err := wire.DecodeBody(env) + if err != nil { + return wire.HelloAccept{}, err + } + hb := body.(wire.HelloBody) + if hb.Accept == nil { + return wire.HelloAccept{}, fmt.Errorf("expected hello accept, got offer") + } + return *hb.Accept, nil +} + +// frameMsg carries one decoded envelope, or the terminal read error that +// ended the connection, from readLoop to connectAndServe's select loop. +type frameMsg struct { + env wire.Envelope + err error +} + +// readLoop pumps decoded envelopes (or the bare ping/pong heartbeat text) +// onto the given channels until the connection dies, then reports the +// terminal error on frames and returns. +func (c *Client) readLoop(ctx context.Context, conn *websocket.Conn, frames chan<- frameMsg, pings, pongs chan<- struct{}) { + for { + typ, data, err := conn.Read(ctx) + if err != nil { + select { + case frames <- frameMsg{err: err}: + case <-ctx.Done(): + } + return + } + if typ == websocket.MessageText { + switch string(data) { + case "ping": + select { + case pings <- struct{}{}: + case <-ctx.Done(): + return + } + continue + case "pong": + select { + case pongs <- struct{}{}: + case <-ctx.Done(): + return + } + continue + } + } + env, derr := wire.DecodeEnvelope(data) + select { + case frames <- frameMsg{env: env, err: derr}: + case <-ctx.Done(): + return + } + } +} + +func (c *Client) replayPending(ctx context.Context, conn *websocket.Conn) { + items, err := c.cfg.Outbox.Pending(ctx, c.cfg.Space) + if err != nil { + c.cfg.Logf("realtime: read pending outbox: %v", err) + return + } + for _, item := range items { + env, err := wire.NewEnvelope(item.Kind, newEnvelopeID(), c.nowMS(), json.RawMessage(item.Body)) + if err != nil { + continue + } + data, err := env.Encode() + if err != nil { + continue + } + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + werr := conn.Write(writeCtx, websocket.MessageText, data) + cancel() + if werr != nil { + return // connection is dying; the outer loop will notice and reconnect + } + } +} + +func (c *Client) flushMetrics(ctx context.Context, conn *websocket.Conn) { + samples, ok := c.metrics.Flush(c.now()) + if !ok { + return + } + body := wire.MetricFrameBody{DeviceID: c.cfg.DeviceID, Metrics: samples} + if err := body.Validate(); err != nil { + c.cfg.Logf("realtime: dropping invalid metric.frame: %v", err) + return + } + env, err := wire.NewEnvelope(wire.TypeMetricFrame, newEnvelopeID(), c.nowMS(), body) + if err != nil { + return + } + data, err := env.Encode() + if err != nil { + return + } + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + _ = conn.Write(writeCtx, websocket.MessageText, data) +} + +func (c *Client) handleFrame(ctx context.Context, conn *websocket.Conn, env wire.Envelope) error { + if !wire.KnownType(env.Type) { + return nil // SPEC.md section 15: ignore, not an error + } + body, err := wire.DecodeBody(env) + if err != nil { + c.cfg.Logf("realtime: dropping malformed %s frame: %v", env.Type, err) + return nil + } + switch env.Type { + case wire.TypeAck: + ab := body.(wire.AckBody) + for _, p := range ab.Acked { + if p.DeviceID != c.cfg.DeviceID { + continue // malformed per SPEC.md 5.3; ignore rather than trust + } + if err := c.cfg.Outbox.Ack(ctx, c.cfg.Space, p.DeviceSeq); err != nil { + c.cfg.Logf("realtime: ack seq %d: %v", p.DeviceSeq, err) + } + } + return nil + case wire.TypeCommand: + cb := body.(wire.CommandBody) + c.handleCommand(ctx, conn, env, cb) + return nil + case wire.TypeError: + eb := body.(wire.ErrorBody) + return c.handleError(eb) + default: + // snapshot/delta/hello/resume/subscribe/... are not meaningful on a + // source connection; ignore rather than kill the connection. + return nil + } +} + +func (c *Client) handleError(eb wire.ErrorBody) error { + if eb.Code == wire.ErrSuperseded { + c.supersededPenalty.Store(true) + c.cfg.Logf("realtime: connection superseded by a newer one (or credential in use elsewhere)") + return fmt.Errorf("%s: %s", eb.Code, eb.Message) + } + if eb.Fatal != nil && *eb.Fatal { + return fmt.Errorf("%s: %s", eb.Code, eb.Message) + } + // Advisory (rate_limited, sequence_gap, ...): log and keep the + // connection. + c.cfg.Logf("realtime: server error %s: %s", eb.Code, eb.Message) + return nil +} + +// handleCommand validates the command's TTL against the local clock +// (SPEC.md section 8: +/- 30s skew allowance), deduplicates by command_id, +// and either applies it (throttle/resume_rate) or forwards it to +// Config.OnCommand. +func (c *Client) handleCommand(ctx context.Context, conn *websocket.Conn, env wire.Envelope, cb wire.CommandBody) { + const skew = 30 * time.Second + now := c.now() + windowStart := time.UnixMilli(env.TS).Add(-skew) + windowEnd := time.UnixMilli(env.TS).Add(time.Duration(cb.TTLMs) * time.Millisecond).Add(skew) + if now.Before(windowStart) || now.After(windowEnd) { + c.cfg.Logf("realtime: dropping expired command %s (action %s)", cb.CommandID, cb.Action) + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + errBody := wire.ErrorBody{ + Code: wire.ErrCommandExpired, + Message: "command ttl window elapsed", + InReplyTo: env.ID, + Retryable: boolPtr(false), + Fatal: boolPtr(false), + } + if eenv, eerr := wire.NewEnvelope(wire.TypeError, newEnvelopeID(), c.nowMS(), errBody); eerr == nil { + if data, derr := eenv.Encode(); derr == nil { + _ = conn.Write(writeCtx, websocket.MessageText, data) + } + } + return + } + + if c.alreadySeen(cb.CommandID) { + return // SPEC.md section 8: MUST NOT execute the same command_id twice + } + + switch cb.Action { + case "throttle": + c.metrics.SetThrottled(true) + case "resume_rate": + c.metrics.SetThrottled(false) + default: + if c.cfg.OnCommand != nil { + c.cfg.OnCommand(CommandAction{ + CommandID: cb.CommandID, + Action: cb.Action, + TaskID: cb.TaskID, + AutomationID: cb.AutomationID, + TargetDeviceID: cb.TargetDeviceID, + }) + } + } +} + +func boolPtr(b bool) *bool { return &b } + +// alreadySeen reports whether commandID has been handled before, recording +// it if not. Bounded so a long-lived connection doesn't grow this set +// forever. +func (c *Client) alreadySeen(commandID string) bool { + const cap = 512 + c.seenMu.Lock() + defer c.seenMu.Unlock() + if c.seen[commandID] { + return true + } + c.seen[commandID] = true + c.seenOrder = append(c.seenOrder, commandID) + if len(c.seenOrder) > cap { + drop := c.seenOrder[0] + c.seenOrder = c.seenOrder[1:] + delete(c.seen, drop) + } + return false +} diff --git a/daemon/internal/realtime/client/client_test.go b/daemon/internal/realtime/client/client_test.go new file mode 100644 index 0000000..fa2f3bc --- /dev/null +++ b/daemon/internal/realtime/client/client_test.go @@ -0,0 +1,363 @@ +package client + +import ( + "context" + "fmt" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/rttest" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +const testDeviceID = "test-device-01" +const testSpace = "space-test" + +func newTestOutbox(t *testing.T) *outbox.Store { + t.Helper() + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +func newTestClient(t *testing.T, url string, store *outbox.Store, mutate func(*Config)) *Client { + t.Helper() + cfg := Config{ + URL: url, + Token: "test-token", + DeviceID: testDeviceID, + Space: testSpace, + Outbox: store, + BackoffBase: 5 * time.Millisecond, + BackoffMax: 20 * time.Millisecond, + BackoffJitter: 0, + ResendInterval: 20 * time.Millisecond, + MetricFlushInterval: 5 * time.Millisecond, + MetricThrottledInterval: 300 * time.Millisecond, + HelloTimeout: 2 * time.Second, + } + if mutate != nil { + mutate(&cfg) + } + c := New(cfg) + t.Cleanup(c.Close) + return c +} + +// waitFor polls cond until it returns true or the timeout elapses. +func waitFor(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(2 * time.Millisecond) + } + if !cond() { + t.Fatalf("condition not met within %v", timeout) + } +} + +func TestClientHelloAndTaskEventAcked(t *testing.T) { + srv := rttest.New(func(conn *rttest.Conn) { + offer, err := conn.HelloAccept("sess-1", 1000) + if err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + if offer.Role != "source" { + t.Errorf("offer.Role = %q, want source", offer.Role) + } + if offer.DeviceID != testDeviceID { + t.Errorf("offer.DeviceID = %q, want %q", offer.DeviceID, testDeviceID) + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type == wire.TypeTaskEvent { + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + if err := conn.Ack(tb.DeviceID, tb.DeviceSeq); err != nil { + return + } + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + if err := c.SendTaskEvent(TaskEvent{ + TaskID: "run-1", + Kind: "started", + OccurredAt: time.Now(), + Title: "Nightly backup", + }); err != nil { + t.Fatalf("SendTaskEvent: %v", err) + } + + waitFor(t, 3*time.Second, func() bool { + pending, err := store.Pending(context.Background(), testSpace) + return err == nil && len(pending) == 0 + }) +} + +// TestClientReconnectReplaysUnacked drops the first connection before +// acking a task.event and asserts the client resends the SAME device_seq +// (in a fresh envelope) on the next connection, per SPEC.md's Agent +// reconnect replay flow (section 5.4). +func TestClientReconnectReplaysUnacked(t *testing.T) { + var connNum int32 + var firstEnvID, secondEnvID string + var firstSeq, secondSeq int64 + + srv := rttest.New(func(conn *rttest.Conn) { + n := atomic.AddInt32(&connNum, 1) + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + switch n { + case 1: + env, err := conn.ReadEnvelope() + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + t.Errorf("connection 1: expected task.event, got %s", env.Type) + return + } + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + firstEnvID = env.ID + firstSeq = tb.DeviceSeq + // Deliberately do NOT ack; close so the client must reconnect. + conn.Close("simulated drop") + case 2: + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type == wire.TypeTaskEvent { + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + secondEnvID = env.ID + secondSeq = tb.DeviceSeq + conn.Ack(tb.DeviceID, tb.DeviceSeq) + } + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + if err := c.SendTaskEvent(TaskEvent{TaskID: "run-1", Kind: "started", OccurredAt: time.Now()}); err != nil { + t.Fatalf("SendTaskEvent: %v", err) + } + + waitFor(t, 3*time.Second, func() bool { + pending, err := store.Pending(context.Background(), testSpace) + return err == nil && len(pending) == 0 + }) + + if firstSeq == 0 || secondSeq == 0 { + t.Fatalf("did not observe both connections' task.event (first=%d second=%d)", firstSeq, secondSeq) + } + if firstSeq != secondSeq { + t.Fatalf("device_seq changed across resend: first=%d second=%d, want identical (SPEC.md 5.4)", firstSeq, secondSeq) + } + if firstEnvID == secondEnvID { + t.Fatalf("envelope id must be freshly generated on resend, got the same id %q both times", firstEnvID) + } +} + +// TestClientCommandThrottleAndResumeRate asserts a server-issued +// command{throttle} switches the metric batcher into its throttled cadence +// and command{resume_rate} switches it back (SPEC.md section 7). The exact +// merge/rate-limit *math* of the batcher (N samples -> 1 frame, the 500ms +// per-metric and overall-interval gates) is covered deterministically, +// without any wall-clock sleeping, by TestMetricBatcher* in metrics_test.go; +// this test's job is only to prove the command actually reaches and flips +// the batcher's state end-to-end over the wire, so it asserts on that state +// transition rather than timing frame arrivals (which would make the test +// flaky under scheduler jitter). +func TestClientCommandThrottleAndResumeRate(t *testing.T) { + connReady := make(chan *rttest.Conn, 1) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + connReady <- conn + for { + _, err := conn.ReadEnvelope() + if err != nil && err != rttest.ErrPing && err != rttest.ErrPong { + return + } + if err == rttest.ErrPing { + conn.WritePong() + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + var conn *rttest.Conn + select { + case conn = <-connReady: + case <-time.After(3 * time.Second): + t.Fatal("server never observed a connection") + } + + if c.MetricsThrottled() { + t.Fatal("expected the client to start un-throttled") + } + + if err := conn.SendCommand(wire.CommandBody{ + CommandID: "cmd-throttle-1", Origin: "server", Action: "throttle", TTLMs: 3600000, + }); err != nil { + t.Fatalf("SendCommand(throttle): %v", err) + } + waitFor(t, time.Second, c.MetricsThrottled) + + if err := conn.SendCommand(wire.CommandBody{ + CommandID: "cmd-resume-1", Origin: "server", Action: "resume_rate", TTLMs: 3600000, + }); err != nil { + t.Fatalf("SendCommand(resume_rate): %v", err) + } + waitFor(t, time.Second, func() bool { return !c.MetricsThrottled() }) +} + +// TestClientCommandTTLRejectsExpired asserts a command whose TTL window has +// already elapsed (even allowing the +/-30s clock-skew grace, SPEC.md +// section 8) is dropped without being executed. +func TestClientCommandTTLRejectsExpired(t *testing.T) { + connReady := make(chan *rttest.Conn, 1) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + connReady <- conn + for { + if _, err := conn.ReadEnvelope(); err != nil { + return + } + } + }) + defer srv.Close() + + var executed int32 + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, func(cfg *Config) { + cfg.OnCommand = func(CommandAction) { atomic.AddInt32(&executed, 1) } + }) + + var conn *rttest.Conn + select { + case conn = <-connReady: + case <-time.After(3 * time.Second): + t.Fatal("server never observed a connection") + } + + // ts is far enough in the past that even the 30s skew allowance and a + // tiny ttl_ms cannot make "now" fall inside the actionable window. + expiredTS := rttest.NowMS() - 10*60*1000 // 10 minutes ago + env, err := wire.NewEnvelope(wire.TypeCommand, "cmd-env-1", expiredTS, wire.CommandBody{ + CommandID: "cmd-pause-expired", Origin: "viewer", IssuedByDeviceID: "viewer-1", + Action: "pause", TaskID: "run-1", TTLMs: 1000, + }) + if err != nil { + t.Fatalf("NewEnvelope: %v", err) + } + if err := conn.WriteEnvelope(env); err != nil { + t.Fatalf("WriteEnvelope: %v", err) + } + + time.Sleep(100 * time.Millisecond) + if n := atomic.LoadInt32(&executed); n != 0 { + t.Fatalf("OnCommand invoked %d times for an expired command, want 0", n) + } +} + +// TestClientCommandDedupeByID asserts the same command_id is never executed +// twice (SPEC.md section 8). +func TestClientCommandDedupeByID(t *testing.T) { + connReady := make(chan *rttest.Conn, 1) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + connReady <- conn + for { + if _, err := conn.ReadEnvelope(); err != nil { + return + } + } + }) + defer srv.Close() + + var executed int32 + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, func(cfg *Config) { + cfg.OnCommand = func(CommandAction) { atomic.AddInt32(&executed, 1) } + }) + + var conn *rttest.Conn + select { + case conn = <-connReady: + case <-time.After(3 * time.Second): + t.Fatal("server never observed a connection") + } + + send := func() { + body := wire.CommandBody{ + CommandID: "cmd-dupe-1", Origin: "viewer", IssuedByDeviceID: "viewer-1", + Action: "pause", TaskID: "run-1", TTLMs: 3600000, + } + env, err := wire.NewEnvelope(wire.TypeCommand, rttestEnvID(), rttest.NowMS(), body) + if err != nil { + t.Fatalf("NewEnvelope: %v", err) + } + if err := conn.WriteEnvelope(env); err != nil { + t.Fatalf("WriteEnvelope: %v", err) + } + } + send() + send() + time.Sleep(100 * time.Millisecond) + + if n := atomic.LoadInt32(&executed); n != 1 { + t.Fatalf("OnCommand invoked %d times for a duplicated command_id, want exactly 1", n) + } +} + +var envCounter int64 + +func rttestEnvID() string { + return fmt.Sprintf("env-%d", atomic.AddInt64(&envCounter, 1)) +} diff --git a/daemon/internal/realtime/client/config.go b/daemon/internal/realtime/client/config.go new file mode 100644 index 0000000..1fec66d --- /dev/null +++ b/daemon/internal/realtime/client/config.go @@ -0,0 +1,149 @@ +// Package client is the daemon-side realtime protocol client +// (proto/realtime/SPEC.md). It connects to the server as a **source** +// device only — the viewer role (subscribe/resume/snapshot/delta) belongs +// to a different consumer (the phone/menu-bar apps) and is out of scope +// here; see the wire package's authorization matrix for the full role set. +// +// Responsibilities: +// - Maintain one outbound WebSocket connection with the mandatory hello +// offer/accept sequence, reconnecting with exponential backoff + jitter +// on any drop. +// - Persist every reliable event (task.event/message.event) to a local +// outbox before attempting delivery, retire it on a matching ack, and +// replay every still-unacked event oldest-first after each reconnect. +// - Merge and rate-limit metric samples into metric.frame batches, +// honoring server-issued throttle/resume_rate commands. +// - Answer the ping/pong heartbeat and detect a silent peer. +package client + +import ( + "math/rand" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" +) + +// Clock abstracts time for components that need deterministic tests. The +// live Client uses time.Now()/time.Sleep directly (see run/connectAndServe); +// Clock exists for the pure, sleep-free pieces (Backoff, metricBatcher) that +// tests drive with an injected instant instead of waiting in real time. +type Clock interface { + Now() time.Time +} + +type realClock struct{} + +func (realClock) Now() time.Time { return time.Now() } + +// Logf is the client's pluggable logger; nil means "discard". +type Logf func(format string, args ...any) + +// CommandAction is a validated, deduplicated, not-yet-expired command this +// client is not itself responsible for executing (anything other than the +// throttle/resume_rate pair it handles internally) — pause/resume/stop/ +// run_now. Forwarded to Config.OnCommand, if set, for the caller to wire +// into process control; this package does not dispatch these itself +// (that plumbing already exists on the HTTP uplink path, see +// internal/uplink.Uplink.Commands, and is out of scope for this package). +type CommandAction struct { + CommandID string + Action string // pause|resume|stop|run_now + TaskID string + AutomationID string + TargetDeviceID string +} + +// Config configures a Client. Zero-value fields with a documented default +// are filled in by New. +type Config struct { + // URL is the ws:// or wss:// endpoint to dial. + URL string + // Token is presented once at connection establishment (SPEC.md section + // 10), as an Authorization: Bearer header — never repeated in an + // envelope. + Token string + // DeviceID and Space identify this connection's hello{device_id} and + // the (device, space) scope its device_seq counter and outbox are + // bound to (SPEC.md section 5.1). One Client serves exactly one space. + DeviceID string + Space string + + // Outbox is the durable reliable-event queue. Required. + Outbox *outbox.Store + + // ProtocolVersions defaults to []int{1}. + ProtocolVersions []int + + // HeartbeatTimeoutFactor multiplies the server-advertised + // heartbeat_interval_ms to decide when a silent peer is dead (SPEC.md + // section 9.3 recommends 2x). Defaults to 2. + HeartbeatTimeoutFactor int + + // Reconnect backoff (SPEC.md work order: "1s起, 2倍, 上限60s, ±20% jitter"). + BackoffBase time.Duration // default 1s + BackoffMax time.Duration // default 60s + BackoffJitter float64 // default 0.2 + + // ResendInterval is how often, while connected, the client re-attempts + // delivery of anything still sitting unacked in the outbox (belt and + // suspenders on top of the mandatory post-reconnect replay). + ResendInterval time.Duration // default 5s + + // MetricFlushInterval is the normal-cadence poll period for the metric + // batcher; MetricThrottledInterval replaces it while under a server + // throttle command (SPEC.md section 7: "an implementation-defined + // lower rate"). The 2 Hz per-metric / 10 Hz per-connection protocol + // ceilings (section 11/12) are enforced independently of this value. + MetricFlushInterval time.Duration // default 200ms + MetricThrottledInterval time.Duration // default 30s + + // HelloTimeout bounds how long to wait for hello{accept} after sending + // the offer (SPEC.md section 9.1 recommends 10s). + HelloTimeout time.Duration // default 10s + + Clock Clock + Rand func() float64 // returns [0,1); defaults to a package-level RNG + + OnCommand func(CommandAction) + Logf Logf +} + +func (c *Config) applyDefaults() { + if len(c.ProtocolVersions) == 0 { + c.ProtocolVersions = []int{1} + } + if c.HeartbeatTimeoutFactor <= 0 { + c.HeartbeatTimeoutFactor = 2 + } + if c.BackoffBase <= 0 { + c.BackoffBase = time.Second + } + if c.BackoffMax <= 0 { + c.BackoffMax = 60 * time.Second + } + if c.BackoffJitter == 0 { + c.BackoffJitter = 0.2 + } + if c.ResendInterval <= 0 { + c.ResendInterval = 5 * time.Second + } + if c.MetricFlushInterval <= 0 { + c.MetricFlushInterval = 200 * time.Millisecond + } + if c.MetricThrottledInterval <= 0 { + c.MetricThrottledInterval = 30 * time.Second + } + if c.HelloTimeout <= 0 { + c.HelloTimeout = 10 * time.Second + } + if c.Clock == nil { + c.Clock = realClock{} + } + if c.Rand == nil { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + c.Rand = rng.Float64 + } + if c.Logf == nil { + c.Logf = func(string, ...any) {} + } +} diff --git a/daemon/internal/realtime/rttest/server.go b/daemon/internal/realtime/rttest/server.go new file mode 100644 index 0000000..22ab4c9 --- /dev/null +++ b/daemon/internal/realtime/rttest/server.go @@ -0,0 +1,187 @@ +// Package rttest is a test-only, minimal WebSocket server used to exercise +// the realtime client (internal/realtime/client) against scripted server +// behavior: hello accept, acking, withholding acks, dropping connections, +// sending commands, and the ping/pong heartbeat. +// +// This is NOT a reimplementation of the real Sitrep server: it does not +// persist state, fold events, track space_revision, manage interest +// leases, or implement anything resembling SpaceHub. It exists purely so +// daemon-side tests can drive the client through the connection sequences +// SPEC.md describes without a network dependency on a real server. It must +// never be described as, or grow into, a self-hosted server implementation. +package rttest + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// Conn wraps one accepted connection with envelope-level read/write helpers. +type Conn struct { + ws *websocket.Conn + ctx context.Context + + // Request carries the *http.Request that established this connection, + // e.g. for asserting on the Authorization header a real client sends + // (SPEC.md section 10: the credential is presented once, out of band, + // at connection establishment). + Request *http.Request +} + +// ReadEnvelope reads one frame and decodes it as an envelope. A literal +// "ping"/"pong" text frame is returned as a sentinel error so callers can +// distinguish it from a decode failure. +var ErrPing = fmt.Errorf("rttest: received ping") +var ErrPong = fmt.Errorf("rttest: received pong") + +func (c *Conn) ReadEnvelope() (wire.Envelope, error) { + typ, data, err := c.ws.Read(c.ctx) + if err != nil { + return wire.Envelope{}, err + } + if typ == websocket.MessageText { + switch string(data) { + case "ping": + return wire.Envelope{}, ErrPing + case "pong": + return wire.Envelope{}, ErrPong + } + } + return wire.DecodeEnvelope(data) +} + +// WriteEnvelope encodes and sends one envelope. +func (c *Conn) WriteEnvelope(env wire.Envelope) error { + data, err := env.Encode() + if err != nil { + return err + } + return c.ws.Write(c.ctx, websocket.MessageText, data) +} + +// WritePing / WritePong send the bare heartbeat text frames (SPEC.md +// section 9.3) — deliberately not JSON envelopes. +func (c *Conn) WritePing() error { return c.ws.Write(c.ctx, websocket.MessageText, []byte("ping")) } +func (c *Conn) WritePong() error { return c.ws.Write(c.ctx, websocket.MessageText, []byte("pong")) } + +// Close closes the connection with the given close code/reason. +func (c *Conn) Close(reason string) error { + return c.ws.Close(websocket.StatusNormalClosure, reason) +} + +// HelloAccept performs the mandatory first exchange (SPEC.md section 9): +// reads the client's hello{stage: offer}, validates it is well-formed, and +// replies hello{stage: accept} with the given session id and heartbeat +// interval. It returns the decoded offer for the caller's assertions. +func (c *Conn) HelloAccept(sessionID string, heartbeatMS int) (wire.HelloOffer, error) { + env, err := c.ReadEnvelope() + if err != nil { + return wire.HelloOffer{}, fmt.Errorf("rttest: read hello offer: %w", err) + } + if env.Type != wire.TypeHello { + return wire.HelloOffer{}, fmt.Errorf("rttest: expected hello, got %q", env.Type) + } + body, err := wire.DecodeBody(env) + if err != nil { + return wire.HelloOffer{}, fmt.Errorf("rttest: decode hello offer: %w", err) + } + hb := body.(wire.HelloBody) + if hb.Offer == nil { + return wire.HelloOffer{}, fmt.Errorf("rttest: expected hello offer, got accept") + } + accept := wire.HelloAccept{ + Stage: "accept", + ProtocolVersion: 1, + SessionID: sessionID, + HeartbeatIntervalMS: heartbeatMS, + } + acceptEnv, err := wire.NewEnvelope(wire.TypeHello, newID(), NowMS(), wire.HelloBody{Accept: &accept}) + if err != nil { + return wire.HelloOffer{}, err + } + if err := c.WriteEnvelope(acceptEnv); err != nil { + return wire.HelloOffer{}, err + } + return *hb.Offer, nil +} + +// Ack acknowledges one reliable event. +func (c *Conn) Ack(deviceID string, seq int64) error { + body := wire.AckBody{Acked: []wire.AckedPair{{DeviceID: deviceID, DeviceSeq: seq}}} + env, err := wire.NewEnvelope(wire.TypeAck, newID(), NowMS(), body) + if err != nil { + return err + } + return c.WriteEnvelope(env) +} + +// SendCommand sends a server-originated command (e.g. throttle/resume_rate). +func (c *Conn) SendCommand(body wire.CommandBody) error { + env, err := wire.NewEnvelope(wire.TypeCommand, newID(), NowMS(), body) + if err != nil { + return err + } + return c.WriteEnvelope(env) +} + +// SendError sends an error envelope (e.g. superseded). +func (c *Conn) SendError(body wire.ErrorBody) error { + env, err := wire.NewEnvelope(wire.TypeError, newID(), NowMS(), body) + if err != nil { + return err + } + return c.WriteEnvelope(env) +} + +// Handler is invoked once per accepted connection, in its own goroutine. +// The handler owns the connection's lifetime: when it returns, the +// connection is closed. +type Handler func(*Conn) + +// Server is a scriptable mock realtime endpoint bound to an httptest server. +type Server struct { + httpSrv *httptest.Server + handler Handler + + mu sync.Mutex + conns []*Conn +} + +// New starts a mock server that invokes handler for every accepted +// connection. Call URL() to get the ws:// address for a client to dial. +func New(handler Handler) *Server { + s := &Server{handler: handler} + mux := http.NewServeMux() + mux.HandleFunc("/", s.serve) + s.httpSrv = httptest.NewServer(mux) + return s +} + +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + conn := &Conn{ws: ws, ctx: r.Context(), Request: r} + s.mu.Lock() + s.conns = append(s.conns, conn) + s.mu.Unlock() + s.handler(conn) + ws.Close(websocket.StatusNormalClosure, "handler done") +} + +// URL returns the ws:// URL clients should dial. +func (s *Server) URL() string { + return "ws" + strings.TrimPrefix(s.httpSrv.URL, "http") +} + +// Close shuts down the underlying HTTP server. +func (s *Server) Close() { s.httpSrv.Close() } diff --git a/daemon/internal/realtime/rttest/util.go b/daemon/internal/realtime/rttest/util.go new file mode 100644 index 0000000..73a2141 --- /dev/null +++ b/daemon/internal/realtime/rttest/util.go @@ -0,0 +1,19 @@ +package rttest + +import ( + "crypto/rand" + "encoding/hex" + "time" +) + +// newID returns an opaque envelope id matching common.schema.json's +// envelope_id pattern. +func newID() string { + b := make([]byte, 12) + _, _ = rand.Read(b) + return "rt" + hex.EncodeToString(b) +} + +// NowMS returns the current time as Unix milliseconds, always within the +// valid unix_ms_timestamp bound (SPEC.md section 3.1). +func NowMS() int64 { return time.Now().UnixMilli() } From 6a396cbacb853c5923e9a1fa9f1f748c58c1d8fc Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:43:09 +0800 Subject: [PATCH 09/60] feat(daemon): integrate realtime uplink behind an opt-in flag Wire internal/realtime/client into the existing HTTP uplink and the resident `sitrep agent` process, off by default: - config: SITREP_REALTIME (or config.json's realtime_enabled) opts in; SITREP_REALTIME_URL overrides the wss:///v2/realtime endpoint this package otherwise derives from the existing server URL. - uplink.Config gains an optional Realtime field. When set, Offer() routes every kind with a realtime-protocol equivalent (task lifecycle -> task.event, message.send -> message.event, metric.update -> metric.frame) to it instead of the HTTP /v2/ingest batch, so the same event is never sent both ways; task.log (no realtime equivalent) always continues over HTTP. Nil Realtime (the default) leaves this package's behavior byte-for-byte unchanged. - cmdAgent builds one shared realtime client (and its SQLite outbox, next to the existing config.json) for the whole process when the flag is on, reused across every automation run. This mirrors the protocol's "at most one connection per device" model (SPEC.md section 9.4); the one-shot `sitrep run` CLI is intentionally left on the existing HTTP-only path; see the handoff notes for why. Covered by a new integration test asserting the flag routes task/ message/metric events to a mock realtime server and never double-sends them over HTTP, and that the flag left off reproduces prior behavior exactly. --- daemon/cmd/sitrep/agent.go | 57 +++++++++- daemon/internal/config/config.go | 52 ++++++++- daemon/internal/uplink/realtime_test.go | 135 ++++++++++++++++++++++++ daemon/internal/uplink/uplink.go | 117 ++++++++++++++++++++ 4 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 daemon/internal/uplink/realtime_test.go diff --git a/daemon/cmd/sitrep/agent.go b/daemon/cmd/sitrep/agent.go index a0b4fae..e6d4255 100644 --- a/daemon/cmd/sitrep/agent.go +++ b/daemon/cmd/sitrep/agent.go @@ -6,7 +6,10 @@ import ( "time" "github.com/QuintinShaw/sitrep/daemon/internal/api" + "github.com/QuintinShaw/sitrep/daemon/internal/config" "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" "github.com/QuintinShaw/sitrep/daemon/internal/runner" "github.com/QuintinShaw/sitrep/daemon/internal/uplink" ) @@ -23,6 +26,16 @@ func cmdAgent(args []string) { } fmt.Fprintf(os.Stderr, "sitrep agent: scheduling automations from %s\n", client.Server) + // Realtime uplink: opt-in feature flag (SITREP_REALTIME=1 or + // config.json's realtime_enabled). One shared connection for the + // whole resident agent process, mirroring the protocol's "at most one + // connection per device" model (proto/realtime/SPEC.md section 9.4) — + // unlike the one-shot `sitrep run` CLI, this process lives long + // enough for a persistent WebSocket connection to make sense. Default + // (flag off) leaves every automation run on the existing HTTP-only + // path, byte-for-byte as before this feature existed. + rt := newAgentRealtimeUplink() + lastRun := map[string]time.Time{} running := map[string]bool{} finished := make(chan string, 32) @@ -72,7 +85,7 @@ func cmdAgent(args []string) { lastRun[automation.ID] = time.Now() running[automation.ID] = true go func(automation api.Automation) { - runAutomation(client, automation) + runAutomation(client, automation, rt) finished <- automation.ID }(automation) } @@ -81,11 +94,47 @@ func cmdAgent(args []string) { } } +// newAgentRealtimeUplink builds the shared realtime client for the resident +// agent process when config.RealtimeEnabled is set, or returns nil (the +// safe, fully-backward-compatible default). A nil *rtclient.Client is +// nil-safe everywhere it's passed (uplink.Config.Realtime nil disables the +// feature entirely, same as today). +func newAgentRealtimeUplink() *rtclient.Client { + cfg := config.Load() + if !cfg.RealtimeEnabled { + return nil + } + if cfg.Server == "" || cfg.DeviceID == "" || cfg.Space == "" { + fmt.Fprintln(os.Stderr, "sitrep agent: realtime uplink enabled but server/device_id/space is not configured; staying on HTTP ingest") + return nil + } + store, err := outbox.Open(config.RealtimeOutboxPath()) + if err != nil { + fmt.Fprintf(os.Stderr, "sitrep agent: realtime outbox: %v; staying on HTTP ingest\n", err) + return nil + } + url := cfg.RealtimeURLFor() + fmt.Fprintf(os.Stderr, "sitrep agent: realtime uplink enabled (%s)\n", url) + return rtclient.New(rtclient.Config{ + URL: url, + Token: cfg.Token, + DeviceID: cfg.DeviceID, + Space: cfg.Space, + Logf: func(format string, args ...any) { + fmt.Fprintf(os.Stderr, "sitrep agent: realtime: "+format+"\n", args...) + }, + Outbox: store, + }) +} + // runAutomation executes one round with a stable automation identity. Task // lifecycle belongs to bounded `sitrep run`; scheduled work emits metrics and -// messages. -func runAutomation(client *api.Client, automation api.Automation) { - up := uplink.New(uplinkConfig("")) +// messages. rt is the shared realtime uplink (nil unless the feature flag +// is on). +func runAutomation(client *api.Client, automation api.Automation, rt *rtclient.Client) { + cfg := uplinkConfig("") + cfg.Realtime = rt + up := uplink.New(cfg) defer up.Close() sourceID := "a" + automation.ID emitAll := runner.MakeEmitter(up, sourceID, os.Getenv("SITREP_DEBUG") != "") diff --git a/daemon/internal/config/config.go b/daemon/internal/config/config.go index 8cf1601..20a07aa 100644 --- a/daemon/internal/config/config.go +++ b/daemon/internal/config/config.go @@ -7,16 +7,29 @@ import ( "encoding/json" "os" "path/filepath" + "strings" ) type Config struct { Server string `json:"server"` Token string `json:"token,omitempty"` // source role: daemon/CLI writes DeviceID string `json:"device_id,omitempty"` - Space string `json:"space,omitempty"` + Space string `json:"space,omitempty"` // ViewerToken is a legacy field for read-side clients; owner tokens // cover both directions in the space model. ViewerToken string `json:"viewer_token,omitempty"` + + // RealtimeEnabled opts into the realtime WebSocket uplink + // (proto/realtime/SPEC.md) for reliable task/message events and + // metric frames, in place of the HTTP /v2/ingest batch for those + // event kinds. Defaults to false: HTTP ingest remains the default + // path until this is explicitly turned on. Overridable with the + // SITREP_REALTIME env var (1/true/yes). + RealtimeEnabled bool `json:"realtime_enabled,omitempty"` + // RealtimeURL overrides the wss:///v2/realtime endpoint this + // package derives from Server by default. Overridable with + // SITREP_REALTIME_URL. + RealtimeURL string `json:"realtime_url,omitempty"` } func Path() string { @@ -27,6 +40,16 @@ func Path() string { return filepath.Join(home, ".config", "sitrep", "config.json") } +// RealtimeOutboxPath is where the realtime uplink's local SQLite outbox +// (internal/realtime/outbox) lives, alongside the existing config file. +func RealtimeOutboxPath() string { + dir := filepath.Dir(Path()) + if dir == "" || dir == "." { + return "realtime-outbox.db" + } + return filepath.Join(dir, "realtime-outbox.db") +} + func Load() Config { var cfg Config if data, err := os.ReadFile(Path()); err == nil { @@ -38,9 +61,36 @@ func Load() Config { } else if v := os.Getenv("SITREP_TOKEN"); v != "" { cfg.Token = v } + if v := os.Getenv("SITREP_REALTIME"); v != "" { + cfg.RealtimeEnabled = v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") + } + if v := os.Getenv("SITREP_REALTIME_URL"); v != "" { + cfg.RealtimeURL = v + } return cfg } +// RealtimeURLFor derives the wss://.../v2/realtime endpoint from cfg.Server +// (http -> ws, https -> wss), unless cfg.RealtimeURL explicitly overrides +// it. The realtime protocol (proto/realtime/SPEC.md) is transport-agnostic +// and does not fix a URL path; "/v2/realtime" is this daemon's own +// convention, matching the existing "/v2/ingest" and "/v2/automations" +// HTTP routes, and MUST match whatever path the server implementation +// exposes once it lands. +func (cfg Config) RealtimeURLFor() string { + if cfg.RealtimeURL != "" { + return cfg.RealtimeURL + } + u := cfg.Server + switch { + case strings.HasPrefix(u, "https://"): + u = "wss://" + strings.TrimPrefix(u, "https://") + case strings.HasPrefix(u, "http://"): + u = "ws://" + strings.TrimPrefix(u, "http://") + } + return strings.TrimRight(u, "/") + "/v2/realtime" +} + func Save(cfg Config) error { path := Path() if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { diff --git a/daemon/internal/uplink/realtime_test.go b/daemon/internal/uplink/realtime_test.go new file mode 100644 index 0000000..53033e1 --- /dev/null +++ b/daemon/internal/uplink/realtime_test.go @@ -0,0 +1,135 @@ +package uplink + +import ( + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/rttest" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestRealtimeFlagRoutesEventsAwayFromHTTP is the integration check for the +// "flag on -> reliable events go over WS, HTTP ingest stops carrying them; +// flag off -> unchanged" requirement: with Config.Realtime set, task +// lifecycle and message.send events must reach the mock realtime server +// (never the HTTP /v2/ingest capture), while task.log keeps going over +// HTTP exactly as before. +func TestRealtimeFlagRoutesEventsAwayFromHTTP(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + received := make(chan wire.Envelope, 16) + rtSrv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + received <- env + if env.Type == wire.TypeTaskEvent || env.Type == wire.TypeMessageEvent { + body, _ := wire.DecodeBody(env) + switch b := body.(type) { + case wire.TaskEventBody: + conn.Ack(b.DeviceID, b.DeviceSeq) + case wire.MessageEventBody: + conn.Ack(b.DeviceID, b.DeviceSeq) + } + } + } + }) + defer rtSrv.Close() + + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + defer store.Close() + + rt := rtclient.New(rtclient.Config{ + URL: rtSrv.URL(), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + }) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 20 * time.Millisecond, Realtime: rt}) + defer u.Close() + + u.Offer(ev(protocol.TaskStart, func(e *Event) { e.Title = "Nightly backup" })) + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "hi"; e.Level = "info" })) + u.Offer(ev(protocol.MetricUpdate, func(e *Event) { e.Key = "cpu.load"; e.Value = "0.5" })) + u.LogLine("s1", "some output line") + + // The realtime server should see the task.event and message.event. + seenTypes := map[string]bool{} + deadline := time.After(2 * time.Second) +loop: + for len(seenTypes) < 2 { + select { + case env := <-received: + seenTypes[env.Type] = true + case <-deadline: + break loop + } + } + if !seenTypes[wire.TypeTaskEvent] { + t.Error("expected the realtime server to see a task.event") + } + if !seenTypes[wire.TypeMessageEvent] { + t.Error("expected the realtime server to see a message.event") + } + + // Give the HTTP flusher a chance to run, then assert it never carried + // the task/message events (only task.log, which has no realtime + // equivalent). + time.Sleep(100 * time.Millisecond) + u.Close() + + for _, e := range httpCap.all() { + if e.Kind == protocol.TaskStart || e.Kind == protocol.MessageSend || e.Kind == protocol.MetricUpdate { + t.Fatalf("HTTP ingest carried a %s event while the realtime flag was on (double write): %+v", e.Kind, e) + } + } + foundLog := false + for _, e := range httpCap.all() { + if e.Kind == protocol.TaskLog { + foundLog = true + } + } + if !foundLog { + t.Error("expected task.log to still flow over HTTP (it has no realtime equivalent)") + } +} + +// TestRealtimeFlagOffPreservesExistingBehavior asserts the zero-value +// (Realtime == nil) path is completely unaffected: everything still goes +// over HTTP, exactly as before this feature existed. +func TestRealtimeFlagOffPreservesExistingBehavior(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: time.Hour}) + u.Offer(ev(protocol.TaskStart, func(e *Event) { e.Title = "t" })) + u.Close() + + got := httpCap.all() + if len(got) != 1 || got[0].Kind != protocol.TaskStart { + t.Fatalf("expected exactly one task.start over HTTP, got %+v", got) + } +} diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index 4f46aa6..7ad128e 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -19,6 +19,8 @@ import ( "time" "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" ) // Event is the wire form of a protocol event (schema/event.schema.json). @@ -47,6 +49,17 @@ type Config struct { // (including empty heartbeats every 3 ticks) carry ?sources= and any // commands in the response are delivered on Commands. CommandSource string + + // Realtime is an opt-in feature flag: when non-nil, every reliable + // event (task lifecycle + message.send) and every metric.update this + // Uplink is Offer()ed is routed to the realtime WebSocket uplink + // (internal/realtime/client) instead of the HTTP /v2/ingest batch — + // the two paths are never used for the same event, so nothing is + // double-written. task.log passthrough output is unaffected: it has + // no realtime-protocol equivalent and always continues over HTTP. + // Nil (the default) preserves this package's exact prior behavior. + // The caller owns Realtime's lifecycle (construction and Close). + Realtime *rtclient.Client } type Uplink struct { @@ -98,6 +111,22 @@ func (u *Uplink) Offer(ev Event) { if u == nil { return } + u.mu.Lock() + if u.closed { + u.mu.Unlock() + return + } + realtime := u.cfg.Realtime + u.mu.Unlock() + + // Feature flag: when a realtime uplink is configured, every kind with a + // realtime-protocol equivalent is routed there instead of the HTTP + // batch below, so the same event is never sent both ways. task.log (no + // realtime equivalent) and any realtime send failure fall through. + if realtime != nil && u.routeToRealtime(realtime, ev) { + return + } + u.mu.Lock() defer u.mu.Unlock() if u.closed { @@ -125,6 +154,94 @@ func (u *Uplink) Offer(ev Event) { } } +// routeToRealtime translates ev into the matching realtime-protocol +// message and hands it to the realtime client. It reports whether it +// handled ev (true) so the caller skips the HTTP batch entirely for that +// event, or leaves it (false) to fall through unchanged — either because +// the kind has no realtime equivalent (task.log) or because the realtime +// send itself failed, in which case losing the event silently would be +// worse than a harmless duplicate over HTTP. +func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { + switch ev.Kind { + case protocol.TaskStart, protocol.TaskProgress, protocol.TaskStep, protocol.TaskDone, protocol.TaskFail: + te := rtclient.TaskEvent{ + TaskID: ev.SourceID, + Kind: taskEventKind(ev.Kind), + OccurredAt: parseEventTime(ev.TS), + Display: displayHints(ev), + } + switch ev.Kind { + case protocol.TaskStart: + te.Title = ev.Title + case protocol.TaskProgress: + p := ev.Percent + te.Percent = &p + te.Step = ev.Step + case protocol.TaskStep: + te.Step = ev.Step + case protocol.TaskDone, protocol.TaskFail: + te.Message = ev.Text + } + return realtime.SendTaskEvent(te) == nil + case protocol.MessageSend: + return realtime.SendMessageEvent(rtclient.MessageEvent{ + Level: ev.Level, + Text: ev.Text, + OccurredAt: parseEventTime(ev.TS), + }) == nil + case protocol.MetricUpdate: + realtime.SendMetric(wire.MetricSample{ + MetricID: ev.Key, + Value: ev.Value, + Label: ev.Label, + TS: parseEventTime(ev.TS).UnixMilli(), + Display: displayHints(ev), + Target: ev.Target, + Min: ev.Min, + Max: ev.Max, + AlertAbove: ev.AlertAbove, + AlertBelow: ev.AlertBelow, + }) + return true // metric.frame is fire-and-forget; there is no failure to react to + default: + return false // e.g. task.log: no realtime-protocol equivalent + } +} + +func taskEventKind(k protocol.Kind) string { + switch k { + case protocol.TaskStart: + return "started" + case protocol.TaskProgress: + return "progress" + case protocol.TaskStep: + return "step" + case protocol.TaskDone: + return "done" + case protocol.TaskFail: + return "failed" + default: + return "" + } +} + +func displayHints(ev Event) *wire.DisplayHints { + if ev.Icon == "" && ev.Tint == "" && ev.Template == "" { + return nil + } + return &wire.DisplayHints{Icon: ev.Icon, Tint: ev.Tint, Template: ev.Template} +} + +// parseEventTime parses the RFC3339 timestamp uplink.Event carries +// (protocol.go / runner.MakeEmitter always stamp one); a malformed or empty +// value falls back to now rather than producing an invalid envelope. +func parseEventTime(ts string) time.Time { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + return t + } + return time.Now() +} + // LogLine buffers one passthrough output line for the task detail view. // Batched into a single task.log event per flush; capped per window. func (u *Uplink) LogLine(sourceID, line string) { From 04b9a7ce148fd67f2aa8de6bdac8b56eea798e28 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:45:31 +0800 Subject: [PATCH 10/60] fix(daemon): target /v3 realtime endpoint The protocol owner settled the realtime WebSocket route on /v3/realtime (the path the server implementation exposes; the Apple client targets the same). Update RealtimeURLFor and its documentation from the provisional /v2/realtime convention, and add a test pinning the derived endpoint (scheme mapping, trailing-slash normalization, and the explicit RealtimeURL override). --- daemon/internal/config/config.go | 14 ++++----- daemon/internal/config/config_test.go | 41 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 daemon/internal/config/config_test.go diff --git a/daemon/internal/config/config.go b/daemon/internal/config/config.go index 20a07aa..1a2cb41 100644 --- a/daemon/internal/config/config.go +++ b/daemon/internal/config/config.go @@ -26,7 +26,7 @@ type Config struct { // path until this is explicitly turned on. Overridable with the // SITREP_REALTIME env var (1/true/yes). RealtimeEnabled bool `json:"realtime_enabled,omitempty"` - // RealtimeURL overrides the wss:///v2/realtime endpoint this + // RealtimeURL overrides the wss:///v3/realtime endpoint this // package derives from Server by default. Overridable with // SITREP_REALTIME_URL. RealtimeURL string `json:"realtime_url,omitempty"` @@ -70,13 +70,13 @@ func Load() Config { return cfg } -// RealtimeURLFor derives the wss://.../v2/realtime endpoint from cfg.Server +// RealtimeURLFor derives the wss://.../v3/realtime endpoint from cfg.Server // (http -> ws, https -> wss), unless cfg.RealtimeURL explicitly overrides // it. The realtime protocol (proto/realtime/SPEC.md) is transport-agnostic -// and does not fix a URL path; "/v2/realtime" is this daemon's own -// convention, matching the existing "/v2/ingest" and "/v2/automations" -// HTTP routes, and MUST match whatever path the server implementation -// exposes once it lands. +// and does not fix a URL path; "/v3/realtime" is the path agreed across +// implementation lines (the server exposes it there, and the Apple client +// targets the same), distinct from the existing "/v2/ingest" and +// "/v2/automations" HTTP routes. func (cfg Config) RealtimeURLFor() string { if cfg.RealtimeURL != "" { return cfg.RealtimeURL @@ -88,7 +88,7 @@ func (cfg Config) RealtimeURLFor() string { case strings.HasPrefix(u, "http://"): u = "ws://" + strings.TrimPrefix(u, "http://") } - return strings.TrimRight(u, "/") + "/v2/realtime" + return strings.TrimRight(u, "/") + "/v3/realtime" } func Save(cfg Config) error { diff --git a/daemon/internal/config/config_test.go b/daemon/internal/config/config_test.go new file mode 100644 index 0000000..dcd5217 --- /dev/null +++ b/daemon/internal/config/config_test.go @@ -0,0 +1,41 @@ +package config + +import "testing" + +// TestRealtimeURLFor pins the agreed cross-implementation realtime +// endpoint path (/v3/realtime) and the http(s) -> ws(s) scheme mapping. +func TestRealtimeURLFor(t *testing.T) { + cases := []struct { + name string + cfg Config + want string + }{ + { + name: "https becomes wss with v3 path", + cfg: Config{Server: "https://sitrep.example.com"}, + want: "wss://sitrep.example.com/v3/realtime", + }, + { + name: "http becomes ws with v3 path", + cfg: Config{Server: "http://localhost:8787"}, + want: "ws://localhost:8787/v3/realtime", + }, + { + name: "trailing slash on server is normalized", + cfg: Config{Server: "https://sitrep.example.com/"}, + want: "wss://sitrep.example.com/v3/realtime", + }, + { + name: "explicit RealtimeURL wins verbatim", + cfg: Config{Server: "https://sitrep.example.com", RealtimeURL: "wss://other.example.com/custom"}, + want: "wss://other.example.com/custom", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.cfg.RealtimeURLFor(); got != c.want { + t.Fatalf("RealtimeURLFor() = %q, want %q", got, c.want) + } + }) + } +} From e7523fca5ec45f9abde152cfa2583c23e5e24cd9 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:48:35 +0800 Subject: [PATCH 11/60] feat(server): add hibernatable realtime space hub Implement SpaceHub, a per-space SQLite-backed Durable Object that speaks the frozen proto/realtime/SPEC.md v1 protocol: hello/subscribe/ resume connection gating, per-(device,space) device_seq dedup for task.event/message.event, deterministic event folding into tasks/ messages/automations tables, chunked snapshot and chained catch-up delta replies, server-minted config.event, interest leases with lazy 1<->0 edge detection driving throttle/resume_rate, and in-memory-only best-effort metric.frame relay. Uses ctx.acceptWebSocket with per-connection attachments (never DO-instance memory) so identity and handshake state survive hibernation/eviction, and ctx.setWebSocketAutoResponse for the ping/pong heartbeat pair. Also adds the hand-written TS protocol types and runtime guards (server/src/realtime/types.ts, guards.ts) mirroring proto/realtime/ schemas field-for-field, plus pure chunking/row-mapping helpers. --- server/src/realtime/attachment.ts | 45 ++ server/src/realtime/chunking.ts | 84 +++ server/src/realtime/guards.ts | 473 ++++++++++++++++ server/src/realtime/protocol.ts | 90 ++++ server/src/realtime/rows.ts | 83 +++ server/src/realtime/space-hub.ts | 863 ++++++++++++++++++++++++++++++ server/src/realtime/types.ts | 272 ++++++++++ 7 files changed, 1910 insertions(+) create mode 100644 server/src/realtime/attachment.ts create mode 100644 server/src/realtime/chunking.ts create mode 100644 server/src/realtime/guards.ts create mode 100644 server/src/realtime/protocol.ts create mode 100644 server/src/realtime/rows.ts create mode 100644 server/src/realtime/space-hub.ts create mode 100644 server/src/realtime/types.ts diff --git a/server/src/realtime/attachment.ts b/server/src/realtime/attachment.ts new file mode 100644 index 0000000..4344148 --- /dev/null +++ b/server/src/realtime/attachment.ts @@ -0,0 +1,45 @@ +// Per-connection identity + handshake state, persisted via +// WebSocket#serializeAttachment so it survives Durable Object hibernation +// and eviction/reconstruction. Cloudflare caps a serialized attachment at +// 2 KiB — this shape stays a handful of short scalar fields specifically to +// stay far under that limit. +// +// The trust boundary: `deviceId`/`role` are set exactly once, in +// SpaceHub#fetch, from headers the Worker attaches AFTER it has already +// authenticated the connection's st2 token (see adapters/workers.ts). The +// DO never re-derives identity from a client-sent `hello` body (SPEC.md +// section 10) — it only ever reads this attachment. + +import type { DeviceRole } from "./types.ts"; + +export interface ConnAttachment { + deviceId: string; + role: DeviceRole; + connectedAt: number; + /** True once this connection has completed the hello offer/accept + * handshake (SPEC.md section 9.1). Every frame before that must be + * answered with hello_required. */ + helloDone: boolean; + /** True once this connection has sent `subscribe` (viewer only) — gates + * the "resume must follow subscribe on the same connection" rule + * (SPEC.md section 6.3). Not set by interest.renew. */ + subscribedThisConn: boolean; + /** True once this connection has received a snapshot-or-delta reply to + * `resume` (SPEC.md section 6.2/6.3) — the sole gate for receiving live + * deltas. An error reply (e.g. revision_unavailable) never sets this. */ + deltaEligible: boolean; + sessionId?: string; +} + +export function isConnAttachment(v: unknown): v is ConnAttachment { + if (typeof v !== "object" || v === null) return false; + const a = v as Record; + return ( + typeof a.deviceId === "string" && + (a.role === "source" || a.role === "viewer") && + typeof a.connectedAt === "number" && + typeof a.helloDone === "boolean" && + typeof a.subscribedThisConn === "boolean" && + typeof a.deltaEligible === "boolean" + ); +} diff --git a/server/src/realtime/chunking.ts b/server/src/realtime/chunking.ts new file mode 100644 index 0000000..7b43b3c --- /dev/null +++ b/server/src/realtime/chunking.ts @@ -0,0 +1,84 @@ +// Pure snapshot/delta chunking helpers — no I/O, unit-testable without a +// Workers runtime. SPEC.md section 6.2/11: every chunk of one snapshot +// shares the same `revision`, `part` numbers consecutively from 1, and the +// last chunk carries `final: true`; a chained catch-up delta series splits +// strictly on event boundaries so `d1.to_revision == d2.from_revision`. + +import { CHUNK_SOFT_MAX_BYTES } from "./protocol.ts"; +import type { + AutomationState, + DeltaBody, + DeltaEventItem, + MessageRecord, + MetricSample, + SnapshotBody, + TaskState, +} from "./types.ts"; + +type SnapshotBuckets = { + tasks: TaskState[]; + metrics: MetricSample[]; + messages: MessageRecord[]; + automations: AutomationState[]; +}; + +/** Packs the four snapshot arrays into one or more chunks under the frame + * size budget. Always emits at least one chunk (part 1, final true), even + * when every array is empty (SPEC.md: "A snapshot MAY be empty"). */ +export function chunkSnapshot(revision: number, arrays: SnapshotBuckets): SnapshotBody[] { + const buckets: Array<{ key: keyof SnapshotBuckets; items: unknown[] }> = [ + { key: "tasks", items: arrays.tasks }, + { key: "metrics", items: arrays.metrics }, + { key: "messages", items: arrays.messages }, + { key: "automations", items: arrays.automations }, + ]; + const chunks: SnapshotBody[] = []; + const empty = (): SnapshotBuckets => ({ tasks: [], metrics: [], messages: [], automations: [] }); + let cur = empty(); + let curSize = 128; // overhead margin for envelope wrapper + fixed keys + const flush = (final: boolean) => { + chunks.push({ revision, part: chunks.length + 1, final, ...cur }); + cur = empty(); + curSize = 128; + }; + for (const bucket of buckets) { + for (const item of bucket.items) { + const size = JSON.stringify(item).length + 2; + const curCount = cur.tasks.length + cur.metrics.length + cur.messages.length + cur.automations.length; + if (curSize + size > CHUNK_SOFT_MAX_BYTES && curCount > 0) flush(false); + (cur[bucket.key] as unknown[]).push(item); + curSize += size; + } + } + flush(true); + return chunks; +} + +/** Splits a catch-up event range into one or more chained deltas whose + * from/to boundaries connect exactly. Always emits at least one delta + * (possibly with an empty `events` array) so a caller can send it as the + * single resume reply. */ +export function chunkDeltaEvents(fromRevision: number, events: DeltaEventItem[]): DeltaBody[] { + const groups: DeltaEventItem[][] = []; + let cur: DeltaEventItem[] = []; + let curSize = 128; + for (const event of events) { + const size = JSON.stringify(event).length + 2; + if (curSize + size > CHUNK_SOFT_MAX_BYTES && cur.length > 0) { + groups.push(cur); + cur = []; + curSize = 128; + } + cur.push(event); + curSize += size; + } + groups.push(cur); + let from = fromRevision; + const out: DeltaBody[] = []; + for (const group of groups) { + const to = from + group.length; + out.push({ from_revision: from, to_revision: to, events: group }); + from = to; + } + return out; +} diff --git a/server/src/realtime/guards.ts b/server/src/realtime/guards.ts new file mode 100644 index 0000000..af7662b --- /dev/null +++ b/server/src/realtime/guards.ts @@ -0,0 +1,473 @@ +// Lightweight, hand-written runtime validation for the realtime protocol. +// Deliberately NOT ajv: proto/realtime/tools/validate.js is the fixture +// conformance tool (its own package, pinned ajv) and is the authority for +// "does this fixture match the schema". This module is the production-path +// guard the server actually runs on every inbound frame — it mirrors the +// same rules (proto/realtime/*.schema.json) by hand, deliberately narrow in +// scope (no generic JSON Schema engine as a production dependency). +// +// Per SPEC.md section 15, the envelope TOP LEVEL is permanently strict +// (unknown top-level field is always malformed) while `body` tolerates +// unknown fields in future minor versions — these guards enforce the exact +// v1.0.0 body shapes (matching the frozen schemas byte for byte, same as +// the fixtures expect), which is stricter than the runtime-tolerance clause +// permits for a hypothetical v1.1 peer. That's an accepted, documented +// trade: see docs/design/realtime-server.md. + +import { + MESSAGE_TYPES, + type AckBody, + type AnyEnvelope, + type AutomationState, + type CommandBody, + type ConfigEventBody, + type DeviceRole, + type ErrorBody, + type ErrorCode, + type HelloBody, + type InterestRenewBody, + type MessageEventBody, + type MessageType, + type MetricFrameBody, + type MetricSample, + type ResumeBody, + type SubscribeBody, + type TaskEventBody, + type UnsubscribeBody, +} from "./types.ts"; + +export const FRAME_MAX_BYTES = 64 * 1024; + +const ENVELOPE_ID_RE = /^[A-Za-z0-9_-]{1,64}$/; +const DEVICE_ID_RE = /^[A-Za-z0-9_-]{1,128}$/; +const METRIC_ID_RE = /^[a-z0-9_.-]{1,64}$/; + +export type ValidationResult = { ok: true; value: T } | { ok: false; code: ErrorCode; message: string }; + +const ok = (value: T): ValidationResult => ({ ok: true, value }); +const fail = (code: ErrorCode, message: string): ValidationResult => ({ ok: false, code, message }); + +export type ParseResult = + | { kind: "ok"; envelope: AnyEnvelope } + | { kind: "unknown_type"; type: string; id?: string } + | { kind: "error"; code: ErrorCode; message: string; inReplyTo?: string }; + +function isPlainObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +export function isUnixMsTimestamp(v: unknown): v is number { + return typeof v === "number" && Number.isInteger(v) && v >= 1_000_000_000_000 && v <= 9_999_999_999_999; +} + +function isFreeText(v: unknown): v is string { + return typeof v === "string" && v.length <= 2048; +} + +function isLabelText(v: unknown): v is string { + return typeof v === "string" && v.length <= 256; +} + +function isDeviceId(v: unknown): v is string { + return typeof v === "string" && DEVICE_ID_RE.test(v); +} + +function isEnvelopeId(v: unknown): v is string { + return typeof v === "string" && ENVELOPE_ID_RE.test(v); +} + +function hasOnlyKeys(obj: Record, allowed: readonly string[]): boolean { + return Object.keys(obj).every((k) => allowed.includes(k)); +} + +function validateDisplayHints(v: unknown): boolean { + if (v === undefined) return true; + if (!isPlainObject(v)) return false; + if (!hasOnlyKeys(v, ["icon", "tint", "template"])) return false; + if (v.icon !== undefined && !(typeof v.icon === "string" && v.icon.length <= 64)) return false; + if (v.tint !== undefined && !(typeof v.tint === "string" && v.tint.length <= 32)) return false; + if (v.template !== undefined && !(typeof v.template === "string" && v.template.length <= 32)) return false; + return true; +} + +function validateMetricSample(v: unknown): v is MetricSample { + if (!isPlainObject(v)) return false; + if (!hasOnlyKeys(v, ["metric_id", "value", "label", "ts", "display", "target", "min", "max", "alert_above", "alert_below"])) return false; + if (typeof v.metric_id !== "string" || !METRIC_ID_RE.test(v.metric_id)) return false; + if (typeof v.value !== "string" || v.value.length > 256) return false; + if (!isUnixMsTimestamp(v.ts)) return false; + if (v.label !== undefined && !isLabelText(v.label)) return false; + if (!validateDisplayHints(v.display)) return false; + for (const key of ["target", "min", "max", "alert_above", "alert_below"] as const) { + if (v[key] !== undefined && !(typeof v[key] === "string" && (v[key] as string).length <= 64)) return false; + } + return true; +} + +export function validateAutomationState(v: unknown): v is AutomationState { + if (!isPlainObject(v)) return false; + if (!hasOnlyKeys(v, ["automation_id", "name", "executor_kind", "schedule", "state", "last_run_at"])) return false; + if (typeof v.automation_id !== "string" || v.automation_id.length < 1 || v.automation_id.length > 128) return false; + if (typeof v.name !== "string" || v.name.length < 1 || v.name.length > 256) return false; + if (!["script", "agent", "hybrid"].includes(v.executor_kind as string)) return false; + const schedule = v.schedule; + if (!isPlainObject(schedule) || schedule.kind !== "interval" || typeof schedule.every_seconds !== "number" || schedule.every_seconds < 1) { + return false; + } + if (!["active", "paused"].includes(v.state as string)) return false; + if (v.last_run_at !== undefined && !isUnixMsTimestamp(v.last_run_at)) return false; + return true; +} + +// ---- per-type body validators ---- + +function validateHelloBody(body: Record): ValidationResult { + if (body.stage === "offer") { + if (!hasOnlyKeys(body, ["stage", "device_id", "role", "protocol_versions", "capabilities"])) { + return fail("malformed", "hello offer: unexpected field"); + } + if (!isDeviceId(body.device_id)) return fail("malformed", "hello offer: invalid device_id"); + if (body.role !== "source" && body.role !== "viewer") return fail("malformed", "hello offer: invalid role"); + if (!Array.isArray(body.protocol_versions) || body.protocol_versions.length < 1) { + return fail("malformed", "hello offer: protocol_versions must be non-empty"); + } + if (!body.protocol_versions.every((n) => Number.isInteger(n) && (n as number) >= 1)) { + return fail("malformed", "hello offer: invalid protocol_versions entry"); + } + if (body.capabilities !== undefined) { + if (!Array.isArray(body.capabilities) || !body.capabilities.every((c) => typeof c === "string" && c.length <= 64)) { + return fail("malformed", "hello offer: invalid capabilities"); + } + } + return ok({ + stage: "offer", + device_id: body.device_id, + role: body.role, + protocol_versions: body.protocol_versions as number[], + capabilities: body.capabilities as string[] | undefined, + }); + } + if (body.stage === "accept") { + if (!hasOnlyKeys(body, ["stage", "protocol_version", "session_id", "heartbeat_interval_ms", "capabilities"])) { + return fail("malformed", "hello accept: unexpected field"); + } + if (!Number.isInteger(body.protocol_version) || (body.protocol_version as number) < 1) { + return fail("malformed", "hello accept: invalid protocol_version"); + } + if (typeof body.session_id !== "string" || body.session_id.length < 1 || body.session_id.length > 64) { + return fail("malformed", "hello accept: invalid session_id"); + } + if (!Number.isInteger(body.heartbeat_interval_ms) || (body.heartbeat_interval_ms as number) < 1000 || (body.heartbeat_interval_ms as number) > 300000) { + return fail("malformed", "hello accept: invalid heartbeat_interval_ms"); + } + return ok({ + stage: "accept", + protocol_version: body.protocol_version as number, + session_id: body.session_id, + heartbeat_interval_ms: body.heartbeat_interval_ms as number, + capabilities: body.capabilities as string[] | undefined, + }); + } + return fail("malformed", "hello: invalid stage"); +} + +function validateResumeBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["last_revision"])) return fail("malformed", "resume: unexpected field"); + if (!Number.isInteger(body.last_revision) || (body.last_revision as number) < 0) { + return fail("malformed", "resume: last_revision must be a non-negative integer"); + } + return ok({ last_revision: body.last_revision as number }); +} + +function validateAckBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["acked", "in_reply_to", "lease"])) return fail("malformed", "ack: unexpected field"); + if (body.acked === undefined && body.in_reply_to === undefined) { + return fail("malformed", "ack: at least one of acked or in_reply_to required"); + } + if (body.acked !== undefined) { + if (!Array.isArray(body.acked) || body.acked.length < 1) return fail("malformed", "ack: acked must be non-empty array"); + for (const pair of body.acked) { + if (!isPlainObject(pair) || !hasOnlyKeys(pair, ["device_id", "device_seq"])) return fail("malformed", "ack: invalid acked entry"); + if (!isDeviceId(pair.device_id)) return fail("malformed", "ack: invalid device_id in acked"); + if (!Number.isInteger(pair.device_seq) || (pair.device_seq as number) < 1) return fail("malformed", "ack: invalid device_seq in acked"); + } + } + if (body.in_reply_to !== undefined && !isEnvelopeId(body.in_reply_to)) return fail("malformed", "ack: invalid in_reply_to"); + if (body.lease !== undefined) { + if (!isPlainObject(body.lease) || !hasOnlyKeys(body.lease, ["expires_at"]) || !isUnixMsTimestamp(body.lease.expires_at)) { + return fail("malformed", "ack: invalid lease"); + } + } + return ok(body as unknown as AckBody); +} + +function validateTaskEventBody(body: Record): ValidationResult { + const allowed = ["device_id", "device_seq", "task_id", "kind", "occurred_at", "title", "percent", "step", "message", "display"]; + if (!hasOnlyKeys(body, allowed)) return fail("malformed", "task.event: unexpected field"); + if (!isDeviceId(body.device_id)) return fail("malformed", "task.event: invalid device_id"); + if (!Number.isInteger(body.device_seq) || (body.device_seq as number) < 1) return fail("malformed", "task.event: invalid device_seq"); + if (typeof body.task_id !== "string" || body.task_id.length < 1 || body.task_id.length > 256) return fail("malformed", "task.event: invalid task_id"); + if (!["started", "progress", "step", "done", "failed"].includes(body.kind as string)) return fail("malformed", "task.event: invalid kind"); + if (!isUnixMsTimestamp(body.occurred_at)) return fail("malformed", "task.event: invalid occurred_at"); + if (body.title !== undefined && !isFreeText(body.title)) return fail("malformed", "task.event: invalid title"); + if (body.step !== undefined && !isFreeText(body.step)) return fail("malformed", "task.event: invalid step"); + if (body.message !== undefined && !isFreeText(body.message)) return fail("malformed", "task.event: invalid message"); + if (body.percent !== undefined && !(Number.isInteger(body.percent) && (body.percent as number) >= 0 && (body.percent as number) <= 100)) { + return fail("malformed", "task.event: invalid percent"); + } + if (!validateDisplayHints(body.display)) return fail("malformed", "task.event: invalid display"); + if (body.kind === "progress" && body.percent === undefined) return fail("malformed", "task.event: progress requires percent"); + return ok(body as unknown as TaskEventBody); +} + +function validateMessageEventBody(body: Record): ValidationResult { + const allowed = ["device_id", "device_seq", "message_id", "level", "text", "occurred_at", "automation_id"]; + if (!hasOnlyKeys(body, allowed)) return fail("malformed", "message.event: unexpected field"); + if (!isDeviceId(body.device_id)) return fail("malformed", "message.event: invalid device_id"); + if (!Number.isInteger(body.device_seq) || (body.device_seq as number) < 1) return fail("malformed", "message.event: invalid device_seq"); + if (typeof body.message_id !== "string" || body.message_id.length < 1 || body.message_id.length > 128) { + return fail("malformed", "message.event: invalid message_id"); + } + if (!["info", "warn", "error"].includes(body.level as string)) return fail("malformed", "message.event: invalid level"); + if (!isFreeText(body.text)) return fail("malformed", "message.event: invalid text"); + if (!isUnixMsTimestamp(body.occurred_at)) return fail("malformed", "message.event: invalid occurred_at"); + if (body.automation_id !== undefined && !(typeof body.automation_id === "string" && body.automation_id.length <= 128)) { + return fail("malformed", "message.event: invalid automation_id"); + } + return ok(body as unknown as MessageEventBody); +} + +function validateMetricFrameBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["device_id", "metrics"])) return fail("malformed", "metric.frame: unexpected field"); + if (!isDeviceId(body.device_id)) return fail("malformed", "metric.frame: invalid device_id"); + if (!Array.isArray(body.metrics) || body.metrics.length < 1) return fail("malformed", "metric.frame: metrics must be non-empty array"); + if (body.metrics.length > 64) return fail("batch_too_large", "metric.frame: metrics exceeds 64 items"); + for (const m of body.metrics) { + if (!validateMetricSample(m)) return fail("malformed", "metric.frame: invalid metric sample"); + } + return ok(body as unknown as MetricFrameBody); +} + +function validateConfigEventBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["kind", "automation_id", "automation", "occurred_at"])) return fail("malformed", "config.event: unexpected field"); + if (!["automation.upserted", "automation.removed"].includes(body.kind as string)) return fail("malformed", "config.event: invalid kind"); + if (typeof body.automation_id !== "string" || body.automation_id.length < 1 || body.automation_id.length > 128) { + return fail("malformed", "config.event: invalid automation_id"); + } + if (!isUnixMsTimestamp(body.occurred_at)) return fail("malformed", "config.event: invalid occurred_at"); + if (body.kind === "automation.upserted" && !validateAutomationState(body.automation)) { + return fail("malformed", "config.event: automation.upserted requires a valid automation"); + } + return ok(body as unknown as ConfigEventBody); +} + +const SUBSCRIBE_TOPICS = ["task", "metric", "message"]; + +function validateSubscribeLikeBody(body: Record, label: string): ValidationResult { + if (!hasOnlyKeys(body, ["topics"])) return fail("malformed", `${label}: unexpected field`); + if (body.topics !== undefined) { + if (!Array.isArray(body.topics) || !body.topics.every((t) => SUBSCRIBE_TOPICS.includes(t as string))) { + return fail("malformed", `${label}: invalid topics`); + } + } + return ok(body as unknown as SubscribeBody); +} + +function validateUnsubscribeBody(body: Record): ValidationResult { + if (Object.keys(body).length > 0) return fail("malformed", "unsubscribe: body must be empty"); + return ok({} as UnsubscribeBody); +} + +function validateCommandBody(body: Record): ValidationResult { + const allowed = ["command_id", "origin", "issued_by_device_id", "action", "task_id", "automation_id", "target_device_id", "ttl_ms", "params"]; + if (!hasOnlyKeys(body, allowed)) return fail("malformed", "command: unexpected field"); + if (typeof body.command_id !== "string" || body.command_id.length < 1 || body.command_id.length > 64) { + return fail("malformed", "command: invalid command_id"); + } + if (body.origin !== "viewer" && body.origin !== "server") return fail("malformed", "command: invalid origin"); + if (!["pause", "resume", "stop", "run_now", "throttle", "resume_rate"].includes(body.action as string)) { + return fail("malformed", "command: invalid action"); + } + if (!Number.isInteger(body.ttl_ms) || (body.ttl_ms as number) < 1 || (body.ttl_ms as number) > 86_400_000) { + return fail("malformed", "command: invalid ttl_ms"); + } + if (body.issued_by_device_id !== undefined && !isDeviceId(body.issued_by_device_id)) { + return fail("malformed", "command: invalid issued_by_device_id"); + } + if (body.target_device_id !== undefined && !isDeviceId(body.target_device_id)) { + return fail("malformed", "command: invalid target_device_id"); + } + if (body.task_id !== undefined && !(typeof body.task_id === "string" && body.task_id.length >= 1 && body.task_id.length <= 256)) { + return fail("malformed", "command: invalid task_id"); + } + if (body.automation_id !== undefined && !(typeof body.automation_id === "string" && body.automation_id.length >= 1 && body.automation_id.length <= 128)) { + return fail("malformed", "command: invalid automation_id"); + } + if (body.params !== undefined && !isPlainObject(body.params)) return fail("malformed", "command: invalid params"); + + const action = body.action as string; + if (action === "pause" || action === "resume" || action === "stop") { + if (body.origin !== "viewer") return fail("malformed", "command: pause/resume/stop require origin viewer"); + if (body.issued_by_device_id === undefined) return fail("malformed", "command: pause/resume/stop require issued_by_device_id"); + if (body.task_id === undefined) return fail("malformed", "command: pause/resume/stop require task_id"); + if (body.automation_id !== undefined) return fail("malformed", "command: pause/resume/stop must not carry automation_id"); + } else if (action === "run_now") { + if (body.origin !== "viewer") return fail("malformed", "command: run_now requires origin viewer"); + if (body.issued_by_device_id === undefined) return fail("malformed", "command: run_now requires issued_by_device_id"); + if (body.automation_id === undefined) return fail("malformed", "command: run_now requires automation_id"); + if (body.task_id !== undefined) return fail("malformed", "command: run_now must not carry task_id"); + } else { + // throttle | resume_rate + if (body.origin !== "server") return fail("malformed", "command: throttle/resume_rate require origin server"); + if (body.task_id !== undefined || body.automation_id !== undefined) { + return fail("malformed", "command: throttle/resume_rate must not carry task_id or automation_id"); + } + } + return ok(body as unknown as CommandBody); +} + +const ERROR_CODES: readonly ErrorCode[] = [ + "version_unsupported", + "hello_required", + "unauthenticated", + "unauthorized", + "malformed", + "rate_limited", + "frame_too_large", + "batch_too_large", + "revision_unavailable", + "command_expired", + "sequence_gap", + "superseded", + "internal_error", +]; + +function validateErrorBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["code", "message", "in_reply_to", "retryable", "fatal"])) return fail("malformed", "error: unexpected field"); + if (!ERROR_CODES.includes(body.code as ErrorCode)) return fail("malformed", "error: invalid code"); + if (typeof body.message !== "string" || body.message.length > 500) return fail("malformed", "error: invalid message"); + if (typeof body.retryable !== "boolean") return fail("malformed", "error: retryable required"); + if (typeof body.fatal !== "boolean") return fail("malformed", "error: fatal required"); + if (body.in_reply_to !== undefined && !isEnvelopeId(body.in_reply_to)) return fail("malformed", "error: invalid in_reply_to"); + return ok(body as unknown as ErrorBody); +} + +function validateBody(type: MessageType, body: Record): ValidationResult { + switch (type) { + case "hello": + return validateHelloBody(body); + case "resume": + return validateResumeBody(body); + case "ack": + return validateAckBody(body); + case "task.event": + return validateTaskEventBody(body); + case "message.event": + return validateMessageEventBody(body); + case "metric.frame": + return validateMetricFrameBody(body); + case "config.event": + return validateConfigEventBody(body); + case "subscribe": + return validateSubscribeLikeBody(body, "subscribe"); + case "unsubscribe": + return validateUnsubscribeBody(body); + case "interest.renew": + return validateSubscribeLikeBody(body, "interest.renew") as ValidationResult; + case "command": + return validateCommandBody(body); + case "error": + return validateErrorBody(body); + case "snapshot": + case "delta": + // Server-only message types: any client-sent instance is rejected by + // the authorization layer (authorizeClientEnvelope), not here. We + // still validate shape for the server's own outbound round-trip test. + return ok(body); + default: + return fail("malformed", `unhandled type ${type as string}`); + } +} + +/** Parses one raw text frame. Returns "unknown_type" (never "malformed") + * for a `type` this version doesn't recognize (SPEC.md section 15) — the + * caller MUST ignore such frames rather than raising an error. */ +export function parseEnvelope(raw: string): ParseResult { + const byteLength = new TextEncoder().encode(raw).length; + if (byteLength > FRAME_MAX_BYTES) { + return { kind: "error", code: "frame_too_large", message: "frame exceeds 64 KiB" }; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { kind: "error", code: "malformed", message: "invalid JSON" }; + } + if (!isPlainObject(parsed)) return { kind: "error", code: "malformed", message: "envelope must be an object" }; + if (!hasOnlyKeys(parsed, ["type", "id", "ts", "body"])) { + return { kind: "error", code: "malformed", message: "envelope carries an unexpected top-level field" }; + } + const { type, id, ts, body } = parsed; + if (typeof type !== "string") return { kind: "error", code: "malformed", message: "type must be a string" }; + if (!isEnvelopeId(id)) return { kind: "error", code: "malformed", message: "invalid envelope id" }; + if (!isUnixMsTimestamp(ts)) return { kind: "error", code: "malformed", message: "invalid envelope ts" }; + if (!isPlainObject(body)) return { kind: "error", code: "malformed", message: "body must be an object" }; + if (!(MESSAGE_TYPES as readonly string[]).includes(type)) { + return { kind: "unknown_type", type, id }; + } + const result = validateBody(type as MessageType, body); + if (!result.ok) return { kind: "error", code: result.code, message: result.message, inReplyTo: id }; + return { kind: "ok", envelope: { type, id, ts, body: result.value } as AnyEnvelope }; +} + +/** Encodes an envelope back to its wire JSON string. Used by the round-trip + * fixture test (decode a valid fixture, encode it, decode again, compare). */ +export function encodeEnvelope(envelope: AnyEnvelope): string { + return JSON.stringify(envelope); +} + +// ---- SPEC.md section 10.1 authorization matrix ---- + +export type AuthorizeResult = { ok: true } | { ok: false; code: "unauthorized" | "hello_required" }; + +const authOk: AuthorizeResult = { ok: true }; +const authUnauthorized: AuthorizeResult = { ok: false, code: "unauthorized" }; +const authHelloRequired: AuthorizeResult = { ok: false, code: "hello_required" }; + +/** Validates a client-originated envelope's `type` against the sending + * connection's role, per SPEC.md section 10.1. Governs client-originated + * envelopes ONLY — server-originated `command` (throttle/resume_rate) is + * not subject to this matrix (SPEC.md section 8). */ +export function authorizeClientEnvelope(role: DeviceRole, envelope: AnyEnvelope): AuthorizeResult { + switch (envelope.type) { + case "hello": { + const body = envelope.body as HelloBody; + if (body.stage === "accept") return authHelloRequired; + return authOk; + } + case "resume": + case "subscribe": + case "unsubscribe": + case "interest.renew": + return role === "viewer" ? authOk : authUnauthorized; + case "task.event": + case "message.event": + case "metric.frame": + case "ack": + return role === "source" ? authOk : authUnauthorized; + case "command": { + const body = envelope.body as CommandBody; + if (role !== "viewer") return authUnauthorized; + if (body.origin !== "viewer") return authUnauthorized; + return authOk; + } + case "error": + return authOk; + case "snapshot": + case "delta": + case "config.event": + return authUnauthorized; + default: + return authUnauthorized; + } +} diff --git a/server/src/realtime/protocol.ts b/server/src/realtime/protocol.ts new file mode 100644 index 0000000..7d3d821 --- /dev/null +++ b/server/src/realtime/protocol.ts @@ -0,0 +1,90 @@ +// Shared constants and envelope-construction helpers for the SpaceHub DO. +// Pure/no I/O so they are cheap to unit test directly. + +import type { + AnyEnvelope, + Envelope, + ErrorBody, + ErrorCode, + MessageType, +} from "./types.ts"; + +export const PROTOCOL_VERSION = 1; +export const SUPPORTED_PROTOCOL_VERSIONS = [1]; + +/** SPEC.md section 7: server MUST choose a lease duration between 30000 ms + * and 60000 ms inclusive; 45000 ms is RECOMMENDED as a default. */ +export const LEASE_MIN_MS = 30_000; +export const LEASE_MAX_MS = 60_000; +export const LEASE_DEFAULT_MS = 45_000; + +/** Heartbeat cadence advertised in hello{stage:accept}. Informative only + * (SPEC.md section 9.3); the server never depends on it for correctness. */ +export const HEARTBEAT_INTERVAL_MS = 30_000; + +/** Reliable-event retention window: how many trailing revisions the event + * log keeps for incremental `resume` catch-up before a viewer is forced + * onto a fresh `snapshot`. Documented normative choice (SPEC.md section + * 6.2 permits any retention policy as long as gaps fall back to snapshot). */ +export const EVENT_LOG_RETENTION_REVISIONS = 1000; + +/** Bounded message history window carried in `snapshot.body.messages` + * (SPEC.md section 6.4 recommends 200). */ +export const MESSAGE_WINDOW = 200; + +/** Soft ceiling used when packing snapshot/delta chunks, comfortably under + * the 64 KiB hard frame limit (SPEC.md section 11) to leave room for + * envelope/JSON overhead. */ +export const CHUNK_SOFT_MAX_BYTES = 56 * 1024; + +/** SPEC.md section 11: a connection MUST NOT emit more than 10 + * metric.frame envelopes per second. */ +export const METRIC_FRAME_RATE_PER_SEC = 10; + +/** SPEC.md section 12: 2 Hz ceiling per metric_id. */ +export const METRIC_SAMPLE_MIN_INTERVAL_MS = 500; + +export function newEnvelopeId(): string { + return crypto.randomUUID(); +} + +export function makeEnvelope(type: T, body: B, ts = Date.now()): Envelope { + return { type, id: newEnvelopeId(), ts, body }; +} + +export function makeError( + code: ErrorCode, + message: string, + opts: { retryable: boolean; fatal: boolean; inReplyTo?: string }, +): AnyEnvelope { + const body: ErrorBody = { + code, + message, + retryable: opts.retryable, + fatal: opts.fatal, + ...(opts.inReplyTo ? { in_reply_to: opts.inReplyTo } : {}), + }; + return makeEnvelope("error", body) as AnyEnvelope; +} + +/** Fixed retryable/fatal semantics per SPEC.md section 13's error code + * table, so call sites never have to repeat these pairings by hand. */ +export const ERROR_SEMANTICS: Record = { + version_unsupported: { retryable: false, fatal: true }, + hello_required: { retryable: true, fatal: true }, + unauthenticated: { retryable: false, fatal: true }, + unauthorized: { retryable: false, fatal: false }, + malformed: { retryable: true, fatal: false }, + rate_limited: { retryable: true, fatal: false }, + frame_too_large: { retryable: true, fatal: false }, + batch_too_large: { retryable: true, fatal: false }, + revision_unavailable: { retryable: true, fatal: false }, + command_expired: { retryable: false, fatal: false }, + sequence_gap: { retryable: false, fatal: false }, + superseded: { retryable: false, fatal: true }, + internal_error: { retryable: true, fatal: false }, +}; + +export function makeStandardError(code: ErrorCode, message: string, inReplyTo?: string): AnyEnvelope { + return makeError(code, message, { ...ERROR_SEMANTICS[code], inReplyTo }); +} diff --git a/server/src/realtime/rows.ts b/server/src/realtime/rows.ts new file mode 100644 index 0000000..253e37f --- /dev/null +++ b/server/src/realtime/rows.ts @@ -0,0 +1,83 @@ +// SQLite row shapes for SpaceHub storage, and pure mapping functions to/from +// the wire types in ./types.ts. Kept separate from space-hub.ts so the +// mapping logic is easy to read and (where pure) easy to unit test. + +import type { AutomationState, DeltaEventItem, MessageRecord, TaskState } from "./types.ts"; + +export interface TaskRow extends Record { + task_id: string; + device_id: string; + title: string | null; + state: string; + percent: number | null; + step: string | null; + message: string | null; + updated_at: number; + display: string | null; +} + +export function rowToTaskState(row: TaskRow): TaskState { + return { + task_id: row.task_id, + device_id: row.device_id, + state: row.state as TaskState["state"], + updated_at: row.updated_at, + ...(row.title !== null ? { title: row.title } : {}), + ...(row.percent !== null ? { percent: row.percent } : {}), + ...(row.step !== null ? { step: row.step } : {}), + ...(row.message !== null ? { message: row.message } : {}), + ...(row.display !== null ? { display: JSON.parse(row.display) } : {}), + }; +} + +export interface AutomationRow extends Record { + automation_id: string; + name: string; + executor_kind: string; + every_seconds: number; + state: string; + last_run_at: number | null; +} + +export function rowToAutomationState(row: AutomationRow): AutomationState { + return { + automation_id: row.automation_id, + name: row.name, + executor_kind: row.executor_kind as AutomationState["executor_kind"], + schedule: { kind: "interval", every_seconds: row.every_seconds }, + state: row.state as AutomationState["state"], + ...(row.last_run_at !== null ? { last_run_at: row.last_run_at } : {}), + }; +} + +export interface MessageRow extends Record { + message_id: string; + device_id: string; + level: string; + text: string; + occurred_at: number; + revision: number; +} + +export function rowToMessageRecord(row: MessageRow): MessageRecord { + return { + message_id: row.message_id, + device_id: row.device_id, + level: row.level as MessageRecord["level"], + text: row.text, + occurred_at: row.occurred_at, + }; +} + +export interface EventLogRow extends Record { + revision: number; + event_type: string; + device_id: string | null; + device_seq: number | null; + occurred_at: number; + payload: string; +} + +export function rowToDeltaEventItem(row: EventLogRow): DeltaEventItem { + return { event_type: row.event_type, event: JSON.parse(row.payload) } as DeltaEventItem; +} diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts new file mode 100644 index 0000000..9cbb56b --- /dev/null +++ b/server/src/realtime/space-hub.ts @@ -0,0 +1,863 @@ +// SpaceHub: one Durable Object per Sitrep space, implementing the realtime +// synchronization protocol (proto/realtime/SPEC.md, frozen v1). SQLite +// backed, Hibernatable-WebSockets based — the DO holds no connection state +// in memory that isn't recoverable from storage + each connection's +// serialized attachment (see ./attachment.ts). +// +// Concurrency note read before editing: every handler below that must +// behave as "one durable transaction" (SPEC.md section 5.2 dedup+apply, +// section 5.5 config.event mint+revision) is written as a sequence of +// synchronous `this.ctx.storage.sql.exec` calls with NO `await` between +// them. Durable Object storage coalesces consecutive synchronous writes — +// and the JS isolate never interleaves another callback into a running +// synchronous function — so this is sufficient for atomicity without a +// separate transaction API. Do not introduce an `await` in the middle of +// one of these methods without re-establishing atomicity another way. +// +// The same single-threaded-synchronous-function argument is what satisfies +// SPEC.md section 6.2's "no other outbound envelope may interleave a +// chunked snapshot": sendSnapshotChunks/sendChainedDeltas build every frame +// up front and `ws.send()` them in a plain synchronous loop, so nothing +// else in this DO can run until the whole reply has gone out. + +import { DurableObject } from "cloudflare:workers"; +import { type ConnAttachment, isConnAttachment } from "./attachment.ts"; +import { chunkDeltaEvents, chunkSnapshot } from "./chunking.ts"; +import { authorizeClientEnvelope, parseEnvelope } from "./guards.ts"; +import { + ERROR_SEMANTICS, + HEARTBEAT_INTERVAL_MS, + LEASE_DEFAULT_MS, + METRIC_FRAME_RATE_PER_SEC, + EVENT_LOG_RETENTION_REVISIONS, + MESSAGE_WINDOW, + PROTOCOL_VERSION, + SUPPORTED_PROTOCOL_VERSIONS, + makeEnvelope, + makeStandardError, + newEnvelopeId, +} from "./protocol.ts"; +import { + rowToAutomationState, + rowToDeltaEventItem, + rowToMessageRecord, + rowToTaskState, + type AutomationRow, + type EventLogRow, + type MessageRow, + type TaskRow, +} from "./rows.ts"; +import type { + AnyEnvelope, + AutomationState, + CommandBody, + ConfigEventBody, + ConfigEventKind, + DeltaBody, + ErrorCode, + HelloOfferBody, + InterestRenewBody, + MessageEventBody, + MessageType, + MetricFrameBody, + MetricSample, + ResumeBody, + SubscribeBody, + TaskEventBody, +} from "./types.ts"; + +interface RateLimiterState { + frameTimestamps: number[]; + perMetricLastTs: Map; +} + +export class SpaceHub extends DurableObject { + /** Best-effort metric cache (SPEC.md section 6.2: snapshot.metrics is a + * convenience cache, allowed to be empty/stale, outside space_revision + * accounting) — deliberately in-memory only, never written to SQLite. */ + private metricsCache = new Map(); + private rateLimiters = new WeakMap(); + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + // Answers a bare "ping" text frame with "pong" entirely inside the + // runtime, without waking a hibernated DO or invoking webSocketMessage + // (SPEC.md section 9.3) — the whole point of the heartbeat being plain + // text rather than a JSON envelope. + this.ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair("ping", "pong")); + this.migrate(); + } + + private migrate(): void { + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS space_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS event_log ( + revision INTEGER PRIMARY KEY, + event_type TEXT NOT NULL, + device_id TEXT, + device_seq INTEGER, + occurred_at INTEGER NOT NULL, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_event_log_device ON event_log(device_id, device_seq); + CREATE TABLE IF NOT EXISTS dedup ( + device_id TEXT NOT NULL, + device_seq INTEGER NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (device_id, device_seq) + ); + CREATE TABLE IF NOT EXISTS tasks ( + task_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + title TEXT, + state TEXT NOT NULL, + percent INTEGER, + step TEXT, + message TEXT, + updated_at INTEGER NOT NULL, + display TEXT + ); + CREATE TABLE IF NOT EXISTS messages ( + message_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + level TEXT NOT NULL, + text TEXT NOT NULL, + occurred_at INTEGER NOT NULL, + revision INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_messages_revision ON messages(revision); + CREATE TABLE IF NOT EXISTS automations ( + automation_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + executor_kind TEXT NOT NULL, + every_seconds INTEGER NOT NULL, + state TEXT NOT NULL, + last_run_at INTEGER + ); + CREATE TABLE IF NOT EXISTS leases ( + device_id TEXT PRIMARY KEY, + expires_at INTEGER NOT NULL, + topics TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS push_outbox ( + event_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS http_idempotency ( + idempotency_key TEXT PRIMARY KEY, + revision INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS pending_commands ( + command_id TEXT PRIMARY KEY, + target_device_id TEXT, + origin_ts INTEGER NOT NULL, + ttl_ms INTEGER NOT NULL, + payload TEXT NOT NULL, + delivered INTEGER NOT NULL DEFAULT 0 + ); + `); + } + + // ---- WebSocket entry point ---- + + async fetch(request: Request): Promise { + if ((request.headers.get("upgrade") ?? "").toLowerCase() !== "websocket") { + return new Response("expected websocket upgrade", { status: 426 }); + } + // Trust boundary: these headers are set by the Worker AFTER it already + // validated the st2 token (adapters/workers.ts) — never derived from + // anything the client sent inside the WebSocket connection itself. + const deviceId = request.headers.get("x-sitrep-device-id"); + const role = request.headers.get("x-sitrep-role"); + if (!deviceId || (role !== "source" && role !== "viewer")) { + return new Response("missing trusted identity", { status: 400 }); + } + + const pair = new WebSocketPair(); + const client = pair[0]; + const server = pair[1]; + this.ctx.acceptWebSocket(server, [deviceId]); + const attachment: ConnAttachment = { + deviceId, + role, + connectedAt: Date.now(), + helloDone: false, + subscribedThisConn: false, + deltaEligible: false, + }; + server.serializeAttachment(attachment); + return new Response(null, { status: 101, webSocket: client }); + } + + async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise { + if (typeof message !== "string") return; // no binary frames in this protocol + if (message === "ping" || message === "pong") return; // handled by auto-response; defensive no-op + + const raw = ws.deserializeAttachment(); + if (!isConnAttachment(raw)) { + ws.close(1011, "missing connection identity"); + return; + } + const att = raw; + + if (!att.helloDone) { + this.handlePreHello(ws, att, message); + return; + } + + const parsed = parseEnvelope(message); + if (parsed.kind === "unknown_type") return; // SPEC.md section 15: ignore, never malformed + if (parsed.kind === "error") { + this.reply(ws, parsed.code, parsed.message, undefined); + return; + } + const envelope = parsed.envelope; + + // A client-sent hello{stage:"accept"} is a handshake violation at ANY + // point in the connection's life (SPEC.md section 9.1), not just before + // the first accept. + if (envelope.type === "hello") { + this.reply(ws, "hello_required", "stage:accept is server-only", envelope.id); + ws.close(1008, "hello_required"); + return; + } + + const authz = authorizeClientEnvelope(att.role, envelope); + if (!authz.ok) { + this.reply(ws, authz.code, `role ${att.role} may not send ${envelope.type}`, envelope.id); + if (ERROR_SEMANTICS[authz.code].fatal) ws.close(1008, authz.code); + return; + } + + this.logHotPath("frame", { type: envelope.type, device_id: att.deviceId }); + + switch (envelope.type) { + case "resume": + this.handleResume(ws, att, envelope.body as ResumeBody, envelope.id); + break; + case "subscribe": + this.handleSubscribe(ws, att, envelope.body as SubscribeBody, envelope.id); + break; + case "unsubscribe": + this.handleUnsubscribe(ws, att, envelope.id); + break; + case "interest.renew": + this.handleInterestRenew(ws, att, envelope.body as InterestRenewBody, envelope.id); + break; + case "task.event": + this.handleTaskEvent(ws, att, envelope.body as TaskEventBody, envelope.id); + break; + case "message.event": + this.handleMessageEvent(ws, att, envelope.body as MessageEventBody, envelope.id); + break; + case "metric.frame": + this.handleMetricFrame(ws, att, envelope.body as MetricFrameBody, envelope.id); + break; + case "command": + this.handleCommand(ws, att, envelope.body as CommandBody, envelope.ts, envelope.id); + break; + case "ack": + case "error": + // Optional/diagnostic client-sent frames: nothing to durably do. + break; + default: + this.reply(ws, "malformed", `unexpected type ${envelope.type}`, envelope.id); + } + } + + async webSocketClose(): Promise { + // Interest leases and the event log are keyed by device/space, not by + // connection (SPEC.md section 7) — nothing to clean up on close beyond + // letting the socket go. + } + + async webSocketError(_ws: WebSocket, error: unknown): Promise { + this.logAlways("ws_error", { message: String(error) }); + } + + // ---- HTTP control-plane entry point (config.event minting) ---- + + /** Mints (or replays, if idempotencyKey was already seen) a config.event + * for an automation upsert/removal. Single synchronous transaction, same + * discipline as applyReliableEvent — see the class-level comment. */ + mintConfigEvent( + idempotencyKey: string | null, + payload: { kind: ConfigEventKind; automation_id: string; automation?: AutomationState }, + ): { revision: number; automation: AutomationState | null } { + if (idempotencyKey) { + const existing = this.ctx.storage.sql + .exec<{ revision: number }>("SELECT revision FROM http_idempotency WHERE idempotency_key = ?", idempotencyKey) + .toArray()[0]; + if (existing) { + return { revision: existing.revision, automation: this.readAutomation(payload.automation_id) }; + } + } + + const occurredAt = Date.now(); + const revision = this.getRevision() + 1; + const body: ConfigEventBody = { + kind: payload.kind, + automation_id: payload.automation_id, + automation: payload.automation, + occurred_at: occurredAt, + }; + const eventPayload = JSON.stringify(body); + + this.ctx.storage.sql.exec( + `INSERT INTO event_log (revision, event_type, device_id, device_seq, occurred_at, payload) VALUES (?, 'config.event', NULL, NULL, ?, ?)`, + revision, + occurredAt, + eventPayload, + ); + this.foldConfigEvent(body); + this.setRevision(revision); + if (idempotencyKey) { + this.ctx.storage.sql.exec( + `INSERT INTO http_idempotency (idempotency_key, revision, created_at) VALUES (?, ?, ?)`, + idempotencyKey, + revision, + occurredAt, + ); + } + this.pruneEventLog(revision); + + this.broadcastDelta({ + from_revision: revision - 1, + to_revision: revision, + events: [{ event_type: "config.event", event: body }], + }); + + return { revision, automation: this.readAutomation(payload.automation_id) }; + } + + automationsSnapshot(): AutomationState[] { + return this.ctx.storage.sql.exec("SELECT * FROM automations ORDER BY automation_id").toArray().map(rowToAutomationState); + } + + // ---- hello / handshake ---- + + private handlePreHello(ws: WebSocket, att: ConnAttachment, message: string): void { + const parsed = parseEnvelope(message); + const offerBody = + parsed.kind === "ok" && parsed.envelope.type === "hello" && (parsed.envelope.body as HelloOfferBody).stage === "offer" + ? (parsed.envelope.body as HelloOfferBody) + : null; + if (!offerBody) { + this.reply(ws, "hello_required", "hello{stage:offer} must be the first frame", undefined); + ws.close(1008, "hello_required"); + return; + } + + const intersection = offerBody.protocol_versions.filter((v) => SUPPORTED_PROTOCOL_VERSIONS.includes(v)); + if (intersection.length === 0) { + this.reply(ws, "version_unsupported", "no shared protocol version", parsed.kind === "ok" ? parsed.envelope.id : undefined); + ws.close(1008, "version_unsupported"); + return; + } + const negotiated = Math.max(...intersection); + + // SPEC.md section 9.4: the new connection completing hello supersedes + // any other open connection already authenticated for this device_id. + for (const other of this.ctx.getWebSockets(att.deviceId)) { + if (other === ws) continue; + const otherAtt = other.deserializeAttachment(); + if (isConnAttachment(otherAtt) && otherAtt.helloDone) { + this.send(other, "error", { + code: "superseded", + message: "device completed hello on a newer connection", + ...ERROR_SEMANTICS.superseded, + }); + other.close(1008, "superseded"); + } + } + + att.helloDone = true; + att.sessionId = crypto.randomUUID(); + ws.serializeAttachment(att); + + this.send(ws, "hello", { + stage: "accept", + protocol_version: negotiated, + session_id: att.sessionId, + heartbeat_interval_ms: HEARTBEAT_INTERVAL_MS, + }); + + if (att.role === "source") this.drainPendingCommands(ws, att.deviceId); + this.reconcileLeaseEdge(); + } + + // ---- resume / snapshot / delta ---- + + private handleResume(ws: WebSocket, att: ConnAttachment, body: ResumeBody, envelopeId: string): void { + if (!att.subscribedThisConn) { + this.reply(ws, "malformed", "resume must follow subscribe on the same connection", envelopeId); + return; + } + const N = body.last_revision; + const R = this.getRevision(); + + if (N > R) { + this.reply(ws, "revision_unavailable", `last_revision ${N} exceeds current revision ${R}`, envelopeId); + return; // deltaEligible stays false — no reply here confers eligibility + } + + if (N === 0) { + this.sendSnapshotChunks(ws, R); + } else if (N === R) { + this.send(ws, "delta", { from_revision: N, to_revision: N, events: [] }); + } else { + const minRetained = this.minRetainedRevision(); + if (minRetained !== null && minRetained <= N + 1) { + this.sendChainedDeltas(ws, N, this.eventLogRange(N, R)); + } else { + this.sendSnapshotChunks(ws, R); + } + } + + att.deltaEligible = true; + ws.serializeAttachment(att); + } + + private sendSnapshotChunks(ws: WebSocket, revision: number): void { + const arrays = this.buildSnapshotArrays(); + const chunks = chunkSnapshot(revision, arrays); + // No `await` in this loop: see the class-level concurrency note. This + // is what guarantees nothing else can interleave an envelope (SPEC.md + // section 6.2) between chunks, including on this same connection. + for (const chunk of chunks) this.send(ws, "snapshot", chunk); + } + + private sendChainedDeltas(ws: WebSocket, fromRevision: number, rows: EventLogRow[]): void { + const events = rows.map(rowToDeltaEventItem); + const deltas = chunkDeltaEvents(fromRevision, events); + for (const delta of deltas) this.send(ws, "delta", delta); + } + + private buildSnapshotArrays() { + const tasks = this.ctx.storage.sql.exec("SELECT * FROM tasks ORDER BY task_id").toArray().map(rowToTaskState); + const automations = this.ctx.storage.sql + .exec("SELECT * FROM automations ORDER BY automation_id") + .toArray() + .map(rowToAutomationState); + const messageRows = this.ctx.storage.sql + .exec("SELECT * FROM messages ORDER BY revision DESC LIMIT ?", MESSAGE_WINDOW) + .toArray(); + messageRows.reverse(); + const messages = messageRows.map(rowToMessageRecord); + const metrics = [...this.metricsCache.values()]; + return { tasks, metrics, messages, automations }; + } + + private minRetainedRevision(): number | null { + const row = this.ctx.storage.sql.exec<{ m: number | null }>("SELECT MIN(revision) as m FROM event_log").toArray()[0]; + return row?.m ?? null; + } + + private eventLogRange(fromExclusive: number, toInclusive: number): EventLogRow[] { + return this.ctx.storage.sql + .exec("SELECT * FROM event_log WHERE revision > ? AND revision <= ? ORDER BY revision", fromExclusive, toInclusive) + .toArray(); + } + + // ---- reliable event application (task.event / message.event) ---- + + private handleTaskEvent(ws: WebSocket, att: ConnAttachment, body: TaskEventBody, envelopeId: string): void { + if (body.device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "device_id does not match the authenticated identity", envelopeId); + return; + } + const { revision, duplicate } = this.applyReliableEvent("task.event", body.device_id, body.device_seq, body, body.occurred_at); + this.send(ws, "ack", { acked: [{ device_id: body.device_id, device_seq: body.device_seq }] }); + if (!duplicate) { + this.broadcastDelta({ from_revision: revision - 1, to_revision: revision, events: [{ event_type: "task.event", event: body }] }); + this.reconcileLeaseEdge(); + } + } + + private handleMessageEvent(ws: WebSocket, att: ConnAttachment, body: MessageEventBody, envelopeId: string): void { + if (body.device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "device_id does not match the authenticated identity", envelopeId); + return; + } + const { revision, duplicate } = this.applyReliableEvent("message.event", body.device_id, body.device_seq, body, body.occurred_at); + this.send(ws, "ack", { acked: [{ device_id: body.device_id, device_seq: body.device_seq }] }); + if (!duplicate) { + this.broadcastDelta({ from_revision: revision - 1, to_revision: revision, events: [{ event_type: "message.event", event: body }] }); + this.reconcileLeaseEdge(); + } + } + + private applyReliableEvent( + eventType: "task.event" | "message.event", + deviceId: string, + deviceSeq: number, + eventBody: TaskEventBody | MessageEventBody, + occurredAt: number, + ): { revision: number; duplicate: boolean } { + const existing = this.ctx.storage.sql + .exec<{ revision: number }>("SELECT revision FROM dedup WHERE device_id = ? AND device_seq = ?", deviceId, deviceSeq) + .toArray()[0]; + if (existing) { + // SPEC.md section 5.2: MUST NOT re-apply, MUST NOT bump revision, + // MUST still ack — the caller sends the ack regardless of `duplicate`. + return { revision: existing.revision, duplicate: true }; + } + + const revision = this.getRevision() + 1; + const payload = JSON.stringify(eventBody); + + this.ctx.storage.sql.exec( + `INSERT INTO event_log (revision, event_type, device_id, device_seq, occurred_at, payload) VALUES (?, ?, ?, ?, ?, ?)`, + revision, + eventType, + deviceId, + deviceSeq, + occurredAt, + payload, + ); + this.ctx.storage.sql.exec(`INSERT INTO dedup (device_id, device_seq, revision) VALUES (?, ?, ?)`, deviceId, deviceSeq, revision); + if (eventType === "task.event") this.foldTaskEvent(eventBody as TaskEventBody); + else this.foldMessageEvent(eventBody as MessageEventBody, revision); + this.setRevision(revision); + // event_id is its own idempotency key: this INSERT only ever runs on + // the non-duplicate branch, so a retried device_seq can never mint a + // second outbox row for the same logical event. + this.ctx.storage.sql.exec( + `INSERT INTO push_outbox (event_id, kind, payload, status, created_at) VALUES (?, ?, ?, 'pending', ?)`, + crypto.randomUUID(), + eventType, + payload, + Date.now(), + ); + this.pruneEventLog(revision); + + return { revision, duplicate: false }; + } + + private foldTaskEvent(body: TaskEventBody): void { + const prev = this.ctx.storage.sql.exec("SELECT * FROM tasks WHERE task_id = ?", body.task_id).toArray()[0]; + + const state = body.kind === "done" ? "done" : body.kind === "failed" ? "failed" : "running"; + + const title = body.title && body.title.length > 0 ? body.title : (prev?.title ?? null); + + let percent = prev?.percent ?? null; + if (state === "running" && body.percent !== undefined) percent = body.percent; + else if (body.kind === "done") percent = 100; + // failed: keep last value (no assignment) + + const step = body.kind === "done" || body.kind === "failed" ? null : (body.step ?? prev?.step ?? null); + const message = body.kind === "done" || body.kind === "failed" ? (body.message ?? null) : null; + const display = body.display ? JSON.stringify(body.display) : (prev?.display ?? null); + + this.ctx.storage.sql.exec( + `INSERT INTO tasks (task_id, device_id, title, state, percent, step, message, updated_at, display) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(task_id) DO UPDATE SET + device_id = excluded.device_id, title = excluded.title, state = excluded.state, + percent = excluded.percent, step = excluded.step, message = excluded.message, + updated_at = excluded.updated_at, display = excluded.display`, + body.task_id, + body.device_id, + title, + state, + percent, + step, + message, + body.occurred_at, + display, + ); + } + + private foldMessageEvent(body: MessageEventBody, revision: number): void { + this.ctx.storage.sql.exec( + `INSERT INTO messages (message_id, device_id, level, text, occurred_at, revision) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(message_id) DO NOTHING`, + body.message_id, + body.device_id, + body.level, + body.text, + body.occurred_at, + revision, + ); + // Bounded window (SPEC.md section 6.4, N=MESSAGE_WINDOW): keep only the + // most recent N rows. No-op while fewer than N rows exist (subquery + // returns NULL, so the DELETE matches nothing). + this.ctx.storage.sql.exec( + `DELETE FROM messages WHERE revision < ( + SELECT revision FROM messages ORDER BY revision DESC LIMIT 1 OFFSET ? + )`, + MESSAGE_WINDOW - 1, + ); + } + + private foldConfigEvent(body: ConfigEventBody): void { + if (body.kind === "automation.upserted" && body.automation) { + const a = body.automation; + this.ctx.storage.sql.exec( + `INSERT INTO automations (automation_id, name, executor_kind, every_seconds, state, last_run_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(automation_id) DO UPDATE SET + name = excluded.name, executor_kind = excluded.executor_kind, every_seconds = excluded.every_seconds, + state = excluded.state, last_run_at = excluded.last_run_at`, + a.automation_id, + a.name, + a.executor_kind, + a.schedule.every_seconds, + a.state, + a.last_run_at ?? null, + ); + } else if (body.kind === "automation.removed") { + this.ctx.storage.sql.exec(`DELETE FROM automations WHERE automation_id = ?`, body.automation_id); + } + } + + private readAutomation(id: string): AutomationState | null { + const row = this.ctx.storage.sql.exec("SELECT * FROM automations WHERE automation_id = ?", id).toArray()[0]; + return row ? rowToAutomationState(row) : null; + } + + private getRevision(): number { + const row = this.ctx.storage.sql.exec<{ value: string }>("SELECT value FROM space_meta WHERE key = 'revision'").toArray()[0]; + return row ? Number(row.value) : 0; + } + + private setRevision(n: number): void { + this.ctx.storage.sql.exec( + `INSERT INTO space_meta (key, value) VALUES ('revision', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + String(n), + ); + } + + private pruneEventLog(currentRevision: number): void { + const floor = currentRevision - EVENT_LOG_RETENTION_REVISIONS; + if (floor > 0) this.ctx.storage.sql.exec(`DELETE FROM event_log WHERE revision <= ?`, floor); + } + + private broadcastDelta(delta: DeltaBody): void { + for (const ws of this.ctx.getWebSockets()) { + const att = ws.deserializeAttachment(); + if (!isConnAttachment(att) || att.role !== "viewer" || !att.deltaEligible) continue; + if (!this.deviceHasActiveLease(att.deviceId)) continue; + this.send(ws, "delta", delta); + } + } + + // ---- metric.frame (best-effort, never persisted, never revisioned) ---- + + private handleMetricFrame(ws: WebSocket, att: ConnAttachment, body: MetricFrameBody, envelopeId: string): void { + if (body.device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "device_id does not match the authenticated identity", envelopeId); + return; + } + const limiter = this.rateLimiterFor(ws); + const now = Date.now(); + limiter.frameTimestamps = limiter.frameTimestamps.filter((t) => now - t < 1000); + if (limiter.frameTimestamps.length >= METRIC_FRAME_RATE_PER_SEC) { + this.reply(ws, "rate_limited", "metric.frame exceeded 10/s on this connection", envelopeId); + return; + } + limiter.frameTimestamps.push(now); + + const accepted: MetricSample[] = []; + for (const sample of body.metrics) { + const lastTs = limiter.perMetricLastTs.get(sample.metric_id) ?? 0; + if (sample.ts <= lastTs) continue; // SPEC.md section 12: discard stale/duplicate + limiter.perMetricLastTs.set(sample.metric_id, sample.ts); + this.metricsCache.set(sample.metric_id, sample); + accepted.push(sample); + } + if (accepted.length > 0) this.broadcastMetricFrame({ device_id: body.device_id, metrics: accepted }); + // Deliberately no this.ctx.storage.sql.exec(...) anywhere in this + // method and no revision change — see the class-level cost notes and + // docs/design/realtime-server.md. + } + + private broadcastMetricFrame(body: MetricFrameBody): void { + const now = Date.now(); + for (const ws of this.ctx.getWebSockets()) { + const att = ws.deserializeAttachment(); + if (!isConnAttachment(att) || att.role !== "viewer" || !att.helloDone) continue; + if (!this.deviceHasMetricInterest(att.deviceId, now)) continue; + this.send(ws, "metric.frame", body); + } + } + + private rateLimiterFor(ws: WebSocket): RateLimiterState { + let state = this.rateLimiters.get(ws); + if (!state) { + state = { frameTimestamps: [], perMetricLastTs: new Map() }; + this.rateLimiters.set(ws, state); + } + return state; + } + + // ---- interest lease ---- + + private handleSubscribe(ws: WebSocket, att: ConnAttachment, body: SubscribeBody, envelopeId: string): void { + const expiresAt = this.upsertLease(att.deviceId, body.topics ?? []); + att.subscribedThisConn = true; + ws.serializeAttachment(att); + this.send(ws, "ack", { in_reply_to: envelopeId, lease: { expires_at: expiresAt } }); + this.reconcileLeaseEdge(); + } + + private handleUnsubscribe(ws: WebSocket, att: ConnAttachment, envelopeId: string): void { + this.ctx.storage.sql.exec(`DELETE FROM leases WHERE device_id = ?`, att.deviceId); + this.send(ws, "ack", { in_reply_to: envelopeId }); + this.reconcileLeaseEdge(); + } + + private handleInterestRenew(ws: WebSocket, att: ConnAttachment, body: InterestRenewBody, envelopeId: string): void { + // SPEC.md section 7: interest.renew always wholesale-replaces topics, + // and if no active lease exists it behaves exactly like a fresh + // subscribe — the upsert below does both without special-casing. + const expiresAt = this.upsertLease(att.deviceId, body.topics ?? []); + this.send(ws, "ack", { in_reply_to: envelopeId, lease: { expires_at: expiresAt } }); + this.reconcileLeaseEdge(); + } + + private upsertLease(deviceId: string, topics: string[]): number { + const expiresAt = Date.now() + LEASE_DEFAULT_MS; + this.ctx.storage.sql.exec( + `INSERT INTO leases (device_id, expires_at, topics) VALUES (?, ?, ?) + ON CONFLICT(device_id) DO UPDATE SET expires_at = excluded.expires_at, topics = excluded.topics`, + deviceId, + expiresAt, + JSON.stringify(topics), + ); + return expiresAt; + } + + private deviceHasActiveLease(deviceId: string): boolean { + const row = this.ctx.storage.sql.exec<{ expires_at: number }>("SELECT expires_at FROM leases WHERE device_id = ?", deviceId).toArray()[0]; + return !!row && row.expires_at > Date.now(); + } + + private deviceHasMetricInterest(deviceId: string, now: number): boolean { + const row = this.ctx.storage.sql + .exec<{ expires_at: number; topics: string }>("SELECT expires_at, topics FROM leases WHERE device_id = ?", deviceId) + .toArray()[0]; + if (!row || row.expires_at <= now) return false; + const topics: string[] = JSON.parse(row.topics); + return topics.length === 0 || topics.includes("metric"); + } + + /** Lazy edge-detection (SPEC.md section 7 permits lazy lease expiry): run + * after every lease mutation and every reliable-event application so a + * lapsed lease is never counted active, without any timer/alarm. */ + private reconcileLeaseEdge(): void { + const now = Date.now(); + const activeNow = this.ctx.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM leases WHERE expires_at > ?", now).toArray()[0].n; + const prevRow = this.ctx.storage.sql.exec<{ value: string }>("SELECT value FROM space_meta WHERE key = 'lease_active_count'").toArray()[0]; + const prev = prevRow ? Number(prevRow.value) : 0; + if (activeNow === prev) return; + this.ctx.storage.sql.exec( + `INSERT INTO space_meta (key, value) VALUES ('lease_active_count', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + String(activeNow), + ); + if (prev > 0 && activeNow === 0) this.broadcastServerCommand("throttle"); + else if (prev === 0 && activeNow > 0) this.broadcastServerCommand("resume_rate"); + } + + private broadcastServerCommand(action: "throttle" | "resume_rate"): void { + const body: CommandBody = { command_id: crypto.randomUUID(), origin: "server", action, ttl_ms: 60_000 }; + for (const ws of this.ctx.getWebSockets()) { + const att = ws.deserializeAttachment(); + if (!isConnAttachment(att) || att.role !== "source" || !att.helloDone) continue; + this.send(ws, "command", body); + } + } + + // ---- command relay (viewer-issued pause/resume/stop/run_now) ---- + + private handleCommand(ws: WebSocket, att: ConnAttachment, body: CommandBody, envelopeTs: number, envelopeId: string): void { + if (body.issued_by_device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "issued_by_device_id must match the authenticated device", envelopeId); + return; + } + if (Date.now() > envelopeTs + body.ttl_ms) { + this.reply(ws, "command_expired", "command TTL elapsed before relay", envelopeId); + return; + } + + const targets = body.target_device_id ? this.ctx.getWebSockets(body.target_device_id) : this.ctx.getWebSockets(); + let delivered = false; + for (const target of targets) { + const targetAtt = target.deserializeAttachment(); + if (!isConnAttachment(targetAtt) || targetAtt.role !== "source" || !targetAtt.helloDone) continue; + if (body.target_device_id && targetAtt.deviceId !== body.target_device_id) continue; + // Relay rules (SPEC.md section 8): preserve the original ts and + // command_id, only the envelope id is fresh. + this.sendRaw(target, { type: "command", id: newEnvelopeId(), ts: envelopeTs, body }); + delivered = true; + } + if (!delivered) { + this.ctx.storage.sql.exec( + `INSERT INTO pending_commands (command_id, target_device_id, origin_ts, ttl_ms, payload, delivered) + VALUES (?, ?, ?, ?, ?, 0) ON CONFLICT(command_id) DO NOTHING`, + body.command_id, + body.target_device_id ?? null, + envelopeTs, + body.ttl_ms, + JSON.stringify(body), + ); + } + } + + /** Best-effort redelivery of commands that missed a disconnected source + * (SPEC.md section 14: command is "persisted... until delivered or + * expired"). Runs once a source completes hello on a fresh connection. */ + private drainPendingCommands(ws: WebSocket, deviceId: string): void { + const now = Date.now(); + const rows = this.ctx.storage.sql + .exec<{ command_id: string; origin_ts: number; ttl_ms: number; payload: string }>( + `SELECT command_id, origin_ts, ttl_ms, payload FROM pending_commands WHERE delivered = 0 AND (target_device_id = ? OR target_device_id IS NULL)`, + deviceId, + ) + .toArray(); + for (const row of rows) { + if (now > row.origin_ts + row.ttl_ms) { + this.ctx.storage.sql.exec(`DELETE FROM pending_commands WHERE command_id = ?`, row.command_id); + continue; + } + const body = JSON.parse(row.payload) as CommandBody; + this.sendRaw(ws, { type: "command", id: newEnvelopeId(), ts: row.origin_ts, body }); + this.ctx.storage.sql.exec(`UPDATE pending_commands SET delivered = 1 WHERE command_id = ?`, row.command_id); + } + } + + // ---- send helpers ---- + + private send(ws: WebSocket, type: MessageType, body: B): void { + ws.send(JSON.stringify(makeEnvelope(type, body))); + } + + private sendRaw(ws: WebSocket, envelope: AnyEnvelope): void { + ws.send(JSON.stringify(envelope)); + } + + private reply(ws: WebSocket, code: ErrorCode, message: string, inReplyTo: string | undefined): void { + this.sendRaw(ws, makeStandardError(code, message, inReplyTo)); + if (code !== "internal_error") this.logAlways("protocol_error", { code, message }); + } + + private logHotPath(event: string, data: Record): void { + console.log(JSON.stringify({ level: "info", event, ...data, ts: Date.now() })); + } + + private logAlways(event: string, data: Record): void { + console.log(JSON.stringify({ level: "error", event, ...data, ts: Date.now() })); + } +} + +// Referenced only for the protocol_version constant it advertises via +// hello{stage:accept}; kept here so the export is exercised by typecheck. +export const SPACE_HUB_PROTOCOL_VERSION = PROTOCOL_VERSION; diff --git a/server/src/realtime/types.ts b/server/src/realtime/types.ts new file mode 100644 index 0000000..499caf1 --- /dev/null +++ b/server/src/realtime/types.ts @@ -0,0 +1,272 @@ +// Hand-written TypeScript mirror of proto/realtime/*.schema.json (protocol +// v1, frozen). This file is a DERIVED artifact: proto/realtime/ is the sole +// source of truth and every shape here must match its schemas field for +// field. Any semantic difference from proto/realtime/ is a bug in this file, +// never an intentional deviation — do not "improve" the protocol here. +// +// proto/realtime/ itself MUST NOT be edited as part of server work. + +export type DeviceRole = "source" | "viewer"; +export type MessageLevel = "info" | "warn" | "error"; +export type TaskKind = "started" | "progress" | "step" | "done" | "failed"; +export type TaskRunState = "running" | "done" | "failed"; +export type AutomationExecutorKind = "script" | "agent" | "hybrid"; +export type AutomationRunState = "active" | "paused"; +export type SubscribeTopic = "task" | "metric" | "message"; +export type CommandOrigin = "viewer" | "server"; +export type CommandAction = "pause" | "resume" | "stop" | "run_now" | "throttle" | "resume_rate"; +export type ConfigEventKind = "automation.upserted" | "automation.removed"; + +export type ErrorCode = + | "version_unsupported" + | "hello_required" + | "unauthenticated" + | "unauthorized" + | "malformed" + | "rate_limited" + | "frame_too_large" + | "batch_too_large" + | "revision_unavailable" + | "command_expired" + | "sequence_gap" + | "superseded" + | "internal_error"; + +export const MESSAGE_TYPES = [ + "hello", + "resume", + "snapshot", + "delta", + "ack", + "task.event", + "message.event", + "metric.frame", + "config.event", + "subscribe", + "unsubscribe", + "interest.renew", + "command", + "error", +] as const; + +export type MessageType = (typeof MESSAGE_TYPES)[number]; + +// ---- common.schema.json $defs ---- + +export interface DisplayHints { + icon?: string; + tint?: string; + template?: string; +} + +export interface TaskState { + task_id: string; + device_id: string; + title?: string; + state: TaskRunState; + percent?: number; + step?: string; + message?: string; + updated_at: number; + display?: DisplayHints; +} + +export interface MetricSample { + metric_id: string; + value: string; + label?: string; + ts: number; + display?: DisplayHints; + target?: string; + min?: string; + max?: string; + alert_above?: string; + alert_below?: string; +} + +export interface AutomationSchedule { + kind: "interval"; + every_seconds: number; +} + +export interface AutomationState { + automation_id: string; + name: string; + executor_kind: AutomationExecutorKind; + schedule: AutomationSchedule; + state: AutomationRunState; + last_run_at?: number; +} + +export interface MessageRecord { + message_id: string; + device_id: string; + level: MessageLevel; + text: string; + occurred_at: number; +} + +// ---- message bodies ---- + +export interface HelloOfferBody { + stage: "offer"; + device_id: string; + role: DeviceRole; + protocol_versions: number[]; + capabilities?: string[]; +} + +export interface HelloAcceptBody { + stage: "accept"; + protocol_version: number; + session_id: string; + heartbeat_interval_ms: number; + capabilities?: string[]; +} + +export type HelloBody = HelloOfferBody | HelloAcceptBody; + +export interface ResumeBody { + last_revision: number; +} + +export interface SnapshotBody { + revision: number; + part: number; + final: boolean; + tasks: TaskState[]; + metrics: MetricSample[]; + messages: MessageRecord[]; + automations: AutomationState[]; +} + +export interface TaskEventBody { + device_id: string; + device_seq: number; + task_id: string; + kind: TaskKind; + occurred_at: number; + title?: string; + percent?: number; + step?: string; + message?: string; + display?: DisplayHints; +} + +export interface MessageEventBody { + device_id: string; + device_seq: number; + message_id: string; + level: MessageLevel; + text: string; + occurred_at: number; + automation_id?: string; +} + +export interface ConfigEventBody { + kind: ConfigEventKind; + automation_id: string; + automation?: AutomationState; + occurred_at: number; +} + +export type DeltaEventItem = + | { event_type: "task.event"; event: TaskEventBody } + | { event_type: "message.event"; event: MessageEventBody } + | { event_type: "config.event"; event: ConfigEventBody }; + +export interface DeltaBody { + from_revision: number; + to_revision: number; + events: DeltaEventItem[]; +} + +export interface AckedPair { + device_id: string; + device_seq: number; +} + +export interface LeaseInfo { + expires_at: number; +} + +export interface AckBody { + acked?: AckedPair[]; + in_reply_to?: string; + lease?: LeaseInfo; +} + +export interface MetricFrameBody { + device_id: string; + metrics: MetricSample[]; +} + +export interface SubscribeBody { + topics?: SubscribeTopic[]; +} + +export type UnsubscribeBody = Record; + +export interface InterestRenewBody { + topics?: SubscribeTopic[]; +} + +export interface CommandBody { + command_id: string; + origin: CommandOrigin; + issued_by_device_id?: string; + action: CommandAction; + task_id?: string; + automation_id?: string; + target_device_id?: string; + ttl_ms: number; + params?: Record; +} + +export interface ErrorBody { + code: ErrorCode; + message: string; + in_reply_to?: string; + retryable: boolean; + fatal: boolean; +} + +// ---- envelope ---- + +export interface Envelope { + type: T; + id: string; + ts: number; + body: B; +} + +export type HelloEnvelope = Envelope<"hello", HelloBody>; +export type ResumeEnvelope = Envelope<"resume", ResumeBody>; +export type SnapshotEnvelope = Envelope<"snapshot", SnapshotBody>; +export type DeltaEnvelope = Envelope<"delta", DeltaBody>; +export type AckEnvelope = Envelope<"ack", AckBody>; +export type TaskEventEnvelope = Envelope<"task.event", TaskEventBody>; +export type MessageEventEnvelope = Envelope<"message.event", MessageEventBody>; +export type MetricFrameEnvelope = Envelope<"metric.frame", MetricFrameBody>; +export type ConfigEventEnvelope = Envelope<"config.event", ConfigEventBody>; +export type SubscribeEnvelope = Envelope<"subscribe", SubscribeBody>; +export type UnsubscribeEnvelope = Envelope<"unsubscribe", UnsubscribeBody>; +export type InterestRenewEnvelope = Envelope<"interest.renew", InterestRenewBody>; +export type CommandEnvelope = Envelope<"command", CommandBody>; +export type ErrorEnvelope = Envelope<"error", ErrorBody>; + +export type AnyEnvelope = + | HelloEnvelope + | ResumeEnvelope + | SnapshotEnvelope + | DeltaEnvelope + | AckEnvelope + | TaskEventEnvelope + | MessageEventEnvelope + | MetricFrameEnvelope + | ConfigEventEnvelope + | SubscribeEnvelope + | UnsubscribeEnvelope + | InterestRenewEnvelope + | CommandEnvelope + | ErrorEnvelope; From a84477007f85e76adcc1fdeb40a067eeea5c1849 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:48:47 +0800 Subject: [PATCH 12/60] feat(server): route /v3/realtime upgrades Add the SPACE_HUB Durable Object binding (new_sqlite_classes migration v2, alongside UserStore's v1) and wire a /v3/realtime WebSocket upgrade route plus a minimal /v3/automations HTTP control-plane companion (upsert/pause/remove, mints a config.event via SpaceHub#mintConfigEvent) into the Workers adapter. The Worker does exactly three things per request: reuse the existing st2 token authenticate() middleware (extended to also cover /v3/*, no behavior change for /v2/*) to resolve role/space/deviceId, reject an invalid or unresolvable token with 401 before ever touching SpaceHub, and forward the upgrade to that space's SpaceHub stub exactly once with the resolved identity attached as trusted headers. /v2/* stays untouched as the existing fallback. --- server/src/adapters/workers.ts | 109 +++++++++++++++++++++++++++++++ server/src/app.ts | 5 ++ server/worker-configuration.d.ts | 5 +- server/wrangler.jsonc | 10 ++- 4 files changed, 125 insertions(+), 4 deletions(-) diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index 4aa8e39..06c476a 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -5,6 +5,13 @@ import { DurableObject } from "cloudflare:workers"; import { createApp, newToken, TOKEN_RE, type Command, type DeviceInfo, type Role, type SpaceRegistry } from "../app.ts"; import { endActivity, sendAlert, startActivity, updateActivity, type ApnsConfig } from "../apns.ts"; +import { SpaceHub } from "../realtime/space-hub.ts"; +import type { AutomationExecutorKind, AutomationState } from "../realtime/types.ts"; + +// Re-exported so wrangler (which binds Durable Object classes by name from +// this entry module, see wrangler.jsonc) can resolve SpaceHub, same as +// UserStore below. +export { SpaceHub }; import { appendTaskLog, EVENT_LOG_CAP, @@ -487,4 +494,106 @@ app.get("/debug/tokens", async (c: any) => { return c.json(await stub(env, "default").debugTokens(c.req.query("full") === "1")); }); +// ---- /v3: realtime WebSocket + its HTTP control-plane companion ---- +// +// Both routes ride the SAME `/v3/*` `authenticate` middleware registered in +// app.ts (st2 token parsing, unchanged from /v2) so an invalid or +// unresolvable token gets its 401 from that shared, already-tested path — +// this handler runs at all ONLY once a token has resolved to a real +// device/role, and it is the only place SpaceHub is ever touched. + +function spaceHubStub(env: WorkerEnv, space: string) { + return env.SPACE_HUB.getByName(space); +} + +app.get("/v3/realtime", async (c: any) => { + const env = c.env as WorkerEnv; + const raw = c.req.raw as Request; + if ((raw.headers.get("upgrade") ?? "").toLowerCase() !== "websocket") { + return c.text("expected websocket upgrade", 426); + } + // authenticate() already ran (see app.use("/v3/*", authenticate) in + // app.ts): an invalid/unresolvable token never reaches this line, and no + // Durable Object of any kind is touched for it beyond the UserStore + // lookup authenticate() itself performs — SpaceHub is instantiated only + // by the single stub.fetch() call below, on the success path. + const deviceId: string | undefined = c.get("deviceId"); + const role = c.get("role") as Role; + if (!deviceId) { + // The legacy bare-admin token (self-host, no registry) resolves to a + // role but no device identity; the realtime protocol requires one. + return c.json({ error: "realtime requires a paired device token" }, 401); + } + const realtimeRole: "source" | "viewer" = role === "source" ? "source" : "viewer"; + + const headers = new Headers(raw.headers); + headers.set("x-sitrep-device-id", deviceId); + headers.set("x-sitrep-role", realtimeRole); + const forwardReq = new Request(raw.url, { method: raw.method, headers }); + + // Exactly one forwarded operation per inbound upgrade request: the DO's + // own fetch() creates the WebSocketPair and calls ctx.acceptWebSocket + // once (see SpaceHub#fetch). + return spaceHubStub(env, c.get("space")).fetch(forwardReq); +}); + +function parseAutomationUpsert(body: any): { name: string; executor_kind: AutomationExecutorKind; every_seconds: number } | null { + const kind = body?.executor_kind; + const everySeconds = Number(body?.schedule?.every_seconds); + if (!body?.name || !["script", "agent", "hybrid"].includes(kind) || !Number.isFinite(everySeconds)) return null; + return { name: String(body.name).slice(0, 256), executor_kind: kind, every_seconds: Math.max(1, everySeconds | 0) }; +} + +app.post("/v3/automations", async (c: any) => { + const role = c.get("role") as Role; + if (!["admin", "owner"].includes(role)) return c.json({ error: "forbidden" }, 403); + const body = await c.req.json().catch(() => null); + const parsed = parseAutomationUpsert(body); + if (!parsed) return c.json({ error: "name, executor_kind and schedule.every_seconds required" }, 400); + const automation: AutomationState = { + automation_id: typeof body.automation_id === "string" && body.automation_id ? body.automation_id : crypto.randomUUID(), + name: parsed.name, + executor_kind: parsed.executor_kind, + schedule: { kind: "interval", every_seconds: parsed.every_seconds }, + state: body.state === "paused" ? "paused" : "active", + }; + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const result = await stub.mintConfigEvent(idempotencyKey, { + kind: "automation.upserted", + automation_id: automation.automation_id, + automation, + }); + return c.json(result); +}); + +app.patch("/v3/automations/:id", async (c: any) => { + const role = c.get("role") as Role; + if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const id = c.req.param("id"); + const existing = (await stub.automationsSnapshot()).find((a: AutomationState) => a.automation_id === id); + if (!existing) return c.json({ error: "not found" }, 404); + const body = await c.req.json().catch(() => null); + const next: AutomationState = { + ...existing, + ...(body?.state === "active" || body?.state === "paused" ? { state: body.state } : {}), + ...(body?.schedule?.every_seconds !== undefined + ? { schedule: { kind: "interval" as const, every_seconds: Math.max(1, Number(body.schedule.every_seconds) | 0) } } + : {}), + }; + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.upserted", automation_id: id, automation: next }); + return c.json(result); +}); + +app.delete("/v3/automations/:id", async (c: any) => { + const role = c.get("role") as Role; + if (!["admin", "owner"].includes(role)) return c.json({ error: "forbidden" }, 403); + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.removed", automation_id: c.req.param("id") }); + return c.json(result); +}); + export default app; diff --git a/server/src/app.ts b/server/src/app.ts index f786f98..0ee853b 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -160,6 +160,10 @@ export function createApp(opts: AppOptions) { // ---- authenticated: resolve space + role from the token ---- + // Shared by /v2/* (existing product API) and /v3/* (realtime + its HTTP + // control-plane companion): identical st2 token parsing and role/space/ + // deviceId resolution, so /v3 gets exactly the same "invalid token never + // reaches a handler" guarantee /v2 already has, with no duplicated logic. const authenticate = async (c: Context, next: Next) => { if (c.req.path === "/v2/spaces" || c.req.path === "/v2/join") return next(); const admin = opts.authToken?.(c); @@ -186,6 +190,7 @@ export function createApp(opts: AppOptions) { await next(); }; app.use("/v2/*", authenticate); + app.use("/v3/*", authenticate); const store = (c: Context) => opts.store(c, c.get("space")); const registry = (c: Context) => opts.registry?.(c, c.get("space")); diff --git a/server/worker-configuration.d.ts b/server/worker-configuration.d.ts index 19ed6da..b275505 100644 --- a/server/worker-configuration.d.ts +++ b/server/worker-configuration.d.ts @@ -1,16 +1,17 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 73e87c808e9c6c4997497b2a096a4dab) +// Generated by Wrangler by running `wrangler types` (hash: 4c0b4da54160bdee5b303d3a4232198a) // Runtime types generated with workerd@1.20260714.1 2026-07-18 nodejs_compat interface __BaseEnv_Env { INVITE_DIR: KVNamespace; APNS_BUNDLE_ID: "dev.sitrep.app"; APNS_HOST: "api.sandbox.push.apple.com"; USER_STORE: DurableObjectNamespace; + SPACE_HUB: DurableObjectNamespace; } declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/adapters/workers"); - durableNamespaces: "UserStore"; + durableNamespaces: "UserStore" | "SpaceHub"; } interface Env extends __BaseEnv_Env {} } diff --git a/server/wrangler.jsonc b/server/wrangler.jsonc index 6d9a5fb..e6884ee 100644 --- a/server/wrangler.jsonc +++ b/server/wrangler.jsonc @@ -12,7 +12,13 @@ "APNS_HOST": "api.sandbox.push.apple.com" }, "durable_objects": { - "bindings": [{ "name": "USER_STORE", "class_name": "UserStore" }] + "bindings": [ + { "name": "USER_STORE", "class_name": "UserStore" }, + { "name": "SPACE_HUB", "class_name": "SpaceHub" } + ] }, - "migrations": [{ "tag": "v1", "new_sqlite_classes": ["UserStore"] }] + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["UserStore"] }, + { "tag": "v2", "new_sqlite_classes": ["SpaceHub"] } + ] } From a1788fa3c33b6d87dc45f85a7d3e9a5e2b924d86 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:49:01 +0800 Subject: [PATCH 13/60] test(server): cover realtime recovery and deduplication Add the second test runway: @cloudflare/vitest-pool-workers running inside local workerd (test:workers), alongside the existing node --test suite (test:unit); `npm test` runs both. Lockfile updated and committed. Coverage: fixture round-trip decode/encode against every proto/realtime/fixtures/valid and invalid fixture (including the role-wrapped authorization-only ones) and pure chunking-boundary tests under test:unit; connection identity recovery via fresh connections/stub references, duplicate device_seq dedup (no double apply, no double revision, still acks), the full resume decision table (snapshot/current-empty-delta/revision_unavailable/retention- miss-falls-back-to-snapshot), push_outbox idempotency, metric.frame's zero-SQL-write guarantee and multi-viewer broadcast, chunked-snapshot ordering under a racing live event, interest-lease throttle/ resume_rate edges, connection gating (hello_required, resume-before- subscribe malformed, no live delta before a connection's own resume reply), connection supersession, and the Worker-layer invalid-token/ reconnect-storm guard that never instantiates a SpaceHub under test:workers. --- server/package-lock.json | 1514 +++++++++++++++++-- server/package.json | 6 +- server/test/realtime/chunking.test.ts | 56 + server/test/realtime/fixtures.test.ts | 72 + server/test/workers/broadcast.workers.ts | 153 ++ server/test/workers/config-event.workers.ts | 63 + server/test/workers/env.d.ts | 7 + server/test/workers/handshake.workers.ts | 122 ++ server/test/workers/helpers.ts | 213 +++ server/test/workers/lease.workers.ts | 68 + server/test/workers/reliability.workers.ts | 205 +++ server/test/workers/token-gate.workers.ts | 48 + server/tsconfig.json | 3 +- server/vitest.config.ts | 13 + 14 files changed, 2373 insertions(+), 170 deletions(-) create mode 100644 server/test/realtime/chunking.test.ts create mode 100644 server/test/realtime/fixtures.test.ts create mode 100644 server/test/workers/broadcast.workers.ts create mode 100644 server/test/workers/config-event.workers.ts create mode 100644 server/test/workers/env.d.ts create mode 100644 server/test/workers/handshake.workers.ts create mode 100644 server/test/workers/helpers.ts create mode 100644 server/test/workers/lease.workers.ts create mode 100644 server/test/workers/reliability.workers.ts create mode 100644 server/test/workers/token-gate.workers.ts create mode 100644 server/vitest.config.ts diff --git a/server/package-lock.json b/server/package-lock.json index 84e9e1d..b56a257 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -11,9 +11,11 @@ "hono": "^4.12.30" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.18.6", "@hono/node-server": "^1.19.0", "@types/node": "^26.1.1", "typescript": "^5.9.3", + "vitest": "^4.1.0", "wrangler": "^4.112.0" } }, @@ -43,6 +45,25 @@ } } }, + "node_modules/@cloudflare/vitest-pool-workers": { + "version": "0.18.6", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-pool-workers/-/vitest-pool-workers-0.18.6.tgz", + "integrity": "sha512-6JGqaQsQRZIVq/6jEC4ouJnShZriPIJ2X0yGndwMm+SiPP93pJi5Dp30dYAoztNrNJC7wWK7ec5slLfMBMZ8jA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cjs-module-lexer": "1.2.3", + "esbuild": "0.28.1", + "miniflare": "4.20260714.0", + "wrangler": "4.112.0", + "zod": "3.25.76" + }, + "peerDependencies": { + "@vitest/runner": "^4.1.0", + "@vitest/snapshot": "^4.1.0", + "vitest": "^4.1.0" + } + }, "node_modules/@cloudflare/workerd-darwin-64": { "version": "1.20260714.1", "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260714.1.tgz", @@ -141,10 +162,10 @@ "node": ">=12" } }, - "node_modules/@emnapi/runtime": { - "version": "1.11.2", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", - "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -705,9 +726,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -725,9 +743,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -745,9 +760,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -765,9 +777,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -785,9 +794,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -805,9 +811,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -825,9 +828,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -845,9 +845,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -865,9 +862,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -891,9 +885,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -917,9 +908,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -943,9 +931,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -969,9 +954,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -995,9 +977,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1021,9 +1000,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1047,9 +1023,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1173,6 +1146,35 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -1202,177 +1204,995 @@ "dev": true, "license": "MIT" }, - "node_modules/@sindresorhus/is": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", - "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "android" + ], "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@speed-highlight/core": { - "version": "1.2.17", - "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.17.tgz", - "integrity": "sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "CC0-1.0" + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@types/node": { - "version": "26.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", - "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "cpu": [ + "x64" + ], "dev": true, "license": "MIT", - "dependencies": { - "undici-types": "~8.3.0" + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "cpu": [ + "arm" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "Apache-2.0", + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=8" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/error-stack-parser-es": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", - "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/antfu" + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/esbuild": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", - "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "cpu": [ + "ppc64" + ], "dev": true, - "hasInstallScript": true, "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.1", - "@esbuild/android-arm": "0.28.1", - "@esbuild/android-arm64": "0.28.1", - "@esbuild/android-x64": "0.28.1", - "@esbuild/darwin-arm64": "0.28.1", - "@esbuild/darwin-x64": "0.28.1", - "@esbuild/freebsd-arm64": "0.28.1", - "@esbuild/freebsd-x64": "0.28.1", - "@esbuild/linux-arm": "0.28.1", - "@esbuild/linux-arm64": "0.28.1", - "@esbuild/linux-ia32": "0.28.1", - "@esbuild/linux-loong64": "0.28.1", - "@esbuild/linux-mips64el": "0.28.1", - "@esbuild/linux-ppc64": "0.28.1", - "@esbuild/linux-riscv64": "0.28.1", - "@esbuild/linux-s390x": "0.28.1", - "@esbuild/linux-x64": "0.28.1", - "@esbuild/netbsd-arm64": "0.28.1", - "@esbuild/netbsd-x64": "0.28.1", - "@esbuild/openbsd-arm64": "0.28.1", - "@esbuild/openbsd-x64": "0.28.1", - "@esbuild/openharmony-arm64": "0.28.1", - "@esbuild/sunos-x64": "0.28.1", - "@esbuild/win32-arm64": "0.28.1", - "@esbuild/win32-ia32": "0.28.1", - "@esbuild/win32-x64": "0.28.1" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "cpu": [ + "s390x" + ], "dev": true, - "hasInstallScript": true, "license": "MIT", "optional": true, "os": [ - "darwin" + "linux" ], "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/hono": { - "version": "4.12.30", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", - "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "cpu": [ + "x64" + ], + "dev": true, "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=16.9.0" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "cpu": [ + "x64" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=6" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/miniflare": { - "version": "4.20260714.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260714.0.tgz", - "integrity": "sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", + "cpu": [ + "wasm32" + ], "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "0.34.5", - "undici": "7.28.0", - "workerd": "1.20260714.1", - "ws": "8.21.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" }, "engines": { - "node": ">=22.0.0" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.17", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.17.tgz", + "integrity": "sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", + "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", + "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", + "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.10", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", + "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", + "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vitest/utils": "4.1.10", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", + "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "@vitest/utils": "4.1.10", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", + "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", + "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz", + "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/hono": { + "version": "4.12.30", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", + "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/miniflare": { + "version": "4.20260714.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260714.0.tgz", + "integrity": "sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.34.5", + "undici": "7.28.0", + "workerd": "1.20260714.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", "dev": true, "license": "MIT" @@ -1384,6 +2204,90 @@ "dev": true, "license": "MIT" }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", + "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.139.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" + } + }, "node_modules/semver": { "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", @@ -1442,6 +2346,37 @@ "@img/sharp-win32-x64": "0.34.5" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, "node_modules/supports-color": { "version": "10.2.2", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", @@ -1455,6 +2390,50 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1500,10 +2479,198 @@ "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "pathe": "^2.0.3" } }, + "node_modules/vite": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", + "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", + "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vitest/expect": "4.1.10", + "@vitest/mocker": "4.1.10", + "@vitest/pretty-format": "4.1.10", + "@vitest/runner": "4.1.10", + "@vitest/snapshot": "4.1.10", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.10", + "@vitest/browser-preview": "4.1.10", + "@vitest/browser-webdriverio": "4.1.10", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/ui": "4.1.10", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/workerd": { "version": "1.20260714.1", "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260714.1.tgz", @@ -1511,6 +2678,7 @@ "dev": true, "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "bin": { "workerd": "bin/workerd" }, @@ -1607,6 +2775,16 @@ "@poppinss/exception": "^1.2.2", "error-stack-parser-es": "^1.0.5" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/server/package.json b/server/package.json index 5e242fe..2dcd3c2 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,9 @@ "scripts": { "dev": "wrangler dev", "dev:node": "node --experimental-strip-types src/adapters/node.ts", - "test": "node --experimental-strip-types --test test/**/*.test.ts", + "test:unit": "node --experimental-strip-types --test \"test/**/*.test.ts\"", + "test:workers": "vitest run", + "test": "npm run test:unit && npm run test:workers", "typecheck": "tsc --noEmit", "deploy": "wrangler deploy" }, @@ -14,9 +16,11 @@ "hono": "^4.12.30" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.18.6", "@hono/node-server": "^1.19.0", "@types/node": "^26.1.1", "typescript": "^5.9.3", + "vitest": "^4.1.0", "wrangler": "^4.112.0" } } diff --git a/server/test/realtime/chunking.test.ts b/server/test/realtime/chunking.test.ts new file mode 100644 index 0000000..be43afc --- /dev/null +++ b/server/test/realtime/chunking.test.ts @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { chunkDeltaEvents, chunkSnapshot } from "../../src/realtime/chunking.ts"; +import type { DeltaEventItem, TaskState } from "../../src/realtime/types.ts"; + +test("chunkSnapshot: empty space still yields exactly one final chunk", () => { + const chunks = chunkSnapshot(0, { tasks: [], metrics: [], messages: [], automations: [] }); + assert.equal(chunks.length, 1); + assert.deepEqual(chunks[0], { revision: 0, part: 1, final: true, tasks: [], metrics: [], messages: [], automations: [] }); +}); + +test("chunkSnapshot: splits large task list across multiple parts, same revision, sequential parts, final on last", () => { + const tasks: TaskState[] = Array.from({ length: 4000 }, (_, i) => ({ + task_id: `task-${i}`, + device_id: "mac-01", + state: "running", + updated_at: 1_700_000_000_000 + i, + title: "x".repeat(50), + })); + const chunks = chunkSnapshot(42, { tasks, metrics: [], messages: [], automations: [] }); + assert.ok(chunks.length > 1, "expected more than one chunk for a large task list"); + for (const [i, chunk] of chunks.entries()) { + assert.equal(chunk.revision, 42); + assert.equal(chunk.part, i + 1); + assert.equal(chunk.final, i === chunks.length - 1); + } + const total = chunks.flatMap((c) => c.tasks); + assert.deepEqual(total, tasks, "concatenating chunks must reproduce the full task list, in order"); +}); + +test("chunkDeltaEvents: chains from/to boundaries across chunks", () => { + const events: DeltaEventItem[] = Array.from({ length: 3000 }, (_, i) => ({ + event_type: "task.event", + event: { + device_id: "mac-01", + device_seq: i + 1, + task_id: `run-${i}`, + kind: "progress", + occurred_at: 1_700_000_000_000 + i, + percent: i % 100, + }, + })); + const deltas = chunkDeltaEvents(100, events); + assert.ok(deltas.length > 1); + assert.equal(deltas[0].from_revision, 100); + for (let i = 1; i < deltas.length; i++) { + assert.equal(deltas[i].from_revision, deltas[i - 1].to_revision, "chained deltas must connect exactly"); + } + assert.equal(deltas.at(-1)!.to_revision, 100 + events.length); + for (const d of deltas) assert.equal(d.to_revision - d.from_revision, d.events.length); +}); + +test("chunkDeltaEvents: empty range still yields one delta with matching from/to", () => { + const deltas = chunkDeltaEvents(5, []); + assert.deepEqual(deltas, [{ from_revision: 5, to_revision: 5, events: [] }]); +}); diff --git a/server/test/realtime/fixtures.test.ts b/server/test/realtime/fixtures.test.ts new file mode 100644 index 0000000..35439f8 --- /dev/null +++ b/server/test/realtime/fixtures.test.ts @@ -0,0 +1,72 @@ +// Fixture round-trip test (required coverage item #11): every fixture under +// proto/realtime/fixtures/valid/ and fixtures/scenarios/**/ must decode, +// re-encode, and decode again to the same value; every fixture under +// fixtures/invalid/ must be rejected by our hand-written guards — either by +// parseEnvelope (schema-shape rejection) or by authorizeClientEnvelope (the +// two role-wrapped fixtures that are only invalid given a sender role). +// +// This is a pure-logic test (no Workers runtime needed), so it runs under +// the plain Node test runner alongside the rest of test:unit. +import assert from "node:assert/strict"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import { authorizeClientEnvelope, encodeEnvelope, parseEnvelope } from "../../src/realtime/guards.ts"; +import type { AnyEnvelope, DeviceRole } from "../../src/realtime/types.ts"; + +const FIXTURES_ROOT = join(import.meta.dirname, "..", "..", "..", "proto", "realtime", "fixtures"); + +function listJsonFilesRecursive(dir: string): string[] { + const out: string[] = []; + for (const name of readdirSync(dir)) { + const full = join(dir, name); + if (statSync(full).isDirectory()) out.push(...listJsonFilesRecursive(full)); + else if (name.endsWith(".json")) out.push(full); + } + return out; +} + +const validFiles = [ + ...listJsonFilesRecursive(join(FIXTURES_ROOT, "valid")), + ...listJsonFilesRecursive(join(FIXTURES_ROOT, "scenarios")), +]; +const invalidFiles = listJsonFilesRecursive(join(FIXTURES_ROOT, "invalid")); + +test(`found valid fixtures (${validFiles.length}) and invalid fixtures (${invalidFiles.length})`, () => { + assert.ok(validFiles.length >= 20, "expected at least the 21 valid+scenario fixtures present at freeze time"); + assert.ok(invalidFiles.length >= 14, "expected at least the 15 invalid fixtures present at freeze time"); +}); + +for (const file of validFiles) { + test(`valid fixture decodes and round-trips: ${file.slice(FIXTURES_ROOT.length + 1)}`, () => { + const raw = readFileSync(file, "utf8"); + const first = parseEnvelope(raw); + assert.equal(first.kind, "ok", `expected ok, got ${JSON.stringify(first)}`); + if (first.kind !== "ok") return; + const reEncoded = encodeEnvelope(first.envelope); + const second = parseEnvelope(reEncoded); + assert.equal(second.kind, "ok"); + if (second.kind !== "ok") return; + assert.deepEqual(second.envelope, first.envelope, "round-trip must be lossless"); + }); +} + +for (const file of invalidFiles) { + test(`invalid fixture is rejected: ${file.slice(FIXTURES_ROOT.length + 1)}`, () => { + const raw = JSON.parse(readFileSync(file, "utf8")) as unknown; + const wrapper = raw as { sender_role?: DeviceRole; frame?: AnyEnvelope }; + if (wrapper.sender_role && wrapper.frame) { + // Role-dependent fixture (proto/realtime/SPEC.md section 16): valid + // shape, but only invalid when sent by the given role/authorization + // rule. Body should parse fine; authorization must reject it. + const parsed = parseEnvelope(JSON.stringify(wrapper.frame)); + assert.equal(parsed.kind, "ok", "wrapped fixture body should be schema-valid on its own"); + if (parsed.kind !== "ok") return; + const authz = authorizeClientEnvelope(wrapper.sender_role, parsed.envelope); + assert.equal(authz.ok, false, "expected authorization to reject this role/frame combination"); + } else { + const parsed = parseEnvelope(JSON.stringify(raw)); + assert.notEqual(parsed.kind, "ok", "expected this fixture to be rejected"); + } + }); +} diff --git a/server/test/workers/broadcast.workers.ts b/server/test/workers/broadcast.workers.ts new file mode 100644 index 0000000..4339273 --- /dev/null +++ b/server/test/workers/broadcast.workers.ts @@ -0,0 +1,153 @@ +// Required coverage #4 (metric.frame never writes SQLite / never bumps +// revision), #5 (one metric.frame reaches multiple viewers), #10 (chunked +// snapshot: same revision, sequential parts, final closes it, nothing else +// interleaves). +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; + +function tableCounts(state: any) { + const tables = ["event_log", "dedup", "tasks", "messages", "automations", "leases", "push_outbox", "space_meta"]; + const counts: Record = {}; + for (const t of tables) { + counts[t] = state.storage.sql.exec<{ n: number }>(`SELECT COUNT(*) as n FROM ${t}`).toArray()[0].n; + } + return counts; +} + +describe("metric.frame", () => { + it("never writes SQLite and never advances space_revision", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient, ["metric"]); + await resume(viewerClient, 0); + + const stub = env.SPACE_HUB.getByName(spaceId); + const before = await runInDurableObject(stub, async (_i: SpaceHub, state) => tableCounts(state)); + + sourceClient.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics: [{ metric_id: "cpu.load", value: "42", ts: Date.now() }] }, + }); + const frame = await viewerClient.recv(); + expect(frame.type).toBe("metric.frame"); + expect(frame.body.metrics[0].metric_id).toBe("cpu.load"); + + const after = await runInDurableObject(stub, async (_i: SpaceHub, state) => tableCounts(state)); + expect(after).toEqual(before); + + sourceClient.close(); + viewerClient.close(); + }); + + it("broadcasts one frame to every eligible viewer", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + const viewer2 = await inviteAndJoin("viewer"); + + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const v1 = await connect(viewer.token); + await helloOffer(v1, viewer.device_id, "viewer"); + await subscribe(v1, ["metric"]); + await resume(v1, 0); + + const v2 = await connect(viewer2.token); + await helloOffer(v2, viewer2.device_id, "viewer"); + await subscribe(v2, ["metric"]); + await resume(v2, 0); + + sourceClient.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics: [{ metric_id: "mem.used", value: "1024", ts: Date.now() }] }, + }); + + const [f1, f2] = await Promise.all([v1.recv(), v2.recv()]); + expect(f1.body.metrics[0].value).toBe("1024"); + expect(f2.body.metrics[0].value).toBe("1024"); + + sourceClient.close(); + v1.close(); + v2.close(); + }); +}); + +describe("chunked snapshot", () => { + it("splits a large snapshot into parts sharing one revision, sequential, final last, with nothing else interleaved", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + + // Seed enough state directly (fast + precise) to force multiple chunks + // well under the 64 KiB frame limit's soft chunk threshold. + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_i: SpaceHub, state) => { + const bigTitle = "x".repeat(2000); + for (let i = 0; i < 60; i++) { + state.storage.sql.exec( + `INSERT INTO tasks (task_id, device_id, title, state, percent, step, message, updated_at, display) + VALUES (?, ?, ?, 'running', 50, NULL, NULL, ?, NULL)`, + `run-${i}`, + source.device_id, + bigTitle, + Date.now(), + ); + } + state.storage.sql.exec( + "INSERT INTO space_meta (key, value) VALUES ('revision', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ); + // A single reliable event so a live delta exists to race against the + // chunk stream below. + state.storage.sql.exec( + `INSERT INTO event_log (revision, event_type, device_id, device_seq, occurred_at, payload) VALUES (1, 'task.event', ?, 1, ?, ?)`, + source.device_id, + Date.now(), + JSON.stringify({ device_id: source.device_id, device_seq: 1, task_id: "run-0", kind: "started", occurred_at: Date.now() }), + ); + state.storage.sql.exec(`INSERT INTO dedup (device_id, device_seq, revision) VALUES (?, 1, 1)`, source.device_id); + }); + + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient); + + viewerClient.send({ type: "resume", id: nextId(), ts: Date.now(), body: { last_revision: 0 } }); + + // Race a live event against the in-flight chunked snapshot: if the + // server ever interleaved it between chunks, the assertions below + // would see a non-"snapshot" envelope before `final`. + sourceClient.send({ + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 2, task_id: "run-1", kind: "done", occurred_at: Date.now() }, + }); + + let final = false; + let partCount = 0; + let sawRevision: number | null = null; + while (!final) { + const msg = await viewerClient.recv(); + expect(msg.type).toBe("snapshot"); + partCount += 1; + expect(msg.body.part).toBe(partCount); + if (sawRevision === null) sawRevision = msg.body.revision; + expect(msg.body.revision).toBe(sawRevision); + final = msg.body.final; + } + expect(partCount).toBeGreaterThan(1); + + sourceClient.close(); + viewerClient.close(); + }); +}); diff --git a/server/test/workers/config-event.workers.ts b/server/test/workers/config-event.workers.ts new file mode 100644 index 0000000..b5871e3 --- /dev/null +++ b/server/test/workers/config-event.workers.ts @@ -0,0 +1,63 @@ +// The /v3 HTTP automation control-plane companion: mints a config.event in +// the same durable transaction as the revision bump, and a retried request +// carrying the same Idempotency-Key must not mint twice (SPEC.md section +// 5.5). Also checks the automation reaches a subscribed viewer as a live +// delta, same as any other reliable event. +import { SELF } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, connect, helloOffer, resume, subscribe } from "./helpers"; + +const ORIGIN = "https://example.com"; + +describe("/v3/automations control plane", () => { + it("upserting twice with the same Idempotency-Key mints exactly one config.event", async () => { + const { ownerToken } = await bootstrapSpace(); + const payload = { + automation_id: "auto-1", + name: "disk watch", + executor_kind: "script", + schedule: { every_seconds: 60 }, + }; + const headers = { + authorization: `Bearer ${ownerToken}`, + "content-type": "application/json", + "idempotency-key": "req-abc", + }; + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: JSON.stringify(payload) }); + const body1 = (await res1.json()) as { revision: number }; + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: JSON.stringify(payload) }); + const body2 = (await res2.json()) as { revision: number }; + expect(body2.revision).toBe(body1.revision); + }); + + it("an upsert reaches a subscribed viewer as a live delta carrying a config.event", async () => { + const { viewer, ownerToken } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + await resume(client, 0); + + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: "auto-2", name: "log rotate", executor_kind: "script", schedule: { every_seconds: 30 } }), + }); + expect(res.status).toBe(200); + + const delta = await client.recv(); + expect(delta.type).toBe("delta"); + expect(delta.body.events[0].event_type).toBe("config.event"); + expect(delta.body.events[0].event.automation_id).toBe("auto-2"); + client.close(); + }); + + it("rejects an upsert from a viewer-role token (only owner/admin may create)", async () => { + const { viewer } = await bootstrapSpace(); + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${viewer.token}`, "content-type": "application/json" }, + body: JSON.stringify({ name: "x", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(res.status).toBe(403); + }); +}); diff --git a/server/test/workers/env.d.ts b/server/test/workers/env.d.ts new file mode 100644 index 0000000..0439735 --- /dev/null +++ b/server/test/workers/env.d.ts @@ -0,0 +1,7 @@ +// Type-only ambient declaration for `cloudflare:test` in this directory. +// Not part of the root `tsc --noEmit` project (see tsconfig.json's +// "exclude") — vitest itself transpiles without type-checking, so this +// file exists purely for editor ergonomics when working on these tests. +declare module "cloudflare:test" { + interface ProvidedEnv extends Env {} +} diff --git a/server/test/workers/handshake.workers.ts b/server/test/workers/handshake.workers.ts new file mode 100644 index 0000000..1e7fc23 --- /dev/null +++ b/server/test/workers/handshake.workers.ts @@ -0,0 +1,122 @@ +// Required coverage #8 (connection gating), #9 (supersede). +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, connect, helloOffer, nextId, resume, subscribe } from "./helpers"; + +describe("connection gating", () => { + it("any frame before hello offer gets hello_required and the connection closes", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + // Send subscribe before ever sending hello{stage:offer}. + client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: {} }); + const err = await client.recv(); + expect(err.type).toBe("error"); + expect(err.body.code).toBe("hello_required"); + expect(err.body.fatal).toBe(true); + await client.waitForClose(); + }); + + it("resume before subscribe on the same connection is rejected as malformed, not silently accepted", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + client.send({ type: "resume", id: nextId(), ts: Date.now(), body: { last_revision: 0 } }); + const err = await client.recv(); + expect(err.type).toBe("error"); + expect(err.body.code).toBe("malformed"); + expect(err.body.retryable).toBe(true); + expect(err.body.fatal).toBe(false); + client.close(); + }); + + it("no live delta reaches a connection before its own resume reply", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient); + // subscribe is acked, but the connection is not yet delta-eligible: a + // reliable event applied right now must produce nothing on this socket + // until resume's reply arrives. + // subscribe just fired the space's 0->1 lease edge, which notifies every + // connected source with command{resume_rate} — drain it before relying + // on ordered recv() below. + await sourceClient.recv(); + + sourceClient.send({ + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 1, task_id: "run-1", kind: "started", occurred_at: Date.now() }, + }); + const ack = await sourceClient.recv(); + expect(ack.type).toBe("ack"); + + await viewerClient.expectSilence(200); + + const reply = await resume(viewerClient, 0); + expect(reply.type).toBe("snapshot"); + sourceClient.close(); + viewerClient.close(); + }); + + it("supersede: an older connection for the same device is closed with `superseded`, not `throttle`", async () => { + const { source, viewer } = await bootstrapSpace(); + + const first = await connect(source.token); + await helloOffer(first, source.device_id, "source"); + + // An unrelated viewer subscribes throughout — if supersession ever + // spuriously toggled the lease count, this connection would see a + // stray throttle/resume_rate command it never asked for. + const bystander = await connect(viewer.token); + await helloOffer(bystander, viewer.device_id, "viewer"); + await subscribe(bystander); + // That subscribe just fired the space's 0->1 lease edge, notifying + // every connected source (including `first`) with resume_rate — drain + // it before asserting on `first`'s next message. + const resumeRate = await first.recv(); + expect(resumeRate.body.action).toBe("resume_rate"); + + const second = await connect(source.token); + await helloOffer(second, source.device_id, "source"); + + const supersededErr = await first.recv(); + expect(supersededErr.type).toBe("error"); + expect(supersededErr.body.code).toBe("superseded"); + expect(supersededErr.body.fatal).toBe(true); + await first.waitForClose(); + + // The new connection is fully functional. + second.send({ + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 1, task_id: "run-1", kind: "started", occurred_at: Date.now() }, + }); + const ack = await second.recv(); + expect(ack.type).toBe("ack"); + expect(ack.body.acked).toEqual([{ device_id: source.device_id, device_seq: 1 }]); + + await bystander.expectSilence(200); + + second.close(); + bystander.close(); + }); + + it("version_unsupported when the offer shares no protocol version with the server", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + client.send({ + type: "hello", + id: nextId(), + ts: Date.now(), + body: { stage: "offer", device_id: viewer.device_id, role: "viewer", protocol_versions: [99] }, + }); + const err = await client.recv(); + expect(err.body.code).toBe("version_unsupported"); + expect(err.body.fatal).toBe(true); + await client.waitForClose(); + }); +}); diff --git a/server/test/workers/helpers.ts b/server/test/workers/helpers.ts new file mode 100644 index 0000000..8bbb6f8 --- /dev/null +++ b/server/test/workers/helpers.ts @@ -0,0 +1,213 @@ +// Shared helpers for the vitest-pool-workers realtime test suite. Not a +// test file itself (doesn't match vitest.config.ts's `*.workers.ts` +// include... actually it does match the extension but has no test()/ +// describe() calls, so it just runs as an empty, harmless module if vitest +// ever loads it directly). +import { SELF } from "cloudflare:test"; + +const ORIGIN = "https://example.com"; + +export interface JoinedDevice { + token: string; + device_id: string; + role: string; +} + +export interface Bootstrapped { + spaceId: string; + ownerToken: string; + source: JoinedDevice; + viewer: JoinedDevice; + inviteAndJoin: (role: "source" | "viewer") => Promise; +} + +/** Creates a fresh space (via the existing /v2 pairing flow) with one + * source device and one viewer device already joined, plus a helper to + * mint more devices of either role. */ +export async function bootstrapSpace(): Promise { + const createRes = await SELF.fetch(`${ORIGIN}/v2/spaces`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ platform: "test", name: "owner-mac" }), + }); + if (createRes.status !== 200) throw new Error(`space creation failed: ${createRes.status} ${await createRes.text()}`); + const { space_id, owner_token } = (await createRes.json()) as { space_id: string; owner_token: string }; + + const inviteAndJoin = async (role: "source" | "viewer"): Promise => { + const inviteRes = await SELF.fetch(`${ORIGIN}/v2/invites`, { + method: "POST", + headers: { authorization: `Bearer ${owner_token}`, "content-type": "application/json" }, + body: JSON.stringify({ role }), + }); + if (inviteRes.status !== 200) throw new Error(`invite failed: ${inviteRes.status} ${await inviteRes.text()}`); + const { code } = (await inviteRes.json()) as { code: string }; + const joinRes = await SELF.fetch(`${ORIGIN}/v2/join`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ code, space: space_id, name: `${role}-device`, platform: "test" }), + }); + if (joinRes.status !== 200) throw new Error(`join failed: ${joinRes.status} ${await joinRes.text()}`); + return (await joinRes.json()) as JoinedDevice; + }; + + const source = await inviteAndJoin("source"); + const viewer = await inviteAndJoin("viewer"); + return { spaceId: space_id, ownerToken: owner_token, source, viewer, inviteAndJoin }; +} + +/** Opens a /v3/realtime WebSocket authenticated with the given token. + * Returns the raw upgrade Response so callers can assert on failed + * upgrades (e.g. invalid token -> 401, no `webSocket`). */ +export async function upgrade(token: string | null): Promise { + const headers = new Headers({ upgrade: "websocket" }); + if (token !== null) headers.set("authorization", `Bearer ${token}`); + return SELF.fetch(`${ORIGIN}/v3/realtime`, { headers }); +} + +/** Thin promise-based wrapper around a Workers-runtime client WebSocket: + * queues incoming text frames so `recv()` can await the next one in order, + * exactly as a real client would process them. */ +export class WsClient { + readonly ws: WebSocket; + private queue: string[] = []; + private waiters: Array<(v: string) => void> = []; + private closed: { code: number; reason: string } | null = null; + private closeWaiters: Array<(v: { code: number; reason: string }) => void> = []; + + constructor(ws: WebSocket) { + this.ws = ws; + ws.accept(); + ws.addEventListener("message", (evt: MessageEvent) => { + const data = typeof evt.data === "string" ? evt.data : new TextDecoder().decode(evt.data as ArrayBuffer); + const waiter = this.waiters.shift(); + if (waiter) waiter(data); + else this.queue.push(data); + }); + ws.addEventListener("close", (evt: CloseEvent) => { + this.closed = { code: evt.code, reason: evt.reason }; + for (const w of this.closeWaiters.splice(0)) w(this.closed); + }); + } + + send(envelope: unknown): void { + this.ws.send(JSON.stringify(envelope)); + } + + sendRaw(text: string): void { + this.ws.send(text); + } + + recvRaw(timeoutMs = 3000): Promise { + const queued = this.queue.shift(); + if (queued !== undefined) return Promise.resolve(queued); + return new Promise((resolve, reject) => { + const waiter = (v: string) => { + clearTimeout(timer); + resolve(v); + }; + const timer = setTimeout(() => { + // Critical: drop this waiter on timeout, or a message that arrives + // later (e.g. right after an expectSilence() timeout) would be + // handed to this already-settled promise and silently vanish + // instead of reaching the next real recv()/expectSilence() call. + const idx = this.waiters.indexOf(waiter); + if (idx !== -1) this.waiters.splice(idx, 1); + reject(new Error(`timed out waiting for a message after ${timeoutMs}ms`)); + }, timeoutMs); + this.waiters.push(waiter); + }); + } + + async recv(timeoutMs = 3000): Promise { + return JSON.parse(await this.recvRaw(timeoutMs)); + } + + /** Resolves true if no message arrives within timeoutMs (used to assert + * silence — e.g. no live delta before a resume reply). */ + async expectSilence(timeoutMs = 200): Promise { + try { + const msg = await this.recvRaw(timeoutMs); + throw new Error(`expected silence but received: ${msg}`); + } catch (err) { + if (err instanceof Error && err.message.startsWith("timed out")) return true; + throw err; + } + } + + waitForClose(timeoutMs = 3000): Promise<{ code: number; reason: string }> { + if (this.closed) return Promise.resolve(this.closed); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("timed out waiting for close")), timeoutMs); + this.closeWaiters.push((v) => { + clearTimeout(timer); + resolve(v); + }); + }); + } + + close(): void { + try { + this.ws.close(); + } catch { + // already closed + } + } +} + +export async function connect(token: string): Promise { + const res = await upgrade(token); + if (res.status !== 101 || !res.webSocket) { + throw new Error(`expected a 101 upgrade, got ${res.status}: ${await res.text().catch(() => "")}`); + } + return new WsClient(res.webSocket); +} + +let idCounter = 0; +export function nextId(): string { + idCounter += 1; + return `t${idCounter}`; +} + +export async function helloOffer(client: WsClient, deviceId: string, role: "source" | "viewer"): Promise { + client.send({ + type: "hello", + id: nextId(), + ts: Date.now(), + body: { stage: "offer", device_id: deviceId, role, protocol_versions: [1] }, + }); + return client.recv(); +} + +export async function subscribe(client: WsClient, topics?: Array<"task" | "metric" | "message">): Promise { + client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: topics ? { topics } : {} }); + return client.recv(); +} + +export async function unsubscribe(client: WsClient): Promise { + client.send({ type: "unsubscribe", id: nextId(), ts: Date.now(), body: {} }); + return client.recv(); +} + +export async function resume(client: WsClient, lastRevision: number): Promise { + client.send({ type: "resume", id: nextId(), ts: Date.now(), body: { last_revision: lastRevision } }); + return client.recv(); +} + +export async function sendTaskEvent( + client: WsClient, + deviceId: string, + deviceSeq: number, + overrides: Partial<{ task_id: string; kind: string; occurred_at: number; percent: number; title: string }> = {}, +): Promise { + const body = { + device_id: deviceId, + device_seq: deviceSeq, + task_id: overrides.task_id ?? "run-1", + kind: overrides.kind ?? "started", + occurred_at: overrides.occurred_at ?? Date.now(), + ...(overrides.percent !== undefined ? { percent: overrides.percent } : {}), + ...(overrides.title !== undefined ? { title: overrides.title } : {}), + }; + client.send({ type: "task.event", id: nextId(), ts: Date.now(), body }); + return client.recv(); +} diff --git a/server/test/workers/lease.workers.ts b/server/test/workers/lease.workers.ts new file mode 100644 index 0000000..500c9ae --- /dev/null +++ b/server/test/workers/lease.workers.ts @@ -0,0 +1,68 @@ +// Required coverage #6: the space's active-lease count's 1<->0 edges +// notify every connected source with command{throttle}/command{resume_rate} +// (SPEC.md section 7). We drive the 1->0 edge via explicit `unsubscribe` +// rather than waiting out the real 30-60s lease TTL — SPEC.md section 7 +// explicitly names unsubscribe as one of exactly two ways a lease ends, +// and both paths run through the same reconcileLeaseEdge() call. +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, connect, helloOffer, subscribe, unsubscribe } from "./helpers"; + +describe("interest lease edges", () => { + it("0->1 sends resume_rate; 1->0 sends throttle; a later 0->1 sends resume_rate again", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const v1 = await connect(viewer.token); + await helloOffer(v1, viewer.device_id, "viewer"); + await subscribe(v1); // 0 -> 1 + + const resumeRate1 = await sourceClient.recv(); + expect(resumeRate1.type).toBe("command"); + expect(resumeRate1.body.origin).toBe("server"); + expect(resumeRate1.body.action).toBe("resume_rate"); + + await unsubscribe(v1); // 1 -> 0 (only lease in the space) + + const throttle = await sourceClient.recv(); + expect(throttle.type).toBe("command"); + expect(throttle.body.origin).toBe("server"); + expect(throttle.body.action).toBe("throttle"); + + const viewer2 = await inviteAndJoin("viewer"); + const v2 = await connect(viewer2.token); + await helloOffer(v2, viewer2.device_id, "viewer"); + await subscribe(v2); // 0 -> 1 again + + const resumeRate2 = await sourceClient.recv(); + expect(resumeRate2.type).toBe("command"); + expect(resumeRate2.body.action).toBe("resume_rate"); + + sourceClient.close(); + v1.close(); + v2.close(); + }); + + it("a second viewer subscribing while one is already active does not re-fire resume_rate", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const v1 = await connect(viewer.token); + await helloOffer(v1, viewer.device_id, "viewer"); + await subscribe(v1); // 0 -> 1 + await sourceClient.recv(); // resume_rate + + const viewer2 = await inviteAndJoin("viewer"); + const v2 = await connect(viewer2.token); + await helloOffer(v2, viewer2.device_id, "viewer"); + await subscribe(v2); // 1 -> 2, not an edge + + await sourceClient.expectSilence(200); + + sourceClient.close(); + v1.close(); + v2.close(); + }); +}); diff --git a/server/test/workers/reliability.workers.ts b/server/test/workers/reliability.workers.ts new file mode 100644 index 0000000..02f987f --- /dev/null +++ b/server/test/workers/reliability.workers.ts @@ -0,0 +1,205 @@ +// Required coverage #2 (dedup), #3 (revision gap/current/unavailable), +// #12 (push_outbox idempotency), and #1 (attachment survives +// reconstruction — see the "recovery" test below and its comment). +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; + +describe("reliable event dedup and revision accounting", () => { + it("a duplicate device_seq re-acks without applying twice or bumping revision again", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient); + await resume(viewerClient, 0); // now delta-eligible + // subscribe's 0->1 lease edge notified the source with resume_rate; + // drain it before relying on ordered recv() on sourceClient below. + await sourceClient.recv(); + + const envelope = { + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 1, task_id: "run-1", kind: "started", occurred_at: Date.now() }, + }; + sourceClient.send(envelope); + const ack1 = await sourceClient.recv(); + expect(ack1.body.acked).toEqual([{ device_id: source.device_id, device_seq: 1 }]); + const delta1 = await viewerClient.recv(); + expect(delta1.type).toBe("delta"); + expect(delta1.body.to_revision).toBe(1); + + // Resend the identical device_seq in a fresh envelope (new id/ts), as a + // real source would after a dropped ack (SPEC.md section 5.4). + sourceClient.send({ ...envelope, id: nextId(), ts: Date.now() }); + const ack2 = await sourceClient.recv(); + expect(ack2.body.acked).toEqual([{ device_id: source.device_id, device_seq: 1 }]); + + // No second delta was broadcast for the retry. + await viewerClient.expectSilence(200); + + // And the space's revision only advanced once. + const current = await resume(viewerClient, 1); + expect(current.type).toBe("delta"); + expect(current.body.from_revision).toBe(1); + expect(current.body.to_revision).toBe(1); + expect(current.body.events).toEqual([]); + + sourceClient.close(); + viewerClient.close(); + }); + + it("event_id in push_outbox is idempotent across a duplicate device_seq retry", async () => { + const { spaceId, source } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + await sendTaskEvent(sourceClient, source.device_id, 1); + await sendTaskEvent(sourceClient, source.device_id, 1); // duplicate + sourceClient.close(); + + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_instance: SpaceHub, state) => { + const outbox = state.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM push_outbox").toArray()[0]; + expect(outbox.n).toBe(1); + const events = state.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM event_log").toArray()[0]; + expect(events.n).toBe(1); + const dedup = state.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM dedup").toArray()[0]; + expect(dedup.n).toBe(1); + }); + }); +}); + +describe("resume decision table", () => { + it("last_revision 0 on a fresh space yields an (empty) snapshot", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 0); + expect(reply.type).toBe("snapshot"); + expect(reply.body.revision).toBe(0); + expect(reply.body.final).toBe(true); + expect(reply.body.tasks).toEqual([]); + client.close(); + }); + + it("last_revision == current revision yields the explicit empty delta", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + await sendTaskEvent(sourceClient, source.device_id, 1); + + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 1); + expect(reply.type).toBe("delta"); + expect(reply.body.from_revision).toBe(1); + expect(reply.body.to_revision).toBe(1); + expect(reply.body.events).toEqual([]); + sourceClient.close(); + client.close(); + }); + + it("last_revision > current revision yields revision_unavailable and no eligibility", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 999); + expect(reply.type).toBe("error"); + expect(reply.body.code).toBe("revision_unavailable"); + expect(reply.body.retryable).toBe(true); + client.close(); + }); + + it("0 < last_revision < current, fully retained, yields a catch-up delta (not a snapshot)", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + await sendTaskEvent(sourceClient, source.device_id, 1, { kind: "started" }); + await sendTaskEvent(sourceClient, source.device_id, 2, { kind: "progress", percent: 50 }); + await sendTaskEvent(sourceClient, source.device_id, 3, { kind: "done" }); + + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 1); + expect(reply.type).toBe("delta"); + expect(reply.body.from_revision).toBe(1); + expect(reply.body.to_revision).toBe(3); + expect(reply.body.events.length).toBe(2); + sourceClient.close(); + client.close(); + }); + + it("a range whose retention was lost falls back to a snapshot instead of an error", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloOffer(sourceClient, source.device_id, "source"); + await sendTaskEvent(sourceClient, source.device_id, 1); + await sendTaskEvent(sourceClient, source.device_id, 2); + await sendTaskEvent(sourceClient, source.device_id, 3); + sourceClient.close(); + + // Simulate the retention window having moved past revision 1 (the real + // window is 1000 revisions; forcing it directly keeps this test fast + // and precise rather than sending 1000+ real events). + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_instance: SpaceHub, state) => { + state.storage.sql.exec("DELETE FROM event_log WHERE revision <= 2"); + }); + + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 1); // wants (1,3], but revision 2 is gone + expect(reply.type).toBe("snapshot"); + expect(reply.body.revision).toBe(3); + client.close(); + }); +}); + +describe("connection identity recovery", () => { + // A true forced eviction/hibernation cycle isn't triggerable from the + // outside in vitest-pool-workers. What we can and do verify: (a) the + // DO's only source of connection identity is the per-connection + // WebSocket attachment (never an in-memory map keyed by connection — + // reviewed in space-hub.ts, every handler calls + // ws.deserializeAttachment() fresh), and (b) all durable facts (revision, + // dedup, folded task state) live in SQLite and are correctly rebuilt by a + // brand new connection/stub reference with no shared JS state, which is + // exactly what a reconstructed DO instance would see after eviction. + it("a fresh connection for the same device, and a fresh stub reference, both observe identical durable state", async () => { + const { spaceId, source } = await bootstrapSpace(); + const first = await connect(source.token); + await helloOffer(first, source.device_id, "source"); + await sendTaskEvent(first, source.device_id, 1, { kind: "started", title: "build" }); + first.close(); + + // A brand new WebSocket connection (fresh attachment, no memory of the + // previous one) still authenticates correctly and can continue the + // device's device_seq sequence — proving identity isn't cached + // anywhere but the attachment + storage. + const second = await connect(source.token); + const accept = await helloOffer(second, source.device_id, "source"); + expect(accept.body.stage).toBe("accept"); + await sendTaskEvent(second, source.device_id, 2, { kind: "done" }); + second.close(); + + // A fresh DurableObjectStub reference (not reusing any object from + // above) reads the exact same durable state straight from storage. + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_instance: SpaceHub, state) => { + const revision = state.storage.sql.exec<{ value: string }>("SELECT value FROM space_meta WHERE key='revision'").toArray()[0]; + expect(Number(revision.value)).toBe(2); + const task = state.storage.sql.exec<{ state: string; title: string }>("SELECT state, title FROM tasks WHERE task_id='run-1'").toArray()[0]; + expect(task.state).toBe("done"); + expect(task.title).toBe("build"); + }); + }); +}); diff --git a/server/test/workers/token-gate.workers.ts b/server/test/workers/token-gate.workers.ts new file mode 100644 index 0000000..cc0ca4d --- /dev/null +++ b/server/test/workers/token-gate.workers.ts @@ -0,0 +1,48 @@ +// Required coverage #7: an invalid token gets 401 at the Worker layer +// without ever instantiating a SpaceHub Durable Object, so a reconnect +// storm of bad/garbage tokens cannot spin up empty spaces. +import { env, listDurableObjectIds } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, upgrade } from "./helpers"; + +describe("token gate", () => { + it("rejects a malformed token with 401 and creates no SpaceHub", async () => { + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + const res = await upgrade("not-a-real-token"); + expect(res.status).toBe(401); + expect(res.webSocket).toBeNull(); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("rejects a missing Authorization header with 401 and creates no SpaceHub", async () => { + const res = await upgrade(null); + expect(res.status).toBe(401); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("rejects a well-formed but unresolvable st2 token with 401 and still creates no SpaceHub", async () => { + // Right shape (matches TOKEN_RE), but no device was ever issued this + // secret — resolveToken() legitimately touches UserStore (pre-existing + // behavior), but must never reach SpaceHub. + const res = await upgrade("st2_zzzzzzzzzz_" + "0".repeat(48)); + expect(res.status).toBe(401); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("a reconnect storm of invalid tokens across many spaces never creates a single SpaceHub", async () => { + const attempts = Array.from({ length: 25 }, (_, i) => upgrade(`st2_space${i}_` + "a".repeat(48))); + const results = await Promise.all(attempts); + for (const res of results) expect(res.status).toBe(401); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("a valid token succeeds and creates exactly one SpaceHub for that space", async () => { + const { source } = await bootstrapSpace(); + const res = await upgrade(source.token); + expect(res.status).toBe(101); + expect(res.webSocket).not.toBeNull(); + res.webSocket?.accept(); + res.webSocket?.close(); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(1); + }); +}); diff --git a/server/tsconfig.json b/server/tsconfig.json index b3c01ce..6bfe394 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -9,5 +9,6 @@ "types": ["node"], "allowImportingTsExtensions": true }, - "include": ["worker-configuration.d.ts", "src/**/*.ts", "test/**/*.ts"] + "include": ["worker-configuration.d.ts", "src/**/*.ts", "test/**/*.ts"], + "exclude": ["test/workers/**/*"] } diff --git a/server/vitest.config.ts b/server/vitest.config.ts new file mode 100644 index 0000000..2719d13 --- /dev/null +++ b/server/vitest.config.ts @@ -0,0 +1,13 @@ +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["test/workers/**/*.workers.ts"], + }, + plugins: [ + cloudflareTest({ + wrangler: { configPath: "./wrangler.jsonc" }, + }), + ], +}); From fdbae770254c72ced2d351b1593d2bda3ca3aaf0 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:49:33 +0800 Subject: [PATCH 14/60] perf(server): sample realtime observability Cap hot-path per-frame logging in SpaceHub to <=1% of messages (injectable sampler, so tests can force full logging), while every protocol_error and ws_error path continues to log unconditionally. Avoids a console.log per inbound frame on a busy connection. --- server/src/realtime/space-hub.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 9cbb56b..8b1ed60 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -77,6 +77,12 @@ export class SpaceHub extends DurableObject { * accounting) — deliberately in-memory only, never written to SQLite. */ private metricsCache = new Map(); private rateLimiters = new WeakMap(); + private hotPathCounter = 0; + + /** Injectable for tests: decides whether the Nth hot-path event gets + * logged. Default samples at <=1%. Errors always log regardless of this + * (see logAlways) — only routine per-frame hot-path logging is sampled. */ + logSampler: (n: number) => boolean = (n) => n % 100 === 0; constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); @@ -850,7 +856,10 @@ export class SpaceHub extends DurableObject { } private logHotPath(event: string, data: Record): void { - console.log(JSON.stringify({ level: "info", event, ...data, ts: Date.now() })); + this.hotPathCounter++; + if (this.logSampler(this.hotPathCounter)) { + console.log(JSON.stringify({ level: "info", event, ...data, ts: Date.now() })); + } } private logAlways(event: string, data: Record): void { From 44a78eaa9e6d116984950c4b4977ba8f6ef7b953 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 10:49:40 +0800 Subject: [PATCH 15/60] docs(server): document realtime cost model Add docs/design/realtime-server.md: a per-message-type table of SQL writes / revision++ / outbox / broadcast aligned with proto/realtime/SPEC.md section 14, a Durable Object billing-axis analysis (requests/duration/storage), and the specific conditions under which SpaceHub can hibernate (no timers/alarms anywhere; lazy lease-edge detection; attachment-only connection identity). --- docs/design/realtime-server.md | 131 +++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 docs/design/realtime-server.md diff --git a/docs/design/realtime-server.md b/docs/design/realtime-server.md new file mode 100644 index 0000000..1a2c248 --- /dev/null +++ b/docs/design/realtime-server.md @@ -0,0 +1,131 @@ +# Realtime server: SpaceHub cost model + +This document is the server-side cost/capacity companion to +`proto/realtime/SPEC.md` section 14. It maps every message type onto the +concrete SQLite writes, `space_revision` accounting, and broadcast fan-out +`server/src/realtime/space-hub.ts` performs, and then analyzes that from a +Durable Object billing perspective (requests, duration, storage). It also +records exactly when this Durable Object can hibernate. + +## Per-message-type cost table + +Columns mirror SPEC.md section 14 plus a `SQL writes` column naming the +actual tables touched, since "persisted" alone doesn't say how many +statements or which tables. + +| `type` | SQL writes | revision++ | outbox | broadcast | +|---|---|---|---|---| +| `hello` | none (attachment only, via `serializeAttachment`) | no | no | no | +| `resume` | none (reads `tasks`/`messages`/`automations`/`event_log`) | no | no | no (unicast reply, possibly chunked) | +| `snapshot` | none | no | no | no | +| `delta` | none (built from `event_log` rows, or from the just-applied event in memory) | no | no | yes, live single-event deltas only; catch-up deltas are unicast | +| `ack` | none | no | no | no | +| `task.event` | `event_log` insert, `dedup` insert, `tasks` upsert, `space_meta` update, `push_outbox` insert (only on the non-duplicate path) | yes (skipped on duplicate) | yes | re-emitted as `delta` to every delta-eligible, leased viewer | +| `message.event` | `event_log` insert, `dedup` insert, `messages` insert + bounded-window prune, `space_meta` update, `push_outbox` insert (non-duplicate path only) | yes (skipped on duplicate) | yes | re-emitted as `delta` | +| `metric.frame` | **none** — no table is ever written for this type | **no** | no | yes, filtered to leases whose `topics` include `metric` (or omit `topics`) | +| `config.event` | `event_log` insert, `automations` upsert/delete, `space_meta` update, `http_idempotency` insert (only when an `Idempotency-Key` was supplied) | yes | no | re-emitted as `delta` | +| `subscribe` | `leases` upsert | no | no | no (its ack is unicast; may trigger a `command` broadcast, see below) | +| `unsubscribe` | `leases` delete | no | no | no (same caveat) | +| `interest.renew` | `leases` upsert | no | no | no (same caveat) | +| `command` (viewer-issued) | `pending_commands` insert, only if no connected source could take live delivery | no | no (viewer command) / yes (targeted at one or all sources) | no (targeted, not fanned out to viewers) | +| `command` (server-issued: `throttle`/`resume_rate`) | none | no | no | yes, to every connected source, on the space's lease-count 1↔0 edge only | +| `error` | none | no | no | no | + +Two things worth calling out because they are easy to get wrong and are +covered by dedicated tests (`server/test/workers/broadcast.workers.ts`): + +- **`metric.frame` never touches SQLite and never advances `space_revision`.** + It is a pure in-memory relay (`SpaceHub#metricsCache`, a `Map`) plus a + websocket fan-out. This is the single biggest cost lever in the whole + design: a source can push metrics at up to 10 Hz per connection + (SPEC.md section 11) with zero storage cost, at the price of the + `snapshot.metrics` cache being best-effort and empty after eviction + (acceptable per spec, section 6.2). +- **Duplicate `device_seq` costs one read, not one write.** The dedup check + (`SELECT revision FROM dedup WHERE device_id = ? AND device_seq = ?`) is + the *only* SQL statement executed on the retry path; the ack the source + needs is still sent. This bounds the cost of a source's at-least-once + resend behavior (SPEC.md section 5.3) to a single indexed read per retry. + +## Durable Object billing perspective + +Cloudflare bills a Durable Object roughly on three axes: **requests** +(wall-clock-billed invocations — HTTP `fetch()` calls and, importantly, +each Hibernatable WebSocket message callback), **duration** (active CPU/wall +time per invocation), and **storage** (row/byte volume plus read/write unit +counts against the attached SQLite database). + +- **Requests.** Every inbound WebSocket frame is one `webSocketMessage` + invocation — this is unavoidable per-message overhead independent of the + handler's cost, which is why the Worker-side gate (rejecting invalid + tokens with a plain 401, `adapters/workers.ts`'s `/v3/realtime` route) + matters: an invalid token or a reconnect storm must never turn into + SpaceHub invocations at all, let alone SpaceHub *instances* (verified by + `server/test/workers/token-gate.workers.ts`). The one exception to "one + request per frame" is the literal `ping`/`pong` heartbeat text, which + `ctx.setWebSocketAutoResponse()` answers entirely inside the runtime + without invoking `webSocketMessage` at all (SPEC.md section 9.3) — this + is a genuine request-count win for a connection that's alive but idle. +- **Duration.** The dominant cost per request is proportional to the + number of `sql.exec` calls a handler issues, not to network I/O (there is + none inside a handler — `ws.send()` is fire-and-forget and never + `await`ed). `task.event`/`message.event` cost roughly 5–6 statements; + `metric.frame` costs zero; `resume` costs a handful of `SELECT`s plus, + for a multi-chunk snapshot, one `ws.send()` per chunk (still no `await`, + so duration scales with payload size, not round trips). Snapshot chunking + (`chunkSnapshot`/`chunkDeltaEvents` in `chunking.ts`) exists specifically + to keep any single `sql.exec`+`JSON.stringify`+`send` unit bounded, not to + reduce request count — one `resume` is still one request no matter how + many chunks it produces, since all of them are sent from inside the same + `webSocketMessage` invocation. +- **Storage.** Two tables grow without bound if left unchecked: + `event_log` (pruned to the last `EVENT_LOG_RETENTION_REVISIONS` = 1000 + revisions — older ones force a `resume` onto the `snapshot` path instead + of erroring, SPEC.md section 6.2) and `messages` (pruned to the last + `MESSAGE_WINDOW` = 200, SPEC.md section 6.4's normative truncation). + `dedup` is intentionally **not** pruned: SPEC.md section 5.2's + deduplication guarantee must hold regardless of how old a retried + `device_seq` is, so it trades unbounded (but tiny — one row per + historical event, `device_id`+`device_seq`+`revision`, all fixed-width) + storage growth for correctness. `push_outbox` and `pending_commands` are + the two "queue" tables in this design; both need a consumer (a future + Alarm/Queue-based push worker, explicitly out of scope for this phase) to + actually shrink — today they only grow, which is an accepted, called-out + gap (see the handoff's "known gaps" section). + +## Hibernation + +SpaceHub is written so that an idle connection costs nothing beyond the +in-memory footprint the runtime itself keeps for a parked WebSocket: + +- Every handler uses `ws.deserializeAttachment()` fresh, every time, + instead of any DO-instance-level `Map` for identity — + this is what makes it safe for the DO's JS instance to be evicted and + reconstructed between messages. `ConnAttachment` (`attachment.ts`) is + deliberately small (five scalars plus an optional `sessionId`) to stay + far under the platform's 2 KiB serialized-attachment cap. +- The only volatile, non-attachment, non-SQLite in-memory state is + `metricsCache` (a best-effort cache the spec explicitly allows to reset + to empty after a restart, section 6.2) and the per-connection rate + limiter state (`rateLimiters`, a `WeakMap`, similarly allowed to reset — + worst case a freshly-evicted-and-restarted connection gets one extra + `metric.frame` or two before the rate limit re-establishes itself). + Neither is required for correctness, only for cost control, so losing + them on eviction is safe by design, not just by accident. +- Nothing in this Durable Object ever calls `setInterval`, `setTimeout`, or + `ctx.storage.setAlarm()`. The interest lease (SPEC.md section 7) + explicitly permits lazy expiry evaluation, which `reconcileLeaseEdge()` + performs by re-checking `COUNT(*) FROM leases WHERE expires_at > ?` every + time a lease is mutated or a reliable event is applied — never on a + timer. This means a space with no active connections and no pending + automation changes has literally nothing scheduled and can hibernate (or + be evicted entirely) indefinitely. +- The heartbeat auto-response (`ctx.setWebSocketAutoResponse`) is itself a + hibernation-compatible primitive: it answers `ping` without waking the DO + instance at all, which is precisely why SPEC.md section 9.3 specifies a + bare text frame rather than a JSON envelope in the first place. + +Net effect: a space's ongoing cost while every connection is idle (no +task/message/metric traffic, no lease churn) is exactly zero DO +invocations — the state sits in SQLite and the connections sit hibernated +until either side has something to say. From 951b7e824351259dfa6307929744597fd1d2ce6e Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:01:17 +0800 Subject: [PATCH 16/60] fix(apple): harden realtime client fault handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address three adversarial-review findings in the SitrepKit client layer: - Terminal errors stop reconnecting (F2): a server error carrying fatal:true and retryable:false (superseded, unauthenticated, version_unsupported) now ends the run loop with the .failed phase preserved instead of feeding the backoff/reconnect cycle; a later start() — user action or the next foreground pass — begins a fresh cycle. Handshake- and subscribe-stage server errors route through handleError so their notices surface and their retryable/fatal semantics reach the loop. A run-generation token stops a stale loop from clobbering its successor's task handle. - Chunked-snapshot interleaving is malformed (F3): per SPEC.md 6.2, any non-snapshot envelope arriving while snapshot chunks are being reassembled (ping/pong never reach dispatch) now discards the buffered chunks and closes the connection for a fresh resume, in both the gate and the client dispatch path. - Malformed frames are contained, not fatal (F5): a frame that fails to decode is skipped per SPEC.md 13 (malformed is non-fatal) and only a run of 5 consecutive malformed frames trips a circuit breaker that reconnects; the DeltaEvent wrapper now ignores unknown sibling fields, matching the body-level tolerance of SPEC.md 15. Also adds Phase.allowsReliableStateOverwrite, the policy the app layer uses to keep HTTP snapshot results from clobbering live delta state (applied in the follow-up commit). --- .../SitrepKit/Realtime/RealtimeBodies.swift | 9 +- .../SitrepKit/Realtime/RealtimeClient.swift | 107 ++++++++++++++++-- .../Realtime/RealtimeResumeGate.swift | 21 ++++ 3 files changed, 125 insertions(+), 12 deletions(-) diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift index 07f6934..0ac6bb1 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift @@ -214,11 +214,10 @@ extension DeltaEvent: Codable { private enum CodingKeys: String, CodingKey { case eventType = "event_type", event } public init(from decoder: Decoder) throws { - let dyn = try decoder.container(keyedBy: AnyKey.self) - let present = Set(dyn.allKeys.map(\.stringValue)) - guard present == ["event_type", "event"] else { - throw RealtimeDecodingError.malformed("delta event must carry exactly event_type/event: \(present)") - } + // Only `event_type` and `event` are defined in v1; unknown sibling + // fields are ignored, matching SPEC.md §15's body-level tolerance + // (the top-level envelope strictness of §3 does not extend to + // objects nested inside `body`). let c = try decoder.container(keyedBy: CodingKeys.self) let type = try c.decode(String.self, forKey: .eventType) switch type { diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift index 72c8d18..797cd86 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift @@ -96,6 +96,18 @@ public actor RealtimeClient { private var consecutiveFailures = 0 private var firedFallback = false private var lastInboundAt: Int = 0 + /// §13: `malformed` is non-fatal — a single undecodable frame is + /// skipped, not a reason to drop the connection. This counts the + /// current run of consecutive malformed frames so a persistently + /// broken peer still trips a circuit breaker (see `receiveFrame`). + private var consecutiveMalformedFrames = 0 + /// Number of consecutive malformed frames after which the connection is + /// torn down and re-established (circuit breaker). + private let malformedFrameCircuitBreaker = 5 + /// Distinguishes which `runLoop` invocation may clear `runTask` on + /// exit, so a stale loop finishing after a stop()/start() cycle can't + /// clobber the newer loop's handle. + private var runGeneration = 0 private let phaseContinuation: AsyncStream.Continuation public nonisolated let phases: AsyncStream @@ -124,12 +136,17 @@ public actor RealtimeClient { // MARK: - Lifecycle /// Idempotent: does nothing if already running. Call when the app enters - /// the foreground. + /// the foreground. Also the restart path after a terminal stop (see + /// `runLoop`): once the loop has exited — including after a + /// fatal-and-not-retryable server error — a later `start()` begins a + /// fresh reconnect cycle. public func start() { guard runTask == nil else { return } consecutiveFailures = 0 firedFallback = false - runTask = Task { [weak self] in await self?.runLoop() } + runGeneration += 1 + let generation = runGeneration + runTask = Task { [weak self] in await self?.runLoop(generation: generation) } } /// Tears down the connection and stops reconnecting. The interest lease @@ -144,13 +161,32 @@ public actor RealtimeClient { phase = .idle } - private func runLoop() async { + /// True for a server error whose own `retryable`/`fatal` flags say + /// "connection closes AND retrying the same thing cannot succeed" — + /// `superseded`, `unauthenticated`, `version_unsupported` (§13). Per + /// ruling, these stop the run loop instead of feeding the reconnect + /// cycle: the notice has already been surfaced, and reconnection waits + /// for an explicit user action or the next foreground `start()`. + private func isTerminal(_ error: Error) -> Bool { + if case RealtimeClientError.serverError(let body) = error { + return body.fatal && !body.retryable + } + return false + } + + private func runLoop(generation: Int) async { + var terminal = false while !Task.isCancelled { do { try await connectOnce() } catch is CancellationError { break } catch { + if isTerminal(error) { + phase = .failed(String(describing: error)) + terminal = true + break + } consecutiveFailures += 1 phase = .failed(String(describing: error)) if consecutiveFailures >= fallbackAfterFailures && !firedFallback { @@ -163,7 +199,14 @@ public actor RealtimeClient { try? await Task.sleep(for: .seconds(backoffDelay(attempt: consecutiveFailures))) } } - phase = .idle + // Only the newest loop may release the handle (a stale loop exiting + // after stop()+start() must not clobber its successor's), and a + // terminal exit keeps the `.failed` phase visible instead of + // resetting to `.idle`. + if runGeneration == generation { + runTask = nil + if !terminal { phase = .idle } + } } private func backoffDelay(attempt: Int) -> Double { @@ -176,6 +219,7 @@ public actor RealtimeClient { private func connectOnce() async throws { phase = .connecting + consecutiveMalformedFrames = 0 var request = URLRequest(url: configuration.url) if let token = configuration.token, !token.isEmpty { request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") @@ -195,7 +239,12 @@ public actor RealtimeClient { on: socket) let ackFrame = try await receiveFrame(on: socket) guard case .ack(let ackEnvelope) = ackFrame, let lease = ackEnvelope.body.lease else { - if case .error(let e) = ackFrame { throw RealtimeClientError.serverError(e.body) } + if case .error(let e) = ackFrame { + // Route through handleError so the notice is surfaced and + // fatal errors (incl. terminal ones) throw with the server's + // own retryable/fatal semantics attached. + try await handleError(e.body, on: socket) + } throw RealtimeClientError.protocolViolation("expected subscribe ack with lease, got \(ackFrame)") } phase = .subscribed @@ -238,7 +287,11 @@ public actor RealtimeClient { } return accept case .error(let env): - throw RealtimeClientError.serverError(env.body) + // handleError surfaces the notice and, for fatal codes (e.g. + // version_unsupported), throws serverError so the run loop can + // apply the terminal/retryable distinction. + try await handleError(env.body, on: socket) + throw RealtimeClientError.protocolViolation("handshake answered with a non-fatal error; reconnecting") default: throw RealtimeClientError.protocolViolation("expected hello accept, got \(frame)") } @@ -247,6 +300,17 @@ public actor RealtimeClient { // MARK: - Frame dispatch (post-handshake, post-subscribe) private func dispatch(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask) async throws { + // §6.2: while a chunked snapshot is in flight, only further snapshot + // chunks (and ping/pong, which never reach dispatch) are legal on + // this connection. Anything else is a malformed sequence: drop the + // buffered chunks and reconnect; the fresh connection resumes anew. + if gate.isSnapshotInFlight { + guard case .snapshot = frame else { + _ = gate.interleavedFrameDuringSnapshot() + throw RealtimeClientError.protocolViolation( + "non-snapshot envelope interleaved during a chunked snapshot") + } + } switch frame { case .snapshot(let env): switch gate.receiveSnapshotChunk(env.body) { @@ -404,9 +468,23 @@ public actor RealtimeClient { continue } do { - return try RealtimeFrame.decode(data) + let frame = try RealtimeFrame.decode(data) + consecutiveMalformedFrames = 0 + return frame } catch RealtimeDecodingError.unknownType { continue // §15: ignore, do not treat as malformed + } catch { + // §13: `malformed` is non-fatal — skip this frame and keep + // the connection. The circuit breaker only trips on a run + // of consecutive malformed frames, which indicates the + // stream itself is broken rather than one bad envelope. + consecutiveMalformedFrames += 1 + if consecutiveMalformedFrames >= malformedFrameCircuitBreaker { + consecutiveMalformedFrames = 0 + throw RealtimeClientError.protocolViolation( + "\(malformedFrameCircuitBreaker) consecutive malformed frames; reconnecting (last: \(error))") + } + continue } } } @@ -427,3 +505,18 @@ public actor RealtimeClient { private func nowMs() -> Int { Int(Date().timeIntervalSince1970 * 1000) } private func newEnvelopeID() -> String { UUID().uuidString } } + +public extension RealtimeClient.Phase { + /// Whether an HTTP snapshot result may overwrite the four reliable + /// collections (tasks/metrics/messages-events/automations). While the + /// realtime connection is `.live`, deltas own that state and an HTTP + /// response — possibly computed seconds ago — must not clobber it; in + /// every other phase the HTTP path is the best available source. + /// + /// Callers MUST evaluate this AFTER their HTTP `await` returns, not + /// before issuing the request: the phase can flip to `.live` while the + /// request is in flight, and it is the state of the world at apply time + /// that decides. REST-only concepts the realtime protocol does not + /// carry (e.g. `presence`) are exempt and always apply. + var allowsReliableStateOverwrite: Bool { self != .live } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift index 0b0af14..5c407a0 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift @@ -44,6 +44,24 @@ public struct RealtimeResumeGate: Sendable, Equatable { case malformedSequence(String) } + /// True while a chunked snapshot is being reassembled (a non-final + /// chunk has arrived and the `final` one has not). Per SPEC.md §6.2 the + /// server MUST NOT interleave any other envelope in this window; the + /// caller must route any non-snapshot frame it receives during it to + /// `interleavedFrameDuringSnapshot()` instead of processing it. + public var isSnapshotInFlight: Bool { !pendingSnapshotChunks.isEmpty } + + /// SPEC.md §6.2: "a viewer that receives any non-`ping`/`pong` envelope + /// between chunks MUST treat it as a malformed sequence and MAY close + /// the connection and reconnect." Discards the buffered chunks (they + /// can never be completed by a conformant continuation now) and reports + /// the malformed sequence; the caller closes and reconnects, and the + /// next connection resumes afresh. + public mutating func interleavedFrameDuringSnapshot() -> Outcome { + pendingSnapshotChunks = [] + return .malformedSequence("non-snapshot envelope interleaved during a chunked snapshot") + } + /// Call once, immediately after sending `resume{last_revision: N}` — /// including the very first resume on a fresh connection and any /// gap-triggered re-resume. @@ -81,6 +99,9 @@ public struct RealtimeResumeGate: Sendable, Equatable { } public mutating func receiveDelta(_ body: DeltaBody) -> Outcome { + // §6.2: nothing but further snapshot chunks may arrive while a + // chunked snapshot is being reassembled. + if isSnapshotInFlight { return interleavedFrameDuringSnapshot() } if let requested = awaitingResumeReply { guard body.fromRevision == requested else { return .discarded } state.apply(delta: body) From b9e21748818f416ab7eddf2202dfed265f79b224 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:01:27 +0800 Subject: [PATCH 17/60] fix(daemon): validate metric_id grammar in metric samples MetricSample.Validate now enforces the metric_id pattern from common.schema.json#/$defs/metric_id (^[a-z0-9_.-]{1,64}$), which also makes an empty id invalid. Previously only the value/label caps and the timestamp bound were checked, so a malformed id could slip into an outbound metric.frame. Add a test covering valid ids and the empty/ uppercase/whitespace/non-ascii/over-length rejections. --- daemon/internal/realtime/wire/validate.go | 9 ++++++++ daemon/internal/realtime/wire/wire_test.go | 25 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/daemon/internal/realtime/wire/validate.go b/daemon/internal/realtime/wire/validate.go index 148038e..35d07d0 100644 --- a/daemon/internal/realtime/wire/validate.go +++ b/daemon/internal/realtime/wire/validate.go @@ -3,8 +3,14 @@ package wire import ( "encoding/json" "fmt" + "regexp" ) +// metricIDPattern mirrors common.schema.json#/$defs/metric_id: the id +// grammar from proto/SPEC.md's metric.update verb, non-empty by +// construction ({1,64}). +var metricIDPattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`) + // Field length caps from common.schema.json. const ( freeTextMax = 2048 // $defs/free_text @@ -217,6 +223,9 @@ func (t TaskState) Validate() error { } func (m MetricSample) Validate() error { + if !metricIDPattern.MatchString(m.MetricID) { + return fmt.Errorf("metric_sample: metric_id %q does not match %s", m.MetricID, metricIDPattern) + } if len(m.Value) > metricValueMax { return fmt.Errorf("metric_sample: value exceeds %d chars", metricValueMax) } diff --git a/daemon/internal/realtime/wire/wire_test.go b/daemon/internal/realtime/wire/wire_test.go index 0f4b0b1..b4a9560 100644 --- a/daemon/internal/realtime/wire/wire_test.go +++ b/daemon/internal/realtime/wire/wire_test.go @@ -200,3 +200,28 @@ func TestUnixMSBoundary(t *testing.T) { } } } + +// TestMetricIDValidation pins the metric_id grammar from +// common.schema.json#/$defs/metric_id (^[a-z0-9_.-]{1,64}$). +func TestMetricIDValidation(t *testing.T) { + valid := []string{"cpu.load", "mem.used_gb", "a", "x_1-2.3"} + for _, id := range valid { + s := MetricSample{MetricID: id, Value: "1", TS: MinUnixMS + 1} + if err := s.Validate(); err != nil { + t.Errorf("Validate(%q) = %v, want nil", id, err) + } + } + invalid := []string{ + "", // empty + "CPU.Load", // uppercase + "has space", // whitespace + "emojié", // non-ascii + string(make([]byte, 65)), // over 64 chars (and NUL bytes) + } + for _, id := range invalid { + s := MetricSample{MetricID: id, Value: "1", TS: MinUnixMS + 1} + if err := s.Validate(); err == nil { + t.Errorf("Validate(%q) = nil, want error", id) + } + } +} From 12e98bda1dc428c6a9beeb094e326321be2b2d21 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:01:37 +0800 Subject: [PATCH 18/60] feat(daemon): bound the realtime outbox with HTTP fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cap the outbox's total row count (default 5000, OpenWithMaxRows for an explicit value). At the cap Enqueue fails with ErrOutboxFull inside the same transaction as the insert — no device_seq is consumed, no row is written — so the existing enqueue-failure fallback in the uplink routes the event over the HTTP ingest path instead: reliability does not degrade while the server is unreachable, and local disk usage stays bounded. Once acks drain the backlog, capacity returns and subsequent events take the realtime path again. Policy documented on ErrOutboxFull and routeToRealtime. Tests: outbox-level (ErrOutboxFull at cap, no seq consumed, capacity restored by an ack) and an end-to-end uplink test (overflow event travels HTTP while the outbox is full, realtime resumes after the backlog drains, non-overflow events never appear on HTTP). --- daemon/internal/realtime/outbox/outbox.go | 47 +++++- .../internal/realtime/outbox/outbox_test.go | 46 ++++++ daemon/internal/uplink/realtime_test.go | 142 ++++++++++++++++++ daemon/internal/uplink/uplink.go | 7 +- 4 files changed, 237 insertions(+), 5 deletions(-) diff --git a/daemon/internal/realtime/outbox/outbox.go b/daemon/internal/realtime/outbox/outbox.go index 5f169b7..078c914 100644 --- a/daemon/internal/realtime/outbox/outbox.go +++ b/daemon/internal/realtime/outbox/outbox.go @@ -16,12 +16,28 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "time" _ "modernc.org/sqlite" ) +// DefaultMaxRows bounds the outbox's total row count (across all spaces). +// The cap keeps local disk usage bounded when the server is unreachable +// for a long stretch; reliability does not degrade at the cap because the +// caller's contract is to fall back to another delivery path (the HTTP +// ingest batch) whenever Enqueue fails — see ErrOutboxFull. +const DefaultMaxRows = 5000 + +// ErrOutboxFull is returned (wrapped) by Enqueue when the outbox already +// holds its maximum number of unacknowledged rows. No device_seq is +// consumed in that case — the failed Enqueue's transaction rolls back +// whole, exactly like any other Enqueue failure — so callers can safely +// route the event to a fallback path (the HTTP ingest batch) and let +// later events use the realtime path again once acks drain the backlog. +var ErrOutboxFull = errors.New("outbox: full") + const schema = ` CREATE TABLE IF NOT EXISTS space_seq_counters ( space TEXT PRIMARY KEY, @@ -50,12 +66,23 @@ type Item struct { // from multiple goroutines: every method serializes through the underlying // *sql.DB, and every mutating operation is one transaction. type Store struct { - db *sql.DB + db *sql.DB + maxRows int } -// Open creates (if needed) and opens the outbox database at path. Callers -// should keep one *Store per daemon process for a given path. +// Open creates (if needed) and opens the outbox database at path with the +// DefaultMaxRows cap. Callers should keep one *Store per daemon process +// for a given path. func Open(path string) (*Store, error) { + return OpenWithMaxRows(path, DefaultMaxRows) +} + +// OpenWithMaxRows is Open with an explicit row cap; maxRows <= 0 falls +// back to DefaultMaxRows. +func OpenWithMaxRows(path string, maxRows int) (*Store, error) { + if maxRows <= 0 { + maxRows = DefaultMaxRows + } // _txlock=immediate: acquire the write lock at BEGIN rather than at // first write, so two overlapping transactions serialize instead of // racing to upgrade a deferred lock (SQLITE_BUSY under concurrent @@ -72,7 +99,7 @@ func Open(path string) (*Store, error) { db.Close() return nil, fmt.Errorf("outbox: init schema: %w", err) } - return &Store{db: db}, nil + return &Store{db: db, maxRows: maxRows}, nil } // Close releases the underlying database handle. @@ -98,6 +125,18 @@ func (s *Store) Enqueue(ctx context.Context, space, kind string, build BuildBody } defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + // Row cap check, inside the same transaction as the insert so two + // concurrent Enqueues cannot both squeeze past the limit. At the cap + // the whole transaction rolls back — no seq consumed, no row written — + // and the caller falls back to its non-realtime delivery path. + var rows int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&rows); err != nil { + return Item{}, fmt.Errorf("outbox: count rows: %w", err) + } + if rows >= s.maxRows { + return Item{}, fmt.Errorf("outbox has %d rows (cap %d): %w", rows, s.maxRows, ErrOutboxFull) + } + seq, err := nextSeqTx(tx, space) if err != nil { return Item{}, err diff --git a/daemon/internal/realtime/outbox/outbox_test.go b/daemon/internal/realtime/outbox/outbox_test.go index 9cf9697..5eb5e87 100644 --- a/daemon/internal/realtime/outbox/outbox_test.go +++ b/daemon/internal/realtime/outbox/outbox_test.go @@ -246,3 +246,49 @@ func TestSurvivesRestart(t *testing.T) { t.Fatalf("seq after restart = %d, want 4 (counter must survive restart)", next.DeviceSeq) } } + +// TestEnqueueFullReturnsErrOutboxFull pins the bounded-outbox policy: at +// the row cap Enqueue fails with ErrOutboxFull, consumes no device_seq +// (the transaction rolls back whole), and capacity freed by an ack makes +// Enqueue work again with the next unconsumed seq. +func TestEnqueueFullReturnsErrOutboxFull(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s, err := OpenWithMaxRows(path, 2) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer s.Close() + ctx := context.Background() + + for i := int64(1); i <= 2; i++ { + if _, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(i)); err != nil { + t.Fatalf("Enqueue %d: %v", i, err) + } + } + + _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) + if !errors.Is(err, ErrOutboxFull) { + t.Fatalf("Enqueue at cap = %v, want ErrOutboxFull", err) + } + + // The failed Enqueue must not have consumed a sequence number. + next, err := s.NextSeq(ctx, "space-a") + if err != nil { + t.Fatalf("NextSeq: %v", err) + } + if next != 3 { + t.Fatalf("NextSeq after full = %d, want 3 (rejected Enqueue must not consume a seq)", next) + } + + // Freeing one row (an ack arrived) restores capacity. + if err := s.Ack(ctx, "space-a", 1); err != nil { + t.Fatalf("Ack: %v", err) + } + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) + if err != nil { + t.Fatalf("Enqueue after ack freed capacity: %v", err) + } + if item.DeviceSeq != 3 { + t.Fatalf("seq after recovery = %d, want 3", item.DeviceSeq) + } +} diff --git a/daemon/internal/uplink/realtime_test.go b/daemon/internal/uplink/realtime_test.go index 53033e1..6503efe 100644 --- a/daemon/internal/uplink/realtime_test.go +++ b/daemon/internal/uplink/realtime_test.go @@ -1,6 +1,7 @@ package uplink import ( + "context" "net/http/httptest" "path/filepath" "testing" @@ -133,3 +134,144 @@ func TestRealtimeFlagOffPreservesExistingBehavior(t *testing.T) { t.Fatalf("expected exactly one task.start over HTTP, got %+v", got) } } + +// TestOutboxFullFallsBackToHTTPAndRecovers pins the bounded-outbox policy +// end to end: when the realtime outbox hits its row cap, further reliable +// events fall back to the HTTP ingest path (reliability does not degrade, +// disk usage stays bounded), and once acks drain the backlog the realtime +// path takes over again. +func TestOutboxFullFallsBackToHTTPAndRecovers(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + wsMessages := make(chan wire.MessageEventBody, 16) + ackRelease := make(chan struct{}) + rtSrv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + released := false + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeMessageEvent { + continue + } + body, _ := wire.DecodeBody(env) + mb := body.(wire.MessageEventBody) + select { + case wsMessages <- mb: + default: + } + if !released { + select { + case <-ackRelease: + released = true + default: + } + } + if released { + conn.Ack(mb.DeviceID, mb.DeviceSeq) + } + } + }) + defer rtSrv.Close() + + // Cap of 1: the first unacked reliable event fills the outbox. + store, err := outbox.OpenWithMaxRows(filepath.Join(t.TempDir(), "outbox.db"), 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer store.Close() + + rt := rtclient.New(rtclient.Config{ + URL: rtSrv.URL(), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + ResendInterval: 20 * time.Millisecond, + }) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 20 * time.Millisecond, Realtime: rt}) + defer u.Close() + + msg := func(text string) Event { + return ev(protocol.MessageSend, func(e *Event) { e.Text = text; e.Level = "info" }) + } + waitForWS := func(text string) { + t.Helper() + deadline := time.After(3 * time.Second) + for { + select { + case mb := <-wsMessages: + if mb.Text == text { + return + } + case <-deadline: + t.Fatalf("realtime server never saw message %q", text) + } + } + } + + // 1. First event: fits in the outbox (row 1/1), goes over WS, unacked. + u.Offer(msg("first")) + waitForWS("first") + + // 2. Second event: outbox is full -> ErrOutboxFull -> HTTP fallback. + u.Offer(msg("second")) + deadline := time.Now().Add(3 * time.Second) + for { + found := false + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend && e.Text == "second" { + found = true + } + } + if found { + break + } + if time.Now().After(deadline) { + t.Fatal("HTTP ingest never carried the overflow event while the outbox was full") + } + time.Sleep(5 * time.Millisecond) + } + + // 3. The server starts acking; the resend loop replays "first", it gets + // acked, and the outbox drains. + close(ackRelease) + waitFor := func(cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("condition not met within 3s") + } + waitFor(func() bool { + n, err := store.Count(context.Background()) + return err == nil && n == 0 + }) + + // 4. With capacity restored, the realtime path takes over again. + u.Offer(msg("third")) + waitForWS("third") + + // "first" and "third" must never have gone over HTTP. + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend && e.Text != "second" { + t.Fatalf("HTTP ingest carried %q, which should have traveled the realtime path only", e.Text) + } + } +} diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index 7ad128e..c3a0443 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -160,7 +160,12 @@ func (u *Uplink) Offer(ev Event) { // event, or leaves it (false) to fall through unchanged — either because // the kind has no realtime equivalent (task.log) or because the realtime // send itself failed, in which case losing the event silently would be -// worse than a harmless duplicate over HTTP. +// worse than a harmless duplicate over HTTP. In particular, a full +// realtime outbox (outbox.ErrOutboxFull — the bounded local queue hit its +// row cap because the server has not been acking) lands here as a send +// failure: the event travels the HTTP ingest path instead, so reliability +// does not degrade while local disk usage stays bounded, and once acks +// drain the backlog subsequent events take the realtime path again. func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { switch ev.Kind { case protocol.TaskStart, protocol.TaskProgress, protocol.TaskStep, protocol.TaskDone, protocol.TaskFail: From cf4d47c77849c33e31dc33b1954d8261fe7b302a Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:01:42 +0800 Subject: [PATCH 19/60] fix(apple): keep live delta state authoritative over http refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two adversarial-review findings in the app wiring: - HTTP refresh no longer bypasses realtime state (F1): while the WebSocket phase is .live, refresh() updates only REST-only concepts (presence, lastSyncAt) and leaves the four reliable collections (tasks/metrics/events/automations) to deltas. The liveness check runs AFTER the await returns, so an in-flight response — a foreground force-refresh, the post-updateMetric refresh, or a fallback-poll response landing after .recovered — can never clobber newer delta state. An HTTP failure while live no longer raises the error banner. - Foreground cycles resume incrementally (F4): AppModel keeps the last SpaceState (with revision C) for the process lifetime and seeds the rebuilt RealtimeClient with it, so returning to the foreground resumes with last_revision C — an incremental delta or empty you-are-current reply — instead of pulling a full snapshot every time. The preserved state is dropped on disconnect and on credential change, where its revision sequence no longer applies. --- apple/SitrepApp/Sitrep/MainTabView.swift | 8 +++-- apple/SitrepApp/Sitrep/SitrepApp.swift | 37 ++++++++++++++++++++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/apple/SitrepApp/Sitrep/MainTabView.swift b/apple/SitrepApp/Sitrep/MainTabView.swift index 5db7f4b..2f689a4 100644 --- a/apple/SitrepApp/Sitrep/MainTabView.swift +++ b/apple/SitrepApp/Sitrep/MainTabView.swift @@ -75,9 +75,11 @@ struct MainTabView: View { } // Coming back to foreground: refresh immediately, (re)connect the // realtime channel, and revive the HTTP fallback poll if anything - // killed it while suspended. Leaving the foreground closes the - // realtime connection — the interest lease simply lapses server-side - // rather than being explicitly released. + // killed it while suspended. While the WebSocket is live, refresh() + // only updates REST-only state (presence) — the reliable collections + // stay owned by deltas. Leaving the foreground closes the realtime + // connection — the interest lease simply lapses server-side rather + // than being explicitly released. .onChange(of: scenePhase) { _, phase in switch phase { case .active: diff --git a/apple/SitrepApp/Sitrep/SitrepApp.swift b/apple/SitrepApp/Sitrep/SitrepApp.swift index de94e2f..908db8d 100644 --- a/apple/SitrepApp/Sitrep/SitrepApp.swift +++ b/apple/SitrepApp/Sitrep/SitrepApp.swift @@ -93,6 +93,12 @@ final class AppModel { private var realtimeClient: RealtimeClient? private var realtimeObservers: [Task] = [] + /// Last SpaceState received from the realtime channel, kept for the + /// process lifetime (not persisted to disk): when the app backgrounds + /// and later rebuilds the client, this — with its revision `C` — seeds + /// the new connection so the foreground resume is an incremental + /// `delta` rather than a full snapshot every time. + private var lastSpaceState: SpaceState? /// Low-frequency (>=30s) HTTP snapshot polling, used ONLY while the /// realtime connection is down for multiple backoff cycles — see /// `RealtimeClient.Notice.fellBackToPolling`/`.recovered`. This replaces @@ -133,9 +139,11 @@ final class AppModel { Task { await PushRegistrar.shared.configure(client: client) } // Credentials changed under an existing connection (e.g. re-paired) // — reconnect with the new ones rather than keep talking under the - // old identity. + // old identity. The preserved SpaceState belongs to the old space's + // revision sequence, so drop it: the next resume starts from 0. if realtimeClient != nil { stopRealtime() + lastSpaceState = nil enterForeground() } } @@ -183,7 +191,10 @@ final class AppModel { } let configuration = RealtimeClient.Configuration( url: url, token: token.isEmpty ? nil : token, deviceID: deviceID) - let rt = RealtimeClient(configuration: configuration) + // Seed with the last known SpaceState so the resume on the new + // connection is incremental (`last_revision: C`) instead of a full + // snapshot on every foreground cycle. + let rt = RealtimeClient(configuration: configuration, initialState: lastSpaceState ?? SpaceState()) realtimeClient = rt realtimeObservers = [ Task { [weak self] in @@ -215,6 +226,7 @@ final class AppModel { } private func applyRealtimeState(_ state: SpaceState) { + lastSpaceState = state tasks = state.uiTasks let sortedMetrics = state.uiMetrics // Widgets refresh on a slow OS budget; while the app is foreground @@ -347,6 +359,7 @@ final class AppModel { func disconnect() async { stopRealtime() stopFallbackPolling() + lastSpaceState = nil if !deviceID.isEmpty { try? await client?.revokeDevice(id: deviceID) } @@ -379,7 +392,19 @@ final class AppModel { guard let client else { return } do { let snapshot = try await client.snapshot() + // presence is REST-only (the realtime protocol does not carry + // it) — updating it is the reason this refresh still runs even + // while realtime is live. presence = snapshot.presence + lastError = nil + lastSyncAt = .now + // Checked AFTER the await, deliberately: while the WebSocket is + // live, deltas own the four reliable collections and an HTTP + // response — possibly computed before the connection went live — + // must not clobber them. Re-checking here (not at request time) + // also covers the in-flight race: a fallback-poll or pull-to- + // refresh response that lands after `.recovered` is dropped. + guard connectionPhase.allowsReliableStateOverwrite else { return } events = snapshot.messages.map(\.state) automations = snapshot.automations.sorted { $0.name < $1.name } tasks = snapshot.tasks.sorted { $0.updatedAt > $1.updatedAt } @@ -391,11 +416,13 @@ final class AppModel { WidgetCenter.shared.reloadAllTimelines() } metrics = sorted - lastError = nil - lastSyncAt = .now adoptOrphanedTasks() } catch { - lastError = error.localizedDescription + // An HTTP hiccup while realtime is live is not a sync outage — + // don't raise the banner over it. + if connectionPhase.allowsReliableStateOverwrite { + lastError = error.localizedDescription + } } } } From b0a7bbfe26611683ca6957e02dab224af3f594a3 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:01:52 +0800 Subject: [PATCH 20/60] test(apple): pin fixture rejection reasons and add realtime regressions Tighten and extend the SitrepKit suite per review (F6): - Every schema-invalid fixture now asserts the SPECIFIC constraint it trips (substring of the thrown error), so a fixture failing for the wrong reason is a test failure; an unmapped new fixture fails loudly. - New RealtimeRegressionTests cover the review fixes: only .live blocks HTTP overwrite of reliable state and the check re-runs after the await (F1, incl. the in-flight race); a delta or any other envelope interleaved during a chunked snapshot is a malformed sequence that discards the buffer (F3); a reconnect seeded with the preserved SpaceState resumes incrementally from C and RealtimeClient honors its initialState (F4); the delta event wrapper ignores unknown sibling fields (F5). - New folding boundary test: an event without step/title leaves the previous values unchanged, while done still clears step (SPEC.md 6.4). --- .../RealtimeMessageFixtureTests.swift | 33 +++- .../Realtime/RealtimeRegressionTests.swift | 176 ++++++++++++++++++ .../Realtime/SpaceStateFoldingTests.swift | 20 ++ 3 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift index dea3803..c636bf0 100644 --- a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift @@ -56,6 +56,27 @@ final class RealtimeMessageFixtureTests: XCTestCase { "role-viewer-command-origin-server.json", ] + /// The specific constraint each schema-invalid fixture must trip, as a + /// substring of the thrown error's description. Pinning the REASON (not + /// just "some error") catches regressions where a fixture starts + /// failing for the wrong reason — e.g. a decoding-order change masking + /// the intended violation with an unrelated one. + private static let expectedRejectionReasons: [String: String] = [ + "ack-neither-acked-nor-in-reply-to.json": "'acked' and/or 'in_reply_to'", + "command-pause-with-automation-id.json": "forbids automation_id", + "command-throttle-with-task-id.json": "forbids task_id and automation_id", + "command-viewer-sends-throttle.json": "requires origin 'server'", + "envelope-carries-credential-field.json": "unexpected field(s): token", + "error-missing-retryable-fatal.json": "retryable", // DecodingError.keyNotFound + "hello-offer-empty-protocol-versions.json": "protocol_versions must not be empty", + "message-event-timestamp-in-seconds.json": "occurred_at is not a Unix ms timestamp", + "metric-frame-oversized-value.json": "value exceeds max length 256", + "resume-negative-revision.json": "last_revision -1 below minimum 0", + "subscribe-unknown-topic.json": "automation", // DecodingError: no such RTTopic case + "task-event-missing-device-seq.json": "device_seq", // DecodingError.keyNotFound + "task-event-progress-missing-percent.json": "percent required when kind == 'progress'", + ] + func testInvalidFixturesFailDecodeOrAreUnauthorized() throws { let names = try FixtureLoader.names(in: "invalid") XCTAssertFalse(names.isEmpty) @@ -64,17 +85,25 @@ final class RealtimeMessageFixtureTests: XCTestCase { if Self.roleTaggedInvalidFixtures.contains(name) { try assertRejectedByAuthorization(fixture: name, wrapperData: bytes) } else { - assertFailsToDecode(fixture: name, data: bytes) + guard let expectedReason = Self.expectedRejectionReasons[name] else { + XCTFail("invalid/\(name): new fixture — add its expected rejection reason to the map") + continue + } + assertFailsToDecode(fixture: name, data: bytes, reasonContaining: expectedReason) } } } - private func assertFailsToDecode(fixture: String, data: Data) { + private func assertFailsToDecode(fixture: String, data: Data, reasonContaining expected: String) { XCTAssertThrowsError(try RealtimeFrame.decode(data), "invalid/\(fixture) should fail to decode") { error in guard error is RealtimeDecodingError || error is DecodingError else { XCTFail("invalid/\(fixture): unexpected error type \(error)") return } + let description = String(describing: error) + XCTAssertTrue( + description.contains(expected), + "invalid/\(fixture): rejected for the wrong reason — expected description containing '\(expected)', got: \(description)") } } diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift new file mode 100644 index 0000000..7f0eb10 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift @@ -0,0 +1,176 @@ +import XCTest +@testable import SitrepKit + +/// Regression coverage for the adversarial-review rulings (F1/F3/F4/F5): +/// HTTP-overwrite gating while the realtime channel is live, interleaved +/// frames during chunked snapshots, incremental resume across foreground +/// cycles, and body-level leniency of the delta event wrapper. +final class RealtimeRegressionTests: XCTestCase { + // MARK: - F1: HTTP snapshot must not overwrite reliable state while live + + func testOnlyLivePhaseBlocksReliableStateOverwrite() { + XCTAssertFalse(RealtimeClient.Phase.live.allowsReliableStateOverwrite, + "while live, deltas own the reliable collections") + XCTAssertTrue(RealtimeClient.Phase.idle.allowsReliableStateOverwrite) + XCTAssertTrue(RealtimeClient.Phase.connecting.allowsReliableStateOverwrite) + XCTAssertTrue(RealtimeClient.Phase.handshaking.allowsReliableStateOverwrite) + XCTAssertTrue(RealtimeClient.Phase.subscribed.allowsReliableStateOverwrite, + "not yet delta-eligible — HTTP is still the best source") + XCTAssertTrue(RealtimeClient.Phase.failed("down").allowsReliableStateOverwrite) + } + + /// The in-flight race: an HTTP request issued while the WebSocket was + /// down must be dropped if the connection went live before the response + /// was applied. Callers implement this by re-evaluating the phase AFTER + /// the await — this test exercises that exact pattern. + func testInFlightHTTPApplyReChecksLivenessAfterAwait() async { + // Shared mutable "current phase", as AppModel's connectionPhase is. + actor PhaseBox { + var phase: RealtimeClient.Phase = .failed("ws down") + func set(_ p: RealtimeClient.Phase) { phase = p } + } + let box = PhaseBox() + var applied = false + + // Request issued while down: a pre-await check would have passed. + let allowedAtRequestTime = await box.phase.allowsReliableStateOverwrite + XCTAssertTrue(allowedAtRequestTime) + + // While the "HTTP response" is in flight, the WS recovers. + await box.set(.live) + + // Apply-time re-check (the pattern refresh() uses) must now refuse. + if await box.phase.allowsReliableStateOverwrite { + applied = true + } + XCTAssertFalse(applied, "a response landing after .recovered must not clobber live delta state") + + // And once the connection is down again, HTTP applies normally. + await box.set(.failed("ws down again")) + if await box.phase.allowsReliableStateOverwrite { + applied = true + } + XCTAssertTrue(applied) + } + + // MARK: - F3: interleaved envelope during a chunked snapshot + + func testDeltaInterleavedDuringChunkedSnapshotIsMalformed() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let part1 = SnapshotBody(revision: 10, part: 1, final: false, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(part1), .snapshotChunkBuffered) + XCTAssertTrue(gate.isSnapshotInFlight) + + // §6.2: any non-snapshot envelope between chunks is a malformed + // sequence — even a delta that would otherwise look like a valid + // resume reply. + let interleaved = DeltaBody(fromRevision: 0, toRevision: 0, events: []) + guard case .malformedSequence = gate.receiveDelta(interleaved) else { + return XCTFail("expected malformedSequence for a delta interleaved mid-snapshot") + } + XCTAssertFalse(gate.isSnapshotInFlight, "buffered chunks are discarded — they can never complete now") + XCTAssertFalse(gate.deltaEligible) + XCTAssertEqual(gate.state.revision, 0, "nothing from the aborted snapshot is applied") + } + + func testInterleavedFrameHelperReportsMalformedAndClearsBuffer() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let part1 = SnapshotBody(revision: 7, part: 1, final: false, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(part1), .snapshotChunkBuffered) + + // The client routes ANY non-snapshot frame (ack, error, metric...) + // through this helper while a snapshot is in flight. + guard case .malformedSequence = gate.interleavedFrameDuringSnapshot() else { + return XCTFail("expected malformedSequence") + } + XCTAssertFalse(gate.isSnapshotInFlight) + } + + // MARK: - F4: foreground cycles resume incrementally with preserved C + + /// Simulates background → foreground: the first "connection" catches up + /// via the scenario fixtures to revision 128; its SpaceState (with C) + /// survives in the app model; the rebuilt gate for the next connection + /// resumes with C and receives an incremental reply — never a snapshot. + func testReconnectResumesIncrementallyWithPreservedRevision() throws { + let files = try FixtureLoader.scenarioFiles("client-resume-delta") + + var firstConnection = RealtimeResumeGate(state: SpaceState(revision: 126)) + firstConnection.beganResume(lastRevision: 126) + guard case .delta(let catchup) = try RealtimeFrame.decode(files[5].data) else { return XCTFail() } + XCTAssertEqual(firstConnection.receiveDelta(catchup.body), .applied) + XCTAssertEqual(firstConnection.state.revision, 128) + + // App backgrounds: connection closes, SpaceState is kept in-process. + let preserved = firstConnection.state + + // App foregrounds: a new gate/connection seeded with the preserved + // state resumes from C = 128 and gets the "you are already current" + // empty delta — an incremental path, not a fresh snapshot. + var secondConnection = RealtimeResumeGate(state: preserved) + secondConnection.beganResume(lastRevision: preserved.revision) + let youAreCurrent = DeltaBody(fromRevision: 128, toRevision: 128, events: []) + XCTAssertEqual(secondConnection.receiveDelta(youAreCurrent), .applied) + XCTAssertTrue(secondConnection.deltaEligible) + XCTAssertEqual(secondConnection.state.revision, 128) + XCTAssertEqual(secondConnection.state.tasks, preserved.tasks, "no state was thrown away by the reconnect") + + // A subsequent live delta continues from the preserved C. + guard case .delta(let live) = try RealtimeFrame.decode(files[6].data) else { return XCTFail() } + XCTAssertEqual(secondConnection.receiveDelta(live.body), .applied) + XCTAssertEqual(secondConnection.state.revision, 129) + } + + func testRealtimeClientHonorsInitialState() async { + let configuration = RealtimeClient.Configuration( + url: URL(string: "wss://example.invalid/v3/realtime")!, token: nil, deviceID: "iphone-test-01") + var seed = SpaceState(revision: 128) + seed.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.5", ts: 1_752_482_000_000), + ])) + let client = RealtimeClient(configuration: configuration, initialState: seed) + let state = await client.currentState + XCTAssertEqual(state.revision, 128, "a rebuilt client must start from the preserved C, not 0") + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.5") + } + + // MARK: - F5: delta event wrapper tolerates unknown sibling fields + + func testDeltaEventIgnoresUnknownSiblingFields() throws { + // A future minor version may annotate delta entries with extra + // fields; SPEC.md §15's body-level tolerance means they are ignored, + // not rejected (§3's strictness covers only the envelope top level). + let json = """ + { + "type": "delta", + "id": "lenient-1", + "ts": 1752482000000, + "body": { + "from_revision": 5, + "to_revision": 6, + "events": [ + { + "event_type": "task.event", + "trace_id": "future-minor-version-annotation", + "event": { + "device_id": "mac-1", + "device_seq": 9, + "task_id": "run-1", + "kind": "started", + "occurred_at": 1752482000000, + "title": "Job" + } + } + ] + } + } + """ + let frame = try RealtimeFrame.decode(Data(json.utf8)) + guard case .delta(let env) = frame, case .taskEvent(let event) = env.body.events[0] else { + return XCTFail("expected the entry to decode as a task.event despite the unknown sibling field") + } + XCTAssertEqual(event.taskID, "run-1") + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift index 053c41c..20c8680 100644 --- a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift @@ -65,6 +65,26 @@ final class SpaceStateFoldingTests: XCTestCase { XCTAssertEqual(task.message, "boom") } + /// §6.4 boundary: an event WITHOUT `step` leaves the previous step + /// unchanged (absence is not a clear); only `done`/`failed` clear it. + /// Same for `title`/`percent`: absent means "unchanged". + func testFoldingEventWithoutStepKeepsPreviousStep() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 40, step: "phase 1")), + .taskEvent(taskEvent(seq: 3, kind: .progress, occurredAt: 3000, percent: 70)), // no step, no title + ] + let state = applyAsDeltaStream(events) + let task = try! XCTUnwrap(state.tasks["run-1"]) + XCTAssertEqual(task.step, "phase 1", "absent step must not clear the previous one") + XCTAssertEqual(task.title, "Job") + XCTAssertEqual(task.percent, 70) + + // ...and done DOES clear it. + let finished = applyAsDeltaStream(events + [.taskEvent(taskEvent(seq: 4, kind: .done, occurredAt: 4000))]) + XCTAssertNil(finished.tasks["run-1"]?.step) + } + func testTaskFoldingMessageStaysAbsentWhileRunning() { let events: [DeltaEvent] = [ .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), From de08949e5ef9b4c00a607ecddfee56d77f1f50b7 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:03:03 +0800 Subject: [PATCH 21/60] fix(daemon): stop reconnecting after fatal handshake rejection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two handshake outcomes previously fed the reconnect loop forever even though a retry could only fail identically: - an error reply that is fatal AND retryable:false (version_unsupported, unauthenticated, ... — SPEC.md section 13's flags are authoritative) - a hello accept naming a protocol version the offer never contained, which section 9.2's intersection rule makes a failed negotiation Both now surface a "giving up, not retryable" log and terminate the reconnect loop; recovery is an explicit restart (a new Client). Tests cover both paths by counting connections at the mock server: exactly one attempt, no loop. --- daemon/internal/realtime/client/client.go | 40 ++++++++++ .../internal/realtime/client/client_test.go | 74 +++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index 398436d..711b5bf 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -22,6 +22,14 @@ const sourceRole = "source" var errClientClosing = errors.New("realtime client: closing") +// errNoRetry marks a handshake failure that is fatal AND not retryable +// (version_unsupported, unauthenticated, or a server accept naming a +// protocol version we never offered): reconnecting cannot succeed without +// an out-of-band change (a software upgrade, a new credential), so the +// reconnect loop must stop instead of hammering the server forever. The +// process recovers by constructing a new Client (i.e. an explicit restart). +var errNoRetry = errors.New("realtime client: fatal, not retryable") + // Client is a reconnecting realtime-protocol source connection. Construct // with New; it starts connecting immediately in the background. Call // SendTaskEvent/SendMessageEvent for reliable events and SendMetric for @@ -219,6 +227,14 @@ func (c *Client) run() { if errors.Is(err, errClientClosing) { return } + if errors.Is(err, errNoRetry) { + // A retry can only fail the same way (wrong protocol version, + // revoked credential, ...): surface it and stop the loop + // entirely rather than reconnecting forever. Recovery is an + // explicit restart (a new Client). + c.cfg.Logf("realtime: giving up, not retryable: %v", err) + return + } c.cfg.Logf("realtime: connection ended: %v", err) c.mu.Lock() @@ -360,6 +376,13 @@ func (c *Client) awaitHelloAccept(ctx context.Context, conn *websocket.Conn) (wi body, derr := wire.DecodeBody(env) if derr == nil { eb := body.(wire.ErrorBody) + // A fatal AND non-retryable rejection (version_unsupported, + // unauthenticated, ...) can only repeat on the next attempt: + // mark it errNoRetry so the reconnect loop stops (SPEC.md + // section 13's retryable flag is authoritative). + if eb.Fatal != nil && *eb.Fatal && eb.Retryable != nil && !*eb.Retryable { + return wire.HelloAccept{}, fmt.Errorf("server rejected hello: %s: %s: %w", eb.Code, eb.Message, errNoRetry) + } return wire.HelloAccept{}, fmt.Errorf("server rejected hello: %s: %s", eb.Code, eb.Message) } return wire.HelloAccept{}, fmt.Errorf("server rejected hello with malformed error body") @@ -375,6 +398,23 @@ func (c *Client) awaitHelloAccept(ctx context.Context, conn *websocket.Conn) (wi if hb.Accept == nil { return wire.HelloAccept{}, fmt.Errorf("expected hello accept, got offer") } + // SPEC.md section 9.2: the server selects from the INTERSECTION of the + // offer's protocol_versions with its own set, so a conformant accept + // always names a version we offered. An accept naming any other version + // is a failed negotiation — retrying with the same offer cannot end + // differently, so treat it like version_unsupported and stop. + offered := false + for _, v := range c.cfg.ProtocolVersions { + if v == hb.Accept.ProtocolVersion { + offered = true + break + } + } + if !offered { + return wire.HelloAccept{}, fmt.Errorf( + "server accepted protocol version %d, which we never offered (%v): %w", + hb.Accept.ProtocolVersion, c.cfg.ProtocolVersions, errNoRetry) + } return *hb.Accept, nil } diff --git a/daemon/internal/realtime/client/client_test.go b/daemon/internal/realtime/client/client_test.go index fa2f3bc..7cd1fec 100644 --- a/daemon/internal/realtime/client/client_test.go +++ b/daemon/internal/realtime/client/client_test.go @@ -361,3 +361,77 @@ var envCounter int64 func rttestEnvID() string { return fmt.Sprintf("env-%d", atomic.AddInt64(&envCounter, 1)) } + +// TestClientStopsAfterFatalNonRetryableHello pins the give-up rule: a +// handshake rejection that is fatal AND not retryable (here +// version_unsupported, SPEC.md section 13) must stop the reconnect loop +// entirely instead of hammering the server forever. +func TestClientStopsAfterFatalNonRetryableHello(t *testing.T) { + var connCount int32 + srv := rttest.New(func(conn *rttest.Conn) { + atomic.AddInt32(&connCount, 1) + if _, err := conn.ReadEnvelope(); err != nil { + return // expected the hello offer + } + conn.SendError(wire.ErrorBody{ + Code: wire.ErrVersionUnsupported, + Message: "no shared protocol version", + Retryable: boolPtr(false), + Fatal: boolPtr(true), + }) + }) + defer srv.Close() + + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, nil) + + // Backoff base is 5ms/cap 20ms: if the client kept reconnecting, this + // window would see dozens of connections. + time.Sleep(300 * time.Millisecond) + if n := atomic.LoadInt32(&connCount); n != 1 { + t.Fatalf("server saw %d connections after a fatal non-retryable hello rejection, want exactly 1 (no reconnect loop)", n) + } +} + +// TestClientStopsWhenAcceptNamesUnofferedVersion pins the accept-side +// version check: SPEC.md section 9.2 requires the server to select from +// the intersection with the offer, so an accept naming a version the +// client never offered is a failed negotiation — treated like +// version_unsupported, and the reconnect loop stops. +func TestClientStopsWhenAcceptNamesUnofferedVersion(t *testing.T) { + var connCount int32 + srv := rttest.New(func(conn *rttest.Conn) { + atomic.AddInt32(&connCount, 1) + if _, err := conn.ReadEnvelope(); err != nil { + return // expected the hello offer + } + accept := wire.HelloAccept{ + Stage: "accept", + ProtocolVersion: 99, // never offered (client offers [1]) + SessionID: "sess-bogus", + HeartbeatIntervalMS: 1000, + } + env, err := wire.NewEnvelope(wire.TypeHello, rttestEnvID(), rttest.NowMS(), wire.HelloBody{Accept: &accept}) + if err != nil { + t.Errorf("NewEnvelope: %v", err) + return + } + _ = conn.WriteEnvelope(env) + // Keep the connection open; the client must abandon it (and not + // come back) on its own. + for { + if _, err := conn.ReadEnvelope(); err != nil { + return + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, nil) + + time.Sleep(300 * time.Millisecond) + if n := atomic.LoadInt32(&connCount); n != 1 { + t.Fatalf("server saw %d connections after an accept naming an unoffered version, want exactly 1 (no reconnect loop)", n) + } +} From 53d59dd31df63aa9b53d5980d1f696545554487d Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:03:31 +0800 Subject: [PATCH 22/60] fix(daemon): reject acks carrying foreign device pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC.md sections 5.3/13 require every pair in an ack to carry the receiving connection's authenticated device_id, and a pair for any other device makes the WHOLE envelope malformed. The client previously skipped foreign pairs but still honored the matching ones; now it drops the entire ack without retiring anything from the outbox (resend keeps this safe — the events stay queued for a later well-formed ack), reports the violation back with error{malformed, retryable, non-fatal}, and keeps the connection. Extract the error-envelope write into a shared sendError helper (also used by the command_expired path). Test: the mock server sends an ack mixing one matching and one foreign pair — nothing is deleted — then a well-formed ack on the same connection still retires the event. --- daemon/internal/realtime/client/client.go | 47 ++++++++-- .../internal/realtime/client/client_test.go | 91 +++++++++++++++++++ 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index 711b5bf..8a81261 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -524,10 +524,27 @@ func (c *Client) handleFrame(ctx context.Context, conn *websocket.Conn, env wire switch env.Type { case wire.TypeAck: ab := body.(wire.AckBody) + // SPEC.md sections 5.3/13: every pair in one ack MUST carry the + // device_id of this connection's authenticated device; a pair for + // any other device makes the WHOLE envelope malformed. Drop the + // ack without retiring anything — the events stay queued for a + // later well-formed ack (resend makes this safe) — report the + // protocol violation back as error{malformed}, and keep the + // connection (malformed is retryable, non-fatal). for _, p := range ab.Acked { if p.DeviceID != c.cfg.DeviceID { - continue // malformed per SPEC.md 5.3; ignore rather than trust + c.cfg.Logf("realtime: protocol violation: ack %s carries a pair for foreign device %q; dropping whole ack", env.ID, p.DeviceID) + c.sendError(ctx, conn, wire.ErrorBody{ + Code: wire.ErrMalformed, + Message: "ack pair for a device other than this connection's", + InReplyTo: env.ID, + Retryable: boolPtr(true), + Fatal: boolPtr(false), + }) + return nil } + } + for _, p := range ab.Acked { if err := c.cfg.Outbox.Ack(ctx, c.cfg.Space, p.DeviceSeq); err != nil { c.cfg.Logf("realtime: ack seq %d: %v", p.DeviceSeq, err) } @@ -573,20 +590,13 @@ func (c *Client) handleCommand(ctx context.Context, conn *websocket.Conn, env wi windowEnd := time.UnixMilli(env.TS).Add(time.Duration(cb.TTLMs) * time.Millisecond).Add(skew) if now.Before(windowStart) || now.After(windowEnd) { c.cfg.Logf("realtime: dropping expired command %s (action %s)", cb.CommandID, cb.Action) - writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - errBody := wire.ErrorBody{ + c.sendError(ctx, conn, wire.ErrorBody{ Code: wire.ErrCommandExpired, Message: "command ttl window elapsed", InReplyTo: env.ID, Retryable: boolPtr(false), Fatal: boolPtr(false), - } - if eenv, eerr := wire.NewEnvelope(wire.TypeError, newEnvelopeID(), c.nowMS(), errBody); eerr == nil { - if data, derr := eenv.Encode(); derr == nil { - _ = conn.Write(writeCtx, websocket.MessageText, data) - } - } + }) return } @@ -614,6 +624,23 @@ func (c *Client) handleCommand(ctx context.Context, conn *websocket.Conn, env wi func boolPtr(b bool) *bool { return &b } +// sendError best-effort writes one error envelope on the current +// connection; failures are ignored (the connection's own health checks +// notice a dying socket independently). +func (c *Client) sendError(ctx context.Context, conn *websocket.Conn, body wire.ErrorBody) { + env, err := wire.NewEnvelope(wire.TypeError, newEnvelopeID(), c.nowMS(), body) + if err != nil { + return + } + data, err := env.Encode() + if err != nil { + return + } + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + _ = conn.Write(writeCtx, websocket.MessageText, data) +} + // alreadySeen reports whether commandID has been handled before, recording // it if not. Bounded so a long-lived connection doesn't grow this set // forever. diff --git a/daemon/internal/realtime/client/client_test.go b/daemon/internal/realtime/client/client_test.go index 7cd1fec..4dc94e8 100644 --- a/daemon/internal/realtime/client/client_test.go +++ b/daemon/internal/realtime/client/client_test.go @@ -435,3 +435,94 @@ func TestClientStopsWhenAcceptNamesUnofferedVersion(t *testing.T) { t.Fatalf("server saw %d connections after an accept naming an unoffered version, want exactly 1 (no reconnect loop)", n) } } + +// TestClientRejectsAckWithForeignDevicePair pins the SPEC.md section +// 5.3/13 rule: an ack carrying any (device_id, device_seq) pair for a +// device other than this connection's makes the WHOLE envelope malformed — +// nothing is retired from the outbox, not even the pairs that did match. +// The connection survives (malformed is non-fatal): a later well-formed +// ack on the same connection still retires the event. +func TestClientRejectsAckWithForeignDevicePair(t *testing.T) { + badAckSent := make(chan struct{}, 1) + sendGoodAck := make(chan struct{}) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + gotFirst := false + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + continue + } + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + if !gotFirst { + gotFirst = true + // A malformed ack: one matching pair AND one foreign pair. + // The client must drop it whole — even the matching pair + // must not be retired. + bad := wire.AckBody{Acked: []wire.AckedPair{ + {DeviceID: tb.DeviceID, DeviceSeq: tb.DeviceSeq}, + {DeviceID: "someone-else", DeviceSeq: tb.DeviceSeq}, + }} + badEnv, err := wire.NewEnvelope(wire.TypeAck, rttestEnvID(), rttest.NowMS(), bad) + if err != nil { + t.Errorf("NewEnvelope: %v", err) + return + } + if err := conn.WriteEnvelope(badEnv); err != nil { + return + } + select { + case badAckSent <- struct{}{}: + default: + } + go func() { + <-sendGoodAck + conn.Ack(tb.DeviceID, tb.DeviceSeq) + }() + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + if err := c.SendTaskEvent(TaskEvent{TaskID: "run-1", Kind: "started", OccurredAt: time.Now()}); err != nil { + t.Fatalf("SendTaskEvent: %v", err) + } + + select { + case <-badAckSent: + case <-time.After(3 * time.Second): + t.Fatal("server never sent the malformed ack") + } + + // Give the client ample time to (mis)process the malformed ack; the + // outbox entry must still be there. + time.Sleep(100 * time.Millisecond) + pending, err := store.Pending(context.Background(), testSpace) + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("outbox has %d items after the malformed ack, want 1 (a foreign pair must void the whole ack)", len(pending)) + } + + // The connection survived: a well-formed ack still works. + close(sendGoodAck) + waitFor(t, 3*time.Second, func() bool { + p, err := store.Pending(context.Background(), testSpace) + return err == nil && len(p) == 0 + }) +} From 40ed17a68fe64c70f4d3469a203126ff108f9ffe Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:03:51 +0800 Subject: [PATCH 23/60] test(daemon): pin throttle state persistence across reconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the agreed semantics in a regression test: a source that was told to throttle keeps its throttled metric cadence across a connection drop (it does not optimistically resume the fast rate on reconnect), and the server-side remedy — re-sending the space's current interest state after the new connection's hello completes — takes effect immediately: a resume_rate on the second connection restores the normal cadence. --- .../internal/realtime/client/client_test.go | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/daemon/internal/realtime/client/client_test.go b/daemon/internal/realtime/client/client_test.go index 4dc94e8..fec506c 100644 --- a/daemon/internal/realtime/client/client_test.go +++ b/daemon/internal/realtime/client/client_test.go @@ -526,3 +526,77 @@ func TestClientRejectsAckWithForeignDevicePair(t *testing.T) { return err == nil && len(p) == 0 }) } + +// TestClientThrottlePreservedAcrossReconnect pins the agreed semantics for +// throttle state vs. reconnects: the daemon KEEPS its throttled state +// across a connection drop (it does not optimistically resume the fast +// cadence), and relies on the server re-sending the space's current +// interest state (throttle or resume_rate) after hello on the new +// connection — here, a resume_rate on the second connection immediately +// restores the normal cadence. +func TestClientThrottlePreservedAcrossReconnect(t *testing.T) { + var connNum int32 + throttleProcessed := make(chan struct{}) + conn2Ready := make(chan *rttest.Conn, 1) + + srv := rttest.New(func(conn *rttest.Conn) { + n := atomic.AddInt32(&connNum, 1) + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + switch n { + case 1: + if err := conn.SendCommand(wire.CommandBody{ + CommandID: "cmd-throttle-x", Origin: "server", Action: "throttle", TTLMs: 3600000, + }); err != nil { + t.Errorf("SendCommand: %v", err) + return + } + // Hold the connection open until the test has observed the + // throttled state, then drop it to force a reconnect. + <-throttleProcessed + default: + conn2Ready <- conn + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + _ = env + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + waitFor(t, 3*time.Second, c.MetricsThrottled) + close(throttleProcessed) // server drops connection 1 + + var conn2 *rttest.Conn + select { + case conn2 = <-conn2Ready: + case <-time.After(3 * time.Second): + t.Fatal("client never reconnected") + } + + // The throttled state survived the reconnect. + if !c.MetricsThrottled() { + t.Fatal("throttled state was lost across the reconnect; it must be preserved until the server says otherwise") + } + + // The server re-sends the space's current interest state after hello: + // a resume_rate on the new connection restores the fast cadence. + if err := conn2.SendCommand(wire.CommandBody{ + CommandID: "cmd-resume-x", Origin: "server", Action: "resume_rate", TTLMs: 3600000, + }); err != nil { + t.Fatalf("SendCommand(resume_rate): %v", err) + } + waitFor(t, 3*time.Second, func() bool { return !c.MetricsThrottled() }) +} From f342ec72de000db09c2dbda35ecb21a000b29052 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:13:17 +0800 Subject: [PATCH 24/60] fix(server): reject duplicate protocol_versions and topics The hello offer schema declares uniqueItems on protocol_versions and the subscribe/interest.renew schema declares it on topics; the hand-written guards accepted duplicates, deviating from the frozen proto/realtime schemas they claim to mirror. Enforce both. --- server/src/realtime/guards.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/realtime/guards.ts b/server/src/realtime/guards.ts index af7662b..c0bdcf7 100644 --- a/server/src/realtime/guards.ts +++ b/server/src/realtime/guards.ts @@ -134,6 +134,10 @@ function validateHelloBody(body: Record): ValidationResult Number.isInteger(n) && (n as number) >= 1)) { return fail("malformed", "hello offer: invalid protocol_versions entry"); } + if (new Set(body.protocol_versions).size !== body.protocol_versions.length) { + // Mirrors the schema's uniqueItems: true on protocol_versions. + return fail("malformed", "hello offer: protocol_versions must not contain duplicates"); + } if (body.capabilities !== undefined) { if (!Array.isArray(body.capabilities) || !body.capabilities.every((c) => typeof c === "string" && c.length <= 64)) { return fail("malformed", "hello offer: invalid capabilities"); @@ -269,6 +273,10 @@ function validateSubscribeLikeBody(body: Record, label: string) if (!Array.isArray(body.topics) || !body.topics.every((t) => SUBSCRIBE_TOPICS.includes(t as string))) { return fail("malformed", `${label}: invalid topics`); } + if (new Set(body.topics).size !== body.topics.length) { + // Mirrors the schema's uniqueItems: true on topics. + return fail("malformed", `${label}: topics must not contain duplicates`); + } } return ok(body as unknown as SubscribeBody); } From 04464af000bd05bc50fdc43df0a7d7c6bb70bf87 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:13:32 +0800 Subject: [PATCH 25/60] fix(server): degrade unexpected realtime errors to internal_error Wrap SpaceHub's webSocketMessage dispatch (and webSocketClose) in a top-level guard: an unexpected handler throw now logs in full and answers the frame with error{internal_error, retryable, non-fatal} whose message carries no stack traces or internal paths (SPEC.md section 13), instead of surfacing as an uncaught DO exception. Also remove reply()'s never-taken internal_error log exclusion so every outbound protocol error is logged unconditionally. Covered by a test that injects a handler fault and verifies the connection survives. --- server/src/realtime/space-hub.ts | 30 +++++++++++++++--- server/test/workers/errors.workers.ts | 44 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 server/test/workers/errors.workers.ts diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 8b1ed60..5f757f3 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -203,6 +203,23 @@ export class SpaceHub extends DurableObject { } async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise { + // Top-level exception guard: an unexpected throw anywhere in a message + // handler must degrade to error{internal_error} (SPEC.md section 13: + // retryable, non-fatal, message free of implementation details), never + // kill the DO instance or silently swallow the frame. + try { + this.dispatchMessage(ws, message); + } catch (e) { + this.logAlways("unhandled_error", { message: String(e), stack: (e as Error)?.stack }); + try { + this.reply(ws, "internal_error", "unexpected server error", undefined); + } catch { + // The socket itself may be broken; nothing more to do. + } + } + } + + private dispatchMessage(ws: WebSocket, message: string | ArrayBuffer): void { if (typeof message !== "string") return; // no binary frames in this protocol if (message === "ping" || message === "pong") return; // handled by auto-response; defensive no-op @@ -279,9 +296,14 @@ export class SpaceHub extends DurableObject { } async webSocketClose(): Promise { - // Interest leases and the event log are keyed by device/space, not by - // connection (SPEC.md section 7) — nothing to clean up on close beyond - // letting the socket go. + try { + // Interest leases and the event log are keyed by device/space, not by + // connection (SPEC.md section 7) — nothing to clean up on close beyond + // letting the socket go. The guard exists so any future cleanup added + // here degrades to a log line instead of an uncaught DO exception. + } catch (e) { + this.logAlways("unhandled_error", { message: String(e), phase: "webSocketClose" }); + } } async webSocketError(_ws: WebSocket, error: unknown): Promise { @@ -852,7 +874,7 @@ export class SpaceHub extends DurableObject { private reply(ws: WebSocket, code: ErrorCode, message: string, inReplyTo: string | undefined): void { this.sendRaw(ws, makeStandardError(code, message, inReplyTo)); - if (code !== "internal_error") this.logAlways("protocol_error", { code, message }); + this.logAlways("protocol_error", { code, message }); } private logHotPath(event: string, data: Record): void { diff --git a/server/test/workers/errors.workers.ts b/server/test/workers/errors.workers.ts new file mode 100644 index 0000000..54182ad --- /dev/null +++ b/server/test/workers/errors.workers.ts @@ -0,0 +1,44 @@ +// Top-level exception guard: an unexpected throw inside any message handler +// must degrade to error{internal_error} (retryable, non-fatal, no internal +// details leaked per SPEC.md section 13) with the connection left usable. +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, nextId, subscribe } from "./helpers"; + +describe("internal_error degradation", () => { + it("an injected handler exception yields error{internal_error} without closing the connection or leaking details", async () => { + const { spaceId, viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + + // Inject a fault into the live DO instance (same isolate as the open + // connection in vitest-pool-workers): the next subscribe throws. + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (instance: SpaceHub) => { + (instance as any).handleSubscribe = () => { + throw new Error("boom /internal/secret/path/space-hub.ts:123"); + }; + }); + + client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: {} }); + const err = await client.recv(); + expect(err.type).toBe("error"); + expect(err.body.code).toBe("internal_error"); + expect(err.body.retryable).toBe(true); + expect(err.body.fatal).toBe(false); + expect(err.body.message).not.toContain("boom"); + expect(err.body.message).not.toContain("space-hub"); + + // Remove the injected fault (restores the prototype method) and verify + // the same connection still works — internal_error is non-fatal. + await runInDurableObject(stub, async (instance: SpaceHub) => { + delete (instance as any).handleSubscribe; + }); + const ack = await subscribe(client); + expect(ack.type).toBe("ack"); + expect(ack.body.lease.expires_at).toBeGreaterThan(Date.now()); + + client.close(); + }); +}); From 05fc7844a2c7113802676796f7629b78cabf7834 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:13:57 +0800 Subject: [PATCH 26/60] feat(server): sync interest state to sources on connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC.md section 7's throttle/resume_rate are pure edge triggers, so a source that was offline when the space's lease count crossed its 1<->0 edge would stay stuck at its last-known rate forever. Immediately after a source's hello accept, SpaceHub now unicasts the CURRENT interest state as command{origin:server} (throttle when no unexpired lease exists, resume_rate otherwise) with a fresh command_id — no new message type and zero protocol schema changes; flagged in-code as a v1.1 clarification candidate per the protocol owner's ruling. Tests cover both the initial throttle and the reconnect-after-missed-edge resume_rate, and the shared helpers drain the new post-hello command. --- server/src/realtime/space-hub.ts | 31 +++++++++++++++++++- server/test/workers/broadcast.workers.ts | 8 +++--- server/test/workers/handshake.workers.ts | 8 +++--- server/test/workers/helpers.ts | 19 +++++++++++-- server/test/workers/lease.workers.ts | 33 ++++++++++++++++++++-- server/test/workers/reliability.workers.ts | 16 +++++------ 6 files changed, 92 insertions(+), 23 deletions(-) diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 5f757f3..22a2e69 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -417,7 +417,19 @@ export class SpaceHub extends DurableObject { heartbeat_interval_ms: HEARTBEAT_INTERVAL_MS, }); - if (att.role === "source") this.drainPendingCommands(ws, att.deviceId); + if (att.role === "source") { + // Interest-state sync on (re)connect. SPEC.md section 7's throttle/ + // resume_rate are pure edge triggers, so a source that was offline + // when the space's lease count crossed an edge would otherwise stay + // stuck at its last-known rate forever (e.g. throttled while viewers + // are actively watching). Immediately after the accept, tell this + // connection the CURRENT interest state. This adds no new message + // type or schema change — it reuses command{origin:server} with a + // fresh command_id — and is a v1.1 protocol clarification candidate + // (ruled by the protocol owner; see the handoff notes). + this.sendCurrentRateState(ws); + this.drainPendingCommands(ws, att.deviceId); + } this.reconcileLeaseEdge(); } @@ -804,6 +816,23 @@ export class SpaceHub extends DurableObject { } } + /** Unicast the space's CURRENT interest state to one freshly-connected + * source (see the call site in handlePreHello for why edges alone are + * not enough). Counts only unexpired leases, same rule as + * reconcileLeaseEdge. */ + private sendCurrentRateState(ws: WebSocket): void { + const active = this.ctx.storage.sql + .exec<{ n: number }>("SELECT COUNT(*) as n FROM leases WHERE expires_at > ?", Date.now()) + .toArray()[0].n; + const body: CommandBody = { + command_id: crypto.randomUUID(), + origin: "server", + action: active > 0 ? "resume_rate" : "throttle", + ttl_ms: 60_000, + }; + this.send(ws, "command", body); + } + // ---- command relay (viewer-issued pause/resume/stop/run_now) ---- private handleCommand(ws: WebSocket, att: ConnAttachment, body: CommandBody, envelopeTs: number, envelopeId: string): void { diff --git a/server/test/workers/broadcast.workers.ts b/server/test/workers/broadcast.workers.ts index 4339273..87f923c 100644 --- a/server/test/workers/broadcast.workers.ts +++ b/server/test/workers/broadcast.workers.ts @@ -5,7 +5,7 @@ import { env, runInDurableObject } from "cloudflare:test"; import { describe, expect, it } from "vitest"; import type { SpaceHub } from "../../src/realtime/space-hub.ts"; -import { bootstrapSpace, connect, helloOffer, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; function tableCounts(state: any) { const tables = ["event_log", "dedup", "tasks", "messages", "automations", "leases", "push_outbox", "space_meta"]; @@ -20,7 +20,7 @@ describe("metric.frame", () => { it("never writes SQLite and never advances space_revision", async () => { const { spaceId, source, viewer } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const viewerClient = await connect(viewer.token); await helloOffer(viewerClient, viewer.device_id, "viewer"); @@ -52,7 +52,7 @@ describe("metric.frame", () => { const viewer2 = await inviteAndJoin("viewer"); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const v1 = await connect(viewer.token); await helloOffer(v1, viewer.device_id, "viewer"); @@ -115,7 +115,7 @@ describe("chunked snapshot", () => { }); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const viewerClient = await connect(viewer.token); await helloOffer(viewerClient, viewer.device_id, "viewer"); diff --git a/server/test/workers/handshake.workers.ts b/server/test/workers/handshake.workers.ts index 1e7fc23..bc9a2e0 100644 --- a/server/test/workers/handshake.workers.ts +++ b/server/test/workers/handshake.workers.ts @@ -1,6 +1,6 @@ // Required coverage #8 (connection gating), #9 (supersede). import { describe, expect, it } from "vitest"; -import { bootstrapSpace, connect, helloOffer, nextId, resume, subscribe } from "./helpers"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, subscribe } from "./helpers"; describe("connection gating", () => { it("any frame before hello offer gets hello_required and the connection closes", async () => { @@ -31,7 +31,7 @@ describe("connection gating", () => { it("no live delta reaches a connection before its own resume reply", async () => { const { source, viewer } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const viewerClient = await connect(viewer.token); await helloOffer(viewerClient, viewer.device_id, "viewer"); @@ -65,7 +65,7 @@ describe("connection gating", () => { const { source, viewer } = await bootstrapSpace(); const first = await connect(source.token); - await helloOffer(first, source.device_id, "source"); + await helloSource(first, source.device_id); // An unrelated viewer subscribes throughout — if supersession ever // spuriously toggled the lease count, this connection would see a @@ -80,7 +80,7 @@ describe("connection gating", () => { expect(resumeRate.body.action).toBe("resume_rate"); const second = await connect(source.token); - await helloOffer(second, source.device_id, "source"); + await helloSource(second, source.device_id); const supersededErr = await first.recv(); expect(supersededErr.type).toBe("error"); diff --git a/server/test/workers/helpers.ts b/server/test/workers/helpers.ts index 8bbb6f8..96b5bce 100644 --- a/server/test/workers/helpers.ts +++ b/server/test/workers/helpers.ts @@ -97,7 +97,7 @@ export class WsClient { this.ws.send(text); } - recvRaw(timeoutMs = 3000): Promise { + recvRaw(timeoutMs = 10_000): Promise { const queued = this.queue.shift(); if (queued !== undefined) return Promise.resolve(queued); return new Promise((resolve, reject) => { @@ -118,7 +118,7 @@ export class WsClient { }); } - async recv(timeoutMs = 3000): Promise { + async recv(timeoutMs = 10_000): Promise { return JSON.parse(await this.recvRaw(timeoutMs)); } @@ -134,7 +134,7 @@ export class WsClient { } } - waitForClose(timeoutMs = 3000): Promise<{ code: number; reason: string }> { + waitForClose(timeoutMs = 10_000): Promise<{ code: number; reason: string }> { if (this.closed) return Promise.resolve(this.closed); return new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error("timed out waiting for close")), timeoutMs); @@ -178,6 +178,19 @@ export async function helloOffer(client: WsClient, deviceId: string, role: "sour return client.recv(); } +/** Completes a source's hello AND consumes the interest-state command the + * server sends every source immediately after its accept (throttle when no + * lease is active, resume_rate otherwise). Returns both so tests that care + * about the initial rate state can assert on it. */ +export async function helloSource(client: WsClient, deviceId: string): Promise<{ accept: any; rate: any }> { + const accept = await helloOffer(client, deviceId, "source"); + const rate = await client.recv(); + if (rate.type !== "command" || rate.body.origin !== "server") { + throw new Error(`expected the post-hello interest-state command, got: ${JSON.stringify(rate)}`); + } + return { accept, rate }; +} + export async function subscribe(client: WsClient, topics?: Array<"task" | "metric" | "message">): Promise { client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: topics ? { topics } : {} }); return client.recv(); diff --git a/server/test/workers/lease.workers.ts b/server/test/workers/lease.workers.ts index 500c9ae..304f34c 100644 --- a/server/test/workers/lease.workers.ts +++ b/server/test/workers/lease.workers.ts @@ -5,14 +5,14 @@ // explicitly names unsubscribe as one of exactly two ways a lease ends, // and both paths run through the same reconcileLeaseEdge() call. import { describe, expect, it } from "vitest"; -import { bootstrapSpace, connect, helloOffer, subscribe, unsubscribe } from "./helpers"; +import { bootstrapSpace, connect, helloOffer, helloSource, subscribe, unsubscribe } from "./helpers"; describe("interest lease edges", () => { it("0->1 sends resume_rate; 1->0 sends throttle; a later 0->1 sends resume_rate again", async () => { const { source, viewer, inviteAndJoin } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const v1 = await connect(viewer.token); await helloOffer(v1, viewer.device_id, "viewer"); @@ -47,7 +47,7 @@ describe("interest lease edges", () => { it("a second viewer subscribing while one is already active does not re-fire resume_rate", async () => { const { source, viewer, inviteAndJoin } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const v1 = await connect(viewer.token); await helloOffer(v1, viewer.device_id, "viewer"); @@ -65,4 +65,31 @@ describe("interest lease edges", () => { v1.close(); v2.close(); }); + + it("a source receives the current interest state immediately after hello, so a missed edge cannot strand it", async () => { + const { source, viewer } = await bootstrapSpace(); + + // No lease exists yet: a connecting source is told to throttle. + const first = await connect(source.token); + const initial = await helloSource(first, source.device_id); + expect(initial.rate.type).toBe("command"); + expect(initial.rate.body.origin).toBe("server"); + expect(initial.rate.body.action).toBe("throttle"); + first.close(); + + // A viewer subscribes while the source is OFFLINE — the 0->1 + // resume_rate edge fires with nobody to hear it. + const v = await connect(viewer.token); + await helloOffer(v, viewer.device_id, "viewer"); + await subscribe(v); + + // The reconnecting source is immediately told the CURRENT state + // (resume_rate), not left stuck at its last-known throttle. + const second = await connect(source.token); + const after = await helloSource(second, source.device_id); + expect(after.rate.body.action).toBe("resume_rate"); + + second.close(); + v.close(); + }); }); diff --git a/server/test/workers/reliability.workers.ts b/server/test/workers/reliability.workers.ts index 02f987f..6c9b094 100644 --- a/server/test/workers/reliability.workers.ts +++ b/server/test/workers/reliability.workers.ts @@ -4,13 +4,13 @@ import { env, runInDurableObject } from "cloudflare:test"; import { describe, expect, it } from "vitest"; import type { SpaceHub } from "../../src/realtime/space-hub.ts"; -import { bootstrapSpace, connect, helloOffer, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; describe("reliable event dedup and revision accounting", () => { it("a duplicate device_seq re-acks without applying twice or bumping revision again", async () => { const { source, viewer } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); const viewerClient = await connect(viewer.token); await helloOffer(viewerClient, viewer.device_id, "viewer"); @@ -56,7 +56,7 @@ describe("reliable event dedup and revision accounting", () => { it("event_id in push_outbox is idempotent across a duplicate device_seq retry", async () => { const { spaceId, source } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); await sendTaskEvent(sourceClient, source.device_id, 1); await sendTaskEvent(sourceClient, source.device_id, 1); // duplicate sourceClient.close(); @@ -90,7 +90,7 @@ describe("resume decision table", () => { it("last_revision == current revision yields the explicit empty delta", async () => { const { source, viewer } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); await sendTaskEvent(sourceClient, source.device_id, 1); const client = await connect(viewer.token); @@ -120,7 +120,7 @@ describe("resume decision table", () => { it("0 < last_revision < current, fully retained, yields a catch-up delta (not a snapshot)", async () => { const { source, viewer } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); await sendTaskEvent(sourceClient, source.device_id, 1, { kind: "started" }); await sendTaskEvent(sourceClient, source.device_id, 2, { kind: "progress", percent: 50 }); await sendTaskEvent(sourceClient, source.device_id, 3, { kind: "done" }); @@ -140,7 +140,7 @@ describe("resume decision table", () => { it("a range whose retention was lost falls back to a snapshot instead of an error", async () => { const { spaceId, source, viewer } = await bootstrapSpace(); const sourceClient = await connect(source.token); - await helloOffer(sourceClient, source.device_id, "source"); + await helloSource(sourceClient, source.device_id); await sendTaskEvent(sourceClient, source.device_id, 1); await sendTaskEvent(sourceClient, source.device_id, 2); await sendTaskEvent(sourceClient, source.device_id, 3); @@ -177,7 +177,7 @@ describe("connection identity recovery", () => { it("a fresh connection for the same device, and a fresh stub reference, both observe identical durable state", async () => { const { spaceId, source } = await bootstrapSpace(); const first = await connect(source.token); - await helloOffer(first, source.device_id, "source"); + await helloSource(first, source.device_id); await sendTaskEvent(first, source.device_id, 1, { kind: "started", title: "build" }); first.close(); @@ -186,7 +186,7 @@ describe("connection identity recovery", () => { // device's device_seq sequence — proving identity isn't cached // anywhere but the attachment + storage. const second = await connect(source.token); - const accept = await helloOffer(second, source.device_id, "source"); + const { accept } = await helloSource(second, source.device_id); expect(accept.body.stage).toBe("accept"); await sendTaskEvent(second, source.device_id, 2, { kind: "done" }); second.close(); From 4dd70c825e8e881d7c2de07abdc6ade66de478c5 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 11:14:18 +0800 Subject: [PATCH 27/60] fix(server): share metric staleness baseline across connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC.md section 12's discard-stale-samples rule was keyed off a per-WebSocket rate-limiter map, so a reconnecting or second source could replay an older sample past a fresher one. The baseline now lives in the space-level metricsCache (keyed by metric_id alone, same lifetime and best-effort semantics as the rest of that cache — reset after eviction is acceptable per section 6.2). Test interleaves two source connections on one metric_id and asserts the older ts is dropped and a genuinely fresher one still flows. --- server/src/realtime/space-hub.ts | 15 ++++--- server/test/workers/broadcast.workers.ts | 51 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 22a2e69..3ebb5a9 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -68,7 +68,6 @@ import type { interface RateLimiterState { frameTimestamps: number[]; - perMetricLastTs: Map; } export class SpaceHub extends DurableObject { @@ -708,9 +707,15 @@ export class SpaceHub extends DurableObject { const accepted: MetricSample[] = []; for (const sample of body.metrics) { - const lastTs = limiter.perMetricLastTs.get(sample.metric_id) ?? 0; - if (sample.ts <= lastTs) continue; // SPEC.md section 12: discard stale/duplicate - limiter.perMetricLastTs.set(sample.metric_id, sample.ts); + // SPEC.md section 12: discard any sample whose ts <= the last-applied + // ts for that metric_id. The baseline is SPACE-level (metricsCache, + // keyed by metric_id alone), NOT per connection: staleness must be + // judged across connections and across devices, or a reconnecting or + // second source could replay an older sample past a fresher one. + // Same lifetime as the rest of metricsCache — reset after eviction + // is acceptable (the whole cache is best-effort, section 6.2). + const cached = this.metricsCache.get(sample.metric_id); + if (cached && sample.ts <= cached.ts) continue; this.metricsCache.set(sample.metric_id, sample); accepted.push(sample); } @@ -733,7 +738,7 @@ export class SpaceHub extends DurableObject { private rateLimiterFor(ws: WebSocket): RateLimiterState { let state = this.rateLimiters.get(ws); if (!state) { - state = { frameTimestamps: [], perMetricLastTs: new Map() }; + state = { frameTimestamps: [] }; this.rateLimiters.set(ws, state); } return state; diff --git a/server/test/workers/broadcast.workers.ts b/server/test/workers/broadcast.workers.ts index 87f923c..8881098 100644 --- a/server/test/workers/broadcast.workers.ts +++ b/server/test/workers/broadcast.workers.ts @@ -79,6 +79,57 @@ describe("metric.frame", () => { v1.close(); v2.close(); }); + + it("staleness baseline is space-level: an older ts from a second source connection is dropped", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + const source2 = await inviteAndJoin("source"); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient, ["metric"]); + await resume(viewerClient, 0); + + const s1 = await connect(source.token); + await helloSource(s1, source.device_id); + const s2 = await connect(source2.token); + await helloSource(s2, source2.device_id); + + const t = Date.now(); + // Source 1 reports cpu.load at t+1000. + s1.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics: [{ metric_id: "cpu.load", value: "80", ts: t + 1000 }] }, + }); + const f1 = await viewerClient.recv(); + expect(f1.body.metrics[0].value).toBe("80"); + + // Source 2 (different device, different connection) reports the SAME + // metric_id with an OLDER ts — SPEC.md section 12's discard rule must + // apply across connections/devices, so the viewer sees nothing. + s2.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source2.device_id, metrics: [{ metric_id: "cpu.load", value: "10", ts: t + 500 }] }, + }); + await viewerClient.expectSilence(200); + + // A genuinely fresher sample from source 2 goes through. + s2.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source2.device_id, metrics: [{ metric_id: "cpu.load", value: "55", ts: t + 2000 }] }, + }); + const f2 = await viewerClient.recv(); + expect(f2.body.metrics[0].value).toBe("55"); + + s1.close(); + s2.close(); + viewerClient.close(); + }); }); describe("chunked snapshot", () => { From 4ec4a0c0dff681f08b745eec4287ceca61db8fd5 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:05:26 +0800 Subject: [PATCH 28/60] fix(server): harden v3 automation control plane Four rulings from the adversarial review, all on the /v3/automations surface: - Idempotency-Key is now bound to request content: mintConfigEvent stores a canonical-JSON fingerprint of (kind, automation_id, automation) and compares it on replay; the same key reused for a different operation returns 409 instead of silently replaying the first operation's result. - http_idempotency growth is bounded lazily on its own write path (no alarm): entries older than 24h are dropped and the table is capped at the 500 most recent rows; documented in the cost model. - Role matrix finalized per the protocol owner (aligned with "watcher schedules are editable from every device, creation only from trusted devices"): POST = owner/admin; PATCH = owner/admin/viewer (viewer editing schedules is intentional); DELETE now also allows viewer, matching /v2 canView. - DELETE of a nonexistent automation returns 404 before minting, so a no-op removal cannot burn a revision (same pattern as PATCH); the /v3 HTTP handlers also gained the internal_error guard (500 with no internal details, full server-side log). Tests cover the 409 conflict, viewer PATCH/DELETE success + POST 403, and the 404-without-revision-burn invariant. --- docs/design/realtime-server.md | 16 ++-- server/src/adapters/workers.ts | 95 +++++++++++++++------ server/src/realtime/space-hub.ts | 41 ++++++++- server/test/workers/config-event.workers.ts | 94 ++++++++++++++++++++ 4 files changed, 210 insertions(+), 36 deletions(-) diff --git a/docs/design/realtime-server.md b/docs/design/realtime-server.md index 1a2c248..9d8c3e9 100644 --- a/docs/design/realtime-server.md +++ b/docs/design/realtime-server.md @@ -87,11 +87,17 @@ counts against the attached SQLite database). deduplication guarantee must hold regardless of how old a retried `device_seq` is, so it trades unbounded (but tiny — one row per historical event, `device_id`+`device_seq`+`revision`, all fixed-width) - storage growth for correctness. `push_outbox` and `pending_commands` are - the two "queue" tables in this design; both need a consumer (a future - Alarm/Queue-based push worker, explicitly out of scope for this phase) to - actually shrink — today they only grow, which is an accepted, called-out - gap (see the handoff's "known gaps" section). + storage growth for correctness. `http_idempotency` is bounded by lazy + cleanup on its own write path (no alarm): every insert also deletes + entries older than 24 hours and caps the table at the 500 most recent + rows — a control-plane retry arriving after both windows simply + re-executes as a fresh request, which is state-level safe because + automation upsert/delete are idempotent operations even without the + key. `push_outbox` and `pending_commands` are the two "queue" tables in + this design; both need a consumer (a future Alarm/Queue-based push + worker, explicitly out of scope for this phase) to actually shrink — + today they only grow, which is an accepted, called-out gap (see the + handoff's "known gaps" section). ## Hibernation diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index 06c476a..f069316 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -544,6 +544,29 @@ function parseAutomationUpsert(body: any): { name: string; executor_kind: Automa return { name: String(body.name).slice(0, 256), executor_kind: kind, every_seconds: Math.max(1, everySeconds | 0) }; } +// /v3/automations role matrix (protocol-owner ruling, aligned with the +// existing product decision "watcher schedules are editable from every +// device; creation only from trusted devices" — same split /v2 makes +// between canEmit-guarded POST and canView-guarded PATCH/DELETE): +// POST (create/upsert) -> owner/admin only +// PATCH (pause/resume, schedule.every_seconds) -> owner/admin/viewer +// (viewer editing the schedule is INTENTIONAL, not an oversight) +// DELETE -> owner/admin/viewer (aligned with /v2 canView's delete) +// A conflict result from mintConfigEvent (same Idempotency-Key, different +// operation content) maps to 409. + +function mintResultResponse(c: any, result: { revision: number; automation: AutomationState | null; conflict?: boolean }) { + if (result.conflict) { + return c.json({ error: "idempotency key was already used for a different operation" }, 409); + } + return c.json({ revision: result.revision, automation: result.automation }); +} + +const internalError = (c: any, e: unknown) => { + console.log(JSON.stringify({ level: "error", event: "v3_http_unhandled", message: String(e) })); + return c.json({ error: "internal error" }, 500); +}; + app.post("/v3/automations", async (c: any) => { const role = c.get("role") as Role; if (!["admin", "owner"].includes(role)) return c.json({ error: "forbidden" }, 403); @@ -558,42 +581,60 @@ app.post("/v3/automations", async (c: any) => { state: body.state === "paused" ? "paused" : "active", }; const idempotencyKey = c.req.header("idempotency-key") ?? null; - const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); - const result = await stub.mintConfigEvent(idempotencyKey, { - kind: "automation.upserted", - automation_id: automation.automation_id, - automation, - }); - return c.json(result); + try { + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const result = await stub.mintConfigEvent(idempotencyKey, { + kind: "automation.upserted", + automation_id: automation.automation_id, + automation, + }); + return mintResultResponse(c, result); + } catch (e) { + return internalError(c, e); + } }); app.patch("/v3/automations/:id", async (c: any) => { const role = c.get("role") as Role; if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); - const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); - const id = c.req.param("id"); - const existing = (await stub.automationsSnapshot()).find((a: AutomationState) => a.automation_id === id); - if (!existing) return c.json({ error: "not found" }, 404); - const body = await c.req.json().catch(() => null); - const next: AutomationState = { - ...existing, - ...(body?.state === "active" || body?.state === "paused" ? { state: body.state } : {}), - ...(body?.schedule?.every_seconds !== undefined - ? { schedule: { kind: "interval" as const, every_seconds: Math.max(1, Number(body.schedule.every_seconds) | 0) } } - : {}), - }; - const idempotencyKey = c.req.header("idempotency-key") ?? null; - const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.upserted", automation_id: id, automation: next }); - return c.json(result); + try { + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const id = c.req.param("id"); + const existing = (await stub.automationsSnapshot()).find((a: AutomationState) => a.automation_id === id); + if (!existing) return c.json({ error: "not found" }, 404); + const body = await c.req.json().catch(() => null); + const next: AutomationState = { + ...existing, + ...(body?.state === "active" || body?.state === "paused" ? { state: body.state } : {}), + ...(body?.schedule?.every_seconds !== undefined + ? { schedule: { kind: "interval" as const, every_seconds: Math.max(1, Number(body.schedule.every_seconds) | 0) } } + : {}), + }; + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.upserted", automation_id: id, automation: next }); + return mintResultResponse(c, result); + } catch (e) { + return internalError(c, e); + } }); app.delete("/v3/automations/:id", async (c: any) => { const role = c.get("role") as Role; - if (!["admin", "owner"].includes(role)) return c.json({ error: "forbidden" }, 403); - const idempotencyKey = c.req.header("idempotency-key") ?? null; - const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); - const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.removed", automation_id: c.req.param("id") }); - return c.json(result); + if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); + try { + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const id = c.req.param("id"); + // 404 before minting: deleting a nonexistent automation must not burn a + // revision on a no-op config.event (same existence-check pattern PATCH + // uses). + const existing = (await stub.automationsSnapshot()).find((a: AutomationState) => a.automation_id === id); + if (!existing) return c.json({ error: "not found" }, 404); + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.removed", automation_id: id }); + return mintResultResponse(c, result); + } catch (e) { + return internalError(c, e); + } }); export default app; diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 3ebb5a9..d6e86ad 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -156,6 +156,7 @@ export class SpaceHub extends DurableObject { ); CREATE TABLE IF NOT EXISTS http_idempotency ( idempotency_key TEXT PRIMARY KEY, + fingerprint TEXT NOT NULL, revision INTEGER NOT NULL, created_at INTEGER NOT NULL ); @@ -313,16 +314,36 @@ export class SpaceHub extends DurableObject { /** Mints (or replays, if idempotencyKey was already seen) a config.event * for an automation upsert/removal. Single synchronous transaction, same - * discipline as applyReliableEvent — see the class-level comment. */ + * discipline as applyReliableEvent — see the class-level comment. + * + * The idempotency key is bound to the request's CONTENT, not just its + * key string: the stored fingerprint is the canonical JSON of + * (kind, automation_id, automation), compared verbatim on a replay. + * (Exact string comparison is strictly stronger than a hash and stays + * synchronous — crypto.subtle.digest is async and would break the + * single-transaction discipline.) A key reused for a DIFFERENT + * operation returns { conflict: true } and mints nothing, instead of + * silently replaying the first operation's result. */ mintConfigEvent( idempotencyKey: string | null, payload: { kind: ConfigEventKind; automation_id: string; automation?: AutomationState }, - ): { revision: number; automation: AutomationState | null } { + ): { revision: number; automation: AutomationState | null; conflict?: boolean } { + const fingerprint = JSON.stringify({ + kind: payload.kind, + automation_id: payload.automation_id, + automation: payload.automation ?? null, + }); if (idempotencyKey) { const existing = this.ctx.storage.sql - .exec<{ revision: number }>("SELECT revision FROM http_idempotency WHERE idempotency_key = ?", idempotencyKey) + .exec<{ revision: number; fingerprint: string }>( + "SELECT revision, fingerprint FROM http_idempotency WHERE idempotency_key = ?", + idempotencyKey, + ) .toArray()[0]; if (existing) { + if (existing.fingerprint !== fingerprint) { + return { revision: existing.revision, automation: null, conflict: true }; + } return { revision: existing.revision, automation: this.readAutomation(payload.automation_id) }; } } @@ -347,11 +368,23 @@ export class SpaceHub extends DurableObject { this.setRevision(revision); if (idempotencyKey) { this.ctx.storage.sql.exec( - `INSERT INTO http_idempotency (idempotency_key, revision, created_at) VALUES (?, ?, ?)`, + `INSERT INTO http_idempotency (idempotency_key, fingerprint, revision, created_at) VALUES (?, ?, ?, ?)`, idempotencyKey, + fingerprint, revision, occurredAt, ); + // Bounded growth, cleaned lazily on the write path (no alarm): drop + // entries older than 24h, and cap the table at the most recent 500 + // rows. A retry arriving after both windows re-executes as a fresh + // request — acceptable, since config.event minting is idempotent at + // the state level (upsert/delete) even without the key. + this.ctx.storage.sql.exec(`DELETE FROM http_idempotency WHERE created_at < ?`, occurredAt - 24 * 3600 * 1000); + this.ctx.storage.sql.exec( + `DELETE FROM http_idempotency WHERE idempotency_key NOT IN ( + SELECT idempotency_key FROM http_idempotency ORDER BY created_at DESC LIMIT 500 + )`, + ); } this.pruneEventLog(revision); diff --git a/server/test/workers/config-event.workers.ts b/server/test/workers/config-event.workers.ts index b5871e3..00b1cfe 100644 --- a/server/test/workers/config-event.workers.ts +++ b/server/test/workers/config-event.workers.ts @@ -60,4 +60,98 @@ describe("/v3/automations control plane", () => { }); expect(res.status).toBe(403); }); + + it("reusing an Idempotency-Key for a DIFFERENT operation returns 409 and mints nothing", async () => { + const { ownerToken } = await bootstrapSpace(); + const headers = { + authorization: `Bearer ${ownerToken}`, + "content-type": "application/json", + "idempotency-key": "req-reused", + }; + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ automation_id: "auto-x", name: "first op", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { revision: number }; + + // Same key, different content — must be rejected, NOT silently + // replayed as the first operation's result. + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ automation_id: "auto-y", name: "second op", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(res2.status).toBe(409); + + // Nothing was minted for the conflicting request: a fresh operation + // lands at exactly body1.revision + 1. + const res3 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: "auto-z", name: "third op", executor_kind: "script", schedule: { every_seconds: 30 } }), + }); + const body3 = (await res3.json()) as { revision: number }; + expect(body3.revision).toBe(body1.revision + 1); + }); + + it("role matrix: viewer may PATCH and DELETE but not POST", async () => { + const { viewer, ownerToken } = await bootstrapSpace(); + // Owner creates two automations for the viewer to act on. + for (const id of ["auto-p", "auto-d"]) { + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: id, name: id, executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(res.status).toBe(200); + } + const viewerHeaders = { authorization: `Bearer ${viewer.token}`, "content-type": "application/json" }; + + // Viewer PATCH (pause + reschedule) succeeds — intentional design: + // watcher schedules are editable from every device. + const patchRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-p`, { + method: "PATCH", + headers: viewerHeaders, + body: JSON.stringify({ state: "paused", schedule: { every_seconds: 120 } }), + }); + expect(patchRes.status).toBe(200); + const patched = (await patchRes.json()) as { automation: { state: string; schedule: { every_seconds: number } } }; + expect(patched.automation.state).toBe("paused"); + expect(patched.automation.schedule.every_seconds).toBe(120); + + // Viewer DELETE succeeds (aligned with /v2 canView's delete). + const deleteRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-d`, { method: "DELETE", headers: viewerHeaders }); + expect(deleteRes.status).toBe(200); + + // Viewer POST (creation) stays forbidden — creation is trusted-device + // only. + const postRes = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: viewerHeaders, + body: JSON.stringify({ name: "nope", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(postRes.status).toBe(403); + }); + + it("DELETE of a nonexistent automation returns 404 without burning a revision", async () => { + const { ownerToken } = await bootstrapSpace(); + const headers = { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }; + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ automation_id: "auto-real", name: "real", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + const body1 = (await res1.json()) as { revision: number }; + + const missingRes = await SELF.fetch(`${ORIGIN}/v3/automations/no-such-automation`, { method: "DELETE", headers }); + expect(missingRes.status).toBe(404); + + // The 404 minted no config.event: the next real operation lands at + // exactly body1.revision + 1. + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations/auto-real`, { method: "DELETE", headers }); + const body2 = (await res2.json()) as { revision: number }; + expect(body2.revision).toBe(body1.revision + 1); + }); }); From d13eee3a6e139286d14e8e50ca25726471a154d9 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:05:34 +0800 Subject: [PATCH 29/60] docs(server): record intentional protocol omissions Note in the cost model why the 10 s handshake timeout (SPEC.md section 9.1, SHOULD) is deliberately not enforced server-side (it would require the per-connection timer machinery this design excludes to stay hibernation-friendly, and the exposure of an offer-less connection is bounded), and why the advisory sequence_gap error (section 5.1, MAY) is not emitted. --- docs/design/realtime-server.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/design/realtime-server.md b/docs/design/realtime-server.md index 9d8c3e9..ad0169e 100644 --- a/docs/design/realtime-server.md +++ b/docs/design/realtime-server.md @@ -135,3 +135,19 @@ Net effect: a space's ongoing cost while every connection is idle (no task/message/metric traffic, no lease churn) is exactly zero DO invocations — the state sits in SQLite and the connections sit hibernated until either side has something to say. + +## Intentional omissions + +- **The 10 s handshake timeout (SPEC.md section 9.1, a SHOULD) is not + implemented.** Enforcing it server-side would require a per-connection + timer — exactly the `setTimeout`/alarm machinery this design excludes to + stay hibernation-friendly. The exposure is bounded: a connection that + never sends an offer holds no lease, receives no deltas, writes nothing, + and costs only the runtime's parked-socket overhead; the client side of + the recommendation (close and reconnect if no accept arrives within + 10 s) is where the liveness benefit actually lives, and belongs to the + client implementations. +- **`sequence_gap` (SPEC.md section 5.1, a MAY) is not emitted.** The + event that did arrive is applied either way; the advisory adds a + per-event bookkeeping read ("what was this device's previous seq") + purely to produce a non-actionable warning. From a92d9c9580caf6bcd1754b866bef888a816b32b0 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:06:52 +0800 Subject: [PATCH 30/60] fix(apple): scope connection resources to their run generation The runGeneration token previously protected only the runTask handle: a stale connectOnce() unwinding after a stop()/start() cycle on the same RealtimeClient instance would run its deferred teardownConnection() and cancel the NEWER generation's socket, lease-renewal task, and watchdog. Unreachable in the app today (it rebuilds the client each cycle) but a hole in the class's own public contract. Teardown of the shared resources now requires the requesting generation to still be current (ConnectionTeardownPolicy, pure and unit-tested); a stale generation limits itself to closing the socket it created and holds by reference. scheduleLeaseRenewal and startHeartbeat gain the same guard so a stale dispatch cannot replace the current generation's tasks either. Adds a stop-then-start same-instance regression test against a refused local port (no real network). --- .../SitrepKit/Realtime/RealtimeClient.swift | 68 ++++++++++++---- .../Realtime/RealtimeGenerationTests.swift | 78 +++++++++++++++++++ 2 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift index 797cd86..3852859 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift @@ -104,9 +104,12 @@ public actor RealtimeClient { /// Number of consecutive malformed frames after which the connection is /// torn down and re-established (circuit breaker). private let malformedFrameCircuitBreaker = 5 - /// Distinguishes which `runLoop` invocation may clear `runTask` on - /// exit, so a stale loop finishing after a stop()/start() cycle can't - /// clobber the newer loop's handle. + /// Distinguishes which `runLoop` invocation currently owns the + /// connection resources (`runTask`, `socket`, `leaseRenewalTask`, + /// `watchdogTask`). A stale generation — a loop still unwinding after a + /// stop()/start() cycle on the same instance — must neither clear the + /// newer loop's task handle nor tear down or replace its connection + /// resources (see `ConnectionTeardownPolicy`). private var runGeneration = 0 private let phaseContinuation: AsyncStream.Continuation @@ -178,7 +181,7 @@ public actor RealtimeClient { var terminal = false while !Task.isCancelled { do { - try await connectOnce() + try await connectOnce(generation: generation) } catch is CancellationError { break } catch { @@ -217,7 +220,7 @@ public actor RealtimeClient { // MARK: - One connection's lifetime - private func connectOnce() async throws { + private func connectOnce(generation: Int) async throws { phase = .connecting consecutiveMalformedFrames = 0 var request = URLRequest(url: configuration.url) @@ -227,11 +230,15 @@ public actor RealtimeClient { let socket = URLSession.shared.webSocketTask(with: request) self.socket = socket socket.resume() - defer { teardownConnection() } + // A stale generation's deferred teardown (this connectOnce resuming + // from suspension after a stop()/start() cycle) must not cancel the + // NEWER generation's socket/lease/watchdog. It still closes its own + // socket, which it holds by reference. + defer { teardownConnection(requestedBy: generation, ownSocket: socket) } phase = .handshaking let accept = try await performHandshake(on: socket) - startHeartbeat(on: socket, intervalMs: accept.heartbeatIntervalMs) + startHeartbeat(on: socket, intervalMs: accept.heartbeatIntervalMs, generation: generation) let subscribeID = newEnvelopeID() try await sendFrame( @@ -248,7 +255,7 @@ public actor RealtimeClient { throw RealtimeClientError.protocolViolation("expected subscribe ack with lease, got \(ackFrame)") } phase = .subscribed - scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket) + scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket, generation: generation) let lastRevision = gate.state.revision gate.beganResume(lastRevision: lastRevision) @@ -258,11 +265,20 @@ public actor RealtimeClient { while !Task.isCancelled { let frame = try await receiveFrame(on: socket) - try await dispatch(frame, on: socket) + try await dispatch(frame, on: socket, generation: generation) } } - private func teardownConnection() { + /// Tears down the shared connection resources — but only when the + /// requesting generation still owns them (`requestedBy: nil` is the + /// unconditional form used by `stop()`). A stale generation's deferred + /// call limits itself to closing the socket it created (`ownSocket`), + /// leaving the newer generation's resources untouched. + private func teardownConnection(requestedBy generation: Int? = nil, ownSocket: URLSessionWebSocketTask? = nil) { + guard ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: generation, current: runGeneration) else { + ownSocket?.cancel(with: .normalClosure, reason: nil) + return + } socket?.cancel(with: .normalClosure, reason: nil) socket = nil leaseRenewalTask?.cancel() @@ -299,7 +315,7 @@ public actor RealtimeClient { // MARK: - Frame dispatch (post-handshake, post-subscribe) - private func dispatch(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask) async throws { + private func dispatch(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask, generation: Int) async throws { // §6.2: while a chunked snapshot is in flight, only further snapshot // chunks (and ping/pong, which never reach dispatch) are legal on // this connection. Anything else is a malformed sequence: drop the @@ -344,7 +360,7 @@ public actor RealtimeClient { stateContinuation.yield(gate.state) case .ack(let env): if let lease = env.body.lease { - scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket) + scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket, generation: generation) } case .error(let env): try await handleError(env.body, on: socket) @@ -391,7 +407,11 @@ public actor RealtimeClient { /// Renews at 1/3 of the window remaining before `expiresAt`, as /// instructed. Rescheduled every time we learn a fresh `expiresAt` (the /// initial subscribe ack, or any subsequent renew ack) — see `dispatch`. - private func scheduleLeaseRenewal(expiresAt: Int, on socket: URLSessionWebSocketTask) { + /// A stale generation (its connection already superseded by a + /// stop()/start() cycle) must not replace the current generation's + /// renewal task. + private func scheduleLeaseRenewal(expiresAt: Int, on socket: URLSessionWebSocketTask, generation: Int) { + guard generation == runGeneration else { return } leaseRenewalTask?.cancel() let window = max(expiresAt - nowMs(), 0) let fireAfterMs = window - window / 3 @@ -408,7 +428,8 @@ public actor RealtimeClient { // MARK: - Heartbeat (§9.3) - private func startHeartbeat(on socket: URLSessionWebSocketTask, intervalMs: Int) { + private func startHeartbeat(on socket: URLSessionWebSocketTask, intervalMs: Int, generation: Int) { + guard generation == runGeneration else { return } watchdogTask?.cancel() lastInboundAt = nowMs() watchdogTask = Task { [weak self] in @@ -506,6 +527,25 @@ public actor RealtimeClient { private func newEnvelopeID() -> String { UUID().uuidString } } +/// Pure decision logic for generation-scoped teardown of the shared +/// connection resources (`socket`/`leaseRenewalTask`/`watchdogTask`), +/// extracted so the guard itself is directly unit-testable without a live +/// transport (`RealtimeGenerationTests`). +enum ConnectionTeardownPolicy { + /// - `requestedBy == nil`: the unconditional form (`stop()`); always + /// tears down. + /// - `requestedBy == current`: the owning generation's own teardown; + /// proceeds. + /// - `requestedBy < current`: a stale generation unwinding after a + /// stop()/start() cycle; it must NOT touch the newer generation's + /// shared resources (it may still close the socket it created and + /// holds by reference). + static func shouldTearDownShared(requestedBy generation: Int?, current: Int) -> Bool { + guard let generation else { return true } + return generation == current + } +} + public extension RealtimeClient.Phase { /// Whether an HTTP snapshot result may overwrite the four reliable /// collections (tasks/metrics/messages-events/automations). While the diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift new file mode 100644 index 0000000..74883c4 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift @@ -0,0 +1,78 @@ +import XCTest +@testable import SitrepKit + +/// Generation-scoped connection-resource ownership: a stale run-loop +/// generation unwinding after a stop()/start() cycle on the SAME +/// RealtimeClient instance must not tear down or replace the newer +/// generation's socket/lease/watchdog resources. No real network is used: +/// the guard itself is pure logic, and the behavioral test connects to a +/// local port that refuses immediately. +final class RealtimeGenerationTests: XCTestCase { + // MARK: - Pure guard matrix + + func testTeardownPolicyMatrix() { + // stop() — unconditional teardown regardless of current generation. + XCTAssertTrue(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: nil, current: 1)) + XCTAssertTrue(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: nil, current: 7)) + + // The owning generation tears down its own shared resources. + XCTAssertTrue(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: 3, current: 3)) + + // A stale generation (superseded by stop()+start()) must NOT touch + // the newer generation's shared resources. + XCTAssertFalse(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: 1, current: 2)) + XCTAssertFalse(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: 2, current: 5)) + } + + // MARK: - stop()/start() on the same instance + + /// The public-contract regression: after stop(), the SAME instance must + /// be restartable — the stale generation's deferred teardown (still + /// unwinding from its suspended connect attempt) must not kill the new + /// generation's run loop or connection attempt. Uses a closed local + /// port, so each connect attempt fails fast without real network. + func testStopThenStartOnSameInstanceRestarts() async throws { + let configuration = RealtimeClient.Configuration( + url: URL(string: "ws://127.0.0.1:1/v3/realtime")!, token: nil, deviceID: "iphone-test-01") + let client = RealtimeClient(configuration: configuration) + + // Generation 1 starts and leaves idle (connecting or already failed). + await client.start() + try await waitUntil("first start leaves .idle") { + await client.currentPhase != .idle + } + + // Stop, then immediately start again on the same instance — the + // window in which generation 1's connectOnce is still unwinding. + await client.stop() + await client.start() + + // Generation 2 must be alive and progressing: it leaves .idle and, + // because the port refuses, reaches .failed — proving the new run + // loop owns the cycle and was not torn down by generation 1's + // deferred cleanup. + try await waitUntil("second start leaves .idle") { + await client.currentPhase != .idle + } + try await waitUntil("second generation reaches .failed on a refused port") { + if case .failed = await client.currentPhase { return true } + return false + } + + await client.stop() + let final = await client.currentPhase + XCTAssertEqual(final, .idle) + } + + private func waitUntil( + _ what: String, timeout: Double = 5, + _ condition: @escaping () async -> Bool + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if await condition() { return } + try await Task.sleep(for: .milliseconds(25)) + } + XCTFail("timed out waiting for: \(what)") + } +} From fdac903ea77dab8253c4c8d3e6941e8f91c04483 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:08:39 +0800 Subject: [PATCH 31/60] fix(apple): decide http overwrites on the actor's authoritative phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The phases and states AsyncStreams are consumed by two independent MainActor tasks, so the cached connectionPhase can momentarily lag the state a delta just applied — a narrow window in which refresh() could read a stale non-live phase and overwrite live delta state. refresh() now queries the RealtimeClient actor's own currentPhase at the write point, removing the dual-stream ordering dependency, and adds a post-write reconciliation: if the connection went live between the check and the writes, the actor's authoritative SpaceState is re-applied so delta-derived data wins; if it flips live after that, the states-stream delivery accompanying the flip is still queued and lands last. The cached connectionPhase remains UI-only (the sync status strip). --- apple/SitrepApp/Sitrep/SitrepApp.swift | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/apple/SitrepApp/Sitrep/SitrepApp.swift b/apple/SitrepApp/Sitrep/SitrepApp.swift index 908db8d..743b36f 100644 --- a/apple/SitrepApp/Sitrep/SitrepApp.swift +++ b/apple/SitrepApp/Sitrep/SitrepApp.swift @@ -388,6 +388,17 @@ final class AppModel { } } + /// The realtime client's own phase, queried on the actor — the + /// authority for HTTP-overwrite decisions. The MainActor-cached + /// `connectionPhase` arrives via a separate AsyncStream from the state + /// updates, so it can momentarily lag behind an already-applied live + /// state; deciding on the cache would open a window where a refresh + /// reads a stale non-live phase and overwrites delta-derived state. + private func authoritativePhase() async -> RealtimeClient.Phase { + guard let realtimeClient else { return .idle } + return await realtimeClient.currentPhase + } + func refresh() async { guard let client else { return } do { @@ -398,13 +409,14 @@ final class AppModel { presence = snapshot.presence lastError = nil lastSyncAt = .now - // Checked AFTER the await, deliberately: while the WebSocket is - // live, deltas own the four reliable collections and an HTTP - // response — possibly computed before the connection went live — - // must not clobber them. Re-checking here (not at request time) - // also covers the in-flight race: a fallback-poll or pull-to- - // refresh response that lands after `.recovered` is dropped. - guard connectionPhase.allowsReliableStateOverwrite else { return } + // Checked AFTER the await, against the ACTOR's phase (not the + // stream-fed MainActor cache): while the WebSocket is live, + // deltas own the four reliable collections and an HTTP response + // — possibly computed before the connection went live — must + // not clobber them. This covers the in-flight race: a fallback- + // poll or pull-to-refresh response that lands after `.recovered` + // is dropped. + guard await authoritativePhase().allowsReliableStateOverwrite else { return } events = snapshot.messages.map(\.state) automations = snapshot.automations.sorted { $0.name < $1.name } tasks = snapshot.tasks.sorted { $0.updatedAt > $1.updatedAt } @@ -417,10 +429,20 @@ final class AppModel { } metrics = sorted adoptOrphanedTasks() + // Post-write reconciliation, closing the last sliver: if the + // connection went live between the check above and these writes, + // re-apply the actor's authoritative state so the delta-derived + // data wins. (If it instead goes live after THIS check, the + // states-stream delivery that accompanied the flip is still + // queued for the MainActor and will overwrite the HTTP data on + // arrival.) + if let rt = realtimeClient, await !rt.currentPhase.allowsReliableStateOverwrite { + applyRealtimeState(await rt.currentState) + } } catch { // An HTTP hiccup while realtime is live is not a sync outage — // don't raise the banner over it. - if connectionPhase.allowsReliableStateOverwrite { + if await authoritativePhase().allowsReliableStateOverwrite { lastError = error.localizedDescription } } From b85a6da8c5050727bfcdd75d7cb83dbf3c64d956 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:12:12 +0800 Subject: [PATCH 32/60] fix(server): keep server-chosen automation ids idempotent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A POST that omitted automation_id minted a fresh crypto.randomUUID() per call, and that random id entered the idempotency fingerprint — so an honest retry of a dropped create (same Idempotency-Key, no explicit id) fingerprint-mismatched into a spurious 409, breaking the idempotency contract exactly on the server-chooses-the-id path. The fingerprint now covers only client-provided content: when the client omits automation_id but supplies an Idempotency-Key, the id is derived deterministically from that key (SHA-256, 32 hex chars), so a retry reconstructs the identical payload and replays the original result — same automation_id, same revision — while a same-key request with different client content still returns 409. The fingerprint itself is now truly canonical JSON (recursively key-sorted via canonicalJson) instead of fixed-construction-order serialization, so a future change in a caller's object construction order cannot introduce false 409s; the comment wording is corrected to match. --- server/src/adapters/workers.ts | 26 +++++++++++++++-- server/src/realtime/space-hub.ts | 30 ++++++++++++++++++-- server/test/workers/config-event.workers.ts | 31 +++++++++++++++++++++ 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index f069316..3c9c654 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -562,6 +562,15 @@ function mintResultResponse(c: any, result: { revision: number; automation: Auto return c.json({ revision: result.revision, automation: result.automation }); } +/** Deterministic automation_id for a POST that omitted one: SHA-256 of the + * Idempotency-Key, hex, truncated to 32 chars (fits the 1-128 char id cap + * and stays collision-safe for this cardinality). Same key -> same id, so + * a retried create replays instead of fingerprint-conflicting. */ +async function deriveAutomationId(idempotencyKey: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(idempotencyKey)); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("").slice(0, 32); +} + const internalError = (c: any, e: unknown) => { console.log(JSON.stringify({ level: "error", event: "v3_http_unhandled", message: String(e) })); return c.json({ error: "internal error" }, 500); @@ -573,14 +582,27 @@ app.post("/v3/automations", async (c: any) => { const body = await c.req.json().catch(() => null); const parsed = parseAutomationUpsert(body); if (!parsed) return c.json({ error: "name, executor_kind and schedule.every_seconds required" }, 400); + const idempotencyKey = c.req.header("idempotency-key") ?? null; + // The idempotency fingerprint must cover only CLIENT-provided content: a + // server-generated random id would enter the fingerprint and make an + // honest retry of a dropped POST (same key, no explicit automation_id) + // fingerprint-mismatch into a spurious 409. When the client omits + // automation_id but supplies an Idempotency-Key, derive the id + // deterministically from that key, so the retry reconstructs the exact + // same automation (including its id) and replays cleanly. + const automationId = + typeof body.automation_id === "string" && body.automation_id + ? body.automation_id + : idempotencyKey + ? await deriveAutomationId(idempotencyKey) + : crypto.randomUUID(); const automation: AutomationState = { - automation_id: typeof body.automation_id === "string" && body.automation_id ? body.automation_id : crypto.randomUUID(), + automation_id: automationId, name: parsed.name, executor_kind: parsed.executor_kind, schedule: { kind: "interval", every_seconds: parsed.every_seconds }, state: body.state === "paused" ? "paused" : "active", }; - const idempotencyKey = c.req.header("idempotency-key") ?? null; try { const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); const result = await stub.mintConfigEvent(idempotencyKey, { diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index d6e86ad..3b58616 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -70,6 +70,21 @@ interface RateLimiterState { frameTimestamps: number[]; } +/** Deterministic JSON with recursively sorted object keys, so two + * structurally-equal values always serialize identically regardless of + * property construction order. Used for idempotency fingerprints. */ +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`); + return `{${entries.join(",")}}`; + } + return JSON.stringify(value); +} + export class SpaceHub extends DurableObject { /** Best-effort metric cache (SPEC.md section 6.2: snapshot.metrics is a * convenience cache, allowed to be empty/stale, outside space_revision @@ -317,18 +332,27 @@ export class SpaceHub extends DurableObject { * discipline as applyReliableEvent — see the class-level comment. * * The idempotency key is bound to the request's CONTENT, not just its - * key string: the stored fingerprint is the canonical JSON of + * key string: the stored fingerprint is truly canonical JSON — + * recursively key-sorted (see canonicalJson), so it cannot drift if a + * caller's object-construction order ever changes — of * (kind, automation_id, automation), compared verbatim on a replay. * (Exact string comparison is strictly stronger than a hash and stays * synchronous — crypto.subtle.digest is async and would break the * single-transaction discipline.) A key reused for a DIFFERENT * operation returns { conflict: true } and mints nothing, instead of - * silently replaying the first operation's result. */ + * silently replaying the first operation's result. + * + * Contract note: the fingerprint must cover only client-provided + * content. When the client omits automation_id on a create, the HTTP + * layer derives it deterministically from the Idempotency-Key BEFORE + * calling this method (see adapters/workers.ts deriveAutomationId), so + * an honest retry reconstructs the identical payload and replays with + * the originally-created automation_id rather than tripping a 409. */ mintConfigEvent( idempotencyKey: string | null, payload: { kind: ConfigEventKind; automation_id: string; automation?: AutomationState }, ): { revision: number; automation: AutomationState | null; conflict?: boolean } { - const fingerprint = JSON.stringify({ + const fingerprint = canonicalJson({ kind: payload.kind, automation_id: payload.automation_id, automation: payload.automation ?? null, diff --git a/server/test/workers/config-event.workers.ts b/server/test/workers/config-event.workers.ts index 00b1cfe..4f1f96d 100644 --- a/server/test/workers/config-event.workers.ts +++ b/server/test/workers/config-event.workers.ts @@ -96,6 +96,37 @@ describe("/v3/automations control plane", () => { expect(body3.revision).toBe(body1.revision + 1); }); + it("retrying a create WITHOUT an explicit automation_id under the same key replays the same automation, not 409", async () => { + const { ownerToken } = await bootstrapSpace(); + const headers = { + authorization: `Bearer ${ownerToken}`, + "content-type": "application/json", + "idempotency-key": "req-serverid", + }; + // No automation_id in the body: the server picks one — deterministically + // from the Idempotency-Key, so a retry of a dropped response must + // replay the original result (same automation_id, same revision). + const payload = JSON.stringify({ name: "no explicit id", executor_kind: "script", schedule: { every_seconds: 60 } }); + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: payload }); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { revision: number; automation: { automation_id: string } }; + expect(body1.automation.automation_id.length).toBeGreaterThan(0); + + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: payload }); + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { revision: number; automation: { automation_id: string } }; + expect(body2.revision).toBe(body1.revision); + expect(body2.automation.automation_id).toBe(body1.automation.automation_id); + + // Same key with DIFFERENT client content still conflicts. + const res3 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ name: "changed content", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(res3.status).toBe(409); + }); + it("role matrix: viewer may PATCH and DELETE but not POST", async () => { const { viewer, ownerToken } = await bootstrapSpace(); // Owner creates two automations for the viewer to act on. From 15aaabf46b91835f955b7f3c9d7a2a6eca9a31cf Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:29:14 +0800 Subject: [PATCH 33/60] test(daemon): add opt-in E2E source lifecycle test against a real server Drives the real daemon realtime client package (hello, task.event, ack, disconnect, reconnect, outbox replay) over an actual WebSocket against a locally running server, gated behind SITREP_E2E_* env vars so it is a no-op under the normal go test ./... regression run. --- daemon/internal/realtime/client/e2e_test.go | 111 ++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 daemon/internal/realtime/client/e2e_test.go diff --git a/daemon/internal/realtime/client/e2e_test.go b/daemon/internal/realtime/client/e2e_test.go new file mode 100644 index 0000000..16bf215 --- /dev/null +++ b/daemon/internal/realtime/client/e2e_test.go @@ -0,0 +1,111 @@ +package client + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" +) + +// TestE2ESourceLifecycle drives a real *daemon* client against a real, +// locally running server (see docs/design/realtime-integration.md for how +// to start one with `wrangler dev`). It is opt-in: it does nothing unless +// SITREP_E2E_URL, SITREP_E2E_TOKEN, SITREP_E2E_DEVICE_ID and +// SITREP_E2E_SPACE are all set, so `go test ./...` in CI/regression never +// touches the network. +// +// Coverage: hello offer/accept over a real WebSocket -> task.event send -> +// server ack (observed as the outbox item retiring) -> disconnect -> a +// second Client sharing the same outbox/device/space reconnects and +// replays the still-pending event -> ack again. This is the cross-line +// interoperability smoke test for the daemon "source" role against the +// real server implementation (as opposed to the in-process rttest fake +// server the rest of this package's tests use). +func TestE2ESourceLifecycle(t *testing.T) { + url := os.Getenv("SITREP_E2E_URL") + token := os.Getenv("SITREP_E2E_TOKEN") + deviceID := os.Getenv("SITREP_E2E_DEVICE_ID") + space := os.Getenv("SITREP_E2E_SPACE") + if url == "" || token == "" || deviceID == "" || space == "" { + t.Skip("set SITREP_E2E_URL, SITREP_E2E_TOKEN, SITREP_E2E_DEVICE_ID, SITREP_E2E_SPACE to run the real-server E2E smoke test") + } + + dir := t.TempDir() + store, err := outbox.Open(filepath.Join(dir, "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + t.Cleanup(func() { store.Close() }) + + mkClient := func() *Client { + return New(Config{ + URL: url, + Token: token, + DeviceID: deviceID, + Space: space, + Outbox: store, + BackoffBase: 50 * time.Millisecond, + BackoffMax: 500 * time.Millisecond, + ResendInterval: 200 * time.Millisecond, + HelloTimeout: 5 * time.Second, + Logf: t.Logf, + }) + } + + c1 := mkClient() + waitFor(t, 5*time.Second, c1.Connected) + t.Log("e2e: hello offer/accept completed against real server (task 1)") + + taskID := "e2e-task-1" + if err := c1.SendTaskEvent(TaskEvent{ + TaskID: taskID, + Kind: "started", + OccurredAt: time.Now(), + Title: "e2e smoke", + }); err != nil { + t.Fatalf("SendTaskEvent(started): %v", err) + } + + waitFor(t, 5*time.Second, func() bool { + n, err := store.Count(context.Background()) + return err == nil && n == 0 + }) + if n, _ := store.Count(context.Background()); n != 0 { + t.Fatalf("expected task.event started to be acked (outbox drained), %d still pending", n) + } + t.Log("e2e: task.event{started} acked by real server, outbox drained") + + c1.Close() + + // Enqueue a second event while fully disconnected: it must sit in the + // outbox until the next connection's post-hello replay picks it up. + percent := 50 + if err := c1.SendTaskEvent(TaskEvent{ + TaskID: taskID, + Kind: "progress", + OccurredAt: time.Now(), + Percent: &percent, + }); err != nil { + t.Fatalf("SendTaskEvent(progress) while offline: %v", err) + } + if n, _ := store.Count(context.Background()); n == 0 { + t.Fatalf("expected the offline progress event to remain pending in the outbox") + } + + c2 := mkClient() + t.Cleanup(c2.Close) + waitFor(t, 5*time.Second, c2.Connected) + t.Log("e2e: reconnect hello offer/accept completed (task 1, replay)") + + waitFor(t, 5*time.Second, func() bool { + n, err := store.Count(context.Background()) + return err == nil && n == 0 + }) + if n, _ := store.Count(context.Background()); n != 0 { + t.Fatalf("expected replayed task.event{progress} to be acked after reconnect, %d still pending", n) + } + t.Log("e2e: task.event{progress} replayed after reconnect and acked, outbox drained") +} From 86b115d30d4f6d57e34917edad1b0c558382ba5a Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:29:16 +0800 Subject: [PATCH 34/60] test(integration): add node viewer E2E smoke scripts scripts/e2e/viewer-smoke.mjs drives two real viewer WebSocket connections (hello -> subscribe -> resume) against a locally running server and verifies fresh-viewer snapshot, live delta revision chaining, and metric.frame fan-out to a second viewer. scripts/e2e/source-drive.mjs is the companion driver that emits the task.event/metric.frame from the source side. Both are plain `ws` scripts independent of the daemon/apple code, so they exercise only the server's wire contract. --- scripts/e2e/package-lock.json | 36 +++++++ scripts/e2e/package.json | 10 ++ scripts/e2e/source-drive.mjs | 105 ++++++++++++++++++ scripts/e2e/viewer-smoke.mjs | 198 ++++++++++++++++++++++++++++++++++ 4 files changed, 349 insertions(+) create mode 100644 scripts/e2e/package-lock.json create mode 100644 scripts/e2e/package.json create mode 100644 scripts/e2e/source-drive.mjs create mode 100644 scripts/e2e/viewer-smoke.mjs diff --git a/scripts/e2e/package-lock.json b/scripts/e2e/package-lock.json new file mode 100644 index 0000000..af0f755 --- /dev/null +++ b/scripts/e2e/package-lock.json @@ -0,0 +1,36 @@ +{ + "name": "sitrep-realtime-e2e-scripts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "sitrep-realtime-e2e-scripts", + "version": "1.0.0", + "dependencies": { + "ws": "^8.18.0" + } + }, + "node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/scripts/e2e/package.json b/scripts/e2e/package.json new file mode 100644 index 0000000..dd4e085 --- /dev/null +++ b/scripts/e2e/package.json @@ -0,0 +1,10 @@ +{ + "name": "sitrep-realtime-e2e-scripts", + "version": "1.0.0", + "private": true, + "description": "Manual/CI-optional cross-line E2E smoke scripts for the realtime protocol viewer role, driven against a real locally running server (wrangler dev). See docs/design/realtime-integration.md for usage. Not part of any package's build or test graph.", + "type": "module", + "dependencies": { + "ws": "^8.18.0" + } +} diff --git a/scripts/e2e/source-drive.mjs b/scripts/e2e/source-drive.mjs new file mode 100644 index 0000000..6a9afb2 --- /dev/null +++ b/scripts/e2e/source-drive.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +// Companion driver for viewer-smoke.mjs: connects as a "source" device and, +// after a short delay (so the viewer script has time to subscribe/resume +// first), emits one task.event (to produce a live delta) and one +// metric.frame (to produce a metric.frame broadcast), then disconnects. +// +// This is a protocol-generic driver used only to exercise the server's +// viewer-side fan-out in scripts/e2e/viewer-smoke.mjs; the daemon (Go) +// source role itself is validated independently and more thoroughly by +// daemon/internal/realtime/client/e2e_test.go (TestE2ESourceLifecycle), +// which drives the real daemon client package against the same server. +// +// Required env: SITREP_E2E_URL, SITREP_E2E_SOURCE_TOKEN, SITREP_E2E_SOURCE_DEVICE_ID. + +import WebSocket from "ws"; + +const URL = requireEnv("SITREP_E2E_URL"); +const TOKEN = requireEnv("SITREP_E2E_SOURCE_TOKEN"); +const DEVICE_ID = requireEnv("SITREP_E2E_SOURCE_DEVICE_ID"); +const DELAY_MS = Number(process.env.SITREP_E2E_SOURCE_DELAY_MS || 4000); + +function requireEnv(name) { + const v = process.env[name]; + if (!v) { + console.error(`missing required env var ${name}`); + process.exit(2); + } + return v; +} + +function log(msg, extra) { + console.log(extra !== undefined ? `[source] ${msg} ${JSON.stringify(extra)}` : `[source] ${msg}`); +} + +let seq = 0; +function envelope(type, body) { + // envelope_id must match ^[A-Za-z0-9_-]{1,64}$ (proto/realtime/common.schema.json) + // so the message `type` (which may contain a literal "." e.g. task.event, + // metric.frame) must NOT be embedded verbatim. + const safeType = type.replace(/[^A-Za-z0-9]/g, ""); + return { type, id: `e2e-src-${safeType}-${++seq}-${Date.now()}`, ts: Date.now(), body }; +} + +const ws = new WebSocket(URL, { headers: { Authorization: `Bearer ${TOKEN}` } }); +let deviceSeq = 1; + +ws.on("open", () => { + const offer = envelope("hello", { + stage: "offer", + device_id: DEVICE_ID, + role: "source", + protocol_versions: [1], + }); + log("send hello offer"); + ws.send(JSON.stringify(offer)); +}); + +ws.on("message", async (data) => { + const text = data.toString(); + if (text === "ping") { + ws.send("pong"); + return; + } + if (text === "pong") return; + const env = JSON.parse(text); + log(`recv ${env.type}`, env.type === "hello" ? env.body : env.type === "ack" ? env.body : undefined); + + if (env.type === "hello" && env.body.stage === "accept") { + log(`hello accepted, waiting ${DELAY_MS}ms before driving events (letting viewers subscribe first)`); + await new Promise((r) => setTimeout(r, DELAY_MS)); + + const taskEvent = envelope("task.event", { + device_id: DEVICE_ID, + device_seq: deviceSeq++, + task_id: "e2e-viewer-drive-task", + kind: "started", + occurred_at: Date.now(), + title: "e2e viewer-smoke drive", + }); + log("send task.event (should trigger a live delta to subscribed viewers)"); + ws.send(JSON.stringify(taskEvent)); + + await new Promise((r) => setTimeout(r, 500)); + + const metricFrame = envelope("metric.frame", { + device_id: DEVICE_ID, + metrics: [{ metric_id: "e2e.cpu.load", value: "0.42", ts: Date.now() }], + }); + log("send metric.frame (should broadcast to metric-subscribed viewers)"); + ws.send(JSON.stringify(metricFrame)); + + await new Promise((r) => setTimeout(r, 1000)); + ws.close(1000, "e2e source drive done"); + log("done, closing"); + } + + if (env.type === "error") { + console.error("[source] server error:", env.body); + } +}); + +ws.on("error", (e) => { + console.error("[source] ws error", e); + process.exit(1); +}); diff --git a/scripts/e2e/viewer-smoke.mjs b/scripts/e2e/viewer-smoke.mjs new file mode 100644 index 0000000..b4edf35 --- /dev/null +++ b/scripts/e2e/viewer-smoke.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node +// Cross-line realtime protocol E2E smoke test: viewer role, driven against a +// real server (see docs/design/realtime-integration.md for how to start one +// locally with `wrangler dev`). Exercises, over two real WebSocket +// connections (two "viewer" devices), everything the apple/SitrepKit +// RealtimeClient does on the wire, using a plain `ws` client so this script +// has no dependency on the Swift/Go implementations: +// +// viewer1: hello{offer} -> hello{accept} -> subscribe -> ack{lease} +// -> resume{last_revision:0} -> snapshot (verifies fresh-viewer +// snapshot path) -> live delta once the source emits an event +// (verifies revision chaining: snapshot.revision -> delta. +// from_revision == snapshot.revision, to_revision == from+1) +// viewer2: same hello/subscribe/resume sequence, then receives the +// metric.frame broadcast triggered by the source (verifies +// server fan-out to a second, independently-subscribed viewer) +// +// All credentials are read from the environment — nothing is hardcoded. +// Required: SITREP_E2E_URL, SITREP_E2E_VIEWER1_TOKEN, SITREP_E2E_VIEWER2_TOKEN. +// The source-side event that drives the live delta/metric.frame is sent by +// a companion process (see docs/design/realtime-integration.md); this +// script only waits for it, with a generous timeout. + +import WebSocket from "ws"; + +const URL = requireEnv("SITREP_E2E_URL"); +const VIEWER1_TOKEN = requireEnv("SITREP_E2E_VIEWER1_TOKEN"); +const VIEWER2_TOKEN = requireEnv("SITREP_E2E_VIEWER2_TOKEN"); +const WAIT_FOR_SOURCE_EVENT_MS = Number(process.env.SITREP_E2E_WAIT_MS || 20000); + +function requireEnv(name) { + const v = process.env[name]; + if (!v) { + console.error(`missing required env var ${name}`); + process.exit(2); + } + return v; +} + +function log(tag, msg, extra) { + const line = `[${tag}] ${msg}`; + console.log(extra !== undefined ? `${line} ${JSON.stringify(extra)}` : line); +} + +let envSeq = 0; +function envelope(type, body) { + // envelope_id must match ^[A-Za-z0-9_-]{1,64}$ (proto/realtime/common.schema.json); + // strip any "." from the message type (e.g. metric.frame) before embedding it. + const safeType = type.replace(/[^A-Za-z0-9]/g, ""); + return { type, id: `e2e-${safeType}-${++envSeq}-${Date.now()}`, ts: Date.now(), body }; +} + +// Connects as a viewer, completes hello -> subscribe -> resume{0}, and +// returns { ws, send, waitFor(predicate) } so the caller can drive the rest +// of the scenario. waitFor resolves the first received frame matching +// predicate, queuing every other frame for subsequent waitFor calls. +function connectViewer(tag, token) { + return new Promise((resolve, reject) => { + const ws = new WebSocket(URL, { headers: { Authorization: `Bearer ${token}` } }); + const queue = []; + const waiters = []; + + ws.on("message", (data) => { + const text = data.toString(); + if (text === "ping") { + ws.send("pong"); + log(tag, "recv ping -> sent pong"); + return; + } + if (text === "pong") return; + const env = JSON.parse(text); + log(tag, `recv ${env.type}`, summarize(env)); + const idx = waiters.findIndex((w) => w.predicate(env)); + if (idx >= 0) { + const [w] = waiters.splice(idx, 1); + w.resolve(env); + } else { + queue.push(env); + } + }); + ws.on("error", reject); + + const send = (type, body) => { + const env = envelope(type, body); + log(tag, `send ${type}`, summarize(env)); + ws.send(JSON.stringify(env)); + return env; + }; + + const waitFor = (predicate, timeoutMs = 10000) => + new Promise((res, rej) => { + const qi = queue.findIndex(predicate); + if (qi >= 0) { + res(queue.splice(qi, 1)[0]); + return; + } + const timer = setTimeout(() => { + const i = waiters.findIndex((w) => w.resolve === wrappedRes); + if (i >= 0) waiters.splice(i, 1); + rej(new Error(`${tag}: timed out waiting for a matching frame`)); + }, timeoutMs); + const wrappedRes = (env) => { + clearTimeout(timer); + res(env); + }; + waiters.push({ predicate, resolve: wrappedRes }); + }); + + ws.on("open", async () => { + try { + send("hello", { + stage: "offer", + device_id: `${tag}-e2e`, + role: "viewer", + protocol_versions: [1], + }); + const accept = await waitFor((e) => e.type === "hello"); + resolve({ ws, send, waitFor, sessionId: accept.body.session_id }); + } catch (e) { + reject(e); + } + }); + }); +} + +function summarize(env) { + switch (env.type) { + case "hello": + return env.body.stage === "accept" + ? { stage: "accept", protocol_version: env.body.protocol_version, session_id: env.body.session_id } + : { stage: "offer", device_id: env.body.device_id, role: env.body.role }; + case "ack": + return { in_reply_to: env.body.in_reply_to, lease: env.body.lease }; + case "snapshot": + return { revision: env.body.revision, part: env.body.part, final: env.body.final, tasks: env.body.tasks?.length, metrics: env.body.metrics?.length }; + case "delta": + return { from_revision: env.body.from_revision, to_revision: env.body.to_revision, events: env.body.events?.length }; + case "metric.frame": + return { device_id: env.body.device_id, metrics: env.body.metrics }; + case "error": + return { code: env.body.code, message: env.body.message, retryable: env.body.retryable, fatal: env.body.fatal }; + default: + return env.body; + } +} + +async function main() { + log("main", `connecting two viewers to ${URL}`); + + const v1 = await connectViewer("viewer1", VIEWER1_TOKEN); + const subAck1 = v1.send("subscribe", { topics: ["task", "metric", "message"] }); + const ack1 = await v1.waitFor((e) => e.type === "ack" && e.body.in_reply_to === subAck1.id); + log("viewer1", "subscribed, lease", ack1.body.lease); + + v1.send("resume", { last_revision: 0 }); + const snapshot = await v1.waitFor((e) => e.type === "snapshot"); + log("viewer1", `fresh-viewer resume(0) correctly produced a snapshot at revision ${snapshot.body.revision}`); + + const v2 = await connectViewer("viewer2", VIEWER2_TOKEN); + const subAck2 = v2.send("subscribe", { topics: ["task", "metric", "message"] }); + const ack2 = await v2.waitFor((e) => e.type === "ack" && e.body.in_reply_to === subAck2.id); + log("viewer2", "subscribed, lease", ack2.body.lease); + v2.send("resume", { last_revision: snapshot.body.revision }); + // viewer2 resumes at the current revision: server should send an + // empty/no-op catch-up or nothing until the next real event — we don't + // block on it, we only need viewer2 subscribed + delta-eligible before + // the source emits its next event. + log("viewer2", "resumed at current revision, now delta-eligible"); + + log("main", `waiting up to ${WAIT_FOR_SOURCE_EVENT_MS}ms for the source's next task.event to arrive as a live delta on viewer1...`); + const delta = await v1.waitFor((e) => e.type === "delta", WAIT_FOR_SOURCE_EVENT_MS); + if (delta.body.from_revision !== snapshot.body.revision) { + throw new Error( + `revision chain broken: snapshot.revision=${snapshot.body.revision} but delta.from_revision=${delta.body.from_revision}`, + ); + } + if (delta.body.to_revision !== delta.body.from_revision + 1 && delta.body.to_revision <= delta.body.from_revision) { + throw new Error(`delta.to_revision (${delta.body.to_revision}) did not advance past from_revision (${delta.body.from_revision})`); + } + log( + "main", + `PASS: live delta chained correctly (snapshot rev ${snapshot.body.revision} -> delta ${delta.body.from_revision}->${delta.body.to_revision})`, + ); + + log("main", `waiting up to ${WAIT_FOR_SOURCE_EVENT_MS}ms for the source's metric.frame broadcast on viewer2...`); + const metricFrame = await v2.waitFor((e) => e.type === "metric.frame", WAIT_FOR_SOURCE_EVENT_MS); + log("main", "PASS: viewer2 received the metric.frame broadcast triggered by the source", summarize(metricFrame)); + + v1.ws.close(1000, "e2e done"); + v2.ws.close(1000, "e2e done"); + log("main", "ALL VIEWER E2E CHECKS PASSED"); + process.exit(0); +} + +main().catch((err) => { + console.error("E2E FAILED:", err); + process.exit(1); +}); From 5c8c88cdf14364803a401e14d651ef353a12efd8 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:29:17 +0800 Subject: [PATCH 35/60] docs(protocol): record realtime cross-line integration verification Captures the full regression results, the six-point wire contract audit (endpoint, auth header, hello, heartbeat, error codes, envelope id/seq), and the real cross-process E2E run (Go source client + node viewer scripts against wrangler dev), including how to reproduce it. --- docs/design/realtime-integration.md | 278 ++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 docs/design/realtime-integration.md diff --git a/docs/design/realtime-integration.md b/docs/design/realtime-integration.md new file mode 100644 index 0000000..c43b3b4 --- /dev/null +++ b/docs/design/realtime-integration.md @@ -0,0 +1,278 @@ +# Realtime protocol cross-line integration + +This document records the result of integrating three independently +developed implementations of `proto/realtime/` onto `integration/realtime`: + +- `server/` (Cloudflare Workers, TypeScript) — the authoritative "SpaceHub+ + v3" server. +- `daemon/` (Go) — the `source` role client (device daemon that pushes task + events). +- `apple/SitrepKit` (Swift) — the `viewer` role client (subscribes to space + state). + +It is not a spec: `proto/realtime/` remains the source of truth. This +records what was verified, how, and how to re-run it. + +## 1. Full regression (per line) + +| Suite | Command | Result | +| --- | --- | --- | +| Protocol fixtures | `cd proto/realtime/tools && npm install && npm run validate` | 81/81 | +| Server | `cd server && npm install && npm run typecheck && npm test` | typecheck clean; 88 (node:test fixtures) + 33 (vitest workers pool) = 121 | +| Daemon | `cd daemon && go build ./... && go vet ./... && go test ./... -race -count=1` | build/vet clean; 38 top-level tests, 0 failures | +| SitrepKit | `cd apple/SitrepKit && swift test` | 30/30 | +| SitrepApp | `cd apple/SitrepApp && xcodebuild build -scheme Sitrep -destination 'generic/platform=iOS Simulator'` | BUILD SUCCEEDED | +| SitrepMenuBar | `cd apple/SitrepMenuBar && swift build` | Build complete | + +No fixes were needed to get all six suites green as merged. + +## 2. Cross-line contract audit + +Six wire-level interface points were checked with file:line evidence from +all three lines. Full detail was produced by a dedicated audit pass; the +verdicts: + +1. **Endpoint URL** (`/v3/realtime`, scheme swap, no query params) — MATCH. + `daemon/internal/config/config.go:80-92` (`RealtimeURLFor`), + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift:14-24`, + `server/src/adapters/workers.ts:509`. +2. **Authorization header** (`Authorization: Bearer `) — MATCH. + `daemon/internal/realtime/client/client.go:266-268`, + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:226-228`, + `server/src/app.ts:175` (case-insensitive `Bearer` strip), + token format `TOKEN_RE` at `server/src/app.ts:22`. +3. **Hello handshake** — MATCH. Both clients send + `{stage, device_id, role, protocol_versions}` + (`daemon/internal/realtime/client/client.go:346-353`, + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:293`); + server negotiates the version intersection and replies + `{stage:"accept", protocol_version, session_id, heartbeat_interval_ms}` + (`server/src/realtime/space-hub.ts:430-489`). One intentional race was + confirmed both in code and live on the wire (§3 below): the server may + push a `command` envelope to a `source` connection immediately after + `hello{accept}`, before any application-level exchange + (`server/src/realtime/space-hub.ts:476-488`). Both clients handle this + correctly (daemon's `readLoop` just dispatches it as the next frame; + apple never receives it, since the push is source-only, and defensively + ignores a stray `command` if one ever arrived). This was reproduced live + during the E2E run below (`scripts/e2e/source-drive.mjs` log shows two + `recv command` frames arriving right after `hello{accept}`, before the + script's own `task.event` was sent). +4. **Heartbeat** (`ping`/`pong` as literal, lower-case, bare-text frames, + either side may initiate) — MATCH. `server/src/realtime/space-hub.ts:107` + (`setWebSocketAutoResponse(new WebSocketRequestResponsePair("ping", + "pong"))`), `daemon/internal/realtime/client/client.go:325,333`, + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:451-481`. +5. **Error codes** — MATCH. All 12 codes in + `server/src/realtime/protocol.ts:72-86` (`ERROR_SEMANTICS`) are mirrored + verbatim in `daemon/internal/realtime/wire/bodies.go:324-336` and + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift:196-209`. + Both clients react to the server-supplied `fatal`/`retryable` flags on + the wire rather than a hardcoded local table, so neither can silently + diverge from server semantics as codes evolve. +6. **Envelope id / device_seq** — MATCH. Schema: + `proto/realtime/common.schema.json` — `envelope_id` pattern + `^[A-Za-z0-9_-]{1,64}$`, `device_seq` minimum `1`. Server generates + `crypto.randomUUID()` ids and validates the same regex + (`server/src/realtime/protocol.ts:47-48`, + `server/src/realtime/guards.ts:41,420`). Daemon generates `"rt" + hex(16 + random bytes)` ids (`daemon/internal/realtime/client/client.go:89-95`) + and starts `device_seq` at 1 (`daemon/internal/realtime/outbox/outbox.go:168-183`). + Apple generates `UUID().uuidString` ids + (`apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:527`) + and validates incoming `device_seq >= 1` + (`apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift:304,406,473`). + +No mismatches were found. No integration fixes were required for the +contract points above. + +## 3. End-to-end smoke test (real cross-process, not a fake/in-process server) + +Both halves ran against a real `wrangler dev` instance (local workerd, not +`--remote`), i.e. real separate OS processes talking real WebSockets over +`127.0.0.1` — not an in-process/simulated transport. + +### Setup + +```bash +cd server +npm install +# The environment used a proxy; wrangler dev only needs to bind locally, so +# no proxy bypass was actually required beyond the existing NO_PROXY setting +# (NO_PROXY=localhost,127.0.0.1,::1,.local was already present). If your +# shell's NO_PROXY doesn't cover 127.0.0.1, unset http_proxy/https_proxy or +# add it before running the command below. +npx wrangler dev --port 8787 --local +``` + +Bootstrapping a test space and three device tokens (source, viewer1, +viewer2) uses the server's existing `/v2/spaces` + `/v2/invites` + `/v2/join` +HTTP endpoints — no new test-only backdoor was added: + +```bash +curl -s -X POST http://127.0.0.1:8787/v2/spaces \ + -H 'content-type: application/json' -d '{"platform":"macos","name":"e2e"}' +# => {"space_id":"...", "owner_token":"st2_..._..."} + +curl -s -X POST http://127.0.0.1:8787/v2/invites \ + -H "Authorization: Bearer $OWNER_TOKEN" -H 'content-type: application/json' \ + -d '{"role":"source"}' # or {"role":"viewer"} +# => {"code":"...", "expires_in":600, "space_id":"..."} + +curl -s -X POST http://127.0.0.1:8787/v2/join \ + -H 'content-type: application/json' \ + -d '{"code":"...", "space":"...", "name":"...", "platform":"..."}' +# => {"token":"st2_..._...", "device_id":"...", "role":"source|viewer", "space_id":"..."} +``` + +### Source role (real Go client, real network) + +`daemon/internal/realtime/client/e2e_test.go` (`TestE2ESourceLifecycle`) is a +new, opt-in test in the daemon's own `client` package. It is a no-op under +plain `go test ./...` (skips immediately) and only runs when four env vars +are set, so it never affects the regular regression run: + +```bash +cd daemon +SITREP_E2E_URL="ws://127.0.0.1:8787/v3/realtime" \ +SITREP_E2E_TOKEN="" \ +SITREP_E2E_DEVICE_ID="" \ +SITREP_E2E_SPACE="" \ +go test ./internal/realtime/client/... -run TestE2ESourceLifecycle -v +``` + +It drives the actual `daemon/internal/realtime/client.Client` (the same +code the daemon binary uses) through: hello offer/accept over the real +socket -> `task.event{started}` -> waits for the outbox to drain (server +ack) -> disconnects -> enqueues a second event while offline -> reconnects +with a fresh `Client` sharing the same on-disk outbox -> confirms the +pending event replays and is acked. Observed run: + +``` +=== RUN TestE2ESourceLifecycle + e2e_test.go:60: e2e: hello offer/accept completed against real server (task 1) + e2e_test.go:79: e2e: task.event{started} acked by real server, outbox drained + e2e_test.go:101: e2e: reconnect hello offer/accept completed (task 1, replay) + e2e_test.go:110: e2e: task.event{progress} replayed after reconnect and acked, outbox drained +--- PASS: TestE2ESourceLifecycle (10.10s) +``` + +### Viewer role (node `ws` script, real network) + +`scripts/e2e/viewer-smoke.mjs` connects two independent viewer devices with +the `ws` npm package (credentials via env vars only) and drives the +mandatory viewer sequence: `hello` -> `subscribe` -> `resume`. A companion +driver, `scripts/e2e/source-drive.mjs`, connects as the source device and, +after a delay (letting the viewers finish subscribing), sends one +`task.event` and one `metric.frame`. + +```bash +cd scripts/e2e +npm install +export SITREP_E2E_URL="ws://127.0.0.1:8787/v3/realtime" +export SITREP_E2E_SOURCE_TOKEN="..." SITREP_E2E_SOURCE_DEVICE_ID="..." +export SITREP_E2E_VIEWER1_TOKEN="..." SITREP_E2E_VIEWER2_TOKEN="..." +node source-drive.mjs & # background: waits, then emits the events +node viewer-smoke.mjs # foreground: asserts the full scenario +``` + +Observed run (trimmed to the load-bearing frames; full output is +reproducible verbatim by re-running the commands above): + +``` +[viewer1] send hello {"stage":"offer","device_id":"viewer1-e2e","role":"viewer"} +[viewer1] recv hello {"stage":"accept","protocol_version":1,"session_id":"0c66d634-..."} +[viewer1] send subscribe {"topics":["task","metric","message"]} +[viewer1] recv ack {"in_reply_to":"e2e-subscribe-2-...","lease":{"expires_at":...}} +[viewer1] send resume {"last_revision":0} +[viewer1] recv snapshot {"revision":0,"part":1,"final":true,"tasks":0,"metrics":0} +[viewer1] fresh-viewer resume(0) correctly produced a snapshot at revision 0 +[viewer2] ... same hello/subscribe/ack sequence ... +[viewer2] send resume {"last_revision":0} +[viewer2] recv snapshot {"revision":0,"part":1,"final":true,...} +[viewer2] recv delta {"from_revision":0,"to_revision":1,"events":1} +[viewer1] recv delta {"from_revision":0,"to_revision":1,"events":1} +[main] PASS: live delta chained correctly (snapshot rev 0 -> delta 0->1) +[viewer2] recv metric.frame {"device_id":"a85ee0f8-...","metrics":[{"metric_id":"e2e.cpu.load","value":"0.42","ts":...}]} +[main] PASS: viewer2 received the metric.frame broadcast triggered by the source +[main] ALL VIEWER E2E CHECKS PASSED +``` + +Source-side log for the same run (note the server's post-accept `command` +push mentioned in §2 item 3, arriving before the driver's own +`task.event`): + +``` +[source] recv hello {"stage":"accept","protocol_version":1,"session_id":"...","heartbeat_interval_ms":30000} +[source] recv command +[source] recv command +[source] send task.event (should trigger a live delta to subscribed viewers) +[source] recv ack {"acked":[{"device_id":"a85ee0f8-...","device_seq":1}]} +[source] send metric.frame (should broadcast to metric-subscribed viewers) +``` + +### What this confirms end-to-end + +- Real TCP/WebSocket connections across three separate Node/Go processes and + one real (local) Workers runtime — not an in-process fake. +- `resume{last_revision:0}` from a fresh viewer correctly returns a + `snapshot` (not a `delta`), matching the revision-gap-snapshot scenario in + `proto/realtime/fixtures/scenarios/client-revision-gap-snapshot/`. +- A live `task.event` from the source is turned into a `delta` broadcast to + every subscribed, lease-holding viewer, with revision continuity + (`snapshot.revision == delta.from_revision`, + `delta.to_revision == delta.from_revision + 1`). +- `metric.frame` from the source is broadcast to a viewer that declared + `metric` interest via `subscribe`, independent of the viewer that received + the live task delta — i.e. server fan-out works across N independent + viewer connections, not just the one that happened to trigger a resume. +- The server's post-hello-accept `command` push to `source` connections + (§2 item 3) is real and was observed live, not just inferred from code. + +No `wrangler dev` startup failure was encountered in this environment +(`NO_PROXY` already covered `127.0.0.1`/`localhost`), so the +vitest-pool-workers fallback described in the work order was not needed. + +### A bug found and fixed in the E2E scripts themselves (not a cross-line defect) + +The first draft of `source-drive.mjs`/`viewer-smoke.mjs` built each +envelope's `id` by embedding the message `type` literally (e.g. +`e2e-src-task.event-1-...`). `task.event` and `metric.frame` contain a `.`, +which `envelope_id`'s pattern (`^[A-Za-z0-9_-]{1,64}$`) forbids — the server +correctly rejected both with `error{code:"malformed", message:"invalid +envelope id"}`. Fixed by stripping non-alphanumeric characters out of the +type before using it in the id. This was a bug in the test fixture, not in +any of the three integrated lines. + +## 4. Integration fixes made in this worktree + +None. No cross-line contract mismatch or regression-suite failure required +a code change in `server/`, `daemon/`, or `apple/`. The only files added on +top of the three merged lines are the new opt-in E2E test/scripts +(`daemon/internal/realtime/client/e2e_test.go`, +`scripts/e2e/source-drive.mjs`, `scripts/e2e/viewer-smoke.mjs`, +`scripts/e2e/package.json`) and this document. + +## 5. Defects requiring a fix on the owning line + +None found. + +## 6. Residual risks + +- The post-hello-accept `command` push to `source` connections + (`server/src/realtime/space-hub.ts:476-488`) is documented in code as a + "v1.1 protocol clarification candidate" but is not yet reflected in + `proto/realtime/messages/hello.schema.json`'s prose. Both clients handle + it safely today; a future client implementation that assumes "nothing + arrives before my first subscribe/resume" would not be safe. Worth an + explicit spec note in a future protocol revision (out of scope here since + `proto/realtime/` is frozen for this integration pass). +- The E2E smoke test only exercises one source + two viewers in one space, + for a few seconds; it does not cover the interest-lease-expiry, duplicate- + connection-supersede, or duplicate-device-seq scenarios end-to-end over a + real network (those are covered at the fixture/unit level in all three + lines, and in the server's own vitest-pool-workers suite, but not by a + live cross-process run in this pass). +- `wrangler dev --local` state is ephemeral (fresh DO storage per run); the + space/device tokens embedded as examples above are from a throwaway local + run and are not valid against any deployed environment. From 6677acbbab0523b588877c6e436b8d05c4d25894 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:41:57 +0800 Subject: [PATCH 36/60] fix(server): log superseded connections as security events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Product invariant 5 requires supersession to never be silent — it can signal credential/session reuse. handlePreHello's supersession branch sent the wire error/close directly, bypassing reply() (the only path calling logAlways), so it left zero server-side trace. Emit an unconditional logAlways("superseded", {device_id, role, superseded_session_id, superseding_session_id}) alongside the existing send/close, without routing through reply() so the wire frames to the superseded peer stay byte-identical. Buffer logAlways events in a capped in-memory securityEventLog on SpaceHub so vitest-pool-workers tests can assert a log fired (console output isn't observable across the workerd/vitest process boundary), and extend the existing supersede test to check the log via runInDurableObject. --- server/src/realtime/space-hub.ts | 27 +++++++++++++++++++++++- server/test/workers/handshake.workers.ts | 20 +++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 3b58616..3910446 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -98,6 +98,15 @@ export class SpaceHub extends DurableObject { * (see logAlways) — only routine per-frame hot-path logging is sampled. */ logSampler: (n: number) => boolean = (n) => n % 100 === 0; + /** Every logAlways() (100%-sampled, security/error tier) event recorded + * by this DO instance, newest last. In-memory only, capped, and never + * persisted or read by production code — it exists purely so + * vitest-pool-workers tests can assert a log fired (e.g. on + * supersession) via `runInDurableObject(stub, (i) => (i as any). + * securityEventLog)`, since console output can't be captured across the + * workerd/vitest process boundary. */ + private securityEventLog: Array<{ event: string; data: Record }> = []; + constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); // Answers a bare "ping" text frame with "pong" entirely inside the @@ -449,6 +458,7 @@ export class SpaceHub extends DurableObject { // SPEC.md section 9.4: the new connection completing hello supersedes // any other open connection already authenticated for this device_id. + const newSessionId = crypto.randomUUID(); for (const other of this.ctx.getWebSockets(att.deviceId)) { if (other === ws) continue; const otherAtt = other.deserializeAttachment(); @@ -458,12 +468,22 @@ export class SpaceHub extends DurableObject { message: "device completed hello on a newer connection", ...ERROR_SEMANTICS.superseded, }); + // Product invariant 5: superseded must never be silent — it is a + // security-relevant event (possible credential/session reuse), so it + // always logs at 100% via logAlways, independent of the wire reply + // above (which must stay byte-identical for the superseded peer). + this.logAlways("superseded", { + device_id: att.deviceId, + role: att.role, + superseded_session_id: otherAtt.sessionId, + superseding_session_id: newSessionId, + }); other.close(1008, "superseded"); } } att.helloDone = true; - att.sessionId = crypto.randomUUID(); + att.sessionId = newSessionId; ws.serializeAttachment(att); this.send(ws, "hello", { @@ -977,6 +997,11 @@ export class SpaceHub extends DurableObject { private logAlways(event: string, data: Record): void { console.log(JSON.stringify({ level: "error", event, ...data, ts: Date.now() })); + this.securityEventLog.push({ event, data }); + // Bounded so a pathological run can't grow this without limit; recent + // history is all tests need, and this is never persisted or read by + // production logic. + if (this.securityEventLog.length > 200) this.securityEventLog.shift(); } } diff --git a/server/test/workers/handshake.workers.ts b/server/test/workers/handshake.workers.ts index bc9a2e0..1fd15ad 100644 --- a/server/test/workers/handshake.workers.ts +++ b/server/test/workers/handshake.workers.ts @@ -1,5 +1,7 @@ // Required coverage #8 (connection gating), #9 (supersede). +import { env, runInDurableObject } from "cloudflare:test"; import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, subscribe } from "./helpers"; describe("connection gating", () => { @@ -62,7 +64,7 @@ describe("connection gating", () => { }); it("supersede: an older connection for the same device is closed with `superseded`, not `throttle`", async () => { - const { source, viewer } = await bootstrapSpace(); + const { source, viewer, spaceId } = await bootstrapSpace(); const first = await connect(source.token); await helloSource(first, source.device_id); @@ -88,6 +90,22 @@ describe("connection gating", () => { expect(supersededErr.body.fatal).toBe(true); await first.waitForClose(); + // Product invariant 5: supersession must never be silent — it's a + // security-relevant event (possible credential/session reuse) and must + // always produce a server-side log, independent of the wire error sent + // above. Console output can't be captured across the workerd/vitest + // process boundary, so inspect the DO's in-memory security event log + // directly via runInDurableObject (same pattern used in errors.workers.ts). + const stub = env.SPACE_HUB.getByName(spaceId); + const securityEvents = await runInDurableObject( + stub, + async (instance: SpaceHub) => (instance as any).securityEventLog as Array<{ event: string; data: Record }>, + ); + const supersededLog = securityEvents.find((e) => e.event === "superseded"); + expect(supersededLog).toBeDefined(); + expect(supersededLog?.data.device_id).toBe(source.device_id); + expect(supersededLog?.data.role).toBe("source"); + // The new connection is fully functional. second.send({ type: "task.event", From 2e5cad76b8b4fbd83da51e0fd706f7f2c0475b35 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 12:46:44 +0800 Subject: [PATCH 37/60] fix(daemon): serialize realtime sends to preserve device_seq wire order Finding 2 (MAJOR, correctness race): on reconnect, client.connected was set true before replayPending drained unacked outbox rows, and a live SendTaskEvent/SendMessageEvent could write directly from the caller's goroutine via trySendNow with nothing ordering it against the replay's writes on the same connection. coder/websocket only guarantees frame integrity under concurrent Write, not ordering, so a freshly-enqueued higher device_seq could reach the wire before an older still-unacked device_seq was resent, letting viewers see task progress regress. Replace the direct-write trySendNow with a coalesced wake signal (wakeSender) consumed only by connectAndServe's own select loop, which already owns the connection's replayPending calls. Every reliable-event write for a given connection - initial post-reconnect replay, periodic resend, and now live sends - goes through replayPending from that single goroutine, so writes for one connection can never race each other onto the wire out of order. A live send during an in-flight replay is simply folded into the same durable-outbox-backed resend stream instead of racing it. Adds TestClientConcurrentSendsPreserveDeviceSeqOrderDuringReplay, which drives many concurrent SendTaskEvent calls across a forced reconnect and asserts the mock server's first-seen device_seq order is strictly increasing; verified it reliably reproduces the old race when run against the pre-fix code. --- daemon/internal/realtime/client/client.go | 67 ++++---- .../internal/realtime/client/client_test.go | 146 ++++++++++++++++++ 2 files changed, 187 insertions(+), 26 deletions(-) diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index 8a81261..33d0b48 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -14,7 +14,6 @@ import ( "github.com/coder/websocket" - "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" ) @@ -42,6 +41,17 @@ type Client struct { connected bool attempt int + // wake coalesces "check the outbox for new work" notifications into the + // single connection goroutine that owns writing reliable events to the + // wire (see connectAndServe's select loop and replayPending). It is + // intentionally never written to directly by anything other than + // wakeSender, and never read by anything other than connectAndServe: + // that single-writer/single-reader discipline is what guarantees + // device_seq order on the wire even when SendTaskEvent/SendMessageEvent + // are called concurrently, including during an in-flight replay after a + // reconnect (SPEC.md section 5.1). + wake chan struct{} + metrics *metricBatcher seenMu sync.Mutex @@ -60,6 +70,7 @@ func New(cfg Config) *Client { cfg.applyDefaults() c := &Client{ cfg: cfg, + wake: make(chan struct{}, 1), metrics: newMetricBatcher(cfg.MetricFlushInterval, cfg.MetricThrottledInterval), seen: make(map[string]bool), closing: make(chan struct{}), @@ -117,7 +128,7 @@ type TaskEvent struct { // survives in the outbox until acknowledged, across any number of // reconnects or process restarts. func (c *Client) SendTaskEvent(ev TaskEvent) error { - item, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeTaskEvent, func(seq int64) (json.RawMessage, error) { + _, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeTaskEvent, func(seq int64) (json.RawMessage, error) { body := wire.TaskEventBody{ DeviceID: c.cfg.DeviceID, DeviceSeq: seq, @@ -138,7 +149,7 @@ func (c *Client) SendTaskEvent(ev TaskEvent) error { if err != nil { return err } - c.trySendNow(item) + c.wakeSender() return nil } @@ -154,7 +165,7 @@ type MessageEvent struct { // SendMessageEvent durably enqueues a message event; see SendTaskEvent. func (c *Client) SendMessageEvent(ev MessageEvent) error { - item, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + _, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { id := ev.MessageID if id == "" { id = fmt.Sprintf("%s:%d", c.cfg.DeviceID, seq) @@ -176,7 +187,7 @@ func (c *Client) SendMessageEvent(ev MessageEvent) error { if err != nil { return err } - c.trySendNow(item) + c.wakeSender() return nil } @@ -188,28 +199,25 @@ func (c *Client) SendMetric(s wire.MetricSample) { c.metrics.Offer(s) } -// trySendNow best-effort writes one outbox item immediately if a -// connection is currently up. On any failure it does nothing further: the -// resend loop and the next reconnect's replay will pick it up. -func (c *Client) trySendNow(item outbox.Item) { - c.mu.Lock() - conn := c.conn - connected := c.connected - c.mu.Unlock() - if !connected || conn == nil { - return - } - env, err := wire.NewEnvelope(item.Kind, newEnvelopeID(), c.nowMS(), json.RawMessage(item.Body)) - if err != nil { - return - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - data, err := env.Encode() - if err != nil { - return +// wakeSender notifies the active connection's single writer goroutine +// (connectAndServe's select loop) that new work may be waiting in the +// outbox. It never writes to the wire itself and is always safe to call, +// including when no connection is currently up (the signal is simply +// coalesced/dropped and the next connection's initial replayPending call +// will pick up the item from durable storage regardless). +// +// This indirection — rather than writing directly from the caller's +// goroutine, as a prior version of this code did — is what guarantees wire +// order matches device_seq order (SPEC.md section 5.1): every reliable +// event write, whether a fresh live send or a reconnect replay, now goes +// through replayPending called from exactly one goroutine per connection, +// so two writes for the same connection can never race each other onto the +// wire out of order. +func (c *Client) wakeSender() { + select { + case c.wake <- struct{}{}: + default: } - _ = conn.Write(ctx, websocket.MessageText, data) } // ---- connection lifecycle ---- @@ -327,6 +335,13 @@ func (c *Client) connectAndServe() error { lastPong = c.now() case <-resendTicker.C: c.replayPending(ctx, conn) + case <-c.wake: + // A live SendTaskEvent/SendMessageEvent call enqueued a new + // item. replayPending re-reads the full outbox (oldest seq + // first) and writes from this same goroutine, so it can never + // race a concurrent replay/resend for this connection — it + // simply folds into the same single-writer stream. + c.replayPending(ctx, conn) case <-metricsTicker.C: c.flushMetrics(ctx, conn) case <-heartbeatTicker.C: diff --git a/daemon/internal/realtime/client/client_test.go b/daemon/internal/realtime/client/client_test.go index fec506c..b3983b4 100644 --- a/daemon/internal/realtime/client/client_test.go +++ b/daemon/internal/realtime/client/client_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "sync" "sync/atomic" "testing" "time" @@ -600,3 +601,148 @@ func TestClientThrottlePreservedAcrossReconnect(t *testing.T) { } waitFor(t, 3*time.Second, func() bool { return !c.MetricsThrottled() }) } + +// TestClientConcurrentSendsPreserveDeviceSeqOrderDuringReplay is a +// regression test for the reconnect race in SPEC.md section 5.1: +// device_seq values MUST appear on the wire in strictly increasing order. +// Before this was fixed, a live SendTaskEvent's write ran directly on the +// caller's own goroutine (trySendNow) with nothing ordering it against +// replayPending's writes on a freshly reconnected connection, so a +// freshly-enqueued higher device_seq could reach the wire before an older, +// still-unacked device_seq was resent. +// +// This test drives that race directly rather than merely asserting on the +// fix's internals: connection 1 accepts exactly one task.event and then +// drops without acking it (forcing a reconnect with a non-empty replay +// backlog), while many goroutines concurrently call SendTaskEvent — some +// landing before the drop, some during the reconnect/replay window on +// connection 2, and some once connection 2 is already steady-state. The +// mock server records every device_seq it observes, across both +// connections, in arrival order. The assertion is on first-occurrence +// order (a seq may legitimately repeat on the wire, e.g. a full-backlog +// resend re-covering an already-sent-but-still-unacked item; only the +// *first* time each seq is ever seen must be strictly increasing). +// +// Run with `go test -race`: the fix serializes all reliable-event writes +// for a connection through a single goroutine (connectAndServe's select +// loop, woken by wakeSender), which this test also exercises for data +// races on the client's own state under concurrent callers. +func TestClientConcurrentSendsPreserveDeviceSeqOrderDuringReplay(t *testing.T) { + const concurrent = 40 + + var mu sync.Mutex + var arrival []int64 + + var connNum int32 + srv := rttest.New(func(conn *rttest.Conn) { + n := atomic.AddInt32(&connNum, 1) + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + if n == 1 { + // Read exactly one task.event, record it, then drop the + // connection without acking — the client must reconnect with + // that event (and possibly more enqueued concurrently) still + // unacked in its outbox. + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + t.Errorf("decode task.event on connection 1: %v", derr) + return + } + tb := body.(wire.TaskEventBody) + mu.Lock() + arrival = append(arrival, tb.DeviceSeq) + mu.Unlock() + conn.Close("simulated drop") + return + } + } + // Every later connection: replay the backlog and everything sent + // afterward lands here. Ack each event as it arrives so the + // outbox eventually drains. + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + t.Errorf("decode task.event on connection %d: %v", n, derr) + continue + } + tb := body.(wire.TaskEventBody) + mu.Lock() + arrival = append(arrival, tb.DeviceSeq) + mu.Unlock() + if err := conn.Ack(tb.DeviceID, tb.DeviceSeq); err != nil { + return + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + var wg sync.WaitGroup + wg.Add(concurrent) + for i := 0; i < concurrent; i++ { + go func(i int) { + defer wg.Done() + if err := c.SendTaskEvent(TaskEvent{ + TaskID: fmt.Sprintf("run-%d", i), + Kind: "started", + OccurredAt: time.Now(), + }); err != nil { + t.Errorf("SendTaskEvent(%d): %v", i, err) + } + }(i) + } + wg.Wait() + + waitFor(t, 5*time.Second, func() bool { + pending, err := store.Pending(context.Background(), testSpace) + return err == nil && len(pending) == 0 + }) + + mu.Lock() + got := append([]int64(nil), arrival...) + mu.Unlock() + + seen := make(map[int64]bool, concurrent) + var order []int64 + for _, seq := range got { + if !seen[seq] { + seen[seq] = true + order = append(order, seq) + } + } + for i := 1; i < len(order); i++ { + if order[i] <= order[i-1] { + t.Fatalf("device_seq out of order on the wire: first-seen order %v (full arrival log %v) — SPEC.md section 5.1 requires strictly increasing device_seq", order, got) + } + } + if len(order) != concurrent { + t.Fatalf("expected %d distinct device_seq values delivered, got %d: %v", concurrent, len(order), order) + } +} From 595452c34ea604508c112ddbf6685e3228ed11ec Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:19:15 +0800 Subject: [PATCH 38/60] feat(daemon): wire realtime outbox row cap to config Add Config.OutboxMaxRows (SITREP_OUTBOX_MAX_ROWS, default unset -> outbox.DefaultMaxRows) so the outbox's 5000-row cap is no longer a hardcoded constant, and pass it through when the agent opens the outbox store. --- daemon/cmd/sitrep/agent.go | 2 +- daemon/internal/config/config.go | 15 ++++++++++++++ daemon/internal/config/config_test.go | 30 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/daemon/cmd/sitrep/agent.go b/daemon/cmd/sitrep/agent.go index e6d4255..23b2920 100644 --- a/daemon/cmd/sitrep/agent.go +++ b/daemon/cmd/sitrep/agent.go @@ -108,7 +108,7 @@ func newAgentRealtimeUplink() *rtclient.Client { fmt.Fprintln(os.Stderr, "sitrep agent: realtime uplink enabled but server/device_id/space is not configured; staying on HTTP ingest") return nil } - store, err := outbox.Open(config.RealtimeOutboxPath()) + store, err := outbox.OpenWithMaxRows(config.RealtimeOutboxPath(), cfg.OutboxMaxRows) if err != nil { fmt.Fprintf(os.Stderr, "sitrep agent: realtime outbox: %v; staying on HTTP ingest\n", err) return nil diff --git a/daemon/internal/config/config.go b/daemon/internal/config/config.go index 1a2cb41..4993440 100644 --- a/daemon/internal/config/config.go +++ b/daemon/internal/config/config.go @@ -7,6 +7,7 @@ import ( "encoding/json" "os" "path/filepath" + "strconv" "strings" ) @@ -30,6 +31,15 @@ type Config struct { // package derives from Server by default. Overridable with // SITREP_REALTIME_URL. RealtimeURL string `json:"realtime_url,omitempty"` + + // OutboxMaxRows bounds the local realtime outbox's row count (see + // internal/realtime/outbox.DefaultMaxRows) — the point at which further + // reliable events must wait for local backpressure rather than being + // enqueued, because the outbox is the sole authority for realtime + // delivery while RealtimeEnabled is on (no legacy-HTTP fallback for + // those events; see internal/uplink.routeToRealtime). Zero/unset uses + // outbox.DefaultMaxRows. Overridable with SITREP_OUTBOX_MAX_ROWS. + OutboxMaxRows int `json:"outbox_max_rows,omitempty"` } func Path() string { @@ -67,6 +77,11 @@ func Load() Config { if v := os.Getenv("SITREP_REALTIME_URL"); v != "" { cfg.RealtimeURL = v } + if v := os.Getenv("SITREP_OUTBOX_MAX_ROWS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + cfg.OutboxMaxRows = n + } + } return cfg } diff --git a/daemon/internal/config/config_test.go b/daemon/internal/config/config_test.go index dcd5217..f2aea44 100644 --- a/daemon/internal/config/config_test.go +++ b/daemon/internal/config/config_test.go @@ -2,6 +2,36 @@ package config import "testing" +// TestLoadOutboxMaxRows pins the SITREP_OUTBOX_MAX_ROWS wiring (the P0 +// review's "cap isn't wired to config" note): Load must surface it as +// Config.OutboxMaxRows, defaulting to 0 (meaning "use +// outbox.DefaultMaxRows") when unset or invalid. +func TestLoadOutboxMaxRows(t *testing.T) { + // HOME points at an empty temp dir so Load never reads a real user + // config.json out from under this test. + t.Setenv("HOME", t.TempDir()) + + cases := []struct { + name string + env string + want int + }{ + {name: "unset defaults to zero (outbox.DefaultMaxRows)", env: "", want: 0}, + {name: "explicit value wins", env: "12345", want: 12345}, + {name: "non-numeric is ignored", env: "not-a-number", want: 0}, + {name: "zero is ignored (not a valid cap)", env: "0", want: 0}, + {name: "negative is ignored", env: "-5", want: 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Setenv("SITREP_OUTBOX_MAX_ROWS", c.env) + if got := Load().OutboxMaxRows; got != c.want { + t.Fatalf("Load().OutboxMaxRows = %d, want %d", got, c.want) + } + }) + } +} + // TestRealtimeURLFor pins the agreed cross-implementation realtime // endpoint path (/v3/realtime) and the http(s) -> ws(s) scheme mapping. func TestRealtimeURLFor(t *testing.T) { From a0dc30d8339f59eadb81646401b1df2fdc4463e0 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:19:17 +0800 Subject: [PATCH 39/60] feat(server): add realtime_enabled kill switch to /v2/snapshot The Apple client auto-connects to /v3/realtime whenever paired, with no server-side gate. Add a top-level realtime_enabled boolean to the /v2/ snapshot response, sourced from the REALTIME_ENABLED wrangler var and defaulting to false so older servers (field absent) and unset envs both read as disabled. No per-space granularity in v1. --- server/src/adapters/workers.ts | 2 ++ server/src/app.ts | 27 +++++++++++++++++++-------- server/test/app.test.ts | 13 +++++++++++++ server/worker-configuration.d.ts | 5 +++-- server/wrangler.jsonc | 6 +++++- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index 3c9c654..cde4d25 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -44,6 +44,7 @@ interface Secrets { APNS_TEAM_ID?: string; APNS_BUNDLE_ID?: string; // var, wrangler.jsonc APNS_HOST?: string; // var, wrangler.jsonc + REALTIME_ENABLED?: boolean; // var, wrangler.jsonc; unset/false disables /v3/realtime } type WorkerEnv = Env & Secrets; @@ -483,6 +484,7 @@ const app = createApp({ await (c.env as WorkerEnv).INVITE_DIR.put(code, space, { expirationTtl: 600 }); }, lookupInvite: (c, code) => (c.env as WorkerEnv).INVITE_DIR.get(code), + realtimeEnabled: (c) => Boolean((c.env as WorkerEnv).REALTIME_ENABLED), }); app.get("/debug/tokens", async (c: any) => { diff --git a/server/src/app.ts b/server/src/app.ts index 0ee853b..6e70f52 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -69,6 +69,11 @@ export interface AppOptions { * joining by code alone needs a code→space lookup with TTL. */ publishInvite?: (c: Context, code: string, space: string) => Promise; lookupInvite?: (c: Context, code: string) => Promise; + /** Realtime kill switch (`/v3/realtime`): a Wrangler `vars` entry + * (`REALTIME_ENABLED`), no per-space granularity in v1. Absent on older + * deployments that don't wire this option, which must mean disabled — so + * the default below is `false`, never `true`. */ + realtimeEnabled?: (c: Context) => boolean; } type Vars = { Variables: { role: Role; space: string; deviceId?: string } }; @@ -242,14 +247,20 @@ export function createApp(opts: AppOptions) { s.automations(), s.presence(), ]); - return c.json(makeSnapshot({ - now: new Date().toISOString(), - presence, - tasks: visibleTasks(tasks, false), - metrics: metrics.map((metric) => mergeMetric(metric, prefs)), - events, - automations, - })); + return c.json({ + ...makeSnapshot({ + now: new Date().toISOString(), + presence, + tasks: visibleTasks(tasks, false), + metrics: metrics.map((metric) => mergeMetric(metric, prefs)), + events, + automations, + }), + // Apple client's server-side kill switch for /v3/realtime auto-connect + // (see AppOptions.realtimeEnabled): missing or unset must read as + // disabled, so this defaults to false rather than being omitted. + realtime_enabled: opts.realtimeEnabled ? opts.realtimeEnabled(c) : false, + }); }); app.get("/v2/automations", async (c) => { diff --git a/server/test/app.test.ts b/server/test/app.test.ts index f28b370..fbe6bd0 100644 --- a/server/test/app.test.ts +++ b/server/test/app.test.ts @@ -8,6 +8,19 @@ function testApp() { return createApp({ store: () => store }); } +test("v2 snapshot realtime_enabled defaults false when unwired", async () => { + const app = testApp(); + const snapshot = await (await app.request("/v2/snapshot")).json() as { realtime_enabled: boolean }; + assert.equal(snapshot.realtime_enabled, false); +}); + +test("v2 snapshot realtime_enabled reflects the configured var", async () => { + const store = new SqliteStore(":memory:"); + const app = createApp({ store: () => store, realtimeEnabled: () => true }); + const snapshot = await (await app.request("/v2/snapshot")).json() as { realtime_enabled: boolean }; + assert.equal(snapshot.realtime_enabled, true); +}); + test("Node ingest returns success and records presence", async () => { const app = testApp(); const response = await app.request("/v2/ingest", { diff --git a/server/worker-configuration.d.ts b/server/worker-configuration.d.ts index b275505..0c91243 100644 --- a/server/worker-configuration.d.ts +++ b/server/worker-configuration.d.ts @@ -1,10 +1,11 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 4c0b4da54160bdee5b303d3a4232198a) +// Generated by Wrangler by running `wrangler types` (hash: 0eac4d6591f9e4c5dd04df5e8b07aced) // Runtime types generated with workerd@1.20260714.1 2026-07-18 nodejs_compat interface __BaseEnv_Env { INVITE_DIR: KVNamespace; APNS_BUNDLE_ID: "dev.sitrep.app"; APNS_HOST: "api.sandbox.push.apple.com"; + REALTIME_ENABLED: false; USER_STORE: DurableObjectNamespace; SPACE_HUB: DurableObjectNamespace; } @@ -20,7 +21,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } // Begin runtime types diff --git a/server/wrangler.jsonc b/server/wrangler.jsonc index e6884ee..6d84bbc 100644 --- a/server/wrangler.jsonc +++ b/server/wrangler.jsonc @@ -9,7 +9,11 @@ "kv_namespaces": [{ "binding": "INVITE_DIR", "id": "490111734d5246cc949f0eec1c0ca0f3" }], "vars": { "APNS_BUNDLE_ID": "dev.sitrep.app", - "APNS_HOST": "api.sandbox.push.apple.com" + "APNS_HOST": "api.sandbox.push.apple.com", + // Server-side kill switch for /v3/realtime auto-connect (Fix A, pre-merge + // review). Absent/false means disabled; flip to true per-environment to + // enable. No per-space granularity in v1. + "REALTIME_ENABLED": false }, "durable_objects": { "bindings": [ From 569eb5838b30b3b2ceaacab13667c20f3b3fb062 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:19:24 +0800 Subject: [PATCH 40/60] fix(daemon): stop forking reliable events to /v2 on outbox-full MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit routeToRealtime previously fell back to legacy /v2 HTTP ingest whenever the realtime outbox returned ErrOutboxFull. /v2 writes UserStore while /v3 viewers resume from SpaceHub, so a diverted task.done/task.fail/ message.send could permanently vanish from a viewer's resume — the two stores forked with no reconciliation. While the realtime flag is on, reliable events (task.event/ message.event) now never take the HTTP path. On ErrOutboxFull the send is queued in-memory and retried oldest-first on every flush tick (Uplink.rtRetry/retryRealtime) until Enqueue succeeds, which self-heals as soon as acks drain the backlog — events already durable in the outbox replay in seq order on their own. This is local backpressure, not a fallback, and Offer() itself still never blocks on the network. Rewrites TestOutboxFullFallsBackToHTTPAndRecovers (now TestOutboxFullNeverForksToHTTPAndRecovers) to assert the opposite of the old behavior: the overflow event must never reach the HTTP ingest spy, and must still arrive over realtime once the outbox drains. --- daemon/internal/uplink/realtime_test.go | 51 ++++++------ daemon/internal/uplink/uplink.go | 106 +++++++++++++++++++++--- 2 files changed, 121 insertions(+), 36 deletions(-) diff --git a/daemon/internal/uplink/realtime_test.go b/daemon/internal/uplink/realtime_test.go index 6503efe..8722b80 100644 --- a/daemon/internal/uplink/realtime_test.go +++ b/daemon/internal/uplink/realtime_test.go @@ -135,12 +135,15 @@ func TestRealtimeFlagOffPreservesExistingBehavior(t *testing.T) { } } -// TestOutboxFullFallsBackToHTTPAndRecovers pins the bounded-outbox policy -// end to end: when the realtime outbox hits its row cap, further reliable -// events fall back to the HTTP ingest path (reliability does not degrade, -// disk usage stays bounded), and once acks drain the backlog the realtime -// path takes over again. -func TestOutboxFullFallsBackToHTTPAndRecovers(t *testing.T) { +// TestOutboxFullNeverForksToHTTPAndRecovers pins the P0 fix: /v2 ingest +// writes UserStore while /v3 viewers resume from SpaceHub, so a reliable +// event (task.event/message.event) diverted to /v2 while the realtime flag +// is on could permanently vanish from a viewer's resume. When the realtime +// outbox hits its row cap, the overflow event must NOT reach the HTTP +// ingest spy at all — it stays queued locally (bounded backpressure) and is +// retried until Enqueue succeeds, then delivered over realtime in seq +// order once acks drain the backlog. +func TestOutboxFullNeverForksToHTTPAndRecovers(t *testing.T) { var httpCap capture httpSrv := httptest.NewServer(httpCap.handler(t)) defer httpSrv.Close() @@ -226,23 +229,17 @@ func TestOutboxFullFallsBackToHTTPAndRecovers(t *testing.T) { u.Offer(msg("first")) waitForWS("first") - // 2. Second event: outbox is full -> ErrOutboxFull -> HTTP fallback. + // 2. Second event: outbox is full -> ErrOutboxFull. Per the P0 fix this + // must NOT fall back to HTTP; it is queued locally (Uplink.rtRetry) and + // retried every flush tick while the outbox stays full. Give it several + // flush intervals to (wrongly) reach HTTP if the fallback regressed, + // then assert it never did. u.Offer(msg("second")) - deadline := time.Now().Add(3 * time.Second) - for { - found := false - for _, e := range httpCap.all() { - if e.Kind == protocol.MessageSend && e.Text == "second" { - found = true - } - } - if found { - break - } - if time.Now().After(deadline) { - t.Fatal("HTTP ingest never carried the overflow event while the outbox was full") + time.Sleep(200 * time.Millisecond) + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend && e.Text == "second" { + t.Fatalf("HTTP ingest carried the overflow event %q while the outbox was full — reliable events must never fork off the realtime path", e.Text) } - time.Sleep(5 * time.Millisecond) } // 3. The server starts acking; the resend loop replays "first", it gets @@ -264,13 +261,19 @@ func TestOutboxFullFallsBackToHTTPAndRecovers(t *testing.T) { return err == nil && n == 0 }) - // 4. With capacity restored, the realtime path takes over again. + // 4. Once the outbox drains, the locally-queued "second" event's retry + // succeeds and it is delivered over realtime — never having touched + // HTTP — in the correct seq order (it was offered before "third"). + waitForWS("second") + + // 5. With capacity restored, the realtime path continues to take + // subsequent events. u.Offer(msg("third")) waitForWS("third") - // "first" and "third" must never have gone over HTTP. + // None of these reliable events may ever have gone over HTTP. for _, e := range httpCap.all() { - if e.Kind == protocol.MessageSend && e.Text != "second" { + if e.Kind == protocol.MessageSend { t.Fatalf("HTTP ingest carried %q, which should have traveled the realtime path only", e.Text) } } diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index c3a0443..8de7d9c 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -12,6 +12,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "strings" @@ -20,6 +21,7 @@ import ( "github.com/QuintinShaw/sitrep/daemon/internal/protocol" rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" ) @@ -60,6 +62,11 @@ type Config struct { // Nil (the default) preserves this package's exact prior behavior. // The caller owns Realtime's lifecycle (construction and Close). Realtime *rtclient.Client + + // Logf is a pluggable logger for conditions worth surfacing but not + // worth failing on (e.g. a reliable event dropped because it could + // never validate). Nil discards. + Logf func(format string, args ...any) } type Uplink struct { @@ -78,6 +85,14 @@ type Uplink struct { kick chan struct{} done chan struct{} + // rtRetry holds reliable events (task.event/message.event) whose + // enqueue into the realtime outbox hit outbox.ErrOutboxFull. They are + // retried, oldest first, on every flush tick until Enqueue succeeds — + // see routeToRealtime and retryRealtime. This is local backpressure, + // not a fallback: these events must not travel the legacy HTTP path + // while the realtime flag is on (see package doc on Config.Realtime). + rtRetry []func() error + ticksSinceSend int } @@ -93,6 +108,9 @@ func New(cfg Config) *Uplink { if cfg.HTTPTimeout <= 0 { cfg.HTTPTimeout = 10 * time.Second } + if cfg.Logf == nil { + cfg.Logf = func(string, ...any) {} + } u := &Uplink{ cfg: cfg, client: &http.Client{Timeout: cfg.HTTPTimeout}, @@ -157,15 +175,15 @@ func (u *Uplink) Offer(ev Event) { // routeToRealtime translates ev into the matching realtime-protocol // message and hands it to the realtime client. It reports whether it // handled ev (true) so the caller skips the HTTP batch entirely for that -// event, or leaves it (false) to fall through unchanged — either because -// the kind has no realtime equivalent (task.log) or because the realtime -// send itself failed, in which case losing the event silently would be -// worse than a harmless duplicate over HTTP. In particular, a full -// realtime outbox (outbox.ErrOutboxFull — the bounded local queue hit its -// row cap because the server has not been acking) lands here as a send -// failure: the event travels the HTTP ingest path instead, so reliability -// does not degrade while local disk usage stays bounded, and once acks -// drain the backlog subsequent events take the realtime path again. +// event, or leaves it (false) to fall through unchanged — only because the +// kind has no realtime equivalent (task.log). +// +// Reliable events (task.event/message.event) NEVER fall back to the legacy +// /v2 HTTP path while the realtime flag is on, even when the send fails: +// /v2 ingest writes UserStore, while /v3 viewers resume from SpaceHub, so +// diverting so much as one terminal task.done/task.fail/message.send would +// let a viewer permanently miss it. See sendReliable for what happens to a +// send failure instead. func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { switch ev.Kind { case protocol.TaskStart, protocol.TaskProgress, protocol.TaskStep, protocol.TaskDone, protocol.TaskFail: @@ -187,13 +205,14 @@ func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { case protocol.TaskDone, protocol.TaskFail: te.Message = ev.Text } - return realtime.SendTaskEvent(te) == nil + return u.sendReliable(func() error { return realtime.SendTaskEvent(te) }) case protocol.MessageSend: - return realtime.SendMessageEvent(rtclient.MessageEvent{ + me := rtclient.MessageEvent{ Level: ev.Level, Text: ev.Text, OccurredAt: parseEventTime(ev.TS), - }) == nil + } + return u.sendReliable(func() error { return realtime.SendMessageEvent(me) }) case protocol.MetricUpdate: realtime.SendMetric(wire.MetricSample{ MetricID: ev.Key, @@ -213,6 +232,68 @@ func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { } } +// sendReliable calls send() once and always reports "handled" (true) for a +// reliable event — the caller (routeToRealtime) must never fall back to +// HTTP for these, so there is nothing left for it to do with the return +// value except decide whether to retry locally. +// +// On outbox.ErrOutboxFull (the bounded local queue hit its row cap because +// the server has not been acking), send is queued for retry: local +// backpressure, not a fallback. It is retried oldest-first on every flush +// tick (retryRealtime) until Enqueue succeeds, which self-heals once acks +// drain the backlog (SPEC.md section 5.3) — events already in the outbox +// are durable in SQLite and replay in seq order on their own. FIFO order +// matters here: device_seq is allocated only by a successful Enqueue, so +// retrying anything but the oldest queued item first could hand a later +// event a lower seq than an earlier one still waiting. +// +// Any other error (e.g. a malformed event failing wire validation) is not +// a transient condition retrying would fix, and per the same no-fallback +// rule it must not be forwarded to /v2 either — it is logged and dropped. +func (u *Uplink) sendReliable(send func() error) bool { + if err := send(); err != nil { + if errors.Is(err, outbox.ErrOutboxFull) { + u.mu.Lock() + u.rtRetry = append(u.rtRetry, send) + u.mu.Unlock() + select { + case u.kick <- struct{}{}: + default: + } + } else { + u.cfg.Logf("uplink: dropping reliable event, realtime enqueue failed: %v", err) + } + } + return true +} + +// retryRealtime re-attempts every event queued by sendReliable, oldest +// first, stopping at the first one that still fails so order is preserved +// (see sendReliable). Called once per flush tick from loop, so a drained +// outbox flushes its whole backlog within one tick. +func (u *Uplink) retryRealtime() { + for { + u.mu.Lock() + if len(u.rtRetry) == 0 { + u.mu.Unlock() + return + } + send := u.rtRetry[0] + u.mu.Unlock() + + err := send() + if err != nil && errors.Is(err, outbox.ErrOutboxFull) { + return // still full; try again next tick + } + if err != nil { + u.cfg.Logf("uplink: dropping reliable event, realtime enqueue failed on retry: %v", err) + } + u.mu.Lock() + u.rtRetry = u.rtRetry[1:] + u.mu.Unlock() + } +} + func taskEventKind(k protocol.Kind) string { switch k { case protocol.TaskStart: @@ -296,6 +377,7 @@ func (u *Uplink) loop() { case <-ticker.C: case <-u.kick: } + u.retryRealtime() batch, closed := u.drain() if len(batch) > 0 { u.send(batch) From 0a0bb07c07ed3bf8c884c39268e04a5e72578f2c Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:19:57 +0800 Subject: [PATCH 41/60] fix(server): configure explicit observability log sampling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "observability": { "enabled": true } alone doesn't bound per-invocation telemetry, so the documented cost model wasn't real. Disable the platform's automatic invocation logs and keep structured logs unsampled (logs.head_sampling_rate: 1) so logAlways() security events (superseded, protocol errors) are never head-sampled away — volume is bounded instead by SpaceHub's existing in-code logSampler. Traces, which have no logAlways equivalent, are head-sampled at 1%. Update the cost model doc with the final config and revised estimate. --- docs/design/realtime-server.md | 51 ++++++++++++++++++++++++++++++++++ server/wrangler.jsonc | 17 +++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/design/realtime-server.md b/docs/design/realtime-server.md index ad0169e..1523ec1 100644 --- a/docs/design/realtime-server.md +++ b/docs/design/realtime-server.md @@ -99,6 +99,57 @@ counts against the attached SQLite database). today they only grow, which is an accepted, called-out gap (see the handoff's "known gaps" section). +## Observability config + +`wrangler.jsonc`'s `observability` block is the other half of the cost +model above — `logSampler`'s ≤1% in-code sampling only bounds what +SpaceHub *chooses* to write to `console.log` (`logHotPath`); it does +nothing about the Workers platform's own per-invocation telemetry, which +fires once per `webSocketMessage` call regardless of what the handler logs. +Final configuration: + +```jsonc +"observability": { + "enabled": true, + "logs": { "enabled": true, "head_sampling_rate": 1, "invocation_logs": false }, + "traces": { "enabled": true, "head_sampling_rate": 0.01 } +} +``` + +- **`logs.head_sampling_rate: 1` (i.e. no head sampling on structured + logs).** This is the deliberate choice, not an oversight: `logAlways()` + (superseded connections, protocol errors, unhandled exceptions — the + security/error tier) is a correctness requirement that must reach 100% + of the time, and Workers Logs head sampling is applied indiscriminately + before the Worker code runs, so any rate below 1.0 here would silently + drop `logAlways` events alongside the hot-path ones. Volume is bounded + instead at the *source*, in-code, by `logSampler` (`≤1%` of + `logHotPath` calls) — a rate this repository controls and tests directly + (`space-hub.ts`'s injectable `logSampler`), rather than a platform knob + that can't distinguish log tiers. +- **`logs.invocation_logs: false`.** This is what actually bounds request + volume: Workers' automatic invocation log is emitted once per request + (once per WebSocket message callback here) independent of anything the + Worker code logs, and cannot be tier-aware the way `logSampler` is. + Disabling it removes the one telemetry source neither `logAlways` nor + `logSampler` was ever designed to control. +- **`traces.head_sampling_rate: 0.01`.** Traces have no `logAlways` + equivalent (nothing security-relevant depends on a trace existing), so + they're sampled at the platform level like any other diagnostic-only + signal. + +Revised cost estimate: per-space WebSocket message volume is bounded by +the protocol itself (10 `metric.frame`/s per connection, SPEC.md section +11, plus whatever rate reliable events arrive at) — call it on the order +of a few thousand `webSocketMessage` invocations/day for an actively-used +space. With `invocation_logs: false`, none of those turn into a platform +invocation-log write; with `logSampler` at ≤1%, `logHotPath` contributes +at most ~1 structured log line per 100 frames; `logAlways` contributes a +handful of lines per space (supersession, occasional protocol errors) — +negligible even at 100% sampling. Net: structured log volume scales with +*intentional* logging, not raw message count, which is what makes the +`head_sampling_rate: 1` choice on `logs` affordable. + ## Hibernation SpaceHub is written so that an idle connection costs nothing beyond the diff --git a/server/wrangler.jsonc b/server/wrangler.jsonc index 6d84bbc..5260d24 100644 --- a/server/wrangler.jsonc +++ b/server/wrangler.jsonc @@ -4,7 +4,22 @@ "main": "src/adapters/workers.ts", "compatibility_date": "2026-07-18", "compatibility_flags": ["nodejs_compat"], - "observability": { "enabled": true }, + // Observability (Fix B, pre-merge review): "enabled": true alone only + // turns telemetry on — it does not bound invocation-log volume, so the + // documented cost model (docs/design/realtime-server.md) wasn't real. + // logAlways() security events (e.g. "superseded") must never be + // head-sampled away, so `logs.head_sampling_rate` stays at 1.0 (its + // default) and the ≤1% in-code `logSampler` in SpaceHub is what actually + // bounds hot-path console.log volume. `invocation_logs: false` is what + // bounds the OTHER, uncappable log source — the platform's automatic + // per-request invocation log, emitted once per webSocketMessage + // regardless of what the handler logs. Traces are diagnostic-only (no + // logAlways equivalent), so they're head-sampled at 1%. + "observability": { + "enabled": true, + "logs": { "enabled": true, "head_sampling_rate": 1, "invocation_logs": false }, + "traces": { "enabled": true, "head_sampling_rate": 0.01 } + }, "routes": [{ "pattern": "sitrep.quintinshaw.com", "custom_domain": true }], "kv_namespaces": [{ "binding": "INVITE_DIR", "id": "490111734d5246cc949f0eec1c0ca0f3" }], "vars": { From e890d50fdaa6708a19f0cf8b5d4c1f235d1ba386 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:20:42 +0800 Subject: [PATCH 42/60] feat(apple): decode realtime_enabled and add capability gate logic SpaceSnapshot gains realtime_enabled (decode-if-present, default false) so old/undeployed servers that never send the field are treated as "no realtime" rather than inferred from URL shape. RealtimeCapabilityGate is pure decision logic (connect/disconnect/none) kept in SitrepKit so the P0 gate is unit-testable without a live socket, and RealtimeClient itself stays capability-agnostic. --- .../SitrepKit/Sources/SitrepKit/Models.swift | 33 ++++++++ .../Realtime/RealtimeCapabilityGate.swift | 54 +++++++++++++ .../Tests/SitrepKitTests/ModelsTests.swift | 36 +++++++++ .../RealtimeCapabilityGateTests.swift | 77 +++++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100644 apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift create mode 100644 apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift diff --git a/apple/SitrepKit/Sources/SitrepKit/Models.swift b/apple/SitrepKit/Sources/SitrepKit/Models.swift index 9db7d2f..4f2d1d7 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Models.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Models.swift @@ -182,10 +182,43 @@ public struct SpaceSnapshot: Codable, Sendable { public var metrics: [SnapshotMetric] public var messages: [SnapshotMessage] public var automations: [AutomationInfo] + /// Server-declared realtime capability gate. Older servers never send + /// this field, and its absence must mean "no realtime" — decode + /// missing/null as `false`, never infer availability from anything else + /// (e.g. the presence of a `/v3/realtime` URL shape). + public var realtimeEnabled: Bool enum CodingKeys: String, CodingKey { case version, presence, tasks, metrics, messages, automations case generatedAt = "generated_at" + case realtimeEnabled = "realtime_enabled" + } + + public init( + version: Int, generatedAt: Date, presence: PresenceInfo, tasks: [TaskState], + metrics: [SnapshotMetric], messages: [SnapshotMessage], automations: [AutomationInfo], + realtimeEnabled: Bool = false + ) { + self.version = version + self.generatedAt = generatedAt + self.presence = presence + self.tasks = tasks + self.metrics = metrics + self.messages = messages + self.automations = automations + self.realtimeEnabled = realtimeEnabled + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + version = try container.decode(Int.self, forKey: .version) + generatedAt = try container.decode(Date.self, forKey: .generatedAt) + presence = try container.decode(PresenceInfo.self, forKey: .presence) + tasks = try container.decode([TaskState].self, forKey: .tasks) + metrics = try container.decode([SnapshotMetric].self, forKey: .metrics) + messages = try container.decode([SnapshotMessage].self, forKey: .messages) + automations = try container.decode([AutomationInfo].self, forKey: .automations) + realtimeEnabled = try container.decodeIfPresent(Bool.self, forKey: .realtimeEnabled) ?? false } } diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift new file mode 100644 index 0000000..da57cd5 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift @@ -0,0 +1,54 @@ +import Foundation + +/// Pure decision logic for the realtime capability P0 gate: the app must +/// only ever open a `RealtimeClient` connection when the most recent +/// successful REST refresh reported `realtime_enabled == true` on +/// `SpaceSnapshot`. Field absent/false (old or not-yet-rolled-out servers) +/// means "do not connect" — never inferred from URL shape or a `/v3` probe. +/// +/// Deliberately free of any RealtimeClient/URLSession/actor dependency so it +/// can be unit-tested without a live socket. `RealtimeClient` itself stays +/// capability-agnostic; the caller (AppModel) owns one of these and asks it +/// what to do each time a REST refresh completes. +public struct RealtimeCapabilityGate: Sendable, Equatable { + /// What the caller should do to the RealtimeClient in response to a + /// refreshed capability value. + public enum Action: Sendable, Equatable { + /// No change — either the value didn't change, or a repeat of an + /// already-applied value. + case none + /// Capability just turned on (including the very first time a + /// refresh reports `true`) — safe to (re)start the connection. + case connect + /// Capability just turned off (server rolled the flag back, or a + /// legacy/absent response) while previously on — tear down any live + /// connection and fall back to plain HTTP; no reconnect attempts + /// until a later refresh reports `true` again. + case disconnect + } + + /// Cold start (before any REST refresh has ever completed) must behave + /// as "not capable" — this default is what makes that safe without any + /// extra bookkeeping at call sites. + public private(set) var isCapable = false + + public init() {} + + /// Feed the `realtime_enabled` value from a just-completed REST + /// refresh. Returns the action the caller should take on its + /// RealtimeClient. Idempotent: repeating the same value is `.none`. + @discardableResult + public mutating func apply(refreshedCapability enabled: Bool) -> Action { + guard enabled != isCapable else { return .none } + isCapable = enabled + return enabled ? .connect : .disconnect + } + + /// Force back to the cold-start state — used when the identity this + /// gate was tracking no longer applies (re-pairing to a different + /// server/space, or disconnecting entirely). The next refresh against + /// the new identity must prove capability again before connecting. + public mutating func reset() { + isCapable = false + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift index 8a4cc31..af1b1a6 100644 --- a/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift +++ b/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift @@ -13,4 +13,40 @@ final class ModelsTests: XCTestCase { XCTAssertEqual(task.status, .running) XCTAssertEqual(task.percent, 45) } + + // MARK: - realtime_enabled (P0 capability gate input) + + private func decodeSnapshot(extraTopLevelField: String) throws -> SpaceSnapshot { + let json = """ + { + "version": 2, + "generated_at": "2026-07-17T12:00:00Z", + "presence": {}, + "tasks": [], + "metrics": [], + "messages": [], + "automations": []\(extraTopLevelField) + } + """ + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return try decoder.decode(SpaceSnapshot.self, from: Data(json.utf8)) + } + + func testSnapshotDecodesRealtimeEnabledTrue() throws { + let snapshot = try decodeSnapshot(extraTopLevelField: #", "realtime_enabled": true"#) + XCTAssertTrue(snapshot.realtimeEnabled) + } + + func testSnapshotDecodesRealtimeEnabledFalse() throws { + let snapshot = try decodeSnapshot(extraTopLevelField: #", "realtime_enabled": false"#) + XCTAssertFalse(snapshot.realtimeEnabled) + } + + /// Old servers never send this field — absence must decode to `false`, + /// never inferred from anything else (P0 fix). + func testSnapshotDecodesRealtimeEnabledAbsentAsFalse() throws { + let snapshot = try decodeSnapshot(extraTopLevelField: "") + XCTAssertFalse(snapshot.realtimeEnabled) + } } diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift new file mode 100644 index 0000000..5f90405 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift @@ -0,0 +1,77 @@ +import XCTest +@testable import SitrepKit + +/// Pure decision-logic tests for the P0 capability gate: whether a +/// RealtimeClient connection may be opened is decided solely from the most +/// recent REST refresh's `realtime_enabled`. No live socket involved — this +/// exercises `RealtimeCapabilityGate` directly. +final class RealtimeCapabilityGateTests: XCTestCase { + func testColdStartDefaultsToNotCapable() { + // Before any refresh has ever completed, the gate must already read + // as "off" — this is what makes a cold start before first refresh + // safe without any extra bookkeeping at call sites. + let gate = RealtimeCapabilityGate() + XCTAssertFalse(gate.isCapable) + } + + func testFirstRefreshTrueConnects() { + var gate = RealtimeCapabilityGate() + let action = gate.apply(refreshedCapability: true) + XCTAssertEqual(action, .connect) + XCTAssertTrue(gate.isCapable) + } + + func testRefreshFalseFromColdStartStaysOff() { + var gate = RealtimeCapabilityGate() + let action = gate.apply(refreshedCapability: false) + XCTAssertEqual(action, .none, "already off, no teardown action needed") + XCTAssertFalse(gate.isCapable) + } + + func testAbsentFieldNeverConnects() { + // Server response with no `realtime_enabled` key decodes to `false` + // upstream (see ModelsTests); feeding that in must never connect. + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: false), .none) + XCTAssertFalse(gate.isCapable) + } + + func testTrueThenFalseTearsDownOnce() { + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + XCTAssertEqual(gate.apply(refreshedCapability: false), .disconnect) + XCTAssertFalse(gate.isCapable) + } + + func testRepeatingSameValueIsANoOp() { + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + // Steady-state refreshes that keep reporting true must not re-fire + // a connect action every 30s poll / pull-to-refresh. + XCTAssertEqual(gate.apply(refreshedCapability: true), .none) + XCTAssertEqual(gate.apply(refreshedCapability: true), .none) + XCTAssertTrue(gate.isCapable) + } + + func testFalseThenTrueReconnectsAfterRollback() { + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + XCTAssertEqual(gate.apply(refreshedCapability: false), .disconnect) + // "no reconnect attempts until a refresh says true again" — the very + // next refresh reporting true is exactly that refresh. + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + XCTAssertTrue(gate.isCapable) + } + + func testResetReturnsToColdStartState() { + var gate = RealtimeCapabilityGate() + _ = gate.apply(refreshedCapability: true) + XCTAssertTrue(gate.isCapable) + gate.reset() + XCTAssertFalse(gate.isCapable) + // Re-pairing to a new server must prove capability again — a stale + // `true` must not survive the reset. + XCTAssertEqual(gate.apply(refreshedCapability: false), .none) + XCTAssertFalse(gate.isCapable) + } +} From 2c482589978ae3cd8e092803605876b18650eaf2 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:20:52 +0800 Subject: [PATCH 43/60] fix(apple): gate realtime auto-connect on server capability (P0) The app previously derived /v3/realtime from the REST URL and connected unconditionally once paired and foregrounded, so a server without realtime enabled could serve a valid legacy /v2 snapshot and then have the UI clobbered by an empty/failed connection state. AppModel now owns a RealtimeCapabilityGate, fed by realtime_enabled on every successful refresh(). startRealtime() is gated on it, so the connection only ever opens once a refresh has confirmed capability; cold start (before any refresh) and legacy/absent responses default to off. If a later refresh reports false/absent while connected, the existing generation-scoped teardown tears the connection down and the app falls back to plain HTTP refresh with no reconnect attempts until a later refresh says true again. Re-pairing (saveSettings) and disconnect() reset the gate since a new identity must prove capability again. MainTabView's sync strip needed no change: gating off keeps connectionPhase at .idle, which already renders as plain HTTP mode. --- apple/SitrepApp/Sitrep/SitrepApp.swift | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/apple/SitrepApp/Sitrep/SitrepApp.swift b/apple/SitrepApp/Sitrep/SitrepApp.swift index 743b36f..1f9f102 100644 --- a/apple/SitrepApp/Sitrep/SitrepApp.swift +++ b/apple/SitrepApp/Sitrep/SitrepApp.swift @@ -91,6 +91,12 @@ final class AppModel { /// once, non-disruptively, then cleared by the view that shows it. var supersededNotice = false + /// P0 gate: decides, from each REST refresh's `realtime_enabled`, + /// whether the RealtimeClient may connect. Pure decision logic lives in + /// `RealtimeCapabilityGate` (SitrepKit) so it's unit-testable without a + /// live socket; this property is the only thing that reads/drives it. + private var realtimeGate = RealtimeCapabilityGate() + var realtimeCapable: Bool { realtimeGate.isCapable } private var realtimeClient: RealtimeClient? private var realtimeObservers: [Task] = [] /// Last SpaceState received from the realtime channel, kept for the @@ -144,6 +150,10 @@ final class AppModel { if realtimeClient != nil { stopRealtime() lastSpaceState = nil + // New credentials may point at a different server entirely — + // don't carry over the old one's capability; wait for this + // space's own refresh to confirm it before reconnecting. + realtimeGate.reset() enterForeground() } } @@ -181,6 +191,10 @@ final class AppModel { } private func startRealtime() { + // The capability gate (P0): only ever connect once a REST refresh has + // confirmed `realtime_enabled == true`. Everything else in this + // method is unchanged pre-existing plumbing. + guard realtimeCapable else { return } guard let restClient = client, let url = restClient.realtimeURL, !deviceID.isEmpty else { return } if let existing = realtimeClient { // Idempotent on the actor side even if already running — this @@ -216,6 +230,20 @@ final class AppModel { Task { await rt.start() } } + /// Apply the capability bit from a fresh REST refresh via the pure gate, + /// then act on whatever it decides: false→true may open a connection + /// (this refresh IS the "a refresh says true again" the P0 fix requires + /// before reconnecting); true→false (server rolled the flag back) tears + /// down any live connection and falls back to plain HTTP, matching the + /// pre-realtime app exactly. + private func applyRealtimeCapability(_ enabled: Bool) { + switch realtimeGate.apply(refreshedCapability: enabled) { + case .none: break + case .connect: startRealtime() + case .disconnect: stopRealtime() + } + } + private func stopRealtime() { guard let rt = realtimeClient else { return } realtimeClient = nil @@ -360,6 +388,9 @@ final class AppModel { stopRealtime() stopFallbackPolling() lastSpaceState = nil + // A future re-pair is a cold start for capability purposes too — no + // connecting on the new space until its own refresh confirms it. + realtimeGate.reset() if !deviceID.isEmpty { try? await client?.revokeDevice(id: deviceID) } @@ -409,6 +440,7 @@ final class AppModel { presence = snapshot.presence lastError = nil lastSyncAt = .now + applyRealtimeCapability(snapshot.realtimeEnabled) // Checked AFTER the await, against the ACTOR's phase (not the // stream-fed MainActor cache): while the WebSocket is live, // deltas own the four reliable collections and an HTTP response From bd4d46a908681e4b59178906c47df4abe83e2c96 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:21:28 +0800 Subject: [PATCH 44/60] fix(server): version-gate SpaceHub schema DDL against hibernation wakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DO constructors re-run on every hibernation wake, and migrate() was unconditionally re-issuing 10+ CREATE TABLE/INDEX IF NOT EXISTS statements each time — conflicting with the sparse-connection cost goals in docs/design/realtime-server.md. Add a _schema_migrations version table: migrate() now does one lightweight SELECT and only runs (and writes) the DDL block when the store is behind, including the fresh-DO case where the version table doesn't exist yet. --- server/src/realtime/space-hub.ts | 54 ++++++++++++++++++++++ server/test/workers/migration.workers.ts | 58 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 server/test/workers/migration.workers.ts diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 3910446..39aa20d 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -86,6 +86,11 @@ function canonicalJson(value: unknown): string { } export class SpaceHub extends DurableObject { + /** Current schema version this build expects. Bump when `migrate()`'s + * DDL block changes, alongside adding whatever new `CREATE TABLE/INDEX + * IF NOT EXISTS` statements the change needs. */ + private static readonly SCHEMA_VERSION = 1; + /** Best-effort metric cache (SPEC.md section 6.2: snapshot.metrics is a * convenience cache, allowed to be empty/stale, outside space_revision * accounting) — deliberately in-memory only, never written to SQLite. */ @@ -93,6 +98,15 @@ export class SpaceHub extends DurableObject { private rateLimiters = new WeakMap(); private hotPathCounter = 0; + /** Set true only when migrate() actually executed the DDL block during + * this construct, as opposed to finding the store already at + * SCHEMA_VERSION and returning without touching SQLite. Exists purely + * so tests can assert the 10+ `CREATE TABLE/INDEX IF NOT EXISTS` + * statements do NOT re-run on every hibernation wake (DO constructors + * re-run on wake; see the constructor's migrate() call and the + * space-hub.ts cost notes) — never read by production logic. */ + private ranMigrationDdl = false; + /** Injectable for tests: decides whether the Nth hot-path event gets * logged. Default samples at <=1%. Errors always log regardless of this * (see logAlways) — only routine per-frame hot-path logging is sampled. */ @@ -117,8 +131,25 @@ export class SpaceHub extends DurableObject { this.migrate(); } + /** Runs the schema DDL exactly once per store lifetime, not once per DO + * construct. DO constructors re-run on every hibernation wake (that's + * the whole point of hibernation — the JS instance is cheap to recreate, + * the SQLite store is what's durable), so unconditionally re-issuing + * 10+ `CREATE TABLE/INDEX IF NOT EXISTS` statements here would mean + * every wake pays for a full schema scan, defeating the sparse- + * connection cost goals documented in realtime-server.md. Instead: one + * lightweight version read, and the DDL block (plus the version write) + * only runs when the store is behind — including the fresh-DO case, + * where the version table itself doesn't exist yet. */ private migrate(): void { + if (this.schemaVersion() >= SpaceHub.SCHEMA_VERSION) return; + + this.ranMigrationDdl = true; this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS _schema_migrations ( + id INTEGER PRIMARY KEY CHECK (id = 1), + version INTEGER NOT NULL + ); CREATE TABLE IF NOT EXISTS space_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -193,6 +224,29 @@ export class SpaceHub extends DurableObject { delivered INTEGER NOT NULL DEFAULT 0 ); `); + // Same synchronous, no-`await`-in-between transaction as the DDL above + // (see the class-level concurrency note) — the version write is + // atomic with the schema it describes. + this.ctx.storage.sql.exec( + `INSERT INTO _schema_migrations (id, version) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET version = excluded.version`, + SpaceHub.SCHEMA_VERSION, + ); + } + + /** One lightweight SELECT: current schema version, or 0 for a fresh DO + * (the `_schema_migrations` table itself doesn't exist yet, which throws + * rather than returning zero rows — SQLite has no "does this table + * exist" query result short of inspecting sqlite_master, and a bare + * SELECT-that-may-throw is cheaper and simpler than checking + * sqlite_master first on every call). */ + private schemaVersion(): number { + try { + const row = this.ctx.storage.sql.exec<{ version: number }>("SELECT version FROM _schema_migrations WHERE id = 1").toArray()[0]; + return row?.version ?? 0; + } catch { + return 0; + } } // ---- WebSocket entry point ---- diff --git a/server/test/workers/migration.workers.ts b/server/test/workers/migration.workers.ts new file mode 100644 index 0000000..1b863c9 --- /dev/null +++ b/server/test/workers/migration.workers.ts @@ -0,0 +1,58 @@ +// Required coverage: DO constructors re-run on every hibernation wake +// (SpaceHub's constructor calls migrate()), so migrate() must not +// unconditionally re-issue its 10+ `CREATE TABLE/INDEX IF NOT EXISTS` +// statements every wake — only when the store's schema version is behind +// what this build expects. See the migrate()/schemaVersion() comments in +// src/realtime/space-hub.ts. +import { env, evictDurableObject, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace } from "./helpers"; + +describe("SpaceHub schema migration", () => { + it("runs the DDL block on a fresh DO (version table missing)", async () => { + const stub = env.SPACE_HUB.getByName(`fresh-${crypto.randomUUID()}`); + const ranDdl = await runInDurableObject(stub, async (instance: SpaceHub) => (instance as any).ranMigrationDdl as boolean); + expect(ranDdl).toBe(true); + + const version = await runInDurableObject(stub, async (_instance: SpaceHub, state) => + state.storage.sql.exec<{ version: number }>("SELECT version FROM _schema_migrations WHERE id = 1").toArray()[0]?.version, + ); + expect(version).toBe(1); + + // The rest of the schema landed too, not just the version table. + const tableExists = await runInDurableObject(stub, async (_instance: SpaceHub, state) => + state.storage.sql + .exec<{ n: number }>("SELECT COUNT(*) as n FROM sqlite_master WHERE type='table' AND name='event_log'") + .toArray()[0].n, + ); + expect(tableExists).toBe(1); + }); + + it("does not re-run the DDL block on a second construct against an already-migrated store", async () => { + // bootstrapSpace() drives a real construct through the app's /v2 + // routes (registry -> UserStore -> SpaceHub stub), exercising the + // exact path a hibernation wake takes in production. + const { spaceId } = await bootstrapSpace(); + const stub = env.SPACE_HUB.getByName(spaceId); + + // bootstrapSpace() only touches UserStore over HTTP — nothing has + // addressed this space's SpaceHub yet, so it isn't "running" for + // evictDurableObject's purposes until something does. + await runInDurableObject(stub, async () => {}); + + // Force a fresh DO *instance* (constructor re-runs) while preserving + // durable SQLite storage — this is precisely what a hibernation wake + // does, and the primitive vitest-pool-workers provides to simulate it. + await evictDurableObject(stub); + + const ranDdl = await runInDurableObject(stub, async (instance: SpaceHub) => (instance as any).ranMigrationDdl as boolean); + expect(ranDdl).toBe(false); + + // Evict and re-check again for good measure: every subsequent wake + // against an up-to-date store must skip the DDL, not just the first. + await evictDurableObject(stub); + const ranDdlAgain = await runInDurableObject(stub, async (instance: SpaceHub) => (instance as any).ranMigrationDdl as boolean); + expect(ranDdlAgain).toBe(false); + }); +}); From 1b5234e91d87ebf86be1ea165e17b1dcc3504a0d Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:22:51 +0800 Subject: [PATCH 45/60] fix(server): cap metricsCache cardinality with LRU eviction metricsCache had per-frame size/rate limits but no cap on distinct metric_ids, so a compromised or misbehaving source rotating metric_ids could grow DO memory without bound until the instance crashes. Add METRIC_CACHE_MAX_METRICS (256, protocol.ts) and evict the least-recently-updated metric_id when a new one would exceed it. Eviction is safe per SPEC.md 6.2 (snapshot.metrics is a best-effort cache); evicted metrics are simply absent from the next snapshot. --- server/src/realtime/protocol.ts | 12 ++++ server/src/realtime/space-hub.ts | 26 +++++++- server/test/workers/metric-cache.workers.ts | 70 +++++++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 server/test/workers/metric-cache.workers.ts diff --git a/server/src/realtime/protocol.ts b/server/src/realtime/protocol.ts index 7d3d821..e0dfe66 100644 --- a/server/src/realtime/protocol.ts +++ b/server/src/realtime/protocol.ts @@ -44,6 +44,18 @@ export const METRIC_FRAME_RATE_PER_SEC = 10; /** SPEC.md section 12: 2 Hz ceiling per metric_id. */ export const METRIC_SAMPLE_MIN_INTERVAL_MS = 500; +/** Cardinality cap on `SpaceHub#metricsCache` (distinct `metric_id`s held + * at once, per space). Without this, a compromised or misbehaving source + * that rotates metric_ids without bound would grow DO memory unboundedly + * until the instance crashes — the existing per-frame size/rate limits + * (METRIC_FRAME_RATE_PER_SEC, the 64-samples-per-frame guard in + * guards.ts) bound how fast the cache can grow but not how big it can + * get. Beyond this cap, the least-recently-updated metric_id is evicted + * to make room (see SpaceHub#touchMetricCache); eviction is safe per + * SPEC.md section 6.2 (snapshot.metrics is a best-effort cache, evicted + * metrics are simply absent from the next snapshot). */ +export const METRIC_CACHE_MAX_METRICS = 256; + export function newEnvelopeId(): string { return crypto.randomUUID(); } diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index 39aa20d..e0fba13 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -28,6 +28,7 @@ import { ERROR_SEMANTICS, HEARTBEAT_INTERVAL_MS, LEASE_DEFAULT_MS, + METRIC_CACHE_MAX_METRICS, METRIC_FRAME_RATE_PER_SEC, EVENT_LOG_RETENTION_REVISIONS, MESSAGE_WINDOW, @@ -93,7 +94,13 @@ export class SpaceHub extends DurableObject { /** Best-effort metric cache (SPEC.md section 6.2: snapshot.metrics is a * convenience cache, allowed to be empty/stale, outside space_revision - * accounting) — deliberately in-memory only, never written to SQLite. */ + * accounting) — deliberately in-memory only, never written to SQLite. + * Capped at METRIC_CACHE_MAX_METRICS distinct metric_ids with + * least-recently-updated eviction (see touchMetricCache) so a source + * rotating metric_ids without bound can't grow this without bound. Map + * iteration order is insertion order and JS re-inserts a key at the end + * on delete+set, which is exactly what touchMetricCache relies on to + * track recency. */ private metricsCache = new Map(); private rateLimiters = new WeakMap(); private hotPathCounter = 0; @@ -847,7 +854,7 @@ export class SpaceHub extends DurableObject { // is acceptable (the whole cache is best-effort, section 6.2). const cached = this.metricsCache.get(sample.metric_id); if (cached && sample.ts <= cached.ts) continue; - this.metricsCache.set(sample.metric_id, sample); + this.touchMetricCache(sample.metric_id, sample); accepted.push(sample); } if (accepted.length > 0) this.broadcastMetricFrame({ device_id: body.device_id, metrics: accepted }); @@ -856,6 +863,21 @@ export class SpaceHub extends DurableObject { // docs/design/realtime-server.md. } + /** Upserts one metric_id into metricsCache, enforcing + * METRIC_CACHE_MAX_METRICS with least-recently-updated eviction. Always + * deletes before re-setting (even for an existing key) so the entry + * moves to the end of Map iteration order — that's what makes "the + * first key in iteration order" equivalent to "the least recently + * updated key" below. */ + private touchMetricCache(metricId: string, sample: MetricSample): void { + this.metricsCache.delete(metricId); + if (this.metricsCache.size >= METRIC_CACHE_MAX_METRICS) { + const oldest = this.metricsCache.keys().next().value; + if (oldest !== undefined) this.metricsCache.delete(oldest); + } + this.metricsCache.set(metricId, sample); + } + private broadcastMetricFrame(body: MetricFrameBody): void { const now = Date.now(); for (const ws of this.ctx.getWebSockets()) { diff --git a/server/test/workers/metric-cache.workers.ts b/server/test/workers/metric-cache.workers.ts new file mode 100644 index 0000000..03e2686 --- /dev/null +++ b/server/test/workers/metric-cache.workers.ts @@ -0,0 +1,70 @@ +// Required coverage: metricsCache (SpaceHub#metricsCache) must not grow +// without bound. A compromised or misbehaving source rotating metric_ids +// past any real cardinality would otherwise grow DO memory until it +// crashes — see METRIC_CACHE_MAX_METRICS in src/realtime/protocol.ts and +// SpaceHub#touchMetricCache. +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { METRIC_CACHE_MAX_METRICS } from "../../src/realtime/protocol.ts"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, subscribe } from "./helpers"; + +describe("metricsCache cardinality cap", () => { + it("evicts least-recently-updated metric_ids beyond the cap, keeping the most recent", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient, ["metric"]); + + // Rotate far more distinct metric_ids than the cap, staying within the + // 64-samples-per-frame guard (guards.ts) and the 10 frames/s limiter + // (METRIC_FRAME_RATE_PER_SEC) by packing many ids per frame across a + // handful of frames rather of one-id-per-frame. + const perFrame = 64; + const frameCount = 6; // 384 distinct ids, comfortably over the 256 cap + const totalIds = perFrame * frameCount; + for (let f = 0; f < frameCount; f++) { + const metrics = Array.from({ length: perFrame }, (_, i) => { + const n = f * perFrame + i; + return { metric_id: `m${n}`, value: String(n), ts: Date.now() + n }; + }); + sourceClient.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics }, + }); + // Drain this frame's broadcast to the subscribed viewer before + // sending the next one, so the viewer's recv queue doesn't leak + // across iterations (not load-bearing for the assertion below, just + // hygiene for the WsClient helper). + await viewerClient.recv(); + } + + const stub = env.SPACE_HUB.getByName(spaceId); + const { size, ids } = await runInDurableObject(stub, async (instance: SpaceHub) => { + const cache = (instance as any).metricsCache as Map; + return { size: cache.size, ids: [...cache.keys()] }; + }); + + expect(size).toBe(METRIC_CACHE_MAX_METRICS); + expect(size).toBeLessThan(totalIds); + + // The most-recently-sent ids (the last frame's) must all have + // survived eviction... + for (let n = totalIds - perFrame; n < totalIds; n++) { + expect(ids).toContain(`m${n}`); + } + // ...while the earliest-sent ids (evicted as least-recently-updated) + // must be gone. + for (let n = 0; n < perFrame; n++) { + expect(ids).not.toContain(`m${n}`); + } + + sourceClient.close(); + viewerClient.close(); + }); +}); From a734dd7b190e27bdeb0339e697cdd238a677bd87 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:23:25 +0800 Subject: [PATCH 46/60] fix(server): log superseded events before the wire reply, not after handlePreHello sent the superseded peer's error frame before calling logAlways(), so a throwing send() would skip the security log entirely. Reorder so the 100%-sampled logAlways fires first; the wire frame and ordering toward the client are unchanged in the success path (still error frame, then close). --- server/src/realtime/space-hub.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index e0fba13..d5a16d3 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -524,21 +524,24 @@ export class SpaceHub extends DurableObject { if (other === ws) continue; const otherAtt = other.deserializeAttachment(); if (isConnAttachment(otherAtt) && otherAtt.helloDone) { - this.send(other, "error", { - code: "superseded", - message: "device completed hello on a newer connection", - ...ERROR_SEMANTICS.superseded, - }); // Product invariant 5: superseded must never be silent — it is a // security-relevant event (possible credential/session reuse), so it - // always logs at 100% via logAlways, independent of the wire reply - // above (which must stay byte-identical for the superseded peer). + // always logs at 100% via logAlways. Logged BEFORE the wire reply + // below: if `this.send(other, ...)` were to throw (e.g. a socket + // that's already gone), the security log must still have fired — + // it must never depend on the send succeeding. The reply itself + // stays byte-identical for the superseded peer in the success path. this.logAlways("superseded", { device_id: att.deviceId, role: att.role, superseded_session_id: otherAtt.sessionId, superseding_session_id: newSessionId, }); + this.send(other, "error", { + code: "superseded", + message: "device completed hello on a newer connection", + ...ERROR_SEMANTICS.superseded, + }); other.close(1008, "superseded"); } } From 27474ac691dba9c5e3d9d5d4e95decedca008e7a Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:34:51 +0800 Subject: [PATCH 47/60] fix(server): strictly parse REALTIME_ENABLED instead of Boolean() Cloudflare dashboard variable overrides always arrive as strings at runtime, even though wrangler.jsonc declares REALTIME_ENABLED as a boolean. Boolean("false") is true, so an operator typing "false" in the dashboard would silently re-enable the realtime kill switch. Add parseRealtimeEnabledFlag() as a strict allow-list (true / "true" / "1", case-insensitive, trimmed) and route the worker adapter through it; widen the Secrets type to string | boolean to match the real runtime shape. Covered by new unit tests in test/app.test.ts. --- server/src/adapters/workers.ts | 10 +++++++--- server/src/app.ts | 17 +++++++++++++++++ server/test/app.test.ts | 19 ++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index cde4d25..9cca830 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -3,7 +3,7 @@ // token in the AUTH_TOKEN secret. Live Activity pushes fire from inside the // DO after each state change (via waitUntil, off the ingest critical path). import { DurableObject } from "cloudflare:workers"; -import { createApp, newToken, TOKEN_RE, type Command, type DeviceInfo, type Role, type SpaceRegistry } from "../app.ts"; +import { createApp, newToken, parseRealtimeEnabledFlag, TOKEN_RE, type Command, type DeviceInfo, type Role, type SpaceRegistry } from "../app.ts"; import { endActivity, sendAlert, startActivity, updateActivity, type ApnsConfig } from "../apns.ts"; import { SpaceHub } from "../realtime/space-hub.ts"; import type { AutomationExecutorKind, AutomationState } from "../realtime/types.ts"; @@ -44,7 +44,11 @@ interface Secrets { APNS_TEAM_ID?: string; APNS_BUNDLE_ID?: string; // var, wrangler.jsonc APNS_HOST?: string; // var, wrangler.jsonc - REALTIME_ENABLED?: boolean; // var, wrangler.jsonc; unset/false disables /v3/realtime + // var, wrangler.jsonc; unset/false disables /v3/realtime. Typed as string | + // boolean because a Cloudflare dashboard variable override always arrives + // as a string at runtime, even though wrangler.jsonc declares it boolean — + // see parseRealtimeEnabledFlag in app.ts for the strict parsing this needs. + REALTIME_ENABLED?: string | boolean; } type WorkerEnv = Env & Secrets; @@ -484,7 +488,7 @@ const app = createApp({ await (c.env as WorkerEnv).INVITE_DIR.put(code, space, { expirationTtl: 600 }); }, lookupInvite: (c, code) => (c.env as WorkerEnv).INVITE_DIR.get(code), - realtimeEnabled: (c) => Boolean((c.env as WorkerEnv).REALTIME_ENABLED), + realtimeEnabled: (c) => parseRealtimeEnabledFlag((c.env as WorkerEnv).REALTIME_ENABLED), }); app.get("/debug/tokens", async (c: any) => { diff --git a/server/src/app.ts b/server/src/app.ts index 6e70f52..6d2e863 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -19,6 +19,23 @@ import { makeSnapshot } from "./domain.ts"; export type Role = "admin" | "owner" | "viewer" | "source"; export type Command = "pause" | "resume" | "stop"; +/** Strict allow-list parser for the REALTIME_ENABLED kill switch. + * + * wrangler.jsonc types this var as a boolean, but a Cloudflare dashboard + * variable override arrives as a STRING at runtime regardless of the + * declared type — `Boolean("false")` is `true`, which would silently flip + * the kill switch on for an operator who typed "false" in the dashboard. + * This flag gates a P0 client behavior, so parsing must be paranoid: only + * the boolean `true`, or the exact strings "true"/"1" (case-insensitive, + * surrounding whitespace trimmed), enable realtime. Everything else — + * `false`, "false", "0", "", undefined, or any other string — disables it. */ +export function parseRealtimeEnabledFlag(value: string | boolean | undefined): boolean { + if (typeof value === "boolean") return value; + if (typeof value !== "string") return false; + const normalized = value.trim().toLowerCase(); + return normalized === "true" || normalized === "1"; +} + export const TOKEN_RE = /^st2_([a-z0-9]{1,16})_[a-f0-9]{48}$/; export interface DeviceInfo { diff --git a/server/test/app.test.ts b/server/test/app.test.ts index fbe6bd0..38876d5 100644 --- a/server/test/app.test.ts +++ b/server/test/app.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { createApp } from "../src/app.ts"; +import { createApp, parseRealtimeEnabledFlag } from "../src/app.ts"; import { SqliteStore } from "../src/sqlite-store.ts"; function testApp() { @@ -21,6 +21,23 @@ test("v2 snapshot realtime_enabled reflects the configured var", async () => { assert.equal(snapshot.realtime_enabled, true); }); +test("parseRealtimeEnabledFlag: only true/'true'/'1' enable, everything else disables", () => { + // Dashboard variable overrides always arrive as strings at runtime even + // though wrangler.jsonc declares REALTIME_ENABLED as boolean — a naive + // Boolean(value) would treat the string "false" as truthy. This parser + // must be a strict allow-list, not a truthiness check. + const enabling: Array = ["true", "TRUE ", " true", "1"]; + for (const value of enabling) { + assert.equal(parseRealtimeEnabledFlag(value), true, `expected ${JSON.stringify(value)} to enable`); + } + assert.equal(parseRealtimeEnabledFlag(true), true); + + const disabling: Array = [false, "false", "0", "", undefined, "yes", "TRUE_ISH", "truex"]; + for (const value of disabling) { + assert.equal(parseRealtimeEnabledFlag(value), false, `expected ${JSON.stringify(value)} to disable`); + } +}); + test("Node ingest returns success and records presence", async () => { const app = testApp(); const response = await app.request("/v2/ingest", { From 80b01f5a18b22a79c471488ff38b5376546614ad Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:34:57 +0800 Subject: [PATCH 48/60] fix(server): don't swallow real SQL errors as fresh-DO in schemaVersion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit schemaVersion()'s bare catch treated any SQL failure — including corruption or I/O errors — as "fresh DO, run migrations" (version 0). The DDL is IF NOT EXISTS so a genuinely fresh DO re-running it is benign, but misdiagnosing real corruption this way hides the failure. Only swallow the specific "no such table" error that means the migrations table doesn't exist yet; rethrow anything else. --- server/src/realtime/space-hub.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts index d5a16d3..821f655 100644 --- a/server/src/realtime/space-hub.ts +++ b/server/src/realtime/space-hub.ts @@ -251,8 +251,13 @@ export class SpaceHub extends DurableObject { try { const row = this.ctx.storage.sql.exec<{ version: number }>("SELECT version FROM _schema_migrations WHERE id = 1").toArray()[0]; return row?.version ?? 0; - } catch { - return 0; + } catch (e) { + // A missing migrations table means this is a genuinely fresh DO — DDL + // below is IF NOT EXISTS, so re-running it is benign. Any other SQL + // error (corruption, I/O) must not be silently treated as "fresh": + // rethrow so it surfaces instead of being misdiagnosed. + if (String(e).includes("no such table")) return 0; + throw e; } } From f781e5cf1e273f1671e7a7d36e59b6ecd0a11d94 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:39:27 +0800 Subject: [PATCH 49/60] fix(daemon): wrap validation failures in a retryable-vs-permanent sentinel Enqueue's BuildBody closures call body.Validate() before the insert, but a validation failure was indistinguishable from a transient BeginTx/COUNT/ INSERT/Commit error (SQLITE_BUSY, disk I/O) once it came back as a plain error. Wrap it in ErrInvalidBody so callers can tell "this body can never validate, drop it" apart from "this Enqueue attempt failed, retry it" via errors.Is. --- daemon/internal/realtime/client/client.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index 33d0b48..639646f 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -29,6 +29,15 @@ var errClientClosing = errors.New("realtime client: closing") // process recovers by constructing a new Client (i.e. an explicit restart). var errNoRetry = errors.New("realtime client: fatal, not retryable") +// ErrInvalidBody wraps a failure from a wire body's Validate(), returned by +// SendTaskEvent/SendMessageEvent when the caller-supplied fields can never +// produce a valid envelope (bad kind, out-of-range percent, oversized free +// text, ...). It is the one Enqueue-adjacent failure callers should treat as +// permanent: unlike outbox.ErrOutboxFull or any other Enqueue error (a +// transient BeginTx/COUNT/INSERT/Commit failure), retrying the exact same +// malformed body can never succeed. Use errors.Is to check for it. +var ErrInvalidBody = errors.New("realtime: event body failed validation") + // Client is a reconnecting realtime-protocol source connection. Construct // with New; it starts connecting immediately in the background. Call // SendTaskEvent/SendMessageEvent for reliable events and SendMetric for @@ -142,7 +151,7 @@ func (c *Client) SendTaskEvent(ev TaskEvent) error { Display: ev.Display, } if err := body.Validate(); err != nil { - return nil, err + return nil, fmt.Errorf("%w: %v", ErrInvalidBody, err) } return json.Marshal(body) }) @@ -180,7 +189,7 @@ func (c *Client) SendMessageEvent(ev MessageEvent) error { AutomationID: ev.AutomationID, } if err := body.Validate(); err != nil { - return nil, err + return nil, fmt.Errorf("%w: %v", ErrInvalidBody, err) } return json.Marshal(body) }) From 1efdac61b44c2039b7a7055d89b960a4fd68130d Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 13:39:36 +0800 Subject: [PATCH 50/60] fix(daemon): serialize reliable retries against fresh Offer()s Two bugs in the outbox-backpressure path: - BLOCKER: sendReliable always attempted Enqueue for a new event, even with older items already waiting in rtRetry. A freed outbox row between an older event's failed attempt and a newer event's Offer() let the newer one grab a lower device_seq, inverting wire order once the older one was retried. Fix: a dedicated rtMu now guards the whole check-then-act step in sendReliable (queue non-empty -> append behind it; queue empty -> attempt Enqueue) and retryRealtime's whole drain, so the two can never interleave. - MAJOR: any non-ErrOutboxFull Enqueue failure was logged and dropped, which silently discarded a reliable task.done/message.send on a transient SQLite error (SQLITE_BUSY, disk I/O). isRetryable now treats only rtclient.ErrInvalidBody as permanent (retrying the same malformed body can't help); everything else, including unrecognized errors, defaults to retry. Adds regression tests for both: a deterministic FIFO-interleaving test that drives sendReliable/retryRealtime directly (no sleeps), a transient- error test asserting eventual delivery, and a permanent-error test asserting drop-not-retry. --- daemon/internal/uplink/rtretry_test.go | 199 +++++++++++++++++++++++++ daemon/internal/uplink/uplink.go | 135 +++++++++++------ 2 files changed, 290 insertions(+), 44 deletions(-) create mode 100644 daemon/internal/uplink/rtretry_test.go diff --git a/daemon/internal/uplink/rtretry_test.go b/daemon/internal/uplink/rtretry_test.go new file mode 100644 index 0000000..a31bb78 --- /dev/null +++ b/daemon/internal/uplink/rtretry_test.go @@ -0,0 +1,199 @@ +package uplink + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "testing" + + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" +) + +// TestSendReliablePreservesFIFOAgainstFreshOffer is the regression test for +// the seq-order-inversion BLOCKER: a fresh Offer() must never win a race +// into outbox.Enqueue ahead of an older event still waiting in rtRetry, even +// when a row frees up between the older event's failed attempt and the +// newer one's Offer(). Reproduces the exact interleaving from the bug +// report by driving sendReliable/retryRealtime directly (white-box, same +// package) instead of racing real goroutines against a flush ticker — +// deterministic, no sleeps. +func TestSendReliablePreservesFIFOAgainstFreshOffer(t *testing.T) { + ctx := context.Background() + store, err := outbox.OpenWithMaxRows(filepath.Join(t.TempDir(), "outbox.db"), 5) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer store.Close() + + const space = "space-1" + body := func(seq int64) (json.RawMessage, error) { + return json.RawMessage(fmt.Sprintf(`{"seq":%d}`, seq)), nil + } + + // Fill the outbox to its cap of 5 with unacked placeholder rows so the + // very next Enqueue attempt (A, below) hits ErrOutboxFull. + var filledSeqs []int64 + for i := 0; i < 5; i++ { + item, err := store.Enqueue(ctx, space, "message.event", body) + if err != nil { + t.Fatalf("prefill Enqueue: %v", err) + } + filledSeqs = append(filledSeqs, item.DeviceSeq) + } + + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + + var seqA, seqB int64 + sendA := func() error { + item, err := store.Enqueue(ctx, space, "message.event", body) + if err != nil { + return err + } + seqA = item.DeviceSeq + return nil + } + sendB := func() error { + item, err := store.Enqueue(ctx, space, "message.event", body) + if err != nil { + return err + } + seqB = item.DeviceSeq + return nil + } + + // Step 1: "outbox full with A in rtRetry". A is offered while completely + // full, so it must queue rather than deliver. + u.sendReliable(sendA) + if len(u.rtRetry) != 1 { + t.Fatalf("expected A queued for retry, rtRetry has %d items", len(u.rtRetry)) + } + if seqA != 0 { + t.Fatalf("A must not have enqueued while the outbox was full; got seq %d", seqA) + } + + // Step 2: "free exactly one row via ack" — an ack arriving on the + // rtclient connection goroutine, independent of the 1s flush tick. + if err := store.Ack(ctx, space, filledSeqs[0]); err != nil { + t.Fatalf("Ack: %v", err) + } + + // Step 3: "Offer(B) BEFORE the next flush tick". With the BLOCKER bug, + // sendReliable would attempt Enqueue unconditionally here, succeeding + // into the just-freed row and handing B a lower device_seq than the + // still-queued A. The fix must instead queue B behind A. + u.sendReliable(sendB) + if len(u.rtRetry) != 2 { + t.Fatalf("expected B queued behind A (FIFO) since rtRetry was non-empty, rtRetry has %d items", len(u.rtRetry)) + } + if seqB != 0 { + t.Fatalf("B must not have enqueued directly while A was still queued for retry; got seq %d", seqB) + } + + // Step 4: the next flush tick drains oldest-first, using the freed row + // for A. + u.retryRealtime() + if seqA == 0 { + t.Fatal("expected A to have been delivered using the freed row") + } + if len(u.rtRetry) != 1 { + t.Fatalf("expected only B still queued after draining A, rtRetry has %d items", len(u.rtRetry)) + } + + // Free a second row so B's retry can succeed too, and drain again. + if err := store.Ack(ctx, space, filledSeqs[1]); err != nil { + t.Fatalf("Ack: %v", err) + } + u.retryRealtime() + if seqB == 0 { + t.Fatal("expected B to have been delivered on the second drain") + } + if len(u.rtRetry) != 0 { + t.Fatalf("expected rtRetry fully drained, has %d items", len(u.rtRetry)) + } + + // The assertion the BLOCKER violates: chronological order (A offered + // before B) must match wire order, i.e. strictly increasing device_seq. + if !(seqA < seqB) { + t.Fatalf("wire order inverted: expected seqA < seqB (A offered first), got seqA=%d seqB=%d", seqA, seqB) + } +} + +// TestSendReliableRetriesTransientEnqueueError is the regression test for +// the MAJOR: a transient outbox.Enqueue failure that is neither +// ErrOutboxFull nor a validation error (e.g. SQLITE_BUSY or a disk I/O blip +// surfacing from BeginTx/COUNT/INSERT/Commit) must be retried, not dropped +// — losing a reliable task.done/message.send to a momentary disk hiccup is +// not acceptable. +func TestSendReliableRetriesTransientEnqueueError(t *testing.T) { + ctx := context.Background() + store, err := outbox.OpenWithMaxRows(filepath.Join(t.TempDir(), "outbox.db"), 10) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer store.Close() + + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + + transientErr := errors.New("outbox: begin: disk I/O error") + attempts := 0 + var seq int64 + send := func() error { + attempts++ + if attempts == 1 { + // Simulate a transient Enqueue failure — not ErrOutboxFull, not + // a validation error, and not a class this code recognizes at + // all. isRetryable must default to "retry" for it. + return transientErr + } + item, err := store.Enqueue(ctx, "space-1", "message.event", func(seq int64) (json.RawMessage, error) { + return json.RawMessage(`{}`), nil + }) + if err != nil { + return err + } + seq = item.DeviceSeq + return nil + } + + handled := u.sendReliable(send) + if !handled { + t.Fatal("sendReliable must always report handled=true for a reliable event") + } + if len(u.rtRetry) != 1 { + t.Fatalf("expected the event queued for retry after an unrecognized transient error, rtRetry has %d items", len(u.rtRetry)) + } + if seq != 0 { + t.Fatal("event must not have been delivered on the failed first attempt") + } + + u.retryRealtime() + if seq == 0 { + t.Fatal("expected the event to be delivered on retry, not dropped") + } + if len(u.rtRetry) != 0 { + t.Fatalf("expected rtRetry drained after the successful retry, has %d items", len(u.rtRetry)) + } +} + +// TestSendReliableDropsPermanentValidationError pins the other half of the +// permanent/transient split: rtclient.ErrInvalidBody (a wire-validation +// failure) is the one Enqueue-adjacent error that must NOT be retried — +// the same malformed body can never validate differently — so it is +// logged and dropped instead of growing rtRetry forever. +func TestSendReliableDropsPermanentValidationError(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + send := func() error { + return fmt.Errorf("%w: task.event: missing task_id", rtclient.ErrInvalidBody) + } + + handled := u.sendReliable(send) + if !handled { + t.Fatal("sendReliable must always report handled=true for a reliable event") + } + if len(u.rtRetry) != 0 { + t.Fatalf("expected a validation failure to be dropped, not queued; rtRetry has %d items", len(u.rtRetry)) + } +} diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index 8de7d9c..e576bae 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -21,7 +21,6 @@ import ( "github.com/QuintinShaw/sitrep/daemon/internal/protocol" rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" - "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" ) @@ -85,8 +84,23 @@ type Uplink struct { kick chan struct{} done chan struct{} - // rtRetry holds reliable events (task.event/message.event) whose - // enqueue into the realtime outbox hit outbox.ErrOutboxFull. They are + // rtMu guards rtRetry and serializes sendReliable's check-then-act step + // against retryRealtime's drain. It is a separate lock from mu (rather + // than reusing it) because sendReliable may hold it across a live + // outbox.Enqueue call (a blocking DB transaction) and must not block + // unrelated Offer() bookkeeping (pending/queue/logs) while doing so. + // + // The invariant this protects: device_seq is allocated only by a + // successful Enqueue, so a new event must never win a race into Enqueue + // while an older one is still waiting in rtRetry — that would hand the + // newer event a lower seq and invert wire order relative to the older + // one once it is retried. See sendReliable/retryRealtime. + rtMu sync.Mutex + + // rtRetry holds reliable events (task.event/message.event) that could + // not be enqueued into the realtime outbox on first attempt — either + // because the outbox hit its row cap (outbox.ErrOutboxFull) or because + // Enqueue itself failed transiently (see sendReliable). They are // retried, oldest first, on every flush tick until Enqueue succeeds — // see routeToRealtime and retryRealtime. This is local backpressure, // not a fallback: these events must not travel the legacy HTTP path @@ -232,34 +246,36 @@ func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { } } -// sendReliable calls send() once and always reports "handled" (true) for a -// reliable event — the caller (routeToRealtime) must never fall back to -// HTTP for these, so there is nothing left for it to do with the return -// value except decide whether to retry locally. +// sendReliable always reports "handled" (true) for a reliable event — the +// caller (routeToRealtime) must never fall back to HTTP for these, so there +// is nothing left for it to do except decide whether to retry locally. +// +// The whole check-then-act step runs under rtMu, held for the duration of +// any direct send() attempt below: // -// On outbox.ErrOutboxFull (the bounded local queue hit its row cap because -// the server has not been acking), send is queued for retry: local -// backpressure, not a fallback. It is retried oldest-first on every flush -// tick (retryRealtime) until Enqueue succeeds, which self-heals once acks -// drain the backlog (SPEC.md section 5.3) — events already in the outbox -// are durable in SQLite and replay in seq order on their own. FIFO order -// matters here: device_seq is allocated only by a successful Enqueue, so -// retrying anything but the oldest queued item first could hand a later -// event a lower seq than an earlier one still waiting. +// - rtRetry non-empty: an older event is still waiting for Enqueue to +// succeed. This new one must NOT attempt Enqueue directly — doing so +// could win a device_seq ahead of the older event, inverting wire order +// once the older one is eventually retried (the BLOCKER this guards +// against). It is appended behind the queue instead, preserving FIFO. +// - rtRetry empty: send() is attempted now. On failure, see isRetryable. // -// Any other error (e.g. a malformed event failing wire validation) is not -// a transient condition retrying would fix, and per the same no-fallback -// rule it must not be forwarded to /v2 either — it is logged and dropped. +// retryRealtime holds the same rtMu across its whole drain, so a concurrent +// Offer can never slip an Enqueue in between two items it is draining, or +// between its "queue empty" check and this function's own check above. func (u *Uplink) sendReliable(send func() error) bool { + u.rtMu.Lock() + defer u.rtMu.Unlock() + + if len(u.rtRetry) > 0 { + u.rtRetry = append(u.rtRetry, send) + u.wakeFlush() + return true + } if err := send(); err != nil { - if errors.Is(err, outbox.ErrOutboxFull) { - u.mu.Lock() + if isRetryable(err) { u.rtRetry = append(u.rtRetry, send) - u.mu.Unlock() - select { - case u.kick <- struct{}{}: - default: - } + u.wakeFlush() } else { u.cfg.Logf("uplink: dropping reliable event, realtime enqueue failed: %v", err) } @@ -267,30 +283,61 @@ func (u *Uplink) sendReliable(send func() error) bool { return true } +// isRetryable classifies a failure from a sendReliable closure (ultimately +// an outbox.Enqueue error) as transient (retry) or permanent (log-and-drop). +// +// outbox.ErrOutboxFull is transient by definition: local backpressure that +// self-heals once acks drain the backlog (SPEC.md section 5.3). +// +// rtclient.ErrInvalidBody is the one permanent case: the event failed wire +// validation (bad kind, out-of-range percent, oversized text, ...), and +// retrying the exact same body can never produce a different result. +// +// Anything else — including an error this function doesn't recognize — is +// treated as transient. Enqueue's non-ErrOutboxFull failures are BeginTx, +// COUNT, INSERT, or Commit errors against the local SQLite file: transient +// conditions (SQLITE_BUSY, a disk I/O blip) that a later retry can plausibly +// clear. Losing a reliable task.done/message.send to a momentary disk +// hiccup is not acceptable, so the default for an unrecognized error is +// retry, never drop. There is no cap on rtRetry's growth during an outage; +// this is the same bounded-by-producer-rate tradeoff already accepted for +// outbox.ErrOutboxFull. +func isRetryable(err error) bool { + if errors.Is(err, rtclient.ErrInvalidBody) { + return false + } + return true +} + // retryRealtime re-attempts every event queued by sendReliable, oldest -// first, stopping at the first one that still fails so order is preserved -// (see sendReliable). Called once per flush tick from loop, so a drained -// outbox flushes its whole backlog within one tick. +// first, stopping at the first one that still fails transiently so order is +// preserved (see sendReliable). Called once per flush tick from loop, so a +// drained outbox flushes its whole backlog within one tick. Holds rtMu for +// its entire drain (not just each item's list-mutation) so a concurrent +// Offer's sendReliable can never observe rtRetry as empty and race an +// Enqueue in ahead of an item this loop is still working through. func (u *Uplink) retryRealtime() { - for { - u.mu.Lock() - if len(u.rtRetry) == 0 { - u.mu.Unlock() - return - } + u.rtMu.Lock() + defer u.rtMu.Unlock() + for len(u.rtRetry) > 0 { send := u.rtRetry[0] - u.mu.Unlock() - - err := send() - if err != nil && errors.Is(err, outbox.ErrOutboxFull) { - return // still full; try again next tick - } - if err != nil { + if err := send(); err != nil { + if isRetryable(err) { + return // still failing transiently; try again next tick + } u.cfg.Logf("uplink: dropping reliable event, realtime enqueue failed on retry: %v", err) } - u.mu.Lock() u.rtRetry = u.rtRetry[1:] - u.mu.Unlock() + } +} + +// wakeFlush nudges the flush loop to run sooner than the next tick, e.g. +// after queuing a reliable event for retry so a freed outbox row is picked +// up promptly rather than waiting a full FlushInterval. +func (u *Uplink) wakeFlush() { + select { + case u.kick <- struct{}{}: + default: } } From e1b87b115073be815e4cfa49d6bcc42aa7cb9720 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:07:57 +0800 Subject: [PATCH 51/60] fix(server): gate /v3/realtime and /v3/automations on REALTIME_ENABLED REALTIME_ENABLED=false only suppressed realtime_enabled in /v2/snapshot; the /v3/realtime upgrade route accepted connections regardless, so a daemon with its local flag on kept writing to SpaceHub while viewers read UserStore (dual authority). The route now rejects with 403 {"error":"realtime_disabled"} using the same parseRealtimeEnabledFlag single source of truth, checked after authenticate() (so an unauthenticated caller still gets its normal 401, not a 403 that would leak flag state) and before the SpaceHub stub.fetch() call, so a disabled deployment never wakes a DO. /v3/automations POST/PATCH/DELETE get the identical gate: they mutate the same SpaceHub DO via mintConfigEvent, so leaving them open would just relocate the dual-authority bug to the automations control plane instead of fixing it. No active-connection kill is needed: flag changes ship via redeploy, which evicts every DO instance and drops its live WebSockets as a side effect (noted at the check site). --- server/src/adapters/workers.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index 9cca830..8a07096 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -530,6 +530,15 @@ app.get("/v3/realtime", async (c: any) => { // role but no device identity; the realtime protocol requires one. return c.json({ error: "realtime requires a paired device token" }, 401); } + // Flag check runs AFTER authenticate() (so an unauthenticated caller still + // gets its normal 401, not a 403 that would leak flag state) and BEFORE + // the spaceHubStub().fetch() below — a disabled deployment must never wake + // a SpaceHub DO. No separate "kill existing connections" path is needed: + // flag changes ship via redeploy, which evicts every DO instance and drops + // all of its live WebSockets as a side effect. + if (!parseRealtimeEnabledFlag(env.REALTIME_ENABLED)) { + return c.json({ error: "realtime_disabled" }, 403); + } const realtimeRole: "source" | "viewer" = role === "source" ? "source" : "viewer"; const headers = new Headers(raw.headers); @@ -582,7 +591,19 @@ const internalError = (c: any, e: unknown) => { return c.json({ error: "internal error" }, 500); }; +// /v3/automations POST/PATCH/DELETE all end in a stub.mintConfigEvent() / +// stub.automationsSnapshot() call against the SAME SpaceHub DO that +// /v3/realtime guards — a half-open control plane (realtime connections +// blocked but automations still mutating SpaceHub state behind the viewers' +// backs) is the dual-authority bug this flag exists to prevent, so these +// routes get the identical 403 gate, checked after authenticate() and before +// any DO call. +function realtimeDisabled(c: any): boolean { + return !parseRealtimeEnabledFlag((c.env as WorkerEnv).REALTIME_ENABLED); +} + app.post("/v3/automations", async (c: any) => { + if (realtimeDisabled(c)) return c.json({ error: "realtime_disabled" }, 403); const role = c.get("role") as Role; if (!["admin", "owner"].includes(role)) return c.json({ error: "forbidden" }, 403); const body = await c.req.json().catch(() => null); @@ -623,6 +644,7 @@ app.post("/v3/automations", async (c: any) => { }); app.patch("/v3/automations/:id", async (c: any) => { + if (realtimeDisabled(c)) return c.json({ error: "realtime_disabled" }, 403); const role = c.get("role") as Role; if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); try { @@ -647,6 +669,7 @@ app.patch("/v3/automations/:id", async (c: any) => { }); app.delete("/v3/automations/:id", async (c: any) => { + if (realtimeDisabled(c)) return c.json({ error: "realtime_disabled" }, 403); const role = c.get("role") as Role; if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); try { From fd3bb11300ca8023ee8d6c4459377c9dec8d9cae Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:08:06 +0800 Subject: [PATCH 52/60] test(server): cover the REALTIME_ENABLED gate on /v3/realtime and /v3/automations Adds realtime-gate.workers.ts: flag off -> /v3/realtime and /v3/automations POST/PATCH/DELETE all return 403 realtime_disabled without waking a SpaceHub DO; flag on -> the upgrade still works (explicit regression check); unauthenticated + flag off -> still 401, not 403. wrangler.jsonc's REALTIME_ENABLED stays false as the production kill-switch default, but the rest of the /v3/realtime protocol suite (handshake, broadcast, reliability, token-gate, ...) exercises the enabled path and would otherwise all start failing with 403. Override it to true for the test pool only via vitest.config.ts's miniflare bindings; this file flips env.REALTIME_ENABLED back to false per-test to reach the disabled branch. --- server/test/workers/realtime-gate.workers.ts | 94 ++++++++++++++++++++ server/vitest.config.ts | 8 ++ 2 files changed, 102 insertions(+) create mode 100644 server/test/workers/realtime-gate.workers.ts diff --git a/server/test/workers/realtime-gate.workers.ts b/server/test/workers/realtime-gate.workers.ts new file mode 100644 index 0000000..b9466c0 --- /dev/null +++ b/server/test/workers/realtime-gate.workers.ts @@ -0,0 +1,94 @@ +// P0-1 (server half): REALTIME_ENABLED=false must actually gate /v3/realtime +// and its HTTP control-plane companion, /v3/automations, BEFORE any +// SpaceHub Durable Object is touched — otherwise a daemon with its local +// flag on keeps writing to SpaceHub while viewers read UserStore (dual +// authority). The rest of the suite runs with the test pool's +// REALTIME_ENABLED override (see vitest.config.ts) on, so this file flips +// `env.REALTIME_ENABLED` back to `false` per-test to exercise the gate. +import { env, listDurableObjectIds } from "cloudflare:test"; +import { describe, expect, it, afterEach } from "vitest"; +import { SELF } from "cloudflare:test"; +import { bootstrapSpace, upgrade } from "./helpers"; + +const ORIGIN = "https://example.com"; + +describe("realtime kill switch", () => { + afterEach(() => { + // Restore the pool-wide override so later files/tests aren't affected + // by this file's per-test flips. + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = true; + }); + + it("flag off: /v3/realtime returns 403 realtime_disabled and wakes no SpaceHub", async () => { + const { viewer } = await bootstrapSpace(); + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + const res = await upgrade(viewer.token); + expect(res.status).toBe(403); + expect(res.webSocket).toBeNull(); + expect(await res.json()).toEqual({ error: "realtime_disabled" }); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("flag off: /v3/automations POST returns 403 realtime_disabled and wakes no SpaceHub", async () => { + const { ownerToken } = await bootstrapSpace(); + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ name: "disk watch", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(res.status).toBe(403); + expect(await res.json()).toEqual({ error: "realtime_disabled" }); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("flag off: /v3/automations PATCH and DELETE also return 403 realtime_disabled", async () => { + const { ownerToken } = await bootstrapSpace(); + // create it first, while the flag is still on for this test. + const createRes = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: "auto-gate", name: "disk watch", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(createRes.status).toBe(200); + + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + const patchRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-gate`, { + method: "PATCH", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ state: "paused" }), + }); + expect(patchRes.status).toBe(403); + expect(await patchRes.json()).toEqual({ error: "realtime_disabled" }); + + const deleteRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-gate`, { + method: "DELETE", + headers: { authorization: `Bearer ${ownerToken}` }, + }); + expect(deleteRes.status).toBe(403); + expect(await deleteRes.json()).toEqual({ error: "realtime_disabled" }); + }); + + it("flag on: /v3/realtime upgrade still works (explicit regression check)", async () => { + const { source } = await bootstrapSpace(); + const res = await upgrade(source.token); + expect(res.status).toBe(101); + expect(res.webSocket).not.toBeNull(); + res.webSocket?.accept(); + res.webSocket?.close(); + }); + + it("flag off: an unauthenticated request still gets 401, not 403 (no flag-state leak)", async () => { + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + const res = await upgrade(null); + expect(res.status).toBe(401); + const badToken = await upgrade("not-a-real-token"); + expect(badToken.status).toBe(401); + }); +}); diff --git a/server/vitest.config.ts b/server/vitest.config.ts index 2719d13..867cd2f 100644 --- a/server/vitest.config.ts +++ b/server/vitest.config.ts @@ -8,6 +8,14 @@ export default defineConfig({ plugins: [ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, + // wrangler.jsonc's REALTIME_ENABLED stays "false" as the production + // kill-switch default (flipped per-environment at deploy time), but + // the /v3/realtime protocol suite needs it on to exercise the + // handshake/broadcast/reliability paths — override it here for the + // test pool only. Tests covering the disabled path flip it back to + // `false` per-test via `env.REALTIME_ENABLED` (see + // realtime-gate.workers.ts). + miniflare: { bindings: { REALTIME_ENABLED: true } }, }), ], }); From 077c9268dc7d32c80cac2289e986965112f91750 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:13:26 +0800 Subject: [PATCH 53/60] fix(daemon): persist overflowed reliable events to a durable outbox_overflow table The outbox's row cap used to reject Enqueue outright (ErrOutboxFull), pushing backpressure onto an in-memory retry queue in the uplink that a short-lived process (agent.go's Uplink.Close()) never drained past a single final flush. Enqueue now durably persists an overflowing event into a new outbox_overflow table in the same SQLite database instead, returning the informational ErrOverflowed. PromoteOverflow moves the oldest overflow row into the seq-bearing outbox (allocating device_seq only at that moment) whenever an ack frees capacity or the store reopens after a restart; the FIFO guard against a fresh event stealing a seq ahead of an older overflowed one is enforced inside Enqueue's own transaction. --- daemon/internal/realtime/outbox/outbox.go | 378 ++++++++++++++++-- .../internal/realtime/outbox/outbox_test.go | 217 +++++++++- 2 files changed, 548 insertions(+), 47 deletions(-) diff --git a/daemon/internal/realtime/outbox/outbox.go b/daemon/internal/realtime/outbox/outbox.go index 078c914..cccf529 100644 --- a/daemon/internal/realtime/outbox/outbox.go +++ b/daemon/internal/realtime/outbox/outbox.go @@ -24,20 +24,31 @@ import ( ) // DefaultMaxRows bounds the outbox's total row count (across all spaces). -// The cap keeps local disk usage bounded when the server is unreachable -// for a long stretch; reliability does not degrade at the cap because the -// caller's contract is to fall back to another delivery path (the HTTP -// ingest batch) whenever Enqueue fails — see ErrOutboxFull. +// The cap keeps local disk usage bounded when the server is unreachable for +// a long stretch. Reliability does not degrade at the cap: an Enqueue that +// would exceed it is transparently redirected, in the same transaction, into +// outbox_overflow (see ErrOverflowed) — a second, uncapped durable table in +// this same database file — rather than being rejected. The event is +// durable either way; only its promotion into the seq-bearing outbox is +// deferred until acks free capacity (see PromoteOverflow). const DefaultMaxRows = 5000 -// ErrOutboxFull is returned (wrapped) by Enqueue when the outbox already -// holds its maximum number of unacknowledged rows. No device_seq is -// consumed in that case — the failed Enqueue's transaction rolls back -// whole, exactly like any other Enqueue failure — so callers can safely -// route the event to a fallback path (the HTTP ingest batch) and let -// later events use the realtime path again once acks drain the backlog. +// ErrOutboxFull is a legacy sentinel kept for API compatibility; Enqueue no +// longer returns it (a full outbox now overflows instead of failing — see +// ErrOverflowed). Nothing in this package returns it anymore. var ErrOutboxFull = errors.New("outbox: full") +// ErrOverflowed is returned (wrapped) by Enqueue when the event was, instead +// of being written straight into the seq-bearing outbox, durably persisted +// into outbox_overflow: the outbox was at its row cap, or an older event for +// the same space was already waiting in overflow (see the FIFO note on +// Enqueue). It is informational, not a failure — by the time it is +// returned, the event has already survived a process crash. Callers must +// not retry or reroute on this error; PromoteOverflow (called on ack-freed +// capacity and at Store open) moves the event into the outbox, allocating +// its device_seq at that moment, once room and turn allow. +var ErrOverflowed = errors.New("outbox: enqueued to overflow (outbox full or an older overflow entry pending)") + const schema = ` CREATE TABLE IF NOT EXISTS space_seq_counters ( space TEXT PRIMARY KEY, @@ -51,6 +62,23 @@ CREATE TABLE IF NOT EXISTS outbox ( created_at INTEGER NOT NULL, PRIMARY KEY (space, device_seq) ); +-- outbox_overflow holds reliable events that arrived while the outbox was +-- at capacity (or while an older overflow entry for the same space was +-- still pending promotion). No device_seq column: a seq is only ever +-- allocated when PromoteOverflow moves a row into outbox, never before, so +-- there is nothing to conflict with the outbox's own counter. "body" holds +-- the pre-seq event material — the same JSON Enqueue would have written, +-- built with a placeholder device_seq of 1 purely to satisfy wire.Validate +-- (which requires device_seq >= 1) — with the real value patched in at +-- promotion time (see setDeviceSeq). id's autoincrement order is the FIFO +-- order promotion must honor. +CREATE TABLE IF NOT EXISTS outbox_overflow ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + space TEXT NOT NULL, + kind TEXT NOT NULL, + body TEXT NOT NULL, + created_at INTEGER NOT NULL +); ` // Item is one persisted, not-yet-acknowledged reliable event. @@ -99,7 +127,19 @@ func OpenWithMaxRows(path string, maxRows int) (*Store, error) { db.Close() return nil, fmt.Errorf("outbox: init schema: %w", err) } - return &Store{db: db, maxRows: maxRows}, nil + store := &Store{db: db, maxRows: maxRows} + + // Restart recovery: anything left in outbox_overflow from a previous + // process survives on disk exactly where it was, but it only becomes + // eligible for delivery once promoted into the seq-bearing outbox. + // Attempt every promotion the current cap allows right away, so a + // daemon that restarts with free capacity (or with an empty outbox) + // doesn't wait for the next ack to notice overflowed work is waiting. + if _, err := store.PromoteOverflow(context.Background(), maxRows); err != nil { + db.Close() + return nil, fmt.Errorf("outbox: promote overflow on open: %w", err) + } + return store, nil } // Close releases the underlying database handle. @@ -110,59 +150,264 @@ func (s *Store) Close() error { return s.db.Close() } // caller never has to allocate a seq itself ahead of time. type BuildBody func(seq int64) (json.RawMessage, error) -// Enqueue allocates the next device_seq for space and durably stores the -// event BuildBody produces for it, atomically: SPEC.md section 5.1 requires -// device_seq to be assigned once, never reused, and strictly increasing per -// (device, space); doing the allocation and the insert in one transaction -// means a process crash between them can never happen — the transaction -// either commits both effects or neither, so a restarted daemon reusing this -// same database file always sees a counter that agrees with what is (or -// isn't) sitting in the outbox. +// enqueueMaxAttempts bounds Enqueue's internal retry of a transient SQLite +// failure (BeginTx/COUNT/INSERT/Commit erroring — e.g. SQLITE_BUSY beyond +// what the driver's own busy_timeout pragma already absorbs, or a momentary +// disk I/O blip). The store's DSN already sets busy_timeout(5000), so most +// lock contention resolves inside a single attempt; this loop is a second, +// small layer on top so a transient failure that busy_timeout doesn't cover +// does not surface to the caller as event loss on the first try. +const enqueueMaxAttempts = 3 + +// enqueueRetryDelay is the (small, fixed) pause between Enqueue's internal +// attempts. Deliberately short: Offer() must not block the caller for long, +// and busy_timeout already handles the common lock-contention case. +const enqueueRetryDelay = 20 * time.Millisecond + +// Enqueue durably persists the event BuildBody produces, allocating its +// device_seq atomically with the insert (SPEC.md section 5.1: a crash +// between "seq chosen" and "event durably stored" can never happen — the +// transaction commits both effects or neither). +// +// If the outbox is at its row cap, OR an older event for this same space is +// still sitting in outbox_overflow awaiting promotion, this event is +// persisted into outbox_overflow instead (no device_seq allocated) and +// Enqueue returns ErrOverflowed — the event is durable either way. The +// overflow-nonempty check happens inside the very same transaction as the +// cap check and the insert, which is what prevents the round-1 +// seq-order-inversion race from resurfacing at the DB layer: without it, a +// fresh Enqueue could race a still-overflowed older event for a freed +// outbox row and win it a lower seq, inverting wire order once the older +// event is eventually promoted. Routing every new arrival to overflow +// whenever overflow is non-empty, checked under the same lock as the +// promotion path, makes that race impossible. +// +// A transient failure (not ErrOverflowed, not a BuildBody/validation error) +// is retried internally up to enqueueMaxAttempts times before it is +// returned to the caller — see enqueueMaxAttempts. func (s *Store) Enqueue(ctx context.Context, space, kind string, build BuildBody) (Item, error) { + var lastErr error + for attempt := 0; attempt < enqueueMaxAttempts; attempt++ { + if attempt > 0 { + select { + case <-time.After(enqueueRetryDelay): + case <-ctx.Done(): + return Item{}, ctx.Err() + } + } + item, terminal, err := s.enqueueAttempt(ctx, space, kind, build) + if err == nil { + return item, nil + } + if terminal { + // ErrOverflowed (success-shaped) or a BuildBody/validation + // failure (the same input can never validate differently): + // neither benefits from a retry. + return item, err + } + lastErr = err + } + return Item{}, fmt.Errorf("outbox: enqueue failed after %d attempts: %w", enqueueMaxAttempts, lastErr) +} + +// enqueueAttempt is one transactional attempt at Enqueue's work. terminal +// reports whether the caller should stop retrying regardless of err being +// non-nil (a BuildBody failure, or the informational ErrOverflowed). +func (s *Store) enqueueAttempt(ctx context.Context, space, kind string, build BuildBody) (item Item, terminal bool, err error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return Item{}, fmt.Errorf("outbox: begin: %w", err) + return Item{}, false, fmt.Errorf("outbox: begin: %w", err) } defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit - // Row cap check, inside the same transaction as the insert so two - // concurrent Enqueues cannot both squeeze past the limit. At the cap - // the whole transaction rolls back — no seq consumed, no row written — - // and the caller falls back to its non-realtime delivery path. + // FIFO guard: while an older event for this space is still waiting in + // overflow, this new one must join it there too, even if the outbox + // below is not currently at cap (e.g. an ack freed a row a moment ago, + // but PromoteOverflow hasn't run yet) — never let a fresher event claim + // a seq ahead of an older still-overflowed one. + var overflowPending int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox_overflow WHERE space = ?`, space).Scan(&overflowPending); err != nil { + return Item{}, false, fmt.Errorf("outbox: count overflow: %w", err) + } + var rows int if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&rows); err != nil { - return Item{}, fmt.Errorf("outbox: count rows: %w", err) + return Item{}, false, fmt.Errorf("outbox: count rows: %w", err) } - if rows >= s.maxRows { - return Item{}, fmt.Errorf("outbox has %d rows (cap %d): %w", rows, s.maxRows, ErrOutboxFull) + + if overflowPending > 0 || rows >= s.maxRows { + if err := s.insertOverflowTx(ctx, tx, space, kind, build); err != nil { + return Item{}, true, err + } + if err := tx.Commit(); err != nil { + return Item{}, false, fmt.Errorf("outbox: commit overflow insert: %w", err) + } + return Item{}, true, fmt.Errorf("outbox has %d rows (cap %d), %d overflow pending for %s: %w", rows, s.maxRows, overflowPending, space, ErrOverflowed) } seq, err := nextSeqTx(tx, space) if err != nil { - return Item{}, err + return Item{}, false, err } body, err := build(seq) if err != nil { - return Item{}, fmt.Errorf("outbox: build body for seq %d: %w", seq, err) + return Item{}, true, fmt.Errorf("outbox: build body for seq %d: %w", seq, err) } now := time.Now().UnixMilli() if _, err := tx.ExecContext(ctx, `INSERT INTO outbox (space, device_seq, kind, body, created_at) VALUES (?, ?, ?, ?, ?)`, space, seq, kind, string(body), now, ); err != nil { - return Item{}, fmt.Errorf("outbox: insert seq %d: %w", seq, err) + return Item{}, false, fmt.Errorf("outbox: insert seq %d: %w", seq, err) } if _, err := tx.ExecContext(ctx, `INSERT INTO space_seq_counters (space, next_seq) VALUES (?, ?) ON CONFLICT(space) DO UPDATE SET next_seq = excluded.next_seq`, space, seq+1, ); err != nil { - return Item{}, fmt.Errorf("outbox: advance counter: %w", err) + return Item{}, false, fmt.Errorf("outbox: advance counter: %w", err) } if err := tx.Commit(); err != nil { - return Item{}, fmt.Errorf("outbox: commit: %w", err) + return Item{}, false, fmt.Errorf("outbox: commit: %w", err) } - return Item{Space: space, DeviceSeq: seq, Kind: kind, Body: body, CreatedAt: now}, nil + return Item{Space: space, DeviceSeq: seq, Kind: kind, Body: body, CreatedAt: now}, false, nil +} + +// insertOverflowTx persists the pre-seq event material into outbox_overflow +// within tx. build is called with a placeholder device_seq of 1 — the +// smallest value wire.Validate accepts — purely so validation can run; the +// real device_seq is patched in at promotion (see setDeviceSeq) and never +// observed on the wire in its placeholder form. +func (s *Store) insertOverflowTx(ctx context.Context, tx *sql.Tx, space, kind string, build BuildBody) error { + const placeholderSeq = 1 + body, err := build(placeholderSeq) + if err != nil { + return fmt.Errorf("outbox: build overflow body: %w", err) + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, + `INSERT INTO outbox_overflow (space, kind, body, created_at) VALUES (?, ?, ?, ?)`, + space, kind, string(body), now, + ); err != nil { + return fmt.Errorf("outbox: insert overflow: %w", err) + } + return nil +} + +// PromoteOverflow moves up to n of the oldest outbox_overflow rows into the +// seq-bearing outbox, allocating each one's device_seq at the moment it is +// promoted (never before). It stops early — before n promotions — once +// either outbox_overflow is empty or the outbox is at its row cap; n is +// simply an upper bound on the number of attempts, so passing a generous +// value (e.g. the store's own maxRows) is always safe. It reports how many +// rows were actually promoted. +// +// Call sites: the ack path (Ack, below) promotes one row per freed outbox +// slot, and Open/OpenWithMaxRows promote everything they can immediately on +// process start (restart recovery) — both are named in the work order as +// the two triggers that must not let a promotable row sit unnoticed. +func (s *Store) PromoteOverflow(ctx context.Context, n int) (int, error) { + promoted := 0 + for i := 0; i < n; i++ { + ok, err := s.promoteOneTx(ctx) + if err != nil { + return promoted, err + } + if !ok { + break + } + promoted++ + } + return promoted, nil +} + +// promoteOneTx promotes exactly the single oldest outbox_overflow row +// (lowest id, across all spaces — id order is overflow's FIFO order) into +// outbox, in one transaction: read the cap, read the oldest row, allocate +// its space's next device_seq, patch that seq into the stored body, insert +// into outbox, advance the counter, and delete the overflow row. ok is +// false (no error) when there was nothing to promote, or no room to promote +// it into. +func (s *Store) promoteOneTx(ctx context.Context) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("outbox: begin promotion: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + + var rows int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&rows); err != nil { + return false, fmt.Errorf("outbox: count rows: %w", err) + } + if rows >= s.maxRows { + return false, nil // no room; caller should stop + } + + var id int64 + var space, kind, body string + err = tx.QueryRowContext(ctx, + `SELECT id, space, kind, body FROM outbox_overflow ORDER BY id ASC LIMIT 1`, + ).Scan(&id, &space, &kind, &body) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("outbox: read oldest overflow row: %w", err) + } + + seq, err := nextSeqTx(tx, space) + if err != nil { + return false, err + } + finalBody, err := setDeviceSeq(json.RawMessage(body), seq) + if err != nil { + return false, fmt.Errorf("outbox: patch device_seq on promotion of overflow id %d: %w", id, err) + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, + `INSERT INTO outbox (space, device_seq, kind, body, created_at) VALUES (?, ?, ?, ?, ?)`, + space, seq, kind, string(finalBody), now, + ); err != nil { + return false, fmt.Errorf("outbox: insert promoted seq %d: %w", seq, err) + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO space_seq_counters (space, next_seq) VALUES (?, ?) + ON CONFLICT(space) DO UPDATE SET next_seq = excluded.next_seq`, + space, seq+1, + ); err != nil { + return false, fmt.Errorf("outbox: advance counter: %w", err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM outbox_overflow WHERE id = ?`, id); err != nil { + return false, fmt.Errorf("outbox: delete promoted overflow id %d: %w", id, err) + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("outbox: commit promotion: %w", err) + } + return true, nil +} + +// setDeviceSeq returns body with its top-level "device_seq" field replaced +// by seq, leaving every other field untouched. It works generically across +// both reliable event kinds (task.event/message.event) because both +// wire.TaskEventBody and wire.MessageEventBody serialize a "device_seq" +// field — this package intentionally does not import the wire package to +// do this via its concrete types, so the overflow mechanism stays agnostic +// to any future reliable event kind that follows the same convention. +func setDeviceSeq(body json.RawMessage, seq int64) (json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(body, &fields); err != nil { + return nil, fmt.Errorf("decode body: %w", err) + } + seqJSON, err := json.Marshal(seq) + if err != nil { + return nil, err + } + fields["device_seq"] = seqJSON + patched, err := json.Marshal(fields) + if err != nil { + return nil, fmt.Errorf("re-encode body: %w", err) + } + return patched, nil } // nextSeqTx reads and does NOT yet persist the next device_seq for space; @@ -199,11 +444,19 @@ func (s *Store) NextSeq(ctx context.Context, space string) (int64, error) { // local resend queue until it receives an ack that covers its device_seq"). // Acking a seq that is not present (already acked, or never enqueued here) // is a no-op, not an error — acks are idempotent (SPEC.md section 5.2). +// +// An ack is also one of the two triggers (the other is Store open) that +// must notice freed outbox capacity and promote a waiting overflow row: it +// attempts exactly one promotion afterward, best-effort. A promotion +// failure here is logged-and-swallowed rather than failing the ack itself +// (the ack succeeded; the event really was acknowledged) — the next ack or +// the caller's own periodic tick gets another chance. func (s *Store) Ack(ctx context.Context, space string, seq int64) error { _, err := s.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ? AND device_seq = ?`, space, seq) if err != nil { return fmt.Errorf("outbox: ack seq %d: %w", seq, err) } + _, _ = s.PromoteOverflow(ctx, 1) //nolint:errcheck // best-effort; see doc comment return nil } @@ -242,3 +495,62 @@ func (s *Store) Count(ctx context.Context) (int, error) { err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&n) return n, err } + +// OverflowItem is one durably-persisted, not-yet-promoted reliable event +// sitting in outbox_overflow. Unlike Item it carries no DeviceSeq — none has +// been allocated yet — and an ID instead, which is both its promotion-FIFO +// key and the handle DeleteOverflow needs. +type OverflowItem struct { + ID int64 + Space string + Kind string + Body json.RawMessage // pre-seq material; device_seq is a placeholder until promoted + CreatedAt int64 +} + +// OverflowPending returns every overflowed item for space, oldest-first by +// id — the same FIFO order PromoteOverflow honors. Intended for a caller +// that needs to drain overflow through a path other than promotion, e.g. +// the daemon's legacy-HTTP fallback when the server has disabled realtime +// entirely (see internal/uplink). +func (s *Store) OverflowPending(ctx context.Context, space string) ([]OverflowItem, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, space, kind, body, created_at FROM outbox_overflow WHERE space = ? ORDER BY id ASC`, + space) + if err != nil { + return nil, fmt.Errorf("outbox: query overflow pending: %w", err) + } + defer rows.Close() + var items []OverflowItem + for rows.Next() { + var it OverflowItem + var body string + if err := rows.Scan(&it.ID, &it.Space, &it.Kind, &body, &it.CreatedAt); err != nil { + return nil, fmt.Errorf("outbox: scan overflow pending: %w", err) + } + it.Body = json.RawMessage(body) + items = append(items, it) + } + return items, rows.Err() +} + +// DeleteOverflow removes exactly the given overflow row by id, without +// promoting it into outbox or allocating it a device_seq — for a caller +// that has durably delivered the event through some other path entirely +// (the legacy-HTTP fallback) and is retiring it the same way Ack retires a +// promoted item. Deleting an id that is not present is a no-op. +func (s *Store) DeleteOverflow(ctx context.Context, id int64) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM outbox_overflow WHERE id = ?`, id) + if err != nil { + return fmt.Errorf("outbox: delete overflow id %d: %w", id, err) + } + return nil +} + +// OverflowCount returns the number of items currently sitting in +// outbox_overflow across every space, mainly for observability/tests. +func (s *Store) OverflowCount(ctx context.Context) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox_overflow`).Scan(&n) + return n, err +} diff --git a/daemon/internal/realtime/outbox/outbox_test.go b/daemon/internal/realtime/outbox/outbox_test.go index 5eb5e87..5d48baa 100644 --- a/daemon/internal/realtime/outbox/outbox_test.go +++ b/daemon/internal/realtime/outbox/outbox_test.go @@ -247,11 +247,13 @@ func TestSurvivesRestart(t *testing.T) { } } -// TestEnqueueFullReturnsErrOutboxFull pins the bounded-outbox policy: at -// the row cap Enqueue fails with ErrOutboxFull, consumes no device_seq -// (the transaction rolls back whole), and capacity freed by an ack makes -// Enqueue work again with the next unconsumed seq. -func TestEnqueueFullReturnsErrOutboxFull(t *testing.T) { +// TestEnqueueFullOverflows pins the bounded-outbox policy: at the row cap, +// Enqueue does not fail — it durably persists the event into +// outbox_overflow and returns ErrOverflowed, consuming no device_seq (no +// seq is ever allocated for an overflowed event until it is promoted). An +// ack freeing outbox capacity then promotes the overflowed event, +// allocating it the next unconsumed seq at that moment. +func TestEnqueueFullOverflows(t *testing.T) { path := filepath.Join(t.TempDir(), "outbox.db") s, err := OpenWithMaxRows(path, 2) if err != nil { @@ -267,28 +269,215 @@ func TestEnqueueFullReturnsErrOutboxFull(t *testing.T) { } _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) - if !errors.Is(err, ErrOutboxFull) { - t.Fatalf("Enqueue at cap = %v, want ErrOutboxFull", err) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("Enqueue at cap = %v, want ErrOverflowed", err) } - // The failed Enqueue must not have consumed a sequence number. + // The overflowed Enqueue must not have consumed a sequence number: no + // seq is allocated until promotion. next, err := s.NextSeq(ctx, "space-a") if err != nil { t.Fatalf("NextSeq: %v", err) } if next != 3 { - t.Fatalf("NextSeq after full = %d, want 3 (rejected Enqueue must not consume a seq)", next) + t.Fatalf("NextSeq after overflow = %d, want 3 (an overflowed Enqueue must not consume a seq)", next) + } + overflowCount, err := s.OverflowCount(ctx) + if err != nil { + t.Fatalf("OverflowCount: %v", err) + } + if overflowCount != 1 { + t.Fatalf("OverflowCount = %d, want 1", overflowCount) } - // Freeing one row (an ack arrived) restores capacity. + // Freeing one row (an ack arrived) promotes the overflowed event, + // allocating it the next unconsumed seq. if err := s.Ack(ctx, "space-a", 1); err != nil { t.Fatalf("Ack: %v", err) } - item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 2 { + t.Fatalf("Pending after ack+promotion = %d items, want 2 (seq 2 and the promoted seq 3): %v", len(pending), pending) + } + if pending[len(pending)-1].DeviceSeq != 3 { + t.Fatalf("promoted item's seq = %d, want 3", pending[len(pending)-1].DeviceSeq) + } + overflowCount, err = s.OverflowCount(ctx) + if err != nil { + t.Fatalf("OverflowCount after promotion: %v", err) + } + if overflowCount != 0 { + t.Fatalf("OverflowCount after promotion = %d, want 0", overflowCount) + } +} + +// TestEnqueueRoutesToOverflowWhileOlderOverflowPending is the DB-layer +// regression test for the round-1 seq-order-inversion race, reproduced at +// the store instead of the uplink: a fresh Enqueue must never claim a seq +// via a just-freed outbox row while an older event for the same space is +// still waiting in overflow — even when the outbox itself currently has +// room. The check must live inside Enqueue's own transaction (not depend on +// a caller doing the right thing), so this test frees a row by a route +// other than Ack/PromoteOverflow (a direct delete) to isolate that +// invariant from Ack's own promotion behavior. +func TestEnqueueRoutesToOverflowWhileOlderOverflowPending(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s, err := OpenWithMaxRows(path, 2) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer s.Close() + ctx := context.Background() + + for i := 0; i < 2; i++ { + if _, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)); err != nil { + t.Fatalf("fill Enqueue %d: %v", i, err) + } + } + // A overflows: the outbox is full. + _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("A's Enqueue at cap = %v, want ErrOverflowed", err) + } + + // Free BOTH filler rows (enough room for both A and B to eventually + // promote) WITHOUT going through Ack, so no promotion runs yet — + // simulates "capacity freed" independent of this store's own + // ack-triggers-promotion wiring, isolating the Enqueue-side guard. + if _, err := s.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ?`, "space-a"); err != nil { + t.Fatalf("simulate freed rows: %v", err) + } + + // B is offered now: the outbox is completely empty (full room), but A + // is still waiting in overflow. B must join it there — claiming a row + // directly would hand B a lower seq than A gets once A is finally + // promoted, inverting wire order. + _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("B's Enqueue while outbox has room but overflow is non-empty = %v, want ErrOverflowed", err) + } + + if _, err := s.PromoteOverflow(ctx, 2); err != nil { + t.Fatalf("PromoteOverflow: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + // A and B, promoted in FIFO order. + if len(pending) != 2 { + t.Fatalf("Pending after promotion = %d items, want 2: %v", len(pending), pending) + } + seqA, seqB := pending[0].DeviceSeq, pending[1].DeviceSeq + if !(seqA < seqB) { + t.Fatalf("wire order inverted: expected A's seq < B's seq (A overflowed first), got A=%d B=%d", seqA, seqB) + } +} + +// TestOverflowPromotesOnOpenWhenCapacityFree pins the second required +// promotion trigger (the first is Ack, above): a fresh Store.Open must +// itself notice a waiting overflow row that has room to be promoted into +// and do so immediately, rather than waiting for the next ack — otherwise a +// daemon that restarts with an already-drained outbox would leave a +// perfectly promotable event stranded until unrelated future traffic +// happened to trigger an ack. +func TestOverflowPromotesOnOpenWhenCapacityFree(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s1, err := OpenWithMaxRows(path, 1) if err != nil { - t.Fatalf("Enqueue after ack freed capacity: %v", err) + t.Fatalf("OpenWithMaxRows: %v", err) + } + ctx := context.Background() + + filler, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if err != nil { + t.Fatalf("filler Enqueue: %v", err) + } + _, err = s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("overflow Enqueue = %v, want ErrOverflowed", err) + } + + // Free the filler's row without running this store's own + // promote-on-ack wiring, so the only remaining trigger that could + // notice the freed capacity is the next Open. + if _, err := s1.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ? AND device_seq = ?`, "space-a", filler.DeviceSeq); err != nil { + t.Fatalf("simulate freed row: %v", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + s2, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + pending, err := s2.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending after reopen: %v", err) + } + if len(pending) != 1 { + t.Fatalf("expected Open to promote the waiting overflow row into freed capacity, got %d pending: %v", len(pending), pending) + } + if n, err := s2.OverflowCount(ctx); err != nil || n != 0 { + t.Fatalf("OverflowCount after reopen-promotion = %d, %v, want 0", n, err) + } +} + +// TestOverflowSurvivesCloseAndReopen is the outbox-layer half of the +// reviewer's exact scenario (the uplink/client-level version lives in +// internal/uplink): an overflowed event, and the outbox rows that keep it +// overflowed, must both survive a process restart (Close + reopen the same +// database file) unchanged, and only promote once an ack actually frees +// capacity on the reopened store. +func TestOverflowSurvivesCloseAndReopen(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s1, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + ctx := context.Background() + + filler, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if err != nil { + t.Fatalf("filler Enqueue: %v", err) + } + _, err = s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("overflow Enqueue = %v, want ErrOverflowed", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + s2, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + // Nothing to promote yet: the outbox is still full (the filler is + // still unacked), so Open's promotion attempt is a no-op. + if n, err := s2.OverflowCount(ctx); err != nil || n != 1 { + t.Fatalf("OverflowCount after reopen = %d, %v, want 1 (outbox still full, nothing promotable)", n, err) + } + + if err := s2.Ack(ctx, "space-a", filler.DeviceSeq); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err := s2.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("Pending after ack+promotion = %d items, want 1: %v", len(pending), pending) } - if item.DeviceSeq != 3 { - t.Fatalf("seq after recovery = %d, want 3", item.DeviceSeq) + if n, err := s2.OverflowCount(ctx); err != nil || n != 0 { + t.Fatalf("OverflowCount after ack+promotion = %d, %v, want 0", n, err) } } From 16c942ba6665a7a5900b1438adf5f12972981222 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:21:13 +0800 Subject: [PATCH 54/60] fix(daemon): replace in-memory reliable-event retry with durable-overflow-aware residual queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rtRetry used to hold every event that hit outbox-full locally, in a slice Close() never fully drained — the exact loss the no-fork rule was meant to prevent for a short-lived per-automation Uplink. Now that outbox.Enqueue persists synchronously (overflowing durably instead of failing), sendReliable only needs a tiny residual queue for the genuinely rare case where Enqueue's own bounded retries are exhausted (a persistent local-disk failure) — ordinary outbox-full backpressure never reaches it. --- daemon/internal/uplink/realtime_test.go | 17 +- daemon/internal/uplink/residual_test.go | 130 ++++++++++++++++ daemon/internal/uplink/rtretry_test.go | 199 ------------------------ daemon/internal/uplink/uplink.go | 145 +++++++---------- 4 files changed, 193 insertions(+), 298 deletions(-) create mode 100644 daemon/internal/uplink/residual_test.go delete mode 100644 daemon/internal/uplink/rtretry_test.go diff --git a/daemon/internal/uplink/realtime_test.go b/daemon/internal/uplink/realtime_test.go index 8722b80..c4bbd06 100644 --- a/daemon/internal/uplink/realtime_test.go +++ b/daemon/internal/uplink/realtime_test.go @@ -140,9 +140,10 @@ func TestRealtimeFlagOffPreservesExistingBehavior(t *testing.T) { // event (task.event/message.event) diverted to /v2 while the realtime flag // is on could permanently vanish from a viewer's resume. When the realtime // outbox hits its row cap, the overflow event must NOT reach the HTTP -// ingest spy at all — it stays queued locally (bounded backpressure) and is -// retried until Enqueue succeeds, then delivered over realtime in seq -// order once acks drain the backlog. +// ingest spy at all — it is durably persisted into outbox_overflow (see +// outbox.ErrOverflowed) rather than rejected, and delivered over realtime, +// in the original offer order, once an ack frees outbox capacity and +// promotes it (outbox.Store.PromoteOverflow). func TestOutboxFullNeverForksToHTTPAndRecovers(t *testing.T) { var httpCap capture httpSrv := httptest.NewServer(httpCap.handler(t)) @@ -229,11 +230,11 @@ func TestOutboxFullNeverForksToHTTPAndRecovers(t *testing.T) { u.Offer(msg("first")) waitForWS("first") - // 2. Second event: outbox is full -> ErrOutboxFull. Per the P0 fix this - // must NOT fall back to HTTP; it is queued locally (Uplink.rtRetry) and - // retried every flush tick while the outbox stays full. Give it several - // flush intervals to (wrongly) reach HTTP if the fallback regressed, - // then assert it never did. + // 2. Second event: outbox is full -> durably persisted into + // outbox_overflow (ErrOverflowed). Per the P0 fix this must NOT fall + // back to HTTP; it stays in overflow until an ack frees outbox capacity + // and promotes it. Give it several flush intervals to (wrongly) reach + // HTTP if the fallback regressed, then assert it never did. u.Offer(msg("second")) time.Sleep(200 * time.Millisecond) for _, e := range httpCap.all() { diff --git a/daemon/internal/uplink/residual_test.go b/daemon/internal/uplink/residual_test.go new file mode 100644 index 0000000..7eb8aef --- /dev/null +++ b/daemon/internal/uplink/residual_test.go @@ -0,0 +1,130 @@ +package uplink + +import ( + "errors" + "fmt" + "testing" + + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" +) + +// TestSendReliableDropsPermanentValidationError pins the permanent/transient +// split: rtclient.ErrInvalidBody (a wire-validation failure) is never +// retried — the same malformed body can never validate differently — so it +// is logged and dropped instead of being held in the residual queue. +func TestSendReliableDropsPermanentValidationError(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + send := func() error { + return fmt.Errorf("%w: task.event: missing task_id", rtclient.ErrInvalidBody) + } + + handled := u.sendReliable(send) + if !handled { + t.Fatal("sendReliable must always report handled=true for a reliable event") + } + if len(u.residual) != 0 { + t.Fatalf("expected a validation failure to be dropped, not held for retry; residual has %d items", len(u.residual)) + } +} + +// TestSendReliableHoldsResidualOnPersistentFailure is the regression test +// for the rare residual-retry path: since outbox.Store.Enqueue now persists +// synchronously (durably, to the outbox or its overflow table, retrying +// transient SQLite failures internally and bounded), the only way send() +// can fail here with something other than rtclient.ErrInvalidBody is a +// genuinely persistent local-disk failure that outlasted Enqueue's own +// bounded retries. In that rare case the event must not be dropped: it is +// held in memory and retried on the next drainResidual call (normally +// driven by the flush loop, once per tick) until it finally persists. +func TestSendReliableHoldsResidualOnPersistentFailure(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + + persistentErr := errors.New("outbox: enqueue failed after 3 attempts: disk I/O error") + attempts := 0 + delivered := false + send := func() error { + attempts++ + if attempts < 3 { + return persistentErr + } + delivered = true + return nil + } + + handled := u.sendReliable(send) + if !handled { + t.Fatal("sendReliable must always report handled=true for a reliable event") + } + if len(u.residual) != 1 { + t.Fatalf("expected the event held in residual after a non-validation failure, residual has %d items", len(u.residual)) + } + if delivered { + t.Fatal("event must not have been delivered on the failed first attempt") + } + + u.drainResidual() // still fails (attempts=2) + if len(u.residual) != 1 { + t.Fatalf("expected the event to remain in residual after a second failed attempt, residual has %d items", len(u.residual)) + } + + u.drainResidual() // succeeds (attempts=3) + if !delivered { + t.Fatal("expected the event to be delivered once send() finally succeeded") + } + if len(u.residual) != 0 { + t.Fatalf("expected residual drained after the successful retry, has %d items", len(u.residual)) + } +} + +// TestDrainResidualPreservesFIFOAndDropsPermanentMidQueue asserts +// drainResidual walks the residual queue oldest-first, stopping at the +// first entry that is still failing (so a later entry can never be +// delivered ahead of an earlier one still stuck), while a permanent +// validation failure encountered mid-queue is dropped in place rather than +// blocking everything behind it. +func TestDrainResidualPreservesFIFOAndDropsPermanentMidQueue(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + + var order []string + transientErr := errors.New("outbox: enqueue failed after 3 attempts: disk I/O error") + + // A: fails once, then succeeds. + aAttempts := 0 + sendA := func() error { + aAttempts++ + if aAttempts == 1 { + return transientErr + } + order = append(order, "A") + return nil + } + // B: permanently invalid. + sendB := func() error { + return fmt.Errorf("%w: bad body", rtclient.ErrInvalidBody) + } + // C: always succeeds once reached. + sendC := func() error { + order = append(order, "C") + return nil + } + + u.sendReliable(sendA) // fails -> held + if len(u.residual) != 1 { + t.Fatalf("expected A held after its first failed attempt, residual has %d items", len(u.residual)) + } + // B and C are queued directly (bypassing sendReliable's own send() + // attempt) to deterministically build a 3-item residual queue without + // depending on timing. + u.residualMu.Lock() + u.residual = append(u.residual, sendB, sendC) + u.residualMu.Unlock() + + u.drainResidual() + + if len(u.residual) != 0 { + t.Fatalf("expected residual fully drained (A recovers, B is permanently dropped, C succeeds), has %d items: order so far %v", len(u.residual), order) + } + if len(order) != 2 || order[0] != "A" || order[1] != "C" { + t.Fatalf("delivery order = %v, want [A C] (B dropped in place, not blocking C)", order) + } +} diff --git a/daemon/internal/uplink/rtretry_test.go b/daemon/internal/uplink/rtretry_test.go deleted file mode 100644 index a31bb78..0000000 --- a/daemon/internal/uplink/rtretry_test.go +++ /dev/null @@ -1,199 +0,0 @@ -package uplink - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "path/filepath" - "testing" - - rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" - "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" -) - -// TestSendReliablePreservesFIFOAgainstFreshOffer is the regression test for -// the seq-order-inversion BLOCKER: a fresh Offer() must never win a race -// into outbox.Enqueue ahead of an older event still waiting in rtRetry, even -// when a row frees up between the older event's failed attempt and the -// newer one's Offer(). Reproduces the exact interleaving from the bug -// report by driving sendReliable/retryRealtime directly (white-box, same -// package) instead of racing real goroutines against a flush ticker — -// deterministic, no sleeps. -func TestSendReliablePreservesFIFOAgainstFreshOffer(t *testing.T) { - ctx := context.Background() - store, err := outbox.OpenWithMaxRows(filepath.Join(t.TempDir(), "outbox.db"), 5) - if err != nil { - t.Fatalf("OpenWithMaxRows: %v", err) - } - defer store.Close() - - const space = "space-1" - body := func(seq int64) (json.RawMessage, error) { - return json.RawMessage(fmt.Sprintf(`{"seq":%d}`, seq)), nil - } - - // Fill the outbox to its cap of 5 with unacked placeholder rows so the - // very next Enqueue attempt (A, below) hits ErrOutboxFull. - var filledSeqs []int64 - for i := 0; i < 5; i++ { - item, err := store.Enqueue(ctx, space, "message.event", body) - if err != nil { - t.Fatalf("prefill Enqueue: %v", err) - } - filledSeqs = append(filledSeqs, item.DeviceSeq) - } - - u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} - - var seqA, seqB int64 - sendA := func() error { - item, err := store.Enqueue(ctx, space, "message.event", body) - if err != nil { - return err - } - seqA = item.DeviceSeq - return nil - } - sendB := func() error { - item, err := store.Enqueue(ctx, space, "message.event", body) - if err != nil { - return err - } - seqB = item.DeviceSeq - return nil - } - - // Step 1: "outbox full with A in rtRetry". A is offered while completely - // full, so it must queue rather than deliver. - u.sendReliable(sendA) - if len(u.rtRetry) != 1 { - t.Fatalf("expected A queued for retry, rtRetry has %d items", len(u.rtRetry)) - } - if seqA != 0 { - t.Fatalf("A must not have enqueued while the outbox was full; got seq %d", seqA) - } - - // Step 2: "free exactly one row via ack" — an ack arriving on the - // rtclient connection goroutine, independent of the 1s flush tick. - if err := store.Ack(ctx, space, filledSeqs[0]); err != nil { - t.Fatalf("Ack: %v", err) - } - - // Step 3: "Offer(B) BEFORE the next flush tick". With the BLOCKER bug, - // sendReliable would attempt Enqueue unconditionally here, succeeding - // into the just-freed row and handing B a lower device_seq than the - // still-queued A. The fix must instead queue B behind A. - u.sendReliable(sendB) - if len(u.rtRetry) != 2 { - t.Fatalf("expected B queued behind A (FIFO) since rtRetry was non-empty, rtRetry has %d items", len(u.rtRetry)) - } - if seqB != 0 { - t.Fatalf("B must not have enqueued directly while A was still queued for retry; got seq %d", seqB) - } - - // Step 4: the next flush tick drains oldest-first, using the freed row - // for A. - u.retryRealtime() - if seqA == 0 { - t.Fatal("expected A to have been delivered using the freed row") - } - if len(u.rtRetry) != 1 { - t.Fatalf("expected only B still queued after draining A, rtRetry has %d items", len(u.rtRetry)) - } - - // Free a second row so B's retry can succeed too, and drain again. - if err := store.Ack(ctx, space, filledSeqs[1]); err != nil { - t.Fatalf("Ack: %v", err) - } - u.retryRealtime() - if seqB == 0 { - t.Fatal("expected B to have been delivered on the second drain") - } - if len(u.rtRetry) != 0 { - t.Fatalf("expected rtRetry fully drained, has %d items", len(u.rtRetry)) - } - - // The assertion the BLOCKER violates: chronological order (A offered - // before B) must match wire order, i.e. strictly increasing device_seq. - if !(seqA < seqB) { - t.Fatalf("wire order inverted: expected seqA < seqB (A offered first), got seqA=%d seqB=%d", seqA, seqB) - } -} - -// TestSendReliableRetriesTransientEnqueueError is the regression test for -// the MAJOR: a transient outbox.Enqueue failure that is neither -// ErrOutboxFull nor a validation error (e.g. SQLITE_BUSY or a disk I/O blip -// surfacing from BeginTx/COUNT/INSERT/Commit) must be retried, not dropped -// — losing a reliable task.done/message.send to a momentary disk hiccup is -// not acceptable. -func TestSendReliableRetriesTransientEnqueueError(t *testing.T) { - ctx := context.Background() - store, err := outbox.OpenWithMaxRows(filepath.Join(t.TempDir(), "outbox.db"), 10) - if err != nil { - t.Fatalf("OpenWithMaxRows: %v", err) - } - defer store.Close() - - u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} - - transientErr := errors.New("outbox: begin: disk I/O error") - attempts := 0 - var seq int64 - send := func() error { - attempts++ - if attempts == 1 { - // Simulate a transient Enqueue failure — not ErrOutboxFull, not - // a validation error, and not a class this code recognizes at - // all. isRetryable must default to "retry" for it. - return transientErr - } - item, err := store.Enqueue(ctx, "space-1", "message.event", func(seq int64) (json.RawMessage, error) { - return json.RawMessage(`{}`), nil - }) - if err != nil { - return err - } - seq = item.DeviceSeq - return nil - } - - handled := u.sendReliable(send) - if !handled { - t.Fatal("sendReliable must always report handled=true for a reliable event") - } - if len(u.rtRetry) != 1 { - t.Fatalf("expected the event queued for retry after an unrecognized transient error, rtRetry has %d items", len(u.rtRetry)) - } - if seq != 0 { - t.Fatal("event must not have been delivered on the failed first attempt") - } - - u.retryRealtime() - if seq == 0 { - t.Fatal("expected the event to be delivered on retry, not dropped") - } - if len(u.rtRetry) != 0 { - t.Fatalf("expected rtRetry drained after the successful retry, has %d items", len(u.rtRetry)) - } -} - -// TestSendReliableDropsPermanentValidationError pins the other half of the -// permanent/transient split: rtclient.ErrInvalidBody (a wire-validation -// failure) is the one Enqueue-adjacent error that must NOT be retried — -// the same malformed body can never validate differently — so it is -// logged and dropped instead of growing rtRetry forever. -func TestSendReliableDropsPermanentValidationError(t *testing.T) { - u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} - send := func() error { - return fmt.Errorf("%w: task.event: missing task_id", rtclient.ErrInvalidBody) - } - - handled := u.sendReliable(send) - if !handled { - t.Fatal("sendReliable must always report handled=true for a reliable event") - } - if len(u.rtRetry) != 0 { - t.Fatalf("expected a validation failure to be dropped, not queued; rtRetry has %d items", len(u.rtRetry)) - } -} diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index e576bae..5619b95 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -84,28 +84,17 @@ type Uplink struct { kick chan struct{} done chan struct{} - // rtMu guards rtRetry and serializes sendReliable's check-then-act step - // against retryRealtime's drain. It is a separate lock from mu (rather - // than reusing it) because sendReliable may hold it across a live - // outbox.Enqueue call (a blocking DB transaction) and must not block - // unrelated Offer() bookkeeping (pending/queue/logs) while doing so. - // - // The invariant this protects: device_seq is allocated only by a - // successful Enqueue, so a new event must never win a race into Enqueue - // while an older one is still waiting in rtRetry — that would hand the - // newer event a lower seq and invert wire order relative to the older - // one once it is retried. See sendReliable/retryRealtime. - rtMu sync.Mutex - - // rtRetry holds reliable events (task.event/message.event) that could - // not be enqueued into the realtime outbox on first attempt — either - // because the outbox hit its row cap (outbox.ErrOutboxFull) or because - // Enqueue itself failed transiently (see sendReliable). They are - // retried, oldest first, on every flush tick until Enqueue succeeds — - // see routeToRealtime and retryRealtime. This is local backpressure, - // not a fallback: these events must not travel the legacy HTTP path - // while the realtime flag is on (see package doc on Config.Realtime). - rtRetry []func() error + // residualMu guards residual, the last-resort in-memory queue for a + // reliable event (task.event/message.event) that could not be durably + // persisted at all — not even into the outbox's overflow table — after + // outbox.Store.Enqueue exhausted its own bounded internal retries (see + // that method's doc comment). This is NOT the local-backpressure path: + // an outbox at its row cap durably overflows instead of failing (see + // outbox.ErrOverflowed), so it never reaches here. residual exists only + // for the rare case of a genuinely failing local disk (e.g. the outbox + // file's filesystem is full or read-only) — see sendReliable. + residualMu sync.Mutex + residual []func() error ticksSinceSend int } @@ -248,86 +237,60 @@ func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { // sendReliable always reports "handled" (true) for a reliable event — the // caller (routeToRealtime) must never fall back to HTTP for these, so there -// is nothing left for it to do except decide whether to retry locally. -// -// The whole check-then-act step runs under rtMu, held for the duration of -// any direct send() attempt below: +// is nothing left for it to do except decide whether to hold it for a +// residual retry. // -// - rtRetry non-empty: an older event is still waiting for Enqueue to -// succeed. This new one must NOT attempt Enqueue directly — doing so -// could win a device_seq ahead of the older event, inverting wire order -// once the older one is eventually retried (the BLOCKER this guards -// against). It is appended behind the queue instead, preserving FIFO. -// - rtRetry empty: send() is attempted now. On failure, see isRetryable. +// send() is outbox.Store.Enqueue underneath (via rtclient.Client.Send*), +// which now persists synchronously: it durably writes the event into either +// the outbox or its overflow table before returning, retrying transient +// SQLite failures internally, bounded (see that method's doc comment). So +// by the time send() returns here, one of three things is true: // -// retryRealtime holds the same rtMu across its whole drain, so a concurrent -// Offer can never slip an Enqueue in between two items it is draining, or -// between its "queue empty" check and this function's own check above. +// - success: the event is durable and (if the outbox had room) already +// handed to the realtime connection to deliver; +// - rtclient.ErrInvalidBody: the event can never validate, at any seq, on +// any retry — logged and dropped, permanently; +// - anything else: Enqueue's own bounded retries were exhausted — a +// persistent local-disk failure. The event was NOT durably written +// anywhere. This is the one case this function still holds the event in +// memory (residual) and keeps retrying every flush tick, so it is not +// silently dropped while the process lives. This residual window exists +// ONLY under that persistent local-disk failure; ordinary outbox-full +// backpressure durably overflows inside Enqueue and never reaches here. func (u *Uplink) sendReliable(send func() error) bool { - u.rtMu.Lock() - defer u.rtMu.Unlock() - - if len(u.rtRetry) > 0 { - u.rtRetry = append(u.rtRetry, send) - u.wakeFlush() - return true - } if err := send(); err != nil { - if isRetryable(err) { - u.rtRetry = append(u.rtRetry, send) - u.wakeFlush() - } else { - u.cfg.Logf("uplink: dropping reliable event, realtime enqueue failed: %v", err) + if errors.Is(err, rtclient.ErrInvalidBody) { + u.cfg.Logf("uplink: dropping reliable event, validation failed: %v", err) + return true } + u.cfg.Logf("uplink: reliable event failed to persist (%v); holding in memory and retrying every flush tick", err) + u.residualMu.Lock() + u.residual = append(u.residual, send) + u.residualMu.Unlock() + u.wakeFlush() } return true } -// isRetryable classifies a failure from a sendReliable closure (ultimately -// an outbox.Enqueue error) as transient (retry) or permanent (log-and-drop). -// -// outbox.ErrOutboxFull is transient by definition: local backpressure that -// self-heals once acks drain the backlog (SPEC.md section 5.3). -// -// rtclient.ErrInvalidBody is the one permanent case: the event failed wire -// validation (bad kind, out-of-range percent, oversized text, ...), and -// retrying the exact same body can never produce a different result. -// -// Anything else — including an error this function doesn't recognize — is -// treated as transient. Enqueue's non-ErrOutboxFull failures are BeginTx, -// COUNT, INSERT, or Commit errors against the local SQLite file: transient -// conditions (SQLITE_BUSY, a disk I/O blip) that a later retry can plausibly -// clear. Losing a reliable task.done/message.send to a momentary disk -// hiccup is not acceptable, so the default for an unrecognized error is -// retry, never drop. There is no cap on rtRetry's growth during an outage; -// this is the same bounded-by-producer-rate tradeoff already accepted for -// outbox.ErrOutboxFull. -func isRetryable(err error) bool { - if errors.Is(err, rtclient.ErrInvalidBody) { - return false - } - return true -} - -// retryRealtime re-attempts every event queued by sendReliable, oldest -// first, stopping at the first one that still fails transiently so order is -// preserved (see sendReliable). Called once per flush tick from loop, so a -// drained outbox flushes its whole backlog within one tick. Holds rtMu for -// its entire drain (not just each item's list-mutation) so a concurrent -// Offer's sendReliable can never observe rtRetry as empty and race an -// Enqueue in ahead of an item this loop is still working through. -func (u *Uplink) retryRealtime() { - u.rtMu.Lock() - defer u.rtMu.Unlock() - for len(u.rtRetry) > 0 { - send := u.rtRetry[0] +// drainResidual re-attempts every event sendReliable could not persist at +// all, oldest first, stopping at the first one that still fails so order +// among residual entries is preserved. Called once per flush tick from +// loop. In the overwhelming majority of runs residual is always empty, so +// this is a cheap no-op check. +func (u *Uplink) drainResidual() { + u.residualMu.Lock() + defer u.residualMu.Unlock() + for len(u.residual) > 0 { + send := u.residual[0] if err := send(); err != nil { - if isRetryable(err) { - return // still failing transiently; try again next tick + if errors.Is(err, rtclient.ErrInvalidBody) { + u.cfg.Logf("uplink: dropping reliable event, validation failed on retry: %v", err) + u.residual = u.residual[1:] + continue } - u.cfg.Logf("uplink: dropping reliable event, realtime enqueue failed on retry: %v", err) + return // still failing to persist; try again next tick } - u.rtRetry = u.rtRetry[1:] + u.residual = u.residual[1:] } } @@ -424,7 +387,7 @@ func (u *Uplink) loop() { case <-ticker.C: case <-u.kick: } - u.retryRealtime() + u.drainResidual() batch, closed := u.drain() if len(batch) > 0 { u.send(batch) From 1114ff96148979d788e14a51e441baa70fd1a196 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:30:15 +0800 Subject: [PATCH 55/60] fix(daemon): obey server's realtime capability, not just the local flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server can now reject /v3/realtime pre-upgrade with HTTP 403 {"error":"realtime_disabled"} when its own REALTIME_ENABLED is off. coder/websocket's Dial keeps the rejected *http.Response on error, so a 403 there is recognized as the new sentinel ErrRealtimeDisabled rather than an ordinary transient dial failure — it no longer feeds the tight reconnect backoff, and instead waits a long, fixed, named interval before re-probing. Uplink.pollRealtimeMode polls Client.ServerDisabled() once per flush tick and switches the whole session to legacy /v2 HTTP routing while it's true — draining every event already sitting durably in the outbox/overflow tables to /v2, in FIFO order, so nothing already-enqueued gets stranded in a store no viewer reads. Authority genuinely moves to UserStore for this case; it is not the forked-reliable-path failure mode the earlier P0 fix guarded against. When the client's periodic re-probe succeeds again, new events resume routing to realtime immediately (legacy mode drained as it went, so there is no backlog to replay). --- daemon/internal/realtime/client/client.go | 95 +++++- daemon/internal/realtime/client/config.go | 18 ++ .../internal/realtime/client/disabled_test.go | 118 ++++++++ daemon/internal/uplink/legacy_test.go | 238 +++++++++++++++ daemon/internal/uplink/realtime_test.go | 7 +- daemon/internal/uplink/uplink.go | 274 +++++++++++++++++- 6 files changed, 738 insertions(+), 12 deletions(-) create mode 100644 daemon/internal/realtime/client/disabled_test.go create mode 100644 daemon/internal/uplink/legacy_test.go diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index 639646f..2d33d8b 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -14,6 +14,7 @@ import ( "github.com/coder/websocket" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" ) @@ -29,6 +30,26 @@ var errClientClosing = errors.New("realtime client: closing") // process recovers by constructing a new Client (i.e. an explicit restart). var errNoRetry = errors.New("realtime client: fatal, not retryable") +// ErrRealtimeDisabled marks a dial rejected with HTTP 403 before the +// WebSocket upgrade completed: the server's own REALTIME_ENABLED flag is +// off, so its /v3/realtime endpoint refuses every connection regardless of +// this device's credentials. This is neither the errNoRetry case (the flag +// can flip back on at any time, with no software upgrade or credential +// change required) nor an ordinary transient dial failure (hammering the +// normal reconnect backoff would be pointless load on a condition that +// cannot self-heal within seconds) — see run's handling and +// serverDisabledRecheckInterval. +var ErrRealtimeDisabled = errors.New("realtime client: server reports realtime disabled (403)") + +// serverDisabledRecheckInterval is how long the client waits before +// re-probing an endpoint that rejected the last dial with +// ErrRealtimeDisabled. Deliberately long and fixed: this is a server +// operator's flag, not something expected to flip from moment to moment, so +// there is no user-facing config for it (see the work order) — only a +// same-package test seam (Config.testDisabledRecheckInterval) so tests +// don't have to wait 5 minutes. +const serverDisabledRecheckInterval = 5 * time.Minute + // ErrInvalidBody wraps a failure from a wire body's Validate(), returned by // SendTaskEvent/SendMessageEvent when the caller-supplied fields can never // produce a valid envelope (bad kind, out-of-range percent, oversized free @@ -69,6 +90,14 @@ type Client struct { supersededPenalty atomic.Bool + // serverDisabled mirrors the outcome of the most recent dial attempt: + // true from the moment a dial is rejected with ErrRealtimeDisabled + // until the next dial actually completes hello (see connectAndServe), + // at which point it is cleared. Uplink polls this (ServerDisabled) to + // decide whether to switch its whole session to legacy /v2 routing — + // see internal/uplink.Uplink.pollRealtimeMode's truth table. + serverDisabled atomic.Bool + closing chan struct{} closed chan struct{} once sync.Once @@ -102,6 +131,24 @@ func (c *Client) Connected() bool { return c.connected } +// ServerDisabled reports whether the most recent dial attempt was rejected +// with ErrRealtimeDisabled (server-side REALTIME_ENABLED off) and no dial +// has completed hello since. See Config's package doc and +// internal/uplink.Uplink.pollRealtimeMode for what the caller does with +// this. +func (c *Client) ServerDisabled() bool { return c.serverDisabled.Load() } + +// Outbox exposes the durable store backing this client, for a caller that +// needs to drain it through a path other than this client's own realtime +// delivery — specifically internal/uplink's legacy-mode fallback, which +// walks Outbox/Overflow directly once ServerDisabled reports the server has +// rejected realtime for this session. +func (c *Client) Outbox() *outbox.Store { return c.cfg.Outbox } + +// Space reports the (device, space) scope this client's outbox and +// device_seq counter are bound to (SPEC.md section 5.1) — see Outbox. +func (c *Client) Space() string { return c.cfg.Space } + // MetricsThrottled reports whether the metric batcher is currently in its // throttled cadence (SPEC.md section 7's command{throttle}/{resume_rate}). func (c *Client) MetricsThrottled() bool { return c.metrics.Throttled() } @@ -252,6 +299,28 @@ func (c *Client) run() { c.cfg.Logf("realtime: giving up, not retryable: %v", err) return } + if errors.Is(err, ErrRealtimeDisabled) { + // The server itself has realtime turned off. This is not a + // transient dial failure (the normal exponential backoff below + // would hammer it every ~60s for no reason: an operator flag + // does not flip back on within seconds) and not the + // give-up-forever case either (it CAN flip back on at any time, + // with no software upgrade or new credential needed) — so wait + // a long, fixed interval and try again, without touching the + // normal reconnect attempt counter at all. + c.serverDisabled.Store(true) + interval := serverDisabledRecheckInterval + if c.cfg.testDisabledRecheckInterval > 0 { + interval = c.cfg.testDisabledRecheckInterval + } + c.cfg.Logf("realtime: server reports realtime disabled; re-checking in %v", interval) + select { + case <-time.After(interval): + case <-c.closing: + return + } + continue + } c.cfg.Logf("realtime: connection ended: %v", err) c.mu.Lock() @@ -284,9 +353,21 @@ func (c *Client) connectAndServe() error { if c.cfg.Token != "" { header.Set("Authorization", "Bearer "+c.cfg.Token) } - conn, _, err := websocket.Dial(dialCtx, c.cfg.URL, &websocket.DialOptions{HTTPHeader: header}) + conn, resp, err := websocket.Dial(dialCtx, c.cfg.URL, &websocket.DialOptions{HTTPHeader: header}) dialCancel() if err != nil { + // A rejection before the WebSocket upgrade completes surfaces here + // as a non-nil err with resp still populated (coder/websocket keeps + // the response, with a small snippet of its body, specifically for + // this kind of diagnosis). Status 403 on this endpoint is + // sufficient signal on its own that the server's REALTIME_ENABLED + // is off — SERVER-side authority has moved to UserStore for this + // session, exactly like the local SITREP_REALTIME flag off has + // always meant "legacy only" on this side; see ErrRealtimeDisabled + // and internal/uplink.Uplink.pollRealtimeMode for what happens next. + if resp != nil && resp.StatusCode == http.StatusForbidden { + return fmt.Errorf("dial: %w", ErrRealtimeDisabled) + } return fmt.Errorf("dial: %w", err) } defer conn.CloseNow() @@ -300,6 +381,12 @@ func (c *Client) connectAndServe() error { } heartbeatInterval := time.Duration(accept.HeartbeatIntervalMS) * time.Millisecond + // A completed hello is proof the server currently accepts realtime, + // whether or not this connection had ever previously been rejected with + // ErrRealtimeDisabled — clear it so pollRealtimeMode's caller resumes + // realtime routing for new events immediately. + c.serverDisabled.Store(false) + c.mu.Lock() c.conn = conn c.connected = true @@ -573,6 +660,12 @@ func (c *Client) handleFrame(ctx context.Context, conn *websocket.Conn, env wire c.cfg.Logf("realtime: ack seq %d: %v", p.DeviceSeq, err) } } + // An ack frees outbox capacity, and Outbox.Ack itself promotes at + // most one waiting overflow row into that freed slot per call (see + // its doc comment). Wake the sender so a freshly-promoted row (now + // bearing a real device_seq) is written to this connection promptly + // via replayPending, rather than waiting for the next resend tick. + c.wakeSender() return nil case wire.TypeCommand: cb := body.(wire.CommandBody) diff --git a/daemon/internal/realtime/client/config.go b/daemon/internal/realtime/client/config.go index 1fec66d..e44aaaf 100644 --- a/daemon/internal/realtime/client/config.go +++ b/daemon/internal/realtime/client/config.go @@ -106,6 +106,24 @@ type Config struct { OnCommand func(CommandAction) Logf Logf + + // testDisabledRecheckInterval overrides serverDisabledRecheckInterval. + // Unexported: there is no public config surface for this (per the work + // order's "no config needed" — a server operator's flag is not + // something a user should be tuning). Set only via + // WithTestDisabledRecheckInterval, and only from tests. + testDisabledRecheckInterval time.Duration +} + +// WithTestDisabledRecheckInterval returns a copy of cfg with the periodic +// re-probe interval (serverDisabledRecheckInterval, normally ~5 minutes) +// shortened. This is a test-only seam for integration tests in sibling +// internal packages (e.g. internal/uplink) that need to observe a +// server-disabled -> re-enabled transition without waiting 5 real minutes; +// production code must never call this. +func WithTestDisabledRecheckInterval(cfg Config, d time.Duration) Config { + cfg.testDisabledRecheckInterval = d + return cfg } func (c *Config) applyDefaults() { diff --git a/daemon/internal/realtime/client/disabled_test.go b/daemon/internal/realtime/client/disabled_test.go new file mode 100644 index 0000000..fff9464 --- /dev/null +++ b/daemon/internal/realtime/client/disabled_test.go @@ -0,0 +1,118 @@ +package client + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestClientDetects403AndDoesNotHammer pins the P0-1 daemon-side fix: a +// dial rejected with HTTP 403 before the WebSocket upgrade completes must be +// recognized as ErrRealtimeDisabled (surfaced via ServerDisabled), not +// treated like an ordinary transient dial failure — which would otherwise +// retry on the normal (much shorter) exponential backoff and hammer an +// endpoint whose REALTIME_ENABLED flag cannot flip back on within +// milliseconds. +func TestClientDetects403AndDoesNotHammer(t *testing.T) { + var dialCount int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&dialCount, 1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + })) + defer srv.Close() + + // A recheck interval well above this test client's normal reconnect + // backoff (BackoffBase 5ms / BackoffMax 20ms, set by newTestClient) so + // the two are clearly distinguishable: if ErrRealtimeDisabled were + // (wrongly) treated as an ordinary transient dial failure, the normal + // backoff would produce roughly 350ms/20ms ~= 17 dials in the window + // below; obeying the recheck interval instead produces roughly + // 350ms/100ms ~= 3-4. + const recheck = 100 * time.Millisecond + const window = 350 * time.Millisecond + + store := newTestOutbox(t) + url := "ws" + strings.TrimPrefix(srv.URL, "http") + c := newTestClient(t, url, store, func(cfg *Config) { + *cfg = WithTestDisabledRecheckInterval(*cfg, recheck) + }) + + waitFor(t, 2*time.Second, c.ServerDisabled) + + time.Sleep(window) + if n := atomic.LoadInt32(&dialCount); n > 8 { + t.Fatalf("dial count = %d over %v with a %v recheck interval; expected roughly window/interval (~3-4), not the ~17 a tight reconnect-backoff loop would produce", n, window, recheck) + } +} + +// TestClientServerDisabledClearsOnceServerAccepts asserts the other half of +// the truth table: once a dial actually completes hello (the server has +// re-enabled realtime), ServerDisabled must clear so the caller (see +// internal/uplink.Uplink.pollRealtimeMode) resumes realtime routing. +func TestClientServerDisabledClearsOnceServerAccepts(t *testing.T) { + var attempts int32 + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&attempts, 1) + if n <= 2 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + return + } + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer ws.Close(websocket.StatusNormalClosure, "done") + ctx := r.Context() + _, data, err := ws.Read(ctx) + if err != nil { + return + } + env, err := wire.DecodeEnvelope(data) + if err != nil || env.Type != wire.TypeHello { + return + } + accept := wire.HelloAccept{Stage: "accept", ProtocolVersion: 1, SessionID: "sess", HeartbeatIntervalMS: 1000} + acceptEnv, err := wire.NewEnvelope(wire.TypeHello, "accept-env-1", time.Now().UnixMilli(), wire.HelloBody{Accept: &accept}) + if err != nil { + return + } + encoded, err := acceptEnv.Encode() + if err != nil { + return + } + if err := ws.Write(ctx, websocket.MessageText, encoded); err != nil { + return + } + for { + if _, _, err := ws.Read(ctx); err != nil { + return + } + } + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + store := newTestOutbox(t) + url := "ws" + strings.TrimPrefix(srv.URL, "http") + c := newTestClient(t, url, store, func(cfg *Config) { + *cfg = WithTestDisabledRecheckInterval(*cfg, 15*time.Millisecond) + }) + + waitFor(t, 2*time.Second, c.ServerDisabled) + waitFor(t, 2*time.Second, c.Connected) + if c.ServerDisabled() { + t.Fatal("expected ServerDisabled to clear once a connection completed hello") + } +} diff --git a/daemon/internal/uplink/legacy_test.go b/daemon/internal/uplink/legacy_test.go new file mode 100644 index 0000000..f968af3 --- /dev/null +++ b/daemon/internal/uplink/legacy_test.go @@ -0,0 +1,238 @@ +package uplink + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +func waitForCond(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + if !cond() { + t.Fatalf("condition not met within %v", timeout) + } +} + +// TestLegacyModeDrainsPendingAndNewEventsWhenServerDisabled is the P0-1 +// integration test: when the server rejects /v3/realtime with HTTP 403 +// realtime_disabled at dial time, the whole Uplink session must switch to +// legacy /v2 routing — including draining anything already sitting durably +// in the outbox from before this session even started (simulating a +// previous process run), so a terminal task state doesn't get stranded in a +// store no viewer reads. New events offered while still in legacy mode must +// also reach /v2, after the drained backlog, in order. No realtime frame +// may ever be sent (the dial never completes hello), and the dial must not +// be hammered. +func TestLegacyModeDrainsPendingAndNewEventsWhenServerDisabled(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + var dialCount int32 + rtSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&dialCount, 1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + })) + defer rtSrv.Close() + + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + defer store.Close() + + // Pre-populate the outbox as if a previous process run had already + // durably enqueued this reliable event before the server was known to + // be disabled. + ctx := context.Background() + if _, err := store.Enqueue(ctx, "space-1", wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + return json.Marshal(wire.MessageEventBody{ + DeviceID: "device-1", DeviceSeq: seq, MessageID: fmt.Sprintf("device-1:%d", seq), + Level: "info", Text: "preexisting", OccurredAt: time.Now().UnixMilli(), + }) + }); err != nil { + t.Fatalf("pre-populate Enqueue: %v", err) + } + + rt := rtclient.New(rtclient.Config{ + URL: "ws" + strings.TrimPrefix(rtSrv.URL, "http"), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + }) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt}) + defer u.Close() + + waitForCond(t, 2*time.Second, rt.ServerDisabled) + + // New reliable events offered while the session is in legacy mode must + // also reach /v2, after the drained backlog. + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "second"; e.Level = "info" })) + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "third"; e.Level = "info" })) + + waitForCond(t, 3*time.Second, func() bool { return len(httpCap.all()) >= 3 }) + + got := httpCap.all() + if len(got) != 3 { + t.Fatalf("got %d events over /v2, want exactly 3: %+v", len(got), got) + } + want := []string{"preexisting", "second", "third"} + for i, w := range want { + if got[i].Text != w { + t.Fatalf("event %d = %q, want %q (full order: %v)", i, got[i].Text, w, got) + } + } + + if n := atomic.LoadInt32(&dialCount); n > 8 { + t.Fatalf("dial count = %d; expected no rapid redial while the server reports realtime disabled", n) + } +} + +// TestLegacyModeResumesRealtimeAfterServerReenables asserts the switch-back +// half: once the realtime client's periodic re-probe succeeds (the server +// re-enables /v3/realtime), the Uplink resumes routing new reliable events +// to realtime and the /v2 spy stops growing. +func TestLegacyModeResumesRealtimeAfterServerReenables(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + var attempts int32 + rtReceived := make(chan wire.Envelope, 16) + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&attempts, 1) + if n <= 2 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + return + } + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer ws.Close(websocket.StatusNormalClosure, "done") + wsCtx := r.Context() + _, data, err := ws.Read(wsCtx) + if err != nil { + return + } + env, err := wire.DecodeEnvelope(data) + if err != nil || env.Type != wire.TypeHello { + return + } + accept := wire.HelloAccept{Stage: "accept", ProtocolVersion: 1, SessionID: "sess", HeartbeatIntervalMS: 1000} + acceptEnv, err := wire.NewEnvelope(wire.TypeHello, "accept-env-1", time.Now().UnixMilli(), wire.HelloBody{Accept: &accept}) + if err != nil { + return + } + encoded, err := acceptEnv.Encode() + if err != nil { + return + } + if err := ws.Write(wsCtx, websocket.MessageText, encoded); err != nil { + return + } + for { + _, data, err := ws.Read(wsCtx) + if err != nil { + return + } + if string(data) == "ping" { + _ = ws.Write(wsCtx, websocket.MessageText, []byte("pong")) + continue + } + e, derr := wire.DecodeEnvelope(data) + if derr != nil { + continue + } + rtReceived <- e + if e.Type == wire.TypeMessageEvent { + body, berr := wire.DecodeBody(e) + if berr == nil { + mb := body.(wire.MessageEventBody) + ackEnv, aerr := wire.NewEnvelope(wire.TypeAck, "ack-1", time.Now().UnixMilli(), + wire.AckBody{Acked: []wire.AckedPair{{DeviceID: mb.DeviceID, DeviceSeq: mb.DeviceSeq}}}) + if aerr == nil { + if encoded, eerr := ackEnv.Encode(); eerr == nil { + _ = ws.Write(wsCtx, websocket.MessageText, encoded) + } + } + } + } + } + }) + rtSrv := httptest.NewServer(mux) + defer rtSrv.Close() + + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + defer store.Close() + + rt := rtclient.New(rtclient.WithTestDisabledRecheckInterval(rtclient.Config{ + URL: "ws" + strings.TrimPrefix(rtSrv.URL, "http"), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + ResendInterval: 20 * time.Millisecond, + }, 30*time.Millisecond)) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt}) + defer u.Close() + + waitForCond(t, 2*time.Second, rt.ServerDisabled) + waitForCond(t, 3*time.Second, func() bool { return !rt.ServerDisabled() }) + waitForCond(t, 3*time.Second, rt.Connected) + // pollRealtimeMode (the Uplink's own tick-driven poll of + // rt.ServerDisabled()) needs one more flush tick after the client-level + // signal clears to flip the Uplink's own legacyMode back off; wait for + // that directly rather than racing it with a fixed sleep. + waitForCond(t, 2*time.Second, func() bool { return !u.inLegacyMode() }) + + baseline := len(httpCap.all()) + + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "afterReenable"; e.Level = "info" })) + + select { + case e := <-rtReceived: + if e.Type != wire.TypeMessageEvent { + t.Fatalf("expected message.event over realtime, got %s", e.Type) + } + case <-time.After(3 * time.Second): + t.Fatal("expected the new event to reach realtime once the server re-enabled it") + } + + time.Sleep(100 * time.Millisecond) + if got := len(httpCap.all()); got != baseline { + t.Fatalf("expected the /v2 spy to stop growing once realtime resumed: was %d, now %d", baseline, got) + } +} diff --git a/daemon/internal/uplink/realtime_test.go b/daemon/internal/uplink/realtime_test.go index c4bbd06..4c4aa08 100644 --- a/daemon/internal/uplink/realtime_test.go +++ b/daemon/internal/uplink/realtime_test.go @@ -119,7 +119,12 @@ loop: // TestRealtimeFlagOffPreservesExistingBehavior asserts the zero-value // (Realtime == nil) path is completely unaffected: everything still goes -// over HTTP, exactly as before this feature existed. +// over HTTP, exactly as before this feature existed. This also pins the +// local-flag-off row of pollRealtimeMode's truth table: with the local +// SITREP_REALTIME flag off, cmd/sitrep's newAgentRealtimeUplink never +// constructs an rtclient.Client at all (see its doc comment), so +// Config.Realtime is nil here exactly as it is in that path — there is no +// Client, so structurally no dial ever happens, let alone a rejected one. func TestRealtimeFlagOffPreservesExistingBehavior(t *testing.T) { var httpCap capture httpSrv := httptest.NewServer(httpCap.handler(t)) diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index 5619b95..7a5bbc4 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -9,6 +9,7 @@ package uplink import ( "bytes" + "context" "crypto/rand" "encoding/hex" "encoding/json" @@ -96,6 +97,13 @@ type Uplink struct { residualMu sync.Mutex residual []func() error + // legacyMu guards legacyMode: whether the server has rejected the + // realtime dial (rtclient.ErrRealtimeDisabled) and this Uplink has + // switched the whole session to routing reliable events over the + // legacy /v2 HTTP path instead (see pollRealtimeMode). + legacyMu sync.Mutex + legacyMode bool + ticksSinceSend int } @@ -140,11 +148,13 @@ func (u *Uplink) Offer(ev Event) { realtime := u.cfg.Realtime u.mu.Unlock() - // Feature flag: when a realtime uplink is configured, every kind with a - // realtime-protocol equivalent is routed there instead of the HTTP - // batch below, so the same event is never sent both ways. task.log (no - // realtime equivalent) and any realtime send failure fall through. - if realtime != nil && u.routeToRealtime(realtime, ev) { + // Feature flag: when a realtime uplink is configured AND the server + // hasn't rejected it (see the truth table on pollRealtimeMode), every + // kind with a realtime-protocol equivalent is routed there instead of + // the HTTP batch below, so the same event is never sent both ways. + // task.log (no realtime equivalent) and any realtime send failure fall + // through. + if realtime != nil && !u.inLegacyMode() && u.routeToRealtime(realtime, ev) { return } @@ -182,11 +192,15 @@ func (u *Uplink) Offer(ev Event) { // kind has no realtime equivalent (task.log). // // Reliable events (task.event/message.event) NEVER fall back to the legacy -// /v2 HTTP path while the realtime flag is on, even when the send fails: -// /v2 ingest writes UserStore, while /v3 viewers resume from SpaceHub, so -// diverting so much as one terminal task.done/task.fail/message.send would -// let a viewer permanently miss it. See sendReliable for what happens to a -// send failure instead. +// /v2 HTTP path while the realtime flag is on AND the server accepts +// realtime, even when a single send attempt fails: /v2 ingest writes +// UserStore, while /v3 viewers resume from SpaceHub, so diverting so much as +// one terminal task.done/task.fail/message.send would let a viewer +// permanently miss it. See sendReliable for what happens to a send failure +// instead. (The one deliberate exception to "never falls back" is the +// server-disabled whole-session legacy mode this Uplink switches into via +// pollRealtimeMode — see that function's truth table — which routeToRealtime +// is not even called under, per Offer's inLegacyMode guard.) func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { switch ev.Kind { case protocol.TaskStart, protocol.TaskProgress, protocol.TaskStep, protocol.TaskDone, protocol.TaskFail: @@ -304,6 +318,245 @@ func (u *Uplink) wakeFlush() { } } +// inLegacyMode reports whether this Uplink is currently routing reliable +// events over the legacy /v2 HTTP path because the server rejected the +// realtime dial (see pollRealtimeMode). +func (u *Uplink) inLegacyMode() bool { + u.legacyMu.Lock() + defer u.legacyMu.Unlock() + return u.legacyMode +} + +// pollRealtimeMode is called once per flush tick from loop. It is the +// daemon-side half of the server-capability truth table: +// +// local SITREP_REALTIME | server /v3/realtime | routing +// -----------------------|----------------------------|-------------------- +// off | (never dialed) | legacy /v2, always +// on | accepts (normal) | realtime +// on | 403 realtime_disabled | legacy /v2, whole +// | | session, until the +// | | client's own long- +// | | interval re-probe +// | | (see rtclient's +// | | serverDisabledRecheckInterval) +// | | succeeds again +// +// The local flag is enforced upstream: newAgentRealtimeUplink (cmd/sitrep) +// never even constructs an rtclient.Client, let alone dials, when +// SITREP_REALTIME is off, so cfg.Realtime is nil and this function is a +// no-op (Offer's nil check on cfg.Realtime already covers that row). +// +// The remaining two rows are what this function drives: rtclient.Client +// dials in the background on its own reconnect loop and exposes +// ServerDisabled() as the result of its last dial attempt. When that flips +// true, this Uplink switches its whole session to legacy routing (not just +// the one event that happened to be in flight) and drains every event +// already sitting durably in the outbox/overflow tables to /v2, in FIFO +// order, so a terminal task state that arrived before the flag flipped +// doesn't sit forever in a store no viewer reads. When it flips back to +// false (the client's periodic re-probe succeeded), new events resume +// routing to realtime immediately — there is no backlog to replay for +// those, since legacy mode drained everything as it went. +func (u *Uplink) pollRealtimeMode() { + rt := u.cfg.Realtime + if rt == nil { + return + } + disabled := rt.ServerDisabled() + + u.legacyMu.Lock() + wasLegacy := u.legacyMode + u.legacyMode = disabled + u.legacyMu.Unlock() + + if !disabled { + if wasLegacy { + u.cfg.Logf("uplink: realtime accepted again; resuming realtime routing for new events") + } + return + } + if !wasLegacy { + u.cfg.Logf("uplink: server rejected realtime (403 realtime_disabled); switching this session to legacy /v2 routing until it re-enables") + } + u.drainOutboxToLegacy(rt) +} + +// drainOutboxToLegacy walks every reliable event durably sitting in rt's +// outbox (real device_seq, ready to deliver) and then its overflow table (no +// device_seq yet), oldest first in each, translating each one back into the +// legacy uplink.Event shape and posting it to /v2/ingest. An item is only +// retired (Ack for outbox items, DeleteOverflow for overflow items) once its +// /v2 send actually succeeds; the first failure stops the whole drain for +// this tick; the next flush tick resumes exactly where it left off, because +// nothing was retired. +// +// Acking an outbox item may itself promote an overflow row into the outbox +// (outbox.Store.Ack's own trigger) — the outer loop re-reads Pending() every +// iteration rather than caching the initial list, so a promoted row is +// picked up and drained in the same pass instead of waiting for the +// overflow-scanning loop below. +func (u *Uplink) drainOutboxToLegacy(rt *rtclient.Client) { + store := rt.Outbox() + space := rt.Space() + if store == nil { + return + } + ctx := context.Background() + + for { + pending, err := store.Pending(ctx, space) + if err != nil { + u.cfg.Logf("uplink: legacy drain: read outbox: %v", err) + return + } + if len(pending) == 0 { + break + } + item := pending[0] + ev, ok := legacyEventFromOutboxItem(item.Kind, item.Body) + if !ok { + u.cfg.Logf("uplink: legacy drain: dropping outbox item with unrecognized kind %q", item.Kind) + if err := store.Ack(ctx, space, item.DeviceSeq); err != nil { + u.cfg.Logf("uplink: legacy drain: ack seq %d: %v", item.DeviceSeq, err) + return + } + continue + } + if !u.sendLegacySync(ev) { + return // /v2 unreachable right now; resume on the next tick + } + if err := store.Ack(ctx, space, item.DeviceSeq); err != nil { + u.cfg.Logf("uplink: legacy drain: ack seq %d: %v", item.DeviceSeq, err) + return + } + } + + for { + overflow, err := store.OverflowPending(ctx, space) + if err != nil { + u.cfg.Logf("uplink: legacy drain: read overflow: %v", err) + return + } + if len(overflow) == 0 { + return + } + ov := overflow[0] + ev, ok := legacyEventFromOutboxItem(ov.Kind, ov.Body) + if !ok { + u.cfg.Logf("uplink: legacy drain: dropping overflow item with unrecognized kind %q", ov.Kind) + if err := store.DeleteOverflow(ctx, ov.ID); err != nil { + u.cfg.Logf("uplink: legacy drain: delete overflow id %d: %v", ov.ID, err) + return + } + continue + } + if !u.sendLegacySync(ev) { + return + } + if err := store.DeleteOverflow(ctx, ov.ID); err != nil { + u.cfg.Logf("uplink: legacy drain: delete overflow id %d: %v", ov.ID, err) + return + } + } +} + +// sendLegacySync posts a single event to /v2/ingest synchronously and +// reports whether the server accepted it (2xx). Unlike send (the normal +// best-effort batch flusher, which retries a few times and then drops on +// failure since telemetry is lossy by design), the legacy drain must know +// definitively whether to retire the durable item it just sent, so it needs +// a real success/failure signal rather than fire-and-forget. +func (u *Uplink) sendLegacySync(ev Event) bool { + body, err := json.Marshal([]Event{ev}) + if err != nil { + return false + } + req, err := http.NewRequest(http.MethodPost, u.cfg.ServerURL+"/v2/ingest", bytes.NewReader(body)) + if err != nil { + return false + } + req.Header.Set("Content-Type", "application/json") + if u.cfg.Token != "" { + req.Header.Set("Authorization", "Bearer "+u.cfg.Token) + } + resp, err := u.client.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode >= 200 && resp.StatusCode < 300 +} + +// legacyEventFromOutboxItem reverse-translates a durably-stored reliable +// event body (wire.TaskEventBody or wire.MessageEventBody JSON, as written +// by routeToRealtime's forward direction) back into the legacy uplink.Event +// shape /v2/ingest expects. ok is false for a kind this function does not +// recognize (defensive; the outbox/overflow tables only ever hold the two +// kinds routeToRealtime writes). +func legacyEventFromOutboxItem(kind string, body json.RawMessage) (Event, bool) { + switch kind { + case wire.TypeTaskEvent: + var b wire.TaskEventBody + if err := json.Unmarshal(body, &b); err != nil { + return Event{}, false + } + k := legacyTaskKind(b.Kind) + if k == "" { + return Event{}, false + } + ev := Event{ + Event: protocol.Event{ + Kind: k, + Title: b.Title, + Step: b.Step, + Text: b.Message, + }, + SourceID: b.TaskID, + TS: time.UnixMilli(b.OccurredAt).UTC().Format(time.RFC3339), + } + if b.Percent != nil { + ev.Percent = *b.Percent + } + if b.Display != nil { + ev.Icon, ev.Tint, ev.Template = b.Display.Icon, b.Display.Tint, b.Display.Template + } + return ev, true + case wire.TypeMessageEvent: + var b wire.MessageEventBody + if err := json.Unmarshal(body, &b); err != nil { + return Event{}, false + } + return Event{ + Event: protocol.Event{Kind: protocol.MessageSend, Text: b.Text, Level: b.Level}, + SourceID: b.AutomationID, + TS: time.UnixMilli(b.OccurredAt).UTC().Format(time.RFC3339), + }, true + default: + return Event{}, false + } +} + +// legacyTaskKind reverses taskEventKind (wire task.event "kind" string back +// to the protocol.Kind the legacy HTTP path expects). Empty return means +// unrecognized. +func legacyTaskKind(wireKind string) protocol.Kind { + switch wireKind { + case "started": + return protocol.TaskStart + case "progress": + return protocol.TaskProgress + case "step": + return protocol.TaskStep + case "done": + return protocol.TaskDone + case "failed": + return protocol.TaskFail + default: + return "" + } +} + func taskEventKind(k protocol.Kind) string { switch k { case protocol.TaskStart: @@ -388,6 +641,7 @@ func (u *Uplink) loop() { case <-u.kick: } u.drainResidual() + u.pollRealtimeMode() batch, closed := u.drain() if len(batch) > 0 { u.send(batch) From 54f13f14c704e485a3763bc51d70fcfabc02fecc Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:53:23 +0800 Subject: [PATCH 56/60] fix(daemon): serialize legacy-mode outbox drain on the shared realtime Client cmd/sitrep/agent.go builds one *rtclient.Client for the whole process but constructs a fresh uplink.Uplink (and its own pollRealtimeMode loop) per concurrently-running automation. Each Uplink's drainOutboxToLegacy read the shared outbox's Pending()/OverflowPending() and sent the row before either had a chance to Ack/DeleteOverflow it, so two automations racing while the server rejected realtime could deliver the same reliable event to /v2/ingest twice. Move drain ownership onto rtclient.Client (already the single shared object owning the outbox and the single-writer send loop): DrainToLegacy takes a per-row LegacySink and serializes every row's read-sink-retire sequence under a new legacyDrainMu, held one row at a time so a reconnecting client is never blocked behind a whole backlog. The same lock guards connectAndServe's post-hello replayPending, closing the narrower switch-back race between a lagging legacy drain and the newly reconnected client's own replay of the same outbox. Uplink.drainOutboxToLegacy now just delegates translation/send to rt.DrainToLegacy via a LegacySink closure; retiring rows is entirely the Client's responsibility. --- daemon/internal/realtime/client/client.go | 147 +++++++++++++++++++++- daemon/internal/uplink/uplink.go | 101 ++++----------- 2 files changed, 167 insertions(+), 81 deletions(-) diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index 2d33d8b..e6e9c30 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -98,6 +98,21 @@ type Client struct { // see internal/uplink.Uplink.pollRealtimeMode's truth table. serverDisabled atomic.Bool + // legacyDrainMu serializes DrainToLegacy against itself and against + // connectAndServe's post-hello replay, one outbox row at a time. This + // Client is shared process-wide (cmd/sitrep/agent.go builds exactly one + // per resident agent, and hands it to a fresh uplink.Uplink for every + // concurrently-running automation, each with its own drain-driving + // poll loop), so without a lock owned HERE — not per-Uplink, which + // would do nothing to stop two different Uplinks from racing — two + // callers could both read the same unacked/un-promoted row via + // Pending()/OverflowPending() before either retired it, and both would + // deliver it: a duplicate reliable event on the wire. See + // DrainToLegacy's doc comment for the full invariant, including how + // this same lock also rules out the drain racing this Client's own + // realtime replay across a switch back from legacy mode. + legacyDrainMu sync.Mutex + closing chan struct{} closed chan struct{} once sync.Once @@ -149,6 +164,119 @@ func (c *Client) Outbox() *outbox.Store { return c.cfg.Outbox } // device_seq counter are bound to (SPEC.md section 5.1) — see Outbox. func (c *Client) Space() string { return c.cfg.Space } +// LegacySink is called once per durable outbox/overflow row by +// DrainToLegacy, oldest first, and must translate + attempt delivery of +// that row over whatever non-realtime path the caller uses (e.g. legacy +// /v2 HTTP). It reports whether the row should be retired: +// +// - true: either delivery succeeded, or the row is permanently +// undeliverable (e.g. an unrecognized kind) and must never be retried — +// either way DrainToLegacy acks/deletes it and moves on to the next row. +// - false: delivery failed for a reason that might succeed later (e.g. +// the legacy endpoint is unreachable right now); DrainToLegacy stops +// immediately, retiring nothing further, so the next call resumes at +// exactly this row. +type LegacySink func(kind string, body json.RawMessage) bool + +// DrainToLegacy walks this Client's shared outbox — every durable reliable +// event with a real device_seq (Pending), then everything still waiting in +// overflow (OverflowPending) — oldest first in each, handing each row to +// sink and retiring it (Ack / DeleteOverflow) only once sink reports true. +// +// This is the single owning entry point for legacy-mode draining: this +// Client is constructed once per resident agent process +// (cmd/sitrep/agent.go) and shared by a fresh uplink.Uplink for every +// concurrently-running automation (see runAutomation), each running its own +// pollRealtimeMode poll loop against the same outbox. Without process-wide +// serialization here, two Uplinks could both read the same unretired row +// before either retired it and both deliver it — a duplicate reliable +// event reaching the server. DrainToLegacy rules that out: every row's +// read-sink-retire sequence runs under legacyDrainMu (see +// drainOneToLegacy), held for exactly that one row, so a second concurrent +// caller blocks until the first has fully retired its row and always sees a +// fresh Pending()/OverflowPending() read with that row already gone. +// +// The same lock also closes the narrower switch-back race the reviewer +// flagged: connectAndServe acquires legacyDrainMu around clearing +// serverDisabled and its initial post-hello replayPending call (see +// connectAndServe), so a drain in flight when the server re-accepts +// realtime either fully retires its current row before that replay reads +// the outbox (the row is already gone by then — no duplicate), or blocks +// until the replay (which reads every row still sitting in the outbox at +// that instant) has already committed to delivering them over realtime — +// drainOneToLegacy then observes ServerDisabled() flip false on its next +// iteration and stops without ever touching those rows itself. Either way a +// given row is delivered exactly once, by exactly one of the two paths. +// +// The lock is held per-row rather than across the whole call so a +// reconnecting Client is never blocked behind an entire backlog — only +// behind whichever single row happens to be mid-flight. +func (c *Client) DrainToLegacy(ctx context.Context, sink LegacySink) { + if c.cfg.Outbox == nil { + return + } + for { + if done := c.drainOneToLegacy(ctx, sink); done { + return + } + } +} + +// drainOneToLegacy drains at most one row (see DrainToLegacy) under +// legacyDrainMu. It reports true when the caller should stop: nothing left +// to drain, sink asked to stop, a store error, or the server has +// re-accepted realtime since the last iteration. +func (c *Client) drainOneToLegacy(ctx context.Context, sink LegacySink) bool { + c.legacyDrainMu.Lock() + defer c.legacyDrainMu.Unlock() + + if !c.ServerDisabled() { + // Realtime has resumed (or was never disabled to begin with, for a + // stray call): every row still sitting in the outbox at this + // instant is guaranteed to be delivered by connectAndServe's + // post-hello replayPending instead, which holds this same lock + // while doing so — see DrainToLegacy's doc comment. Stopping here + // cannot drop or duplicate anything. + return true + } + + store := c.cfg.Outbox + pending, err := store.Pending(ctx, c.cfg.Space) + if err != nil { + c.cfg.Logf("realtime: legacy drain: read outbox: %v", err) + return true + } + if len(pending) > 0 { + item := pending[0] + if !sink(item.Kind, item.Body) { + return true + } + if err := store.Ack(ctx, c.cfg.Space, item.DeviceSeq); err != nil { + c.cfg.Logf("realtime: legacy drain: ack seq %d: %v", item.DeviceSeq, err) + return true + } + return false + } + + overflow, err := store.OverflowPending(ctx, c.cfg.Space) + if err != nil { + c.cfg.Logf("realtime: legacy drain: read overflow: %v", err) + return true + } + if len(overflow) == 0 { + return true + } + ov := overflow[0] + if !sink(ov.Kind, ov.Body) { + return true + } + if err := store.DeleteOverflow(ctx, ov.ID); err != nil { + c.cfg.Logf("realtime: legacy drain: delete overflow id %d: %v", ov.ID, err) + return true + } + return false +} + // MetricsThrottled reports whether the metric batcher is currently in its // throttled cadence (SPEC.md section 7's command{throttle}/{resume_rate}). func (c *Client) MetricsThrottled() bool { return c.metrics.Throttled() } @@ -381,12 +509,6 @@ func (c *Client) connectAndServe() error { } heartbeatInterval := time.Duration(accept.HeartbeatIntervalMS) * time.Millisecond - // A completed hello is proof the server currently accepts realtime, - // whether or not this connection had ever previously been rejected with - // ErrRealtimeDisabled — clear it so pollRealtimeMode's caller resumes - // realtime routing for new events immediately. - c.serverDisabled.Store(false) - c.mu.Lock() c.conn = conn c.connected = true @@ -399,7 +521,20 @@ func (c *Client) connectAndServe() error { c.mu.Unlock() }() + // A completed hello is proof the server currently accepts realtime, + // whether or not this connection had ever previously been rejected with + // ErrRealtimeDisabled — clear it so pollRealtimeMode's caller resumes + // realtime routing for new events immediately, and replay everything + // still sitting in the outbox over this connection. + // + // Both happen under legacyDrainMu — the same lock DrainToLegacy holds + // per row — so a legacy drain in flight when the server re-accepts + // realtime cannot race this replay onto the same row: see + // DrainToLegacy's doc comment for the full invariant. + c.legacyDrainMu.Lock() + c.serverDisabled.Store(false) c.replayPending(ctx, conn) + c.legacyDrainMu.Unlock() frames := make(chan frameMsg, 8) pings := make(chan struct{}, 8) diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index 7a5bbc4..4671112 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -382,83 +382,34 @@ func (u *Uplink) pollRealtimeMode() { u.drainOutboxToLegacy(rt) } -// drainOutboxToLegacy walks every reliable event durably sitting in rt's -// outbox (real device_seq, ready to deliver) and then its overflow table (no -// device_seq yet), oldest first in each, translating each one back into the -// legacy uplink.Event shape and posting it to /v2/ingest. An item is only -// retired (Ack for outbox items, DeleteOverflow for overflow items) once its -// /v2 send actually succeeds; the first failure stops the whole drain for -// this tick; the next flush tick resumes exactly where it left off, because -// nothing was retired. -// -// Acking an outbox item may itself promote an overflow row into the outbox -// (outbox.Store.Ack's own trigger) — the outer loop re-reads Pending() every -// iteration rather than caching the initial list, so a promoted row is -// picked up and drained in the same pass instead of waiting for the -// overflow-scanning loop below. +// drainOutboxToLegacy hands rt.DrainToLegacy the translate-and-send step for +// every reliable event durably sitting in rt's shared outbox (real +// device_seq, ready to deliver) and then its overflow table (no device_seq +// yet), oldest first in each. rt owns retiring each row (Ack/DeleteOverflow) +// and — critically — owns serializing that read-send-retire sequence +// process-wide: rt is the one *rtclient.Client shared by every automation's +// Uplink (see cmd/sitrep/agent.go), so without rt's own per-row locking two +// Uplinks polling concurrently could both read and send the same unretired +// row. See rtclient.Client.DrainToLegacy's doc comment for the full +// invariant, including why this is also safe across a switch back to +// realtime mid-drain. func (u *Uplink) drainOutboxToLegacy(rt *rtclient.Client) { - store := rt.Outbox() - space := rt.Space() - if store == nil { - return - } - ctx := context.Background() - - for { - pending, err := store.Pending(ctx, space) - if err != nil { - u.cfg.Logf("uplink: legacy drain: read outbox: %v", err) - return - } - if len(pending) == 0 { - break - } - item := pending[0] - ev, ok := legacyEventFromOutboxItem(item.Kind, item.Body) - if !ok { - u.cfg.Logf("uplink: legacy drain: dropping outbox item with unrecognized kind %q", item.Kind) - if err := store.Ack(ctx, space, item.DeviceSeq); err != nil { - u.cfg.Logf("uplink: legacy drain: ack seq %d: %v", item.DeviceSeq, err) - return - } - continue - } - if !u.sendLegacySync(ev) { - return // /v2 unreachable right now; resume on the next tick - } - if err := store.Ack(ctx, space, item.DeviceSeq); err != nil { - u.cfg.Logf("uplink: legacy drain: ack seq %d: %v", item.DeviceSeq, err) - return - } - } + rt.DrainToLegacy(context.Background(), u.legacyDrainSink) +} - for { - overflow, err := store.OverflowPending(ctx, space) - if err != nil { - u.cfg.Logf("uplink: legacy drain: read overflow: %v", err) - return - } - if len(overflow) == 0 { - return - } - ov := overflow[0] - ev, ok := legacyEventFromOutboxItem(ov.Kind, ov.Body) - if !ok { - u.cfg.Logf("uplink: legacy drain: dropping overflow item with unrecognized kind %q", ov.Kind) - if err := store.DeleteOverflow(ctx, ov.ID); err != nil { - u.cfg.Logf("uplink: legacy drain: delete overflow id %d: %v", ov.ID, err) - return - } - continue - } - if !u.sendLegacySync(ev) { - return - } - if err := store.DeleteOverflow(ctx, ov.ID); err != nil { - u.cfg.Logf("uplink: legacy drain: delete overflow id %d: %v", ov.ID, err) - return - } - } +// legacyDrainSink is the rtclient.LegacySink for drainOutboxToLegacy: +// translate the durable row back into the legacy uplink.Event shape and post +// it to /v2/ingest synchronously. An unrecognized kind can never be +// delivered on any retry, so it is logged and reported as retirable (true) +// without ever calling sendLegacySync — matching the previous drain's +// behavior of discarding it rather than blocking the backlog behind it. +func (u *Uplink) legacyDrainSink(kind string, body json.RawMessage) bool { + ev, ok := legacyEventFromOutboxItem(kind, body) + if !ok { + u.cfg.Logf("uplink: legacy drain: dropping outbox item with unrecognized kind %q", kind) + return true + } + return u.sendLegacySync(ev) } // sendLegacySync posts a single event to /v2/ingest synchronously and From f8d635bc3109f67b3efd51d5bcfb6d4a9c320ae7 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:54:50 +0800 Subject: [PATCH 57/60] test(daemon): regress duplicate-delivery bug in shared-Client legacy drain Add TestDrainToLegacyConcurrentCallersDeliverEachRowExactlyOnce: many goroutines call Client.DrainToLegacy concurrently against one Client/outbox pre-loaded with a deterministic backlog, simulating several automations' Uplinks polling the same shared realtime Client at once (the normal case commit 1114ff9's review flagged as racy). Asserts every row reaches the sink exactly once, in device_seq order, across all callers combined. Meaningful under -race; passes at -count=20. --- .../realtime/client/legacy_drain_test.go | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 daemon/internal/realtime/client/legacy_drain_test.go diff --git a/daemon/internal/realtime/client/legacy_drain_test.go b/daemon/internal/realtime/client/legacy_drain_test.go new file mode 100644 index 0000000..a7bda18 --- /dev/null +++ b/daemon/internal/realtime/client/legacy_drain_test.go @@ -0,0 +1,128 @@ +package client + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestDrainToLegacyConcurrentCallersDeliverEachRowExactlyOnce is the +// regression test for the MAJOR concurrency defect introduced by commit +// 1114ff9: cmd/sitrep/agent.go builds exactly one *rtclient.Client for the +// whole resident agent process, but spawns a fresh uplink.Uplink — each +// running its own pollRealtimeMode poll loop — per concurrently-running +// automation. Before DrainToLegacy moved drain ownership (and its locking) +// onto this shared Client, every one of those Uplinks drained the SAME +// outbox directly: two callers could both read the same unretired row via +// Pending()/OverflowPending() before either Ack'd/DeleteOverflow'd it, and +// both would "deliver" it — a duplicate reliable event reaching the server +// in production. +// +// This test drives that exact shape at the layer the fix lives in: many +// goroutines call DrainToLegacy concurrently against one Client/outbox +// pre-loaded with a deterministic backlog (simulating several Uplinks' +// pollRealtimeMode loops firing at once, which is the normal case — the +// scheduler in cmd/sitrep/agent.go polls every 2s and multiple automations +// can easily be mid-run together). It asserts every row is observed by the +// sink EXACTLY ONCE, in device_seq order, across all callers combined — the +// order assertion holds only because legacyDrainMu is held across a whole +// row's read+sink+retire, so at any instant the row with the lowest +// remaining device_seq is always the next (and only) one any caller can be +// looking at, regardless of how the goroutines are scheduled. Must be +// meaningful under -race (run with -race -count=20). +func TestDrainToLegacyConcurrentCallersDeliverEachRowExactlyOnce(t *testing.T) { + const rows = 40 + const drainers = 8 + + // A permanently-403 endpoint: DrainToLegacy only acts while + // ServerDisabled() is true (see drainOneToLegacy), matching the + // production precondition (the server has rejected /v3/realtime). + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + })) + defer srv.Close() + + store := newTestOutbox(t) + ctx := context.Background() + + // Deterministic backlog, oldest-first by construction (Enqueue is + // called sequentially, not concurrently, so device_seq order is fixed + // and known ahead of time). + for i := 0; i < rows; i++ { + if _, err := store.Enqueue(ctx, testSpace, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + return json.Marshal(wire.MessageEventBody{ + DeviceID: testDeviceID, DeviceSeq: seq, + MessageID: "m", Level: "info", Text: "unused", OccurredAt: time.Now().UnixMilli(), + AutomationID: strconv.Itoa(i), + }) + }); err != nil { + t.Fatalf("pre-populate Enqueue %d: %v", i, err) + } + } + + url := "ws" + srv.URL[len("http"):] + c := newTestClient(t, url, store, nil) + waitFor(t, 2*time.Second, c.ServerDisabled) + + var mu sync.Mutex + var delivered []int64 // device_seq, in the order the sink observed them + seen := map[int64]int{} // device_seq -> times observed + + sink := func(kind string, body json.RawMessage) bool { + var b wire.MessageEventBody + if err := json.Unmarshal(body, &b); err != nil { + t.Errorf("sink: decode body: %v", err) + return true + } + mu.Lock() + delivered = append(delivered, b.DeviceSeq) + seen[b.DeviceSeq]++ + mu.Unlock() + return true + } + + var wg sync.WaitGroup + wg.Add(drainers) + for i := 0; i < drainers; i++ { + go func() { + defer wg.Done() + c.DrainToLegacy(ctx, sink) + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + + if len(delivered) != rows { + t.Fatalf("sink observed %d rows, want %d (delivered=%v)", len(delivered), rows, delivered) + } + for seq, n := range seen { + if n != 1 { + t.Fatalf("device_seq %d delivered %d times, want exactly 1 (this is the duplicate-delivery bug the fix closes)", seq, n) + } + } + for i, seq := range delivered { + want := int64(i + 1) // device_seq starts at 1 (SPEC.md section 5.1) + if seq != want { + t.Fatalf("delivery order[%d] = device_seq %d, want %d (out of order): full order %v", i, seq, want, delivered) + } + } + + pending, err := store.Pending(ctx, testSpace) + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 0 { + t.Fatalf("expected outbox fully drained, %d rows remain", len(pending)) + } +} From 90c95626dc72f0490d6801ef357b20aa2459ee0b Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:59:18 +0800 Subject: [PATCH 58/60] fix(daemon): stop rtclient.Client from residual-retrying overflowed events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SendTaskEvent/SendMessageEvent returned outbox.ErrOverflowed to the caller like any other Enqueue failure, even though ErrOverflowed's own doc comment calls it informational (the event is already durably persisted into outbox_overflow) and says callers must not retry or reroute on it. uplink.Uplink.sendReliable's doc comment already assumed this never reached it ("ordinary outbox-full backpressure durably overflows inside Enqueue and never reaches here") — but nothing enforced it, so an overflowed event fell into sendReliable's persist-failure/ residual-retry branch and got re-Enqueued on every flush tick until capacity freed, each attempt durably creating another outbox_overflow row for the same logical event (Enqueue never deduplicates): unbounded duplicate rows in overflow's uncapped table, only one of which is ever promoted and delivered. Swallow ErrOverflowed in both Send*Event methods so it never reaches the caller as an error, matching the invariant both doc comments already assumed. --- daemon/internal/realtime/client/client.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go index e6e9c30..a7da427 100644 --- a/daemon/internal/realtime/client/client.go +++ b/daemon/internal/realtime/client/client.go @@ -311,6 +311,19 @@ type TaskEvent struct { // its device_seq) and attempts immediate delivery if connected. The event // survives in the outbox until acknowledged, across any number of // reconnects or process restarts. +// +// outbox.ErrOverflowed is deliberately not surfaced as an error: per its own +// doc comment it is informational, not a failure — the event is already +// durably persisted (into outbox_overflow instead of outbox) by the time +// Enqueue returns it, and "[c]allers must not retry or reroute on this +// error". uplink.Uplink.sendReliable's own doc comment already assumes this +// ("ordinary outbox-full backpressure durably overflows inside Enqueue and +// never reaches here"): if this returned ErrOverflowed as a plain error, +// sendReliable's persist-failure branch would re-attempt the exact same +// Enqueue call every flush tick until capacity frees, each attempt durably +// creating ANOTHER outbox_overflow row for the same logical event (Enqueue +// does not deduplicate) — unbounded duplicate rows in an uncapped table, +// only one of which is ever promoted and delivered. func (c *Client) SendTaskEvent(ev TaskEvent) error { _, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeTaskEvent, func(seq int64) (json.RawMessage, error) { body := wire.TaskEventBody{ @@ -330,7 +343,7 @@ func (c *Client) SendTaskEvent(ev TaskEvent) error { } return json.Marshal(body) }) - if err != nil { + if err != nil && !errors.Is(err, outbox.ErrOverflowed) { return err } c.wakeSender() @@ -347,7 +360,8 @@ type MessageEvent struct { AutomationID string } -// SendMessageEvent durably enqueues a message event; see SendTaskEvent. +// SendMessageEvent durably enqueues a message event; see SendTaskEvent, +// including why outbox.ErrOverflowed is not surfaced as an error here. func (c *Client) SendMessageEvent(ev MessageEvent) error { _, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { id := ev.MessageID @@ -368,7 +382,7 @@ func (c *Client) SendMessageEvent(ev MessageEvent) error { } return json.Marshal(body) }) - if err != nil { + if err != nil && !errors.Is(err, outbox.ErrOverflowed) { return err } c.wakeSender() From f3c9523334932ca76d08c6823a08e575757c0b22 Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 14:59:25 +0800 Subject: [PATCH 59/60] test(daemon): cover overflow promotion across a genuine process restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TestOverflowSurvivesProcessRestartAndDeliversInOrder: fill a cap-1 outbox, overflow a second reliable event, shut the Uplink/Client/Store down exactly as a process exit would, open a brand new Store handle on the same file (never reusing the old one), ack the still-unacked older event once reconnected, and assert the overflowed event is promoted in its original order and delivered over realtime — with the /v2 HTTP spy seeing nothing at all throughout. --- daemon/internal/uplink/restart_test.go | 219 +++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 daemon/internal/uplink/restart_test.go diff --git a/daemon/internal/uplink/restart_test.go b/daemon/internal/uplink/restart_test.go new file mode 100644 index 0000000..dd6feb2 --- /dev/null +++ b/daemon/internal/uplink/restart_test.go @@ -0,0 +1,219 @@ +package uplink + +import ( + "context" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/rttest" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestOverflowSurvivesProcessRestartAndDeliversInOrder is the reviewer's +// exact combined scenario: a full outbox forces a reliable event into +// overflow, the Uplink (and the realtime Client, and the outbox handle +// underneath it) shut down exactly as a process exit would, a brand new +// process opens a brand new Store handle on the same file (not the same +// open handle — genuinely closed and reopened, so this exercises the real +// on-disk recovery path rather than an in-memory one), capacity frees once +// the still-unacked older event is finally acked, the overflowed event is +// promoted into the outbox in its original offer order, and it is +// delivered over realtime to a mock server — with the /v2 HTTP spy seeing +// nothing at all, at any point in the test (reliable events must never +// fork off the realtime path, restart or not). +func TestOverflowSurvivesProcessRestartAndDeliversInOrder(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + dbPath := filepath.Join(t.TempDir(), "outbox.db") + const deviceID = "device-1" + const space = "space-1" + + msg := func(text string) Event { + return ev(protocol.MessageSend, func(e *Event) { e.Text = text; e.Level = "info" }) + } + + // ---- "process 1": fill the (cap-1) outbox, overflow the second event, + // then shut down without ever acking the first event — simulating a + // crash/restart while the server (or the connection to it) is down. + firstSeen := make(chan wire.MessageEventBody, 4) + rtSrv1 := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess-1", 1000); err != nil { + t.Errorf("phase 1 HelloAccept: %v", err) + return + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeMessageEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + continue + } + select { + case firstSeen <- body.(wire.MessageEventBody): + default: + } + // Deliberately never ack: the event must still be sitting + // unacked in the outbox when process 1 shuts down. + } + }) + + store1, err := outbox.OpenWithMaxRows(dbPath, 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + + rt1 := rtclient.New(rtclient.Config{ + URL: rtSrv1.URL(), DeviceID: deviceID, Space: space, Outbox: store1, + }) + + u1 := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt1}) + + // 1. Fills the cap-1 outbox (device_seq 1), delivered over WS, left + // unacked on purpose. + u1.Offer(msg("first")) + select { + case mb := <-firstSeen: + if mb.Text != "first" { + t.Fatalf("phase 1 saw %q, want %q", mb.Text, "first") + } + case <-time.After(3 * time.Second): + t.Fatal("realtime server never saw the first event") + } + + // 2. Offer() enqueues synchronously (Enqueue is called directly from + // the calling goroutine, not the flush loop — see routeToRealtime), so + // by the time Offer returns the outbox-full check has already run: this + // event is durably in outbox_overflow, not the outbox, no device_seq + // allocated yet. + u1.Offer(msg("second")) + if n, err := store1.OverflowCount(context.Background()); err != nil || n != 1 { + t.Fatalf("OverflowCount = %d, err = %v; want exactly 1 overflowed event after the outbox filled", n, err) + } + + // 3. Shut down exactly as a process exit would: Uplink, then the + // realtime Client, then the Store handle itself. + u1.Close() + rt1.Close() + rtSrv1.Close() + if err := store1.Close(); err != nil { + t.Fatalf("store1.Close: %v", err) + } + + // ---- "process 2": a brand new Store handle on the same file (never + // reusing store1/rt1/u1) — this is what makes it a genuine restart test + // rather than an in-memory one. + secondSeen := make(chan wire.MessageEventBody, 4) + rtSrv2 := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess-2", 1000); err != nil { + t.Errorf("phase 2 HelloAccept: %v", err) + return + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeMessageEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + continue + } + mb := body.(wire.MessageEventBody) + select { + case secondSeen <- mb: + default: + } + // Ack everything this time: this is what frees the cap-1 + // outbox's one slot and lets Store.Ack's own promotion trigger + // move the overflowed event in. + _ = conn.Ack(mb.DeviceID, mb.DeviceSeq) + } + }) + defer rtSrv2.Close() + + store2, err := outbox.OpenWithMaxRows(dbPath, 1) + if err != nil { + t.Fatalf("reopen OpenWithMaxRows: %v", err) + } + defer store2.Close() + + // Reopening at the same cap must NOT have promoted the overflowed event + // on its own: the outbox still holds the never-acked "first" row, so + // there is no free capacity yet — promotion only happens once that row + // is finally acked below. + if n, err := store2.Count(context.Background()); err != nil || n != 1 { + t.Fatalf("outbox count after reopen = %d, err = %v; want 1 (the still-unacked first event)", n, err) + } + if n, err := store2.OverflowCount(context.Background()); err != nil || n != 1 { + t.Fatalf("overflow count after reopen = %d, err = %v; want 1 (the second event, still not promoted)", n, err) + } + + rt2 := rtclient.New(rtclient.Config{ + URL: rtSrv2.URL(), DeviceID: deviceID, Space: space, Outbox: store2, + ResendInterval: 20 * time.Millisecond, + }) + defer rt2.Close() + + u2 := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt2}) + defer u2.Close() + + // 4/5. On reconnect, replayPending resends the still-unacked "first"; + // the mock server acks it, which frees the cap-1 outbox's one slot and + // promotes "second" out of overflow (Store.Ack's own trigger) with the + // next device_seq in this space's sequence (2) — never reusing seq 1, + // and never skipping ahead of it. Assert both are delivered, in that + // order, entirely over realtime. + var got []string + deadline := time.After(3 * time.Second) + for len(got) < 2 { + select { + case mb := <-secondSeen: + got = append(got, mb.Text) + case <-deadline: + t.Fatalf("timed out waiting for both events after restart; got so far: %v", got) + } + } + if got[0] != "first" || got[1] != "second" { + t.Fatalf("post-restart delivery order = %v, want [first second] (the promoted event must not jump ahead of the still-unacked older one)", got) + } + + waitForCond(t, 3*time.Second, func() bool { + n, err := store2.Count(context.Background()) + return err == nil && n == 0 + }) + if n, err := store2.OverflowCount(context.Background()); err != nil || n != 0 { + t.Fatalf("overflow count at end = %d, err = %v; want 0 (promoted and delivered)", n, err) + } + + // Throughout the whole test — fill, overflow, restart, promotion, + // delivery — the /v2 HTTP spy must never have carried either reliable + // event: it is not idempotent for a viewer resuming from SpaceHub the + // way /v3 is, so a fork here would be a silent, permanent miss. + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend { + t.Fatalf("HTTP /v2 ingest carried %q; reliable events must travel the realtime path only, restart or not", e.Text) + } + } +} From edad342e8da04e78713ece42ef396035463c4afc Mon Sep 17 00:00:00 2001 From: QuintinShaw Date: Sun, 19 Jul 2026 15:01:19 +0800 Subject: [PATCH 60/60] test(daemon): regress Enqueue's retry loop against real SQLite lock contention Add TestEnqueueRecoversFromRealTransientLockContention: a second, real sql.DB connection to the same outbox file takes SQLite's write lock (BEGIN IMMEDIATE) for a duration that lands inside Enqueue's second internal attempt window, so enqueueAttempt observes a genuine SQLITE_BUSY (not a faked send/build error) on its first attempt and recovers via its own bounded retry (enqueueMaxAttempts/ enqueueRetryDelay) once the lock actually releases. Asserts the event ends up durable and readable, never dropped. Add the openWithBusyTimeoutMS test seam (OpenWithMaxRows now delegates to it with the production 5000ms) so the test can use a short busy_timeout and stay fast instead of waiting out the real 5s default. --- daemon/internal/realtime/outbox/outbox.go | 14 +++- .../internal/realtime/outbox/outbox_test.go | 81 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/daemon/internal/realtime/outbox/outbox.go b/daemon/internal/realtime/outbox/outbox.go index cccf529..808fc55 100644 --- a/daemon/internal/realtime/outbox/outbox.go +++ b/daemon/internal/realtime/outbox/outbox.go @@ -108,6 +108,18 @@ func Open(path string) (*Store, error) { // OpenWithMaxRows is Open with an explicit row cap; maxRows <= 0 falls // back to DefaultMaxRows. func OpenWithMaxRows(path string, maxRows int) (*Store, error) { + return openWithBusyTimeoutMS(path, maxRows, 5000) +} + +// openWithBusyTimeoutMS is OpenWithMaxRows with the SQLite busy_timeout +// pragma exposed, in milliseconds. Production always goes through +// OpenWithMaxRows (5000ms, see the _txlock=immediate comment below); this +// exists as a same-package test seam so a test can drive a genuinely fast +// SQLITE_BUSY through enqueueAttempt's own bounded retry loop (see +// enqueueMaxAttempts) with a short busy_timeout, rather than needing to hold +// a competing write lock for the real 5s default just to prove that retry +// path recovers a transient error. +func openWithBusyTimeoutMS(path string, maxRows, busyTimeoutMS int) (*Store, error) { if maxRows <= 0 { maxRows = DefaultMaxRows } @@ -115,7 +127,7 @@ func OpenWithMaxRows(path string, maxRows int) (*Store, error) { // first write, so two overlapping transactions serialize instead of // racing to upgrade a deferred lock (SQLITE_BUSY under concurrent // Enqueue calls from multiple goroutines). - dsn := path + "?_pragma=busy_timeout(5000)&_txlock=immediate" + dsn := fmt.Sprintf("%s?_pragma=busy_timeout(%d)&_txlock=immediate", path, busyTimeoutMS) db, err := sql.Open("sqlite", dsn) if err != nil { return nil, fmt.Errorf("outbox: open %s: %w", path, err) diff --git a/daemon/internal/realtime/outbox/outbox_test.go b/daemon/internal/realtime/outbox/outbox_test.go index 5d48baa..23a5958 100644 --- a/daemon/internal/realtime/outbox/outbox_test.go +++ b/daemon/internal/realtime/outbox/outbox_test.go @@ -2,12 +2,14 @@ package outbox import ( "context" + "database/sql" "encoding/json" "errors" "fmt" "path/filepath" "sync" "testing" + "time" ) func open(t *testing.T) *Store { @@ -481,3 +483,82 @@ func TestOverflowSurvivesCloseAndReopen(t *testing.T) { t.Fatalf("OverflowCount after ack+promotion = %d, %v, want 0", n, err) } } + +// TestEnqueueRecoversFromRealTransientLockContention drives a genuine +// SQLITE_BUSY through enqueueAttempt's own bounded retry loop +// (enqueueMaxAttempts / enqueueRetryDelay) — not a fabricated send/build +// callback error, but an actual second connection holding the write lock a +// real Enqueue call has to wait out. It asserts the event still ends up +// durable and readable (never dropped) once the lock clears within the +// retry budget. +// +// A short busy_timeout (via the openWithBusyTimeoutMS test seam, rather +// than production's 5000ms) keeps this fast: with the competing lock held +// for holdDuration — comfortably inside the window of Enqueue's SECOND +// attempt (busyTimeoutMS + enqueueRetryDelay through 2*busyTimeoutMS + +// enqueueRetryDelay) and comfortably past its first (which must therefore +// see a real SQLITE_BUSY and retry) — the first attempt is guaranteed to +// fail on genuine lock contention, and the second is guaranteed to recover +// once the lock actually releases, with margin on both sides against +// scheduler jitter. +func TestEnqueueRecoversFromRealTransientLockContention(t *testing.T) { + const busyTimeoutMS = 100 + const holdDuration = 170 * time.Millisecond + + path := filepath.Join(t.TempDir(), "outbox.db") + + s, err := openWithBusyTimeoutMS(path, DefaultMaxRows, busyTimeoutMS) + if err != nil { + t.Fatalf("openWithBusyTimeoutMS: %v", err) + } + defer s.Close() + + // A second, independent connection to the same database file. Its own + // BEGIN (immediate, via the same _txlock=immediate DSN param + // OpenWithMaxRows itself relies on) genuinely acquires SQLite's write + // lock — this is real lock contention, not a simulated error. + dsn := fmt.Sprintf("%s?_pragma=busy_timeout(%d)&_txlock=immediate", path, busyTimeoutMS) + contender, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("open contender connection: %v", err) + } + defer contender.Close() + contender.SetMaxOpenConns(1) + + lockTx, err := contender.Begin() + if err != nil { + t.Fatalf("contender BEGIN IMMEDIATE: %v", err) + } + released := make(chan struct{}) + go func() { + time.Sleep(holdDuration) + if err := lockTx.Commit(); err != nil { + t.Errorf("contender commit: %v", err) + } + close(released) + }() + t.Cleanup(func() { <-released }) + + start := time.Now() + item, err := s.Enqueue(context.Background(), "space-a", "message.event", bodyFor(0)) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("Enqueue: %v (expected it to recover once the competing lock released after %v; Enqueue took %v)", err, holdDuration, elapsed) + } + if item.DeviceSeq != 1 { + t.Fatalf("Enqueue seq = %d, want 1", item.DeviceSeq) + } + if elapsed < holdDuration { + t.Fatalf("Enqueue returned after %v, before the competing lock even released at %v — contention was never actually exercised", elapsed, holdDuration) + } + + // Durable and readable, not silently dropped. + pending, err := s.Pending(context.Background(), "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 || pending[0].DeviceSeq != 1 { + t.Fatalf("Pending = %+v, want exactly the one durably-enqueued event", pending) + } +}