diff --git a/apple/SitrepApp/Sitrep/MainTabView.swift b/apple/SitrepApp/Sitrep/MainTabView.swift index 3cf4513c..2f689a40 100644 --- a/apple/SitrepApp/Sitrep/MainTabView.swift +++ b/apple/SitrepApp/Sitrep/MainTabView.swift @@ -15,9 +15,16 @@ struct MainTabView: View { var body: some View { configuredTabs .safeAreaInset(edge: .top, spacing: 0) { - // Honesty first: a monitoring tool must SAY when it can't see. - if let err = model.lastError { - ErrorBanner(text: err) + VStack(spacing: 0) { + // Honesty first: a monitoring tool must SAY when it can't see. + if let err = model.lastError { + ErrorBanner(text: err) + } else if let status = SyncStatusStrip.label(for: model.connectionPhase) { + SyncStatusStrip(color: status.color, text: status.text) + } + if model.supersededNotice { + SupersededBanner(isPresented: $model.supersededNotice) + } } } } @@ -66,12 +73,22 @@ struct MainTabView: View { .onReceive(NotificationCenter.default.publisher(for: .sitrepOpenEvents)) { _ in selection = 2 } - // Coming back to foreground: refresh immediately and revive the - // poll loop if anything killed it while suspended. + // Coming back to foreground: refresh immediately, (re)connect the + // realtime channel, and revive the HTTP fallback poll if anything + // killed it while suspended. While the WebSocket is live, refresh() + // only updates REST-only state (presence) — the reliable collections + // stay owned by deltas. Leaving the foreground closes the realtime + // connection — the interest lease simply lapses server-side rather + // than being explicitly released. .onChange(of: scenePhase) { _, phase in - if phase == .active { + switch phase { + case .active: model.ensurePolling() Task { await model.refresh() } + case .background: + model.leaveBackground() + default: + break } } } @@ -94,6 +111,70 @@ struct ErrorBanner: View { } } +/// A subtle, transient strip about THIS PHONE's realtime connection to the +/// server — distinct from `PresencePill`/`PresenceRow`, which report +/// whether the Mac/Agent is reporting in. Reuses the same dot-plus-caption +/// language rather than a new visual system; shows nothing once the +/// connection is `.live` (the common case), so it stays out of the way. +struct SyncStatusStrip: View { + let color: Color + let text: String + + /// nil means "healthy, show nothing" — only `.live` and `.idle` (not yet + /// started) count as nothing-to-say; every other phase is transient and + /// worth a one-line, low-emphasis mention. + static func label(for phase: RealtimeClient.Phase) -> (color: Color, text: String)? { + switch phase { + case .idle, .live: nil + case .connecting, .handshaking: (.gray, "正在连接实时同步…") + case .subscribed: (.gray, "正在同步…") + case .failed: (.orange, "实时同步中断，已切换到低频刷新") + } + } + + var body: some View { + HStack(spacing: 6) { + Circle().fill(color).frame(width: 6, height: 6) + Text(text).font(.caption2.weight(.medium)).foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 4) + .background(.background.secondary) + .transition(.move(edge: .top).combined(with: .opacity)) + .animation(.default, value: text) + } +} + +/// One-shot, non-disruptive notice for SPEC.md §9.4 `superseded`: this +/// device's realtime connection was replaced by another connection using +/// the same credential — most often the same phone's own connection racing +/// itself across a fast background/foreground cycle, but possibly a sign +/// the credential is in use elsewhere. Tap to dismiss; not repeated. +struct SupersededBanner: View { + @Binding var isPresented: Bool + + var body: some View { + Button { + withAnimation { isPresented = false } + } label: { + Label("这台设备的实时连接被另一个连接取代，可能是该凭据在别处也在使用", systemImage: "info.circle") + .font(.caption.weight(.medium)) + .lineLimit(2) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 12) + .padding(.vertical, 7) + } + .buttonStyle(.plain) + .background(.blue.opacity(0.12)) + .foregroundStyle(.blue) + .transition(.move(edge: .top).combined(with: .opacity)) + .task { + try? await Task.sleep(for: .seconds(6)) + withAnimation { isPresented = false } + } + } +} + /// "Is my Mac alive" — the first question a monitoring tool must answer. /// Green: agent heartbeat or ingest within a minute. Orange: minutes quiet. /// Red: nothing for 10+ minutes. diff --git a/apple/SitrepApp/Sitrep/SitrepApp.swift b/apple/SitrepApp/Sitrep/SitrepApp.swift index 1c5a0623..1f9f1025 100644 --- a/apple/SitrepApp/Sitrep/SitrepApp.swift +++ b/apple/SitrepApp/Sitrep/SitrepApp.swift @@ -81,7 +81,41 @@ final class AppModel { /// computed against this, not against server timestamps. var lastSyncAt: Date? - private var pollTask: Task? + /// The realtime WebSocket connection's lifecycle (SitrepKit's + /// `RealtimeClient`). Drives the subtle sync-status indicator; `.live` + /// means tasks/metrics/events/automations are updating from `delta`s in + /// real time rather than from the HTTP fallback poll below. + var connectionPhase: RealtimeClient.Phase = .idle + /// One-shot: this device's realtime connection was superseded by + /// another connection using the same credential (SPEC.md §9.4). Surfaced + /// once, non-disruptively, then cleared by the view that shows it. + var supersededNotice = false + + /// P0 gate: decides, from each REST refresh's `realtime_enabled`, + /// whether the RealtimeClient may connect. Pure decision logic lives in + /// `RealtimeCapabilityGate` (SitrepKit) so it's unit-testable without a + /// live socket; this property is the only thing that reads/drives it. + private var realtimeGate = RealtimeCapabilityGate() + var realtimeCapable: Bool { realtimeGate.isCapable } + private var realtimeClient: RealtimeClient? + private var realtimeObservers: [Task] = [] + /// Last SpaceState received from the realtime channel, kept for the + /// process lifetime (not persisted to disk): when the app backgrounds + /// and later rebuilds the client, this — with its revision `C` — seeds + /// the new connection so the foreground resume is an incremental + /// `delta` rather than a full snapshot every time. + private var lastSpaceState: SpaceState? + /// Low-frequency (>=30s) HTTP snapshot polling, used ONLY while the + /// realtime connection is down for multiple backoff cycles — see + /// `RealtimeClient.Notice.fellBackToPolling`/`.recovered`. This replaces + /// what used to be an always-on 3s poll loop: normal sync now comes from + /// `delta`s over the WebSocket. + private var fallbackPollTask: Task? + /// Whether the fallback poll is *supposed* to be running (independent of + /// whether `fallbackPollTask` is currently non-nil) — lets `ensurePolling()` + /// revive it defensively if something silently killed the task while + /// the realtime connection was still down. + private var fallbackActive = false var lastError: String? // Keychain-backed (survives reinstalls); UserDefaults is a legacy @@ -109,6 +143,19 @@ final class AppModel { WidgetCenter.shared.reloadAllTimelines() let client = self.client Task { await PushRegistrar.shared.configure(client: client) } + // Credentials changed under an existing connection (e.g. re-paired) + // — reconnect with the new ones rather than keep talking under the + // old identity. The preserved SpaceState belongs to the old space's + // revision sequence, so drop it: the next resume starts from 0. + if realtimeClient != nil { + stopRealtime() + lastSpaceState = nil + // New credentials may point at a different server entirely — + // don't carry over the old one's capability; wait for this + // space's own refresh to confirm it before reconnecting. + realtimeGate.reset() + enterForeground() + } } func start() async { @@ -121,7 +168,139 @@ final class AppModel { } await PushRegistrar.shared.configure(client: client) await PushRegistrar.shared.startObserving() - ensurePolling() + // Seed the UI with one HTTP snapshot immediately; the realtime + // connection (started below) then takes over as the live source. + await refresh() + enterForeground() + } + + // MARK: - Realtime sync (foreground WebSocket + HTTP fallback) + + /// Call when the app becomes foreground-active. Idempotent. + func enterForeground() { + guard !needsOnboarding else { return } + startRealtime() + } + + /// Call when the app leaves the foreground. The interest lease simply + /// lapses server-side (SPEC.md §7) rather than being released with an + /// explicit `unsubscribe` — closing the connection is enough. + func leaveBackground() { + stopRealtime() + stopFallbackPolling() + } + + private func startRealtime() { + // The capability gate (P0): only ever connect once a REST refresh has + // confirmed `realtime_enabled == true`. Everything else in this + // method is unchanged pre-existing plumbing. + guard realtimeCapable else { return } + guard let restClient = client, let url = restClient.realtimeURL, !deviceID.isEmpty else { return } + if let existing = realtimeClient { + // Idempotent on the actor side even if already running — this + // doubles as the revival path if something silently killed the + // connection's internal loop without going through `stop()`. + Task { await existing.start() } + return + } + let configuration = RealtimeClient.Configuration( + url: url, token: token.isEmpty ? nil : token, deviceID: deviceID) + // Seed with the last known SpaceState so the resume on the new + // connection is incremental (`last_revision: C`) instead of a full + // snapshot on every foreground cycle. + let rt = RealtimeClient(configuration: configuration, initialState: lastSpaceState ?? SpaceState()) + realtimeClient = rt + realtimeObservers = [ + Task { [weak self] in + for await phase in rt.phases { + self?.connectionPhase = phase + } + }, + Task { [weak self] in + for await state in rt.states { + self?.applyRealtimeState(state) + } + }, + Task { [weak self] in + for await notice in rt.notices { + self?.handleRealtimeNotice(notice) + } + }, + ] + Task { await rt.start() } + } + + /// Apply the capability bit from a fresh REST refresh via the pure gate, + /// then act on whatever it decides: false→true may open a connection + /// (this refresh IS the "a refresh says true again" the P0 fix requires + /// before reconnecting); true→false (server rolled the flag back) tears + /// down any live connection and falls back to plain HTTP, matching the + /// pre-realtime app exactly. + private func applyRealtimeCapability(_ enabled: Bool) { + switch realtimeGate.apply(refreshedCapability: enabled) { + case .none: break + case .connect: startRealtime() + case .disconnect: stopRealtime() + } + } + + private func stopRealtime() { + guard let rt = realtimeClient else { return } + realtimeClient = nil + for observer in realtimeObservers { observer.cancel() } + realtimeObservers = [] + connectionPhase = .idle + Task { await rt.stop() } + } + + private func applyRealtimeState(_ state: SpaceState) { + lastSpaceState = state + tasks = state.uiTasks + let sortedMetrics = state.uiMetrics + // Widgets refresh on a slow OS budget; while the app is foreground + // we piggyback any change onto them so the two surfaces never + // disagree (same rule `refresh()` applies for the HTTP path). + if sortedMetrics != metrics { + WidgetCenter.shared.reloadAllTimelines() + } + metrics = sortedMetrics + events = state.uiEvents + automations = state.uiAutomations + lastError = nil + lastSyncAt = .now + adoptOrphanedTasks() + } + + private func handleRealtimeNotice(_ notice: RealtimeClient.Notice) { + switch notice { + case .superseded: + supersededNotice = true + case .serverError(let code, let message): + lastError = "\(code.rawValue): \(message)" + case .revisionGap: + break // transparent to the user; the client re-resumes on its own + case .fellBackToPolling: + startFallbackPolling() + case .recovered: + stopFallbackPolling() + } + } + + private func startFallbackPolling() { + fallbackActive = true + guard fallbackPollTask == nil else { return } + fallbackPollTask = Task { [weak self] in + while !Task.isCancelled { + await self?.refresh() + try? await Task.sleep(for: .seconds(30)) + } + } + } + + private func stopFallbackPolling() { + fallbackActive = false + fallbackPollTask?.cancel() + fallbackPollTask = nil } /// v2 intentionally breaks the old space/auth model. Keychain values @@ -154,17 +333,16 @@ final class AppModel { } } - /// The poll loop is owned, observable, and restartable — whatever kills - /// it (suspension edge cases, cancellation), the next foreground pass - /// revives it. + /// Called on every `scenePhase == .active` transition (see + /// `MainTabView`). Both the realtime connection and the HTTP fallback + /// poll are owned, observable, and restartable — whatever kills them + /// (suspension edge cases, cancellation), the next foreground pass + /// revives them. This is the same self-healing shape the old always-on + /// 3s poll loop used; it now applies to `RealtimeClient.start()` + /// (idempotent even if already running) and to the fallback poll task. func ensurePolling() { - if let pollTask, !pollTask.isCancelled { return } - pollTask = Task { [weak self] in - while !Task.isCancelled { - await self?.refresh() - try? await Task.sleep(for: .seconds(3)) - } - } + enterForeground() + if fallbackActive { startFallbackPolling() } } /// Local fallback for remote push-to-start: if the server has running @@ -207,6 +385,12 @@ final class AppModel { /// Revoke this phone's device record, clear credentials, return to /// onboarding. Other devices in the space are untouched. func disconnect() async { + stopRealtime() + stopFallbackPolling() + lastSpaceState = nil + // A future re-pair is a cold start for capability purposes too — no + // connecting on the new space until its own refresh confirms it. + realtimeGate.reset() if !deviceID.isEmpty { try? await client?.revokeDevice(id: deviceID) } @@ -235,11 +419,36 @@ final class AppModel { } } + /// The realtime client's own phase, queried on the actor — the + /// authority for HTTP-overwrite decisions. The MainActor-cached + /// `connectionPhase` arrives via a separate AsyncStream from the state + /// updates, so it can momentarily lag behind an already-applied live + /// state; deciding on the cache would open a window where a refresh + /// reads a stale non-live phase and overwrites delta-derived state. + private func authoritativePhase() async -> RealtimeClient.Phase { + guard let realtimeClient else { return .idle } + return await realtimeClient.currentPhase + } + func refresh() async { guard let client else { return } do { let snapshot = try await client.snapshot() + // presence is REST-only (the realtime protocol does not carry + // it) — updating it is the reason this refresh still runs even + // while realtime is live. presence = snapshot.presence + lastError = nil + lastSyncAt = .now + applyRealtimeCapability(snapshot.realtimeEnabled) + // Checked AFTER the await, against the ACTOR's phase (not the + // stream-fed MainActor cache): while the WebSocket is live, + // deltas own the four reliable collections and an HTTP response + // — possibly computed before the connection went live — must + // not clobber them. This covers the in-flight race: a fallback- + // poll or pull-to-refresh response that lands after `.recovered` + // is dropped. + guard await authoritativePhase().allowsReliableStateOverwrite else { return } events = snapshot.messages.map(\.state) automations = snapshot.automations.sorted { $0.name < $1.name } tasks = snapshot.tasks.sorted { $0.updatedAt > $1.updatedAt } @@ -251,11 +460,23 @@ final class AppModel { WidgetCenter.shared.reloadAllTimelines() } metrics = sorted - lastError = nil - lastSyncAt = .now adoptOrphanedTasks() + // Post-write reconciliation, closing the last sliver: if the + // connection went live between the check above and these writes, + // re-apply the actor's authoritative state so the delta-derived + // data wins. (If it instead goes live after THIS check, the + // states-stream delivery that accompanied the flip is still + // queued for the MainActor and will overwrite the HTTP data on + // arrival.) + if let rt = realtimeClient, await !rt.currentPhase.allowsReliableStateOverwrite { + applyRealtimeState(await rt.currentState) + } } catch { - lastError = error.localizedDescription + // An HTTP hiccup while realtime is live is not a sync outage — + // don't raise the banner over it. + if await authoritativePhase().allowsReliableStateOverwrite { + lastError = error.localizedDescription + } } } } diff --git a/apple/SitrepKit/Sources/SitrepKit/Models.swift b/apple/SitrepKit/Sources/SitrepKit/Models.swift index a7beff50..4f2d1d7c 100644 --- a/apple/SitrepKit/Sources/SitrepKit/Models.swift +++ b/apple/SitrepKit/Sources/SitrepKit/Models.swift @@ -111,14 +111,30 @@ public struct AutomationInfo: Codable, Identifiable, Sendable, Equatable { public var state: State public var lastRun: Date? + public init(id: String, name: String, executor: Executor, schedule: Schedule, state: State, lastRun: Date? = nil) { + self.id = id + self.name = name + self.executor = executor + self.schedule = schedule + self.state = state + self.lastRun = lastRun + } + public struct Executor: Codable, Sendable, Equatable { public var kind: String + + public init(kind: String) { self.kind = kind } } public struct Schedule: Codable, Sendable, Equatable { public var kind: String public var everySeconds: Int + public init(kind: String, everySeconds: Int) { + self.kind = kind + self.everySeconds = everySeconds + } + enum CodingKeys: String, CodingKey { case kind case everySeconds = "every_seconds" @@ -166,10 +182,43 @@ public struct SpaceSnapshot: Codable, Sendable { public var metrics: [SnapshotMetric] public var messages: [SnapshotMessage] public var automations: [AutomationInfo] + /// Server-declared realtime capability gate. Older servers never send + /// this field, and its absence must mean "no realtime" — decode + /// missing/null as `false`, never infer availability from anything else + /// (e.g. the presence of a `/v3/realtime` URL shape). + public var realtimeEnabled: Bool enum CodingKeys: String, CodingKey { case version, presence, tasks, metrics, messages, automations case generatedAt = "generated_at" + case realtimeEnabled = "realtime_enabled" + } + + public init( + version: Int, generatedAt: Date, presence: PresenceInfo, tasks: [TaskState], + metrics: [SnapshotMetric], messages: [SnapshotMessage], automations: [AutomationInfo], + realtimeEnabled: Bool = false + ) { + self.version = version + self.generatedAt = generatedAt + self.presence = presence + self.tasks = tasks + self.metrics = metrics + self.messages = messages + self.automations = automations + self.realtimeEnabled = realtimeEnabled + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + version = try container.decode(Int.self, forKey: .version) + generatedAt = try container.decode(Date.self, forKey: .generatedAt) + presence = try container.decode(PresenceInfo.self, forKey: .presence) + tasks = try container.decode([TaskState].self, forKey: .tasks) + metrics = try container.decode([SnapshotMetric].self, forKey: .metrics) + messages = try container.decode([SnapshotMessage].self, forKey: .messages) + automations = try container.decode([AutomationInfo].self, forKey: .automations) + realtimeEnabled = try container.decodeIfPresent(Bool.self, forKey: .realtimeEnabled) ?? false } } diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift new file mode 100644 index 00000000..f573676d --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeAuthorization.swift @@ -0,0 +1,46 @@ +import Foundation + +/// A narrow mirror of SPEC.md §10.1's authorization matrix, scoped to what a +/// *client* (source or viewer) may legally send — the rules a conformant +/// server enforces against inbound client traffic. SitrepApp only ever acts +/// as a viewer, so `RealtimeClient` only ever needs the viewer column, but +/// this stays general so the two role-tagged invalid fixtures +/// (`fixtures/invalid/role-client-hello-accept.json`, +/// `fixtures/invalid/role-viewer-command-origin-server.json`) can be checked +/// exactly as `proto/realtime/tools/validate.js` checks them, and so a +/// future source-role implementation (e.g. the menu bar app, stage 5) can +/// reuse it instead of re-deriving the matrix. +/// +/// This is a client-side sanity net, not a substitute for the server's own +/// enforcement: the server is the authority per §10.1; this only prevents +/// *this* implementation from ever constructing or accepting a frame the +/// matrix forbids. +public enum RealtimeAuthorization { + /// Whether a client authenticated with `role` may send a frame of this + /// shape. Only encodes the rules exercised by the two role-tagged + /// invalid fixtures (hello stage/role, command origin) — see SPEC.md + /// §10.1 for the full matrix, most of which server-side implementations + /// need and a viewer-only client does not. + public static func clientMaySend(_ frame: RealtimeFrame, as role: RTDeviceRole) -> Bool { + switch frame { + case .hello(let e): + // §9.1: a client may only ever send stage "offer"; "accept" + // belongs exclusively to the server. + if case .accept = e.body { return false } + return true + case .command(let e): + // §8: a client-sent command with origin "server" is always + // unauthorized, regardless of role or action. + return e.body.origin != .server + case .resume, .subscribe, .unsubscribe, .interestRenew: + return role == .viewer + case .taskEvent, .messageEvent, .metricFrame: + return role == .source + case .snapshot, .delta, .configEvent: + // Server-only in both directions; no client may send these. + return false + case .ack, .error: + return true + } + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift new file mode 100644 index 00000000..0ac6bb14 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift @@ -0,0 +1,774 @@ +import Foundation + +// One type per messages/*.schema.json body. Ordered to match SPEC.md §4's +// table. Each body's Decodable init enforces exactly the constraints its +// schema file expresses (required fields via non-optional properties, +// everything else via explicit checks); unrecognized extra keys are ignored +// for free by Codable, matching the body-level leniency in SPEC.md §15. + +// MARK: - hello + +/// `hello{body.stage: "offer"}` — sent by the connecting device, in both +/// directions of the realtime protocol (only source/viewer send offers). +public struct HelloOffer: Codable, Sendable, Equatable { + public var deviceID: String + public var role: RTDeviceRole + public var protocolVersions: [Int] + public var capabilities: [String] + + public init(deviceID: String, role: RTDeviceRole, protocolVersions: [Int], capabilities: [String] = []) { + self.deviceID = deviceID + self.role = role + self.protocolVersions = protocolVersions + self.capabilities = capabilities + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + role = try c.decode(RTDeviceRole.self, forKey: .role) + protocolVersions = try RealtimeValidation.uniqueItems( + RealtimeValidation.nonEmpty(c.decode([Int].self, forKey: .protocolVersions), field: "protocol_versions"), + field: "protocol_versions") + capabilities = try c.decodeIfPresent([String].self, forKey: .capabilities) ?? [] + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(role, forKey: .role) + try c.encode(protocolVersions, forKey: .protocolVersions) + try c.encode(capabilities, forKey: .capabilities) + } + + private enum CodingKeys: String, CodingKey { + case deviceID = "device_id", role + case protocolVersions = "protocol_versions", capabilities + } +} + +/// `hello{body.stage: "accept"}` — server-only. +public struct HelloAccept: Codable, Sendable, Equatable { + public var protocolVersion: Int + public var sessionID: String + public var heartbeatIntervalMs: Int + public var capabilities: [String] + + public init(protocolVersion: Int, sessionID: String, heartbeatIntervalMs: Int, capabilities: [String] = []) { + self.protocolVersion = protocolVersion + self.sessionID = sessionID + self.heartbeatIntervalMs = heartbeatIntervalMs + self.capabilities = capabilities + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + protocolVersion = try RealtimeValidation.minimum( + c.decode(Int.self, forKey: .protocolVersion), 1, field: "protocol_version") + sessionID = try RealtimeValidation.length(c.decode(String.self, forKey: .sessionID), 1...64, field: "session_id") + heartbeatIntervalMs = try RealtimeValidation.range( + c.decode(Int.self, forKey: .heartbeatIntervalMs), 1000...300_000, field: "heartbeat_interval_ms") + capabilities = try c.decodeIfPresent([String].self, forKey: .capabilities) ?? [] + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(protocolVersion, forKey: .protocolVersion) + try c.encode(sessionID, forKey: .sessionID) + try c.encode(heartbeatIntervalMs, forKey: .heartbeatIntervalMs) + try c.encode(capabilities, forKey: .capabilities) + } + + private enum CodingKeys: String, CodingKey { + case protocolVersion = "protocol_version", sessionID = "session_id" + case heartbeatIntervalMs = "heartbeat_interval_ms", capabilities + } +} + +/// `hello.body`: `oneOf(offer, accept)` discriminated by `stage`. +public enum HelloBody: RealtimeBody { + case offer(HelloOffer) + case accept(HelloAccept) + + public static let messageType = "hello" + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + let stage = try c.decode(String.self, forKey: .stage) + switch stage { + case "offer": self = .offer(try HelloOffer(from: decoder)) + case "accept": self = .accept(try HelloAccept(from: decoder)) + default: throw RealtimeDecodingError.malformed("hello.stage must be 'offer' or 'accept': \(stage)") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .offer(let o): + try c.encode("offer", forKey: .stage) + try o.encode(to: encoder) + case .accept(let a): + try c.encode("accept", forKey: .stage) + try a.encode(to: encoder) + } + } + + private enum CodingKeys: String, CodingKey { case stage } +} + +// MARK: - resume + +public struct ResumeBody: RealtimeBody { + public static let messageType = "resume" + + /// The space_revision this viewer last fully applied. 0 means "no prior + /// state, send me a snapshot" (§6.3). + public var lastRevision: Int + + public init(lastRevision: Int) { self.lastRevision = lastRevision } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + lastRevision = try RealtimeValidation.minimum( + c.decode(Int.self, forKey: .lastRevision), 0, field: "last_revision") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(lastRevision, forKey: .lastRevision) + } + + private enum CodingKeys: String, CodingKey { case lastRevision = "last_revision" } +} + +// MARK: - snapshot + +public struct SnapshotBody: RealtimeBody { + public static let messageType = "snapshot" + + public var revision: Int + public var part: Int + public var final: Bool + public var tasks: [RTTaskState] + public var metrics: [RTMetricSample] + public var messages: [RTMessageRecord] + public var automations: [RTAutomationState] + + public init(revision: Int, part: Int, final: Bool, tasks: [RTTaskState], metrics: [RTMetricSample], + messages: [RTMessageRecord], automations: [RTAutomationState]) { + self.revision = revision + self.part = part + self.final = final + self.tasks = tasks + self.metrics = metrics + self.messages = messages + self.automations = automations + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + revision = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .revision), 0, field: "revision") + part = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .part), 1, field: "part") + final = try c.decode(Bool.self, forKey: .final) + tasks = try c.decode([RTTaskState].self, forKey: .tasks) + metrics = try c.decode([RTMetricSample].self, forKey: .metrics) + messages = try c.decode([RTMessageRecord].self, forKey: .messages) + automations = try c.decode([RTAutomationState].self, forKey: .automations) + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(revision, forKey: .revision) + try c.encode(part, forKey: .part) + try c.encode(final, forKey: .final) + try c.encode(tasks, forKey: .tasks) + try c.encode(metrics, forKey: .metrics) + try c.encode(messages, forKey: .messages) + try c.encode(automations, forKey: .automations) + } + + private enum CodingKeys: String, CodingKey { case revision, part, final, tasks, metrics, messages, automations } +} + +// MARK: - delta + +/// One entry of `delta.body.events`: `oneOf` tagged by `event_type`. An +/// `event_type` this build does not recognize is kept as `.unknown` rather +/// than failing the whole delta — a future minor version could add a new +/// reliable event kind, and dropping just that one entry (while the +/// revision still advances by the full `events.length`, per §6.2's exact +/// arithmetic) is safer than losing revision continuity over it. This is a +/// defensive interpretation beyond what SPEC.md §15 states explicitly (which +/// only names *envelope* `type`, not nested `event_type`) — flagged in the +/// handoff as a place a protocol owner may want to rule on explicitly. +public enum DeltaEvent: Sendable, Equatable { + case taskEvent(TaskEventBody) + case messageEvent(MessageEventBody) + case configEvent(ConfigEventBody) + case unknown(eventType: String) +} + +extension DeltaEvent: Codable { + private enum CodingKeys: String, CodingKey { case eventType = "event_type", event } + + public init(from decoder: Decoder) throws { + // Only `event_type` and `event` are defined in v1; unknown sibling + // fields are ignored, matching SPEC.md §15's body-level tolerance + // (the top-level envelope strictness of §3 does not extend to + // objects nested inside `body`). + let c = try decoder.container(keyedBy: CodingKeys.self) + let type = try c.decode(String.self, forKey: .eventType) + switch type { + case "task.event": self = .taskEvent(try c.decode(TaskEventBody.self, forKey: .event)) + case "message.event": self = .messageEvent(try c.decode(MessageEventBody.self, forKey: .event)) + case "config.event": self = .configEvent(try c.decode(ConfigEventBody.self, forKey: .event)) + default: self = .unknown(eventType: type) + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .taskEvent(let b): + try c.encode("task.event", forKey: .eventType) + try c.encode(b, forKey: .event) + case .messageEvent(let b): + try c.encode("message.event", forKey: .eventType) + try c.encode(b, forKey: .event) + case .configEvent(let b): + try c.encode("config.event", forKey: .eventType) + try c.encode(b, forKey: .event) + case .unknown(let type): + // Never constructed from an outbound encode in this client (we + // never originate delta), but keep encode total. + try c.encode(type, forKey: .eventType) + try c.encode([String: String](), forKey: .event) + } + } +} + +public struct DeltaBody: RealtimeBody { + public static let messageType = "delta" + + public var fromRevision: Int + public var toRevision: Int + public var events: [DeltaEvent] + + public init(fromRevision: Int, toRevision: Int, events: [DeltaEvent]) { + self.fromRevision = fromRevision + self.toRevision = toRevision + self.events = events + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + fromRevision = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .fromRevision), 0, field: "from_revision") + toRevision = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .toRevision), 0, field: "to_revision") + events = try c.decode([DeltaEvent].self, forKey: .events) + // §6.2: to_revision - from_revision MUST equal events.length exactly. + guard toRevision - fromRevision == events.count else { + throw RealtimeDecodingError.malformed( + "delta arithmetic violated: to_revision(\(toRevision)) - from_revision(\(fromRevision)) != events.length(\(events.count))") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(fromRevision, forKey: .fromRevision) + try c.encode(toRevision, forKey: .toRevision) + try c.encode(events, forKey: .events) + } + + private enum CodingKeys: String, CodingKey { + case fromRevision = "from_revision", toRevision = "to_revision", events + } +} + +// MARK: - ack + +public struct DeviceSeqPair: Codable, Sendable, Equatable { + public var deviceID: String + public var deviceSeq: Int + + public init(deviceID: String, deviceSeq: Int) { + self.deviceID = deviceID + self.deviceSeq = deviceSeq + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + deviceSeq = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .deviceSeq), 1, field: "device_seq") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(deviceSeq, forKey: .deviceSeq) + } + + private enum CodingKeys: String, CodingKey { case deviceID = "device_id", deviceSeq = "device_seq" } +} + +public struct LeaseInfo: Codable, Sendable, Equatable { + /// Absolute deadline; the viewer must `interest.renew` before this to + /// keep the lease alive (SPEC.md §7). + public var expiresAt: Int + + public init(expiresAt: Int) { self.expiresAt = expiresAt } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + expiresAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .expiresAt), field: "lease.expires_at") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(expiresAt, forKey: .expiresAt) + } + + private enum CodingKeys: String, CodingKey { case expiresAt = "expires_at" } +} + +public struct AckBody: RealtimeBody { + public static let messageType = "ack" + + public var acked: [DeviceSeqPair]? + public var inReplyTo: String? + public var lease: LeaseInfo? + + public init(acked: [DeviceSeqPair]? = nil, inReplyTo: String? = nil, lease: LeaseInfo? = nil) { + self.acked = acked + self.inReplyTo = inReplyTo + self.lease = lease + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + acked = try c.decodeIfPresent([DeviceSeqPair].self, forKey: .acked) + if let acked { try _ = RealtimeValidation.nonEmpty(acked, field: "acked") } + inReplyTo = try c.decodeIfPresent(String.self, forKey: .inReplyTo) + lease = try c.decodeIfPresent(LeaseInfo.self, forKey: .lease) + // anyOf(acked, in_reply_to) — see fixtures/invalid/ack-neither-acked-nor-in-reply-to.json. + guard acked != nil || inReplyTo != nil else { + throw RealtimeDecodingError.malformed("ack.body must carry 'acked' and/or 'in_reply_to'") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encodeIfPresent(acked, forKey: .acked) + try c.encodeIfPresent(inReplyTo, forKey: .inReplyTo) + try c.encodeIfPresent(lease, forKey: .lease) + } + + private enum CodingKeys: String, CodingKey { case acked, inReplyTo = "in_reply_to", lease } +} + +// MARK: - task.event + +public struct TaskEventBody: RealtimeBody { + public static let messageType = "task.event" + + public var deviceID: String + public var deviceSeq: Int + public var taskID: String + public var kind: RTTaskKind + public var occurredAt: Int + public var title: String? + public var percent: Int? + public var step: String? + public var message: String? + public var display: RTDisplayHints? + + public init(deviceID: String, deviceSeq: Int, taskID: String, kind: RTTaskKind, occurredAt: Int, + title: String? = nil, percent: Int? = nil, step: String? = nil, message: String? = nil, + display: RTDisplayHints? = nil) { + self.deviceID = deviceID + self.deviceSeq = deviceSeq + self.taskID = taskID + self.kind = kind + self.occurredAt = occurredAt + self.title = title + self.percent = percent + self.step = step + self.message = message + self.display = display + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + deviceSeq = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .deviceSeq), 1, field: "device_seq") + taskID = try RealtimeValidation.length(c.decode(String.self, forKey: .taskID), 1...256, field: "task_id") + kind = try c.decode(RTTaskKind.self, forKey: .kind) + occurredAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .occurredAt), field: "occurred_at") + title = try c.decodeIfPresent(String.self, forKey: .title) + if let title { try _ = RealtimeValidation.maxLength(title, 2048, field: "title") } + percent = try c.decodeIfPresent(Int.self, forKey: .percent) + if let percent { try _ = RealtimeValidation.range(percent, 0...100, field: "percent") } + step = try c.decodeIfPresent(String.self, forKey: .step) + if let step { try _ = RealtimeValidation.maxLength(step, 2048, field: "step") } + message = try c.decodeIfPresent(String.self, forKey: .message) + if let message { try _ = RealtimeValidation.maxLength(message, 2048, field: "message") } + display = try c.decodeIfPresent(RTDisplayHints.self, forKey: .display) + // if kind == progress then percent required. + if kind == .progress && percent == nil { + throw RealtimeDecodingError.malformed("task.event.body.percent required when kind == 'progress'") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(deviceSeq, forKey: .deviceSeq) + try c.encode(taskID, forKey: .taskID) + try c.encode(kind, forKey: .kind) + try c.encode(occurredAt, forKey: .occurredAt) + try c.encodeIfPresent(title, forKey: .title) + try c.encodeIfPresent(percent, forKey: .percent) + try c.encodeIfPresent(step, forKey: .step) + try c.encodeIfPresent(message, forKey: .message) + try c.encodeIfPresent(display, forKey: .display) + } + + private enum CodingKeys: String, CodingKey { + case deviceID = "device_id", deviceSeq = "device_seq", taskID = "task_id", kind + case occurredAt = "occurred_at", title, percent, step, message, display + } +} + +// MARK: - message.event + +public struct MessageEventBody: RealtimeBody { + public static let messageType = "message.event" + + public var deviceID: String + public var deviceSeq: Int + public var messageID: String + public var level: RTMessageLevel + public var text: String + public var occurredAt: Int + public var automationID: String? + + public init(deviceID: String, deviceSeq: Int, messageID: String, level: RTMessageLevel, text: String, + occurredAt: Int, automationID: String? = nil) { + self.deviceID = deviceID + self.deviceSeq = deviceSeq + self.messageID = messageID + self.level = level + self.text = text + self.occurredAt = occurredAt + self.automationID = automationID + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + deviceSeq = try RealtimeValidation.minimum(c.decode(Int.self, forKey: .deviceSeq), 1, field: "device_seq") + messageID = try RealtimeValidation.length(c.decode(String.self, forKey: .messageID), 1...128, field: "message_id") + level = try c.decode(RTMessageLevel.self, forKey: .level) + text = try RealtimeValidation.maxLength(c.decode(String.self, forKey: .text), 2048, field: "text") + occurredAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .occurredAt), field: "occurred_at") + automationID = try c.decodeIfPresent(String.self, forKey: .automationID) + if let automationID { try _ = RealtimeValidation.maxLength(automationID, 128, field: "automation_id") } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(deviceSeq, forKey: .deviceSeq) + try c.encode(messageID, forKey: .messageID) + try c.encode(level, forKey: .level) + try c.encode(text, forKey: .text) + try c.encode(occurredAt, forKey: .occurredAt) + try c.encodeIfPresent(automationID, forKey: .automationID) + } + + private enum CodingKeys: String, CodingKey { + case deviceID = "device_id", deviceSeq = "device_seq", messageID = "message_id" + case level, text, occurredAt = "occurred_at", automationID = "automation_id" + } +} + +// MARK: - metric.frame + +public struct MetricFrameBody: RealtimeBody { + public static let messageType = "metric.frame" + + public var deviceID: String + public var metrics: [RTMetricSample] + + public init(deviceID: String, metrics: [RTMetricSample]) { + self.deviceID = deviceID + self.metrics = metrics + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + deviceID = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .deviceID), "^[A-Za-z0-9_-]{1,128}$", field: "device_id") + let decoded = try c.decode([RTMetricSample].self, forKey: .metrics) + metrics = try RealtimeValidation.maxItems( + RealtimeValidation.nonEmpty(decoded, field: "metrics"), 64, field: "metrics") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(deviceID, forKey: .deviceID) + try c.encode(metrics, forKey: .metrics) + } + + private enum CodingKeys: String, CodingKey { case deviceID = "device_id", metrics } +} + +// MARK: - config.event + +public struct ConfigEventBody: RealtimeBody { + public static let messageType = "config.event" + + public enum Kind: String, Codable, Sendable, Equatable { + case upserted = "automation.upserted" + case removed = "automation.removed" + } + + public var kind: Kind + public var automationID: String + public var automation: RTAutomationState? + public var occurredAt: Int + + public init(kind: Kind, automationID: String, automation: RTAutomationState? = nil, occurredAt: Int) { + self.kind = kind + self.automationID = automationID + self.automation = automation + self.occurredAt = occurredAt + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + kind = try c.decode(Kind.self, forKey: .kind) + automationID = try RealtimeValidation.length(c.decode(String.self, forKey: .automationID), 1...128, field: "automation_id") + automation = try c.decodeIfPresent(RTAutomationState.self, forKey: .automation) + occurredAt = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .occurredAt), field: "occurred_at") + if kind == .upserted && automation == nil { + throw RealtimeDecodingError.malformed("config.event.body.automation required when kind == 'automation.upserted'") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(kind, forKey: .kind) + try c.encode(automationID, forKey: .automationID) + try c.encodeIfPresent(automation, forKey: .automation) + try c.encode(occurredAt, forKey: .occurredAt) + } + + private enum CodingKeys: String, CodingKey { + case kind, automationID = "automation_id", automation, occurredAt = "occurred_at" + } +} + +// MARK: - subscribe / unsubscribe / interest.renew + +/// Shared body shape for `subscribe`, sent as a viewer declares (or wholly +/// replaces) its device's interest lease. +public struct SubscribeBody: RealtimeBody { + public static let messageType = "subscribe" + + public var topics: [RTTopic] + + public init(topics: [RTTopic] = []) { self.topics = topics } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + let decoded = try c.decodeIfPresent([RTTopic].self, forKey: .topics) ?? [] + topics = try RealtimeValidation.uniqueItems(decoded, field: "topics") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(topics, forKey: .topics) + } + + private enum CodingKeys: String, CodingKey { case topics } +} + +public struct UnsubscribeBody: RealtimeBody { + public static let messageType = "unsubscribe" + + public init() {} + public init(from decoder: Decoder) throws { _ = try decoder.container(keyedBy: AnyKey.self) } + public func encode(to encoder: Encoder) throws { _ = encoder.container(keyedBy: AnyKey.self) } +} + +/// Same shape as `SubscribeBody` (interest.renew.schema.json literally +/// `$ref`s subscribe's body) but a distinct Swift type since `messageType` +/// differs and each schema file is versioned independently. +public struct InterestRenewBody: RealtimeBody { + public static let messageType = "interest.renew" + + public var topics: [RTTopic] + + public init(topics: [RTTopic] = []) { self.topics = topics } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + let decoded = try c.decodeIfPresent([RTTopic].self, forKey: .topics) ?? [] + topics = try RealtimeValidation.uniqueItems(decoded, field: "topics") + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(topics, forKey: .topics) + } + + private enum CodingKeys: String, CodingKey { case topics } +} + +// MARK: - command + +public struct CommandBody: RealtimeBody { + public static let messageType = "command" + + public var commandID: String + public var origin: RTCommandOrigin + public var issuedByDeviceID: String? + public var action: RTCommandAction + public var taskID: String? + public var automationID: String? + public var targetDeviceID: String? + public var ttlMs: Int + public var params: [String: JSONValue]? + + public init(commandID: String, origin: RTCommandOrigin, issuedByDeviceID: String? = nil, + action: RTCommandAction, taskID: String? = nil, automationID: String? = nil, + targetDeviceID: String? = nil, ttlMs: Int, params: [String: JSONValue]? = nil) { + self.commandID = commandID + self.origin = origin + self.issuedByDeviceID = issuedByDeviceID + self.action = action + self.taskID = taskID + self.automationID = automationID + self.targetDeviceID = targetDeviceID + self.ttlMs = ttlMs + self.params = params + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + commandID = try RealtimeValidation.length(c.decode(String.self, forKey: .commandID), 1...64, field: "command_id") + origin = try c.decode(RTCommandOrigin.self, forKey: .origin) + issuedByDeviceID = try c.decodeIfPresent(String.self, forKey: .issuedByDeviceID) + action = try c.decode(RTCommandAction.self, forKey: .action) + taskID = try c.decodeIfPresent(String.self, forKey: .taskID) + if let taskID { try _ = RealtimeValidation.length(taskID, 1...256, field: "task_id") } + automationID = try c.decodeIfPresent(String.self, forKey: .automationID) + if let automationID { try _ = RealtimeValidation.length(automationID, 1...128, field: "automation_id") } + targetDeviceID = try c.decodeIfPresent(String.self, forKey: .targetDeviceID) + ttlMs = try RealtimeValidation.range(c.decode(Int.self, forKey: .ttlMs), 1...86_400_000, field: "ttl_ms") + params = try c.decodeIfPresent([String: JSONValue].self, forKey: .params) + + // §8 per-action required/forbidden field matrix. + switch action { + case .pause, .resume, .stop: + guard origin == .viewer else { + throw RealtimeDecodingError.malformed("command.action \(action) requires origin 'viewer'") + } + guard issuedByDeviceID != nil else { + throw RealtimeDecodingError.malformed("command.action \(action) requires issued_by_device_id") + } + guard taskID != nil else { + throw RealtimeDecodingError.malformed("command.action \(action) requires task_id") + } + guard automationID == nil else { + throw RealtimeDecodingError.malformed("command.action \(action) forbids automation_id") + } + case .runNow: + guard origin == .viewer else { + throw RealtimeDecodingError.malformed("command.action run_now requires origin 'viewer'") + } + guard issuedByDeviceID != nil else { + throw RealtimeDecodingError.malformed("command.action run_now requires issued_by_device_id") + } + guard automationID != nil else { + throw RealtimeDecodingError.malformed("command.action run_now requires automation_id") + } + guard taskID == nil else { + throw RealtimeDecodingError.malformed("command.action run_now forbids task_id") + } + case .throttle, .resumeRate: + guard origin == .server else { + throw RealtimeDecodingError.malformed("command.action \(action) requires origin 'server'") + } + guard taskID == nil && automationID == nil else { + throw RealtimeDecodingError.malformed("command.action \(action) forbids task_id and automation_id") + } + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(commandID, forKey: .commandID) + try c.encode(origin, forKey: .origin) + try c.encodeIfPresent(issuedByDeviceID, forKey: .issuedByDeviceID) + try c.encode(action, forKey: .action) + try c.encodeIfPresent(taskID, forKey: .taskID) + try c.encodeIfPresent(automationID, forKey: .automationID) + try c.encodeIfPresent(targetDeviceID, forKey: .targetDeviceID) + try c.encode(ttlMs, forKey: .ttlMs) + try c.encodeIfPresent(params, forKey: .params) + } + + private enum CodingKeys: String, CodingKey { + case commandID = "command_id", origin, issuedByDeviceID = "issued_by_device_id", action + case taskID = "task_id", automationID = "automation_id", targetDeviceID = "target_device_id" + case ttlMs = "ttl_ms", params + } +} + +// MARK: - error + +public struct ErrorBody: RealtimeBody { + public static let messageType = "error" + + public var code: RTErrorCode + public var message: String + public var inReplyTo: String? + public var retryable: Bool + public var fatal: Bool + + public init(code: RTErrorCode, message: String, inReplyTo: String? = nil, retryable: Bool, fatal: Bool) { + self.code = code + self.message = message + self.inReplyTo = inReplyTo + self.retryable = retryable + self.fatal = fatal + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + code = try c.decode(RTErrorCode.self, forKey: .code) + message = try RealtimeValidation.maxLength(c.decode(String.self, forKey: .message), 500, field: "message") + inReplyTo = try c.decodeIfPresent(String.self, forKey: .inReplyTo) + retryable = try c.decode(Bool.self, forKey: .retryable) + fatal = try c.decode(Bool.self, forKey: .fatal) + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(code, forKey: .code) + try c.encode(message, forKey: .message) + try c.encodeIfPresent(inReplyTo, forKey: .inReplyTo) + try c.encode(retryable, forKey: .retryable) + try c.encode(fatal, forKey: .fatal) + } + + private enum CodingKeys: String, CodingKey { + case code, message, inReplyTo = "in_reply_to", retryable, fatal + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift new file mode 100644 index 00000000..da57cd5b --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCapabilityGate.swift @@ -0,0 +1,54 @@ +import Foundation + +/// Pure decision logic for the realtime capability P0 gate: the app must +/// only ever open a `RealtimeClient` connection when the most recent +/// successful REST refresh reported `realtime_enabled == true` on +/// `SpaceSnapshot`. Field absent/false (old or not-yet-rolled-out servers) +/// means "do not connect" — never inferred from URL shape or a `/v3` probe. +/// +/// Deliberately free of any RealtimeClient/URLSession/actor dependency so it +/// can be unit-tested without a live socket. `RealtimeClient` itself stays +/// capability-agnostic; the caller (AppModel) owns one of these and asks it +/// what to do each time a REST refresh completes. +public struct RealtimeCapabilityGate: Sendable, Equatable { + /// What the caller should do to the RealtimeClient in response to a + /// refreshed capability value. + public enum Action: Sendable, Equatable { + /// No change — either the value didn't change, or a repeat of an + /// already-applied value. + case none + /// Capability just turned on (including the very first time a + /// refresh reports `true`) — safe to (re)start the connection. + case connect + /// Capability just turned off (server rolled the flag back, or a + /// legacy/absent response) while previously on — tear down any live + /// connection and fall back to plain HTTP; no reconnect attempts + /// until a later refresh reports `true` again. + case disconnect + } + + /// Cold start (before any REST refresh has ever completed) must behave + /// as "not capable" — this default is what makes that safe without any + /// extra bookkeeping at call sites. + public private(set) var isCapable = false + + public init() {} + + /// Feed the `realtime_enabled` value from a just-completed REST + /// refresh. Returns the action the caller should take on its + /// RealtimeClient. Idempotent: repeating the same value is `.none`. + @discardableResult + public mutating func apply(refreshedCapability enabled: Bool) -> Action { + guard enabled != isCapable else { return .none } + isCapable = enabled + return enabled ? .connect : .disconnect + } + + /// Force back to the cold-start state — used when the identity this + /// gate was tracking no longer applies (re-pairing to a different + /// server/space, or disconnecting entirely). The next refresh against + /// the new identity must prove capability again before connecting. + public mutating func reset() { + isCapable = false + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift new file mode 100644 index 00000000..38528593 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift @@ -0,0 +1,562 @@ +import Foundation + +/// Errors `RealtimeClient` raises itself (as opposed to a `RTErrorCode` the +/// server sent, which arrives wrapped in `.serverError`). +public enum RealtimeClientError: Error, Sendable { + case timeout + case protocolViolation(String) + case serverError(ErrorBody) +} + +/// A viewer-role client for the Sitrep realtime protocol +/// (`proto/realtime/SPEC.md`). Owns one WebSocket connection at a time and +/// drives the mandatory `hello → subscribe → resume` gate, snapshot chunk +/// reassembly, delta application, interest-lease renewal, and reconnect +/// with backoff. SitrepApp is a viewer only (it observes a Mac Agent's +/// tasks; it never executes or reports them), so this client never sends +/// `task.event`/`message.event`/`metric.frame` uplink — only `hello`, +/// `subscribe`, `unsubscribe`, `resume`, `interest.renew`, and viewer +/// `command`s. +/// +/// All connection state lives inside this actor; callers observe it through +/// the three `AsyncStream`s (`phases`, `states`, `notices`) rather than by +/// reading actor-isolated properties directly. +public actor RealtimeClient { + /// Connection lifecycle, per the brief: idle/connecting/handshaking/ + /// subscribed/live/failed. + public enum Phase: Sendable, Equatable { + case idle + case connecting + case handshaking + case subscribed + case live + case failed(String) + } + + /// One-off, non-state notifications a UI layer may want to surface. + public enum Notice: Sendable, Equatable { + /// This device's credential/connection was superseded by another + /// connection (SPEC.md §9.4). SitrepApp never intentionally opens a + /// second concurrent connection, so this always means "surprising" + /// per §9.4's client guidance — surface a one-time, non-disruptive + /// banner ("this session may be in use elsewhere"), don't reconnect + /// in a tight loop. + case superseded + case serverError(RTErrorCode, message: String) + case revisionGap(from: Int) + /// WS has failed for `fallbackAfterFailures` consecutive backoff + /// cycles; the caller should fall back to low-frequency HTTP + /// polling until `.recovered`. + case fellBackToPolling(reason: String) + /// WS reached `.live` again after a fallback; the caller should stop + /// the HTTP polling fallback. + case recovered + } + + public struct Configuration: Sendable { + /// The realtime WebSocket endpoint (see `APIClient.realtimeURL`). + public var url: URL + /// Presented once at connection establishment (SPEC.md §10), as an + /// `Authorization: Bearer` header — never repeated inside an + /// envelope afterward. + public var token: String? + public var deviceID: String + public var protocolVersions: [Int] + public var topics: [RTTopic] + + public init(url: URL, token: String?, deviceID: String, protocolVersions: [Int] = [1], topics: [RTTopic] = []) { + self.url = url + self.token = token + self.deviceID = deviceID + self.protocolVersions = protocolVersions + self.topics = topics + } + } + + /// Consecutive failed connection attempts (full backoff cycles) before + /// signalling `.fellBackToPolling`. + public var fallbackAfterFailures = 3 + + private let configuration: Configuration + + private var runTask: Task? + private var socket: URLSessionWebSocketTask? + private var leaseRenewalTask: Task? + private var watchdogTask: Task? + + /// All resume/delta/snapshot gating and folding decisions are delegated + /// to this shared, transport-agnostic state machine (SPEC.md §6.2/§6.3) + /// — see `RealtimeResumeGateScenarioTests` for the same logic driven + /// directly by scenario fixtures. + private var gate: RealtimeResumeGate + private var phase: Phase = .idle { + didSet { phaseContinuation.yield(phase) } + } + + private var consecutiveFailures = 0 + private var firedFallback = false + private var lastInboundAt: Int = 0 + /// §13: `malformed` is non-fatal — a single undecodable frame is + /// skipped, not a reason to drop the connection. This counts the + /// current run of consecutive malformed frames so a persistently + /// broken peer still trips a circuit breaker (see `receiveFrame`). + private var consecutiveMalformedFrames = 0 + /// Number of consecutive malformed frames after which the connection is + /// torn down and re-established (circuit breaker). + private let malformedFrameCircuitBreaker = 5 + /// Distinguishes which `runLoop` invocation currently owns the + /// connection resources (`runTask`, `socket`, `leaseRenewalTask`, + /// `watchdogTask`). A stale generation — a loop still unwinding after a + /// stop()/start() cycle on the same instance — must neither clear the + /// newer loop's task handle nor tear down or replace its connection + /// resources (see `ConnectionTeardownPolicy`). + private var runGeneration = 0 + + private let phaseContinuation: AsyncStream.Continuation + public nonisolated let phases: AsyncStream + private let noticeContinuation: AsyncStream.Continuation + public nonisolated let notices: AsyncStream + private let stateContinuation: AsyncStream.Continuation + public nonisolated let states: AsyncStream + + public init(configuration: Configuration, initialState: SpaceState = SpaceState()) { + self.configuration = configuration + self.gate = RealtimeResumeGate(state: initialState) + var pc: AsyncStream.Continuation! + self.phases = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { pc = $0 } + self.phaseContinuation = pc + var nc: AsyncStream.Continuation! + self.notices = AsyncStream(bufferingPolicy: .bufferingNewest(8)) { nc = $0 } + self.noticeContinuation = nc + var sc: AsyncStream.Continuation! + self.states = AsyncStream(bufferingPolicy: .bufferingNewest(1)) { sc = $0 } + self.stateContinuation = sc + } + + public var currentState: SpaceState { gate.state } + public var currentPhase: Phase { phase } + + // MARK: - Lifecycle + + /// Idempotent: does nothing if already running. Call when the app enters + /// the foreground. Also the restart path after a terminal stop (see + /// `runLoop`): once the loop has exited — including after a + /// fatal-and-not-retryable server error — a later `start()` begins a + /// fresh reconnect cycle. + public func start() { + guard runTask == nil else { return } + consecutiveFailures = 0 + firedFallback = false + runGeneration += 1 + let generation = runGeneration + runTask = Task { [weak self] in await self?.runLoop(generation: generation) } + } + + /// Tears down the connection and stops reconnecting. The interest lease + /// naturally expires on the server side (SPEC.md §7) — this deliberately + /// does not send `unsubscribe`, matching the brief: backgrounding closes + /// the connection and lets the lease lapse rather than releasing it + /// explicitly. Call when the app leaves the foreground. + public func stop() { + runTask?.cancel() + runTask = nil + teardownConnection() + phase = .idle + } + + /// True for a server error whose own `retryable`/`fatal` flags say + /// "connection closes AND retrying the same thing cannot succeed" — + /// `superseded`, `unauthenticated`, `version_unsupported` (§13). Per + /// ruling, these stop the run loop instead of feeding the reconnect + /// cycle: the notice has already been surfaced, and reconnection waits + /// for an explicit user action or the next foreground `start()`. + private func isTerminal(_ error: Error) -> Bool { + if case RealtimeClientError.serverError(let body) = error { + return body.fatal && !body.retryable + } + return false + } + + private func runLoop(generation: Int) async { + var terminal = false + while !Task.isCancelled { + do { + try await connectOnce(generation: generation) + } catch is CancellationError { + break + } catch { + if isTerminal(error) { + phase = .failed(String(describing: error)) + terminal = true + break + } + consecutiveFailures += 1 + phase = .failed(String(describing: error)) + if consecutiveFailures >= fallbackAfterFailures && !firedFallback { + firedFallback = true + noticeContinuation.yield(.fellBackToPolling(reason: String(describing: error))) + } + } + guard !Task.isCancelled else { break } + if consecutiveFailures > 0 { + try? await Task.sleep(for: .seconds(backoffDelay(attempt: consecutiveFailures))) + } + } + // Only the newest loop may release the handle (a stale loop exiting + // after stop()+start() must not clobber its successor's), and a + // terminal exit keeps the `.failed` phase visible instead of + // resetting to `.idle`. + if runGeneration == generation { + runTask = nil + if !terminal { phase = .idle } + } + } + + private func backoffDelay(attempt: Int) -> Double { + let capped = min(attempt, 6) + let base = min(30.0, pow(2.0, Double(capped))) + return base + Double.random(in: 0...(base * 0.3)) + } + + // MARK: - One connection's lifetime + + private func connectOnce(generation: Int) async throws { + phase = .connecting + consecutiveMalformedFrames = 0 + var request = URLRequest(url: configuration.url) + if let token = configuration.token, !token.isEmpty { + request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + } + let socket = URLSession.shared.webSocketTask(with: request) + self.socket = socket + socket.resume() + // A stale generation's deferred teardown (this connectOnce resuming + // from suspension after a stop()/start() cycle) must not cancel the + // NEWER generation's socket/lease/watchdog. It still closes its own + // socket, which it holds by reference. + defer { teardownConnection(requestedBy: generation, ownSocket: socket) } + + phase = .handshaking + let accept = try await performHandshake(on: socket) + startHeartbeat(on: socket, intervalMs: accept.heartbeatIntervalMs, generation: generation) + + let subscribeID = newEnvelopeID() + try await sendFrame( + .subscribe(Envelope(id: subscribeID, ts: nowMs(), body: SubscribeBody(topics: configuration.topics))), + on: socket) + let ackFrame = try await receiveFrame(on: socket) + guard case .ack(let ackEnvelope) = ackFrame, let lease = ackEnvelope.body.lease else { + if case .error(let e) = ackFrame { + // Route through handleError so the notice is surfaced and + // fatal errors (incl. terminal ones) throw with the server's + // own retryable/fatal semantics attached. + try await handleError(e.body, on: socket) + } + throw RealtimeClientError.protocolViolation("expected subscribe ack with lease, got \(ackFrame)") + } + phase = .subscribed + scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket, generation: generation) + + let lastRevision = gate.state.revision + gate.beganResume(lastRevision: lastRevision) + try await sendFrame( + .resume(Envelope(id: newEnvelopeID(), ts: nowMs(), body: ResumeBody(lastRevision: lastRevision))), + on: socket) + + while !Task.isCancelled { + let frame = try await receiveFrame(on: socket) + try await dispatch(frame, on: socket, generation: generation) + } + } + + /// Tears down the shared connection resources — but only when the + /// requesting generation still owns them (`requestedBy: nil` is the + /// unconditional form used by `stop()`). A stale generation's deferred + /// call limits itself to closing the socket it created (`ownSocket`), + /// leaving the newer generation's resources untouched. + private func teardownConnection(requestedBy generation: Int? = nil, ownSocket: URLSessionWebSocketTask? = nil) { + guard ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: generation, current: runGeneration) else { + ownSocket?.cancel(with: .normalClosure, reason: nil) + return + } + socket?.cancel(with: .normalClosure, reason: nil) + socket = nil + leaseRenewalTask?.cancel() + leaseRenewalTask = nil + watchdogTask?.cancel() + watchdogTask = nil + } + + // MARK: - Handshake (§9) + + private func performHandshake(on socket: URLSessionWebSocketTask) async throws -> HelloAccept { + let offer = HelloOffer(deviceID: configuration.deviceID, role: .viewer, protocolVersions: configuration.protocolVersions) + try await sendFrame(.hello(Envelope(id: newEnvelopeID(), ts: nowMs(), body: .offer(offer))), on: socket) + // §9.1: the client MUST NOT send anything else until the accept + // arrives; we simply don't send anything else here. Recommended 10s + // timeout on the client side. + let frame = try await withTimeout(seconds: 10) { try await self.receiveFrame(on: socket) } + switch frame { + case .hello(let env): + guard case .accept(let accept) = env.body else { + throw RealtimeClientError.protocolViolation("client received a hello offer, expected accept") + } + return accept + case .error(let env): + // handleError surfaces the notice and, for fatal codes (e.g. + // version_unsupported), throws serverError so the run loop can + // apply the terminal/retryable distinction. + try await handleError(env.body, on: socket) + throw RealtimeClientError.protocolViolation("handshake answered with a non-fatal error; reconnecting") + default: + throw RealtimeClientError.protocolViolation("expected hello accept, got \(frame)") + } + } + + // MARK: - Frame dispatch (post-handshake, post-subscribe) + + private func dispatch(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask, generation: Int) async throws { + // §6.2: while a chunked snapshot is in flight, only further snapshot + // chunks (and ping/pong, which never reach dispatch) are legal on + // this connection. Anything else is a malformed sequence: drop the + // buffered chunks and reconnect; the fresh connection resumes anew. + if gate.isSnapshotInFlight { + guard case .snapshot = frame else { + _ = gate.interleavedFrameDuringSnapshot() + throw RealtimeClientError.protocolViolation( + "non-snapshot envelope interleaved during a chunked snapshot") + } + } + switch frame { + case .snapshot(let env): + switch gate.receiveSnapshotChunk(env.body) { + case .applied: + markDeltaEligible() + stateContinuation.yield(gate.state) + case .snapshotChunkBuffered, .discarded: + break + case .gapDetected: + break // not reachable for a snapshot chunk + case .malformedSequence(let reason): + throw RealtimeClientError.protocolViolation("snapshot chunk out of sequence: \(reason)") + } + case .delta(let env): + switch gate.receiveDelta(env.body) { + case .applied: + markDeltaEligible() + stateContinuation.yield(gate.state) + case .discarded: + break + case .gapDetected(let resendLastRevision): + noticeContinuation.yield(.revisionGap(from: resendLastRevision)) + try await sendFrame( + .resume(Envelope(id: newEnvelopeID(), ts: nowMs(), body: ResumeBody(lastRevision: resendLastRevision))), + on: socket) + case .snapshotChunkBuffered, .malformedSequence: + break // not reachable for a delta + } + case .metricFrame(let env): + gate.receiveMetricFrame(env.body) + stateContinuation.yield(gate.state) + case .ack(let env): + if let lease = env.body.lease { + scheduleLeaseRenewal(expiresAt: lease.expiresAt, on: socket, generation: generation) + } + case .error(let env): + try await handleError(env.body, on: socket) + case .command, .hello, .subscribe, .unsubscribe, .interestRenew, .taskEvent, .messageEvent, .configEvent, .resume: + // A conformant server never sends any of these to a viewer + // (§10.1: server-only or source-only in the other direction). + // Ignore defensively rather than tearing down the connection. + break + } + } + + private func markDeltaEligible() { + phase = .live + if consecutiveFailures > 0 { consecutiveFailures = 0 } + if firedFallback { + firedFallback = false + noticeContinuation.yield(.recovered) + } + } + + private func handleError(_ body: ErrorBody, on socket: URLSessionWebSocketTask) async throws { + // §6.3: revision_unavailable answering our outstanding resume ⇒ + // retry with last_revision: 0, staying in the "awaiting reply" state. + if gate.receiveErrorAnswersOutstandingResume(body) { + try await sendFrame( + .resume(Envelope(id: newEnvelopeID(), ts: nowMs(), body: ResumeBody(lastRevision: 0))), + on: socket) + return + } + + if body.code == .superseded { + noticeContinuation.yield(.superseded) + } else { + noticeContinuation.yield(.serverError(body.code, message: body.message)) + } + + if body.fatal { + throw RealtimeClientError.serverError(body) + } + } + + // MARK: - Interest lease (§7) + + /// Renews at 1/3 of the window remaining before `expiresAt`, as + /// instructed. Rescheduled every time we learn a fresh `expiresAt` (the + /// initial subscribe ack, or any subsequent renew ack) — see `dispatch`. + /// A stale generation (its connection already superseded by a + /// stop()/start() cycle) must not replace the current generation's + /// renewal task. + private func scheduleLeaseRenewal(expiresAt: Int, on socket: URLSessionWebSocketTask, generation: Int) { + guard generation == runGeneration else { return } + leaseRenewalTask?.cancel() + let window = max(expiresAt - nowMs(), 0) + let fireAfterMs = window - window / 3 + leaseRenewalTask = Task { [weak self] in + try? await Task.sleep(for: .milliseconds(max(fireAfterMs, 0))) + guard !Task.isCancelled else { return } + try? await self?.sendFrame( + .interestRenew(Envelope( + id: self?.newEnvelopeID() ?? UUID().uuidString, ts: self?.nowMs() ?? 0, + body: InterestRenewBody(topics: self?.configuration.topics ?? []))), + on: socket) + } + } + + // MARK: - Heartbeat (§9.3) + + private func startHeartbeat(on socket: URLSessionWebSocketTask, intervalMs: Int, generation: Int) { + guard generation == runGeneration else { return } + watchdogTask?.cancel() + lastInboundAt = nowMs() + watchdogTask = Task { [weak self] in + while !Task.isCancelled { + try? await Task.sleep(for: .milliseconds(intervalMs)) + guard !Task.isCancelled, let self else { return } + try? await self.sendPing(on: socket) + let idleFor = await self.millisecondsSinceLastInbound() + if idleFor > intervalMs * 2 { + // RECOMMENDED in §9.3: no pong (or any traffic) within + // 2x the interval ⇒ treat the connection as dead. + socket.cancel(with: .abnormalClosure, reason: nil) + return + } + } + } + } + + private func sendPing(on socket: URLSessionWebSocketTask) async throws { + try await socket.send(.string("ping")) + } + + private func millisecondsSinceLastInbound() -> Int { nowMs() - lastInboundAt } + + // MARK: - Transport helpers + + private func sendFrame(_ frame: RealtimeFrame, on socket: URLSessionWebSocketTask) async throws { + let data = try frame.encoded() + guard let text = String(data: data, encoding: .utf8) else { + throw RealtimeClientError.protocolViolation("failed to encode outbound frame as UTF-8") + } + try await socket.send(.string(text)) + } + + /// Reads the next real envelope off the socket, transparently answering + /// `ping` with `pong` and dropping stray `pong`/unrecognized-type frames + /// (§9.3, §15) without surfacing them to `dispatch`. + private func receiveFrame(on socket: URLSessionWebSocketTask) async throws -> RealtimeFrame { + while true { + let message = try await socket.receive() + lastInboundAt = nowMs() + let data: Data + switch message { + case .string(let text): + if text == "ping" { + try? await socket.send(.string("pong")) + continue + } + if text == "pong" { + continue + } + guard let d = text.data(using: .utf8) else { continue } + data = d + case .data(let d): + data = d + @unknown default: + continue + } + do { + let frame = try RealtimeFrame.decode(data) + consecutiveMalformedFrames = 0 + return frame + } catch RealtimeDecodingError.unknownType { + continue // §15: ignore, do not treat as malformed + } catch { + // §13: `malformed` is non-fatal — skip this frame and keep + // the connection. The circuit breaker only trips on a run + // of consecutive malformed frames, which indicates the + // stream itself is broken rather than one bad envelope. + consecutiveMalformedFrames += 1 + if consecutiveMalformedFrames >= malformedFrameCircuitBreaker { + consecutiveMalformedFrames = 0 + throw RealtimeClientError.protocolViolation( + "\(malformedFrameCircuitBreaker) consecutive malformed frames; reconnecting (last: \(error))") + } + continue + } + } + } + + private func withTimeout(seconds: Double, operation: @escaping @Sendable () async throws -> T) async throws -> T { + try await withThrowingTaskGroup(of: T.self) { group in + group.addTask { try await operation() } + group.addTask { + try await Task.sleep(for: .seconds(seconds)) + throw RealtimeClientError.timeout + } + guard let result = try await group.next() else { throw RealtimeClientError.timeout } + group.cancelAll() + return result + } + } + + private func nowMs() -> Int { Int(Date().timeIntervalSince1970 * 1000) } + private func newEnvelopeID() -> String { UUID().uuidString } +} + +/// Pure decision logic for generation-scoped teardown of the shared +/// connection resources (`socket`/`leaseRenewalTask`/`watchdogTask`), +/// extracted so the guard itself is directly unit-testable without a live +/// transport (`RealtimeGenerationTests`). +enum ConnectionTeardownPolicy { + /// - `requestedBy == nil`: the unconditional form (`stop()`); always + /// tears down. + /// - `requestedBy == current`: the owning generation's own teardown; + /// proceeds. + /// - `requestedBy < current`: a stale generation unwinding after a + /// stop()/start() cycle; it must NOT touch the newer generation's + /// shared resources (it may still close the socket it created and + /// holds by reference). + static func shouldTearDownShared(requestedBy generation: Int?, current: Int) -> Bool { + guard let generation else { return true } + return generation == current + } +} + +public extension RealtimeClient.Phase { + /// Whether an HTTP snapshot result may overwrite the four reliable + /// collections (tasks/metrics/messages-events/automations). While the + /// realtime connection is `.live`, deltas own that state and an HTTP + /// response — possibly computed seconds ago — must not clobber it; in + /// every other phase the HTTP path is the best available source. + /// + /// Callers MUST evaluate this AFTER their HTTP `await` returns, not + /// before issuing the request: the phase can flip to `.live` while the + /// request is in flight, and it is the state of the world at apply time + /// that decides. REST-only concepts the realtime protocol does not + /// carry (e.g. `presence`) are exempt and always apply. + var allowsReliableStateOverwrite: Bool { self != .live } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift new file mode 100644 index 00000000..605f7dc3 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift @@ -0,0 +1,483 @@ +import Foundation + +// Swift mirror of proto/realtime/. This is a derived artifact: field names, +// enum values, and validation rules MUST match the frozen schemas under +// proto/realtime/ byte-for-byte in semantics. Any deviation is a bug, not a +// style choice. Do not "clean up" a name here without checking the schema +// first. +// +// Strictness split (SPEC.md §3, §15): +// - The envelope's top level (type/id/ts/body) is permanently strict: an +// unknown top-level field is always `malformed`. `Envelope` enforces this +// explicitly (see RealtimeEnvelope.swift). +// - Fields *inside* `body` are lenient at runtime: a receiver MUST ignore +// unrecognized body fields. Swift's `Decodable` already does this for any +// JSON key not named in a type's `CodingKeys`, so the body structs below +// get that leniency for free and only need to enforce the fields this +// version of the protocol actually defines (required-ness, bounds, enums). + +/// Every error a fixture in `proto/realtime/fixtures/invalid/` is expected to +/// produce when decoded by a conformant receiver. +public enum RealtimeDecodingError: Error, Sendable, Equatable, CustomStringConvertible { + /// The envelope or body violates a constraint this protocol version + /// defines (§13 `malformed`): an unknown top-level envelope field, a + /// missing required field, a value outside its schema bound, or a + /// same-message cross-field rule (e.g. `command` action/field + /// exclusivity, `ack` requiring `acked` or `in_reply_to`). + case malformed(String) + /// `envelope.type` is not one of the 14 known message types. Per §15 + /// this is NOT malformed — a receiver MUST ignore it (forward + /// compatibility with a future additive message type). + case unknownType(String) + + public var description: String { + switch self { + case .malformed(let reason): "realtime malformed: \(reason)" + case .unknownType(let type): "realtime unknown type (ignored per SPEC.md §15): \(type)" + } + } +} + +/// Schema-bound checks mirroring `common.schema.json` `$defs`. Every check +/// here corresponds to a named constraint in that file so a reviewer can +/// diff the two side by side. +enum RealtimeValidation { + /// `$defs/unix_ms_timestamp`: 13-digit millisecond bound. This is the + /// mechanical guard against the historical seconds/milliseconds bug — + /// see `fixtures/invalid/message-event-timestamp-in-seconds.json`. + static func unixMs(_ value: Int, field: String) throws -> Int { + guard value >= 1_000_000_000_000 && value <= 9_999_999_999_999 else { + throw RealtimeDecodingError.malformed("\(field) is not a Unix ms timestamp (13 digits): \(value)") + } + return value + } + + static func maxLength(_ s: String, _ n: Int, field: String) throws -> String { + guard s.count <= n else { + throw RealtimeDecodingError.malformed("\(field) exceeds max length \(n): \(s.count) chars") + } + return s + } + + static func length(_ s: String, _ range: ClosedRange, field: String) throws -> String { + guard range.contains(s.count) else { + throw RealtimeDecodingError.malformed("\(field) length \(s.count) outside \(range)") + } + return s + } + + static func minimum(_ v: Int, _ lower: Int, field: String) throws -> Int { + guard v >= lower else { + throw RealtimeDecodingError.malformed("\(field) \(v) below minimum \(lower)") + } + return v + } + + static func range(_ v: Int, _ r: ClosedRange, field: String) throws -> Int { + guard r.contains(v) else { + throw RealtimeDecodingError.malformed("\(field) \(v) outside \(r)") + } + return v + } + + static func nonEmpty(_ arr: [T], field: String) throws -> [T] { + guard !arr.isEmpty else { + throw RealtimeDecodingError.malformed("\(field) must not be empty") + } + return arr + } + + static func maxItems(_ arr: [T], _ n: Int, field: String) throws -> [T] { + guard arr.count <= n else { + throw RealtimeDecodingError.malformed("\(field) exceeds max items \(n): \(arr.count)") + } + return arr + } + + static func uniqueItems(_ arr: [T], field: String) throws -> [T] { + guard Set(arr).count == arr.count else { + throw RealtimeDecodingError.malformed("\(field) contains duplicate items") + } + return arr + } + + static func pattern(_ s: String, _ pattern: String, field: String) throws -> String { + guard s.range(of: pattern, options: .regularExpression) != nil, + s.range(of: pattern, options: .regularExpression) == s.startIndex..: Sendable, Equatable { + public var id: String + public var ts: Int + public var body: Body + + public init(id: String, ts: Int, body: Body) { + self.id = id + self.ts = ts + self.body = body + } +} + +extension Envelope: Codable { + private enum CodingKeys: String, CodingKey { case type, id, ts, body } + + public init(from decoder: Decoder) throws { + // Strict top-level check: exactly {type, id, ts, body}, no more, no + // less. A fixed CodingKeys-keyed container can't see keys it doesn't + // name, so this uses a dynamic-key container to enumerate what's + // actually present in the JSON object. + let dyn = try decoder.container(keyedBy: AnyKey.self) + let present = Set(dyn.allKeys.map(\.stringValue)) + let allowed: Set = ["type", "id", "ts", "body"] + guard present == allowed else { + let extra = present.subtracting(allowed) + let missing = allowed.subtracting(present) + throw RealtimeDecodingError.malformed( + "envelope must carry exactly type/id/ts/body" + + (extra.isEmpty ? "" : "; unexpected field(s): \(extra.sorted().joined(separator: ", "))") + + (missing.isEmpty ? "" : "; missing field(s): \(missing.sorted().joined(separator: ", "))") + ) + } + + let c = try decoder.container(keyedBy: CodingKeys.self) + let type = try c.decode(String.self, forKey: .type) + guard type == Body.messageType else { + throw RealtimeDecodingError.malformed("expected envelope.type '\(Body.messageType)', got '\(type)'") + } + id = try RealtimeValidation.pattern( + c.decode(String.self, forKey: .id), "^[A-Za-z0-9_-]{1,64}$", field: "id") + ts = try RealtimeValidation.unixMs(c.decode(Int.self, forKey: .ts), field: "ts") + body = try c.decode(Body.self, forKey: .body) + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(Body.messageType, forKey: .type) + try c.encode(id, forKey: .id) + try c.encode(ts, forKey: .ts) + try c.encode(body, forKey: .body) + } +} + +/// One of the 14 message types (SPEC.md §4), fully decoded. `RealtimeFrame` +/// is the unit `RealtimeClient` sends/receives; a value round-trips through +/// `decode(_:)`/`encoded()` byte-for-byte in semantics (field order may +/// differ, JSON does not guarantee it). +public enum RealtimeFrame: Sendable, Equatable { + case hello(Envelope) + case resume(Envelope) + case snapshot(Envelope) + case delta(Envelope) + case ack(Envelope) + case taskEvent(Envelope) + case messageEvent(Envelope) + case metricFrame(Envelope) + case configEvent(Envelope) + case subscribe(Envelope) + case unsubscribe(Envelope) + case interestRenew(Envelope) + case command(Envelope) + case error(Envelope) + + private struct TypePeek: Decodable { let type: String } + + /// Decodes one JSON frame (one WebSocket text frame carries exactly one + /// envelope, SPEC.md §11). Throws `RealtimeDecodingError.unknownType` + /// for a `type` outside the 14 known ones (§15: a receiver MUST ignore + /// this, not treat it as malformed) and `.malformed` for every other + /// schema/sequence violation. + public static func decode(_ data: Data) throws -> RealtimeFrame { + let decoder = JSONDecoder() + let peek: TypePeek + do { + peek = try decoder.decode(TypePeek.self, from: data) + } catch { + throw RealtimeDecodingError.malformed("envelope missing/invalid 'type': \(error)") + } + switch peek.type { + case HelloBody.messageType: return .hello(try decoder.decode(Envelope.self, from: data)) + case ResumeBody.messageType: return .resume(try decoder.decode(Envelope.self, from: data)) + case SnapshotBody.messageType: return .snapshot(try decoder.decode(Envelope.self, from: data)) + case DeltaBody.messageType: return .delta(try decoder.decode(Envelope.self, from: data)) + case AckBody.messageType: return .ack(try decoder.decode(Envelope.self, from: data)) + case TaskEventBody.messageType: return .taskEvent(try decoder.decode(Envelope.self, from: data)) + case MessageEventBody.messageType: return .messageEvent(try decoder.decode(Envelope.self, from: data)) + case MetricFrameBody.messageType: return .metricFrame(try decoder.decode(Envelope.self, from: data)) + case ConfigEventBody.messageType: return .configEvent(try decoder.decode(Envelope.self, from: data)) + case SubscribeBody.messageType: return .subscribe(try decoder.decode(Envelope.self, from: data)) + case UnsubscribeBody.messageType: return .unsubscribe(try decoder.decode(Envelope.self, from: data)) + case InterestRenewBody.messageType: return .interestRenew(try decoder.decode(Envelope.self, from: data)) + case CommandBody.messageType: return .command(try decoder.decode(Envelope.self, from: data)) + case ErrorBody.messageType: return .error(try decoder.decode(Envelope.self, from: data)) + default: + throw RealtimeDecodingError.unknownType(peek.type) + } + } + + public func encoded() throws -> Data { + let encoder = JSONEncoder() + switch self { + case .hello(let e): return try encoder.encode(e) + case .resume(let e): return try encoder.encode(e) + case .snapshot(let e): return try encoder.encode(e) + case .delta(let e): return try encoder.encode(e) + case .ack(let e): return try encoder.encode(e) + case .taskEvent(let e): return try encoder.encode(e) + case .messageEvent(let e): return try encoder.encode(e) + case .metricFrame(let e): return try encoder.encode(e) + case .configEvent(let e): return try encoder.encode(e) + case .subscribe(let e): return try encoder.encode(e) + case .unsubscribe(let e): return try encoder.encode(e) + case .interestRenew(let e): return try encoder.encode(e) + case .command(let e): return try encoder.encode(e) + case .error(let e): return try encoder.encode(e) + } + } + + /// The envelope `id` shared by every case, for logging/`in_reply_to` + /// correlation. + public var envelopeID: String { + switch self { + case .hello(let e): e.id + case .resume(let e): e.id + case .snapshot(let e): e.id + case .delta(let e): e.id + case .ack(let e): e.id + case .taskEvent(let e): e.id + case .messageEvent(let e): e.id + case .metricFrame(let e): e.id + case .configEvent(let e): e.id + case .subscribe(let e): e.id + case .unsubscribe(let e): e.id + case .interestRenew(let e): e.id + case .command(let e): e.id + case .error(let e): e.id + } + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift new file mode 100644 index 00000000..5c407a03 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeResumeGate.swift @@ -0,0 +1,140 @@ +import Foundation + +/// The pure decision logic behind SPEC.md §6.2/§6.3's resume/delta/snapshot +/// gating and §6.4's folding, extracted from `RealtimeClient` so it has +/// exactly one implementation shared by the live client and by tests that +/// drive it directly with decoded fixtures — no WebSocket transport +/// required. `RealtimeClient` is the only thing that owns a socket; this +/// type only ever sees already-decoded `RealtimeFrame` bodies and reports +/// what happened, never performs I/O itself. +public struct RealtimeResumeGate: Sendable, Equatable { + public private(set) var state: SpaceState + /// True once this connection has received a snapshot-or-delta reply to + /// its most recent `resume` (SPEC.md §6.2/§6.3) — i.e. it may now accept + /// live deltas via the ordinary `from_revision` comparison. + public private(set) var deltaEligible = false + /// Non-nil while a resume reply (initial, or a gap-triggered re-resume) + /// is outstanding; holds the `last_revision` that was requested, so a + /// racing stray delta (one that doesn't match) can be told apart from + /// the actual reply. + public private(set) var awaitingResumeReply: Int? + private var pendingSnapshotChunks: [SnapshotBody] = [] + + public init(state: SpaceState = SpaceState()) { + self.state = state + } + + public enum Outcome: Sendable, Equatable { + /// The frame was folded into `state` (a snapshot final chunk, a + /// delta that matched, or the resume reply itself). + case applied + /// Silently ignored per SPEC.md §6.3: a pre-reply delta, a stale + /// `from_revision < C` delta, or a stray that doesn't match an + /// outstanding resume request. + case discarded + /// `from_revision > C` on an already-eligible connection: the + /// caller MUST send a fresh `resume{last_revision: resendLastRevision}`. + case gapDetected(resendLastRevision: Int) + /// A non-final snapshot chunk was buffered; nothing applied yet. + case snapshotChunkBuffered + /// The peer violated §6.2's chunk sequencing (SPEC.md: "a viewer + /// that receives any non-ping/pong envelope between chunks MUST + /// treat it as malformed"; the same applies to a genuinely + /// out-of-sequence chunk). The caller should close and reconnect. + case malformedSequence(String) + } + + /// True while a chunked snapshot is being reassembled (a non-final + /// chunk has arrived and the `final` one has not). Per SPEC.md §6.2 the + /// server MUST NOT interleave any other envelope in this window; the + /// caller must route any non-snapshot frame it receives during it to + /// `interleavedFrameDuringSnapshot()` instead of processing it. + public var isSnapshotInFlight: Bool { !pendingSnapshotChunks.isEmpty } + + /// SPEC.md §6.2: "a viewer that receives any non-`ping`/`pong` envelope + /// between chunks MUST treat it as a malformed sequence and MAY close + /// the connection and reconnect." Discards the buffered chunks (they + /// can never be completed by a conformant continuation now) and reports + /// the malformed sequence; the caller closes and reconnects, and the + /// next connection resumes afresh. + public mutating func interleavedFrameDuringSnapshot() -> Outcome { + pendingSnapshotChunks = [] + return .malformedSequence("non-snapshot envelope interleaved during a chunked snapshot") + } + + /// Call once, immediately after sending `resume{last_revision: N}` — + /// including the very first resume on a fresh connection and any + /// gap-triggered re-resume. + public mutating func beganResume(lastRevision: Int) { + awaitingResumeReply = lastRevision + deltaEligible = false + pendingSnapshotChunks = [] + } + + public mutating func receiveSnapshotChunk(_ chunk: SnapshotBody) -> Outcome { + if let first = pendingSnapshotChunks.first { + guard chunk.revision == first.revision, chunk.part == pendingSnapshotChunks.count + 1 else { + return .malformedSequence( + "expected revision \(first.revision) part \(pendingSnapshotChunks.count + 1), got revision \(chunk.revision) part \(chunk.part)") + } + } else { + guard chunk.part == 1 else { + return .malformedSequence("snapshot chunk sequence must start at part 1, got \(chunk.part)") + } + } + pendingSnapshotChunks.append(chunk) + guard chunk.final else { return .snapshotChunkBuffered } + + let aggregated = SnapshotBody( + revision: chunk.revision, part: 1, final: true, + tasks: pendingSnapshotChunks.flatMap(\.tasks), + metrics: pendingSnapshotChunks.flatMap(\.metrics), + messages: pendingSnapshotChunks.flatMap(\.messages), + automations: pendingSnapshotChunks.flatMap(\.automations)) + pendingSnapshotChunks = [] + state.apply(snapshot: aggregated) + awaitingResumeReply = nil + deltaEligible = true + return .applied + } + + public mutating func receiveDelta(_ body: DeltaBody) -> Outcome { + // §6.2: nothing but further snapshot chunks may arrive while a + // chunked snapshot is being reassembled. + if isSnapshotInFlight { return interleavedFrameDuringSnapshot() } + if let requested = awaitingResumeReply { + guard body.fromRevision == requested else { return .discarded } + state.apply(delta: body) + awaitingResumeReply = nil + deltaEligible = true + return .applied + } + guard deltaEligible else { return .discarded } + + if body.fromRevision < state.revision { + return .discarded + } else if body.fromRevision == state.revision { + state.apply(delta: body) + return .applied + } else { + let resendFrom = state.revision + awaitingResumeReply = resendFrom + deltaEligible = false + return .gapDetected(resendLastRevision: resendFrom) + } + } + + public mutating func receiveMetricFrame(_ body: MetricFrameBody) { + state.apply(metricFrame: body) + } + + /// `true` if this error is the reply to our outstanding resume + /// (`revision_unavailable`, SPEC.md §6.3's decision table): the caller + /// MUST retry with `resume{last_revision: 0}`. The gate stays in the + /// "awaiting reply" state (now for revision 0) either way. + public mutating func receiveErrorAnswersOutstandingResume(_ body: ErrorBody) -> Bool { + guard awaitingResumeReply != nil, body.code == .revisionUnavailable else { return false } + awaitingResumeReply = 0 + return true + } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift new file mode 100644 index 00000000..0ddabb24 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeUIBridge.swift @@ -0,0 +1,70 @@ +import Foundation + +// Adapters from the realtime wire model (RTTaskState/RTMetricSample/...) to +// the existing poll-era UI models (TaskState/MetricState/EventLogEntry/ +// AutomationInfo), so SitrepApp's views keep reading the same types whether +// their data came from an HTTP snapshot or from `SpaceState`. This mirrors +// the existing `SnapshotMetric.state`/`SnapshotMessage.state` adapters in +// Models.swift, which do the equivalent job for the REST `/v2/snapshot` +// response shape. + +private extension Date { + init(msSinceEpoch ms: Int) { self.init(timeIntervalSince1970: Double(ms) / 1000) } +} + +public extension RTTaskState { + var uiState: TaskState { + let status: TaskStatus + switch state { + case .running: status = .running + case .done: status = .done + case .failed: status = .failed + } + var t = TaskState( + sourceID: taskID, title: title ?? "", status: status, percent: percent, step: step, + updatedAt: Date(msSinceEpoch: updatedAt)) + t.icon = display?.icon + t.tint = display?.tint + t.template = display?.template + return t + } +} + +public extension RTMetricSample { + var uiState: MetricState { + var m = MetricState(key: metricID, value: value, label: label, updatedAt: Date(msSinceEpoch: ts)) + m.icon = display?.icon + m.tint = display?.tint + m.template = display?.template + m.target = target + m.min = min + m.max = max + m.alertAbove = alertAbove + m.alertBelow = alertBelow + return m + } +} + +public extension RTMessageRecord { + var uiEvent: EventLogEntry { + EventLogEntry(serverID: messageID, text: text, level: level.rawValue, ts: Date(msSinceEpoch: occurredAt), source: nil) + } +} + +public extension RTAutomationState { + var uiInfo: AutomationInfo { + AutomationInfo( + id: automationID, name: name, + executor: .init(kind: executorKind), + schedule: .init(kind: "interval", everySeconds: scheduleEverySeconds), + state: state == .active ? .active : .paused, + lastRun: lastRunAt.map { Date(msSinceEpoch: $0) }) + } +} + +public extension SpaceState { + var uiTasks: [TaskState] { tasks.values.map(\.uiState).sorted { $0.updatedAt > $1.updatedAt } } + var uiMetrics: [MetricState] { metrics.values.map(\.uiState).sorted { $0.key < $1.key } } + var uiEvents: [EventLogEntry] { messages.map(\.uiEvent) } + var uiAutomations: [AutomationInfo] { automations.values.map(\.uiInfo).sorted { $0.name < $1.name } } +} diff --git a/apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift b/apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift new file mode 100644 index 00000000..fcaffa86 --- /dev/null +++ b/apple/SitrepKit/Sources/SitrepKit/Realtime/SpaceState.swift @@ -0,0 +1,144 @@ +import Foundation + +/// Unified realtime read model for one space: `tasks`/`messages`/ +/// `automations`/`metrics` plus the local `revision` (`C` in SPEC.md §6.3). +/// +/// Two convergent paths build this exact same shape (§6.4): `apply(snapshot:)` +/// replaces it wholesale from a resume reply's folded state, and +/// `apply(delta:)` folds reliable events into it one at a time. Both MUST +/// reach identical `tasks`/`automations` state for the same event history — +/// that equivalence is what `SpaceStateFoldingTests` asserts. +/// +/// This is a plain value type with no transport/connection knowledge; +/// `RealtimeClient` owns the connection state machine and mutates one of +/// these as frames arrive. +public struct SpaceState: Sendable, Equatable { + /// Local current revision (`C`). 0 before the first resume reply. + public private(set) var revision: Int + + public private(set) var tasks: [String: RTTaskState] = [:] + public private(set) var automations: [String: RTAutomationState] = [:] + /// Oldest-first. Capped locally (`messageWindow`) so a very long-lived + /// connection's delta-appended history doesn't grow unbounded — the + /// server's own snapshot window (SPEC.md §6.4 recommends N=200) bounds + /// what a fresh snapshot carries, but nothing bounds how many + /// `message.event`s a live connection can accumulate over days, so the + /// client enforces its own cap defensively. This is a client-local + /// memory bound, not a protocol requirement. + public private(set) var messages: [RTMessageRecord] = [] + /// Latest known sample per metric_id. `metric.frame` is best-effort and + /// out-of-band from `revision` (§12); ordering is enforced purely by + /// `ts`, independent of arrival or revision order. + public private(set) var metrics: [String: RTMetricSample] = [:] + + public var messageWindow: Int = 200 + + public init(revision: Int = 0) { + self.revision = revision + } + + /// Replaces all four collections wholesale from an aggregated (already + /// chunk-reassembled) snapshot, and sets `revision` to the snapshot's + /// revision. This is always safe to call — a snapshot is the + /// authoritative folded state at its revision, never a partial update. + public mutating func apply(snapshot: SnapshotBody) { + revision = snapshot.revision + tasks = Dictionary(uniqueKeysWithValues: snapshot.tasks.map { ($0.taskID, $0) }) + automations = Dictionary(uniqueKeysWithValues: snapshot.automations.map { ($0.automationID, $0) }) + messages = Array(snapshot.messages.suffix(messageWindow)) + metrics = Dictionary(uniqueKeysWithValues: snapshot.metrics.map { ($0.metricID, $0) }) + } + + /// Folds one delta's events into state and advances `revision` to + /// `delta.toRevision`. The caller (`RealtimeClient`) is responsible for + /// the §6.3 gating decision (apply only when `fromRevision == revision`) + /// — this method unconditionally applies, so tests and the client share + /// one folding implementation without duplicating the gate. + public mutating func apply(delta: DeltaBody) { + for event in delta.events { + switch event { + case .taskEvent(let e): foldTaskEvent(e) + case .messageEvent(let e): foldMessageEvent(e) + case .configEvent(let e): foldConfigEvent(e) + case .unknown: break // §15-style forward-compat: ignore what we don't recognize. + } + } + revision = delta.toRevision + } + + /// `metric.frame` is out-of-band from `revision` (§12): apply + /// independent of delta/snapshot state, discarding any sample whose + /// `ts` does not strictly advance past the last-applied `ts` for that + /// `metric_id`. + public mutating func apply(metricFrame: MetricFrameBody) { + for sample in metricFrame.metrics { + if let existing = metrics[sample.metricID], sample.ts <= existing.ts { continue } + metrics[sample.metricID] = sample + } + } + + // MARK: - §6.4 deterministic folding + + private mutating func foldTaskEvent(_ e: TaskEventBody) { + var state = tasks[e.taskID] ?? RTTaskState( + taskID: e.taskID, deviceID: e.deviceID, title: nil, state: .running, + percent: nil, step: nil, message: nil, updatedAt: e.occurredAt, display: nil) + + switch e.kind { + case .started, .progress, .step: + state.state = .running + case .done: + state.state = .done + case .failed: + state.state = .failed + } + + if let title = e.title, !title.isEmpty { + state.title = title + } + + switch e.kind { + case .started, .progress, .step: + if let percent = e.percent { state.percent = percent } + if let step = e.step { state.step = step } + case .done: + state.percent = 100 + state.step = nil + case .failed: + // percent keeps its last running value; step cleared. + state.step = nil + } + + switch e.kind { + case .done, .failed: + if let message = e.message { state.message = message } + case .started, .progress, .step: + break // message stays absent while running (never set here). + } + + if let display = e.display { state.display = display } + state.updatedAt = e.occurredAt + + tasks[e.taskID] = state + } + + private mutating func foldMessageEvent(_ e: MessageEventBody) { + let record = RTMessageRecord( + messageID: e.messageID, deviceID: e.deviceID, level: e.level, text: e.text, occurredAt: e.occurredAt) + messages.append(record) + if messages.count > messageWindow { + messages.removeFirst(messages.count - messageWindow) + } + } + + private mutating func foldConfigEvent(_ e: ConfigEventBody) { + switch e.kind { + case .upserted: + if let automation = e.automation { + automations[e.automationID] = automation + } + case .removed: + automations.removeValue(forKey: e.automationID) + } + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift index 8a4cc314..af1b1a60 100644 --- a/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift +++ b/apple/SitrepKit/Tests/SitrepKitTests/ModelsTests.swift @@ -13,4 +13,40 @@ final class ModelsTests: XCTestCase { XCTAssertEqual(task.status, .running) XCTAssertEqual(task.percent, 45) } + + // MARK: - realtime_enabled (P0 capability gate input) + + private func decodeSnapshot(extraTopLevelField: String) throws -> SpaceSnapshot { + let json = """ + { + "version": 2, + "generated_at": "2026-07-17T12:00:00Z", + "presence": {}, + "tasks": [], + "metrics": [], + "messages": [], + "automations": []\(extraTopLevelField) + } + """ + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return try decoder.decode(SpaceSnapshot.self, from: Data(json.utf8)) + } + + func testSnapshotDecodesRealtimeEnabledTrue() throws { + let snapshot = try decodeSnapshot(extraTopLevelField: #", "realtime_enabled": true"#) + XCTAssertTrue(snapshot.realtimeEnabled) + } + + func testSnapshotDecodesRealtimeEnabledFalse() throws { + let snapshot = try decodeSnapshot(extraTopLevelField: #", "realtime_enabled": false"#) + XCTAssertFalse(snapshot.realtimeEnabled) + } + + /// Old servers never send this field — absence must decode to `false`, + /// never inferred from anything else (P0 fix). + func testSnapshotDecodesRealtimeEnabledAbsentAsFalse() throws { + let snapshot = try decodeSnapshot(extraTopLevelField: "") + XCTAssertFalse(snapshot.realtimeEnabled) + } } diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift new file mode 100644 index 00000000..fb70a52f --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift @@ -0,0 +1,43 @@ +import Foundation + +/// Loads fixtures directly from `proto/realtime/fixtures/` (the frozen +/// protocol repository, never copied or duplicated into `apple/`) using +/// this test file's own `#filePath` to locate the repo root. This keeps the +/// Swift test suite testing the SAME fixtures the spec's own +/// `tools/validate.js` checks, with zero drift risk from a stale copy. +enum FixtureLoader { + /// `.../apple/SitrepKit/Tests/SitrepKitTests/Realtime/FixtureLoader.swift` + /// → walk up to the repo root. + static var repoRoot: URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // FixtureLoader.swift -> Realtime/ + .deletingLastPathComponent() // Realtime/ -> SitrepKitTests/ + .deletingLastPathComponent() // SitrepKitTests/ -> Tests/ + .deletingLastPathComponent() // Tests/ -> SitrepKit/ + .deletingLastPathComponent() // SitrepKit/ -> apple/ + .deletingLastPathComponent() // apple/ -> repo root + } + + static var fixturesRoot: URL { + repoRoot.appendingPathComponent("proto/realtime/fixtures") + } + + static func data(_ relativePath: String) throws -> Data { + try Data(contentsOf: fixturesRoot.appendingPathComponent(relativePath)) + } + + static func names(in subdirectory: String) throws -> [String] { + let dir = fixturesRoot.appendingPathComponent(subdirectory) + return try FileManager.default.contentsOfDirectory(atPath: dir.path) + .filter { $0.hasSuffix(".json") } + .sorted() + } + + /// Ordered `.json` files in a scenario directory (excludes `README.md`), + /// sorted by filename — the scenarios are numbered `01-...json`, + /// `02-...json`, so lexicographic order is message order. + static func scenarioFiles(_ scenario: String) throws -> [(name: String, data: Data)] { + let names = try names(in: "scenarios/\(scenario)") + return try names.map { ($0, try data("scenarios/\(scenario)/\($0)")) } + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift new file mode 100644 index 00000000..5f90405a --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeCapabilityGateTests.swift @@ -0,0 +1,77 @@ +import XCTest +@testable import SitrepKit + +/// Pure decision-logic tests for the P0 capability gate: whether a +/// RealtimeClient connection may be opened is decided solely from the most +/// recent REST refresh's `realtime_enabled`. No live socket involved — this +/// exercises `RealtimeCapabilityGate` directly. +final class RealtimeCapabilityGateTests: XCTestCase { + func testColdStartDefaultsToNotCapable() { + // Before any refresh has ever completed, the gate must already read + // as "off" — this is what makes a cold start before first refresh + // safe without any extra bookkeeping at call sites. + let gate = RealtimeCapabilityGate() + XCTAssertFalse(gate.isCapable) + } + + func testFirstRefreshTrueConnects() { + var gate = RealtimeCapabilityGate() + let action = gate.apply(refreshedCapability: true) + XCTAssertEqual(action, .connect) + XCTAssertTrue(gate.isCapable) + } + + func testRefreshFalseFromColdStartStaysOff() { + var gate = RealtimeCapabilityGate() + let action = gate.apply(refreshedCapability: false) + XCTAssertEqual(action, .none, "already off, no teardown action needed") + XCTAssertFalse(gate.isCapable) + } + + func testAbsentFieldNeverConnects() { + // Server response with no `realtime_enabled` key decodes to `false` + // upstream (see ModelsTests); feeding that in must never connect. + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: false), .none) + XCTAssertFalse(gate.isCapable) + } + + func testTrueThenFalseTearsDownOnce() { + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + XCTAssertEqual(gate.apply(refreshedCapability: false), .disconnect) + XCTAssertFalse(gate.isCapable) + } + + func testRepeatingSameValueIsANoOp() { + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + // Steady-state refreshes that keep reporting true must not re-fire + // a connect action every 30s poll / pull-to-refresh. + XCTAssertEqual(gate.apply(refreshedCapability: true), .none) + XCTAssertEqual(gate.apply(refreshedCapability: true), .none) + XCTAssertTrue(gate.isCapable) + } + + func testFalseThenTrueReconnectsAfterRollback() { + var gate = RealtimeCapabilityGate() + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + XCTAssertEqual(gate.apply(refreshedCapability: false), .disconnect) + // "no reconnect attempts until a refresh says true again" — the very + // next refresh reporting true is exactly that refresh. + XCTAssertEqual(gate.apply(refreshedCapability: true), .connect) + XCTAssertTrue(gate.isCapable) + } + + func testResetReturnsToColdStartState() { + var gate = RealtimeCapabilityGate() + _ = gate.apply(refreshedCapability: true) + XCTAssertTrue(gate.isCapable) + gate.reset() + XCTAssertFalse(gate.isCapable) + // Re-pairing to a new server must prove capability again — a stale + // `true` must not survive the reset. + XCTAssertEqual(gate.apply(refreshedCapability: false), .none) + XCTAssertFalse(gate.isCapable) + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift new file mode 100644 index 00000000..74883c40 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeGenerationTests.swift @@ -0,0 +1,78 @@ +import XCTest +@testable import SitrepKit + +/// Generation-scoped connection-resource ownership: a stale run-loop +/// generation unwinding after a stop()/start() cycle on the SAME +/// RealtimeClient instance must not tear down or replace the newer +/// generation's socket/lease/watchdog resources. No real network is used: +/// the guard itself is pure logic, and the behavioral test connects to a +/// local port that refuses immediately. +final class RealtimeGenerationTests: XCTestCase { + // MARK: - Pure guard matrix + + func testTeardownPolicyMatrix() { + // stop() — unconditional teardown regardless of current generation. + XCTAssertTrue(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: nil, current: 1)) + XCTAssertTrue(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: nil, current: 7)) + + // The owning generation tears down its own shared resources. + XCTAssertTrue(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: 3, current: 3)) + + // A stale generation (superseded by stop()+start()) must NOT touch + // the newer generation's shared resources. + XCTAssertFalse(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: 1, current: 2)) + XCTAssertFalse(ConnectionTeardownPolicy.shouldTearDownShared(requestedBy: 2, current: 5)) + } + + // MARK: - stop()/start() on the same instance + + /// The public-contract regression: after stop(), the SAME instance must + /// be restartable — the stale generation's deferred teardown (still + /// unwinding from its suspended connect attempt) must not kill the new + /// generation's run loop or connection attempt. Uses a closed local + /// port, so each connect attempt fails fast without real network. + func testStopThenStartOnSameInstanceRestarts() async throws { + let configuration = RealtimeClient.Configuration( + url: URL(string: "ws://127.0.0.1:1/v3/realtime")!, token: nil, deviceID: "iphone-test-01") + let client = RealtimeClient(configuration: configuration) + + // Generation 1 starts and leaves idle (connecting or already failed). + await client.start() + try await waitUntil("first start leaves .idle") { + await client.currentPhase != .idle + } + + // Stop, then immediately start again on the same instance — the + // window in which generation 1's connectOnce is still unwinding. + await client.stop() + await client.start() + + // Generation 2 must be alive and progressing: it leaves .idle and, + // because the port refuses, reaches .failed — proving the new run + // loop owns the cycle and was not torn down by generation 1's + // deferred cleanup. + try await waitUntil("second start leaves .idle") { + await client.currentPhase != .idle + } + try await waitUntil("second generation reaches .failed on a refused port") { + if case .failed = await client.currentPhase { return true } + return false + } + + await client.stop() + let final = await client.currentPhase + XCTAssertEqual(final, .idle) + } + + private func waitUntil( + _ what: String, timeout: Double = 5, + _ condition: @escaping () async -> Bool + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if await condition() { return } + try await Task.sleep(for: .milliseconds(25)) + } + XCTFail("timed out waiting for: \(what)") + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift new file mode 100644 index 00000000..c636bf0b --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeMessageFixtureTests.swift @@ -0,0 +1,140 @@ +import XCTest +@testable import SitrepKit + +/// Fixture-driven coverage of the Swift protocol layer against +/// `proto/realtime/fixtures/`, mirroring what `tools/validate.js` checks for +/// the TS/Go implementations: every valid fixture decodes and round-trips; +/// every invalid fixture is rejected, either by decoding (schema-equivalent +/// validation) or by the client authorization matrix for the two +/// role-tagged fixtures. +final class RealtimeMessageFixtureTests: XCTestCase { + // MARK: - valid/ + + func testAllValidFixturesDecodeAndRoundTrip() throws { + let names = try FixtureLoader.names(in: "valid") + XCTAssertFalse(names.isEmpty, "expected at least one valid fixture") + for name in names { + try assertDecodesAndRoundTrips(fixture: "valid/\(name)") + } + } + + // MARK: - scenarios/**/*.json (every message in every scenario is individually valid) + + func testAllScenarioMessagesDecodeAndRoundTrip() throws { + let scenarioDirs = try scenarioDirectoryNames() + XCTAssertFalse(scenarioDirs.isEmpty) + for dir in scenarioDirs { + for file in try FixtureLoader.scenarioFiles(dir) { + try assertDecodesAndRoundTrips(fixture: "scenarios/\(dir)/\(file.name)", data: file.data) + } + } + } + + private func assertDecodesAndRoundTrips(fixture: String, data: Data? = nil) throws { + let bytes = try data ?? FixtureLoader.data(fixture) + let frame: RealtimeFrame + do { + frame = try RealtimeFrame.decode(bytes) + } catch { + XCTFail("\(fixture): expected successful decode, got \(error)") + return + } + let reencoded = try frame.encoded() + let redecoded = try RealtimeFrame.decode(reencoded) + XCTAssertEqual(frame, redecoded, "\(fixture): round-trip changed semantics") + } + + // MARK: - invalid/ + + /// Fixtures whose invalidity is a role/authorization violation rather + /// than a schema violation: the frame body decodes fine on its own (it's + /// well-formed JSON matching its schema), but the sender's role forbids + /// sending it (SPEC.md §10.1). These use the + /// `{"sender_role": ..., "frame": {...}}` wrapper described in §16. + private static let roleTaggedInvalidFixtures: Set = [ + "role-client-hello-accept.json", + "role-viewer-command-origin-server.json", + ] + + /// The specific constraint each schema-invalid fixture must trip, as a + /// substring of the thrown error's description. Pinning the REASON (not + /// just "some error") catches regressions where a fixture starts + /// failing for the wrong reason — e.g. a decoding-order change masking + /// the intended violation with an unrelated one. + private static let expectedRejectionReasons: [String: String] = [ + "ack-neither-acked-nor-in-reply-to.json": "'acked' and/or 'in_reply_to'", + "command-pause-with-automation-id.json": "forbids automation_id", + "command-throttle-with-task-id.json": "forbids task_id and automation_id", + "command-viewer-sends-throttle.json": "requires origin 'server'", + "envelope-carries-credential-field.json": "unexpected field(s): token", + "error-missing-retryable-fatal.json": "retryable", // DecodingError.keyNotFound + "hello-offer-empty-protocol-versions.json": "protocol_versions must not be empty", + "message-event-timestamp-in-seconds.json": "occurred_at is not a Unix ms timestamp", + "metric-frame-oversized-value.json": "value exceeds max length 256", + "resume-negative-revision.json": "last_revision -1 below minimum 0", + "subscribe-unknown-topic.json": "automation", // DecodingError: no such RTTopic case + "task-event-missing-device-seq.json": "device_seq", // DecodingError.keyNotFound + "task-event-progress-missing-percent.json": "percent required when kind == 'progress'", + ] + + func testInvalidFixturesFailDecodeOrAreUnauthorized() throws { + let names = try FixtureLoader.names(in: "invalid") + XCTAssertFalse(names.isEmpty) + for name in names { + let bytes = try FixtureLoader.data("invalid/\(name)") + if Self.roleTaggedInvalidFixtures.contains(name) { + try assertRejectedByAuthorization(fixture: name, wrapperData: bytes) + } else { + guard let expectedReason = Self.expectedRejectionReasons[name] else { + XCTFail("invalid/\(name): new fixture — add its expected rejection reason to the map") + continue + } + assertFailsToDecode(fixture: name, data: bytes, reasonContaining: expectedReason) + } + } + } + + private func assertFailsToDecode(fixture: String, data: Data, reasonContaining expected: String) { + XCTAssertThrowsError(try RealtimeFrame.decode(data), "invalid/\(fixture) should fail to decode") { error in + guard error is RealtimeDecodingError || error is DecodingError else { + XCTFail("invalid/\(fixture): unexpected error type \(error)") + return + } + let description = String(describing: error) + XCTAssertTrue( + description.contains(expected), + "invalid/\(fixture): rejected for the wrong reason — expected description containing '\(expected)', got: \(description)") + } + } + + private func assertRejectedByAuthorization(fixture: String, wrapperData: Data) throws { + guard let object = try JSONSerialization.jsonObject(with: wrapperData) as? [String: Any], + let senderRoleRaw = object["sender_role"] as? String, + let senderRole = RTDeviceRole(rawValue: senderRoleRaw), + let frameObject = object["frame"] + else { + XCTFail("invalid/\(fixture): expected {sender_role, frame} wrapper") + return + } + let frameData = try JSONSerialization.data(withJSONObject: frameObject) + // The wrapped frame is schema-valid on its own — decoding it must + // succeed; the rejection is purely a role/authorization matter. + let frame = try RealtimeFrame.decode(frameData) + XCTAssertFalse( + RealtimeAuthorization.clientMaySend(frame, as: senderRole), + "invalid/\(fixture): expected role '\(senderRole)' to be disallowed from sending this frame") + } + + // MARK: - helpers + + private func scenarioDirectoryNames() throws -> [String] { + let dir = FixtureLoader.fixturesRoot.appendingPathComponent("scenarios") + return try FileManager.default.contentsOfDirectory(atPath: dir.path) + .filter { name in + var isDir: ObjCBool = false + FileManager.default.fileExists(atPath: dir.appendingPathComponent(name).path, isDirectory: &isDir) + return isDir.boolValue + } + .sorted() + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift new file mode 100644 index 00000000..7f0eb10b --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeRegressionTests.swift @@ -0,0 +1,176 @@ +import XCTest +@testable import SitrepKit + +/// Regression coverage for the adversarial-review rulings (F1/F3/F4/F5): +/// HTTP-overwrite gating while the realtime channel is live, interleaved +/// frames during chunked snapshots, incremental resume across foreground +/// cycles, and body-level leniency of the delta event wrapper. +final class RealtimeRegressionTests: XCTestCase { + // MARK: - F1: HTTP snapshot must not overwrite reliable state while live + + func testOnlyLivePhaseBlocksReliableStateOverwrite() { + XCTAssertFalse(RealtimeClient.Phase.live.allowsReliableStateOverwrite, + "while live, deltas own the reliable collections") + XCTAssertTrue(RealtimeClient.Phase.idle.allowsReliableStateOverwrite) + XCTAssertTrue(RealtimeClient.Phase.connecting.allowsReliableStateOverwrite) + XCTAssertTrue(RealtimeClient.Phase.handshaking.allowsReliableStateOverwrite) + XCTAssertTrue(RealtimeClient.Phase.subscribed.allowsReliableStateOverwrite, + "not yet delta-eligible — HTTP is still the best source") + XCTAssertTrue(RealtimeClient.Phase.failed("down").allowsReliableStateOverwrite) + } + + /// The in-flight race: an HTTP request issued while the WebSocket was + /// down must be dropped if the connection went live before the response + /// was applied. Callers implement this by re-evaluating the phase AFTER + /// the await — this test exercises that exact pattern. + func testInFlightHTTPApplyReChecksLivenessAfterAwait() async { + // Shared mutable "current phase", as AppModel's connectionPhase is. + actor PhaseBox { + var phase: RealtimeClient.Phase = .failed("ws down") + func set(_ p: RealtimeClient.Phase) { phase = p } + } + let box = PhaseBox() + var applied = false + + // Request issued while down: a pre-await check would have passed. + let allowedAtRequestTime = await box.phase.allowsReliableStateOverwrite + XCTAssertTrue(allowedAtRequestTime) + + // While the "HTTP response" is in flight, the WS recovers. + await box.set(.live) + + // Apply-time re-check (the pattern refresh() uses) must now refuse. + if await box.phase.allowsReliableStateOverwrite { + applied = true + } + XCTAssertFalse(applied, "a response landing after .recovered must not clobber live delta state") + + // And once the connection is down again, HTTP applies normally. + await box.set(.failed("ws down again")) + if await box.phase.allowsReliableStateOverwrite { + applied = true + } + XCTAssertTrue(applied) + } + + // MARK: - F3: interleaved envelope during a chunked snapshot + + func testDeltaInterleavedDuringChunkedSnapshotIsMalformed() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let part1 = SnapshotBody(revision: 10, part: 1, final: false, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(part1), .snapshotChunkBuffered) + XCTAssertTrue(gate.isSnapshotInFlight) + + // §6.2: any non-snapshot envelope between chunks is a malformed + // sequence — even a delta that would otherwise look like a valid + // resume reply. + let interleaved = DeltaBody(fromRevision: 0, toRevision: 0, events: []) + guard case .malformedSequence = gate.receiveDelta(interleaved) else { + return XCTFail("expected malformedSequence for a delta interleaved mid-snapshot") + } + XCTAssertFalse(gate.isSnapshotInFlight, "buffered chunks are discarded — they can never complete now") + XCTAssertFalse(gate.deltaEligible) + XCTAssertEqual(gate.state.revision, 0, "nothing from the aborted snapshot is applied") + } + + func testInterleavedFrameHelperReportsMalformedAndClearsBuffer() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let part1 = SnapshotBody(revision: 7, part: 1, final: false, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(part1), .snapshotChunkBuffered) + + // The client routes ANY non-snapshot frame (ack, error, metric...) + // through this helper while a snapshot is in flight. + guard case .malformedSequence = gate.interleavedFrameDuringSnapshot() else { + return XCTFail("expected malformedSequence") + } + XCTAssertFalse(gate.isSnapshotInFlight) + } + + // MARK: - F4: foreground cycles resume incrementally with preserved C + + /// Simulates background → foreground: the first "connection" catches up + /// via the scenario fixtures to revision 128; its SpaceState (with C) + /// survives in the app model; the rebuilt gate for the next connection + /// resumes with C and receives an incremental reply — never a snapshot. + func testReconnectResumesIncrementallyWithPreservedRevision() throws { + let files = try FixtureLoader.scenarioFiles("client-resume-delta") + + var firstConnection = RealtimeResumeGate(state: SpaceState(revision: 126)) + firstConnection.beganResume(lastRevision: 126) + guard case .delta(let catchup) = try RealtimeFrame.decode(files[5].data) else { return XCTFail() } + XCTAssertEqual(firstConnection.receiveDelta(catchup.body), .applied) + XCTAssertEqual(firstConnection.state.revision, 128) + + // App backgrounds: connection closes, SpaceState is kept in-process. + let preserved = firstConnection.state + + // App foregrounds: a new gate/connection seeded with the preserved + // state resumes from C = 128 and gets the "you are already current" + // empty delta — an incremental path, not a fresh snapshot. + var secondConnection = RealtimeResumeGate(state: preserved) + secondConnection.beganResume(lastRevision: preserved.revision) + let youAreCurrent = DeltaBody(fromRevision: 128, toRevision: 128, events: []) + XCTAssertEqual(secondConnection.receiveDelta(youAreCurrent), .applied) + XCTAssertTrue(secondConnection.deltaEligible) + XCTAssertEqual(secondConnection.state.revision, 128) + XCTAssertEqual(secondConnection.state.tasks, preserved.tasks, "no state was thrown away by the reconnect") + + // A subsequent live delta continues from the preserved C. + guard case .delta(let live) = try RealtimeFrame.decode(files[6].data) else { return XCTFail() } + XCTAssertEqual(secondConnection.receiveDelta(live.body), .applied) + XCTAssertEqual(secondConnection.state.revision, 129) + } + + func testRealtimeClientHonorsInitialState() async { + let configuration = RealtimeClient.Configuration( + url: URL(string: "wss://example.invalid/v3/realtime")!, token: nil, deviceID: "iphone-test-01") + var seed = SpaceState(revision: 128) + seed.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.5", ts: 1_752_482_000_000), + ])) + let client = RealtimeClient(configuration: configuration, initialState: seed) + let state = await client.currentState + XCTAssertEqual(state.revision, 128, "a rebuilt client must start from the preserved C, not 0") + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.5") + } + + // MARK: - F5: delta event wrapper tolerates unknown sibling fields + + func testDeltaEventIgnoresUnknownSiblingFields() throws { + // A future minor version may annotate delta entries with extra + // fields; SPEC.md §15's body-level tolerance means they are ignored, + // not rejected (§3's strictness covers only the envelope top level). + let json = """ + { + "type": "delta", + "id": "lenient-1", + "ts": 1752482000000, + "body": { + "from_revision": 5, + "to_revision": 6, + "events": [ + { + "event_type": "task.event", + "trace_id": "future-minor-version-annotation", + "event": { + "device_id": "mac-1", + "device_seq": 9, + "task_id": "run-1", + "kind": "started", + "occurred_at": 1752482000000, + "title": "Job" + } + } + ] + } + } + """ + let frame = try RealtimeFrame.decode(Data(json.utf8)) + guard case .delta(let env) = frame, case .taskEvent(let event) = env.body.events[0] else { + return XCTFail("expected the entry to decode as a task.event despite the unknown sibling field") + } + XCTAssertEqual(event.taskID, "run-1") + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift new file mode 100644 index 00000000..96624c67 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/RealtimeResumeGateScenarioTests.swift @@ -0,0 +1,234 @@ +import XCTest +@testable import SitrepKit + +/// Drives `RealtimeResumeGate` — the same gating logic `RealtimeClient` uses +/// — directly with the scenario fixtures under `proto/realtime/fixtures/scenarios/`, +/// feeding each directory's messages in filename order (as `tools/validate.js` +/// does) and asserting the terminal state matches each scenario's own +/// README. This exercises the viewer-relevant scenarios end to end without +/// a real WebSocket transport. +final class RealtimeResumeGateScenarioTests: XCTestCase { + /// Decodes one scenario fixture file, expecting envelope `type` to be + /// one of the given cases (a lightweight helper since scenario fixtures + /// are plain envelopes, not the `{sender_role, frame}` wrapper). + private func decode(_ data: Data) throws -> RealtimeFrame { + try RealtimeFrame.decode(data) + } + + // MARK: - client-resume-delta + + /// 126 -> (catch-up delta, 126->128) -> (live delta, 128->129). + func testClientResumeDelta() throws { + let files = try FixtureLoader.scenarioFiles("client-resume-delta") + var gate = RealtimeResumeGate(state: SpaceState(revision: 126)) + + // 01 hello offer / 02 hello accept — connection-setup, not gate concerns. + // 03 subscribe / 04 ack — lease, not gate concerns. + // 05 resume{last_revision: 126} + let resumeFrame = try decode(files[4].data) + guard case .resume(let resumeEnv) = resumeFrame else { return XCTFail("expected resume") } + XCTAssertEqual(resumeEnv.body.lastRevision, 126) + gate.beganResume(lastRevision: resumeEnv.body.lastRevision) + XCTAssertFalse(gate.deltaEligible, "must not be delta-eligible before the resume reply") + + // 06 catch-up delta 126 -> 128 (the resume reply itself) + guard case .delta(let catchupEnv) = try decode(files[5].data) else { return XCTFail("expected delta") } + XCTAssertEqual(gate.receiveDelta(catchupEnv.body), .applied) + XCTAssertTrue(gate.deltaEligible, "connection becomes delta-eligible on its first snapshot-or-delta reply") + XCTAssertEqual(gate.state.revision, 128) + XCTAssertEqual(gate.state.tasks["run-2f9a"]?.percent, 40) + XCTAssertEqual(gate.state.messages.count, 1) + + // 07 live delta 128 -> 129 + guard case .delta(let liveEnv) = try decode(files[6].data) else { return XCTFail("expected delta") } + XCTAssertEqual(gate.receiveDelta(liveEnv.body), .applied) + XCTAssertEqual(gate.state.revision, 129) + XCTAssertEqual(gate.state.tasks["run-2f9a"]?.percent, 60) + + XCTAssertEqual(files.count, 7) + } + + /// A delta that arrives before the resume reply must be discarded + /// (SPEC.md §6.3's defense-in-depth rule), never applied and never + /// advancing C. + func testClientResumeDeltaDiscardsPreReplyDelta() throws { + let files = try FixtureLoader.scenarioFiles("client-resume-delta") + var gate = RealtimeResumeGate(state: SpaceState(revision: 126)) + gate.beganResume(lastRevision: 126) + + // Simulate a (non-conformant-server) delta racing ahead of the + // reply: feed the LIVE delta (from step 07, from_revision 128) + // before the catch-up reply (step 06). + guard case .delta(let liveEnv) = try decode(files[6].data) else { return XCTFail() } + XCTAssertEqual(gate.receiveDelta(liveEnv.body), .discarded) + XCTAssertEqual(gate.state.revision, 126, "a pre-reply delta must not advance C") + XCTAssertFalse(gate.deltaEligible) + + // The real reply still applies normally afterward. + guard case .delta(let catchupEnv) = try decode(files[5].data) else { return XCTFail() } + XCTAssertEqual(gate.receiveDelta(catchupEnv.body), .applied) + XCTAssertEqual(gate.state.revision, 128) + } + + // MARK: - client-revision-gap-snapshot + + /// last_revision 40, server can't serve incrementally -> two-chunk + /// snapshot at revision 150. + func testClientRevisionGapSnapshot() throws { + let files = try FixtureLoader.scenarioFiles("client-revision-gap-snapshot") + XCTAssertEqual(files.count, 7) + var gate = RealtimeResumeGate(state: SpaceState(revision: 999)) // arbitrary stale prior state + + guard case .resume(let resumeEnv) = try decode(files[4].data) else { return XCTFail("expected resume") } + XCTAssertEqual(resumeEnv.body.lastRevision, 40) + gate.beganResume(lastRevision: resumeEnv.body.lastRevision) + + guard case .snapshot(let part1) = try decode(files[5].data) else { return XCTFail("expected snapshot part 1") } + XCTAssertEqual(part1.body.part, 1) + XCTAssertFalse(part1.body.final) + XCTAssertEqual(gate.receiveSnapshotChunk(part1.body), .snapshotChunkBuffered) + XCTAssertFalse(gate.deltaEligible, "nothing applies until the final chunk") + XCTAssertEqual(gate.state.revision, 999, "prior stale state must not change before the final chunk") + + guard case .snapshot(let part2) = try decode(files[6].data) else { return XCTFail("expected snapshot part 2") } + XCTAssertEqual(part2.body.part, 2) + XCTAssertTrue(part2.body.final) + XCTAssertEqual(gate.receiveSnapshotChunk(part2.body), .applied) + + XCTAssertTrue(gate.deltaEligible) + XCTAssertEqual(gate.state.revision, 150) + // Concatenated across both chunks. + XCTAssertEqual(gate.state.tasks.count, 1) + XCTAssertEqual(gate.state.automations.count, 1) + XCTAssertEqual(gate.state.metrics.count, 1) + XCTAssertEqual(gate.state.messages.count, 1) + + // A live delta with from_revision == 150 is now applicable. + var followUp = gate + let liveDelta = DeltaBody( + fromRevision: 150, toRevision: 151, + events: [.taskEvent(TaskEventBody( + deviceID: "mac-quintin-01", deviceSeq: 200, taskID: "run-2f9a", kind: .done, occurredAt: 1_752_483_100_000))]) + XCTAssertEqual(followUp.receiveDelta(liveDelta), .applied) + XCTAssertEqual(followUp.state.revision, 151) + } + + /// A non-conformant server interleaving something other than the next + /// chunk (or ping/pong, which never reaches the gate at all — see + /// `RealtimeClient.receiveFrame`) between snapshot chunks must be + /// treated as a malformed sequence. + func testSnapshotChunkOutOfSequenceIsMalformed() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let part1 = SnapshotBody(revision: 10, part: 1, final: false, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(part1), .snapshotChunkBuffered) + + // Wrong revision. + let wrongRevision = SnapshotBody(revision: 11, part: 2, final: true, tasks: [], metrics: [], messages: [], automations: []) + guard case .malformedSequence = gate.receiveSnapshotChunk(wrongRevision) else { + return XCTFail("expected malformed sequence for a revision change mid-snapshot") + } + } + + // MARK: - gap detection -> re-resume, and revision_unavailable -> retry with 0 + + func testGapDetectionTriggersReResume() { + var gate = RealtimeResumeGate(state: SpaceState()) + gate.beganResume(lastRevision: 0) + let empty = DeltaBody(fromRevision: 0, toRevision: 0, events: []) + XCTAssertEqual(gate.receiveDelta(empty), .applied) + XCTAssertTrue(gate.deltaEligible) + + // A delta arrives with from_revision > C: a gap. + let gapped = DeltaBody( + fromRevision: 5, toRevision: 6, + events: [.taskEvent(TaskEventBody(deviceID: "d1", deviceSeq: 1, taskID: "t1", kind: .started, occurredAt: 1_752_000_000_000))]) + guard case .gapDetected(let resendFrom) = gate.receiveDelta(gapped) else { + return XCTFail("expected a gap") + } + XCTAssertEqual(resendFrom, 0, "must resend resume with the last revision actually applied") + XCTAssertEqual(gate.state.revision, 0, "the gapped delta itself is never applied") + XCTAssertFalse(gate.deltaEligible, "not delta-eligible again until the fresh resume's reply") + + // Simulate the caller re-issuing resume and the server replying. + gate.beganResume(lastRevision: resendFrom) + let reply = DeltaBody(fromRevision: 0, toRevision: 1, + events: [.taskEvent(TaskEventBody(deviceID: "d1", deviceSeq: 1, taskID: "t1", kind: .started, occurredAt: 1_752_000_000_000))]) + XCTAssertEqual(gate.receiveDelta(reply), .applied) + XCTAssertEqual(gate.state.revision, 1) + } + + /// A stray live delta racing an outstanding re-resume (from_revision + /// doesn't match what was requested) must be discarded, not misapplied + /// as if it were the reply and not re-triggering yet another resume. + func testRacingDeltaDuringOutstandingResumeIsDiscarded() { + var gate = RealtimeResumeGate(state: SpaceState(revision: 5)) + gate.beganResume(lastRevision: 5) + let racing = DeltaBody( + fromRevision: 7, toRevision: 8, + events: [.taskEvent(TaskEventBody(deviceID: "d1", deviceSeq: 9, taskID: "t1", kind: .progress, occurredAt: 1_752_000_000_000, percent: 10))]) + XCTAssertEqual(gate.receiveDelta(racing), .discarded) + XCTAssertEqual(gate.state.revision, 5) + XCTAssertNotNil(gate.awaitingResumeReply, "still waiting for the true reply") + } + + func testRevisionUnavailableRetriesWithZero() { + var gate = RealtimeResumeGate(state: SpaceState(revision: 200)) + gate.beganResume(lastRevision: 200) + let error = ErrorBody(code: .revisionUnavailable, message: "gone", retryable: true, fatal: false) + XCTAssertTrue(gate.receiveErrorAnswersOutstandingResume(error)) + XCTAssertEqual(gate.awaitingResumeReply, 0) + + let snapshot = SnapshotBody(revision: 5, part: 1, final: true, tasks: [], metrics: [], messages: [], automations: []) + XCTAssertEqual(gate.receiveSnapshotChunk(snapshot), .applied) + XCTAssertEqual(gate.state.revision, 5) + } + + // MARK: - interest-lease-expiry (viewer-visible parts: subscribe/renew acks and the broadcast commands) + + func testInterestLeaseExpiryScenarioShapes() throws { + let files = try FixtureLoader.scenarioFiles("interest-lease-expiry") + XCTAssertEqual(files.count, 6) + + guard case .ack(let ack1) = try decode(files[1].data), let lease1 = ack1.body.lease else { + return XCTFail("expected subscribe ack with lease") + } + XCTAssertEqual(lease1.expiresAt - ack1.ts, 45000) // within [30000, 60000] of envelope ts + + guard case .command(let throttle) = try decode(files[2].data) else { return XCTFail("expected throttle command") } + XCTAssertEqual(throttle.body.origin, .server) + XCTAssertEqual(throttle.body.action, .throttle) + XCTAssertNil(throttle.body.targetDeviceID, "broadcasts to every source device in the space") + // A viewer must never be allowed to originate this itself. + XCTAssertFalse(RealtimeAuthorization.clientMaySend(.command(throttle), as: .viewer)) + + guard case .ack(let ack2) = try decode(files[4].data), let lease2 = ack2.body.lease else { + return XCTFail("expected second viewer's subscribe ack with lease") + } + XCTAssertTrue((30_000...60_000).contains(lease2.expiresAt - ack2.ts)) + + guard case .command(let resumeRate) = try decode(files[5].data) else { return XCTFail("expected resume_rate command") } + XCTAssertEqual(resumeRate.body.origin, .server) + XCTAssertEqual(resumeRate.body.action, .resumeRate) + } + + // MARK: - duplicate-connection-supersede + + func testDuplicateConnectionSupersedeScenarioShapes() throws { + let files = try FixtureLoader.scenarioFiles("duplicate-connection-supersede") + XCTAssertEqual(files.count, 5) + + guard case .hello(let offer1) = try decode(files[0].data), case .offer(let o1) = offer1.body else { + return XCTFail("expected hello offer") + } + guard case .hello(let offer2) = try decode(files[2].data), case .offer(let o2) = offer2.body else { + return XCTFail("expected second hello offer") + } + XCTAssertEqual(o1.deviceID, o2.deviceID, "same device_id opens a second connection") + + guard case .error(let superseded) = try decode(files[4].data) else { return XCTFail("expected superseded error") } + XCTAssertEqual(superseded.body.code, .superseded) + XCTAssertTrue(superseded.body.fatal) + XCTAssertFalse(superseded.body.retryable, "not retryable on the dying connection itself") + } +} diff --git a/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift new file mode 100644 index 00000000..20c86803 --- /dev/null +++ b/apple/SitrepKit/Tests/SitrepKitTests/Realtime/SpaceStateFoldingTests.swift @@ -0,0 +1,171 @@ +import XCTest +@testable import SitrepKit + +/// SPEC.md §6.4: two viewers — one that replayed every delta, one that took +/// a snapshot — MUST converge on identical task and automation state for +/// the same event history. These tests build one event history, fold it +/// two ways (delta-by-delta vs. a directly-applied snapshot equivalent to +/// what a server would have computed), and assert the two `SpaceState` +/// values agree on `tasks`/`automations`. +final class SpaceStateFoldingTests: XCTestCase { + private func taskEvent( + seq: Int, kind: RTTaskKind, occurredAt: Int, title: String? = nil, percent: Int? = nil, + step: String? = nil, message: String? = nil, display: RTDisplayHints? = nil + ) -> TaskEventBody { + TaskEventBody( + deviceID: "mac-1", deviceSeq: seq, taskID: "run-1", kind: kind, occurredAt: occurredAt, + title: title, percent: percent, step: step, message: message, display: display) + } + + private func applyAsDeltaStream(_ events: [DeltaEvent]) -> SpaceState { + var state = SpaceState() + for (index, event) in events.enumerated() { + state.apply(delta: DeltaBody(fromRevision: index, toRevision: index + 1, events: [event])) + } + return state + } + + /// started -> progress(40, "uploading") -> progress(80, no title change) -> done("all good"). + func testTaskFoldingConvergesDeltaVsSnapshot() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Nightly backup", display: RTDisplayHints(icon: "moon"))), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 40, step: "uploading")), + .taskEvent(taskEvent(seq: 3, kind: .progress, occurredAt: 3000, title: "", percent: 80)), // empty title must NOT overwrite + .taskEvent(taskEvent(seq: 4, kind: .done, occurredAt: 4000, message: "all good")), + ] + + let viaDeltas = applyAsDeltaStream(events) + + // A snapshot is the server's already-folded state at this revision; + // construct it by hand per the exact §6.4 rules and apply it + // wholesale, simulating "the other viewer took a snapshot". + let expectedFolded = RTTaskState( + taskID: "run-1", deviceID: "mac-1", title: "Nightly backup", state: .done, + percent: 100, step: nil, message: "all good", updatedAt: 4000, + display: RTDisplayHints(icon: "moon")) + var viaSnapshot = SpaceState() + viaSnapshot.apply(snapshot: SnapshotBody( + revision: 4, part: 1, final: true, tasks: [expectedFolded], metrics: [], messages: [], automations: [])) + + XCTAssertEqual(viaDeltas.tasks, viaSnapshot.tasks) + XCTAssertEqual(viaDeltas.tasks["run-1"], expectedFolded) + } + + func testTaskFoldingFailedKeepsLastPercentAndClearsStep() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 55, step: "step 2")), + .taskEvent(taskEvent(seq: 3, kind: .failed, occurredAt: 3000, message: "boom")), + ] + let state = applyAsDeltaStream(events) + let task = try! XCTUnwrap(state.tasks["run-1"]) + XCTAssertEqual(task.state, .failed) + XCTAssertEqual(task.percent, 55, "percent keeps its last running value on failure") + XCTAssertNil(task.step, "step is cleared on done/failed") + XCTAssertEqual(task.message, "boom") + } + + /// §6.4 boundary: an event WITHOUT `step` leaves the previous step + /// unchanged (absence is not a clear); only `done`/`failed` clear it. + /// Same for `title`/`percent`: absent means "unchanged". + func testFoldingEventWithoutStepKeepsPreviousStep() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 40, step: "phase 1")), + .taskEvent(taskEvent(seq: 3, kind: .progress, occurredAt: 3000, percent: 70)), // no step, no title + ] + let state = applyAsDeltaStream(events) + let task = try! XCTUnwrap(state.tasks["run-1"]) + XCTAssertEqual(task.step, "phase 1", "absent step must not clear the previous one") + XCTAssertEqual(task.title, "Job") + XCTAssertEqual(task.percent, 70) + + // ...and done DOES clear it. + let finished = applyAsDeltaStream(events + [.taskEvent(taskEvent(seq: 4, kind: .done, occurredAt: 4000))]) + XCTAssertNil(finished.tasks["run-1"]?.step) + } + + func testTaskFoldingMessageStaysAbsentWhileRunning() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job")), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 10)), + ] + let state = applyAsDeltaStream(events) + XCTAssertNil(state.tasks["run-1"]?.message) + } + + func testTaskFoldingDisplayReplacesWholesaleNotMerged() { + let events: [DeltaEvent] = [ + .taskEvent(taskEvent(seq: 1, kind: .started, occurredAt: 1000, title: "Job", display: RTDisplayHints(icon: "a", tint: "blue"))), + .taskEvent(taskEvent(seq: 2, kind: .progress, occurredAt: 2000, percent: 10, display: RTDisplayHints(tint: "red"))), + ] + let state = applyAsDeltaStream(events) + // The second event's display object wins WHOLESALE: `icon` is gone, + // not preserved from the first event. + XCTAssertEqual(state.tasks["run-1"]?.display, RTDisplayHints(tint: "red")) + } + + // MARK: - automation folding (config.event) + + func testAutomationUpsertThenRemoveConvergesDeltaVsSnapshot() { + let automationV1 = RTAutomationState( + automationID: "auto-1", name: "Disk watch", executorKind: "script", + scheduleEverySeconds: 300, state: .active, lastRunAt: nil) + let automationV2 = RTAutomationState( + automationID: "auto-1", name: "Disk watch (paused)", executorKind: "script", + scheduleEverySeconds: 600, state: .paused, lastRunAt: 5000) + + let events: [DeltaEvent] = [ + .configEvent(ConfigEventBody(kind: .upserted, automationID: "auto-1", automation: automationV1, occurredAt: 1000)), + .configEvent(ConfigEventBody(kind: .upserted, automationID: "auto-1", automation: automationV2, occurredAt: 2000)), + ] + let viaDeltas = applyAsDeltaStream(events) + XCTAssertEqual(viaDeltas.automations["auto-1"], automationV2, "upsert replaces wholesale, no per-field merge") + + let removed = events + [.configEvent(ConfigEventBody(kind: .removed, automationID: "auto-1", occurredAt: 3000))] + let viaDeltasRemoved = applyAsDeltaStream(removed) + XCTAssertNil(viaDeltasRemoved.automations["auto-1"]) + + var viaSnapshot = SpaceState() + viaSnapshot.apply(snapshot: SnapshotBody( + revision: 2, part: 1, final: true, tasks: [], metrics: [], messages: [], automations: [automationV2])) + XCTAssertEqual(viaDeltas.automations, viaSnapshot.automations) + } + + // MARK: - message.event append-only history + + func testMessageEventsAppendInRevisionOrder() { + let events: [DeltaEvent] = [ + .messageEvent(MessageEventBody(deviceID: "mac-1", deviceSeq: 1, messageID: "m1", level: .info, text: "first", occurredAt: 1000)), + .messageEvent(MessageEventBody(deviceID: "mac-1", deviceSeq: 2, messageID: "m2", level: .warn, text: "second", occurredAt: 2000)), + ] + let state = applyAsDeltaStream(events) + XCTAssertEqual(state.messages.map(\.text), ["first", "second"]) + } + + // MARK: - metric.frame ts-based discard + + func testMetricFrameDiscardsStaleOrDuplicateTimestamps() { + var state = SpaceState() + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.5", ts: 2000), + ])) + // Same ts again: discarded (ts <= last-applied ts). + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.9", ts: 2000), + ])) + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.5") + + // Earlier ts: discarded. + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.1", ts: 1000), + ])) + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.5") + + // Later ts: applied. + state.apply(metricFrame: MetricFrameBody(deviceID: "mac-1", metrics: [ + RTMetricSample(metricID: "cpu.load", value: "0.7", ts: 3000), + ])) + XCTAssertEqual(state.metrics["cpu.load"]?.value, "0.7") + } +} diff --git a/daemon/cmd/sitrep/agent.go b/daemon/cmd/sitrep/agent.go index a0b4fae1..23b29206 100644 --- a/daemon/cmd/sitrep/agent.go +++ b/daemon/cmd/sitrep/agent.go @@ -6,7 +6,10 @@ import ( "time" "github.com/QuintinShaw/sitrep/daemon/internal/api" + "github.com/QuintinShaw/sitrep/daemon/internal/config" "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" "github.com/QuintinShaw/sitrep/daemon/internal/runner" "github.com/QuintinShaw/sitrep/daemon/internal/uplink" ) @@ -23,6 +26,16 @@ func cmdAgent(args []string) { } fmt.Fprintf(os.Stderr, "sitrep agent: scheduling automations from %s\n", client.Server) + // Realtime uplink: opt-in feature flag (SITREP_REALTIME=1 or + // config.json's realtime_enabled). One shared connection for the + // whole resident agent process, mirroring the protocol's "at most one + // connection per device" model (proto/realtime/SPEC.md section 9.4) — + // unlike the one-shot `sitrep run` CLI, this process lives long + // enough for a persistent WebSocket connection to make sense. Default + // (flag off) leaves every automation run on the existing HTTP-only + // path, byte-for-byte as before this feature existed. + rt := newAgentRealtimeUplink() + lastRun := map[string]time.Time{} running := map[string]bool{} finished := make(chan string, 32) @@ -72,7 +85,7 @@ func cmdAgent(args []string) { lastRun[automation.ID] = time.Now() running[automation.ID] = true go func(automation api.Automation) { - runAutomation(client, automation) + runAutomation(client, automation, rt) finished <- automation.ID }(automation) } @@ -81,11 +94,47 @@ func cmdAgent(args []string) { } } +// newAgentRealtimeUplink builds the shared realtime client for the resident +// agent process when config.RealtimeEnabled is set, or returns nil (the +// safe, fully-backward-compatible default). A nil *rtclient.Client is +// nil-safe everywhere it's passed (uplink.Config.Realtime nil disables the +// feature entirely, same as today). +func newAgentRealtimeUplink() *rtclient.Client { + cfg := config.Load() + if !cfg.RealtimeEnabled { + return nil + } + if cfg.Server == "" || cfg.DeviceID == "" || cfg.Space == "" { + fmt.Fprintln(os.Stderr, "sitrep agent: realtime uplink enabled but server/device_id/space is not configured; staying on HTTP ingest") + return nil + } + store, err := outbox.OpenWithMaxRows(config.RealtimeOutboxPath(), cfg.OutboxMaxRows) + if err != nil { + fmt.Fprintf(os.Stderr, "sitrep agent: realtime outbox: %v; staying on HTTP ingest\n", err) + return nil + } + url := cfg.RealtimeURLFor() + fmt.Fprintf(os.Stderr, "sitrep agent: realtime uplink enabled (%s)\n", url) + return rtclient.New(rtclient.Config{ + URL: url, + Token: cfg.Token, + DeviceID: cfg.DeviceID, + Space: cfg.Space, + Logf: func(format string, args ...any) { + fmt.Fprintf(os.Stderr, "sitrep agent: realtime: "+format+"\n", args...) + }, + Outbox: store, + }) +} + // runAutomation executes one round with a stable automation identity. Task // lifecycle belongs to bounded `sitrep run`; scheduled work emits metrics and -// messages. -func runAutomation(client *api.Client, automation api.Automation) { - up := uplink.New(uplinkConfig("")) +// messages. rt is the shared realtime uplink (nil unless the feature flag +// is on). +func runAutomation(client *api.Client, automation api.Automation, rt *rtclient.Client) { + cfg := uplinkConfig("") + cfg.Realtime = rt + up := uplink.New(cfg) defer up.Close() sourceID := "a" + automation.ID emitAll := runner.MakeEmitter(up, sourceID, os.Getenv("SITREP_DEBUG") != "") diff --git a/daemon/go.mod b/daemon/go.mod index b1a7e47a..003711f8 100644 --- a/daemon/go.mod +++ b/daemon/go.mod @@ -1,3 +1,21 @@ module github.com/QuintinShaw/sitrep/daemon go 1.26 + +require ( + github.com/coder/websocket v1.8.15 + modernc.org/sqlite v1.39.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/sys v0.34.0 // indirect + modernc.org/libc v1.66.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/daemon/go.sum b/daemon/go.sum new file mode 100644 index 00000000..56481fba --- /dev/null +++ b/daemon/go.sum @@ -0,0 +1,51 @@ +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= +modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= +modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= +modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/daemon/internal/config/config.go b/daemon/internal/config/config.go index 8cf16019..49934402 100644 --- a/daemon/internal/config/config.go +++ b/daemon/internal/config/config.go @@ -7,16 +7,39 @@ import ( "encoding/json" "os" "path/filepath" + "strconv" + "strings" ) type Config struct { Server string `json:"server"` Token string `json:"token,omitempty"` // source role: daemon/CLI writes DeviceID string `json:"device_id,omitempty"` - Space string `json:"space,omitempty"` + Space string `json:"space,omitempty"` // ViewerToken is a legacy field for read-side clients; owner tokens // cover both directions in the space model. ViewerToken string `json:"viewer_token,omitempty"` + + // RealtimeEnabled opts into the realtime WebSocket uplink + // (proto/realtime/SPEC.md) for reliable task/message events and + // metric frames, in place of the HTTP /v2/ingest batch for those + // event kinds. Defaults to false: HTTP ingest remains the default + // path until this is explicitly turned on. Overridable with the + // SITREP_REALTIME env var (1/true/yes). + RealtimeEnabled bool `json:"realtime_enabled,omitempty"` + // RealtimeURL overrides the wss:///v3/realtime endpoint this + // package derives from Server by default. Overridable with + // SITREP_REALTIME_URL. + RealtimeURL string `json:"realtime_url,omitempty"` + + // OutboxMaxRows bounds the local realtime outbox's row count (see + // internal/realtime/outbox.DefaultMaxRows) — the point at which further + // reliable events must wait for local backpressure rather than being + // enqueued, because the outbox is the sole authority for realtime + // delivery while RealtimeEnabled is on (no legacy-HTTP fallback for + // those events; see internal/uplink.routeToRealtime). Zero/unset uses + // outbox.DefaultMaxRows. Overridable with SITREP_OUTBOX_MAX_ROWS. + OutboxMaxRows int `json:"outbox_max_rows,omitempty"` } func Path() string { @@ -27,6 +50,16 @@ func Path() string { return filepath.Join(home, ".config", "sitrep", "config.json") } +// RealtimeOutboxPath is where the realtime uplink's local SQLite outbox +// (internal/realtime/outbox) lives, alongside the existing config file. +func RealtimeOutboxPath() string { + dir := filepath.Dir(Path()) + if dir == "" || dir == "." { + return "realtime-outbox.db" + } + return filepath.Join(dir, "realtime-outbox.db") +} + func Load() Config { var cfg Config if data, err := os.ReadFile(Path()); err == nil { @@ -38,9 +71,41 @@ func Load() Config { } else if v := os.Getenv("SITREP_TOKEN"); v != "" { cfg.Token = v } + if v := os.Getenv("SITREP_REALTIME"); v != "" { + cfg.RealtimeEnabled = v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") + } + if v := os.Getenv("SITREP_REALTIME_URL"); v != "" { + cfg.RealtimeURL = v + } + if v := os.Getenv("SITREP_OUTBOX_MAX_ROWS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + cfg.OutboxMaxRows = n + } + } return cfg } +// RealtimeURLFor derives the wss://.../v3/realtime endpoint from cfg.Server +// (http -> ws, https -> wss), unless cfg.RealtimeURL explicitly overrides +// it. The realtime protocol (proto/realtime/SPEC.md) is transport-agnostic +// and does not fix a URL path; "/v3/realtime" is the path agreed across +// implementation lines (the server exposes it there, and the Apple client +// targets the same), distinct from the existing "/v2/ingest" and +// "/v2/automations" HTTP routes. +func (cfg Config) RealtimeURLFor() string { + if cfg.RealtimeURL != "" { + return cfg.RealtimeURL + } + u := cfg.Server + switch { + case strings.HasPrefix(u, "https://"): + u = "wss://" + strings.TrimPrefix(u, "https://") + case strings.HasPrefix(u, "http://"): + u = "ws://" + strings.TrimPrefix(u, "http://") + } + return strings.TrimRight(u, "/") + "/v3/realtime" +} + func Save(cfg Config) error { path := Path() if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { diff --git a/daemon/internal/config/config_test.go b/daemon/internal/config/config_test.go new file mode 100644 index 00000000..f2aea441 --- /dev/null +++ b/daemon/internal/config/config_test.go @@ -0,0 +1,71 @@ +package config + +import "testing" + +// TestLoadOutboxMaxRows pins the SITREP_OUTBOX_MAX_ROWS wiring (the P0 +// review's "cap isn't wired to config" note): Load must surface it as +// Config.OutboxMaxRows, defaulting to 0 (meaning "use +// outbox.DefaultMaxRows") when unset or invalid. +func TestLoadOutboxMaxRows(t *testing.T) { + // HOME points at an empty temp dir so Load never reads a real user + // config.json out from under this test. + t.Setenv("HOME", t.TempDir()) + + cases := []struct { + name string + env string + want int + }{ + {name: "unset defaults to zero (outbox.DefaultMaxRows)", env: "", want: 0}, + {name: "explicit value wins", env: "12345", want: 12345}, + {name: "non-numeric is ignored", env: "not-a-number", want: 0}, + {name: "zero is ignored (not a valid cap)", env: "0", want: 0}, + {name: "negative is ignored", env: "-5", want: 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Setenv("SITREP_OUTBOX_MAX_ROWS", c.env) + if got := Load().OutboxMaxRows; got != c.want { + t.Fatalf("Load().OutboxMaxRows = %d, want %d", got, c.want) + } + }) + } +} + +// TestRealtimeURLFor pins the agreed cross-implementation realtime +// endpoint path (/v3/realtime) and the http(s) -> ws(s) scheme mapping. +func TestRealtimeURLFor(t *testing.T) { + cases := []struct { + name string + cfg Config + want string + }{ + { + name: "https becomes wss with v3 path", + cfg: Config{Server: "https://sitrep.example.com"}, + want: "wss://sitrep.example.com/v3/realtime", + }, + { + name: "http becomes ws with v3 path", + cfg: Config{Server: "http://localhost:8787"}, + want: "ws://localhost:8787/v3/realtime", + }, + { + name: "trailing slash on server is normalized", + cfg: Config{Server: "https://sitrep.example.com/"}, + want: "wss://sitrep.example.com/v3/realtime", + }, + { + name: "explicit RealtimeURL wins verbatim", + cfg: Config{Server: "https://sitrep.example.com", RealtimeURL: "wss://other.example.com/custom"}, + want: "wss://other.example.com/custom", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.cfg.RealtimeURLFor(); got != c.want { + t.Fatalf("RealtimeURLFor() = %q, want %q", got, c.want) + } + }) + } +} diff --git a/daemon/internal/realtime/client/backoff.go b/daemon/internal/realtime/client/backoff.go new file mode 100644 index 00000000..5bc6e836 --- /dev/null +++ b/daemon/internal/realtime/client/backoff.go @@ -0,0 +1,45 @@ +package client + +import "time" + +// Backoff computes the delay before reconnect attempt N (0-based: attempt 0 +// is the wait before the first retry after the first connection failure). +// The base delay doubles per attempt and is capped at max, then jittered by +// +/- jitterFrac (proportionally) using randFn, which must return a value +// in [0,1). This is a pure function — no sleeping, no shared state — so it +// can be tested by simply calling it with a fixed randFn and asserting on +// the numbers it returns. +func Backoff(attempt int, base, max time.Duration, jitterFrac float64, randFn func() float64) time.Duration { + if attempt < 0 { + attempt = 0 + } + if base <= 0 { + base = time.Second + } + if max < base { + max = base + } + d := base + for i := 0; i < attempt && d < max; i++ { + d *= 2 + if d > max { + d = max + } + } + if d > max { + d = max + } + if jitterFrac <= 0 || randFn == nil { + return d + } + r := randFn() + if r < 0 { + r = 0 + } + if r > 1 { + r = 1 + } + // Map r in [0,1) to a factor in [1-jitterFrac, 1+jitterFrac]. + factor := 1 + (r*2-1)*jitterFrac + return time.Duration(float64(d) * factor) +} diff --git a/daemon/internal/realtime/client/backoff_test.go b/daemon/internal/realtime/client/backoff_test.go new file mode 100644 index 00000000..45db2a5a --- /dev/null +++ b/daemon/internal/realtime/client/backoff_test.go @@ -0,0 +1,85 @@ +package client + +import ( + "testing" + "time" +) + +func fixedRand(v float64) func() float64 { return func() float64 { return v } } + +func TestBackoffDoublesAndCaps(t *testing.T) { + base := time.Second + max := 60 * time.Second + mid := fixedRand(0.5) // r=0.5 => factor=1 => no jitter distortion + + cases := []struct { + attempt int + want time.Duration + }{ + {0, 1 * time.Second}, + {1, 2 * time.Second}, + {2, 4 * time.Second}, + {3, 8 * time.Second}, + {4, 16 * time.Second}, + {5, 32 * time.Second}, + {6, 60 * time.Second}, // would be 64s, capped + {7, 60 * time.Second}, + {100, 60 * time.Second}, + } + for _, c := range cases { + got := Backoff(c.attempt, base, max, 0.2, mid) + if got != c.want { + t.Errorf("Backoff(%d) = %v, want %v", c.attempt, got, c.want) + } + } +} + +func TestBackoffJitterBounds(t *testing.T) { + base := time.Second + max := 60 * time.Second + attempt := 3 // nominal 8s + nominal := 8 * time.Second + jitter := 0.2 + + for _, r := range []float64{0, 0.25, 0.5, 0.75, 1} { + got := Backoff(attempt, base, max, jitter, fixedRand(r)) + lo := time.Duration(float64(nominal) * (1 - jitter)) + hi := time.Duration(float64(nominal) * (1 + jitter)) + if got < lo || got > hi { + t.Errorf("Backoff with r=%v = %v, want within [%v, %v]", r, got, lo, hi) + } + } +} + +func TestBackoffZeroJitterIsDeterministic(t *testing.T) { + got1 := Backoff(2, time.Second, 60*time.Second, 0, fixedRand(0.99)) + got2 := Backoff(2, time.Second, 60*time.Second, 0, fixedRand(0.01)) + if got1 != got2 || got1 != 4*time.Second { + t.Fatalf("zero jitter should ignore randFn entirely: got %v and %v, want 4s both", got1, got2) + } +} + +func TestBackoffNegativeAttemptClampsToZero(t *testing.T) { + got := Backoff(-5, time.Second, 60*time.Second, 0, fixedRand(0.5)) + if got != time.Second { + t.Fatalf("Backoff(-5) = %v, want base (1s)", got) + } +} + +// TestBackoffMonotonicNonDecreasing pins the "1s起, 2倍, 上限60s" shape +// across a full attempt sweep without ever sleeping — the whole point of +// keeping Backoff a pure function. +func TestBackoffMonotonicNonDecreasing(t *testing.T) { + base, max := time.Second, 60*time.Second + prev := time.Duration(0) + for attempt := 0; attempt < 10; attempt++ { + got := Backoff(attempt, base, max, 0, nil) + if got < prev { + t.Fatalf("attempt %d: backoff %v < previous %v (must be non-decreasing)", attempt, got, prev) + } + if got > max { + t.Fatalf("attempt %d: backoff %v exceeds cap %v", attempt, got, max) + } + prev = got + } +} diff --git a/daemon/internal/realtime/client/client.go b/daemon/internal/realtime/client/client.go new file mode 100644 index 00000000..a7da4278 --- /dev/null +++ b/daemon/internal/realtime/client/client.go @@ -0,0 +1,928 @@ +package client + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +const sourceRole = "source" + +var errClientClosing = errors.New("realtime client: closing") + +// errNoRetry marks a handshake failure that is fatal AND not retryable +// (version_unsupported, unauthenticated, or a server accept naming a +// protocol version we never offered): reconnecting cannot succeed without +// an out-of-band change (a software upgrade, a new credential), so the +// reconnect loop must stop instead of hammering the server forever. The +// process recovers by constructing a new Client (i.e. an explicit restart). +var errNoRetry = errors.New("realtime client: fatal, not retryable") + +// ErrRealtimeDisabled marks a dial rejected with HTTP 403 before the +// WebSocket upgrade completed: the server's own REALTIME_ENABLED flag is +// off, so its /v3/realtime endpoint refuses every connection regardless of +// this device's credentials. This is neither the errNoRetry case (the flag +// can flip back on at any time, with no software upgrade or credential +// change required) nor an ordinary transient dial failure (hammering the +// normal reconnect backoff would be pointless load on a condition that +// cannot self-heal within seconds) — see run's handling and +// serverDisabledRecheckInterval. +var ErrRealtimeDisabled = errors.New("realtime client: server reports realtime disabled (403)") + +// serverDisabledRecheckInterval is how long the client waits before +// re-probing an endpoint that rejected the last dial with +// ErrRealtimeDisabled. Deliberately long and fixed: this is a server +// operator's flag, not something expected to flip from moment to moment, so +// there is no user-facing config for it (see the work order) — only a +// same-package test seam (Config.testDisabledRecheckInterval) so tests +// don't have to wait 5 minutes. +const serverDisabledRecheckInterval = 5 * time.Minute + +// ErrInvalidBody wraps a failure from a wire body's Validate(), returned by +// SendTaskEvent/SendMessageEvent when the caller-supplied fields can never +// produce a valid envelope (bad kind, out-of-range percent, oversized free +// text, ...). It is the one Enqueue-adjacent failure callers should treat as +// permanent: unlike outbox.ErrOutboxFull or any other Enqueue error (a +// transient BeginTx/COUNT/INSERT/Commit failure), retrying the exact same +// malformed body can never succeed. Use errors.Is to check for it. +var ErrInvalidBody = errors.New("realtime: event body failed validation") + +// Client is a reconnecting realtime-protocol source connection. Construct +// with New; it starts connecting immediately in the background. Call +// SendTaskEvent/SendMessageEvent for reliable events and SendMetric for +// best-effort metric samples; call Close to shut down. +type Client struct { + cfg Config + + mu sync.Mutex + conn *websocket.Conn + connected bool + attempt int + + // wake coalesces "check the outbox for new work" notifications into the + // single connection goroutine that owns writing reliable events to the + // wire (see connectAndServe's select loop and replayPending). It is + // intentionally never written to directly by anything other than + // wakeSender, and never read by anything other than connectAndServe: + // that single-writer/single-reader discipline is what guarantees + // device_seq order on the wire even when SendTaskEvent/SendMessageEvent + // are called concurrently, including during an in-flight replay after a + // reconnect (SPEC.md section 5.1). + wake chan struct{} + + metrics *metricBatcher + + seenMu sync.Mutex + seenOrder []string + seen map[string]bool + + supersededPenalty atomic.Bool + + // serverDisabled mirrors the outcome of the most recent dial attempt: + // true from the moment a dial is rejected with ErrRealtimeDisabled + // until the next dial actually completes hello (see connectAndServe), + // at which point it is cleared. Uplink polls this (ServerDisabled) to + // decide whether to switch its whole session to legacy /v2 routing — + // see internal/uplink.Uplink.pollRealtimeMode's truth table. + serverDisabled atomic.Bool + + // legacyDrainMu serializes DrainToLegacy against itself and against + // connectAndServe's post-hello replay, one outbox row at a time. This + // Client is shared process-wide (cmd/sitrep/agent.go builds exactly one + // per resident agent, and hands it to a fresh uplink.Uplink for every + // concurrently-running automation, each with its own drain-driving + // poll loop), so without a lock owned HERE — not per-Uplink, which + // would do nothing to stop two different Uplinks from racing — two + // callers could both read the same unacked/un-promoted row via + // Pending()/OverflowPending() before either retired it, and both would + // deliver it: a duplicate reliable event on the wire. See + // DrainToLegacy's doc comment for the full invariant, including how + // this same lock also rules out the drain racing this Client's own + // realtime replay across a switch back from legacy mode. + legacyDrainMu sync.Mutex + + closing chan struct{} + closed chan struct{} + once sync.Once +} + +// New starts a Client connecting in the background. +func New(cfg Config) *Client { + cfg.applyDefaults() + c := &Client{ + cfg: cfg, + wake: make(chan struct{}, 1), + metrics: newMetricBatcher(cfg.MetricFlushInterval, cfg.MetricThrottledInterval), + seen: make(map[string]bool), + closing: make(chan struct{}), + closed: make(chan struct{}), + } + go c.run() + return c +} + +// Close stops the client and waits for its background goroutine to exit. +func (c *Client) Close() { + c.once.Do(func() { close(c.closing) }) + <-c.closed +} + +// Connected reports whether a hello-completed connection is currently up. +func (c *Client) Connected() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.connected +} + +// ServerDisabled reports whether the most recent dial attempt was rejected +// with ErrRealtimeDisabled (server-side REALTIME_ENABLED off) and no dial +// has completed hello since. See Config's package doc and +// internal/uplink.Uplink.pollRealtimeMode for what the caller does with +// this. +func (c *Client) ServerDisabled() bool { return c.serverDisabled.Load() } + +// Outbox exposes the durable store backing this client, for a caller that +// needs to drain it through a path other than this client's own realtime +// delivery — specifically internal/uplink's legacy-mode fallback, which +// walks Outbox/Overflow directly once ServerDisabled reports the server has +// rejected realtime for this session. +func (c *Client) Outbox() *outbox.Store { return c.cfg.Outbox } + +// Space reports the (device, space) scope this client's outbox and +// device_seq counter are bound to (SPEC.md section 5.1) — see Outbox. +func (c *Client) Space() string { return c.cfg.Space } + +// LegacySink is called once per durable outbox/overflow row by +// DrainToLegacy, oldest first, and must translate + attempt delivery of +// that row over whatever non-realtime path the caller uses (e.g. legacy +// /v2 HTTP). It reports whether the row should be retired: +// +// - true: either delivery succeeded, or the row is permanently +// undeliverable (e.g. an unrecognized kind) and must never be retried — +// either way DrainToLegacy acks/deletes it and moves on to the next row. +// - false: delivery failed for a reason that might succeed later (e.g. +// the legacy endpoint is unreachable right now); DrainToLegacy stops +// immediately, retiring nothing further, so the next call resumes at +// exactly this row. +type LegacySink func(kind string, body json.RawMessage) bool + +// DrainToLegacy walks this Client's shared outbox — every durable reliable +// event with a real device_seq (Pending), then everything still waiting in +// overflow (OverflowPending) — oldest first in each, handing each row to +// sink and retiring it (Ack / DeleteOverflow) only once sink reports true. +// +// This is the single owning entry point for legacy-mode draining: this +// Client is constructed once per resident agent process +// (cmd/sitrep/agent.go) and shared by a fresh uplink.Uplink for every +// concurrently-running automation (see runAutomation), each running its own +// pollRealtimeMode poll loop against the same outbox. Without process-wide +// serialization here, two Uplinks could both read the same unretired row +// before either retired it and both deliver it — a duplicate reliable +// event reaching the server. DrainToLegacy rules that out: every row's +// read-sink-retire sequence runs under legacyDrainMu (see +// drainOneToLegacy), held for exactly that one row, so a second concurrent +// caller blocks until the first has fully retired its row and always sees a +// fresh Pending()/OverflowPending() read with that row already gone. +// +// The same lock also closes the narrower switch-back race the reviewer +// flagged: connectAndServe acquires legacyDrainMu around clearing +// serverDisabled and its initial post-hello replayPending call (see +// connectAndServe), so a drain in flight when the server re-accepts +// realtime either fully retires its current row before that replay reads +// the outbox (the row is already gone by then — no duplicate), or blocks +// until the replay (which reads every row still sitting in the outbox at +// that instant) has already committed to delivering them over realtime — +// drainOneToLegacy then observes ServerDisabled() flip false on its next +// iteration and stops without ever touching those rows itself. Either way a +// given row is delivered exactly once, by exactly one of the two paths. +// +// The lock is held per-row rather than across the whole call so a +// reconnecting Client is never blocked behind an entire backlog — only +// behind whichever single row happens to be mid-flight. +func (c *Client) DrainToLegacy(ctx context.Context, sink LegacySink) { + if c.cfg.Outbox == nil { + return + } + for { + if done := c.drainOneToLegacy(ctx, sink); done { + return + } + } +} + +// drainOneToLegacy drains at most one row (see DrainToLegacy) under +// legacyDrainMu. It reports true when the caller should stop: nothing left +// to drain, sink asked to stop, a store error, or the server has +// re-accepted realtime since the last iteration. +func (c *Client) drainOneToLegacy(ctx context.Context, sink LegacySink) bool { + c.legacyDrainMu.Lock() + defer c.legacyDrainMu.Unlock() + + if !c.ServerDisabled() { + // Realtime has resumed (or was never disabled to begin with, for a + // stray call): every row still sitting in the outbox at this + // instant is guaranteed to be delivered by connectAndServe's + // post-hello replayPending instead, which holds this same lock + // while doing so — see DrainToLegacy's doc comment. Stopping here + // cannot drop or duplicate anything. + return true + } + + store := c.cfg.Outbox + pending, err := store.Pending(ctx, c.cfg.Space) + if err != nil { + c.cfg.Logf("realtime: legacy drain: read outbox: %v", err) + return true + } + if len(pending) > 0 { + item := pending[0] + if !sink(item.Kind, item.Body) { + return true + } + if err := store.Ack(ctx, c.cfg.Space, item.DeviceSeq); err != nil { + c.cfg.Logf("realtime: legacy drain: ack seq %d: %v", item.DeviceSeq, err) + return true + } + return false + } + + overflow, err := store.OverflowPending(ctx, c.cfg.Space) + if err != nil { + c.cfg.Logf("realtime: legacy drain: read overflow: %v", err) + return true + } + if len(overflow) == 0 { + return true + } + ov := overflow[0] + if !sink(ov.Kind, ov.Body) { + return true + } + if err := store.DeleteOverflow(ctx, ov.ID); err != nil { + c.cfg.Logf("realtime: legacy drain: delete overflow id %d: %v", ov.ID, err) + return true + } + return false +} + +// MetricsThrottled reports whether the metric batcher is currently in its +// throttled cadence (SPEC.md section 7's command{throttle}/{resume_rate}). +func (c *Client) MetricsThrottled() bool { return c.metrics.Throttled() } + +func newEnvelopeID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return fmt.Sprintf("rt%d", time.Now().UnixNano()) + } + return "rt" + hex.EncodeToString(b) +} + +func (c *Client) now() time.Time { return c.cfg.Clock.Now() } +func (c *Client) nowMS() int64 { return c.now().UnixMilli() } + +// ---- public send API ---- + +// TaskEvent is the caller-facing shape for SendTaskEvent; see +// wire.TaskEventBody for field semantics. +type TaskEvent struct { + TaskID string + Kind string // started|progress|step|done|failed + OccurredAt time.Time + Title string + Percent *int + Step string + Message string + Display *wire.DisplayHints +} + +// SendTaskEvent durably enqueues a task lifecycle/progress event (assigning +// its device_seq) and attempts immediate delivery if connected. The event +// survives in the outbox until acknowledged, across any number of +// reconnects or process restarts. +// +// outbox.ErrOverflowed is deliberately not surfaced as an error: per its own +// doc comment it is informational, not a failure — the event is already +// durably persisted (into outbox_overflow instead of outbox) by the time +// Enqueue returns it, and "[c]allers must not retry or reroute on this +// error". uplink.Uplink.sendReliable's own doc comment already assumes this +// ("ordinary outbox-full backpressure durably overflows inside Enqueue and +// never reaches here"): if this returned ErrOverflowed as a plain error, +// sendReliable's persist-failure branch would re-attempt the exact same +// Enqueue call every flush tick until capacity frees, each attempt durably +// creating ANOTHER outbox_overflow row for the same logical event (Enqueue +// does not deduplicate) — unbounded duplicate rows in an uncapped table, +// only one of which is ever promoted and delivered. +func (c *Client) SendTaskEvent(ev TaskEvent) error { + _, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeTaskEvent, func(seq int64) (json.RawMessage, error) { + body := wire.TaskEventBody{ + DeviceID: c.cfg.DeviceID, + DeviceSeq: seq, + TaskID: ev.TaskID, + Kind: ev.Kind, + OccurredAt: ev.OccurredAt.UnixMilli(), + Title: ev.Title, + Percent: ev.Percent, + Step: ev.Step, + Message: ev.Message, + Display: ev.Display, + } + if err := body.Validate(); err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidBody, err) + } + return json.Marshal(body) + }) + if err != nil && !errors.Is(err, outbox.ErrOverflowed) { + return err + } + c.wakeSender() + return nil +} + +// MessageEvent is the caller-facing shape for SendMessageEvent. If +// MessageID is empty it defaults to ":". +type MessageEvent struct { + MessageID string + Level string // info|warn|error + Text string + OccurredAt time.Time + AutomationID string +} + +// SendMessageEvent durably enqueues a message event; see SendTaskEvent, +// including why outbox.ErrOverflowed is not surfaced as an error here. +func (c *Client) SendMessageEvent(ev MessageEvent) error { + _, err := c.cfg.Outbox.Enqueue(context.Background(), c.cfg.Space, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + id := ev.MessageID + if id == "" { + id = fmt.Sprintf("%s:%d", c.cfg.DeviceID, seq) + } + body := wire.MessageEventBody{ + DeviceID: c.cfg.DeviceID, + DeviceSeq: seq, + MessageID: id, + Level: ev.Level, + Text: ev.Text, + OccurredAt: ev.OccurredAt.UnixMilli(), + AutomationID: ev.AutomationID, + } + if err := body.Validate(); err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidBody, err) + } + return json.Marshal(body) + }) + if err != nil && !errors.Is(err, outbox.ErrOverflowed) { + return err + } + c.wakeSender() + return nil +} + +// SendMetric offers one best-effort metric sample. It is merged with any +// other pending sample for the same MetricID and flushed on the client's +// own cadence (SPEC.md section 12); it is never persisted, never +// retransmitted, and silently superseded by the next sample if lost. +func (c *Client) SendMetric(s wire.MetricSample) { + c.metrics.Offer(s) +} + +// wakeSender notifies the active connection's single writer goroutine +// (connectAndServe's select loop) that new work may be waiting in the +// outbox. It never writes to the wire itself and is always safe to call, +// including when no connection is currently up (the signal is simply +// coalesced/dropped and the next connection's initial replayPending call +// will pick up the item from durable storage regardless). +// +// This indirection — rather than writing directly from the caller's +// goroutine, as a prior version of this code did — is what guarantees wire +// order matches device_seq order (SPEC.md section 5.1): every reliable +// event write, whether a fresh live send or a reconnect replay, now goes +// through replayPending called from exactly one goroutine per connection, +// so two writes for the same connection can never race each other onto the +// wire out of order. +func (c *Client) wakeSender() { + select { + case c.wake <- struct{}{}: + default: + } +} + +// ---- connection lifecycle ---- + +func (c *Client) run() { + defer close(c.closed) + for { + select { + case <-c.closing: + return + default: + } + + err := c.connectAndServe() + if errors.Is(err, errClientClosing) { + return + } + if errors.Is(err, errNoRetry) { + // A retry can only fail the same way (wrong protocol version, + // revoked credential, ...): surface it and stop the loop + // entirely rather than reconnecting forever. Recovery is an + // explicit restart (a new Client). + c.cfg.Logf("realtime: giving up, not retryable: %v", err) + return + } + if errors.Is(err, ErrRealtimeDisabled) { + // The server itself has realtime turned off. This is not a + // transient dial failure (the normal exponential backoff below + // would hammer it every ~60s for no reason: an operator flag + // does not flip back on within seconds) and not the + // give-up-forever case either (it CAN flip back on at any time, + // with no software upgrade or new credential needed) — so wait + // a long, fixed interval and try again, without touching the + // normal reconnect attempt counter at all. + c.serverDisabled.Store(true) + interval := serverDisabledRecheckInterval + if c.cfg.testDisabledRecheckInterval > 0 { + interval = c.cfg.testDisabledRecheckInterval + } + c.cfg.Logf("realtime: server reports realtime disabled; re-checking in %v", interval) + select { + case <-time.After(interval): + case <-c.closing: + return + } + continue + } + c.cfg.Logf("realtime: connection ended: %v", err) + + c.mu.Lock() + attempt := c.attempt + c.attempt++ + c.mu.Unlock() + + delay := Backoff(attempt, c.cfg.BackoffBase, c.cfg.BackoffMax, c.cfg.BackoffJitter, c.cfg.Rand) + if c.supersededPenalty.Swap(false) { + // Superseded means our own newer connection won, or our + // credential is compromised — either way, reconnecting + // aggressively is the wrong instinct; stretch the wait to the + // backoff ceiling instead of a reconnect storm. + delay = c.cfg.BackoffMax + } + select { + case <-time.After(delay): + case <-c.closing: + return + } + } +} + +func (c *Client) connectAndServe() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dialCtx, dialCancel := context.WithTimeout(ctx, c.cfg.HelloTimeout) + header := http.Header{} + if c.cfg.Token != "" { + header.Set("Authorization", "Bearer "+c.cfg.Token) + } + conn, resp, err := websocket.Dial(dialCtx, c.cfg.URL, &websocket.DialOptions{HTTPHeader: header}) + dialCancel() + if err != nil { + // A rejection before the WebSocket upgrade completes surfaces here + // as a non-nil err with resp still populated (coder/websocket keeps + // the response, with a small snippet of its body, specifically for + // this kind of diagnosis). Status 403 on this endpoint is + // sufficient signal on its own that the server's REALTIME_ENABLED + // is off — SERVER-side authority has moved to UserStore for this + // session, exactly like the local SITREP_REALTIME flag off has + // always meant "legacy only" on this side; see ErrRealtimeDisabled + // and internal/uplink.Uplink.pollRealtimeMode for what happens next. + if resp != nil && resp.StatusCode == http.StatusForbidden { + return fmt.Errorf("dial: %w", ErrRealtimeDisabled) + } + return fmt.Errorf("dial: %w", err) + } + defer conn.CloseNow() + + if err := c.sendHelloOffer(ctx, conn); err != nil { + return fmt.Errorf("hello offer: %w", err) + } + accept, err := c.awaitHelloAccept(ctx, conn) + if err != nil { + return fmt.Errorf("hello accept: %w", err) + } + heartbeatInterval := time.Duration(accept.HeartbeatIntervalMS) * time.Millisecond + + c.mu.Lock() + c.conn = conn + c.connected = true + c.attempt = 0 + c.mu.Unlock() + defer func() { + c.mu.Lock() + c.connected = false + c.conn = nil + c.mu.Unlock() + }() + + // A completed hello is proof the server currently accepts realtime, + // whether or not this connection had ever previously been rejected with + // ErrRealtimeDisabled — clear it so pollRealtimeMode's caller resumes + // realtime routing for new events immediately, and replay everything + // still sitting in the outbox over this connection. + // + // Both happen under legacyDrainMu — the same lock DrainToLegacy holds + // per row — so a legacy drain in flight when the server re-accepts + // realtime cannot race this replay onto the same row: see + // DrainToLegacy's doc comment for the full invariant. + c.legacyDrainMu.Lock() + c.serverDisabled.Store(false) + c.replayPending(ctx, conn) + c.legacyDrainMu.Unlock() + + frames := make(chan frameMsg, 8) + pings := make(chan struct{}, 8) + pongs := make(chan struct{}, 8) + go c.readLoop(ctx, conn, frames, pings, pongs) + + resendTicker := time.NewTicker(c.cfg.ResendInterval) + defer resendTicker.Stop() + metricsTicker := time.NewTicker(c.cfg.MetricFlushInterval) + defer metricsTicker.Stop() + heartbeatTicker := time.NewTicker(heartbeatInterval) + defer heartbeatTicker.Stop() + + lastPong := c.now() + heartbeatTimeout := time.Duration(c.cfg.HeartbeatTimeoutFactor) * heartbeatInterval + + for { + select { + case fm := <-frames: + if fm.err != nil { + return fm.err + } + if err := c.handleFrame(ctx, conn, fm.env); err != nil { + return err + } + case <-pings: + _ = conn.Write(ctx, websocket.MessageText, []byte("pong")) + case <-pongs: + lastPong = c.now() + case <-resendTicker.C: + c.replayPending(ctx, conn) + case <-c.wake: + // A live SendTaskEvent/SendMessageEvent call enqueued a new + // item. replayPending re-reads the full outbox (oldest seq + // first) and writes from this same goroutine, so it can never + // race a concurrent replay/resend for this connection — it + // simply folds into the same single-writer stream. + c.replayPending(ctx, conn) + case <-metricsTicker.C: + c.flushMetrics(ctx, conn) + case <-heartbeatTicker.C: + if err := conn.Write(ctx, websocket.MessageText, []byte("ping")); err != nil { + return fmt.Errorf("heartbeat write: %w", err) + } + if c.now().Sub(lastPong) > heartbeatTimeout { + return fmt.Errorf("heartbeat timeout: no pong within %v", heartbeatTimeout) + } + case <-c.closing: + _ = conn.Close(websocket.StatusNormalClosure, "client closing") + return errClientClosing + } + } +} + +func (c *Client) sendHelloOffer(ctx context.Context, conn *websocket.Conn) error { + offer := wire.HelloOffer{ + Stage: "offer", + DeviceID: c.cfg.DeviceID, + Role: sourceRole, + ProtocolVersions: c.cfg.ProtocolVersions, + } + env, err := wire.NewEnvelope(wire.TypeHello, newEnvelopeID(), c.nowMS(), wire.HelloBody{Offer: &offer}) + if err != nil { + return err + } + data, err := env.Encode() + if err != nil { + return err + } + return conn.Write(ctx, websocket.MessageText, data) +} + +func (c *Client) awaitHelloAccept(ctx context.Context, conn *websocket.Conn) (wire.HelloAccept, error) { + ctx, cancel := context.WithTimeout(ctx, c.cfg.HelloTimeout) + defer cancel() + _, data, err := conn.Read(ctx) + if err != nil { + return wire.HelloAccept{}, err + } + env, err := wire.DecodeEnvelope(data) + if err != nil { + return wire.HelloAccept{}, err + } + if env.Type == wire.TypeError { + body, derr := wire.DecodeBody(env) + if derr == nil { + eb := body.(wire.ErrorBody) + // A fatal AND non-retryable rejection (version_unsupported, + // unauthenticated, ...) can only repeat on the next attempt: + // mark it errNoRetry so the reconnect loop stops (SPEC.md + // section 13's retryable flag is authoritative). + if eb.Fatal != nil && *eb.Fatal && eb.Retryable != nil && !*eb.Retryable { + return wire.HelloAccept{}, fmt.Errorf("server rejected hello: %s: %s: %w", eb.Code, eb.Message, errNoRetry) + } + return wire.HelloAccept{}, fmt.Errorf("server rejected hello: %s: %s", eb.Code, eb.Message) + } + return wire.HelloAccept{}, fmt.Errorf("server rejected hello with malformed error body") + } + if env.Type != wire.TypeHello { + return wire.HelloAccept{}, fmt.Errorf("expected hello, got %q", env.Type) + } + body, err := wire.DecodeBody(env) + if err != nil { + return wire.HelloAccept{}, err + } + hb := body.(wire.HelloBody) + if hb.Accept == nil { + return wire.HelloAccept{}, fmt.Errorf("expected hello accept, got offer") + } + // SPEC.md section 9.2: the server selects from the INTERSECTION of the + // offer's protocol_versions with its own set, so a conformant accept + // always names a version we offered. An accept naming any other version + // is a failed negotiation — retrying with the same offer cannot end + // differently, so treat it like version_unsupported and stop. + offered := false + for _, v := range c.cfg.ProtocolVersions { + if v == hb.Accept.ProtocolVersion { + offered = true + break + } + } + if !offered { + return wire.HelloAccept{}, fmt.Errorf( + "server accepted protocol version %d, which we never offered (%v): %w", + hb.Accept.ProtocolVersion, c.cfg.ProtocolVersions, errNoRetry) + } + return *hb.Accept, nil +} + +// frameMsg carries one decoded envelope, or the terminal read error that +// ended the connection, from readLoop to connectAndServe's select loop. +type frameMsg struct { + env wire.Envelope + err error +} + +// readLoop pumps decoded envelopes (or the bare ping/pong heartbeat text) +// onto the given channels until the connection dies, then reports the +// terminal error on frames and returns. +func (c *Client) readLoop(ctx context.Context, conn *websocket.Conn, frames chan<- frameMsg, pings, pongs chan<- struct{}) { + for { + typ, data, err := conn.Read(ctx) + if err != nil { + select { + case frames <- frameMsg{err: err}: + case <-ctx.Done(): + } + return + } + if typ == websocket.MessageText { + switch string(data) { + case "ping": + select { + case pings <- struct{}{}: + case <-ctx.Done(): + return + } + continue + case "pong": + select { + case pongs <- struct{}{}: + case <-ctx.Done(): + return + } + continue + } + } + env, derr := wire.DecodeEnvelope(data) + select { + case frames <- frameMsg{env: env, err: derr}: + case <-ctx.Done(): + return + } + } +} + +func (c *Client) replayPending(ctx context.Context, conn *websocket.Conn) { + items, err := c.cfg.Outbox.Pending(ctx, c.cfg.Space) + if err != nil { + c.cfg.Logf("realtime: read pending outbox: %v", err) + return + } + for _, item := range items { + env, err := wire.NewEnvelope(item.Kind, newEnvelopeID(), c.nowMS(), json.RawMessage(item.Body)) + if err != nil { + continue + } + data, err := env.Encode() + if err != nil { + continue + } + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + werr := conn.Write(writeCtx, websocket.MessageText, data) + cancel() + if werr != nil { + return // connection is dying; the outer loop will notice and reconnect + } + } +} + +func (c *Client) flushMetrics(ctx context.Context, conn *websocket.Conn) { + samples, ok := c.metrics.Flush(c.now()) + if !ok { + return + } + body := wire.MetricFrameBody{DeviceID: c.cfg.DeviceID, Metrics: samples} + if err := body.Validate(); err != nil { + c.cfg.Logf("realtime: dropping invalid metric.frame: %v", err) + return + } + env, err := wire.NewEnvelope(wire.TypeMetricFrame, newEnvelopeID(), c.nowMS(), body) + if err != nil { + return + } + data, err := env.Encode() + if err != nil { + return + } + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + _ = conn.Write(writeCtx, websocket.MessageText, data) +} + +func (c *Client) handleFrame(ctx context.Context, conn *websocket.Conn, env wire.Envelope) error { + if !wire.KnownType(env.Type) { + return nil // SPEC.md section 15: ignore, not an error + } + body, err := wire.DecodeBody(env) + if err != nil { + c.cfg.Logf("realtime: dropping malformed %s frame: %v", env.Type, err) + return nil + } + switch env.Type { + case wire.TypeAck: + ab := body.(wire.AckBody) + // SPEC.md sections 5.3/13: every pair in one ack MUST carry the + // device_id of this connection's authenticated device; a pair for + // any other device makes the WHOLE envelope malformed. Drop the + // ack without retiring anything — the events stay queued for a + // later well-formed ack (resend makes this safe) — report the + // protocol violation back as error{malformed}, and keep the + // connection (malformed is retryable, non-fatal). + for _, p := range ab.Acked { + if p.DeviceID != c.cfg.DeviceID { + c.cfg.Logf("realtime: protocol violation: ack %s carries a pair for foreign device %q; dropping whole ack", env.ID, p.DeviceID) + c.sendError(ctx, conn, wire.ErrorBody{ + Code: wire.ErrMalformed, + Message: "ack pair for a device other than this connection's", + InReplyTo: env.ID, + Retryable: boolPtr(true), + Fatal: boolPtr(false), + }) + return nil + } + } + for _, p := range ab.Acked { + if err := c.cfg.Outbox.Ack(ctx, c.cfg.Space, p.DeviceSeq); err != nil { + c.cfg.Logf("realtime: ack seq %d: %v", p.DeviceSeq, err) + } + } + // An ack frees outbox capacity, and Outbox.Ack itself promotes at + // most one waiting overflow row into that freed slot per call (see + // its doc comment). Wake the sender so a freshly-promoted row (now + // bearing a real device_seq) is written to this connection promptly + // via replayPending, rather than waiting for the next resend tick. + c.wakeSender() + return nil + case wire.TypeCommand: + cb := body.(wire.CommandBody) + c.handleCommand(ctx, conn, env, cb) + return nil + case wire.TypeError: + eb := body.(wire.ErrorBody) + return c.handleError(eb) + default: + // snapshot/delta/hello/resume/subscribe/... are not meaningful on a + // source connection; ignore rather than kill the connection. + return nil + } +} + +func (c *Client) handleError(eb wire.ErrorBody) error { + if eb.Code == wire.ErrSuperseded { + c.supersededPenalty.Store(true) + c.cfg.Logf("realtime: connection superseded by a newer one (or credential in use elsewhere)") + return fmt.Errorf("%s: %s", eb.Code, eb.Message) + } + if eb.Fatal != nil && *eb.Fatal { + return fmt.Errorf("%s: %s", eb.Code, eb.Message) + } + // Advisory (rate_limited, sequence_gap, ...): log and keep the + // connection. + c.cfg.Logf("realtime: server error %s: %s", eb.Code, eb.Message) + return nil +} + +// handleCommand validates the command's TTL against the local clock +// (SPEC.md section 8: +/- 30s skew allowance), deduplicates by command_id, +// and either applies it (throttle/resume_rate) or forwards it to +// Config.OnCommand. +func (c *Client) handleCommand(ctx context.Context, conn *websocket.Conn, env wire.Envelope, cb wire.CommandBody) { + const skew = 30 * time.Second + now := c.now() + windowStart := time.UnixMilli(env.TS).Add(-skew) + windowEnd := time.UnixMilli(env.TS).Add(time.Duration(cb.TTLMs) * time.Millisecond).Add(skew) + if now.Before(windowStart) || now.After(windowEnd) { + c.cfg.Logf("realtime: dropping expired command %s (action %s)", cb.CommandID, cb.Action) + c.sendError(ctx, conn, wire.ErrorBody{ + Code: wire.ErrCommandExpired, + Message: "command ttl window elapsed", + InReplyTo: env.ID, + Retryable: boolPtr(false), + Fatal: boolPtr(false), + }) + return + } + + if c.alreadySeen(cb.CommandID) { + return // SPEC.md section 8: MUST NOT execute the same command_id twice + } + + switch cb.Action { + case "throttle": + c.metrics.SetThrottled(true) + case "resume_rate": + c.metrics.SetThrottled(false) + default: + if c.cfg.OnCommand != nil { + c.cfg.OnCommand(CommandAction{ + CommandID: cb.CommandID, + Action: cb.Action, + TaskID: cb.TaskID, + AutomationID: cb.AutomationID, + TargetDeviceID: cb.TargetDeviceID, + }) + } + } +} + +func boolPtr(b bool) *bool { return &b } + +// sendError best-effort writes one error envelope on the current +// connection; failures are ignored (the connection's own health checks +// notice a dying socket independently). +func (c *Client) sendError(ctx context.Context, conn *websocket.Conn, body wire.ErrorBody) { + env, err := wire.NewEnvelope(wire.TypeError, newEnvelopeID(), c.nowMS(), body) + if err != nil { + return + } + data, err := env.Encode() + if err != nil { + return + } + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + _ = conn.Write(writeCtx, websocket.MessageText, data) +} + +// alreadySeen reports whether commandID has been handled before, recording +// it if not. Bounded so a long-lived connection doesn't grow this set +// forever. +func (c *Client) alreadySeen(commandID string) bool { + const cap = 512 + c.seenMu.Lock() + defer c.seenMu.Unlock() + if c.seen[commandID] { + return true + } + c.seen[commandID] = true + c.seenOrder = append(c.seenOrder, commandID) + if len(c.seenOrder) > cap { + drop := c.seenOrder[0] + c.seenOrder = c.seenOrder[1:] + delete(c.seen, drop) + } + return false +} diff --git a/daemon/internal/realtime/client/client_test.go b/daemon/internal/realtime/client/client_test.go new file mode 100644 index 00000000..b3983b46 --- /dev/null +++ b/daemon/internal/realtime/client/client_test.go @@ -0,0 +1,748 @@ +package client + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/rttest" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +const testDeviceID = "test-device-01" +const testSpace = "space-test" + +func newTestOutbox(t *testing.T) *outbox.Store { + t.Helper() + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +func newTestClient(t *testing.T, url string, store *outbox.Store, mutate func(*Config)) *Client { + t.Helper() + cfg := Config{ + URL: url, + Token: "test-token", + DeviceID: testDeviceID, + Space: testSpace, + Outbox: store, + BackoffBase: 5 * time.Millisecond, + BackoffMax: 20 * time.Millisecond, + BackoffJitter: 0, + ResendInterval: 20 * time.Millisecond, + MetricFlushInterval: 5 * time.Millisecond, + MetricThrottledInterval: 300 * time.Millisecond, + HelloTimeout: 2 * time.Second, + } + if mutate != nil { + mutate(&cfg) + } + c := New(cfg) + t.Cleanup(c.Close) + return c +} + +// waitFor polls cond until it returns true or the timeout elapses. +func waitFor(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(2 * time.Millisecond) + } + if !cond() { + t.Fatalf("condition not met within %v", timeout) + } +} + +func TestClientHelloAndTaskEventAcked(t *testing.T) { + srv := rttest.New(func(conn *rttest.Conn) { + offer, err := conn.HelloAccept("sess-1", 1000) + if err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + if offer.Role != "source" { + t.Errorf("offer.Role = %q, want source", offer.Role) + } + if offer.DeviceID != testDeviceID { + t.Errorf("offer.DeviceID = %q, want %q", offer.DeviceID, testDeviceID) + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type == wire.TypeTaskEvent { + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + if err := conn.Ack(tb.DeviceID, tb.DeviceSeq); err != nil { + return + } + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + if err := c.SendTaskEvent(TaskEvent{ + TaskID: "run-1", + Kind: "started", + OccurredAt: time.Now(), + Title: "Nightly backup", + }); err != nil { + t.Fatalf("SendTaskEvent: %v", err) + } + + waitFor(t, 3*time.Second, func() bool { + pending, err := store.Pending(context.Background(), testSpace) + return err == nil && len(pending) == 0 + }) +} + +// TestClientReconnectReplaysUnacked drops the first connection before +// acking a task.event and asserts the client resends the SAME device_seq +// (in a fresh envelope) on the next connection, per SPEC.md's Agent +// reconnect replay flow (section 5.4). +func TestClientReconnectReplaysUnacked(t *testing.T) { + var connNum int32 + var firstEnvID, secondEnvID string + var firstSeq, secondSeq int64 + + srv := rttest.New(func(conn *rttest.Conn) { + n := atomic.AddInt32(&connNum, 1) + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + switch n { + case 1: + env, err := conn.ReadEnvelope() + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + t.Errorf("connection 1: expected task.event, got %s", env.Type) + return + } + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + firstEnvID = env.ID + firstSeq = tb.DeviceSeq + // Deliberately do NOT ack; close so the client must reconnect. + conn.Close("simulated drop") + case 2: + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type == wire.TypeTaskEvent { + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + secondEnvID = env.ID + secondSeq = tb.DeviceSeq + conn.Ack(tb.DeviceID, tb.DeviceSeq) + } + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + if err := c.SendTaskEvent(TaskEvent{TaskID: "run-1", Kind: "started", OccurredAt: time.Now()}); err != nil { + t.Fatalf("SendTaskEvent: %v", err) + } + + waitFor(t, 3*time.Second, func() bool { + pending, err := store.Pending(context.Background(), testSpace) + return err == nil && len(pending) == 0 + }) + + if firstSeq == 0 || secondSeq == 0 { + t.Fatalf("did not observe both connections' task.event (first=%d second=%d)", firstSeq, secondSeq) + } + if firstSeq != secondSeq { + t.Fatalf("device_seq changed across resend: first=%d second=%d, want identical (SPEC.md 5.4)", firstSeq, secondSeq) + } + if firstEnvID == secondEnvID { + t.Fatalf("envelope id must be freshly generated on resend, got the same id %q both times", firstEnvID) + } +} + +// TestClientCommandThrottleAndResumeRate asserts a server-issued +// command{throttle} switches the metric batcher into its throttled cadence +// and command{resume_rate} switches it back (SPEC.md section 7). The exact +// merge/rate-limit *math* of the batcher (N samples -> 1 frame, the 500ms +// per-metric and overall-interval gates) is covered deterministically, +// without any wall-clock sleeping, by TestMetricBatcher* in metrics_test.go; +// this test's job is only to prove the command actually reaches and flips +// the batcher's state end-to-end over the wire, so it asserts on that state +// transition rather than timing frame arrivals (which would make the test +// flaky under scheduler jitter). +func TestClientCommandThrottleAndResumeRate(t *testing.T) { + connReady := make(chan *rttest.Conn, 1) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + connReady <- conn + for { + _, err := conn.ReadEnvelope() + if err != nil && err != rttest.ErrPing && err != rttest.ErrPong { + return + } + if err == rttest.ErrPing { + conn.WritePong() + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + var conn *rttest.Conn + select { + case conn = <-connReady: + case <-time.After(3 * time.Second): + t.Fatal("server never observed a connection") + } + + if c.MetricsThrottled() { + t.Fatal("expected the client to start un-throttled") + } + + if err := conn.SendCommand(wire.CommandBody{ + CommandID: "cmd-throttle-1", Origin: "server", Action: "throttle", TTLMs: 3600000, + }); err != nil { + t.Fatalf("SendCommand(throttle): %v", err) + } + waitFor(t, time.Second, c.MetricsThrottled) + + if err := conn.SendCommand(wire.CommandBody{ + CommandID: "cmd-resume-1", Origin: "server", Action: "resume_rate", TTLMs: 3600000, + }); err != nil { + t.Fatalf("SendCommand(resume_rate): %v", err) + } + waitFor(t, time.Second, func() bool { return !c.MetricsThrottled() }) +} + +// TestClientCommandTTLRejectsExpired asserts a command whose TTL window has +// already elapsed (even allowing the +/-30s clock-skew grace, SPEC.md +// section 8) is dropped without being executed. +func TestClientCommandTTLRejectsExpired(t *testing.T) { + connReady := make(chan *rttest.Conn, 1) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + connReady <- conn + for { + if _, err := conn.ReadEnvelope(); err != nil { + return + } + } + }) + defer srv.Close() + + var executed int32 + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, func(cfg *Config) { + cfg.OnCommand = func(CommandAction) { atomic.AddInt32(&executed, 1) } + }) + + var conn *rttest.Conn + select { + case conn = <-connReady: + case <-time.After(3 * time.Second): + t.Fatal("server never observed a connection") + } + + // ts is far enough in the past that even the 30s skew allowance and a + // tiny ttl_ms cannot make "now" fall inside the actionable window. + expiredTS := rttest.NowMS() - 10*60*1000 // 10 minutes ago + env, err := wire.NewEnvelope(wire.TypeCommand, "cmd-env-1", expiredTS, wire.CommandBody{ + CommandID: "cmd-pause-expired", Origin: "viewer", IssuedByDeviceID: "viewer-1", + Action: "pause", TaskID: "run-1", TTLMs: 1000, + }) + if err != nil { + t.Fatalf("NewEnvelope: %v", err) + } + if err := conn.WriteEnvelope(env); err != nil { + t.Fatalf("WriteEnvelope: %v", err) + } + + time.Sleep(100 * time.Millisecond) + if n := atomic.LoadInt32(&executed); n != 0 { + t.Fatalf("OnCommand invoked %d times for an expired command, want 0", n) + } +} + +// TestClientCommandDedupeByID asserts the same command_id is never executed +// twice (SPEC.md section 8). +func TestClientCommandDedupeByID(t *testing.T) { + connReady := make(chan *rttest.Conn, 1) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + connReady <- conn + for { + if _, err := conn.ReadEnvelope(); err != nil { + return + } + } + }) + defer srv.Close() + + var executed int32 + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, func(cfg *Config) { + cfg.OnCommand = func(CommandAction) { atomic.AddInt32(&executed, 1) } + }) + + var conn *rttest.Conn + select { + case conn = <-connReady: + case <-time.After(3 * time.Second): + t.Fatal("server never observed a connection") + } + + send := func() { + body := wire.CommandBody{ + CommandID: "cmd-dupe-1", Origin: "viewer", IssuedByDeviceID: "viewer-1", + Action: "pause", TaskID: "run-1", TTLMs: 3600000, + } + env, err := wire.NewEnvelope(wire.TypeCommand, rttestEnvID(), rttest.NowMS(), body) + if err != nil { + t.Fatalf("NewEnvelope: %v", err) + } + if err := conn.WriteEnvelope(env); err != nil { + t.Fatalf("WriteEnvelope: %v", err) + } + } + send() + send() + time.Sleep(100 * time.Millisecond) + + if n := atomic.LoadInt32(&executed); n != 1 { + t.Fatalf("OnCommand invoked %d times for a duplicated command_id, want exactly 1", n) + } +} + +var envCounter int64 + +func rttestEnvID() string { + return fmt.Sprintf("env-%d", atomic.AddInt64(&envCounter, 1)) +} + +// TestClientStopsAfterFatalNonRetryableHello pins the give-up rule: a +// handshake rejection that is fatal AND not retryable (here +// version_unsupported, SPEC.md section 13) must stop the reconnect loop +// entirely instead of hammering the server forever. +func TestClientStopsAfterFatalNonRetryableHello(t *testing.T) { + var connCount int32 + srv := rttest.New(func(conn *rttest.Conn) { + atomic.AddInt32(&connCount, 1) + if _, err := conn.ReadEnvelope(); err != nil { + return // expected the hello offer + } + conn.SendError(wire.ErrorBody{ + Code: wire.ErrVersionUnsupported, + Message: "no shared protocol version", + Retryable: boolPtr(false), + Fatal: boolPtr(true), + }) + }) + defer srv.Close() + + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, nil) + + // Backoff base is 5ms/cap 20ms: if the client kept reconnecting, this + // window would see dozens of connections. + time.Sleep(300 * time.Millisecond) + if n := atomic.LoadInt32(&connCount); n != 1 { + t.Fatalf("server saw %d connections after a fatal non-retryable hello rejection, want exactly 1 (no reconnect loop)", n) + } +} + +// TestClientStopsWhenAcceptNamesUnofferedVersion pins the accept-side +// version check: SPEC.md section 9.2 requires the server to select from +// the intersection with the offer, so an accept naming a version the +// client never offered is a failed negotiation — treated like +// version_unsupported, and the reconnect loop stops. +func TestClientStopsWhenAcceptNamesUnofferedVersion(t *testing.T) { + var connCount int32 + srv := rttest.New(func(conn *rttest.Conn) { + atomic.AddInt32(&connCount, 1) + if _, err := conn.ReadEnvelope(); err != nil { + return // expected the hello offer + } + accept := wire.HelloAccept{ + Stage: "accept", + ProtocolVersion: 99, // never offered (client offers [1]) + SessionID: "sess-bogus", + HeartbeatIntervalMS: 1000, + } + env, err := wire.NewEnvelope(wire.TypeHello, rttestEnvID(), rttest.NowMS(), wire.HelloBody{Accept: &accept}) + if err != nil { + t.Errorf("NewEnvelope: %v", err) + return + } + _ = conn.WriteEnvelope(env) + // Keep the connection open; the client must abandon it (and not + // come back) on its own. + for { + if _, err := conn.ReadEnvelope(); err != nil { + return + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + newTestClient(t, srv.URL(), store, nil) + + time.Sleep(300 * time.Millisecond) + if n := atomic.LoadInt32(&connCount); n != 1 { + t.Fatalf("server saw %d connections after an accept naming an unoffered version, want exactly 1 (no reconnect loop)", n) + } +} + +// TestClientRejectsAckWithForeignDevicePair pins the SPEC.md section +// 5.3/13 rule: an ack carrying any (device_id, device_seq) pair for a +// device other than this connection's makes the WHOLE envelope malformed — +// nothing is retired from the outbox, not even the pairs that did match. +// The connection survives (malformed is non-fatal): a later well-formed +// ack on the same connection still retires the event. +func TestClientRejectsAckWithForeignDevicePair(t *testing.T) { + badAckSent := make(chan struct{}, 1) + sendGoodAck := make(chan struct{}) + srv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + gotFirst := false + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + continue + } + body, _ := wire.DecodeBody(env) + tb := body.(wire.TaskEventBody) + if !gotFirst { + gotFirst = true + // A malformed ack: one matching pair AND one foreign pair. + // The client must drop it whole — even the matching pair + // must not be retired. + bad := wire.AckBody{Acked: []wire.AckedPair{ + {DeviceID: tb.DeviceID, DeviceSeq: tb.DeviceSeq}, + {DeviceID: "someone-else", DeviceSeq: tb.DeviceSeq}, + }} + badEnv, err := wire.NewEnvelope(wire.TypeAck, rttestEnvID(), rttest.NowMS(), bad) + if err != nil { + t.Errorf("NewEnvelope: %v", err) + return + } + if err := conn.WriteEnvelope(badEnv); err != nil { + return + } + select { + case badAckSent <- struct{}{}: + default: + } + go func() { + <-sendGoodAck + conn.Ack(tb.DeviceID, tb.DeviceSeq) + }() + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + if err := c.SendTaskEvent(TaskEvent{TaskID: "run-1", Kind: "started", OccurredAt: time.Now()}); err != nil { + t.Fatalf("SendTaskEvent: %v", err) + } + + select { + case <-badAckSent: + case <-time.After(3 * time.Second): + t.Fatal("server never sent the malformed ack") + } + + // Give the client ample time to (mis)process the malformed ack; the + // outbox entry must still be there. + time.Sleep(100 * time.Millisecond) + pending, err := store.Pending(context.Background(), testSpace) + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("outbox has %d items after the malformed ack, want 1 (a foreign pair must void the whole ack)", len(pending)) + } + + // The connection survived: a well-formed ack still works. + close(sendGoodAck) + waitFor(t, 3*time.Second, func() bool { + p, err := store.Pending(context.Background(), testSpace) + return err == nil && len(p) == 0 + }) +} + +// TestClientThrottlePreservedAcrossReconnect pins the agreed semantics for +// throttle state vs. reconnects: the daemon KEEPS its throttled state +// across a connection drop (it does not optimistically resume the fast +// cadence), and relies on the server re-sending the space's current +// interest state (throttle or resume_rate) after hello on the new +// connection — here, a resume_rate on the second connection immediately +// restores the normal cadence. +func TestClientThrottlePreservedAcrossReconnect(t *testing.T) { + var connNum int32 + throttleProcessed := make(chan struct{}) + conn2Ready := make(chan *rttest.Conn, 1) + + srv := rttest.New(func(conn *rttest.Conn) { + n := atomic.AddInt32(&connNum, 1) + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + switch n { + case 1: + if err := conn.SendCommand(wire.CommandBody{ + CommandID: "cmd-throttle-x", Origin: "server", Action: "throttle", TTLMs: 3600000, + }); err != nil { + t.Errorf("SendCommand: %v", err) + return + } + // Hold the connection open until the test has observed the + // throttled state, then drop it to force a reconnect. + <-throttleProcessed + default: + conn2Ready <- conn + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + _ = env + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + waitFor(t, 3*time.Second, c.MetricsThrottled) + close(throttleProcessed) // server drops connection 1 + + var conn2 *rttest.Conn + select { + case conn2 = <-conn2Ready: + case <-time.After(3 * time.Second): + t.Fatal("client never reconnected") + } + + // The throttled state survived the reconnect. + if !c.MetricsThrottled() { + t.Fatal("throttled state was lost across the reconnect; it must be preserved until the server says otherwise") + } + + // The server re-sends the space's current interest state after hello: + // a resume_rate on the new connection restores the fast cadence. + if err := conn2.SendCommand(wire.CommandBody{ + CommandID: "cmd-resume-x", Origin: "server", Action: "resume_rate", TTLMs: 3600000, + }); err != nil { + t.Fatalf("SendCommand(resume_rate): %v", err) + } + waitFor(t, 3*time.Second, func() bool { return !c.MetricsThrottled() }) +} + +// TestClientConcurrentSendsPreserveDeviceSeqOrderDuringReplay is a +// regression test for the reconnect race in SPEC.md section 5.1: +// device_seq values MUST appear on the wire in strictly increasing order. +// Before this was fixed, a live SendTaskEvent's write ran directly on the +// caller's own goroutine (trySendNow) with nothing ordering it against +// replayPending's writes on a freshly reconnected connection, so a +// freshly-enqueued higher device_seq could reach the wire before an older, +// still-unacked device_seq was resent. +// +// This test drives that race directly rather than merely asserting on the +// fix's internals: connection 1 accepts exactly one task.event and then +// drops without acking it (forcing a reconnect with a non-empty replay +// backlog), while many goroutines concurrently call SendTaskEvent — some +// landing before the drop, some during the reconnect/replay window on +// connection 2, and some once connection 2 is already steady-state. The +// mock server records every device_seq it observes, across both +// connections, in arrival order. The assertion is on first-occurrence +// order (a seq may legitimately repeat on the wire, e.g. a full-backlog +// resend re-covering an already-sent-but-still-unacked item; only the +// *first* time each seq is ever seen must be strictly increasing). +// +// Run with `go test -race`: the fix serializes all reliable-event writes +// for a connection through a single goroutine (connectAndServe's select +// loop, woken by wakeSender), which this test also exercises for data +// races on the client's own state under concurrent callers. +func TestClientConcurrentSendsPreserveDeviceSeqOrderDuringReplay(t *testing.T) { + const concurrent = 40 + + var mu sync.Mutex + var arrival []int64 + + var connNum int32 + srv := rttest.New(func(conn *rttest.Conn) { + n := atomic.AddInt32(&connNum, 1) + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + if n == 1 { + // Read exactly one task.event, record it, then drop the + // connection without acking — the client must reconnect with + // that event (and possibly more enqueued concurrently) still + // unacked in its outbox. + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + t.Errorf("decode task.event on connection 1: %v", derr) + return + } + tb := body.(wire.TaskEventBody) + mu.Lock() + arrival = append(arrival, tb.DeviceSeq) + mu.Unlock() + conn.Close("simulated drop") + return + } + } + // Every later connection: replay the backlog and everything sent + // afterward lands here. Ack each event as it arrives so the + // outbox eventually drains. + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeTaskEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + t.Errorf("decode task.event on connection %d: %v", n, derr) + continue + } + tb := body.(wire.TaskEventBody) + mu.Lock() + arrival = append(arrival, tb.DeviceSeq) + mu.Unlock() + if err := conn.Ack(tb.DeviceID, tb.DeviceSeq); err != nil { + return + } + } + }) + defer srv.Close() + + store := newTestOutbox(t) + c := newTestClient(t, srv.URL(), store, nil) + + var wg sync.WaitGroup + wg.Add(concurrent) + for i := 0; i < concurrent; i++ { + go func(i int) { + defer wg.Done() + if err := c.SendTaskEvent(TaskEvent{ + TaskID: fmt.Sprintf("run-%d", i), + Kind: "started", + OccurredAt: time.Now(), + }); err != nil { + t.Errorf("SendTaskEvent(%d): %v", i, err) + } + }(i) + } + wg.Wait() + + waitFor(t, 5*time.Second, func() bool { + pending, err := store.Pending(context.Background(), testSpace) + return err == nil && len(pending) == 0 + }) + + mu.Lock() + got := append([]int64(nil), arrival...) + mu.Unlock() + + seen := make(map[int64]bool, concurrent) + var order []int64 + for _, seq := range got { + if !seen[seq] { + seen[seq] = true + order = append(order, seq) + } + } + for i := 1; i < len(order); i++ { + if order[i] <= order[i-1] { + t.Fatalf("device_seq out of order on the wire: first-seen order %v (full arrival log %v) — SPEC.md section 5.1 requires strictly increasing device_seq", order, got) + } + } + if len(order) != concurrent { + t.Fatalf("expected %d distinct device_seq values delivered, got %d: %v", concurrent, len(order), order) + } +} diff --git a/daemon/internal/realtime/client/config.go b/daemon/internal/realtime/client/config.go new file mode 100644 index 00000000..e44aaaf0 --- /dev/null +++ b/daemon/internal/realtime/client/config.go @@ -0,0 +1,167 @@ +// Package client is the daemon-side realtime protocol client +// (proto/realtime/SPEC.md). It connects to the server as a **source** +// device only — the viewer role (subscribe/resume/snapshot/delta) belongs +// to a different consumer (the phone/menu-bar apps) and is out of scope +// here; see the wire package's authorization matrix for the full role set. +// +// Responsibilities: +// - Maintain one outbound WebSocket connection with the mandatory hello +// offer/accept sequence, reconnecting with exponential backoff + jitter +// on any drop. +// - Persist every reliable event (task.event/message.event) to a local +// outbox before attempting delivery, retire it on a matching ack, and +// replay every still-unacked event oldest-first after each reconnect. +// - Merge and rate-limit metric samples into metric.frame batches, +// honoring server-issued throttle/resume_rate commands. +// - Answer the ping/pong heartbeat and detect a silent peer. +package client + +import ( + "math/rand" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" +) + +// Clock abstracts time for components that need deterministic tests. The +// live Client uses time.Now()/time.Sleep directly (see run/connectAndServe); +// Clock exists for the pure, sleep-free pieces (Backoff, metricBatcher) that +// tests drive with an injected instant instead of waiting in real time. +type Clock interface { + Now() time.Time +} + +type realClock struct{} + +func (realClock) Now() time.Time { return time.Now() } + +// Logf is the client's pluggable logger; nil means "discard". +type Logf func(format string, args ...any) + +// CommandAction is a validated, deduplicated, not-yet-expired command this +// client is not itself responsible for executing (anything other than the +// throttle/resume_rate pair it handles internally) — pause/resume/stop/ +// run_now. Forwarded to Config.OnCommand, if set, for the caller to wire +// into process control; this package does not dispatch these itself +// (that plumbing already exists on the HTTP uplink path, see +// internal/uplink.Uplink.Commands, and is out of scope for this package). +type CommandAction struct { + CommandID string + Action string // pause|resume|stop|run_now + TaskID string + AutomationID string + TargetDeviceID string +} + +// Config configures a Client. Zero-value fields with a documented default +// are filled in by New. +type Config struct { + // URL is the ws:// or wss:// endpoint to dial. + URL string + // Token is presented once at connection establishment (SPEC.md section + // 10), as an Authorization: Bearer header — never repeated in an + // envelope. + Token string + // DeviceID and Space identify this connection's hello{device_id} and + // the (device, space) scope its device_seq counter and outbox are + // bound to (SPEC.md section 5.1). One Client serves exactly one space. + DeviceID string + Space string + + // Outbox is the durable reliable-event queue. Required. + Outbox *outbox.Store + + // ProtocolVersions defaults to []int{1}. + ProtocolVersions []int + + // HeartbeatTimeoutFactor multiplies the server-advertised + // heartbeat_interval_ms to decide when a silent peer is dead (SPEC.md + // section 9.3 recommends 2x). Defaults to 2. + HeartbeatTimeoutFactor int + + // Reconnect backoff (SPEC.md work order: "1s起, 2倍, 上限60s, ±20% jitter"). + BackoffBase time.Duration // default 1s + BackoffMax time.Duration // default 60s + BackoffJitter float64 // default 0.2 + + // ResendInterval is how often, while connected, the client re-attempts + // delivery of anything still sitting unacked in the outbox (belt and + // suspenders on top of the mandatory post-reconnect replay). + ResendInterval time.Duration // default 5s + + // MetricFlushInterval is the normal-cadence poll period for the metric + // batcher; MetricThrottledInterval replaces it while under a server + // throttle command (SPEC.md section 7: "an implementation-defined + // lower rate"). The 2 Hz per-metric / 10 Hz per-connection protocol + // ceilings (section 11/12) are enforced independently of this value. + MetricFlushInterval time.Duration // default 200ms + MetricThrottledInterval time.Duration // default 30s + + // HelloTimeout bounds how long to wait for hello{accept} after sending + // the offer (SPEC.md section 9.1 recommends 10s). + HelloTimeout time.Duration // default 10s + + Clock Clock + Rand func() float64 // returns [0,1); defaults to a package-level RNG + + OnCommand func(CommandAction) + Logf Logf + + // testDisabledRecheckInterval overrides serverDisabledRecheckInterval. + // Unexported: there is no public config surface for this (per the work + // order's "no config needed" — a server operator's flag is not + // something a user should be tuning). Set only via + // WithTestDisabledRecheckInterval, and only from tests. + testDisabledRecheckInterval time.Duration +} + +// WithTestDisabledRecheckInterval returns a copy of cfg with the periodic +// re-probe interval (serverDisabledRecheckInterval, normally ~5 minutes) +// shortened. This is a test-only seam for integration tests in sibling +// internal packages (e.g. internal/uplink) that need to observe a +// server-disabled -> re-enabled transition without waiting 5 real minutes; +// production code must never call this. +func WithTestDisabledRecheckInterval(cfg Config, d time.Duration) Config { + cfg.testDisabledRecheckInterval = d + return cfg +} + +func (c *Config) applyDefaults() { + if len(c.ProtocolVersions) == 0 { + c.ProtocolVersions = []int{1} + } + if c.HeartbeatTimeoutFactor <= 0 { + c.HeartbeatTimeoutFactor = 2 + } + if c.BackoffBase <= 0 { + c.BackoffBase = time.Second + } + if c.BackoffMax <= 0 { + c.BackoffMax = 60 * time.Second + } + if c.BackoffJitter == 0 { + c.BackoffJitter = 0.2 + } + if c.ResendInterval <= 0 { + c.ResendInterval = 5 * time.Second + } + if c.MetricFlushInterval <= 0 { + c.MetricFlushInterval = 200 * time.Millisecond + } + if c.MetricThrottledInterval <= 0 { + c.MetricThrottledInterval = 30 * time.Second + } + if c.HelloTimeout <= 0 { + c.HelloTimeout = 10 * time.Second + } + if c.Clock == nil { + c.Clock = realClock{} + } + if c.Rand == nil { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + c.Rand = rng.Float64 + } + if c.Logf == nil { + c.Logf = func(string, ...any) {} + } +} diff --git a/daemon/internal/realtime/client/disabled_test.go b/daemon/internal/realtime/client/disabled_test.go new file mode 100644 index 00000000..fff94642 --- /dev/null +++ b/daemon/internal/realtime/client/disabled_test.go @@ -0,0 +1,118 @@ +package client + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestClientDetects403AndDoesNotHammer pins the P0-1 daemon-side fix: a +// dial rejected with HTTP 403 before the WebSocket upgrade completes must be +// recognized as ErrRealtimeDisabled (surfaced via ServerDisabled), not +// treated like an ordinary transient dial failure — which would otherwise +// retry on the normal (much shorter) exponential backoff and hammer an +// endpoint whose REALTIME_ENABLED flag cannot flip back on within +// milliseconds. +func TestClientDetects403AndDoesNotHammer(t *testing.T) { + var dialCount int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&dialCount, 1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + })) + defer srv.Close() + + // A recheck interval well above this test client's normal reconnect + // backoff (BackoffBase 5ms / BackoffMax 20ms, set by newTestClient) so + // the two are clearly distinguishable: if ErrRealtimeDisabled were + // (wrongly) treated as an ordinary transient dial failure, the normal + // backoff would produce roughly 350ms/20ms ~= 17 dials in the window + // below; obeying the recheck interval instead produces roughly + // 350ms/100ms ~= 3-4. + const recheck = 100 * time.Millisecond + const window = 350 * time.Millisecond + + store := newTestOutbox(t) + url := "ws" + strings.TrimPrefix(srv.URL, "http") + c := newTestClient(t, url, store, func(cfg *Config) { + *cfg = WithTestDisabledRecheckInterval(*cfg, recheck) + }) + + waitFor(t, 2*time.Second, c.ServerDisabled) + + time.Sleep(window) + if n := atomic.LoadInt32(&dialCount); n > 8 { + t.Fatalf("dial count = %d over %v with a %v recheck interval; expected roughly window/interval (~3-4), not the ~17 a tight reconnect-backoff loop would produce", n, window, recheck) + } +} + +// TestClientServerDisabledClearsOnceServerAccepts asserts the other half of +// the truth table: once a dial actually completes hello (the server has +// re-enabled realtime), ServerDisabled must clear so the caller (see +// internal/uplink.Uplink.pollRealtimeMode) resumes realtime routing. +func TestClientServerDisabledClearsOnceServerAccepts(t *testing.T) { + var attempts int32 + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&attempts, 1) + if n <= 2 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + return + } + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer ws.Close(websocket.StatusNormalClosure, "done") + ctx := r.Context() + _, data, err := ws.Read(ctx) + if err != nil { + return + } + env, err := wire.DecodeEnvelope(data) + if err != nil || env.Type != wire.TypeHello { + return + } + accept := wire.HelloAccept{Stage: "accept", ProtocolVersion: 1, SessionID: "sess", HeartbeatIntervalMS: 1000} + acceptEnv, err := wire.NewEnvelope(wire.TypeHello, "accept-env-1", time.Now().UnixMilli(), wire.HelloBody{Accept: &accept}) + if err != nil { + return + } + encoded, err := acceptEnv.Encode() + if err != nil { + return + } + if err := ws.Write(ctx, websocket.MessageText, encoded); err != nil { + return + } + for { + if _, _, err := ws.Read(ctx); err != nil { + return + } + } + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + store := newTestOutbox(t) + url := "ws" + strings.TrimPrefix(srv.URL, "http") + c := newTestClient(t, url, store, func(cfg *Config) { + *cfg = WithTestDisabledRecheckInterval(*cfg, 15*time.Millisecond) + }) + + waitFor(t, 2*time.Second, c.ServerDisabled) + waitFor(t, 2*time.Second, c.Connected) + if c.ServerDisabled() { + t.Fatal("expected ServerDisabled to clear once a connection completed hello") + } +} diff --git a/daemon/internal/realtime/client/e2e_test.go b/daemon/internal/realtime/client/e2e_test.go new file mode 100644 index 00000000..16bf2150 --- /dev/null +++ b/daemon/internal/realtime/client/e2e_test.go @@ -0,0 +1,111 @@ +package client + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" +) + +// TestE2ESourceLifecycle drives a real *daemon* client against a real, +// locally running server (see docs/design/realtime-integration.md for how +// to start one with `wrangler dev`). It is opt-in: it does nothing unless +// SITREP_E2E_URL, SITREP_E2E_TOKEN, SITREP_E2E_DEVICE_ID and +// SITREP_E2E_SPACE are all set, so `go test ./...` in CI/regression never +// touches the network. +// +// Coverage: hello offer/accept over a real WebSocket -> task.event send -> +// server ack (observed as the outbox item retiring) -> disconnect -> a +// second Client sharing the same outbox/device/space reconnects and +// replays the still-pending event -> ack again. This is the cross-line +// interoperability smoke test for the daemon "source" role against the +// real server implementation (as opposed to the in-process rttest fake +// server the rest of this package's tests use). +func TestE2ESourceLifecycle(t *testing.T) { + url := os.Getenv("SITREP_E2E_URL") + token := os.Getenv("SITREP_E2E_TOKEN") + deviceID := os.Getenv("SITREP_E2E_DEVICE_ID") + space := os.Getenv("SITREP_E2E_SPACE") + if url == "" || token == "" || deviceID == "" || space == "" { + t.Skip("set SITREP_E2E_URL, SITREP_E2E_TOKEN, SITREP_E2E_DEVICE_ID, SITREP_E2E_SPACE to run the real-server E2E smoke test") + } + + dir := t.TempDir() + store, err := outbox.Open(filepath.Join(dir, "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + t.Cleanup(func() { store.Close() }) + + mkClient := func() *Client { + return New(Config{ + URL: url, + Token: token, + DeviceID: deviceID, + Space: space, + Outbox: store, + BackoffBase: 50 * time.Millisecond, + BackoffMax: 500 * time.Millisecond, + ResendInterval: 200 * time.Millisecond, + HelloTimeout: 5 * time.Second, + Logf: t.Logf, + }) + } + + c1 := mkClient() + waitFor(t, 5*time.Second, c1.Connected) + t.Log("e2e: hello offer/accept completed against real server (task 1)") + + taskID := "e2e-task-1" + if err := c1.SendTaskEvent(TaskEvent{ + TaskID: taskID, + Kind: "started", + OccurredAt: time.Now(), + Title: "e2e smoke", + }); err != nil { + t.Fatalf("SendTaskEvent(started): %v", err) + } + + waitFor(t, 5*time.Second, func() bool { + n, err := store.Count(context.Background()) + return err == nil && n == 0 + }) + if n, _ := store.Count(context.Background()); n != 0 { + t.Fatalf("expected task.event started to be acked (outbox drained), %d still pending", n) + } + t.Log("e2e: task.event{started} acked by real server, outbox drained") + + c1.Close() + + // Enqueue a second event while fully disconnected: it must sit in the + // outbox until the next connection's post-hello replay picks it up. + percent := 50 + if err := c1.SendTaskEvent(TaskEvent{ + TaskID: taskID, + Kind: "progress", + OccurredAt: time.Now(), + Percent: &percent, + }); err != nil { + t.Fatalf("SendTaskEvent(progress) while offline: %v", err) + } + if n, _ := store.Count(context.Background()); n == 0 { + t.Fatalf("expected the offline progress event to remain pending in the outbox") + } + + c2 := mkClient() + t.Cleanup(c2.Close) + waitFor(t, 5*time.Second, c2.Connected) + t.Log("e2e: reconnect hello offer/accept completed (task 1, replay)") + + waitFor(t, 5*time.Second, func() bool { + n, err := store.Count(context.Background()) + return err == nil && n == 0 + }) + if n, _ := store.Count(context.Background()); n != 0 { + t.Fatalf("expected replayed task.event{progress} to be acked after reconnect, %d still pending", n) + } + t.Log("e2e: task.event{progress} replayed after reconnect and acked, outbox drained") +} diff --git a/daemon/internal/realtime/client/legacy_drain_test.go b/daemon/internal/realtime/client/legacy_drain_test.go new file mode 100644 index 00000000..a7bda180 --- /dev/null +++ b/daemon/internal/realtime/client/legacy_drain_test.go @@ -0,0 +1,128 @@ +package client + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestDrainToLegacyConcurrentCallersDeliverEachRowExactlyOnce is the +// regression test for the MAJOR concurrency defect introduced by commit +// 1114ff9: cmd/sitrep/agent.go builds exactly one *rtclient.Client for the +// whole resident agent process, but spawns a fresh uplink.Uplink — each +// running its own pollRealtimeMode poll loop — per concurrently-running +// automation. Before DrainToLegacy moved drain ownership (and its locking) +// onto this shared Client, every one of those Uplinks drained the SAME +// outbox directly: two callers could both read the same unretired row via +// Pending()/OverflowPending() before either Ack'd/DeleteOverflow'd it, and +// both would "deliver" it — a duplicate reliable event reaching the server +// in production. +// +// This test drives that exact shape at the layer the fix lives in: many +// goroutines call DrainToLegacy concurrently against one Client/outbox +// pre-loaded with a deterministic backlog (simulating several Uplinks' +// pollRealtimeMode loops firing at once, which is the normal case — the +// scheduler in cmd/sitrep/agent.go polls every 2s and multiple automations +// can easily be mid-run together). It asserts every row is observed by the +// sink EXACTLY ONCE, in device_seq order, across all callers combined — the +// order assertion holds only because legacyDrainMu is held across a whole +// row's read+sink+retire, so at any instant the row with the lowest +// remaining device_seq is always the next (and only) one any caller can be +// looking at, regardless of how the goroutines are scheduled. Must be +// meaningful under -race (run with -race -count=20). +func TestDrainToLegacyConcurrentCallersDeliverEachRowExactlyOnce(t *testing.T) { + const rows = 40 + const drainers = 8 + + // A permanently-403 endpoint: DrainToLegacy only acts while + // ServerDisabled() is true (see drainOneToLegacy), matching the + // production precondition (the server has rejected /v3/realtime). + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + })) + defer srv.Close() + + store := newTestOutbox(t) + ctx := context.Background() + + // Deterministic backlog, oldest-first by construction (Enqueue is + // called sequentially, not concurrently, so device_seq order is fixed + // and known ahead of time). + for i := 0; i < rows; i++ { + if _, err := store.Enqueue(ctx, testSpace, wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + return json.Marshal(wire.MessageEventBody{ + DeviceID: testDeviceID, DeviceSeq: seq, + MessageID: "m", Level: "info", Text: "unused", OccurredAt: time.Now().UnixMilli(), + AutomationID: strconv.Itoa(i), + }) + }); err != nil { + t.Fatalf("pre-populate Enqueue %d: %v", i, err) + } + } + + url := "ws" + srv.URL[len("http"):] + c := newTestClient(t, url, store, nil) + waitFor(t, 2*time.Second, c.ServerDisabled) + + var mu sync.Mutex + var delivered []int64 // device_seq, in the order the sink observed them + seen := map[int64]int{} // device_seq -> times observed + + sink := func(kind string, body json.RawMessage) bool { + var b wire.MessageEventBody + if err := json.Unmarshal(body, &b); err != nil { + t.Errorf("sink: decode body: %v", err) + return true + } + mu.Lock() + delivered = append(delivered, b.DeviceSeq) + seen[b.DeviceSeq]++ + mu.Unlock() + return true + } + + var wg sync.WaitGroup + wg.Add(drainers) + for i := 0; i < drainers; i++ { + go func() { + defer wg.Done() + c.DrainToLegacy(ctx, sink) + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + + if len(delivered) != rows { + t.Fatalf("sink observed %d rows, want %d (delivered=%v)", len(delivered), rows, delivered) + } + for seq, n := range seen { + if n != 1 { + t.Fatalf("device_seq %d delivered %d times, want exactly 1 (this is the duplicate-delivery bug the fix closes)", seq, n) + } + } + for i, seq := range delivered { + want := int64(i + 1) // device_seq starts at 1 (SPEC.md section 5.1) + if seq != want { + t.Fatalf("delivery order[%d] = device_seq %d, want %d (out of order): full order %v", i, seq, want, delivered) + } + } + + pending, err := store.Pending(ctx, testSpace) + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 0 { + t.Fatalf("expected outbox fully drained, %d rows remain", len(pending)) + } +} diff --git a/daemon/internal/realtime/client/metrics.go b/daemon/internal/realtime/client/metrics.go new file mode 100644 index 00000000..b36529c5 --- /dev/null +++ b/daemon/internal/realtime/client/metrics.go @@ -0,0 +1,117 @@ +package client + +import ( + "sync" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// metricBatcher merges pending metric samples per metric_id (SPEC.md +// section 12: "A sender MUST merge multiple pending updates to the same +// metric_id into a single sample ... before emitting a frame") and gates +// emission at a bounded cadence: no metric_id may be re-included more than +// once per 500ms (the protocol's 2 Hz per-metric ceiling), and Flush is +// itself only "due" at the batcher's current interval (normal or, while +// throttled, the slower one), which — kept at or above 100ms by the +// client's poll loop — also keeps the connection under the 10 frames/sec +// cap (section 11). +// +// It is a pure, clock-driven data structure with no goroutine or timer of +// its own: the caller decides when to call Flush. This makes it directly +// unit-testable (inject instants, no sleeping) and keeps the "when do we +// actually write to the socket" decision in the Client, which knows whether +// a connection currently exists. +// +// metric.frame is never persisted and never acknowledged (SPEC.md section +// 12): a sample that arrives while disconnected simply overwrites the +// pending entry for its metric_id, same as any other merge — nothing is +// queued, and only the latest value survives to be sent after reconnect. +type metricBatcher struct { + mu sync.Mutex + + pending map[string]wire.MetricSample + + normalInterval time.Duration + throttledInterval time.Duration + throttled bool + + lastFlushAt time.Time // wall-clock time of the last non-empty Flush + lastSentAt map[string]time.Time // per metric_id: wall-clock time last included in a sent frame +} + +func newMetricBatcher(normal, throttled time.Duration) *metricBatcher { + return &metricBatcher{ + pending: make(map[string]wire.MetricSample), + lastSentAt: make(map[string]time.Time), + normalInterval: normal, + throttledInterval: throttled, + } +} + +// Offer merges one sample in: last value wins per metric_id. +func (b *metricBatcher) Offer(s wire.MetricSample) { + b.mu.Lock() + defer b.mu.Unlock() + b.pending[s.MetricID] = s +} + +// SetThrottled switches the batcher between its normal and throttled +// cadence (SPEC.md section 7: command{throttle}/{resume_rate}). +func (b *metricBatcher) SetThrottled(throttled bool) { + b.mu.Lock() + defer b.mu.Unlock() + b.throttled = throttled +} + +// Throttled reports the batcher's current cadence state, mainly for +// observability and tests. +func (b *metricBatcher) Throttled() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.throttled +} + +func (b *metricBatcher) interval() time.Duration { + if b.throttled { + return b.throttledInterval + } + return b.normalInterval +} + +// Flush returns the next metric.frame's worth of samples if the batcher's +// current interval has elapsed since the last emission AND at least one +// pending sample clears its own per-metric_id 500ms gate. Samples that +// don't clear the gate are held back — merged with whatever arrives next — +// rather than dropped; a call that finds nothing eligible returns +// (Body{}, false) and does not disturb any bookkeeping. +func (b *metricBatcher) Flush(now time.Time) ([]wire.MetricSample, bool) { + const perMetricGate = 500 * time.Millisecond + + b.mu.Lock() + defer b.mu.Unlock() + + if !b.lastFlushAt.IsZero() && now.Sub(b.lastFlushAt) < b.interval() { + return nil, false + } + if len(b.pending) == 0 { + return nil, false + } + + var out []wire.MetricSample + for id, sample := range b.pending { + if last, ok := b.lastSentAt[id]; ok && now.Sub(last) < perMetricGate { + continue // held back for a later cycle, not dropped + } + out = append(out, sample) + } + if len(out) == 0 { + return nil, false + } + for _, s := range out { + delete(b.pending, s.MetricID) + b.lastSentAt[s.MetricID] = now + } + b.lastFlushAt = now + return out, true +} diff --git a/daemon/internal/realtime/client/metrics_test.go b/daemon/internal/realtime/client/metrics_test.go new file mode 100644 index 00000000..14de6416 --- /dev/null +++ b/daemon/internal/realtime/client/metrics_test.go @@ -0,0 +1,117 @@ +package client + +import ( + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +func sample(id, value string) wire.MetricSample { + return wire.MetricSample{MetricID: id, Value: value, TS: wire.MinUnixMS + 1000} +} + +func TestMetricBatcherMergesMultipleSamplesIntoOneFrame(t *testing.T) { + b := newMetricBatcher(100*time.Millisecond, 30*time.Second) + base := time.Unix(1700000000, 0) + + // Three updates to the same metric_id before any flush: only the + // latest must survive (SPEC.md section 12). + b.Offer(sample("cpu.load", "0.1")) + b.Offer(sample("cpu.load", "0.2")) + b.Offer(sample("cpu.load", "0.3")) + b.Offer(sample("mem.used_gb", "18.4")) + + out, ok := b.Flush(base) + if !ok { + t.Fatal("Flush: expected a due frame on first call") + } + if len(out) != 2 { + t.Fatalf("Flush returned %d samples, want 2 (one per metric_id)", len(out)) + } + byID := map[string]wire.MetricSample{} + for _, s := range out { + byID[s.MetricID] = s + } + if byID["cpu.load"].Value != "0.3" { + t.Fatalf("cpu.load = %q, want latest value 0.3", byID["cpu.load"].Value) + } +} + +func TestMetricBatcherRateLimitsPerMetric(t *testing.T) { + b := newMetricBatcher(10*time.Millisecond, 30*time.Second) + base := time.Unix(1700000000, 0) + + b.Offer(sample("cpu.load", "0.1")) + out, ok := b.Flush(base) + if !ok || len(out) != 1 { + t.Fatalf("first flush = %v, %v; want 1 sample", out, ok) + } + + // Immediately offer a new value and try to flush again well within the + // per-metric 500ms gate (even though the batcher's own interval has + // elapsed) — the sample must be held back, not dropped. + b.Offer(sample("cpu.load", "0.9")) + out, ok = b.Flush(base.Add(50 * time.Millisecond)) + if ok { + t.Fatalf("expected no due frame within the 500ms per-metric gate, got %v", out) + } + + // After the gate clears, the latest pending value (0.9) must appear. + out, ok = b.Flush(base.Add(600 * time.Millisecond)) + if !ok || len(out) != 1 || out[0].Value != "0.9" { + t.Fatalf("flush after gate = %v, %v; want [0.9]", out, ok) + } +} + +func TestMetricBatcherOverallIntervalGate(t *testing.T) { + b := newMetricBatcher(200*time.Millisecond, 30*time.Second) + base := time.Unix(1700000000, 0) + + b.Offer(sample("cpu.load", "0.1")) + if _, ok := b.Flush(base); !ok { + t.Fatal("expected first flush to be due") + } + b.Offer(sample("mem.used_gb", "1")) + if _, ok := b.Flush(base.Add(50 * time.Millisecond)); ok { + t.Fatal("expected flush before the batcher interval elapsed to be a no-op") + } + if out, ok := b.Flush(base.Add(250 * time.Millisecond)); !ok || len(out) != 1 { + t.Fatalf("flush after interval elapsed = %v, %v; want 1 sample", out, ok) + } +} + +func TestMetricBatcherThrottleSwitchesCadence(t *testing.T) { + b := newMetricBatcher(100*time.Millisecond, 10*time.Second) + base := time.Unix(1700000000, 0) + + b.Offer(sample("cpu.load", "0.1")) + if _, ok := b.Flush(base); !ok { + t.Fatal("expected first flush to be due") + } + + b.SetThrottled(true) + b.Offer(sample("cpu.load", "0.2")) + // Well past the *normal* interval but nowhere near the throttled one. + if _, ok := b.Flush(base.Add(500 * time.Millisecond)); ok { + t.Fatal("expected throttled batcher to withhold the frame") + } + if out, ok := b.Flush(base.Add(11 * time.Second)); !ok || len(out) != 1 { + t.Fatalf("flush past throttled interval = %v, %v; want 1 sample", out, ok) + } + + b.SetThrottled(false) + b.Offer(sample("cpu.load", "0.3")) + // +700ms past the last flush at 11s: clears both the batcher's now-fast + // interval and the per-metric 500ms gate. + if _, ok := b.Flush(base.Add(11700 * time.Millisecond)); !ok { + t.Fatal("expected resume_rate to restore the normal (fast) cadence") + } +} + +func TestMetricBatcherEmptyIsNeverDue(t *testing.T) { + b := newMetricBatcher(time.Millisecond, time.Second) + if out, ok := b.Flush(time.Now()); ok { + t.Fatalf("expected no frame with nothing pending, got %v", out) + } +} diff --git a/daemon/internal/realtime/outbox/outbox.go b/daemon/internal/realtime/outbox/outbox.go new file mode 100644 index 00000000..808fc554 --- /dev/null +++ b/daemon/internal/realtime/outbox/outbox.go @@ -0,0 +1,568 @@ +// Package outbox is the local, crash-safe queue of reliable realtime events +// (task.event / message.event, SPEC.md section 5) awaiting acknowledgement +// from the server. It also owns the per-(device, space) device_seq counter +// (SPEC.md section 5.1): allocating the next sequence number and durably +// enqueueing the event it belongs to happen in a single SQLite transaction, +// so a crash between "I picked seq N" and "event N is safely on disk" is +// impossible — either both happened, or neither did, and the next call sees +// the same N again. +// +// Storage is modernc.org/sqlite, a pure-Go (no cgo) SQLite driver, chosen so +// the daemon keeps its zero-cgo build. The database file lives alongside +// the existing ~/.config/sitrep/config.json (internal/config.Path). +package outbox + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + _ "modernc.org/sqlite" +) + +// DefaultMaxRows bounds the outbox's total row count (across all spaces). +// The cap keeps local disk usage bounded when the server is unreachable for +// a long stretch. Reliability does not degrade at the cap: an Enqueue that +// would exceed it is transparently redirected, in the same transaction, into +// outbox_overflow (see ErrOverflowed) — a second, uncapped durable table in +// this same database file — rather than being rejected. The event is +// durable either way; only its promotion into the seq-bearing outbox is +// deferred until acks free capacity (see PromoteOverflow). +const DefaultMaxRows = 5000 + +// ErrOutboxFull is a legacy sentinel kept for API compatibility; Enqueue no +// longer returns it (a full outbox now overflows instead of failing — see +// ErrOverflowed). Nothing in this package returns it anymore. +var ErrOutboxFull = errors.New("outbox: full") + +// ErrOverflowed is returned (wrapped) by Enqueue when the event was, instead +// of being written straight into the seq-bearing outbox, durably persisted +// into outbox_overflow: the outbox was at its row cap, or an older event for +// the same space was already waiting in overflow (see the FIFO note on +// Enqueue). It is informational, not a failure — by the time it is +// returned, the event has already survived a process crash. Callers must +// not retry or reroute on this error; PromoteOverflow (called on ack-freed +// capacity and at Store open) moves the event into the outbox, allocating +// its device_seq at that moment, once room and turn allow. +var ErrOverflowed = errors.New("outbox: enqueued to overflow (outbox full or an older overflow entry pending)") + +const schema = ` +CREATE TABLE IF NOT EXISTS space_seq_counters ( + space TEXT PRIMARY KEY, + next_seq INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS outbox ( + space TEXT NOT NULL, + device_seq INTEGER NOT NULL, + kind TEXT NOT NULL, + body TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (space, device_seq) +); +-- outbox_overflow holds reliable events that arrived while the outbox was +-- at capacity (or while an older overflow entry for the same space was +-- still pending promotion). No device_seq column: a seq is only ever +-- allocated when PromoteOverflow moves a row into outbox, never before, so +-- there is nothing to conflict with the outbox's own counter. "body" holds +-- the pre-seq event material — the same JSON Enqueue would have written, +-- built with a placeholder device_seq of 1 purely to satisfy wire.Validate +-- (which requires device_seq >= 1) — with the real value patched in at +-- promotion time (see setDeviceSeq). id's autoincrement order is the FIFO +-- order promotion must honor. +CREATE TABLE IF NOT EXISTS outbox_overflow ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + space TEXT NOT NULL, + kind TEXT NOT NULL, + body TEXT NOT NULL, + created_at INTEGER NOT NULL +); +` + +// Item is one persisted, not-yet-acknowledged reliable event. +type Item struct { + Space string + DeviceSeq int64 + Kind string // "task.event" | "message.event" + Body json.RawMessage + CreatedAt int64 // unix ms +} + +// Store is the outbox's SQLite-backed handle. It is safe for concurrent use +// from multiple goroutines: every method serializes through the underlying +// *sql.DB, and every mutating operation is one transaction. +type Store struct { + db *sql.DB + maxRows int +} + +// Open creates (if needed) and opens the outbox database at path with the +// DefaultMaxRows cap. Callers should keep one *Store per daemon process +// for a given path. +func Open(path string) (*Store, error) { + return OpenWithMaxRows(path, DefaultMaxRows) +} + +// OpenWithMaxRows is Open with an explicit row cap; maxRows <= 0 falls +// back to DefaultMaxRows. +func OpenWithMaxRows(path string, maxRows int) (*Store, error) { + return openWithBusyTimeoutMS(path, maxRows, 5000) +} + +// openWithBusyTimeoutMS is OpenWithMaxRows with the SQLite busy_timeout +// pragma exposed, in milliseconds. Production always goes through +// OpenWithMaxRows (5000ms, see the _txlock=immediate comment below); this +// exists as a same-package test seam so a test can drive a genuinely fast +// SQLITE_BUSY through enqueueAttempt's own bounded retry loop (see +// enqueueMaxAttempts) with a short busy_timeout, rather than needing to hold +// a competing write lock for the real 5s default just to prove that retry +// path recovers a transient error. +func openWithBusyTimeoutMS(path string, maxRows, busyTimeoutMS int) (*Store, error) { + if maxRows <= 0 { + maxRows = DefaultMaxRows + } + // _txlock=immediate: acquire the write lock at BEGIN rather than at + // first write, so two overlapping transactions serialize instead of + // racing to upgrade a deferred lock (SQLITE_BUSY under concurrent + // Enqueue calls from multiple goroutines). + dsn := fmt.Sprintf("%s?_pragma=busy_timeout(%d)&_txlock=immediate", path, busyTimeoutMS) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("outbox: open %s: %w", path, err) + } + // SQLite allows only one writer at a time; a single connection avoids + // spurious SQLITE_BUSY errors from the driver opening a second one. + db.SetMaxOpenConns(1) + if _, err := db.Exec(schema); err != nil { + db.Close() + return nil, fmt.Errorf("outbox: init schema: %w", err) + } + store := &Store{db: db, maxRows: maxRows} + + // Restart recovery: anything left in outbox_overflow from a previous + // process survives on disk exactly where it was, but it only becomes + // eligible for delivery once promoted into the seq-bearing outbox. + // Attempt every promotion the current cap allows right away, so a + // daemon that restarts with free capacity (or with an empty outbox) + // doesn't wait for the next ack to notice overflowed work is waiting. + if _, err := store.PromoteOverflow(context.Background(), maxRows); err != nil { + db.Close() + return nil, fmt.Errorf("outbox: promote overflow on open: %w", err) + } + return store, nil +} + +// Close releases the underlying database handle. +func (s *Store) Close() error { return s.db.Close() } + +// BuildBody produces the JSON body for a reliable event once its device_seq +// is known — Enqueue calls it inside the allocating transaction so the +// caller never has to allocate a seq itself ahead of time. +type BuildBody func(seq int64) (json.RawMessage, error) + +// enqueueMaxAttempts bounds Enqueue's internal retry of a transient SQLite +// failure (BeginTx/COUNT/INSERT/Commit erroring — e.g. SQLITE_BUSY beyond +// what the driver's own busy_timeout pragma already absorbs, or a momentary +// disk I/O blip). The store's DSN already sets busy_timeout(5000), so most +// lock contention resolves inside a single attempt; this loop is a second, +// small layer on top so a transient failure that busy_timeout doesn't cover +// does not surface to the caller as event loss on the first try. +const enqueueMaxAttempts = 3 + +// enqueueRetryDelay is the (small, fixed) pause between Enqueue's internal +// attempts. Deliberately short: Offer() must not block the caller for long, +// and busy_timeout already handles the common lock-contention case. +const enqueueRetryDelay = 20 * time.Millisecond + +// Enqueue durably persists the event BuildBody produces, allocating its +// device_seq atomically with the insert (SPEC.md section 5.1: a crash +// between "seq chosen" and "event durably stored" can never happen — the +// transaction commits both effects or neither). +// +// If the outbox is at its row cap, OR an older event for this same space is +// still sitting in outbox_overflow awaiting promotion, this event is +// persisted into outbox_overflow instead (no device_seq allocated) and +// Enqueue returns ErrOverflowed — the event is durable either way. The +// overflow-nonempty check happens inside the very same transaction as the +// cap check and the insert, which is what prevents the round-1 +// seq-order-inversion race from resurfacing at the DB layer: without it, a +// fresh Enqueue could race a still-overflowed older event for a freed +// outbox row and win it a lower seq, inverting wire order once the older +// event is eventually promoted. Routing every new arrival to overflow +// whenever overflow is non-empty, checked under the same lock as the +// promotion path, makes that race impossible. +// +// A transient failure (not ErrOverflowed, not a BuildBody/validation error) +// is retried internally up to enqueueMaxAttempts times before it is +// returned to the caller — see enqueueMaxAttempts. +func (s *Store) Enqueue(ctx context.Context, space, kind string, build BuildBody) (Item, error) { + var lastErr error + for attempt := 0; attempt < enqueueMaxAttempts; attempt++ { + if attempt > 0 { + select { + case <-time.After(enqueueRetryDelay): + case <-ctx.Done(): + return Item{}, ctx.Err() + } + } + item, terminal, err := s.enqueueAttempt(ctx, space, kind, build) + if err == nil { + return item, nil + } + if terminal { + // ErrOverflowed (success-shaped) or a BuildBody/validation + // failure (the same input can never validate differently): + // neither benefits from a retry. + return item, err + } + lastErr = err + } + return Item{}, fmt.Errorf("outbox: enqueue failed after %d attempts: %w", enqueueMaxAttempts, lastErr) +} + +// enqueueAttempt is one transactional attempt at Enqueue's work. terminal +// reports whether the caller should stop retrying regardless of err being +// non-nil (a BuildBody failure, or the informational ErrOverflowed). +func (s *Store) enqueueAttempt(ctx context.Context, space, kind string, build BuildBody) (item Item, terminal bool, err error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Item{}, false, fmt.Errorf("outbox: begin: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + + // FIFO guard: while an older event for this space is still waiting in + // overflow, this new one must join it there too, even if the outbox + // below is not currently at cap (e.g. an ack freed a row a moment ago, + // but PromoteOverflow hasn't run yet) — never let a fresher event claim + // a seq ahead of an older still-overflowed one. + var overflowPending int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox_overflow WHERE space = ?`, space).Scan(&overflowPending); err != nil { + return Item{}, false, fmt.Errorf("outbox: count overflow: %w", err) + } + + var rows int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&rows); err != nil { + return Item{}, false, fmt.Errorf("outbox: count rows: %w", err) + } + + if overflowPending > 0 || rows >= s.maxRows { + if err := s.insertOverflowTx(ctx, tx, space, kind, build); err != nil { + return Item{}, true, err + } + if err := tx.Commit(); err != nil { + return Item{}, false, fmt.Errorf("outbox: commit overflow insert: %w", err) + } + return Item{}, true, fmt.Errorf("outbox has %d rows (cap %d), %d overflow pending for %s: %w", rows, s.maxRows, overflowPending, space, ErrOverflowed) + } + + seq, err := nextSeqTx(tx, space) + if err != nil { + return Item{}, false, err + } + body, err := build(seq) + if err != nil { + return Item{}, true, fmt.Errorf("outbox: build body for seq %d: %w", seq, err) + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, + `INSERT INTO outbox (space, device_seq, kind, body, created_at) VALUES (?, ?, ?, ?, ?)`, + space, seq, kind, string(body), now, + ); err != nil { + return Item{}, false, fmt.Errorf("outbox: insert seq %d: %w", seq, err) + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO space_seq_counters (space, next_seq) VALUES (?, ?) + ON CONFLICT(space) DO UPDATE SET next_seq = excluded.next_seq`, + space, seq+1, + ); err != nil { + return Item{}, false, fmt.Errorf("outbox: advance counter: %w", err) + } + if err := tx.Commit(); err != nil { + return Item{}, false, fmt.Errorf("outbox: commit: %w", err) + } + return Item{Space: space, DeviceSeq: seq, Kind: kind, Body: body, CreatedAt: now}, false, nil +} + +// insertOverflowTx persists the pre-seq event material into outbox_overflow +// within tx. build is called with a placeholder device_seq of 1 — the +// smallest value wire.Validate accepts — purely so validation can run; the +// real device_seq is patched in at promotion (see setDeviceSeq) and never +// observed on the wire in its placeholder form. +func (s *Store) insertOverflowTx(ctx context.Context, tx *sql.Tx, space, kind string, build BuildBody) error { + const placeholderSeq = 1 + body, err := build(placeholderSeq) + if err != nil { + return fmt.Errorf("outbox: build overflow body: %w", err) + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, + `INSERT INTO outbox_overflow (space, kind, body, created_at) VALUES (?, ?, ?, ?)`, + space, kind, string(body), now, + ); err != nil { + return fmt.Errorf("outbox: insert overflow: %w", err) + } + return nil +} + +// PromoteOverflow moves up to n of the oldest outbox_overflow rows into the +// seq-bearing outbox, allocating each one's device_seq at the moment it is +// promoted (never before). It stops early — before n promotions — once +// either outbox_overflow is empty or the outbox is at its row cap; n is +// simply an upper bound on the number of attempts, so passing a generous +// value (e.g. the store's own maxRows) is always safe. It reports how many +// rows were actually promoted. +// +// Call sites: the ack path (Ack, below) promotes one row per freed outbox +// slot, and Open/OpenWithMaxRows promote everything they can immediately on +// process start (restart recovery) — both are named in the work order as +// the two triggers that must not let a promotable row sit unnoticed. +func (s *Store) PromoteOverflow(ctx context.Context, n int) (int, error) { + promoted := 0 + for i := 0; i < n; i++ { + ok, err := s.promoteOneTx(ctx) + if err != nil { + return promoted, err + } + if !ok { + break + } + promoted++ + } + return promoted, nil +} + +// promoteOneTx promotes exactly the single oldest outbox_overflow row +// (lowest id, across all spaces — id order is overflow's FIFO order) into +// outbox, in one transaction: read the cap, read the oldest row, allocate +// its space's next device_seq, patch that seq into the stored body, insert +// into outbox, advance the counter, and delete the overflow row. ok is +// false (no error) when there was nothing to promote, or no room to promote +// it into. +func (s *Store) promoteOneTx(ctx context.Context) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("outbox: begin promotion: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + + var rows int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&rows); err != nil { + return false, fmt.Errorf("outbox: count rows: %w", err) + } + if rows >= s.maxRows { + return false, nil // no room; caller should stop + } + + var id int64 + var space, kind, body string + err = tx.QueryRowContext(ctx, + `SELECT id, space, kind, body FROM outbox_overflow ORDER BY id ASC LIMIT 1`, + ).Scan(&id, &space, &kind, &body) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("outbox: read oldest overflow row: %w", err) + } + + seq, err := nextSeqTx(tx, space) + if err != nil { + return false, err + } + finalBody, err := setDeviceSeq(json.RawMessage(body), seq) + if err != nil { + return false, fmt.Errorf("outbox: patch device_seq on promotion of overflow id %d: %w", id, err) + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, + `INSERT INTO outbox (space, device_seq, kind, body, created_at) VALUES (?, ?, ?, ?, ?)`, + space, seq, kind, string(finalBody), now, + ); err != nil { + return false, fmt.Errorf("outbox: insert promoted seq %d: %w", seq, err) + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO space_seq_counters (space, next_seq) VALUES (?, ?) + ON CONFLICT(space) DO UPDATE SET next_seq = excluded.next_seq`, + space, seq+1, + ); err != nil { + return false, fmt.Errorf("outbox: advance counter: %w", err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM outbox_overflow WHERE id = ?`, id); err != nil { + return false, fmt.Errorf("outbox: delete promoted overflow id %d: %w", id, err) + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("outbox: commit promotion: %w", err) + } + return true, nil +} + +// setDeviceSeq returns body with its top-level "device_seq" field replaced +// by seq, leaving every other field untouched. It works generically across +// both reliable event kinds (task.event/message.event) because both +// wire.TaskEventBody and wire.MessageEventBody serialize a "device_seq" +// field — this package intentionally does not import the wire package to +// do this via its concrete types, so the overflow mechanism stays agnostic +// to any future reliable event kind that follows the same convention. +func setDeviceSeq(body json.RawMessage, seq int64) (json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(body, &fields); err != nil { + return nil, fmt.Errorf("decode body: %w", err) + } + seqJSON, err := json.Marshal(seq) + if err != nil { + return nil, err + } + fields["device_seq"] = seqJSON + patched, err := json.Marshal(fields) + if err != nil { + return nil, fmt.Errorf("re-encode body: %w", err) + } + return patched, nil +} + +// nextSeqTx reads and does NOT yet persist the next device_seq for space; +// the caller is expected to write space_seq_counters within the same +// transaction once it has successfully built and inserted the event that +// consumes this value (see Enqueue). Device_seq starts at 1 (SPEC.md +// section 5.1). +func nextSeqTx(tx *sql.Tx, space string) (int64, error) { + var next int64 + err := tx.QueryRow(`SELECT next_seq FROM space_seq_counters WHERE space = ?`, space).Scan(&next) + switch { + case err == sql.ErrNoRows: + return 1, nil + case err != nil: + return 0, fmt.Errorf("outbox: read counter: %w", err) + default: + return next, nil + } +} + +// NextSeq reports the device_seq the next Enqueue(space, ...) call will +// allocate, without allocating it. Intended for tests and diagnostics. +func (s *Store) NextSeq(ctx context.Context, space string) (int64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() //nolint:errcheck + return nextSeqTx(tx, space) +} + +// Ack deletes exactly the (space, device_seq) pair, per SPEC.md section 5.3 +// ("A source device MUST keep every sent task.event/message.event in a +// local resend queue until it receives an ack that covers its device_seq"). +// Acking a seq that is not present (already acked, or never enqueued here) +// is a no-op, not an error — acks are idempotent (SPEC.md section 5.2). +// +// An ack is also one of the two triggers (the other is Store open) that +// must notice freed outbox capacity and promote a waiting overflow row: it +// attempts exactly one promotion afterward, best-effort. A promotion +// failure here is logged-and-swallowed rather than failing the ack itself +// (the ack succeeded; the event really was acknowledged) — the next ack or +// the caller's own periodic tick gets another chance. +func (s *Store) Ack(ctx context.Context, space string, seq int64) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ? AND device_seq = ?`, space, seq) + if err != nil { + return fmt.Errorf("outbox: ack seq %d: %w", seq, err) + } + _, _ = s.PromoteOverflow(ctx, 1) //nolint:errcheck // best-effort; see doc comment + return nil +} + +// Pending returns every outstanding (unacknowledged) item for space, +// oldest-first by device_seq — the order SPEC.md section 5.3/5.4 requires +// for reconnect replay ("resend every still-unacked event, oldest first, +// immediately after hello/hello-accept completes on a new connection"). +// Because the outbox is durable, this also serves a process restart: the +// queue is exactly as it was left, so a freshly started daemon replays from +// disk with no special-cased recovery path. +func (s *Store) Pending(ctx context.Context, space string) ([]Item, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT space, device_seq, kind, body, created_at FROM outbox WHERE space = ? ORDER BY device_seq ASC`, + space) + if err != nil { + return nil, fmt.Errorf("outbox: query pending: %w", err) + } + defer rows.Close() + var items []Item + for rows.Next() { + var it Item + var body string + if err := rows.Scan(&it.Space, &it.DeviceSeq, &it.Kind, &body, &it.CreatedAt); err != nil { + return nil, fmt.Errorf("outbox: scan pending: %w", err) + } + it.Body = json.RawMessage(body) + items = append(items, it) + } + return items, rows.Err() +} + +// Count returns the number of unacknowledged items across every space, +// mainly for observability/tests. +func (s *Store) Count(ctx context.Context) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox`).Scan(&n) + return n, err +} + +// OverflowItem is one durably-persisted, not-yet-promoted reliable event +// sitting in outbox_overflow. Unlike Item it carries no DeviceSeq — none has +// been allocated yet — and an ID instead, which is both its promotion-FIFO +// key and the handle DeleteOverflow needs. +type OverflowItem struct { + ID int64 + Space string + Kind string + Body json.RawMessage // pre-seq material; device_seq is a placeholder until promoted + CreatedAt int64 +} + +// OverflowPending returns every overflowed item for space, oldest-first by +// id — the same FIFO order PromoteOverflow honors. Intended for a caller +// that needs to drain overflow through a path other than promotion, e.g. +// the daemon's legacy-HTTP fallback when the server has disabled realtime +// entirely (see internal/uplink). +func (s *Store) OverflowPending(ctx context.Context, space string) ([]OverflowItem, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, space, kind, body, created_at FROM outbox_overflow WHERE space = ? ORDER BY id ASC`, + space) + if err != nil { + return nil, fmt.Errorf("outbox: query overflow pending: %w", err) + } + defer rows.Close() + var items []OverflowItem + for rows.Next() { + var it OverflowItem + var body string + if err := rows.Scan(&it.ID, &it.Space, &it.Kind, &body, &it.CreatedAt); err != nil { + return nil, fmt.Errorf("outbox: scan overflow pending: %w", err) + } + it.Body = json.RawMessage(body) + items = append(items, it) + } + return items, rows.Err() +} + +// DeleteOverflow removes exactly the given overflow row by id, without +// promoting it into outbox or allocating it a device_seq — for a caller +// that has durably delivered the event through some other path entirely +// (the legacy-HTTP fallback) and is retiring it the same way Ack retires a +// promoted item. Deleting an id that is not present is a no-op. +func (s *Store) DeleteOverflow(ctx context.Context, id int64) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM outbox_overflow WHERE id = ?`, id) + if err != nil { + return fmt.Errorf("outbox: delete overflow id %d: %w", id, err) + } + return nil +} + +// OverflowCount returns the number of items currently sitting in +// outbox_overflow across every space, mainly for observability/tests. +func (s *Store) OverflowCount(ctx context.Context) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM outbox_overflow`).Scan(&n) + return n, err +} diff --git a/daemon/internal/realtime/outbox/outbox_test.go b/daemon/internal/realtime/outbox/outbox_test.go new file mode 100644 index 00000000..23a59588 --- /dev/null +++ b/daemon/internal/realtime/outbox/outbox_test.go @@ -0,0 +1,564 @@ +package outbox + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "sync" + "testing" + "time" +) + +func open(t *testing.T) *Store { + t.Helper() + path := filepath.Join(t.TempDir(), "outbox.db") + s, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func bodyFor(seq int64) BuildBody { + return func(s int64) (json.RawMessage, error) { + return json.RawMessage(fmt.Sprintf(`{"device_seq":%d}`, s)), nil + } +} + +func TestEnqueueAllocatesMonotonicSeq(t *testing.T) { + s := open(t) + ctx := context.Background() + for want := int64(1); want <= 5; want++ { + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(want)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if item.DeviceSeq != want { + t.Fatalf("Enqueue seq = %d, want %d", item.DeviceSeq, want) + } + } + // A second space starts its own counter at 1 (device_seq is scoped to + // one (device, space) pair, SPEC.md section 5.1). + item, err := s.Enqueue(ctx, "space-b", "task.event", bodyFor(1)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if item.DeviceSeq != 1 { + t.Fatalf("space-b first seq = %d, want 1", item.DeviceSeq) + } +} + +func TestEnqueueConcurrentIsMonotonicAndUnique(t *testing.T) { + s := open(t) + ctx := context.Background() + const n = 50 + seqs := make([]int64, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if err != nil { + t.Errorf("Enqueue: %v", err) + return + } + seqs[i] = item.DeviceSeq + }(i) + } + wg.Wait() + seen := make(map[int64]bool, n) + for _, seq := range seqs { + if seq < 1 { + t.Fatalf("got invalid seq %d", seq) + } + if seen[seq] { + t.Fatalf("seq %d allocated twice", seq) + } + seen[seq] = true + } + if len(seen) != n { + t.Fatalf("got %d unique seqs, want %d", len(seen), n) + } +} + +// TestEnqueueFailureLeavesNoGap asserts the crash-safety property required +// by the work order: allocating a device_seq and durably enqueueing the +// event it belongs to happen in ONE transaction. If building the event body +// fails after the seq was chosen but before the transaction commits, the +// whole transaction MUST roll back — including the counter advance — so the +// next successful Enqueue reuses the same seq rather than skipping it. This +// stands in for an actual process crash between "seq chosen" and "event +// durably stored": from the database's perspective the two are +// indistinguishable, and both must leave the counter exactly where it was. +func TestEnqueueFailureLeavesNoGap(t *testing.T) { + s := open(t) + ctx := context.Background() + + boom := errors.New("boom") + _, err := s.Enqueue(ctx, "space-a", "task.event", func(seq int64) (json.RawMessage, error) { + if seq != 1 { + t.Fatalf("expected first attempted seq to be 1, got %d", seq) + } + return nil, boom + }) + if !errors.Is(err, boom) { + t.Fatalf("Enqueue error = %v, want wrapping %v", err, boom) + } + + next, err := s.NextSeq(ctx, "space-a") + if err != nil { + t.Fatalf("NextSeq: %v", err) + } + if next != 1 { + t.Fatalf("NextSeq after failed build = %d, want 1 (no gap left by the aborted transaction)", next) + } + + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(1)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if item.DeviceSeq != 1 { + t.Fatalf("seq after retry = %d, want 1 (reused, not skipped)", item.DeviceSeq) + } + + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("Pending returned %d items, want exactly 1 (the failed build must not have left a row)", len(pending)) + } +} + +func TestEnqueueAckDeletes(t *testing.T) { + s := open(t) + ctx := context.Background() + item, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(1)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil || len(pending) != 1 { + t.Fatalf("Pending = %v, %v; want 1 item", pending, err) + } + if err := s.Ack(ctx, "space-a", item.DeviceSeq); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err = s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 0 { + t.Fatalf("Pending after ack = %v, want empty", pending) + } + // Acking an already-acked (or never-enqueued) seq is a no-op, not an + // error (SPEC.md section 5.2: acking a duplicate is always safe). + if err := s.Ack(ctx, "space-a", item.DeviceSeq); err != nil { + t.Fatalf("double Ack: %v", err) + } + if err := s.Ack(ctx, "space-a", 999); err != nil { + t.Fatalf("Ack of unknown seq: %v", err) + } +} + +func TestPendingOrderedOldestFirst(t *testing.T) { + s := open(t) + ctx := context.Background() + for i := int64(1); i <= 5; i++ { + if _, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(i)); err != nil { + t.Fatalf("Enqueue: %v", err) + } + } + // Ack the middle one out of order to make sure ordering survives gaps. + if err := s.Ack(ctx, "space-a", 3); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + want := []int64{1, 2, 4, 5} + if len(pending) != len(want) { + t.Fatalf("Pending = %d items, want %d", len(pending), len(want)) + } + for i, w := range want { + if pending[i].DeviceSeq != w { + t.Fatalf("Pending[%d].DeviceSeq = %d, want %d (order: %v)", i, pending[i].DeviceSeq, w, pending) + } + } +} + +// TestSurvivesRestart simulates a daemon process restart: closing the Store +// and reopening the same database file must preserve both the pending +// queue (for replay) and the device_seq counter (so a restarted source +// never reuses or skips a sequence number). +func TestSurvivesRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + ctx := context.Background() + + s1, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + if _, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(1)); err != nil { + t.Fatalf("Enqueue: %v", err) + } + if _, err := s1.Enqueue(ctx, "space-a", "message.event", bodyFor(2)); err != nil { + t.Fatalf("Enqueue: %v", err) + } + // This one gets acked before the "restart" — it must NOT reappear. + item3, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if err := s1.Ack(ctx, "space-a", item3.DeviceSeq); err != nil { + t.Fatalf("Ack: %v", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + s2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + pending, err := s2.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending after restart: %v", err) + } + if len(pending) != 2 { + t.Fatalf("Pending after restart = %d items, want 2 (got %v)", len(pending), pending) + } + if pending[0].DeviceSeq != 1 || pending[1].DeviceSeq != 2 { + t.Fatalf("Pending after restart out of order: %v", pending) + } + + next, err := s2.Enqueue(ctx, "space-a", "task.event", bodyFor(4)) + if err != nil { + t.Fatalf("Enqueue after restart: %v", err) + } + if next.DeviceSeq != 4 { + t.Fatalf("seq after restart = %d, want 4 (counter must survive restart)", next.DeviceSeq) + } +} + +// TestEnqueueFullOverflows pins the bounded-outbox policy: at the row cap, +// Enqueue does not fail — it durably persists the event into +// outbox_overflow and returns ErrOverflowed, consuming no device_seq (no +// seq is ever allocated for an overflowed event until it is promoted). An +// ack freeing outbox capacity then promotes the overflowed event, +// allocating it the next unconsumed seq at that moment. +func TestEnqueueFullOverflows(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s, err := OpenWithMaxRows(path, 2) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer s.Close() + ctx := context.Background() + + for i := int64(1); i <= 2; i++ { + if _, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(i)); err != nil { + t.Fatalf("Enqueue %d: %v", i, err) + } + } + + _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(3)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("Enqueue at cap = %v, want ErrOverflowed", err) + } + + // The overflowed Enqueue must not have consumed a sequence number: no + // seq is allocated until promotion. + next, err := s.NextSeq(ctx, "space-a") + if err != nil { + t.Fatalf("NextSeq: %v", err) + } + if next != 3 { + t.Fatalf("NextSeq after overflow = %d, want 3 (an overflowed Enqueue must not consume a seq)", next) + } + overflowCount, err := s.OverflowCount(ctx) + if err != nil { + t.Fatalf("OverflowCount: %v", err) + } + if overflowCount != 1 { + t.Fatalf("OverflowCount = %d, want 1", overflowCount) + } + + // Freeing one row (an ack arrived) promotes the overflowed event, + // allocating it the next unconsumed seq. + if err := s.Ack(ctx, "space-a", 1); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 2 { + t.Fatalf("Pending after ack+promotion = %d items, want 2 (seq 2 and the promoted seq 3): %v", len(pending), pending) + } + if pending[len(pending)-1].DeviceSeq != 3 { + t.Fatalf("promoted item's seq = %d, want 3", pending[len(pending)-1].DeviceSeq) + } + overflowCount, err = s.OverflowCount(ctx) + if err != nil { + t.Fatalf("OverflowCount after promotion: %v", err) + } + if overflowCount != 0 { + t.Fatalf("OverflowCount after promotion = %d, want 0", overflowCount) + } +} + +// TestEnqueueRoutesToOverflowWhileOlderOverflowPending is the DB-layer +// regression test for the round-1 seq-order-inversion race, reproduced at +// the store instead of the uplink: a fresh Enqueue must never claim a seq +// via a just-freed outbox row while an older event for the same space is +// still waiting in overflow — even when the outbox itself currently has +// room. The check must live inside Enqueue's own transaction (not depend on +// a caller doing the right thing), so this test frees a row by a route +// other than Ack/PromoteOverflow (a direct delete) to isolate that +// invariant from Ack's own promotion behavior. +func TestEnqueueRoutesToOverflowWhileOlderOverflowPending(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s, err := OpenWithMaxRows(path, 2) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer s.Close() + ctx := context.Background() + + for i := 0; i < 2; i++ { + if _, err := s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)); err != nil { + t.Fatalf("fill Enqueue %d: %v", i, err) + } + } + // A overflows: the outbox is full. + _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("A's Enqueue at cap = %v, want ErrOverflowed", err) + } + + // Free BOTH filler rows (enough room for both A and B to eventually + // promote) WITHOUT going through Ack, so no promotion runs yet — + // simulates "capacity freed" independent of this store's own + // ack-triggers-promotion wiring, isolating the Enqueue-side guard. + if _, err := s.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ?`, "space-a"); err != nil { + t.Fatalf("simulate freed rows: %v", err) + } + + // B is offered now: the outbox is completely empty (full room), but A + // is still waiting in overflow. B must join it there — claiming a row + // directly would hand B a lower seq than A gets once A is finally + // promoted, inverting wire order. + _, err = s.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("B's Enqueue while outbox has room but overflow is non-empty = %v, want ErrOverflowed", err) + } + + if _, err := s.PromoteOverflow(ctx, 2); err != nil { + t.Fatalf("PromoteOverflow: %v", err) + } + pending, err := s.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + // A and B, promoted in FIFO order. + if len(pending) != 2 { + t.Fatalf("Pending after promotion = %d items, want 2: %v", len(pending), pending) + } + seqA, seqB := pending[0].DeviceSeq, pending[1].DeviceSeq + if !(seqA < seqB) { + t.Fatalf("wire order inverted: expected A's seq < B's seq (A overflowed first), got A=%d B=%d", seqA, seqB) + } +} + +// TestOverflowPromotesOnOpenWhenCapacityFree pins the second required +// promotion trigger (the first is Ack, above): a fresh Store.Open must +// itself notice a waiting overflow row that has room to be promoted into +// and do so immediately, rather than waiting for the next ack — otherwise a +// daemon that restarts with an already-drained outbox would leave a +// perfectly promotable event stranded until unrelated future traffic +// happened to trigger an ack. +func TestOverflowPromotesOnOpenWhenCapacityFree(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s1, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + ctx := context.Background() + + filler, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if err != nil { + t.Fatalf("filler Enqueue: %v", err) + } + _, err = s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("overflow Enqueue = %v, want ErrOverflowed", err) + } + + // Free the filler's row without running this store's own + // promote-on-ack wiring, so the only remaining trigger that could + // notice the freed capacity is the next Open. + if _, err := s1.db.ExecContext(ctx, `DELETE FROM outbox WHERE space = ? AND device_seq = ?`, "space-a", filler.DeviceSeq); err != nil { + t.Fatalf("simulate freed row: %v", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + s2, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + pending, err := s2.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending after reopen: %v", err) + } + if len(pending) != 1 { + t.Fatalf("expected Open to promote the waiting overflow row into freed capacity, got %d pending: %v", len(pending), pending) + } + if n, err := s2.OverflowCount(ctx); err != nil || n != 0 { + t.Fatalf("OverflowCount after reopen-promotion = %d, %v, want 0", n, err) + } +} + +// TestOverflowSurvivesCloseAndReopen is the outbox-layer half of the +// reviewer's exact scenario (the uplink/client-level version lives in +// internal/uplink): an overflowed event, and the outbox rows that keep it +// overflowed, must both survive a process restart (Close + reopen the same +// database file) unchanged, and only promote once an ack actually frees +// capacity on the reopened store. +func TestOverflowSurvivesCloseAndReopen(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.db") + s1, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + ctx := context.Background() + + filler, err := s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if err != nil { + t.Fatalf("filler Enqueue: %v", err) + } + _, err = s1.Enqueue(ctx, "space-a", "task.event", bodyFor(0)) + if !errors.Is(err, ErrOverflowed) { + t.Fatalf("overflow Enqueue = %v, want ErrOverflowed", err) + } + if err := s1.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + s2, err := OpenWithMaxRows(path, 1) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + // Nothing to promote yet: the outbox is still full (the filler is + // still unacked), so Open's promotion attempt is a no-op. + if n, err := s2.OverflowCount(ctx); err != nil || n != 1 { + t.Fatalf("OverflowCount after reopen = %d, %v, want 1 (outbox still full, nothing promotable)", n, err) + } + + if err := s2.Ack(ctx, "space-a", filler.DeviceSeq); err != nil { + t.Fatalf("Ack: %v", err) + } + pending, err := s2.Pending(ctx, "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 { + t.Fatalf("Pending after ack+promotion = %d items, want 1: %v", len(pending), pending) + } + if n, err := s2.OverflowCount(ctx); err != nil || n != 0 { + t.Fatalf("OverflowCount after ack+promotion = %d, %v, want 0", n, err) + } +} + +// TestEnqueueRecoversFromRealTransientLockContention drives a genuine +// SQLITE_BUSY through enqueueAttempt's own bounded retry loop +// (enqueueMaxAttempts / enqueueRetryDelay) — not a fabricated send/build +// callback error, but an actual second connection holding the write lock a +// real Enqueue call has to wait out. It asserts the event still ends up +// durable and readable (never dropped) once the lock clears within the +// retry budget. +// +// A short busy_timeout (via the openWithBusyTimeoutMS test seam, rather +// than production's 5000ms) keeps this fast: with the competing lock held +// for holdDuration — comfortably inside the window of Enqueue's SECOND +// attempt (busyTimeoutMS + enqueueRetryDelay through 2*busyTimeoutMS + +// enqueueRetryDelay) and comfortably past its first (which must therefore +// see a real SQLITE_BUSY and retry) — the first attempt is guaranteed to +// fail on genuine lock contention, and the second is guaranteed to recover +// once the lock actually releases, with margin on both sides against +// scheduler jitter. +func TestEnqueueRecoversFromRealTransientLockContention(t *testing.T) { + const busyTimeoutMS = 100 + const holdDuration = 170 * time.Millisecond + + path := filepath.Join(t.TempDir(), "outbox.db") + + s, err := openWithBusyTimeoutMS(path, DefaultMaxRows, busyTimeoutMS) + if err != nil { + t.Fatalf("openWithBusyTimeoutMS: %v", err) + } + defer s.Close() + + // A second, independent connection to the same database file. Its own + // BEGIN (immediate, via the same _txlock=immediate DSN param + // OpenWithMaxRows itself relies on) genuinely acquires SQLite's write + // lock — this is real lock contention, not a simulated error. + dsn := fmt.Sprintf("%s?_pragma=busy_timeout(%d)&_txlock=immediate", path, busyTimeoutMS) + contender, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("open contender connection: %v", err) + } + defer contender.Close() + contender.SetMaxOpenConns(1) + + lockTx, err := contender.Begin() + if err != nil { + t.Fatalf("contender BEGIN IMMEDIATE: %v", err) + } + released := make(chan struct{}) + go func() { + time.Sleep(holdDuration) + if err := lockTx.Commit(); err != nil { + t.Errorf("contender commit: %v", err) + } + close(released) + }() + t.Cleanup(func() { <-released }) + + start := time.Now() + item, err := s.Enqueue(context.Background(), "space-a", "message.event", bodyFor(0)) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("Enqueue: %v (expected it to recover once the competing lock released after %v; Enqueue took %v)", err, holdDuration, elapsed) + } + if item.DeviceSeq != 1 { + t.Fatalf("Enqueue seq = %d, want 1", item.DeviceSeq) + } + if elapsed < holdDuration { + t.Fatalf("Enqueue returned after %v, before the competing lock even released at %v — contention was never actually exercised", elapsed, holdDuration) + } + + // Durable and readable, not silently dropped. + pending, err := s.Pending(context.Background(), "space-a") + if err != nil { + t.Fatalf("Pending: %v", err) + } + if len(pending) != 1 || pending[0].DeviceSeq != 1 { + t.Fatalf("Pending = %+v, want exactly the one durably-enqueued event", pending) + } +} diff --git a/daemon/internal/realtime/rttest/server.go b/daemon/internal/realtime/rttest/server.go new file mode 100644 index 00000000..22ab4c9d --- /dev/null +++ b/daemon/internal/realtime/rttest/server.go @@ -0,0 +1,187 @@ +// Package rttest is a test-only, minimal WebSocket server used to exercise +// the realtime client (internal/realtime/client) against scripted server +// behavior: hello accept, acking, withholding acks, dropping connections, +// sending commands, and the ping/pong heartbeat. +// +// This is NOT a reimplementation of the real Sitrep server: it does not +// persist state, fold events, track space_revision, manage interest +// leases, or implement anything resembling SpaceHub. It exists purely so +// daemon-side tests can drive the client through the connection sequences +// SPEC.md describes without a network dependency on a real server. It must +// never be described as, or grow into, a self-hosted server implementation. +package rttest + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// Conn wraps one accepted connection with envelope-level read/write helpers. +type Conn struct { + ws *websocket.Conn + ctx context.Context + + // Request carries the *http.Request that established this connection, + // e.g. for asserting on the Authorization header a real client sends + // (SPEC.md section 10: the credential is presented once, out of band, + // at connection establishment). + Request *http.Request +} + +// ReadEnvelope reads one frame and decodes it as an envelope. A literal +// "ping"/"pong" text frame is returned as a sentinel error so callers can +// distinguish it from a decode failure. +var ErrPing = fmt.Errorf("rttest: received ping") +var ErrPong = fmt.Errorf("rttest: received pong") + +func (c *Conn) ReadEnvelope() (wire.Envelope, error) { + typ, data, err := c.ws.Read(c.ctx) + if err != nil { + return wire.Envelope{}, err + } + if typ == websocket.MessageText { + switch string(data) { + case "ping": + return wire.Envelope{}, ErrPing + case "pong": + return wire.Envelope{}, ErrPong + } + } + return wire.DecodeEnvelope(data) +} + +// WriteEnvelope encodes and sends one envelope. +func (c *Conn) WriteEnvelope(env wire.Envelope) error { + data, err := env.Encode() + if err != nil { + return err + } + return c.ws.Write(c.ctx, websocket.MessageText, data) +} + +// WritePing / WritePong send the bare heartbeat text frames (SPEC.md +// section 9.3) — deliberately not JSON envelopes. +func (c *Conn) WritePing() error { return c.ws.Write(c.ctx, websocket.MessageText, []byte("ping")) } +func (c *Conn) WritePong() error { return c.ws.Write(c.ctx, websocket.MessageText, []byte("pong")) } + +// Close closes the connection with the given close code/reason. +func (c *Conn) Close(reason string) error { + return c.ws.Close(websocket.StatusNormalClosure, reason) +} + +// HelloAccept performs the mandatory first exchange (SPEC.md section 9): +// reads the client's hello{stage: offer}, validates it is well-formed, and +// replies hello{stage: accept} with the given session id and heartbeat +// interval. It returns the decoded offer for the caller's assertions. +func (c *Conn) HelloAccept(sessionID string, heartbeatMS int) (wire.HelloOffer, error) { + env, err := c.ReadEnvelope() + if err != nil { + return wire.HelloOffer{}, fmt.Errorf("rttest: read hello offer: %w", err) + } + if env.Type != wire.TypeHello { + return wire.HelloOffer{}, fmt.Errorf("rttest: expected hello, got %q", env.Type) + } + body, err := wire.DecodeBody(env) + if err != nil { + return wire.HelloOffer{}, fmt.Errorf("rttest: decode hello offer: %w", err) + } + hb := body.(wire.HelloBody) + if hb.Offer == nil { + return wire.HelloOffer{}, fmt.Errorf("rttest: expected hello offer, got accept") + } + accept := wire.HelloAccept{ + Stage: "accept", + ProtocolVersion: 1, + SessionID: sessionID, + HeartbeatIntervalMS: heartbeatMS, + } + acceptEnv, err := wire.NewEnvelope(wire.TypeHello, newID(), NowMS(), wire.HelloBody{Accept: &accept}) + if err != nil { + return wire.HelloOffer{}, err + } + if err := c.WriteEnvelope(acceptEnv); err != nil { + return wire.HelloOffer{}, err + } + return *hb.Offer, nil +} + +// Ack acknowledges one reliable event. +func (c *Conn) Ack(deviceID string, seq int64) error { + body := wire.AckBody{Acked: []wire.AckedPair{{DeviceID: deviceID, DeviceSeq: seq}}} + env, err := wire.NewEnvelope(wire.TypeAck, newID(), NowMS(), body) + if err != nil { + return err + } + return c.WriteEnvelope(env) +} + +// SendCommand sends a server-originated command (e.g. throttle/resume_rate). +func (c *Conn) SendCommand(body wire.CommandBody) error { + env, err := wire.NewEnvelope(wire.TypeCommand, newID(), NowMS(), body) + if err != nil { + return err + } + return c.WriteEnvelope(env) +} + +// SendError sends an error envelope (e.g. superseded). +func (c *Conn) SendError(body wire.ErrorBody) error { + env, err := wire.NewEnvelope(wire.TypeError, newID(), NowMS(), body) + if err != nil { + return err + } + return c.WriteEnvelope(env) +} + +// Handler is invoked once per accepted connection, in its own goroutine. +// The handler owns the connection's lifetime: when it returns, the +// connection is closed. +type Handler func(*Conn) + +// Server is a scriptable mock realtime endpoint bound to an httptest server. +type Server struct { + httpSrv *httptest.Server + handler Handler + + mu sync.Mutex + conns []*Conn +} + +// New starts a mock server that invokes handler for every accepted +// connection. Call URL() to get the ws:// address for a client to dial. +func New(handler Handler) *Server { + s := &Server{handler: handler} + mux := http.NewServeMux() + mux.HandleFunc("/", s.serve) + s.httpSrv = httptest.NewServer(mux) + return s +} + +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + conn := &Conn{ws: ws, ctx: r.Context(), Request: r} + s.mu.Lock() + s.conns = append(s.conns, conn) + s.mu.Unlock() + s.handler(conn) + ws.Close(websocket.StatusNormalClosure, "handler done") +} + +// URL returns the ws:// URL clients should dial. +func (s *Server) URL() string { + return "ws" + strings.TrimPrefix(s.httpSrv.URL, "http") +} + +// Close shuts down the underlying HTTP server. +func (s *Server) Close() { s.httpSrv.Close() } diff --git a/daemon/internal/realtime/rttest/util.go b/daemon/internal/realtime/rttest/util.go new file mode 100644 index 00000000..73a2141f --- /dev/null +++ b/daemon/internal/realtime/rttest/util.go @@ -0,0 +1,19 @@ +package rttest + +import ( + "crypto/rand" + "encoding/hex" + "time" +) + +// newID returns an opaque envelope id matching common.schema.json's +// envelope_id pattern. +func newID() string { + b := make([]byte, 12) + _, _ = rand.Read(b) + return "rt" + hex.EncodeToString(b) +} + +// NowMS returns the current time as Unix milliseconds, always within the +// valid unix_ms_timestamp bound (SPEC.md section 3.1). +func NowMS() int64 { return time.Now().UnixMilli() } diff --git a/daemon/internal/realtime/wire/authz.go b/daemon/internal/realtime/wire/authz.go new file mode 100644 index 00000000..bca590fc --- /dev/null +++ b/daemon/internal/realtime/wire/authz.go @@ -0,0 +1,77 @@ +package wire + +import "fmt" + +// Role is the realtime-protocol role of a connecting device (SPEC.md +// section 9.2) — coarser than and distinct from the HTTP-layer roles used +// for pairing/REST. +type Role string + +const ( + RoleSource Role = "source" + RoleViewer Role = "viewer" +) + +// Authorize checks a client-originated envelope against the section 10.1 +// authorization matrix, plus the two rules that apply regardless of the +// declared role: +// - stage:"accept" belongs exclusively to the server (section 9.1); any +// client-sent accept is a handshake violation, answered with +// error{code: hello_required} rather than the ordinary +// error{code: unauthorized} the matrix would otherwise suggest. +// - command{origin: "server"} can never be sent by a client, regardless +// of the sender's role or the action named (section 8). +// +// body must be the already-decoded, already-Validate()'d body value +// returned by DecodeBody for the same envelope. Authorize does not itself +// re-run schema validation. +func Authorize(role Role, envType string, body any) error { + switch envType { + case TypeHello: + hb, ok := body.(HelloBody) + if !ok { + return fmt.Errorf("authorize: expected HelloBody, got %T", body) + } + if hb.Accept != nil { + return fmt.Errorf("%s: stage \"accept\" may only be sent by the server", ErrHelloRequired) + } + // stage "offer" is permitted from both source and viewer. + return nil + case TypeResume, TypeSubscribe, TypeUnsubscribe, TypeInterestRenew: + if role != RoleViewer { + return fmt.Errorf("%s: %q may only be sent by a viewer", ErrUnauthorized, envType) + } + return nil + case TypeSnapshot, TypeDelta, TypeConfigEvent: + return fmt.Errorf("%s: %q is server-only, a client MUST NOT send it", ErrUnauthorized, envType) + case TypeTaskEvent, TypeMessageEvent, TypeMetricFrame: + if role != RoleSource { + return fmt.Errorf("%s: %q may only be sent by a source", ErrUnauthorized, envType) + } + return nil + case TypeAck: + if role != RoleSource { + return fmt.Errorf("%s: only a source may send ack (optional command-delivery ack)", ErrUnauthorized) + } + return nil + case TypeCommand: + cb, ok := body.(CommandBody) + if !ok { + return fmt.Errorf("authorize: expected CommandBody, got %T", body) + } + // This rule applies regardless of role: a client can never + // originate a server-side command, whatever action it names. + if cb.Origin == "server" { + return fmt.Errorf("%s: a client may never send command with origin \"server\"", ErrUnauthorized) + } + if role != RoleViewer { + return fmt.Errorf("%s: command may only be sent by a viewer (with origin viewer)", ErrUnauthorized) + } + return nil + case TypeError: + return nil // either role may send error + default: + // Unrecognized type: SPEC.md section 15 says ignore, not reject. + return nil + } +} diff --git a/daemon/internal/realtime/wire/bodies.go b/daemon/internal/realtime/wire/bodies.go new file mode 100644 index 00000000..d9c47cc5 --- /dev/null +++ b/daemon/internal/realtime/wire/bodies.go @@ -0,0 +1,337 @@ +package wire + +import ( + "encoding/json" + "fmt" +) + +// DisplayHints mirrors common.schema.json#/$defs/display_hints. +type DisplayHints struct { + Icon string `json:"icon,omitempty"` + Tint string `json:"tint,omitempty"` + Template string `json:"template,omitempty"` +} + +// TaskState mirrors common.schema.json#/$defs/task_state. +type TaskState struct { + TaskID string `json:"task_id"` + DeviceID string `json:"device_id"` + Title string `json:"title,omitempty"` + State string `json:"state"` // running|done|failed + Percent *int `json:"percent,omitempty"` + Step string `json:"step,omitempty"` + Message string `json:"message,omitempty"` + UpdatedAt int64 `json:"updated_at"` + Display *DisplayHints `json:"display,omitempty"` +} + +// MetricSample mirrors common.schema.json#/$defs/metric_sample. +type MetricSample struct { + MetricID string `json:"metric_id"` + Value string `json:"value"` + Label string `json:"label,omitempty"` + TS int64 `json:"ts"` + Display *DisplayHints `json:"display,omitempty"` + Target string `json:"target,omitempty"` + Min string `json:"min,omitempty"` + Max string `json:"max,omitempty"` + AlertAbove string `json:"alert_above,omitempty"` + AlertBelow string `json:"alert_below,omitempty"` +} + +// Schedule mirrors the schedule object nested in automation_state. +type Schedule struct { + Kind string `json:"kind"` // always "interval" in v1 + EverySeconds int `json:"every_seconds"` +} + +// AutomationState mirrors common.schema.json#/$defs/automation_state. +type AutomationState struct { + AutomationID string `json:"automation_id"` + Name string `json:"name"` + ExecutorKind string `json:"executor_kind"` // script|agent|hybrid + Schedule Schedule `json:"schedule"` + State string `json:"state"` // active|paused + LastRunAt int64 `json:"last_run_at,omitempty"` +} + +// MessageRecord mirrors common.schema.json#/$defs/message_record. +type MessageRecord struct { + MessageID string `json:"message_id"` + DeviceID string `json:"device_id"` + Level string `json:"level"` + Text string `json:"text"` + OccurredAt int64 `json:"occurred_at"` +} + +// ---- hello (messages/hello.schema.json) ---- + +// HelloOffer is sent by the connecting device, client -> server. +// +// Capabilities is a pointer so an explicitly-present-but-empty +// "capabilities": [] round-trips distinctly from an entirely absent field +// (both are valid on the wire; schema default is [] when omitted). +type HelloOffer struct { + Stage string `json:"stage"` // const "offer" + DeviceID string `json:"device_id"` + Role string `json:"role"` // source|viewer + ProtocolVersions []int `json:"protocol_versions"` + Capabilities *[]string `json:"capabilities,omitempty"` +} + +// HelloAccept is sent by the server once it has negotiated a version. +type HelloAccept struct { + Stage string `json:"stage"` // const "accept" + ProtocolVersion int `json:"protocol_version"` + SessionID string `json:"session_id"` + Capabilities *[]string `json:"capabilities,omitempty"` + HeartbeatIntervalMS int `json:"heartbeat_interval_ms"` +} + +// HelloBody is the oneOf{offer, accept} discriminated union keyed by `stage`. +// Exactly one of Offer/Accept is non-nil after decode. +type HelloBody struct { + Offer *HelloOffer + Accept *HelloAccept +} + +func (h HelloBody) MarshalJSON() ([]byte, error) { + switch { + case h.Offer != nil: + return json.Marshal(h.Offer) + case h.Accept != nil: + return json.Marshal(h.Accept) + default: + return nil, fmt.Errorf("hello body: neither offer nor accept set") + } +} + +func (h *HelloBody) UnmarshalJSON(data []byte) error { + var disc struct { + Stage string `json:"stage"` + } + if err := json.Unmarshal(data, &disc); err != nil { + return err + } + switch disc.Stage { + case "offer": + var o HelloOffer + if err := json.Unmarshal(data, &o); err != nil { + return err + } + h.Offer = &o + case "accept": + var a HelloAccept + if err := json.Unmarshal(data, &a); err != nil { + return err + } + h.Accept = &a + default: + return fmt.Errorf("hello body: unknown stage %q", disc.Stage) + } + return nil +} + +// ---- resume (messages/resume.schema.json) ---- + +type ResumeBody struct { + LastRevision int64 `json:"last_revision"` +} + +// ---- snapshot (messages/snapshot.schema.json) ---- + +type SnapshotBody struct { + Revision int64 `json:"revision"` + Part int `json:"part"` + Final bool `json:"final"` + Tasks []TaskState `json:"tasks"` + Metrics []MetricSample `json:"metrics"` + Messages []MessageRecord `json:"messages"` + Automations []AutomationState `json:"automations"` +} + +// ---- reliable events ---- + +// TaskEventBody mirrors messages/task.event.schema.json#/$defs/body. +type TaskEventBody struct { + DeviceID string `json:"device_id"` + DeviceSeq int64 `json:"device_seq"` + TaskID string `json:"task_id"` + Kind string `json:"kind"` // started|progress|step|done|failed + OccurredAt int64 `json:"occurred_at"` + Title string `json:"title,omitempty"` + Percent *int `json:"percent,omitempty"` + Step string `json:"step,omitempty"` + Message string `json:"message,omitempty"` + Display *DisplayHints `json:"display,omitempty"` +} + +// MessageEventBody mirrors messages/message.event.schema.json#/$defs/body. +type MessageEventBody struct { + DeviceID string `json:"device_id"` + DeviceSeq int64 `json:"device_seq"` + MessageID string `json:"message_id"` + Level string `json:"level"` // info|warn|error + Text string `json:"text"` + OccurredAt int64 `json:"occurred_at"` + AutomationID string `json:"automation_id,omitempty"` +} + +// ConfigEventBody mirrors messages/config.event.schema.json#/$defs/body. +// Server-minted only: no device_id/device_seq (SPEC.md section 5.5). +type ConfigEventBody struct { + Kind string `json:"kind"` // automation.upserted|automation.removed + AutomationID string `json:"automation_id"` + Automation *AutomationState `json:"automation,omitempty"` + OccurredAt int64 `json:"occurred_at"` +} + +// ---- metric.frame (messages/metric.frame.schema.json) ---- + +type MetricFrameBody struct { + DeviceID string `json:"device_id"` + Metrics []MetricSample `json:"metrics"` +} + +// ---- delta (messages/delta.schema.json) ---- + +// DeltaEventItem is one oneOf{task.event, message.event, config.event} entry +// of delta.body.events, tagged by event_type. +type DeltaEventItem struct { + EventType string + TaskEvent *TaskEventBody + MessageEvent *MessageEventBody + ConfigEvent *ConfigEventBody +} + +func (d DeltaEventItem) MarshalJSON() ([]byte, error) { + var wrapper struct { + EventType string `json:"event_type"` + Event any `json:"event"` + } + wrapper.EventType = d.EventType + switch d.EventType { + case TypeTaskEvent: + wrapper.Event = d.TaskEvent + case TypeMessageEvent: + wrapper.Event = d.MessageEvent + case TypeConfigEvent: + wrapper.Event = d.ConfigEvent + default: + return nil, fmt.Errorf("delta event: unknown event_type %q", d.EventType) + } + return json.Marshal(wrapper) +} + +func (d *DeltaEventItem) UnmarshalJSON(data []byte) error { + var wrapper struct { + EventType string `json:"event_type"` + Event json.RawMessage `json:"event"` + } + if err := json.Unmarshal(data, &wrapper); err != nil { + return err + } + d.EventType = wrapper.EventType + switch wrapper.EventType { + case TypeTaskEvent: + var b TaskEventBody + if err := json.Unmarshal(wrapper.Event, &b); err != nil { + return err + } + d.TaskEvent = &b + case TypeMessageEvent: + var b MessageEventBody + if err := json.Unmarshal(wrapper.Event, &b); err != nil { + return err + } + d.MessageEvent = &b + case TypeConfigEvent: + var b ConfigEventBody + if err := json.Unmarshal(wrapper.Event, &b); err != nil { + return err + } + d.ConfigEvent = &b + default: + return fmt.Errorf("delta event: unknown event_type %q", wrapper.EventType) + } + return nil +} + +type DeltaBody struct { + FromRevision int64 `json:"from_revision"` + ToRevision int64 `json:"to_revision"` + Events []DeltaEventItem `json:"events"` +} + +// ---- ack (messages/ack.schema.json) ---- + +type AckedPair struct { + DeviceID string `json:"device_id"` + DeviceSeq int64 `json:"device_seq"` +} + +type Lease struct { + ExpiresAt int64 `json:"expires_at"` +} + +type AckBody struct { + Acked []AckedPair `json:"acked,omitempty"` + InReplyTo string `json:"in_reply_to,omitempty"` + Lease *Lease `json:"lease,omitempty"` +} + +// ---- subscribe / interest.renew (messages/subscribe.schema.json) ---- + +// SubscribeBody is shared verbatim by subscribe and interest.renew +// (interest.renew.schema.json $refs subscribe's body). +type SubscribeBody struct { + Topics []string `json:"topics,omitempty"` +} + +// UnsubscribeBody is intentionally empty (messages/unsubscribe.schema.json). +type UnsubscribeBody struct{} + +// ---- command (messages/command.schema.json) ---- + +type CommandBody struct { + CommandID string `json:"command_id"` + Origin string `json:"origin"` // viewer|server + IssuedByDeviceID string `json:"issued_by_device_id,omitempty"` + Action string `json:"action"` + TaskID string `json:"task_id,omitempty"` + AutomationID string `json:"automation_id,omitempty"` + TargetDeviceID string `json:"target_device_id,omitempty"` + TTLMs int64 `json:"ttl_ms"` + // Params is a pointer for the same reason as HelloOffer.Capabilities: + // an explicit "params": {} must round-trip distinctly from an absent + // field, even though both mean "no extension keys" (protocol v1 defines + // none) per messages/command.schema.json. + Params *map[string]any `json:"params,omitempty"` +} + +// ---- error (messages/error.schema.json) ---- + +type ErrorBody struct { + Code string `json:"code"` + Message string `json:"message"` + InReplyTo string `json:"in_reply_to,omitempty"` + Retryable *bool `json:"retryable"` + Fatal *bool `json:"fatal"` +} + +// Error codes, SPEC.md section 13. +const ( + ErrVersionUnsupported = "version_unsupported" + ErrHelloRequired = "hello_required" + ErrUnauthenticated = "unauthenticated" + ErrUnauthorized = "unauthorized" + ErrMalformed = "malformed" + ErrRateLimited = "rate_limited" + ErrFrameTooLarge = "frame_too_large" + ErrBatchTooLarge = "batch_too_large" + ErrRevisionUnavailable = "revision_unavailable" + ErrCommandExpired = "command_expired" + ErrSequenceGap = "sequence_gap" + ErrSuperseded = "superseded" + ErrInternal = "internal_error" +) diff --git a/daemon/internal/realtime/wire/envelope.go b/daemon/internal/realtime/wire/envelope.go new file mode 100644 index 00000000..8f16ee99 --- /dev/null +++ b/daemon/internal/realtime/wire/envelope.go @@ -0,0 +1,133 @@ +// Package wire implements the Go representation of the frozen Sitrep +// realtime protocol (proto/realtime/SPEC.md, v1.0.0). It mirrors the JSON +// Schemas under proto/realtime/ field-for-field: this package MUST NOT +// diverge from that spec, which is the sole source of truth. When in doubt, +// re-read proto/realtime/SPEC.md and the relevant messages/*.schema.json +// before changing anything here. +// +// The envelope's top level is permanently strict (SPEC.md section 3): every +// frame carries exactly type/id/ts/body and nothing else. The tolerance for +// unrecognized fields described in section 15 applies only inside `body`, +// which is why body structs are decoded with the standard (lenient) +// encoding/json behavior while the envelope itself is decoded with +// DisallowUnknownFields. +package wire + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" +) + +// MessageType enumerates the complete, closed set of 14 message types for +// protocol version 1 (SPEC.md section 4). No implementation may invent an +// additional type without a version negotiation both peers agree to. +const ( + TypeHello = "hello" + TypeResume = "resume" + TypeSnapshot = "snapshot" + TypeDelta = "delta" + TypeAck = "ack" + TypeTaskEvent = "task.event" + TypeMessageEvent = "message.event" + TypeMetricFrame = "metric.frame" + TypeConfigEvent = "config.event" + TypeSubscribe = "subscribe" + TypeUnsubscribe = "unsubscribe" + TypeInterestRenew = "interest.renew" + TypeCommand = "command" + TypeError = "error" +) + +// knownTypes is used by callers that must decide, per SPEC.md section 15, +// whether an unrecognized type should be silently ignored (not an error) +// rather than rejected as malformed. +var knownTypes = map[string]bool{ + TypeHello: true, TypeResume: true, TypeSnapshot: true, TypeDelta: true, + TypeAck: true, TypeTaskEvent: true, TypeMessageEvent: true, + TypeMetricFrame: true, TypeConfigEvent: true, TypeSubscribe: true, + TypeUnsubscribe: true, TypeInterestRenew: true, TypeCommand: true, + TypeError: true, +} + +// KnownType reports whether typ is one of the 14 registered message types. +func KnownType(typ string) bool { return knownTypes[typ] } + +// Envelope is the generic frame shape shared by every message +// (envelope.schema.json). Body is kept as raw JSON so a receiver can dispatch +// on Type before committing to a concrete body shape. +type Envelope struct { + Type string `json:"type"` + ID string `json:"id"` + TS int64 `json:"ts"` + Body json.RawMessage `json:"body"` +} + +// envelopeIDPattern / deviceIDPattern mirror common.schema.json's $defs. +var ( + envelopeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + deviceIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) +) + +// ValidEnvelopeID reports whether id satisfies common.schema.json#/$defs/envelope_id. +func ValidEnvelopeID(id string) bool { return envelopeIDPattern.MatchString(id) } + +// ValidDeviceID reports whether id satisfies common.schema.json#/$defs/device_id. +func ValidDeviceID(id string) bool { return deviceIDPattern.MatchString(id) } + +// MinUnixMS / MaxUnixMS mirror common.schema.json#/$defs/unix_ms_timestamp's +// bound: any value in this range cannot plausibly be Unix seconds (10 digits, +// under 1e10) and is therefore guaranteed to be milliseconds. This is a +// deliberate mechanical guard against the historical seconds/milliseconds +// confusion called out in SPEC.md section 3.1. +const ( + MinUnixMS int64 = 1_000_000_000_000 + MaxUnixMS int64 = 9_999_999_999_999 +) + +// ValidUnixMS reports whether ts is a plausible Unix-milliseconds timestamp. +func ValidUnixMS(ts int64) bool { return ts >= MinUnixMS && ts <= MaxUnixMS } + +// DecodeEnvelope parses one frame's top level. It enforces the permanently +// strict envelope shape: exactly type/id/ts/body, no other top-level field +// (SPEC.md section 3) — an envelope carrying, e.g., a smuggled "token" field +// is rejected here with the same effect as error{code: malformed}. +func DecodeEnvelope(data []byte) (Envelope, error) { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + var env Envelope + if err := dec.Decode(&env); err != nil { + return Envelope{}, fmt.Errorf("malformed envelope: %w", err) + } + if env.Type == "" { + return Envelope{}, fmt.Errorf("malformed envelope: missing type") + } + if env.ID == "" { + return Envelope{}, fmt.Errorf("malformed envelope: missing id") + } + if !ValidEnvelopeID(env.ID) { + return Envelope{}, fmt.Errorf("malformed envelope: id %q does not match envelope_id pattern", env.ID) + } + if !ValidUnixMS(env.TS) { + return Envelope{}, fmt.Errorf("malformed envelope: ts %d is not a plausible unix-ms timestamp", env.TS) + } + if env.Body == nil { + return Envelope{}, fmt.Errorf("malformed envelope: missing body") + } + return env, nil +} + +// NewEnvelope marshals body and wraps it in an Envelope ready for encoding. +func NewEnvelope(typ, id string, ts int64, body any) (Envelope, error) { + raw, err := json.Marshal(body) + if err != nil { + return Envelope{}, err + } + return Envelope{Type: typ, ID: id, TS: ts, Body: raw}, nil +} + +// Encode serializes the envelope to its wire form. The envelope struct's own +// field set is exactly {type,id,ts,body}, so plain json.Marshal already +// satisfies the strict top level. +func (e Envelope) Encode() ([]byte, error) { return json.Marshal(e) } diff --git a/daemon/internal/realtime/wire/validate.go b/daemon/internal/realtime/wire/validate.go new file mode 100644 index 00000000..35d07d0c --- /dev/null +++ b/daemon/internal/realtime/wire/validate.go @@ -0,0 +1,479 @@ +package wire + +import ( + "encoding/json" + "fmt" + "regexp" +) + +// metricIDPattern mirrors common.schema.json#/$defs/metric_id: the id +// grammar from proto/SPEC.md's metric.update verb, non-empty by +// construction ({1,64}). +var metricIDPattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`) + +// Field length caps from common.schema.json. +const ( + freeTextMax = 2048 // $defs/free_text + labelTextMax = 256 // $defs/label_text + metricValueMax = 256 // metric_sample.value +) + +// DecodeBody decodes env.Body into the concrete Go type for env.Type and +// validates it against the semantic rules this package mirrors from the +// JSON Schemas (required fields, enums, ranges, conditional requirements). +// It returns an error for any envelope whose type is not one of the 14 +// registered types; per SPEC.md section 15 a caller that only needs +// forward-compatible tolerance should check KnownType(env.Type) before +// calling DecodeBody and ignore (not reject) an unrecognized type. +func DecodeBody(env Envelope) (any, error) { + switch env.Type { + case TypeHello: + var b HelloBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeResume: + var b ResumeBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeSnapshot: + var b SnapshotBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeDelta: + var b DeltaBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeAck: + var b AckBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeTaskEvent: + var b TaskEventBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeMessageEvent: + var b MessageEventBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeMetricFrame: + var b MetricFrameBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeConfigEvent: + var b ConfigEventBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeSubscribe, TypeInterestRenew: + var b SubscribeBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeUnsubscribe: + var b UnsubscribeBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + return b, nil + case TypeCommand: + var b CommandBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + case TypeError: + var b ErrorBody + if err := json.Unmarshal(env.Body, &b); err != nil { + return nil, err + } + if err := b.Validate(); err != nil { + return nil, err + } + return b, nil + default: + return nil, fmt.Errorf("unknown message type %q", env.Type) + } +} + +func (h HelloBody) Validate() error { + switch { + case h.Offer != nil: + o := h.Offer + if o.DeviceID == "" || !ValidDeviceID(o.DeviceID) { + return fmt.Errorf("hello offer: invalid device_id %q", o.DeviceID) + } + if o.Role != "source" && o.Role != "viewer" { + return fmt.Errorf("hello offer: invalid role %q", o.Role) + } + if len(o.ProtocolVersions) == 0 { + return fmt.Errorf("hello offer: protocol_versions must not be empty") + } + for _, v := range o.ProtocolVersions { + if v < 1 { + return fmt.Errorf("hello offer: protocol_versions entries must be >= 1") + } + } + return nil + case h.Accept != nil: + a := h.Accept + if a.ProtocolVersion < 1 { + return fmt.Errorf("hello accept: invalid protocol_version %d", a.ProtocolVersion) + } + if a.SessionID == "" || len(a.SessionID) > 64 { + return fmt.Errorf("hello accept: invalid session_id") + } + if a.HeartbeatIntervalMS < 1000 || a.HeartbeatIntervalMS > 300000 { + return fmt.Errorf("hello accept: heartbeat_interval_ms out of range") + } + return nil + default: + return fmt.Errorf("hello body: neither offer nor accept present") + } +} + +func (r ResumeBody) Validate() error { + if r.LastRevision < 0 { + return fmt.Errorf("resume: last_revision must be >= 0, got %d", r.LastRevision) + } + return nil +} + +func (s SnapshotBody) Validate() error { + if s.Revision < 0 { + return fmt.Errorf("snapshot: revision must be >= 0") + } + if s.Part < 1 { + return fmt.Errorf("snapshot: part must be >= 1") + } + for i, t := range s.Tasks { + if err := t.Validate(); err != nil { + return fmt.Errorf("snapshot: tasks[%d]: %w", i, err) + } + } + for i, m := range s.Metrics { + if err := m.Validate(); err != nil { + return fmt.Errorf("snapshot: metrics[%d]: %w", i, err) + } + } + return nil +} + +func (t TaskState) Validate() error { + if t.State != "running" && t.State != "done" && t.State != "failed" { + return fmt.Errorf("task_state: invalid state %q", t.State) + } + if !ValidUnixMS(t.UpdatedAt) { + return fmt.Errorf("task_state: updated_at is not a plausible unix-ms timestamp") + } + if len(t.Title) > freeTextMax || len(t.Step) > freeTextMax || len(t.Message) > freeTextMax { + return fmt.Errorf("task_state: free-text field exceeds %d chars", freeTextMax) + } + return nil +} + +func (m MetricSample) Validate() error { + if !metricIDPattern.MatchString(m.MetricID) { + return fmt.Errorf("metric_sample: metric_id %q does not match %s", m.MetricID, metricIDPattern) + } + if len(m.Value) > metricValueMax { + return fmt.Errorf("metric_sample: value exceeds %d chars", metricValueMax) + } + if len(m.Label) > labelTextMax { + return fmt.Errorf("metric_sample: label exceeds %d chars", labelTextMax) + } + if !ValidUnixMS(m.TS) { + return fmt.Errorf("metric_sample: ts is not a plausible unix-ms timestamp") + } + return nil +} + +func (b DeltaBody) Validate() error { + if b.FromRevision < 0 || b.ToRevision < 0 { + return fmt.Errorf("delta: revisions must be >= 0") + } + if b.ToRevision-b.FromRevision != int64(len(b.Events)) { + return fmt.Errorf("delta: to_revision - from_revision (%d) must equal events.length (%d)", + b.ToRevision-b.FromRevision, len(b.Events)) + } + for i, e := range b.Events { + switch e.EventType { + case TypeTaskEvent: + if e.TaskEvent == nil { + return fmt.Errorf("delta: events[%d]: missing task.event body", i) + } + if err := e.TaskEvent.Validate(); err != nil { + return fmt.Errorf("delta: events[%d]: %w", i, err) + } + case TypeMessageEvent: + if e.MessageEvent == nil { + return fmt.Errorf("delta: events[%d]: missing message.event body", i) + } + if err := e.MessageEvent.Validate(); err != nil { + return fmt.Errorf("delta: events[%d]: %w", i, err) + } + case TypeConfigEvent: + if e.ConfigEvent == nil { + return fmt.Errorf("delta: events[%d]: missing config.event body", i) + } + if err := e.ConfigEvent.Validate(); err != nil { + return fmt.Errorf("delta: events[%d]: %w", i, err) + } + default: + return fmt.Errorf("delta: events[%d]: unknown event_type %q", i, e.EventType) + } + } + return nil +} + +func (a AckBody) Validate() error { + if len(a.Acked) == 0 && a.InReplyTo == "" { + return fmt.Errorf("ack: at least one of acked or in_reply_to is required") + } + if a.InReplyTo != "" && !ValidEnvelopeID(a.InReplyTo) { + return fmt.Errorf("ack: in_reply_to is not a valid envelope id") + } + for i, p := range a.Acked { + if !ValidDeviceID(p.DeviceID) { + return fmt.Errorf("ack: acked[%d]: invalid device_id", i) + } + if p.DeviceSeq < 1 { + return fmt.Errorf("ack: acked[%d]: device_seq must be >= 1", i) + } + } + if a.Lease != nil && !ValidUnixMS(a.Lease.ExpiresAt) { + return fmt.Errorf("ack: lease.expires_at is not a plausible unix-ms timestamp") + } + return nil +} + +func (t TaskEventBody) Validate() error { + if !ValidDeviceID(t.DeviceID) { + return fmt.Errorf("task.event: invalid device_id %q", t.DeviceID) + } + if t.DeviceSeq < 1 { + return fmt.Errorf("task.event: device_seq must be >= 1 (missing or invalid)") + } + if t.TaskID == "" { + return fmt.Errorf("task.event: missing task_id") + } + switch t.Kind { + case "started", "progress", "step", "done", "failed": + default: + return fmt.Errorf("task.event: invalid kind %q", t.Kind) + } + if !ValidUnixMS(t.OccurredAt) { + return fmt.Errorf("task.event: occurred_at is not a plausible unix-ms timestamp") + } + if t.Kind == "progress" && t.Percent == nil { + return fmt.Errorf("task.event: percent is required when kind is progress") + } + if t.Percent != nil && (*t.Percent < 0 || *t.Percent > 100) { + return fmt.Errorf("task.event: percent out of range") + } + if len(t.Title) > freeTextMax || len(t.Step) > freeTextMax || len(t.Message) > freeTextMax { + return fmt.Errorf("task.event: free-text field exceeds %d chars", freeTextMax) + } + return nil +} + +func (m MessageEventBody) Validate() error { + if !ValidDeviceID(m.DeviceID) { + return fmt.Errorf("message.event: invalid device_id %q", m.DeviceID) + } + if m.DeviceSeq < 1 { + return fmt.Errorf("message.event: device_seq must be >= 1 (missing or invalid)") + } + if m.MessageID == "" { + return fmt.Errorf("message.event: missing message_id") + } + switch m.Level { + case "info", "warn", "error": + default: + return fmt.Errorf("message.event: invalid level %q", m.Level) + } + if len(m.Text) > freeTextMax { + return fmt.Errorf("message.event: text exceeds %d chars", freeTextMax) + } + if !ValidUnixMS(m.OccurredAt) { + return fmt.Errorf("message.event: occurred_at is not a plausible unix-ms timestamp") + } + return nil +} + +func (c ConfigEventBody) Validate() error { + switch c.Kind { + case "automation.upserted", "automation.removed": + default: + return fmt.Errorf("config.event: invalid kind %q", c.Kind) + } + if c.AutomationID == "" { + return fmt.Errorf("config.event: missing automation_id") + } + if c.Kind == "automation.upserted" && c.Automation == nil { + return fmt.Errorf("config.event: automation is required when kind is automation.upserted") + } + if !ValidUnixMS(c.OccurredAt) { + return fmt.Errorf("config.event: occurred_at is not a plausible unix-ms timestamp") + } + return nil +} + +func (m MetricFrameBody) Validate() error { + if !ValidDeviceID(m.DeviceID) { + return fmt.Errorf("metric.frame: invalid device_id %q", m.DeviceID) + } + if len(m.Metrics) == 0 { + return fmt.Errorf("metric.frame: metrics must not be empty") + } + if len(m.Metrics) > 64 { + return fmt.Errorf("metric.frame: metrics exceeds 64-item cap") + } + seen := make(map[string]bool, len(m.Metrics)) + for i, s := range m.Metrics { + if err := s.Validate(); err != nil { + return fmt.Errorf("metric.frame: metrics[%d]: %w", i, err) + } + if seen[s.MetricID] { + return fmt.Errorf("metric.frame: duplicate metric_id %q in one frame", s.MetricID) + } + seen[s.MetricID] = true + } + return nil +} + +var validTopics = map[string]bool{"task": true, "metric": true, "message": true} + +func (s SubscribeBody) Validate() error { + seen := make(map[string]bool, len(s.Topics)) + for _, t := range s.Topics { + if !validTopics[t] { + return fmt.Errorf("subscribe: unknown topic %q", t) + } + if seen[t] { + return fmt.Errorf("subscribe: duplicate topic %q", t) + } + seen[t] = true + } + return nil +} + +func (c CommandBody) Validate() error { + if c.CommandID == "" || len(c.CommandID) > 64 { + return fmt.Errorf("command: invalid command_id") + } + if c.Origin != "viewer" && c.Origin != "server" { + return fmt.Errorf("command: invalid origin %q", c.Origin) + } + if c.TTLMs < 1 || c.TTLMs > 86400000 { + return fmt.Errorf("command: ttl_ms out of range") + } + switch c.Action { + case "pause", "resume", "stop": + if c.Origin != "viewer" { + return fmt.Errorf("command: action %q requires origin viewer", c.Action) + } + if c.IssuedByDeviceID == "" { + return fmt.Errorf("command: action %q requires issued_by_device_id", c.Action) + } + if c.TaskID == "" { + return fmt.Errorf("command: action %q requires task_id", c.Action) + } + if c.AutomationID != "" { + return fmt.Errorf("command: action %q must not carry automation_id", c.Action) + } + case "run_now": + if c.Origin != "viewer" { + return fmt.Errorf("command: action run_now requires origin viewer") + } + if c.IssuedByDeviceID == "" { + return fmt.Errorf("command: action run_now requires issued_by_device_id") + } + if c.AutomationID == "" { + return fmt.Errorf("command: action run_now requires automation_id") + } + if c.TaskID != "" { + return fmt.Errorf("command: action run_now must not carry task_id") + } + case "throttle", "resume_rate": + if c.Origin != "server" { + return fmt.Errorf("command: action %q requires origin server", c.Action) + } + if c.TaskID != "" || c.AutomationID != "" { + return fmt.Errorf("command: action %q must not carry task_id or automation_id", c.Action) + } + default: + return fmt.Errorf("command: invalid action %q", c.Action) + } + return nil +} + +func (e ErrorBody) Validate() error { + if e.Code == "" { + return fmt.Errorf("error: missing code") + } + switch e.Code { + case ErrVersionUnsupported, ErrHelloRequired, ErrUnauthenticated, ErrUnauthorized, + ErrMalformed, ErrRateLimited, ErrFrameTooLarge, ErrBatchTooLarge, + ErrRevisionUnavailable, ErrCommandExpired, ErrSequenceGap, ErrSuperseded, ErrInternal: + default: + return fmt.Errorf("error: unknown code %q", e.Code) + } + if len(e.Message) > 500 { + return fmt.Errorf("error: message exceeds 500 chars") + } + if e.Retryable == nil || e.Fatal == nil { + return fmt.Errorf("error: retryable and fatal are both required") + } + return nil +} diff --git a/daemon/internal/realtime/wire/wire_test.go b/daemon/internal/realtime/wire/wire_test.go new file mode 100644 index 00000000..b4a9560e --- /dev/null +++ b/daemon/internal/realtime/wire/wire_test.go @@ -0,0 +1,227 @@ +package wire + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" +) + +// fixturesDir locates proto/realtime/fixtures relative to this source file, +// independent of the test runner's working directory. +func fixturesDir(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + dir := filepath.Dir(file) // .../daemon/internal/realtime/wire + root := filepath.Join(dir, "..", "..", "..", "..") + fixtures := filepath.Join(root, "proto", "realtime", "fixtures") + if _, err := os.Stat(fixtures); err != nil { + t.Fatalf("fixtures dir not found at %s: %v", fixtures, err) + } + return fixtures +} + +func listJSONFiles(t *testing.T, dir string) []string { + t.Helper() + var out []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(path) == ".json" { + out = append(out, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + sort.Strings(out) + return out +} + +// normalize decodes JSON into a generic interface{} tree so two +// differently-formatted (but semantically identical) documents compare +// equal with reflect.DeepEqual. +func normalize(t *testing.T, data []byte) any { + t.Helper() + var v any + if err := json.Unmarshal(data, &v); err != nil { + t.Fatalf("normalize: %v", err) + } + return v +} + +// TestValidFixturesRoundTrip decodes every bare-envelope fixture under +// fixtures/valid and fixtures/scenarios, re-encodes it through the Go +// types, and asserts the result is semantically identical to the original. +// This is the "Go types match the schema byte for byte" check called for by +// the realtime daemon work order: if a field were missing, renamed, or +// mistyped in bodies.go, this test would lose or corrupt it on round trip. +func TestValidFixturesRoundTrip(t *testing.T) { + fixtures := fixturesDir(t) + dirs := []string{ + filepath.Join(fixtures, "valid"), + filepath.Join(fixtures, "scenarios"), + } + tested := 0 + for _, dir := range dirs { + for _, path := range listJSONFiles(t, dir) { + if filepath.Base(path) == "README.md" { + continue + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + t.Run(relPath(fixtures, path), func(t *testing.T) { + env, err := DecodeEnvelope(data) + if err != nil { + t.Fatalf("DecodeEnvelope: %v", err) + } + body, err := DecodeBody(env) + if err != nil { + t.Fatalf("DecodeBody: %v", err) + } + reEncodedBody, err := json.Marshal(body) + if err != nil { + t.Fatalf("re-marshal body: %v", err) + } + roundTripped := Envelope{Type: env.Type, ID: env.ID, TS: env.TS, Body: reEncodedBody} + out, err := roundTripped.Encode() + if err != nil { + t.Fatalf("Encode: %v", err) + } + want := normalize(t, data) + got := normalize(t, out) + if !reflect.DeepEqual(want, got) { + t.Fatalf("round trip mismatch:\n original: %s\n produced: %s", data, out) + } + }) + tested++ + } + } + if tested == 0 { + t.Fatal("no valid fixtures were exercised") + } + t.Logf("round-tripped %d fixtures", tested) +} + +func relPath(base, path string) string { + rel, err := filepath.Rel(base, path) + if err != nil { + return path + } + return rel +} + +// roleFixture is the {"sender_role": ..., "frame": {...}} wrapper format +// used by a handful of invalid fixtures to test the section 10.1 +// authorization matrix rather than pure schema shape (SPEC.md section 16). +type roleFixture struct { + SenderRole string `json:"sender_role"` + Frame json.RawMessage `json:"frame"` +} + +func isRoleFixture(data []byte) (roleFixture, bool) { + var rf roleFixture + if err := json.Unmarshal(data, &rf); err != nil { + return roleFixture{}, false + } + return rf, rf.SenderRole != "" && rf.Frame != nil +} + +// TestInvalidFixturesRejected asserts every fixture under fixtures/invalid +// is rejected by this package, either at envelope decode, body decode/ +// validate, or (for the sender_role-wrapped fixtures) the authorization +// matrix. +func TestInvalidFixturesRejected(t *testing.T) { + fixtures := fixturesDir(t) + dir := filepath.Join(fixtures, "invalid") + tested := 0 + for _, path := range listJSONFiles(t, dir) { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + t.Run(relPath(fixtures, path), func(t *testing.T) { + if rf, ok := isRoleFixture(data); ok { + env, err := DecodeEnvelope(rf.Frame) + if err != nil { + return // rejected at envelope level: acceptable + } + body, err := DecodeBody(env) + if err != nil { + return // rejected at body validation: acceptable + } + if err := Authorize(Role(rf.SenderRole), env.Type, body); err == nil { + t.Fatalf("expected rejection for role %q sending %q, got none", rf.SenderRole, env.Type) + } + return + } + env, err := DecodeEnvelope(data) + if err != nil { + return // rejected at envelope level: acceptable + } + if _, err := DecodeBody(env); err == nil { + t.Fatalf("expected DecodeBody to reject fixture, got none") + } + }) + tested++ + } + if tested == 0 { + t.Fatal("no invalid fixtures were exercised") + } + t.Logf("checked %d invalid fixtures", tested) +} + +// TestUnixMSBoundary pins the seconds/milliseconds guard from +// common.schema.json#/$defs/unix_ms_timestamp (SPEC.md section 3.1). +func TestUnixMSBoundary(t *testing.T) { + cases := []struct { + ts int64 + want bool + }{ + {1000000000000, true}, // exact minimum + {9999999999999, true}, // exact maximum + {999999999999, false}, // one below minimum + {10000000000000, false}, // one above maximum + {1752480006, false}, // a plausible *seconds* value must be rejected + } + for _, c := range cases { + if got := ValidUnixMS(c.ts); got != c.want { + t.Errorf("ValidUnixMS(%d) = %v, want %v", c.ts, got, c.want) + } + } +} + +// TestMetricIDValidation pins the metric_id grammar from +// common.schema.json#/$defs/metric_id (^[a-z0-9_.-]{1,64}$). +func TestMetricIDValidation(t *testing.T) { + valid := []string{"cpu.load", "mem.used_gb", "a", "x_1-2.3"} + for _, id := range valid { + s := MetricSample{MetricID: id, Value: "1", TS: MinUnixMS + 1} + if err := s.Validate(); err != nil { + t.Errorf("Validate(%q) = %v, want nil", id, err) + } + } + invalid := []string{ + "", // empty + "CPU.Load", // uppercase + "has space", // whitespace + "emojié", // non-ascii + string(make([]byte, 65)), // over 64 chars (and NUL bytes) + } + for _, id := range invalid { + s := MetricSample{MetricID: id, Value: "1", TS: MinUnixMS + 1} + if err := s.Validate(); err == nil { + t.Errorf("Validate(%q) = nil, want error", id) + } + } +} diff --git a/daemon/internal/uplink/legacy_test.go b/daemon/internal/uplink/legacy_test.go new file mode 100644 index 00000000..f968af3f --- /dev/null +++ b/daemon/internal/uplink/legacy_test.go @@ -0,0 +1,238 @@ +package uplink + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +func waitForCond(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + if !cond() { + t.Fatalf("condition not met within %v", timeout) + } +} + +// TestLegacyModeDrainsPendingAndNewEventsWhenServerDisabled is the P0-1 +// integration test: when the server rejects /v3/realtime with HTTP 403 +// realtime_disabled at dial time, the whole Uplink session must switch to +// legacy /v2 routing — including draining anything already sitting durably +// in the outbox from before this session even started (simulating a +// previous process run), so a terminal task state doesn't get stranded in a +// store no viewer reads. New events offered while still in legacy mode must +// also reach /v2, after the drained backlog, in order. No realtime frame +// may ever be sent (the dial never completes hello), and the dial must not +// be hammered. +func TestLegacyModeDrainsPendingAndNewEventsWhenServerDisabled(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + var dialCount int32 + rtSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&dialCount, 1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + })) + defer rtSrv.Close() + + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + defer store.Close() + + // Pre-populate the outbox as if a previous process run had already + // durably enqueued this reliable event before the server was known to + // be disabled. + ctx := context.Background() + if _, err := store.Enqueue(ctx, "space-1", wire.TypeMessageEvent, func(seq int64) (json.RawMessage, error) { + return json.Marshal(wire.MessageEventBody{ + DeviceID: "device-1", DeviceSeq: seq, MessageID: fmt.Sprintf("device-1:%d", seq), + Level: "info", Text: "preexisting", OccurredAt: time.Now().UnixMilli(), + }) + }); err != nil { + t.Fatalf("pre-populate Enqueue: %v", err) + } + + rt := rtclient.New(rtclient.Config{ + URL: "ws" + strings.TrimPrefix(rtSrv.URL, "http"), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + }) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt}) + defer u.Close() + + waitForCond(t, 2*time.Second, rt.ServerDisabled) + + // New reliable events offered while the session is in legacy mode must + // also reach /v2, after the drained backlog. + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "second"; e.Level = "info" })) + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "third"; e.Level = "info" })) + + waitForCond(t, 3*time.Second, func() bool { return len(httpCap.all()) >= 3 }) + + got := httpCap.all() + if len(got) != 3 { + t.Fatalf("got %d events over /v2, want exactly 3: %+v", len(got), got) + } + want := []string{"preexisting", "second", "third"} + for i, w := range want { + if got[i].Text != w { + t.Fatalf("event %d = %q, want %q (full order: %v)", i, got[i].Text, w, got) + } + } + + if n := atomic.LoadInt32(&dialCount); n > 8 { + t.Fatalf("dial count = %d; expected no rapid redial while the server reports realtime disabled", n) + } +} + +// TestLegacyModeResumesRealtimeAfterServerReenables asserts the switch-back +// half: once the realtime client's periodic re-probe succeeds (the server +// re-enables /v3/realtime), the Uplink resumes routing new reliable events +// to realtime and the /v2 spy stops growing. +func TestLegacyModeResumesRealtimeAfterServerReenables(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + var attempts int32 + rtReceived := make(chan wire.Envelope, 16) + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&attempts, 1) + if n <= 2 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"realtime_disabled"}`)) + return + } + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer ws.Close(websocket.StatusNormalClosure, "done") + wsCtx := r.Context() + _, data, err := ws.Read(wsCtx) + if err != nil { + return + } + env, err := wire.DecodeEnvelope(data) + if err != nil || env.Type != wire.TypeHello { + return + } + accept := wire.HelloAccept{Stage: "accept", ProtocolVersion: 1, SessionID: "sess", HeartbeatIntervalMS: 1000} + acceptEnv, err := wire.NewEnvelope(wire.TypeHello, "accept-env-1", time.Now().UnixMilli(), wire.HelloBody{Accept: &accept}) + if err != nil { + return + } + encoded, err := acceptEnv.Encode() + if err != nil { + return + } + if err := ws.Write(wsCtx, websocket.MessageText, encoded); err != nil { + return + } + for { + _, data, err := ws.Read(wsCtx) + if err != nil { + return + } + if string(data) == "ping" { + _ = ws.Write(wsCtx, websocket.MessageText, []byte("pong")) + continue + } + e, derr := wire.DecodeEnvelope(data) + if derr != nil { + continue + } + rtReceived <- e + if e.Type == wire.TypeMessageEvent { + body, berr := wire.DecodeBody(e) + if berr == nil { + mb := body.(wire.MessageEventBody) + ackEnv, aerr := wire.NewEnvelope(wire.TypeAck, "ack-1", time.Now().UnixMilli(), + wire.AckBody{Acked: []wire.AckedPair{{DeviceID: mb.DeviceID, DeviceSeq: mb.DeviceSeq}}}) + if aerr == nil { + if encoded, eerr := ackEnv.Encode(); eerr == nil { + _ = ws.Write(wsCtx, websocket.MessageText, encoded) + } + } + } + } + } + }) + rtSrv := httptest.NewServer(mux) + defer rtSrv.Close() + + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + defer store.Close() + + rt := rtclient.New(rtclient.WithTestDisabledRecheckInterval(rtclient.Config{ + URL: "ws" + strings.TrimPrefix(rtSrv.URL, "http"), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + ResendInterval: 20 * time.Millisecond, + }, 30*time.Millisecond)) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt}) + defer u.Close() + + waitForCond(t, 2*time.Second, rt.ServerDisabled) + waitForCond(t, 3*time.Second, func() bool { return !rt.ServerDisabled() }) + waitForCond(t, 3*time.Second, rt.Connected) + // pollRealtimeMode (the Uplink's own tick-driven poll of + // rt.ServerDisabled()) needs one more flush tick after the client-level + // signal clears to flip the Uplink's own legacyMode back off; wait for + // that directly rather than racing it with a fixed sleep. + waitForCond(t, 2*time.Second, func() bool { return !u.inLegacyMode() }) + + baseline := len(httpCap.all()) + + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "afterReenable"; e.Level = "info" })) + + select { + case e := <-rtReceived: + if e.Type != wire.TypeMessageEvent { + t.Fatalf("expected message.event over realtime, got %s", e.Type) + } + case <-time.After(3 * time.Second): + t.Fatal("expected the new event to reach realtime once the server re-enabled it") + } + + time.Sleep(100 * time.Millisecond) + if got := len(httpCap.all()); got != baseline { + t.Fatalf("expected the /v2 spy to stop growing once realtime resumed: was %d, now %d", baseline, got) + } +} diff --git a/daemon/internal/uplink/realtime_test.go b/daemon/internal/uplink/realtime_test.go new file mode 100644 index 00000000..4c4aa083 --- /dev/null +++ b/daemon/internal/uplink/realtime_test.go @@ -0,0 +1,286 @@ +package uplink + +import ( + "context" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/rttest" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestRealtimeFlagRoutesEventsAwayFromHTTP is the integration check for the +// "flag on -> reliable events go over WS, HTTP ingest stops carrying them; +// flag off -> unchanged" requirement: with Config.Realtime set, task +// lifecycle and message.send events must reach the mock realtime server +// (never the HTTP /v2/ingest capture), while task.log keeps going over +// HTTP exactly as before. +func TestRealtimeFlagRoutesEventsAwayFromHTTP(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + received := make(chan wire.Envelope, 16) + rtSrv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + received <- env + if env.Type == wire.TypeTaskEvent || env.Type == wire.TypeMessageEvent { + body, _ := wire.DecodeBody(env) + switch b := body.(type) { + case wire.TaskEventBody: + conn.Ack(b.DeviceID, b.DeviceSeq) + case wire.MessageEventBody: + conn.Ack(b.DeviceID, b.DeviceSeq) + } + } + } + }) + defer rtSrv.Close() + + store, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.db")) + if err != nil { + t.Fatalf("outbox.Open: %v", err) + } + defer store.Close() + + rt := rtclient.New(rtclient.Config{ + URL: rtSrv.URL(), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + }) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 20 * time.Millisecond, Realtime: rt}) + defer u.Close() + + u.Offer(ev(protocol.TaskStart, func(e *Event) { e.Title = "Nightly backup" })) + u.Offer(ev(protocol.MessageSend, func(e *Event) { e.Text = "hi"; e.Level = "info" })) + u.Offer(ev(protocol.MetricUpdate, func(e *Event) { e.Key = "cpu.load"; e.Value = "0.5" })) + u.LogLine("s1", "some output line") + + // The realtime server should see the task.event and message.event. + seenTypes := map[string]bool{} + deadline := time.After(2 * time.Second) +loop: + for len(seenTypes) < 2 { + select { + case env := <-received: + seenTypes[env.Type] = true + case <-deadline: + break loop + } + } + if !seenTypes[wire.TypeTaskEvent] { + t.Error("expected the realtime server to see a task.event") + } + if !seenTypes[wire.TypeMessageEvent] { + t.Error("expected the realtime server to see a message.event") + } + + // Give the HTTP flusher a chance to run, then assert it never carried + // the task/message events (only task.log, which has no realtime + // equivalent). + time.Sleep(100 * time.Millisecond) + u.Close() + + for _, e := range httpCap.all() { + if e.Kind == protocol.TaskStart || e.Kind == protocol.MessageSend || e.Kind == protocol.MetricUpdate { + t.Fatalf("HTTP ingest carried a %s event while the realtime flag was on (double write): %+v", e.Kind, e) + } + } + foundLog := false + for _, e := range httpCap.all() { + if e.Kind == protocol.TaskLog { + foundLog = true + } + } + if !foundLog { + t.Error("expected task.log to still flow over HTTP (it has no realtime equivalent)") + } +} + +// TestRealtimeFlagOffPreservesExistingBehavior asserts the zero-value +// (Realtime == nil) path is completely unaffected: everything still goes +// over HTTP, exactly as before this feature existed. This also pins the +// local-flag-off row of pollRealtimeMode's truth table: with the local +// SITREP_REALTIME flag off, cmd/sitrep's newAgentRealtimeUplink never +// constructs an rtclient.Client at all (see its doc comment), so +// Config.Realtime is nil here exactly as it is in that path — there is no +// Client, so structurally no dial ever happens, let alone a rejected one. +func TestRealtimeFlagOffPreservesExistingBehavior(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: time.Hour}) + u.Offer(ev(protocol.TaskStart, func(e *Event) { e.Title = "t" })) + u.Close() + + got := httpCap.all() + if len(got) != 1 || got[0].Kind != protocol.TaskStart { + t.Fatalf("expected exactly one task.start over HTTP, got %+v", got) + } +} + +// TestOutboxFullNeverForksToHTTPAndRecovers pins the P0 fix: /v2 ingest +// writes UserStore while /v3 viewers resume from SpaceHub, so a reliable +// event (task.event/message.event) diverted to /v2 while the realtime flag +// is on could permanently vanish from a viewer's resume. When the realtime +// outbox hits its row cap, the overflow event must NOT reach the HTTP +// ingest spy at all — it is durably persisted into outbox_overflow (see +// outbox.ErrOverflowed) rather than rejected, and delivered over realtime, +// in the original offer order, once an ack frees outbox capacity and +// promotes it (outbox.Store.PromoteOverflow). +func TestOutboxFullNeverForksToHTTPAndRecovers(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + wsMessages := make(chan wire.MessageEventBody, 16) + ackRelease := make(chan struct{}) + rtSrv := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess", 1000); err != nil { + t.Errorf("HelloAccept: %v", err) + return + } + released := false + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeMessageEvent { + continue + } + body, _ := wire.DecodeBody(env) + mb := body.(wire.MessageEventBody) + select { + case wsMessages <- mb: + default: + } + if !released { + select { + case <-ackRelease: + released = true + default: + } + } + if released { + conn.Ack(mb.DeviceID, mb.DeviceSeq) + } + } + }) + defer rtSrv.Close() + + // Cap of 1: the first unacked reliable event fills the outbox. + store, err := outbox.OpenWithMaxRows(filepath.Join(t.TempDir(), "outbox.db"), 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + defer store.Close() + + rt := rtclient.New(rtclient.Config{ + URL: rtSrv.URL(), + DeviceID: "device-1", + Space: "space-1", + Outbox: store, + ResendInterval: 20 * time.Millisecond, + }) + defer rt.Close() + + u := New(Config{ServerURL: httpSrv.URL, FlushInterval: 20 * time.Millisecond, Realtime: rt}) + defer u.Close() + + msg := func(text string) Event { + return ev(protocol.MessageSend, func(e *Event) { e.Text = text; e.Level = "info" }) + } + waitForWS := func(text string) { + t.Helper() + deadline := time.After(3 * time.Second) + for { + select { + case mb := <-wsMessages: + if mb.Text == text { + return + } + case <-deadline: + t.Fatalf("realtime server never saw message %q", text) + } + } + } + + // 1. First event: fits in the outbox (row 1/1), goes over WS, unacked. + u.Offer(msg("first")) + waitForWS("first") + + // 2. Second event: outbox is full -> durably persisted into + // outbox_overflow (ErrOverflowed). Per the P0 fix this must NOT fall + // back to HTTP; it stays in overflow until an ack frees outbox capacity + // and promotes it. Give it several flush intervals to (wrongly) reach + // HTTP if the fallback regressed, then assert it never did. + u.Offer(msg("second")) + time.Sleep(200 * time.Millisecond) + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend && e.Text == "second" { + t.Fatalf("HTTP ingest carried the overflow event %q while the outbox was full — reliable events must never fork off the realtime path", e.Text) + } + } + + // 3. The server starts acking; the resend loop replays "first", it gets + // acked, and the outbox drains. + close(ackRelease) + waitFor := func(cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("condition not met within 3s") + } + waitFor(func() bool { + n, err := store.Count(context.Background()) + return err == nil && n == 0 + }) + + // 4. Once the outbox drains, the locally-queued "second" event's retry + // succeeds and it is delivered over realtime — never having touched + // HTTP — in the correct seq order (it was offered before "third"). + waitForWS("second") + + // 5. With capacity restored, the realtime path continues to take + // subsequent events. + u.Offer(msg("third")) + waitForWS("third") + + // None of these reliable events may ever have gone over HTTP. + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend { + t.Fatalf("HTTP ingest carried %q, which should have traveled the realtime path only", e.Text) + } + } +} diff --git a/daemon/internal/uplink/residual_test.go b/daemon/internal/uplink/residual_test.go new file mode 100644 index 00000000..7eb8aef4 --- /dev/null +++ b/daemon/internal/uplink/residual_test.go @@ -0,0 +1,130 @@ +package uplink + +import ( + "errors" + "fmt" + "testing" + + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" +) + +// TestSendReliableDropsPermanentValidationError pins the permanent/transient +// split: rtclient.ErrInvalidBody (a wire-validation failure) is never +// retried — the same malformed body can never validate differently — so it +// is logged and dropped instead of being held in the residual queue. +func TestSendReliableDropsPermanentValidationError(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + send := func() error { + return fmt.Errorf("%w: task.event: missing task_id", rtclient.ErrInvalidBody) + } + + handled := u.sendReliable(send) + if !handled { + t.Fatal("sendReliable must always report handled=true for a reliable event") + } + if len(u.residual) != 0 { + t.Fatalf("expected a validation failure to be dropped, not held for retry; residual has %d items", len(u.residual)) + } +} + +// TestSendReliableHoldsResidualOnPersistentFailure is the regression test +// for the rare residual-retry path: since outbox.Store.Enqueue now persists +// synchronously (durably, to the outbox or its overflow table, retrying +// transient SQLite failures internally and bounded), the only way send() +// can fail here with something other than rtclient.ErrInvalidBody is a +// genuinely persistent local-disk failure that outlasted Enqueue's own +// bounded retries. In that rare case the event must not be dropped: it is +// held in memory and retried on the next drainResidual call (normally +// driven by the flush loop, once per tick) until it finally persists. +func TestSendReliableHoldsResidualOnPersistentFailure(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + + persistentErr := errors.New("outbox: enqueue failed after 3 attempts: disk I/O error") + attempts := 0 + delivered := false + send := func() error { + attempts++ + if attempts < 3 { + return persistentErr + } + delivered = true + return nil + } + + handled := u.sendReliable(send) + if !handled { + t.Fatal("sendReliable must always report handled=true for a reliable event") + } + if len(u.residual) != 1 { + t.Fatalf("expected the event held in residual after a non-validation failure, residual has %d items", len(u.residual)) + } + if delivered { + t.Fatal("event must not have been delivered on the failed first attempt") + } + + u.drainResidual() // still fails (attempts=2) + if len(u.residual) != 1 { + t.Fatalf("expected the event to remain in residual after a second failed attempt, residual has %d items", len(u.residual)) + } + + u.drainResidual() // succeeds (attempts=3) + if !delivered { + t.Fatal("expected the event to be delivered once send() finally succeeded") + } + if len(u.residual) != 0 { + t.Fatalf("expected residual drained after the successful retry, has %d items", len(u.residual)) + } +} + +// TestDrainResidualPreservesFIFOAndDropsPermanentMidQueue asserts +// drainResidual walks the residual queue oldest-first, stopping at the +// first entry that is still failing (so a later entry can never be +// delivered ahead of an earlier one still stuck), while a permanent +// validation failure encountered mid-queue is dropped in place rather than +// blocking everything behind it. +func TestDrainResidualPreservesFIFOAndDropsPermanentMidQueue(t *testing.T) { + u := &Uplink{cfg: Config{Logf: func(string, ...any) {}}} + + var order []string + transientErr := errors.New("outbox: enqueue failed after 3 attempts: disk I/O error") + + // A: fails once, then succeeds. + aAttempts := 0 + sendA := func() error { + aAttempts++ + if aAttempts == 1 { + return transientErr + } + order = append(order, "A") + return nil + } + // B: permanently invalid. + sendB := func() error { + return fmt.Errorf("%w: bad body", rtclient.ErrInvalidBody) + } + // C: always succeeds once reached. + sendC := func() error { + order = append(order, "C") + return nil + } + + u.sendReliable(sendA) // fails -> held + if len(u.residual) != 1 { + t.Fatalf("expected A held after its first failed attempt, residual has %d items", len(u.residual)) + } + // B and C are queued directly (bypassing sendReliable's own send() + // attempt) to deterministically build a 3-item residual queue without + // depending on timing. + u.residualMu.Lock() + u.residual = append(u.residual, sendB, sendC) + u.residualMu.Unlock() + + u.drainResidual() + + if len(u.residual) != 0 { + t.Fatalf("expected residual fully drained (A recovers, B is permanently dropped, C succeeds), has %d items: order so far %v", len(u.residual), order) + } + if len(order) != 2 || order[0] != "A" || order[1] != "C" { + t.Fatalf("delivery order = %v, want [A C] (B dropped in place, not blocking C)", order) + } +} diff --git a/daemon/internal/uplink/restart_test.go b/daemon/internal/uplink/restart_test.go new file mode 100644 index 00000000..dd6feb2b --- /dev/null +++ b/daemon/internal/uplink/restart_test.go @@ -0,0 +1,219 @@ +package uplink + +import ( + "context" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/outbox" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/rttest" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" +) + +// TestOverflowSurvivesProcessRestartAndDeliversInOrder is the reviewer's +// exact combined scenario: a full outbox forces a reliable event into +// overflow, the Uplink (and the realtime Client, and the outbox handle +// underneath it) shut down exactly as a process exit would, a brand new +// process opens a brand new Store handle on the same file (not the same +// open handle — genuinely closed and reopened, so this exercises the real +// on-disk recovery path rather than an in-memory one), capacity frees once +// the still-unacked older event is finally acked, the overflowed event is +// promoted into the outbox in its original offer order, and it is +// delivered over realtime to a mock server — with the /v2 HTTP spy seeing +// nothing at all, at any point in the test (reliable events must never +// fork off the realtime path, restart or not). +func TestOverflowSurvivesProcessRestartAndDeliversInOrder(t *testing.T) { + var httpCap capture + httpSrv := httptest.NewServer(httpCap.handler(t)) + defer httpSrv.Close() + + dbPath := filepath.Join(t.TempDir(), "outbox.db") + const deviceID = "device-1" + const space = "space-1" + + msg := func(text string) Event { + return ev(protocol.MessageSend, func(e *Event) { e.Text = text; e.Level = "info" }) + } + + // ---- "process 1": fill the (cap-1) outbox, overflow the second event, + // then shut down without ever acking the first event — simulating a + // crash/restart while the server (or the connection to it) is down. + firstSeen := make(chan wire.MessageEventBody, 4) + rtSrv1 := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess-1", 1000); err != nil { + t.Errorf("phase 1 HelloAccept: %v", err) + return + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeMessageEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + continue + } + select { + case firstSeen <- body.(wire.MessageEventBody): + default: + } + // Deliberately never ack: the event must still be sitting + // unacked in the outbox when process 1 shuts down. + } + }) + + store1, err := outbox.OpenWithMaxRows(dbPath, 1) + if err != nil { + t.Fatalf("OpenWithMaxRows: %v", err) + } + + rt1 := rtclient.New(rtclient.Config{ + URL: rtSrv1.URL(), DeviceID: deviceID, Space: space, Outbox: store1, + }) + + u1 := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt1}) + + // 1. Fills the cap-1 outbox (device_seq 1), delivered over WS, left + // unacked on purpose. + u1.Offer(msg("first")) + select { + case mb := <-firstSeen: + if mb.Text != "first" { + t.Fatalf("phase 1 saw %q, want %q", mb.Text, "first") + } + case <-time.After(3 * time.Second): + t.Fatal("realtime server never saw the first event") + } + + // 2. Offer() enqueues synchronously (Enqueue is called directly from + // the calling goroutine, not the flush loop — see routeToRealtime), so + // by the time Offer returns the outbox-full check has already run: this + // event is durably in outbox_overflow, not the outbox, no device_seq + // allocated yet. + u1.Offer(msg("second")) + if n, err := store1.OverflowCount(context.Background()); err != nil || n != 1 { + t.Fatalf("OverflowCount = %d, err = %v; want exactly 1 overflowed event after the outbox filled", n, err) + } + + // 3. Shut down exactly as a process exit would: Uplink, then the + // realtime Client, then the Store handle itself. + u1.Close() + rt1.Close() + rtSrv1.Close() + if err := store1.Close(); err != nil { + t.Fatalf("store1.Close: %v", err) + } + + // ---- "process 2": a brand new Store handle on the same file (never + // reusing store1/rt1/u1) — this is what makes it a genuine restart test + // rather than an in-memory one. + secondSeen := make(chan wire.MessageEventBody, 4) + rtSrv2 := rttest.New(func(conn *rttest.Conn) { + if _, err := conn.HelloAccept("sess-2", 1000); err != nil { + t.Errorf("phase 2 HelloAccept: %v", err) + return + } + for { + env, err := conn.ReadEnvelope() + if err == rttest.ErrPing { + _ = conn.WritePong() + continue + } + if err != nil { + return + } + if env.Type != wire.TypeMessageEvent { + continue + } + body, derr := wire.DecodeBody(env) + if derr != nil { + continue + } + mb := body.(wire.MessageEventBody) + select { + case secondSeen <- mb: + default: + } + // Ack everything this time: this is what frees the cap-1 + // outbox's one slot and lets Store.Ack's own promotion trigger + // move the overflowed event in. + _ = conn.Ack(mb.DeviceID, mb.DeviceSeq) + } + }) + defer rtSrv2.Close() + + store2, err := outbox.OpenWithMaxRows(dbPath, 1) + if err != nil { + t.Fatalf("reopen OpenWithMaxRows: %v", err) + } + defer store2.Close() + + // Reopening at the same cap must NOT have promoted the overflowed event + // on its own: the outbox still holds the never-acked "first" row, so + // there is no free capacity yet — promotion only happens once that row + // is finally acked below. + if n, err := store2.Count(context.Background()); err != nil || n != 1 { + t.Fatalf("outbox count after reopen = %d, err = %v; want 1 (the still-unacked first event)", n, err) + } + if n, err := store2.OverflowCount(context.Background()); err != nil || n != 1 { + t.Fatalf("overflow count after reopen = %d, err = %v; want 1 (the second event, still not promoted)", n, err) + } + + rt2 := rtclient.New(rtclient.Config{ + URL: rtSrv2.URL(), DeviceID: deviceID, Space: space, Outbox: store2, + ResendInterval: 20 * time.Millisecond, + }) + defer rt2.Close() + + u2 := New(Config{ServerURL: httpSrv.URL, FlushInterval: 15 * time.Millisecond, Realtime: rt2}) + defer u2.Close() + + // 4/5. On reconnect, replayPending resends the still-unacked "first"; + // the mock server acks it, which frees the cap-1 outbox's one slot and + // promotes "second" out of overflow (Store.Ack's own trigger) with the + // next device_seq in this space's sequence (2) — never reusing seq 1, + // and never skipping ahead of it. Assert both are delivered, in that + // order, entirely over realtime. + var got []string + deadline := time.After(3 * time.Second) + for len(got) < 2 { + select { + case mb := <-secondSeen: + got = append(got, mb.Text) + case <-deadline: + t.Fatalf("timed out waiting for both events after restart; got so far: %v", got) + } + } + if got[0] != "first" || got[1] != "second" { + t.Fatalf("post-restart delivery order = %v, want [first second] (the promoted event must not jump ahead of the still-unacked older one)", got) + } + + waitForCond(t, 3*time.Second, func() bool { + n, err := store2.Count(context.Background()) + return err == nil && n == 0 + }) + if n, err := store2.OverflowCount(context.Background()); err != nil || n != 0 { + t.Fatalf("overflow count at end = %d, err = %v; want 0 (promoted and delivered)", n, err) + } + + // Throughout the whole test — fill, overflow, restart, promotion, + // delivery — the /v2 HTTP spy must never have carried either reliable + // event: it is not idempotent for a viewer resuming from SpaceHub the + // way /v3 is, so a fork here would be a silent, permanent miss. + for _, e := range httpCap.all() { + if e.Kind == protocol.MessageSend { + t.Fatalf("HTTP /v2 ingest carried %q; reliable events must travel the realtime path only, restart or not", e.Text) + } + } +} diff --git a/daemon/internal/uplink/uplink.go b/daemon/internal/uplink/uplink.go index 4f46aa62..46711121 100644 --- a/daemon/internal/uplink/uplink.go +++ b/daemon/internal/uplink/uplink.go @@ -9,9 +9,11 @@ package uplink import ( "bytes" + "context" "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "strings" @@ -19,6 +21,8 @@ import ( "time" "github.com/QuintinShaw/sitrep/daemon/internal/protocol" + rtclient "github.com/QuintinShaw/sitrep/daemon/internal/realtime/client" + "github.com/QuintinShaw/sitrep/daemon/internal/realtime/wire" ) // Event is the wire form of a protocol event (schema/event.schema.json). @@ -47,6 +51,22 @@ type Config struct { // (including empty heartbeats every 3 ticks) carry ?sources= and any // commands in the response are delivered on Commands. CommandSource string + + // Realtime is an opt-in feature flag: when non-nil, every reliable + // event (task lifecycle + message.send) and every metric.update this + // Uplink is Offer()ed is routed to the realtime WebSocket uplink + // (internal/realtime/client) instead of the HTTP /v2/ingest batch — + // the two paths are never used for the same event, so nothing is + // double-written. task.log passthrough output is unaffected: it has + // no realtime-protocol equivalent and always continues over HTTP. + // Nil (the default) preserves this package's exact prior behavior. + // The caller owns Realtime's lifecycle (construction and Close). + Realtime *rtclient.Client + + // Logf is a pluggable logger for conditions worth surfacing but not + // worth failing on (e.g. a reliable event dropped because it could + // never validate). Nil discards. + Logf func(format string, args ...any) } type Uplink struct { @@ -65,6 +85,25 @@ type Uplink struct { kick chan struct{} done chan struct{} + // residualMu guards residual, the last-resort in-memory queue for a + // reliable event (task.event/message.event) that could not be durably + // persisted at all — not even into the outbox's overflow table — after + // outbox.Store.Enqueue exhausted its own bounded internal retries (see + // that method's doc comment). This is NOT the local-backpressure path: + // an outbox at its row cap durably overflows instead of failing (see + // outbox.ErrOverflowed), so it never reaches here. residual exists only + // for the rare case of a genuinely failing local disk (e.g. the outbox + // file's filesystem is full or read-only) — see sendReliable. + residualMu sync.Mutex + residual []func() error + + // legacyMu guards legacyMode: whether the server has rejected the + // realtime dial (rtclient.ErrRealtimeDisabled) and this Uplink has + // switched the whole session to routing reliable events over the + // legacy /v2 HTTP path instead (see pollRealtimeMode). + legacyMu sync.Mutex + legacyMode bool + ticksSinceSend int } @@ -80,6 +119,9 @@ func New(cfg Config) *Uplink { if cfg.HTTPTimeout <= 0 { cfg.HTTPTimeout = 10 * time.Second } + if cfg.Logf == nil { + cfg.Logf = func(string, ...any) {} + } u := &Uplink{ cfg: cfg, client: &http.Client{Timeout: cfg.HTTPTimeout}, @@ -98,6 +140,24 @@ func (u *Uplink) Offer(ev Event) { if u == nil { return } + u.mu.Lock() + if u.closed { + u.mu.Unlock() + return + } + realtime := u.cfg.Realtime + u.mu.Unlock() + + // Feature flag: when a realtime uplink is configured AND the server + // hasn't rejected it (see the truth table on pollRealtimeMode), every + // kind with a realtime-protocol equivalent is routed there instead of + // the HTTP batch below, so the same event is never sent both ways. + // task.log (no realtime equivalent) and any realtime send failure fall + // through. + if realtime != nil && !u.inLegacyMode() && u.routeToRealtime(realtime, ev) { + return + } + u.mu.Lock() defer u.mu.Unlock() if u.closed { @@ -125,6 +185,363 @@ func (u *Uplink) Offer(ev Event) { } } +// routeToRealtime translates ev into the matching realtime-protocol +// message and hands it to the realtime client. It reports whether it +// handled ev (true) so the caller skips the HTTP batch entirely for that +// event, or leaves it (false) to fall through unchanged — only because the +// kind has no realtime equivalent (task.log). +// +// Reliable events (task.event/message.event) NEVER fall back to the legacy +// /v2 HTTP path while the realtime flag is on AND the server accepts +// realtime, even when a single send attempt fails: /v2 ingest writes +// UserStore, while /v3 viewers resume from SpaceHub, so diverting so much as +// one terminal task.done/task.fail/message.send would let a viewer +// permanently miss it. See sendReliable for what happens to a send failure +// instead. (The one deliberate exception to "never falls back" is the +// server-disabled whole-session legacy mode this Uplink switches into via +// pollRealtimeMode — see that function's truth table — which routeToRealtime +// is not even called under, per Offer's inLegacyMode guard.) +func (u *Uplink) routeToRealtime(realtime *rtclient.Client, ev Event) bool { + switch ev.Kind { + case protocol.TaskStart, protocol.TaskProgress, protocol.TaskStep, protocol.TaskDone, protocol.TaskFail: + te := rtclient.TaskEvent{ + TaskID: ev.SourceID, + Kind: taskEventKind(ev.Kind), + OccurredAt: parseEventTime(ev.TS), + Display: displayHints(ev), + } + switch ev.Kind { + case protocol.TaskStart: + te.Title = ev.Title + case protocol.TaskProgress: + p := ev.Percent + te.Percent = &p + te.Step = ev.Step + case protocol.TaskStep: + te.Step = ev.Step + case protocol.TaskDone, protocol.TaskFail: + te.Message = ev.Text + } + return u.sendReliable(func() error { return realtime.SendTaskEvent(te) }) + case protocol.MessageSend: + me := rtclient.MessageEvent{ + Level: ev.Level, + Text: ev.Text, + OccurredAt: parseEventTime(ev.TS), + } + return u.sendReliable(func() error { return realtime.SendMessageEvent(me) }) + case protocol.MetricUpdate: + realtime.SendMetric(wire.MetricSample{ + MetricID: ev.Key, + Value: ev.Value, + Label: ev.Label, + TS: parseEventTime(ev.TS).UnixMilli(), + Display: displayHints(ev), + Target: ev.Target, + Min: ev.Min, + Max: ev.Max, + AlertAbove: ev.AlertAbove, + AlertBelow: ev.AlertBelow, + }) + return true // metric.frame is fire-and-forget; there is no failure to react to + default: + return false // e.g. task.log: no realtime-protocol equivalent + } +} + +// sendReliable always reports "handled" (true) for a reliable event — the +// caller (routeToRealtime) must never fall back to HTTP for these, so there +// is nothing left for it to do except decide whether to hold it for a +// residual retry. +// +// send() is outbox.Store.Enqueue underneath (via rtclient.Client.Send*), +// which now persists synchronously: it durably writes the event into either +// the outbox or its overflow table before returning, retrying transient +// SQLite failures internally, bounded (see that method's doc comment). So +// by the time send() returns here, one of three things is true: +// +// - success: the event is durable and (if the outbox had room) already +// handed to the realtime connection to deliver; +// - rtclient.ErrInvalidBody: the event can never validate, at any seq, on +// any retry — logged and dropped, permanently; +// - anything else: Enqueue's own bounded retries were exhausted — a +// persistent local-disk failure. The event was NOT durably written +// anywhere. This is the one case this function still holds the event in +// memory (residual) and keeps retrying every flush tick, so it is not +// silently dropped while the process lives. This residual window exists +// ONLY under that persistent local-disk failure; ordinary outbox-full +// backpressure durably overflows inside Enqueue and never reaches here. +func (u *Uplink) sendReliable(send func() error) bool { + if err := send(); err != nil { + if errors.Is(err, rtclient.ErrInvalidBody) { + u.cfg.Logf("uplink: dropping reliable event, validation failed: %v", err) + return true + } + u.cfg.Logf("uplink: reliable event failed to persist (%v); holding in memory and retrying every flush tick", err) + u.residualMu.Lock() + u.residual = append(u.residual, send) + u.residualMu.Unlock() + u.wakeFlush() + } + return true +} + +// drainResidual re-attempts every event sendReliable could not persist at +// all, oldest first, stopping at the first one that still fails so order +// among residual entries is preserved. Called once per flush tick from +// loop. In the overwhelming majority of runs residual is always empty, so +// this is a cheap no-op check. +func (u *Uplink) drainResidual() { + u.residualMu.Lock() + defer u.residualMu.Unlock() + for len(u.residual) > 0 { + send := u.residual[0] + if err := send(); err != nil { + if errors.Is(err, rtclient.ErrInvalidBody) { + u.cfg.Logf("uplink: dropping reliable event, validation failed on retry: %v", err) + u.residual = u.residual[1:] + continue + } + return // still failing to persist; try again next tick + } + u.residual = u.residual[1:] + } +} + +// wakeFlush nudges the flush loop to run sooner than the next tick, e.g. +// after queuing a reliable event for retry so a freed outbox row is picked +// up promptly rather than waiting a full FlushInterval. +func (u *Uplink) wakeFlush() { + select { + case u.kick <- struct{}{}: + default: + } +} + +// inLegacyMode reports whether this Uplink is currently routing reliable +// events over the legacy /v2 HTTP path because the server rejected the +// realtime dial (see pollRealtimeMode). +func (u *Uplink) inLegacyMode() bool { + u.legacyMu.Lock() + defer u.legacyMu.Unlock() + return u.legacyMode +} + +// pollRealtimeMode is called once per flush tick from loop. It is the +// daemon-side half of the server-capability truth table: +// +// local SITREP_REALTIME | server /v3/realtime | routing +// -----------------------|----------------------------|-------------------- +// off | (never dialed) | legacy /v2, always +// on | accepts (normal) | realtime +// on | 403 realtime_disabled | legacy /v2, whole +// | | session, until the +// | | client's own long- +// | | interval re-probe +// | | (see rtclient's +// | | serverDisabledRecheckInterval) +// | | succeeds again +// +// The local flag is enforced upstream: newAgentRealtimeUplink (cmd/sitrep) +// never even constructs an rtclient.Client, let alone dials, when +// SITREP_REALTIME is off, so cfg.Realtime is nil and this function is a +// no-op (Offer's nil check on cfg.Realtime already covers that row). +// +// The remaining two rows are what this function drives: rtclient.Client +// dials in the background on its own reconnect loop and exposes +// ServerDisabled() as the result of its last dial attempt. When that flips +// true, this Uplink switches its whole session to legacy routing (not just +// the one event that happened to be in flight) and drains every event +// already sitting durably in the outbox/overflow tables to /v2, in FIFO +// order, so a terminal task state that arrived before the flag flipped +// doesn't sit forever in a store no viewer reads. When it flips back to +// false (the client's periodic re-probe succeeded), new events resume +// routing to realtime immediately — there is no backlog to replay for +// those, since legacy mode drained everything as it went. +func (u *Uplink) pollRealtimeMode() { + rt := u.cfg.Realtime + if rt == nil { + return + } + disabled := rt.ServerDisabled() + + u.legacyMu.Lock() + wasLegacy := u.legacyMode + u.legacyMode = disabled + u.legacyMu.Unlock() + + if !disabled { + if wasLegacy { + u.cfg.Logf("uplink: realtime accepted again; resuming realtime routing for new events") + } + return + } + if !wasLegacy { + u.cfg.Logf("uplink: server rejected realtime (403 realtime_disabled); switching this session to legacy /v2 routing until it re-enables") + } + u.drainOutboxToLegacy(rt) +} + +// drainOutboxToLegacy hands rt.DrainToLegacy the translate-and-send step for +// every reliable event durably sitting in rt's shared outbox (real +// device_seq, ready to deliver) and then its overflow table (no device_seq +// yet), oldest first in each. rt owns retiring each row (Ack/DeleteOverflow) +// and — critically — owns serializing that read-send-retire sequence +// process-wide: rt is the one *rtclient.Client shared by every automation's +// Uplink (see cmd/sitrep/agent.go), so without rt's own per-row locking two +// Uplinks polling concurrently could both read and send the same unretired +// row. See rtclient.Client.DrainToLegacy's doc comment for the full +// invariant, including why this is also safe across a switch back to +// realtime mid-drain. +func (u *Uplink) drainOutboxToLegacy(rt *rtclient.Client) { + rt.DrainToLegacy(context.Background(), u.legacyDrainSink) +} + +// legacyDrainSink is the rtclient.LegacySink for drainOutboxToLegacy: +// translate the durable row back into the legacy uplink.Event shape and post +// it to /v2/ingest synchronously. An unrecognized kind can never be +// delivered on any retry, so it is logged and reported as retirable (true) +// without ever calling sendLegacySync — matching the previous drain's +// behavior of discarding it rather than blocking the backlog behind it. +func (u *Uplink) legacyDrainSink(kind string, body json.RawMessage) bool { + ev, ok := legacyEventFromOutboxItem(kind, body) + if !ok { + u.cfg.Logf("uplink: legacy drain: dropping outbox item with unrecognized kind %q", kind) + return true + } + return u.sendLegacySync(ev) +} + +// sendLegacySync posts a single event to /v2/ingest synchronously and +// reports whether the server accepted it (2xx). Unlike send (the normal +// best-effort batch flusher, which retries a few times and then drops on +// failure since telemetry is lossy by design), the legacy drain must know +// definitively whether to retire the durable item it just sent, so it needs +// a real success/failure signal rather than fire-and-forget. +func (u *Uplink) sendLegacySync(ev Event) bool { + body, err := json.Marshal([]Event{ev}) + if err != nil { + return false + } + req, err := http.NewRequest(http.MethodPost, u.cfg.ServerURL+"/v2/ingest", bytes.NewReader(body)) + if err != nil { + return false + } + req.Header.Set("Content-Type", "application/json") + if u.cfg.Token != "" { + req.Header.Set("Authorization", "Bearer "+u.cfg.Token) + } + resp, err := u.client.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode >= 200 && resp.StatusCode < 300 +} + +// legacyEventFromOutboxItem reverse-translates a durably-stored reliable +// event body (wire.TaskEventBody or wire.MessageEventBody JSON, as written +// by routeToRealtime's forward direction) back into the legacy uplink.Event +// shape /v2/ingest expects. ok is false for a kind this function does not +// recognize (defensive; the outbox/overflow tables only ever hold the two +// kinds routeToRealtime writes). +func legacyEventFromOutboxItem(kind string, body json.RawMessage) (Event, bool) { + switch kind { + case wire.TypeTaskEvent: + var b wire.TaskEventBody + if err := json.Unmarshal(body, &b); err != nil { + return Event{}, false + } + k := legacyTaskKind(b.Kind) + if k == "" { + return Event{}, false + } + ev := Event{ + Event: protocol.Event{ + Kind: k, + Title: b.Title, + Step: b.Step, + Text: b.Message, + }, + SourceID: b.TaskID, + TS: time.UnixMilli(b.OccurredAt).UTC().Format(time.RFC3339), + } + if b.Percent != nil { + ev.Percent = *b.Percent + } + if b.Display != nil { + ev.Icon, ev.Tint, ev.Template = b.Display.Icon, b.Display.Tint, b.Display.Template + } + return ev, true + case wire.TypeMessageEvent: + var b wire.MessageEventBody + if err := json.Unmarshal(body, &b); err != nil { + return Event{}, false + } + return Event{ + Event: protocol.Event{Kind: protocol.MessageSend, Text: b.Text, Level: b.Level}, + SourceID: b.AutomationID, + TS: time.UnixMilli(b.OccurredAt).UTC().Format(time.RFC3339), + }, true + default: + return Event{}, false + } +} + +// legacyTaskKind reverses taskEventKind (wire task.event "kind" string back +// to the protocol.Kind the legacy HTTP path expects). Empty return means +// unrecognized. +func legacyTaskKind(wireKind string) protocol.Kind { + switch wireKind { + case "started": + return protocol.TaskStart + case "progress": + return protocol.TaskProgress + case "step": + return protocol.TaskStep + case "done": + return protocol.TaskDone + case "failed": + return protocol.TaskFail + default: + return "" + } +} + +func taskEventKind(k protocol.Kind) string { + switch k { + case protocol.TaskStart: + return "started" + case protocol.TaskProgress: + return "progress" + case protocol.TaskStep: + return "step" + case protocol.TaskDone: + return "done" + case protocol.TaskFail: + return "failed" + default: + return "" + } +} + +func displayHints(ev Event) *wire.DisplayHints { + if ev.Icon == "" && ev.Tint == "" && ev.Template == "" { + return nil + } + return &wire.DisplayHints{Icon: ev.Icon, Tint: ev.Tint, Template: ev.Template} +} + +// parseEventTime parses the RFC3339 timestamp uplink.Event carries +// (protocol.go / runner.MakeEmitter always stamp one); a malformed or empty +// value falls back to now rather than producing an invalid envelope. +func parseEventTime(ts string) time.Time { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + return t + } + return time.Now() +} + // LogLine buffers one passthrough output line for the task detail view. // Batched into a single task.log event per flush; capped per window. func (u *Uplink) LogLine(sourceID, line string) { @@ -174,6 +591,8 @@ func (u *Uplink) loop() { case <-ticker.C: case <-u.kick: } + u.drainResidual() + u.pollRealtimeMode() batch, closed := u.drain() if len(batch) > 0 { u.send(batch) diff --git a/docs/design/realtime-integration.md b/docs/design/realtime-integration.md new file mode 100644 index 00000000..c43b3b4f --- /dev/null +++ b/docs/design/realtime-integration.md @@ -0,0 +1,278 @@ +# Realtime protocol cross-line integration + +This document records the result of integrating three independently +developed implementations of `proto/realtime/` onto `integration/realtime`: + +- `server/` (Cloudflare Workers, TypeScript) — the authoritative "SpaceHub+ + v3" server. +- `daemon/` (Go) — the `source` role client (device daemon that pushes task + events). +- `apple/SitrepKit` (Swift) — the `viewer` role client (subscribes to space + state). + +It is not a spec: `proto/realtime/` remains the source of truth. This +records what was verified, how, and how to re-run it. + +## 1. Full regression (per line) + +| Suite | Command | Result | +| --- | --- | --- | +| Protocol fixtures | `cd proto/realtime/tools && npm install && npm run validate` | 81/81 | +| Server | `cd server && npm install && npm run typecheck && npm test` | typecheck clean; 88 (node:test fixtures) + 33 (vitest workers pool) = 121 | +| Daemon | `cd daemon && go build ./... && go vet ./... && go test ./... -race -count=1` | build/vet clean; 38 top-level tests, 0 failures | +| SitrepKit | `cd apple/SitrepKit && swift test` | 30/30 | +| SitrepApp | `cd apple/SitrepApp && xcodebuild build -scheme Sitrep -destination 'generic/platform=iOS Simulator'` | BUILD SUCCEEDED | +| SitrepMenuBar | `cd apple/SitrepMenuBar && swift build` | Build complete | + +No fixes were needed to get all six suites green as merged. + +## 2. Cross-line contract audit + +Six wire-level interface points were checked with file:line evidence from +all three lines. Full detail was produced by a dedicated audit pass; the +verdicts: + +1. **Endpoint URL** (`/v3/realtime`, scheme swap, no query params) — MATCH. + `daemon/internal/config/config.go:80-92` (`RealtimeURLFor`), + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeEndpoint.swift:14-24`, + `server/src/adapters/workers.ts:509`. +2. **Authorization header** (`Authorization: Bearer `) — MATCH. + `daemon/internal/realtime/client/client.go:266-268`, + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:226-228`, + `server/src/app.ts:175` (case-insensitive `Bearer` strip), + token format `TOKEN_RE` at `server/src/app.ts:22`. +3. **Hello handshake** — MATCH. Both clients send + `{stage, device_id, role, protocol_versions}` + (`daemon/internal/realtime/client/client.go:346-353`, + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:293`); + server negotiates the version intersection and replies + `{stage:"accept", protocol_version, session_id, heartbeat_interval_ms}` + (`server/src/realtime/space-hub.ts:430-489`). One intentional race was + confirmed both in code and live on the wire (§3 below): the server may + push a `command` envelope to a `source` connection immediately after + `hello{accept}`, before any application-level exchange + (`server/src/realtime/space-hub.ts:476-488`). Both clients handle this + correctly (daemon's `readLoop` just dispatches it as the next frame; + apple never receives it, since the push is source-only, and defensively + ignores a stray `command` if one ever arrived). This was reproduced live + during the E2E run below (`scripts/e2e/source-drive.mjs` log shows two + `recv command` frames arriving right after `hello{accept}`, before the + script's own `task.event` was sent). +4. **Heartbeat** (`ping`/`pong` as literal, lower-case, bare-text frames, + either side may initiate) — MATCH. `server/src/realtime/space-hub.ts:107` + (`setWebSocketAutoResponse(new WebSocketRequestResponsePair("ping", + "pong"))`), `daemon/internal/realtime/client/client.go:325,333`, + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:451-481`. +5. **Error codes** — MATCH. All 12 codes in + `server/src/realtime/protocol.ts:72-86` (`ERROR_SEMANTICS`) are mirrored + verbatim in `daemon/internal/realtime/wire/bodies.go:324-336` and + `apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeCommon.swift:196-209`. + Both clients react to the server-supplied `fatal`/`retryable` flags on + the wire rather than a hardcoded local table, so neither can silently + diverge from server semantics as codes evolve. +6. **Envelope id / device_seq** — MATCH. Schema: + `proto/realtime/common.schema.json` — `envelope_id` pattern + `^[A-Za-z0-9_-]{1,64}$`, `device_seq` minimum `1`. Server generates + `crypto.randomUUID()` ids and validates the same regex + (`server/src/realtime/protocol.ts:47-48`, + `server/src/realtime/guards.ts:41,420`). Daemon generates `"rt" + hex(16 + random bytes)` ids (`daemon/internal/realtime/client/client.go:89-95`) + and starts `device_seq` at 1 (`daemon/internal/realtime/outbox/outbox.go:168-183`). + Apple generates `UUID().uuidString` ids + (`apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeClient.swift:527`) + and validates incoming `device_seq >= 1` + (`apple/SitrepKit/Sources/SitrepKit/Realtime/RealtimeBodies.swift:304,406,473`). + +No mismatches were found. No integration fixes were required for the +contract points above. + +## 3. End-to-end smoke test (real cross-process, not a fake/in-process server) + +Both halves ran against a real `wrangler dev` instance (local workerd, not +`--remote`), i.e. real separate OS processes talking real WebSockets over +`127.0.0.1` — not an in-process/simulated transport. + +### Setup + +```bash +cd server +npm install +# The environment used a proxy; wrangler dev only needs to bind locally, so +# no proxy bypass was actually required beyond the existing NO_PROXY setting +# (NO_PROXY=localhost,127.0.0.1,::1,.local was already present). If your +# shell's NO_PROXY doesn't cover 127.0.0.1, unset http_proxy/https_proxy or +# add it before running the command below. +npx wrangler dev --port 8787 --local +``` + +Bootstrapping a test space and three device tokens (source, viewer1, +viewer2) uses the server's existing `/v2/spaces` + `/v2/invites` + `/v2/join` +HTTP endpoints — no new test-only backdoor was added: + +```bash +curl -s -X POST http://127.0.0.1:8787/v2/spaces \ + -H 'content-type: application/json' -d '{"platform":"macos","name":"e2e"}' +# => {"space_id":"...", "owner_token":"st2_..._..."} + +curl -s -X POST http://127.0.0.1:8787/v2/invites \ + -H "Authorization: Bearer $OWNER_TOKEN" -H 'content-type: application/json' \ + -d '{"role":"source"}' # or {"role":"viewer"} +# => {"code":"...", "expires_in":600, "space_id":"..."} + +curl -s -X POST http://127.0.0.1:8787/v2/join \ + -H 'content-type: application/json' \ + -d '{"code":"...", "space":"...", "name":"...", "platform":"..."}' +# => {"token":"st2_..._...", "device_id":"...", "role":"source|viewer", "space_id":"..."} +``` + +### Source role (real Go client, real network) + +`daemon/internal/realtime/client/e2e_test.go` (`TestE2ESourceLifecycle`) is a +new, opt-in test in the daemon's own `client` package. It is a no-op under +plain `go test ./...` (skips immediately) and only runs when four env vars +are set, so it never affects the regular regression run: + +```bash +cd daemon +SITREP_E2E_URL="ws://127.0.0.1:8787/v3/realtime" \ +SITREP_E2E_TOKEN="" \ +SITREP_E2E_DEVICE_ID="" \ +SITREP_E2E_SPACE="" \ +go test ./internal/realtime/client/... -run TestE2ESourceLifecycle -v +``` + +It drives the actual `daemon/internal/realtime/client.Client` (the same +code the daemon binary uses) through: hello offer/accept over the real +socket -> `task.event{started}` -> waits for the outbox to drain (server +ack) -> disconnects -> enqueues a second event while offline -> reconnects +with a fresh `Client` sharing the same on-disk outbox -> confirms the +pending event replays and is acked. Observed run: + +``` +=== RUN TestE2ESourceLifecycle + e2e_test.go:60: e2e: hello offer/accept completed against real server (task 1) + e2e_test.go:79: e2e: task.event{started} acked by real server, outbox drained + e2e_test.go:101: e2e: reconnect hello offer/accept completed (task 1, replay) + e2e_test.go:110: e2e: task.event{progress} replayed after reconnect and acked, outbox drained +--- PASS: TestE2ESourceLifecycle (10.10s) +``` + +### Viewer role (node `ws` script, real network) + +`scripts/e2e/viewer-smoke.mjs` connects two independent viewer devices with +the `ws` npm package (credentials via env vars only) and drives the +mandatory viewer sequence: `hello` -> `subscribe` -> `resume`. A companion +driver, `scripts/e2e/source-drive.mjs`, connects as the source device and, +after a delay (letting the viewers finish subscribing), sends one +`task.event` and one `metric.frame`. + +```bash +cd scripts/e2e +npm install +export SITREP_E2E_URL="ws://127.0.0.1:8787/v3/realtime" +export SITREP_E2E_SOURCE_TOKEN="..." SITREP_E2E_SOURCE_DEVICE_ID="..." +export SITREP_E2E_VIEWER1_TOKEN="..." SITREP_E2E_VIEWER2_TOKEN="..." +node source-drive.mjs & # background: waits, then emits the events +node viewer-smoke.mjs # foreground: asserts the full scenario +``` + +Observed run (trimmed to the load-bearing frames; full output is +reproducible verbatim by re-running the commands above): + +``` +[viewer1] send hello {"stage":"offer","device_id":"viewer1-e2e","role":"viewer"} +[viewer1] recv hello {"stage":"accept","protocol_version":1,"session_id":"0c66d634-..."} +[viewer1] send subscribe {"topics":["task","metric","message"]} +[viewer1] recv ack {"in_reply_to":"e2e-subscribe-2-...","lease":{"expires_at":...}} +[viewer1] send resume {"last_revision":0} +[viewer1] recv snapshot {"revision":0,"part":1,"final":true,"tasks":0,"metrics":0} +[viewer1] fresh-viewer resume(0) correctly produced a snapshot at revision 0 +[viewer2] ... same hello/subscribe/ack sequence ... +[viewer2] send resume {"last_revision":0} +[viewer2] recv snapshot {"revision":0,"part":1,"final":true,...} +[viewer2] recv delta {"from_revision":0,"to_revision":1,"events":1} +[viewer1] recv delta {"from_revision":0,"to_revision":1,"events":1} +[main] PASS: live delta chained correctly (snapshot rev 0 -> delta 0->1) +[viewer2] recv metric.frame {"device_id":"a85ee0f8-...","metrics":[{"metric_id":"e2e.cpu.load","value":"0.42","ts":...}]} +[main] PASS: viewer2 received the metric.frame broadcast triggered by the source +[main] ALL VIEWER E2E CHECKS PASSED +``` + +Source-side log for the same run (note the server's post-accept `command` +push mentioned in §2 item 3, arriving before the driver's own +`task.event`): + +``` +[source] recv hello {"stage":"accept","protocol_version":1,"session_id":"...","heartbeat_interval_ms":30000} +[source] recv command +[source] recv command +[source] send task.event (should trigger a live delta to subscribed viewers) +[source] recv ack {"acked":[{"device_id":"a85ee0f8-...","device_seq":1}]} +[source] send metric.frame (should broadcast to metric-subscribed viewers) +``` + +### What this confirms end-to-end + +- Real TCP/WebSocket connections across three separate Node/Go processes and + one real (local) Workers runtime — not an in-process fake. +- `resume{last_revision:0}` from a fresh viewer correctly returns a + `snapshot` (not a `delta`), matching the revision-gap-snapshot scenario in + `proto/realtime/fixtures/scenarios/client-revision-gap-snapshot/`. +- A live `task.event` from the source is turned into a `delta` broadcast to + every subscribed, lease-holding viewer, with revision continuity + (`snapshot.revision == delta.from_revision`, + `delta.to_revision == delta.from_revision + 1`). +- `metric.frame` from the source is broadcast to a viewer that declared + `metric` interest via `subscribe`, independent of the viewer that received + the live task delta — i.e. server fan-out works across N independent + viewer connections, not just the one that happened to trigger a resume. +- The server's post-hello-accept `command` push to `source` connections + (§2 item 3) is real and was observed live, not just inferred from code. + +No `wrangler dev` startup failure was encountered in this environment +(`NO_PROXY` already covered `127.0.0.1`/`localhost`), so the +vitest-pool-workers fallback described in the work order was not needed. + +### A bug found and fixed in the E2E scripts themselves (not a cross-line defect) + +The first draft of `source-drive.mjs`/`viewer-smoke.mjs` built each +envelope's `id` by embedding the message `type` literally (e.g. +`e2e-src-task.event-1-...`). `task.event` and `metric.frame` contain a `.`, +which `envelope_id`'s pattern (`^[A-Za-z0-9_-]{1,64}$`) forbids — the server +correctly rejected both with `error{code:"malformed", message:"invalid +envelope id"}`. Fixed by stripping non-alphanumeric characters out of the +type before using it in the id. This was a bug in the test fixture, not in +any of the three integrated lines. + +## 4. Integration fixes made in this worktree + +None. No cross-line contract mismatch or regression-suite failure required +a code change in `server/`, `daemon/`, or `apple/`. The only files added on +top of the three merged lines are the new opt-in E2E test/scripts +(`daemon/internal/realtime/client/e2e_test.go`, +`scripts/e2e/source-drive.mjs`, `scripts/e2e/viewer-smoke.mjs`, +`scripts/e2e/package.json`) and this document. + +## 5. Defects requiring a fix on the owning line + +None found. + +## 6. Residual risks + +- The post-hello-accept `command` push to `source` connections + (`server/src/realtime/space-hub.ts:476-488`) is documented in code as a + "v1.1 protocol clarification candidate" but is not yet reflected in + `proto/realtime/messages/hello.schema.json`'s prose. Both clients handle + it safely today; a future client implementation that assumes "nothing + arrives before my first subscribe/resume" would not be safe. Worth an + explicit spec note in a future protocol revision (out of scope here since + `proto/realtime/` is frozen for this integration pass). +- The E2E smoke test only exercises one source + two viewers in one space, + for a few seconds; it does not cover the interest-lease-expiry, duplicate- + connection-supersede, or duplicate-device-seq scenarios end-to-end over a + real network (those are covered at the fixture/unit level in all three + lines, and in the server's own vitest-pool-workers suite, but not by a + live cross-process run in this pass). +- `wrangler dev --local` state is ephemeral (fresh DO storage per run); the + space/device tokens embedded as examples above are from a throwaway local + run and are not valid against any deployed environment. diff --git a/docs/design/realtime-server.md b/docs/design/realtime-server.md new file mode 100644 index 00000000..1523ec1e --- /dev/null +++ b/docs/design/realtime-server.md @@ -0,0 +1,204 @@ +# Realtime server: SpaceHub cost model + +This document is the server-side cost/capacity companion to +`proto/realtime/SPEC.md` section 14. It maps every message type onto the +concrete SQLite writes, `space_revision` accounting, and broadcast fan-out +`server/src/realtime/space-hub.ts` performs, and then analyzes that from a +Durable Object billing perspective (requests, duration, storage). It also +records exactly when this Durable Object can hibernate. + +## Per-message-type cost table + +Columns mirror SPEC.md section 14 plus a `SQL writes` column naming the +actual tables touched, since "persisted" alone doesn't say how many +statements or which tables. + +| `type` | SQL writes | revision++ | outbox | broadcast | +|---|---|---|---|---| +| `hello` | none (attachment only, via `serializeAttachment`) | no | no | no | +| `resume` | none (reads `tasks`/`messages`/`automations`/`event_log`) | no | no | no (unicast reply, possibly chunked) | +| `snapshot` | none | no | no | no | +| `delta` | none (built from `event_log` rows, or from the just-applied event in memory) | no | no | yes, live single-event deltas only; catch-up deltas are unicast | +| `ack` | none | no | no | no | +| `task.event` | `event_log` insert, `dedup` insert, `tasks` upsert, `space_meta` update, `push_outbox` insert (only on the non-duplicate path) | yes (skipped on duplicate) | yes | re-emitted as `delta` to every delta-eligible, leased viewer | +| `message.event` | `event_log` insert, `dedup` insert, `messages` insert + bounded-window prune, `space_meta` update, `push_outbox` insert (non-duplicate path only) | yes (skipped on duplicate) | yes | re-emitted as `delta` | +| `metric.frame` | **none** — no table is ever written for this type | **no** | no | yes, filtered to leases whose `topics` include `metric` (or omit `topics`) | +| `config.event` | `event_log` insert, `automations` upsert/delete, `space_meta` update, `http_idempotency` insert (only when an `Idempotency-Key` was supplied) | yes | no | re-emitted as `delta` | +| `subscribe` | `leases` upsert | no | no | no (its ack is unicast; may trigger a `command` broadcast, see below) | +| `unsubscribe` | `leases` delete | no | no | no (same caveat) | +| `interest.renew` | `leases` upsert | no | no | no (same caveat) | +| `command` (viewer-issued) | `pending_commands` insert, only if no connected source could take live delivery | no | no (viewer command) / yes (targeted at one or all sources) | no (targeted, not fanned out to viewers) | +| `command` (server-issued: `throttle`/`resume_rate`) | none | no | no | yes, to every connected source, on the space's lease-count 1↔0 edge only | +| `error` | none | no | no | no | + +Two things worth calling out because they are easy to get wrong and are +covered by dedicated tests (`server/test/workers/broadcast.workers.ts`): + +- **`metric.frame` never touches SQLite and never advances `space_revision`.** + It is a pure in-memory relay (`SpaceHub#metricsCache`, a `Map`) plus a + websocket fan-out. This is the single biggest cost lever in the whole + design: a source can push metrics at up to 10 Hz per connection + (SPEC.md section 11) with zero storage cost, at the price of the + `snapshot.metrics` cache being best-effort and empty after eviction + (acceptable per spec, section 6.2). +- **Duplicate `device_seq` costs one read, not one write.** The dedup check + (`SELECT revision FROM dedup WHERE device_id = ? AND device_seq = ?`) is + the *only* SQL statement executed on the retry path; the ack the source + needs is still sent. This bounds the cost of a source's at-least-once + resend behavior (SPEC.md section 5.3) to a single indexed read per retry. + +## Durable Object billing perspective + +Cloudflare bills a Durable Object roughly on three axes: **requests** +(wall-clock-billed invocations — HTTP `fetch()` calls and, importantly, +each Hibernatable WebSocket message callback), **duration** (active CPU/wall +time per invocation), and **storage** (row/byte volume plus read/write unit +counts against the attached SQLite database). + +- **Requests.** Every inbound WebSocket frame is one `webSocketMessage` + invocation — this is unavoidable per-message overhead independent of the + handler's cost, which is why the Worker-side gate (rejecting invalid + tokens with a plain 401, `adapters/workers.ts`'s `/v3/realtime` route) + matters: an invalid token or a reconnect storm must never turn into + SpaceHub invocations at all, let alone SpaceHub *instances* (verified by + `server/test/workers/token-gate.workers.ts`). The one exception to "one + request per frame" is the literal `ping`/`pong` heartbeat text, which + `ctx.setWebSocketAutoResponse()` answers entirely inside the runtime + without invoking `webSocketMessage` at all (SPEC.md section 9.3) — this + is a genuine request-count win for a connection that's alive but idle. +- **Duration.** The dominant cost per request is proportional to the + number of `sql.exec` calls a handler issues, not to network I/O (there is + none inside a handler — `ws.send()` is fire-and-forget and never + `await`ed). `task.event`/`message.event` cost roughly 5–6 statements; + `metric.frame` costs zero; `resume` costs a handful of `SELECT`s plus, + for a multi-chunk snapshot, one `ws.send()` per chunk (still no `await`, + so duration scales with payload size, not round trips). Snapshot chunking + (`chunkSnapshot`/`chunkDeltaEvents` in `chunking.ts`) exists specifically + to keep any single `sql.exec`+`JSON.stringify`+`send` unit bounded, not to + reduce request count — one `resume` is still one request no matter how + many chunks it produces, since all of them are sent from inside the same + `webSocketMessage` invocation. +- **Storage.** Two tables grow without bound if left unchecked: + `event_log` (pruned to the last `EVENT_LOG_RETENTION_REVISIONS` = 1000 + revisions — older ones force a `resume` onto the `snapshot` path instead + of erroring, SPEC.md section 6.2) and `messages` (pruned to the last + `MESSAGE_WINDOW` = 200, SPEC.md section 6.4's normative truncation). + `dedup` is intentionally **not** pruned: SPEC.md section 5.2's + deduplication guarantee must hold regardless of how old a retried + `device_seq` is, so it trades unbounded (but tiny — one row per + historical event, `device_id`+`device_seq`+`revision`, all fixed-width) + storage growth for correctness. `http_idempotency` is bounded by lazy + cleanup on its own write path (no alarm): every insert also deletes + entries older than 24 hours and caps the table at the 500 most recent + rows — a control-plane retry arriving after both windows simply + re-executes as a fresh request, which is state-level safe because + automation upsert/delete are idempotent operations even without the + key. `push_outbox` and `pending_commands` are the two "queue" tables in + this design; both need a consumer (a future Alarm/Queue-based push + worker, explicitly out of scope for this phase) to actually shrink — + today they only grow, which is an accepted, called-out gap (see the + handoff's "known gaps" section). + +## Observability config + +`wrangler.jsonc`'s `observability` block is the other half of the cost +model above — `logSampler`'s ≤1% in-code sampling only bounds what +SpaceHub *chooses* to write to `console.log` (`logHotPath`); it does +nothing about the Workers platform's own per-invocation telemetry, which +fires once per `webSocketMessage` call regardless of what the handler logs. +Final configuration: + +```jsonc +"observability": { + "enabled": true, + "logs": { "enabled": true, "head_sampling_rate": 1, "invocation_logs": false }, + "traces": { "enabled": true, "head_sampling_rate": 0.01 } +} +``` + +- **`logs.head_sampling_rate: 1` (i.e. no head sampling on structured + logs).** This is the deliberate choice, not an oversight: `logAlways()` + (superseded connections, protocol errors, unhandled exceptions — the + security/error tier) is a correctness requirement that must reach 100% + of the time, and Workers Logs head sampling is applied indiscriminately + before the Worker code runs, so any rate below 1.0 here would silently + drop `logAlways` events alongside the hot-path ones. Volume is bounded + instead at the *source*, in-code, by `logSampler` (`≤1%` of + `logHotPath` calls) — a rate this repository controls and tests directly + (`space-hub.ts`'s injectable `logSampler`), rather than a platform knob + that can't distinguish log tiers. +- **`logs.invocation_logs: false`.** This is what actually bounds request + volume: Workers' automatic invocation log is emitted once per request + (once per WebSocket message callback here) independent of anything the + Worker code logs, and cannot be tier-aware the way `logSampler` is. + Disabling it removes the one telemetry source neither `logAlways` nor + `logSampler` was ever designed to control. +- **`traces.head_sampling_rate: 0.01`.** Traces have no `logAlways` + equivalent (nothing security-relevant depends on a trace existing), so + they're sampled at the platform level like any other diagnostic-only + signal. + +Revised cost estimate: per-space WebSocket message volume is bounded by +the protocol itself (10 `metric.frame`/s per connection, SPEC.md section +11, plus whatever rate reliable events arrive at) — call it on the order +of a few thousand `webSocketMessage` invocations/day for an actively-used +space. With `invocation_logs: false`, none of those turn into a platform +invocation-log write; with `logSampler` at ≤1%, `logHotPath` contributes +at most ~1 structured log line per 100 frames; `logAlways` contributes a +handful of lines per space (supersession, occasional protocol errors) — +negligible even at 100% sampling. Net: structured log volume scales with +*intentional* logging, not raw message count, which is what makes the +`head_sampling_rate: 1` choice on `logs` affordable. + +## Hibernation + +SpaceHub is written so that an idle connection costs nothing beyond the +in-memory footprint the runtime itself keeps for a parked WebSocket: + +- Every handler uses `ws.deserializeAttachment()` fresh, every time, + instead of any DO-instance-level `Map` for identity — + this is what makes it safe for the DO's JS instance to be evicted and + reconstructed between messages. `ConnAttachment` (`attachment.ts`) is + deliberately small (five scalars plus an optional `sessionId`) to stay + far under the platform's 2 KiB serialized-attachment cap. +- The only volatile, non-attachment, non-SQLite in-memory state is + `metricsCache` (a best-effort cache the spec explicitly allows to reset + to empty after a restart, section 6.2) and the per-connection rate + limiter state (`rateLimiters`, a `WeakMap`, similarly allowed to reset — + worst case a freshly-evicted-and-restarted connection gets one extra + `metric.frame` or two before the rate limit re-establishes itself). + Neither is required for correctness, only for cost control, so losing + them on eviction is safe by design, not just by accident. +- Nothing in this Durable Object ever calls `setInterval`, `setTimeout`, or + `ctx.storage.setAlarm()`. The interest lease (SPEC.md section 7) + explicitly permits lazy expiry evaluation, which `reconcileLeaseEdge()` + performs by re-checking `COUNT(*) FROM leases WHERE expires_at > ?` every + time a lease is mutated or a reliable event is applied — never on a + timer. This means a space with no active connections and no pending + automation changes has literally nothing scheduled and can hibernate (or + be evicted entirely) indefinitely. +- The heartbeat auto-response (`ctx.setWebSocketAutoResponse`) is itself a + hibernation-compatible primitive: it answers `ping` without waking the DO + instance at all, which is precisely why SPEC.md section 9.3 specifies a + bare text frame rather than a JSON envelope in the first place. + +Net effect: a space's ongoing cost while every connection is idle (no +task/message/metric traffic, no lease churn) is exactly zero DO +invocations — the state sits in SQLite and the connections sit hibernated +until either side has something to say. + +## Intentional omissions + +- **The 10 s handshake timeout (SPEC.md section 9.1, a SHOULD) is not + implemented.** Enforcing it server-side would require a per-connection + timer — exactly the `setTimeout`/alarm machinery this design excludes to + stay hibernation-friendly. The exposure is bounded: a connection that + never sends an offer holds no lease, receives no deltas, writes nothing, + and costs only the runtime's parked-socket overhead; the client side of + the recommendation (close and reconnect if no accept arrives within + 10 s) is where the liveness benefit actually lives, and belongs to the + client implementations. +- **`sequence_gap` (SPEC.md section 5.1, a MAY) is not emitted.** The + event that did arrive is applied either way; the advisory adds a + per-event bookkeeping read ("what was this device's previous seq") + purely to produce a non-actionable warning. diff --git a/scripts/e2e/package-lock.json b/scripts/e2e/package-lock.json new file mode 100644 index 00000000..af0f755f --- /dev/null +++ b/scripts/e2e/package-lock.json @@ -0,0 +1,36 @@ +{ + "name": "sitrep-realtime-e2e-scripts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "sitrep-realtime-e2e-scripts", + "version": "1.0.0", + "dependencies": { + "ws": "^8.18.0" + } + }, + "node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/scripts/e2e/package.json b/scripts/e2e/package.json new file mode 100644 index 00000000..dd4e085c --- /dev/null +++ b/scripts/e2e/package.json @@ -0,0 +1,10 @@ +{ + "name": "sitrep-realtime-e2e-scripts", + "version": "1.0.0", + "private": true, + "description": "Manual/CI-optional cross-line E2E smoke scripts for the realtime protocol viewer role, driven against a real locally running server (wrangler dev). See docs/design/realtime-integration.md for usage. Not part of any package's build or test graph.", + "type": "module", + "dependencies": { + "ws": "^8.18.0" + } +} diff --git a/scripts/e2e/source-drive.mjs b/scripts/e2e/source-drive.mjs new file mode 100644 index 00000000..6a9afb29 --- /dev/null +++ b/scripts/e2e/source-drive.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +// Companion driver for viewer-smoke.mjs: connects as a "source" device and, +// after a short delay (so the viewer script has time to subscribe/resume +// first), emits one task.event (to produce a live delta) and one +// metric.frame (to produce a metric.frame broadcast), then disconnects. +// +// This is a protocol-generic driver used only to exercise the server's +// viewer-side fan-out in scripts/e2e/viewer-smoke.mjs; the daemon (Go) +// source role itself is validated independently and more thoroughly by +// daemon/internal/realtime/client/e2e_test.go (TestE2ESourceLifecycle), +// which drives the real daemon client package against the same server. +// +// Required env: SITREP_E2E_URL, SITREP_E2E_SOURCE_TOKEN, SITREP_E2E_SOURCE_DEVICE_ID. + +import WebSocket from "ws"; + +const URL = requireEnv("SITREP_E2E_URL"); +const TOKEN = requireEnv("SITREP_E2E_SOURCE_TOKEN"); +const DEVICE_ID = requireEnv("SITREP_E2E_SOURCE_DEVICE_ID"); +const DELAY_MS = Number(process.env.SITREP_E2E_SOURCE_DELAY_MS || 4000); + +function requireEnv(name) { + const v = process.env[name]; + if (!v) { + console.error(`missing required env var ${name}`); + process.exit(2); + } + return v; +} + +function log(msg, extra) { + console.log(extra !== undefined ? `[source] ${msg} ${JSON.stringify(extra)}` : `[source] ${msg}`); +} + +let seq = 0; +function envelope(type, body) { + // envelope_id must match ^[A-Za-z0-9_-]{1,64}$ (proto/realtime/common.schema.json) + // so the message `type` (which may contain a literal "." e.g. task.event, + // metric.frame) must NOT be embedded verbatim. + const safeType = type.replace(/[^A-Za-z0-9]/g, ""); + return { type, id: `e2e-src-${safeType}-${++seq}-${Date.now()}`, ts: Date.now(), body }; +} + +const ws = new WebSocket(URL, { headers: { Authorization: `Bearer ${TOKEN}` } }); +let deviceSeq = 1; + +ws.on("open", () => { + const offer = envelope("hello", { + stage: "offer", + device_id: DEVICE_ID, + role: "source", + protocol_versions: [1], + }); + log("send hello offer"); + ws.send(JSON.stringify(offer)); +}); + +ws.on("message", async (data) => { + const text = data.toString(); + if (text === "ping") { + ws.send("pong"); + return; + } + if (text === "pong") return; + const env = JSON.parse(text); + log(`recv ${env.type}`, env.type === "hello" ? env.body : env.type === "ack" ? env.body : undefined); + + if (env.type === "hello" && env.body.stage === "accept") { + log(`hello accepted, waiting ${DELAY_MS}ms before driving events (letting viewers subscribe first)`); + await new Promise((r) => setTimeout(r, DELAY_MS)); + + const taskEvent = envelope("task.event", { + device_id: DEVICE_ID, + device_seq: deviceSeq++, + task_id: "e2e-viewer-drive-task", + kind: "started", + occurred_at: Date.now(), + title: "e2e viewer-smoke drive", + }); + log("send task.event (should trigger a live delta to subscribed viewers)"); + ws.send(JSON.stringify(taskEvent)); + + await new Promise((r) => setTimeout(r, 500)); + + const metricFrame = envelope("metric.frame", { + device_id: DEVICE_ID, + metrics: [{ metric_id: "e2e.cpu.load", value: "0.42", ts: Date.now() }], + }); + log("send metric.frame (should broadcast to metric-subscribed viewers)"); + ws.send(JSON.stringify(metricFrame)); + + await new Promise((r) => setTimeout(r, 1000)); + ws.close(1000, "e2e source drive done"); + log("done, closing"); + } + + if (env.type === "error") { + console.error("[source] server error:", env.body); + } +}); + +ws.on("error", (e) => { + console.error("[source] ws error", e); + process.exit(1); +}); diff --git a/scripts/e2e/viewer-smoke.mjs b/scripts/e2e/viewer-smoke.mjs new file mode 100644 index 00000000..b4edf35c --- /dev/null +++ b/scripts/e2e/viewer-smoke.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node +// Cross-line realtime protocol E2E smoke test: viewer role, driven against a +// real server (see docs/design/realtime-integration.md for how to start one +// locally with `wrangler dev`). Exercises, over two real WebSocket +// connections (two "viewer" devices), everything the apple/SitrepKit +// RealtimeClient does on the wire, using a plain `ws` client so this script +// has no dependency on the Swift/Go implementations: +// +// viewer1: hello{offer} -> hello{accept} -> subscribe -> ack{lease} +// -> resume{last_revision:0} -> snapshot (verifies fresh-viewer +// snapshot path) -> live delta once the source emits an event +// (verifies revision chaining: snapshot.revision -> delta. +// from_revision == snapshot.revision, to_revision == from+1) +// viewer2: same hello/subscribe/resume sequence, then receives the +// metric.frame broadcast triggered by the source (verifies +// server fan-out to a second, independently-subscribed viewer) +// +// All credentials are read from the environment — nothing is hardcoded. +// Required: SITREP_E2E_URL, SITREP_E2E_VIEWER1_TOKEN, SITREP_E2E_VIEWER2_TOKEN. +// The source-side event that drives the live delta/metric.frame is sent by +// a companion process (see docs/design/realtime-integration.md); this +// script only waits for it, with a generous timeout. + +import WebSocket from "ws"; + +const URL = requireEnv("SITREP_E2E_URL"); +const VIEWER1_TOKEN = requireEnv("SITREP_E2E_VIEWER1_TOKEN"); +const VIEWER2_TOKEN = requireEnv("SITREP_E2E_VIEWER2_TOKEN"); +const WAIT_FOR_SOURCE_EVENT_MS = Number(process.env.SITREP_E2E_WAIT_MS || 20000); + +function requireEnv(name) { + const v = process.env[name]; + if (!v) { + console.error(`missing required env var ${name}`); + process.exit(2); + } + return v; +} + +function log(tag, msg, extra) { + const line = `[${tag}] ${msg}`; + console.log(extra !== undefined ? `${line} ${JSON.stringify(extra)}` : line); +} + +let envSeq = 0; +function envelope(type, body) { + // envelope_id must match ^[A-Za-z0-9_-]{1,64}$ (proto/realtime/common.schema.json); + // strip any "." from the message type (e.g. metric.frame) before embedding it. + const safeType = type.replace(/[^A-Za-z0-9]/g, ""); + return { type, id: `e2e-${safeType}-${++envSeq}-${Date.now()}`, ts: Date.now(), body }; +} + +// Connects as a viewer, completes hello -> subscribe -> resume{0}, and +// returns { ws, send, waitFor(predicate) } so the caller can drive the rest +// of the scenario. waitFor resolves the first received frame matching +// predicate, queuing every other frame for subsequent waitFor calls. +function connectViewer(tag, token) { + return new Promise((resolve, reject) => { + const ws = new WebSocket(URL, { headers: { Authorization: `Bearer ${token}` } }); + const queue = []; + const waiters = []; + + ws.on("message", (data) => { + const text = data.toString(); + if (text === "ping") { + ws.send("pong"); + log(tag, "recv ping -> sent pong"); + return; + } + if (text === "pong") return; + const env = JSON.parse(text); + log(tag, `recv ${env.type}`, summarize(env)); + const idx = waiters.findIndex((w) => w.predicate(env)); + if (idx >= 0) { + const [w] = waiters.splice(idx, 1); + w.resolve(env); + } else { + queue.push(env); + } + }); + ws.on("error", reject); + + const send = (type, body) => { + const env = envelope(type, body); + log(tag, `send ${type}`, summarize(env)); + ws.send(JSON.stringify(env)); + return env; + }; + + const waitFor = (predicate, timeoutMs = 10000) => + new Promise((res, rej) => { + const qi = queue.findIndex(predicate); + if (qi >= 0) { + res(queue.splice(qi, 1)[0]); + return; + } + const timer = setTimeout(() => { + const i = waiters.findIndex((w) => w.resolve === wrappedRes); + if (i >= 0) waiters.splice(i, 1); + rej(new Error(`${tag}: timed out waiting for a matching frame`)); + }, timeoutMs); + const wrappedRes = (env) => { + clearTimeout(timer); + res(env); + }; + waiters.push({ predicate, resolve: wrappedRes }); + }); + + ws.on("open", async () => { + try { + send("hello", { + stage: "offer", + device_id: `${tag}-e2e`, + role: "viewer", + protocol_versions: [1], + }); + const accept = await waitFor((e) => e.type === "hello"); + resolve({ ws, send, waitFor, sessionId: accept.body.session_id }); + } catch (e) { + reject(e); + } + }); + }); +} + +function summarize(env) { + switch (env.type) { + case "hello": + return env.body.stage === "accept" + ? { stage: "accept", protocol_version: env.body.protocol_version, session_id: env.body.session_id } + : { stage: "offer", device_id: env.body.device_id, role: env.body.role }; + case "ack": + return { in_reply_to: env.body.in_reply_to, lease: env.body.lease }; + case "snapshot": + return { revision: env.body.revision, part: env.body.part, final: env.body.final, tasks: env.body.tasks?.length, metrics: env.body.metrics?.length }; + case "delta": + return { from_revision: env.body.from_revision, to_revision: env.body.to_revision, events: env.body.events?.length }; + case "metric.frame": + return { device_id: env.body.device_id, metrics: env.body.metrics }; + case "error": + return { code: env.body.code, message: env.body.message, retryable: env.body.retryable, fatal: env.body.fatal }; + default: + return env.body; + } +} + +async function main() { + log("main", `connecting two viewers to ${URL}`); + + const v1 = await connectViewer("viewer1", VIEWER1_TOKEN); + const subAck1 = v1.send("subscribe", { topics: ["task", "metric", "message"] }); + const ack1 = await v1.waitFor((e) => e.type === "ack" && e.body.in_reply_to === subAck1.id); + log("viewer1", "subscribed, lease", ack1.body.lease); + + v1.send("resume", { last_revision: 0 }); + const snapshot = await v1.waitFor((e) => e.type === "snapshot"); + log("viewer1", `fresh-viewer resume(0) correctly produced a snapshot at revision ${snapshot.body.revision}`); + + const v2 = await connectViewer("viewer2", VIEWER2_TOKEN); + const subAck2 = v2.send("subscribe", { topics: ["task", "metric", "message"] }); + const ack2 = await v2.waitFor((e) => e.type === "ack" && e.body.in_reply_to === subAck2.id); + log("viewer2", "subscribed, lease", ack2.body.lease); + v2.send("resume", { last_revision: snapshot.body.revision }); + // viewer2 resumes at the current revision: server should send an + // empty/no-op catch-up or nothing until the next real event — we don't + // block on it, we only need viewer2 subscribed + delta-eligible before + // the source emits its next event. + log("viewer2", "resumed at current revision, now delta-eligible"); + + log("main", `waiting up to ${WAIT_FOR_SOURCE_EVENT_MS}ms for the source's next task.event to arrive as a live delta on viewer1...`); + const delta = await v1.waitFor((e) => e.type === "delta", WAIT_FOR_SOURCE_EVENT_MS); + if (delta.body.from_revision !== snapshot.body.revision) { + throw new Error( + `revision chain broken: snapshot.revision=${snapshot.body.revision} but delta.from_revision=${delta.body.from_revision}`, + ); + } + if (delta.body.to_revision !== delta.body.from_revision + 1 && delta.body.to_revision <= delta.body.from_revision) { + throw new Error(`delta.to_revision (${delta.body.to_revision}) did not advance past from_revision (${delta.body.from_revision})`); + } + log( + "main", + `PASS: live delta chained correctly (snapshot rev ${snapshot.body.revision} -> delta ${delta.body.from_revision}->${delta.body.to_revision})`, + ); + + log("main", `waiting up to ${WAIT_FOR_SOURCE_EVENT_MS}ms for the source's metric.frame broadcast on viewer2...`); + const metricFrame = await v2.waitFor((e) => e.type === "metric.frame", WAIT_FOR_SOURCE_EVENT_MS); + log("main", "PASS: viewer2 received the metric.frame broadcast triggered by the source", summarize(metricFrame)); + + v1.ws.close(1000, "e2e done"); + v2.ws.close(1000, "e2e done"); + log("main", "ALL VIEWER E2E CHECKS PASSED"); + process.exit(0); +} + +main().catch((err) => { + console.error("E2E FAILED:", err); + process.exit(1); +}); diff --git a/server/package-lock.json b/server/package-lock.json index 84e9e1d0..b56a2571 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -11,9 +11,11 @@ "hono": "^4.12.30" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.18.6", "@hono/node-server": "^1.19.0", "@types/node": "^26.1.1", "typescript": "^5.9.3", + "vitest": "^4.1.0", "wrangler": "^4.112.0" } }, @@ -43,6 +45,25 @@ } } }, + "node_modules/@cloudflare/vitest-pool-workers": { + "version": "0.18.6", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-pool-workers/-/vitest-pool-workers-0.18.6.tgz", + "integrity": "sha512-6JGqaQsQRZIVq/6jEC4ouJnShZriPIJ2X0yGndwMm+SiPP93pJi5Dp30dYAoztNrNJC7wWK7ec5slLfMBMZ8jA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cjs-module-lexer": "1.2.3", + "esbuild": "0.28.1", + "miniflare": "4.20260714.0", + "wrangler": "4.112.0", + "zod": "3.25.76" + }, + "peerDependencies": { + "@vitest/runner": "^4.1.0", + "@vitest/snapshot": "^4.1.0", + "vitest": "^4.1.0" + } + }, "node_modules/@cloudflare/workerd-darwin-64": { "version": "1.20260714.1", "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260714.1.tgz", @@ -141,10 +162,10 @@ "node": ">=12" } }, - "node_modules/@emnapi/runtime": { - "version": "1.11.2", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", - "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -705,9 +726,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -725,9 +743,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -745,9 +760,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -765,9 +777,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -785,9 +794,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -805,9 +811,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -825,9 +828,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -845,9 +845,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -865,9 +862,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -891,9 +885,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -917,9 +908,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -943,9 +931,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -969,9 +954,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -995,9 +977,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1021,9 +1000,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1047,9 +1023,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1173,6 +1146,35 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -1202,177 +1204,995 @@ "dev": true, "license": "MIT" }, - "node_modules/@sindresorhus/is": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", - "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "android" + ], "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@speed-highlight/core": { - "version": "1.2.17", - "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.17.tgz", - "integrity": "sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "CC0-1.0" + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@types/node": { - "version": "26.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", - "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "cpu": [ + "x64" + ], "dev": true, "license": "MIT", - "dependencies": { - "undici-types": "~8.3.0" + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "cpu": [ + "arm" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "Apache-2.0", + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=8" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/error-stack-parser-es": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", - "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/antfu" + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/esbuild": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", - "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "cpu": [ + "ppc64" + ], "dev": true, - "hasInstallScript": true, "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.1", - "@esbuild/android-arm": "0.28.1", - "@esbuild/android-arm64": "0.28.1", - "@esbuild/android-x64": "0.28.1", - "@esbuild/darwin-arm64": "0.28.1", - "@esbuild/darwin-x64": "0.28.1", - "@esbuild/freebsd-arm64": "0.28.1", - "@esbuild/freebsd-x64": "0.28.1", - "@esbuild/linux-arm": "0.28.1", - "@esbuild/linux-arm64": "0.28.1", - "@esbuild/linux-ia32": "0.28.1", - "@esbuild/linux-loong64": "0.28.1", - "@esbuild/linux-mips64el": "0.28.1", - "@esbuild/linux-ppc64": "0.28.1", - "@esbuild/linux-riscv64": "0.28.1", - "@esbuild/linux-s390x": "0.28.1", - "@esbuild/linux-x64": "0.28.1", - "@esbuild/netbsd-arm64": "0.28.1", - "@esbuild/netbsd-x64": "0.28.1", - "@esbuild/openbsd-arm64": "0.28.1", - "@esbuild/openbsd-x64": "0.28.1", - "@esbuild/openharmony-arm64": "0.28.1", - "@esbuild/sunos-x64": "0.28.1", - "@esbuild/win32-arm64": "0.28.1", - "@esbuild/win32-ia32": "0.28.1", - "@esbuild/win32-x64": "0.28.1" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "cpu": [ + "s390x" + ], "dev": true, - "hasInstallScript": true, "license": "MIT", "optional": true, "os": [ - "darwin" + "linux" ], "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/hono": { - "version": "4.12.30", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", - "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "cpu": [ + "x64" + ], + "dev": true, "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=16.9.0" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "cpu": [ + "x64" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">=6" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/miniflare": { - "version": "4.20260714.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260714.0.tgz", - "integrity": "sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", + "cpu": [ + "wasm32" + ], "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "0.34.5", - "undici": "7.28.0", - "workerd": "1.20260714.1", - "ws": "8.21.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" }, "engines": { - "node": ">=22.0.0" + "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.17", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.17.tgz", + "integrity": "sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", + "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", + "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", + "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.10", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", + "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", + "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vitest/utils": "4.1.10", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", + "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "@vitest/utils": "4.1.10", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", + "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", + "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "dev": true, + "license": "MIT" + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz", + "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/hono": { + "version": "4.12.30", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", + "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/miniflare": { + "version": "4.20260714.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260714.0.tgz", + "integrity": "sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.34.5", + "undici": "7.28.0", + "workerd": "1.20260714.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", "dev": true, "license": "MIT" @@ -1384,6 +2204,90 @@ "dev": true, "license": "MIT" }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", + "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.139.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" + } + }, "node_modules/semver": { "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", @@ -1442,6 +2346,37 @@ "@img/sharp-win32-x64": "0.34.5" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, "node_modules/supports-color": { "version": "10.2.2", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", @@ -1455,6 +2390,50 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1500,10 +2479,198 @@ "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "pathe": "^2.0.3" } }, + "node_modules/vite": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", + "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", + "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vitest/expect": "4.1.10", + "@vitest/mocker": "4.1.10", + "@vitest/pretty-format": "4.1.10", + "@vitest/runner": "4.1.10", + "@vitest/snapshot": "4.1.10", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.10", + "@vitest/browser-preview": "4.1.10", + "@vitest/browser-webdriverio": "4.1.10", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/ui": "4.1.10", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/workerd": { "version": "1.20260714.1", "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260714.1.tgz", @@ -1511,6 +2678,7 @@ "dev": true, "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "bin": { "workerd": "bin/workerd" }, @@ -1607,6 +2775,16 @@ "@poppinss/exception": "^1.2.2", "error-stack-parser-es": "^1.0.5" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/server/package.json b/server/package.json index 5e242fe3..2dcd3c25 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,9 @@ "scripts": { "dev": "wrangler dev", "dev:node": "node --experimental-strip-types src/adapters/node.ts", - "test": "node --experimental-strip-types --test test/**/*.test.ts", + "test:unit": "node --experimental-strip-types --test \"test/**/*.test.ts\"", + "test:workers": "vitest run", + "test": "npm run test:unit && npm run test:workers", "typecheck": "tsc --noEmit", "deploy": "wrangler deploy" }, @@ -14,9 +16,11 @@ "hono": "^4.12.30" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.18.6", "@hono/node-server": "^1.19.0", "@types/node": "^26.1.1", "typescript": "^5.9.3", + "vitest": "^4.1.0", "wrangler": "^4.112.0" } } diff --git a/server/src/adapters/workers.ts b/server/src/adapters/workers.ts index 4aa8e39b..8a070968 100644 --- a/server/src/adapters/workers.ts +++ b/server/src/adapters/workers.ts @@ -3,8 +3,15 @@ // token in the AUTH_TOKEN secret. Live Activity pushes fire from inside the // DO after each state change (via waitUntil, off the ingest critical path). import { DurableObject } from "cloudflare:workers"; -import { createApp, newToken, TOKEN_RE, type Command, type DeviceInfo, type Role, type SpaceRegistry } from "../app.ts"; +import { createApp, newToken, parseRealtimeEnabledFlag, TOKEN_RE, type Command, type DeviceInfo, type Role, type SpaceRegistry } from "../app.ts"; import { endActivity, sendAlert, startActivity, updateActivity, type ApnsConfig } from "../apns.ts"; +import { SpaceHub } from "../realtime/space-hub.ts"; +import type { AutomationExecutorKind, AutomationState } from "../realtime/types.ts"; + +// Re-exported so wrangler (which binds Durable Object classes by name from +// this entry module, see wrangler.jsonc) can resolve SpaceHub, same as +// UserStore below. +export { SpaceHub }; import { appendTaskLog, EVENT_LOG_CAP, @@ -37,6 +44,11 @@ interface Secrets { APNS_TEAM_ID?: string; APNS_BUNDLE_ID?: string; // var, wrangler.jsonc APNS_HOST?: string; // var, wrangler.jsonc + // var, wrangler.jsonc; unset/false disables /v3/realtime. Typed as string | + // boolean because a Cloudflare dashboard variable override always arrives + // as a string at runtime, even though wrangler.jsonc declares it boolean — + // see parseRealtimeEnabledFlag in app.ts for the strict parsing this needs. + REALTIME_ENABLED?: string | boolean; } type WorkerEnv = Env & Secrets; @@ -476,6 +488,7 @@ const app = createApp({ await (c.env as WorkerEnv).INVITE_DIR.put(code, space, { expirationTtl: 600 }); }, lookupInvite: (c, code) => (c.env as WorkerEnv).INVITE_DIR.get(code), + realtimeEnabled: (c) => parseRealtimeEnabledFlag((c.env as WorkerEnv).REALTIME_ENABLED), }); app.get("/debug/tokens", async (c: any) => { @@ -487,4 +500,192 @@ app.get("/debug/tokens", async (c: any) => { return c.json(await stub(env, "default").debugTokens(c.req.query("full") === "1")); }); +// ---- /v3: realtime WebSocket + its HTTP control-plane companion ---- +// +// Both routes ride the SAME `/v3/*` `authenticate` middleware registered in +// app.ts (st2 token parsing, unchanged from /v2) so an invalid or +// unresolvable token gets its 401 from that shared, already-tested path — +// this handler runs at all ONLY once a token has resolved to a real +// device/role, and it is the only place SpaceHub is ever touched. + +function spaceHubStub(env: WorkerEnv, space: string) { + return env.SPACE_HUB.getByName(space); +} + +app.get("/v3/realtime", async (c: any) => { + const env = c.env as WorkerEnv; + const raw = c.req.raw as Request; + if ((raw.headers.get("upgrade") ?? "").toLowerCase() !== "websocket") { + return c.text("expected websocket upgrade", 426); + } + // authenticate() already ran (see app.use("/v3/*", authenticate) in + // app.ts): an invalid/unresolvable token never reaches this line, and no + // Durable Object of any kind is touched for it beyond the UserStore + // lookup authenticate() itself performs — SpaceHub is instantiated only + // by the single stub.fetch() call below, on the success path. + const deviceId: string | undefined = c.get("deviceId"); + const role = c.get("role") as Role; + if (!deviceId) { + // The legacy bare-admin token (self-host, no registry) resolves to a + // role but no device identity; the realtime protocol requires one. + return c.json({ error: "realtime requires a paired device token" }, 401); + } + // Flag check runs AFTER authenticate() (so an unauthenticated caller still + // gets its normal 401, not a 403 that would leak flag state) and BEFORE + // the spaceHubStub().fetch() below — a disabled deployment must never wake + // a SpaceHub DO. No separate "kill existing connections" path is needed: + // flag changes ship via redeploy, which evicts every DO instance and drops + // all of its live WebSockets as a side effect. + if (!parseRealtimeEnabledFlag(env.REALTIME_ENABLED)) { + return c.json({ error: "realtime_disabled" }, 403); + } + const realtimeRole: "source" | "viewer" = role === "source" ? "source" : "viewer"; + + const headers = new Headers(raw.headers); + headers.set("x-sitrep-device-id", deviceId); + headers.set("x-sitrep-role", realtimeRole); + const forwardReq = new Request(raw.url, { method: raw.method, headers }); + + // Exactly one forwarded operation per inbound upgrade request: the DO's + // own fetch() creates the WebSocketPair and calls ctx.acceptWebSocket + // once (see SpaceHub#fetch). + return spaceHubStub(env, c.get("space")).fetch(forwardReq); +}); + +function parseAutomationUpsert(body: any): { name: string; executor_kind: AutomationExecutorKind; every_seconds: number } | null { + const kind = body?.executor_kind; + const everySeconds = Number(body?.schedule?.every_seconds); + if (!body?.name || !["script", "agent", "hybrid"].includes(kind) || !Number.isFinite(everySeconds)) return null; + return { name: String(body.name).slice(0, 256), executor_kind: kind, every_seconds: Math.max(1, everySeconds | 0) }; +} + +// /v3/automations role matrix (protocol-owner ruling, aligned with the +// existing product decision "watcher schedules are editable from every +// device; creation only from trusted devices" — same split /v2 makes +// between canEmit-guarded POST and canView-guarded PATCH/DELETE): +// POST (create/upsert) -> owner/admin only +// PATCH (pause/resume, schedule.every_seconds) -> owner/admin/viewer +// (viewer editing the schedule is INTENTIONAL, not an oversight) +// DELETE -> owner/admin/viewer (aligned with /v2 canView's delete) +// A conflict result from mintConfigEvent (same Idempotency-Key, different +// operation content) maps to 409. + +function mintResultResponse(c: any, result: { revision: number; automation: AutomationState | null; conflict?: boolean }) { + if (result.conflict) { + return c.json({ error: "idempotency key was already used for a different operation" }, 409); + } + return c.json({ revision: result.revision, automation: result.automation }); +} + +/** Deterministic automation_id for a POST that omitted one: SHA-256 of the + * Idempotency-Key, hex, truncated to 32 chars (fits the 1-128 char id cap + * and stays collision-safe for this cardinality). Same key -> same id, so + * a retried create replays instead of fingerprint-conflicting. */ +async function deriveAutomationId(idempotencyKey: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(idempotencyKey)); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("").slice(0, 32); +} + +const internalError = (c: any, e: unknown) => { + console.log(JSON.stringify({ level: "error", event: "v3_http_unhandled", message: String(e) })); + return c.json({ error: "internal error" }, 500); +}; + +// /v3/automations POST/PATCH/DELETE all end in a stub.mintConfigEvent() / +// stub.automationsSnapshot() call against the SAME SpaceHub DO that +// /v3/realtime guards — a half-open control plane (realtime connections +// blocked but automations still mutating SpaceHub state behind the viewers' +// backs) is the dual-authority bug this flag exists to prevent, so these +// routes get the identical 403 gate, checked after authenticate() and before +// any DO call. +function realtimeDisabled(c: any): boolean { + return !parseRealtimeEnabledFlag((c.env as WorkerEnv).REALTIME_ENABLED); +} + +app.post("/v3/automations", async (c: any) => { + if (realtimeDisabled(c)) return c.json({ error: "realtime_disabled" }, 403); + const role = c.get("role") as Role; + if (!["admin", "owner"].includes(role)) return c.json({ error: "forbidden" }, 403); + const body = await c.req.json().catch(() => null); + const parsed = parseAutomationUpsert(body); + if (!parsed) return c.json({ error: "name, executor_kind and schedule.every_seconds required" }, 400); + const idempotencyKey = c.req.header("idempotency-key") ?? null; + // The idempotency fingerprint must cover only CLIENT-provided content: a + // server-generated random id would enter the fingerprint and make an + // honest retry of a dropped POST (same key, no explicit automation_id) + // fingerprint-mismatch into a spurious 409. When the client omits + // automation_id but supplies an Idempotency-Key, derive the id + // deterministically from that key, so the retry reconstructs the exact + // same automation (including its id) and replays cleanly. + const automationId = + typeof body.automation_id === "string" && body.automation_id + ? body.automation_id + : idempotencyKey + ? await deriveAutomationId(idempotencyKey) + : crypto.randomUUID(); + const automation: AutomationState = { + automation_id: automationId, + name: parsed.name, + executor_kind: parsed.executor_kind, + schedule: { kind: "interval", every_seconds: parsed.every_seconds }, + state: body.state === "paused" ? "paused" : "active", + }; + try { + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const result = await stub.mintConfigEvent(idempotencyKey, { + kind: "automation.upserted", + automation_id: automation.automation_id, + automation, + }); + return mintResultResponse(c, result); + } catch (e) { + return internalError(c, e); + } +}); + +app.patch("/v3/automations/:id", async (c: any) => { + if (realtimeDisabled(c)) return c.json({ error: "realtime_disabled" }, 403); + const role = c.get("role") as Role; + if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); + try { + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const id = c.req.param("id"); + const existing = (await stub.automationsSnapshot()).find((a: AutomationState) => a.automation_id === id); + if (!existing) return c.json({ error: "not found" }, 404); + const body = await c.req.json().catch(() => null); + const next: AutomationState = { + ...existing, + ...(body?.state === "active" || body?.state === "paused" ? { state: body.state } : {}), + ...(body?.schedule?.every_seconds !== undefined + ? { schedule: { kind: "interval" as const, every_seconds: Math.max(1, Number(body.schedule.every_seconds) | 0) } } + : {}), + }; + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.upserted", automation_id: id, automation: next }); + return mintResultResponse(c, result); + } catch (e) { + return internalError(c, e); + } +}); + +app.delete("/v3/automations/:id", async (c: any) => { + if (realtimeDisabled(c)) return c.json({ error: "realtime_disabled" }, 403); + const role = c.get("role") as Role; + if (!["admin", "owner", "viewer"].includes(role)) return c.json({ error: "forbidden" }, 403); + try { + const stub = spaceHubStub(c.env as WorkerEnv, c.get("space")); + const id = c.req.param("id"); + // 404 before minting: deleting a nonexistent automation must not burn a + // revision on a no-op config.event (same existence-check pattern PATCH + // uses). + const existing = (await stub.automationsSnapshot()).find((a: AutomationState) => a.automation_id === id); + if (!existing) return c.json({ error: "not found" }, 404); + const idempotencyKey = c.req.header("idempotency-key") ?? null; + const result = await stub.mintConfigEvent(idempotencyKey, { kind: "automation.removed", automation_id: id }); + return mintResultResponse(c, result); + } catch (e) { + return internalError(c, e); + } +}); + export default app; diff --git a/server/src/app.ts b/server/src/app.ts index f786f980..6d2e8638 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -19,6 +19,23 @@ import { makeSnapshot } from "./domain.ts"; export type Role = "admin" | "owner" | "viewer" | "source"; export type Command = "pause" | "resume" | "stop"; +/** Strict allow-list parser for the REALTIME_ENABLED kill switch. + * + * wrangler.jsonc types this var as a boolean, but a Cloudflare dashboard + * variable override arrives as a STRING at runtime regardless of the + * declared type — `Boolean("false")` is `true`, which would silently flip + * the kill switch on for an operator who typed "false" in the dashboard. + * This flag gates a P0 client behavior, so parsing must be paranoid: only + * the boolean `true`, or the exact strings "true"/"1" (case-insensitive, + * surrounding whitespace trimmed), enable realtime. Everything else — + * `false`, "false", "0", "", undefined, or any other string — disables it. */ +export function parseRealtimeEnabledFlag(value: string | boolean | undefined): boolean { + if (typeof value === "boolean") return value; + if (typeof value !== "string") return false; + const normalized = value.trim().toLowerCase(); + return normalized === "true" || normalized === "1"; +} + export const TOKEN_RE = /^st2_([a-z0-9]{1,16})_[a-f0-9]{48}$/; export interface DeviceInfo { @@ -69,6 +86,11 @@ export interface AppOptions { * joining by code alone needs a code→space lookup with TTL. */ publishInvite?: (c: Context, code: string, space: string) => Promise; lookupInvite?: (c: Context, code: string) => Promise; + /** Realtime kill switch (`/v3/realtime`): a Wrangler `vars` entry + * (`REALTIME_ENABLED`), no per-space granularity in v1. Absent on older + * deployments that don't wire this option, which must mean disabled — so + * the default below is `false`, never `true`. */ + realtimeEnabled?: (c: Context) => boolean; } type Vars = { Variables: { role: Role; space: string; deviceId?: string } }; @@ -160,6 +182,10 @@ export function createApp(opts: AppOptions) { // ---- authenticated: resolve space + role from the token ---- + // Shared by /v2/* (existing product API) and /v3/* (realtime + its HTTP + // control-plane companion): identical st2 token parsing and role/space/ + // deviceId resolution, so /v3 gets exactly the same "invalid token never + // reaches a handler" guarantee /v2 already has, with no duplicated logic. const authenticate = async (c: Context, next: Next) => { if (c.req.path === "/v2/spaces" || c.req.path === "/v2/join") return next(); const admin = opts.authToken?.(c); @@ -186,6 +212,7 @@ export function createApp(opts: AppOptions) { await next(); }; app.use("/v2/*", authenticate); + app.use("/v3/*", authenticate); const store = (c: Context) => opts.store(c, c.get("space")); const registry = (c: Context) => opts.registry?.(c, c.get("space")); @@ -237,14 +264,20 @@ export function createApp(opts: AppOptions) { s.automations(), s.presence(), ]); - return c.json(makeSnapshot({ - now: new Date().toISOString(), - presence, - tasks: visibleTasks(tasks, false), - metrics: metrics.map((metric) => mergeMetric(metric, prefs)), - events, - automations, - })); + return c.json({ + ...makeSnapshot({ + now: new Date().toISOString(), + presence, + tasks: visibleTasks(tasks, false), + metrics: metrics.map((metric) => mergeMetric(metric, prefs)), + events, + automations, + }), + // Apple client's server-side kill switch for /v3/realtime auto-connect + // (see AppOptions.realtimeEnabled): missing or unset must read as + // disabled, so this defaults to false rather than being omitted. + realtime_enabled: opts.realtimeEnabled ? opts.realtimeEnabled(c) : false, + }); }); app.get("/v2/automations", async (c) => { diff --git a/server/src/realtime/attachment.ts b/server/src/realtime/attachment.ts new file mode 100644 index 00000000..43441480 --- /dev/null +++ b/server/src/realtime/attachment.ts @@ -0,0 +1,45 @@ +// Per-connection identity + handshake state, persisted via +// WebSocket#serializeAttachment so it survives Durable Object hibernation +// and eviction/reconstruction. Cloudflare caps a serialized attachment at +// 2 KiB — this shape stays a handful of short scalar fields specifically to +// stay far under that limit. +// +// The trust boundary: `deviceId`/`role` are set exactly once, in +// SpaceHub#fetch, from headers the Worker attaches AFTER it has already +// authenticated the connection's st2 token (see adapters/workers.ts). The +// DO never re-derives identity from a client-sent `hello` body (SPEC.md +// section 10) — it only ever reads this attachment. + +import type { DeviceRole } from "./types.ts"; + +export interface ConnAttachment { + deviceId: string; + role: DeviceRole; + connectedAt: number; + /** True once this connection has completed the hello offer/accept + * handshake (SPEC.md section 9.1). Every frame before that must be + * answered with hello_required. */ + helloDone: boolean; + /** True once this connection has sent `subscribe` (viewer only) — gates + * the "resume must follow subscribe on the same connection" rule + * (SPEC.md section 6.3). Not set by interest.renew. */ + subscribedThisConn: boolean; + /** True once this connection has received a snapshot-or-delta reply to + * `resume` (SPEC.md section 6.2/6.3) — the sole gate for receiving live + * deltas. An error reply (e.g. revision_unavailable) never sets this. */ + deltaEligible: boolean; + sessionId?: string; +} + +export function isConnAttachment(v: unknown): v is ConnAttachment { + if (typeof v !== "object" || v === null) return false; + const a = v as Record; + return ( + typeof a.deviceId === "string" && + (a.role === "source" || a.role === "viewer") && + typeof a.connectedAt === "number" && + typeof a.helloDone === "boolean" && + typeof a.subscribedThisConn === "boolean" && + typeof a.deltaEligible === "boolean" + ); +} diff --git a/server/src/realtime/chunking.ts b/server/src/realtime/chunking.ts new file mode 100644 index 00000000..7b43b3c0 --- /dev/null +++ b/server/src/realtime/chunking.ts @@ -0,0 +1,84 @@ +// Pure snapshot/delta chunking helpers — no I/O, unit-testable without a +// Workers runtime. SPEC.md section 6.2/11: every chunk of one snapshot +// shares the same `revision`, `part` numbers consecutively from 1, and the +// last chunk carries `final: true`; a chained catch-up delta series splits +// strictly on event boundaries so `d1.to_revision == d2.from_revision`. + +import { CHUNK_SOFT_MAX_BYTES } from "./protocol.ts"; +import type { + AutomationState, + DeltaBody, + DeltaEventItem, + MessageRecord, + MetricSample, + SnapshotBody, + TaskState, +} from "./types.ts"; + +type SnapshotBuckets = { + tasks: TaskState[]; + metrics: MetricSample[]; + messages: MessageRecord[]; + automations: AutomationState[]; +}; + +/** Packs the four snapshot arrays into one or more chunks under the frame + * size budget. Always emits at least one chunk (part 1, final true), even + * when every array is empty (SPEC.md: "A snapshot MAY be empty"). */ +export function chunkSnapshot(revision: number, arrays: SnapshotBuckets): SnapshotBody[] { + const buckets: Array<{ key: keyof SnapshotBuckets; items: unknown[] }> = [ + { key: "tasks", items: arrays.tasks }, + { key: "metrics", items: arrays.metrics }, + { key: "messages", items: arrays.messages }, + { key: "automations", items: arrays.automations }, + ]; + const chunks: SnapshotBody[] = []; + const empty = (): SnapshotBuckets => ({ tasks: [], metrics: [], messages: [], automations: [] }); + let cur = empty(); + let curSize = 128; // overhead margin for envelope wrapper + fixed keys + const flush = (final: boolean) => { + chunks.push({ revision, part: chunks.length + 1, final, ...cur }); + cur = empty(); + curSize = 128; + }; + for (const bucket of buckets) { + for (const item of bucket.items) { + const size = JSON.stringify(item).length + 2; + const curCount = cur.tasks.length + cur.metrics.length + cur.messages.length + cur.automations.length; + if (curSize + size > CHUNK_SOFT_MAX_BYTES && curCount > 0) flush(false); + (cur[bucket.key] as unknown[]).push(item); + curSize += size; + } + } + flush(true); + return chunks; +} + +/** Splits a catch-up event range into one or more chained deltas whose + * from/to boundaries connect exactly. Always emits at least one delta + * (possibly with an empty `events` array) so a caller can send it as the + * single resume reply. */ +export function chunkDeltaEvents(fromRevision: number, events: DeltaEventItem[]): DeltaBody[] { + const groups: DeltaEventItem[][] = []; + let cur: DeltaEventItem[] = []; + let curSize = 128; + for (const event of events) { + const size = JSON.stringify(event).length + 2; + if (curSize + size > CHUNK_SOFT_MAX_BYTES && cur.length > 0) { + groups.push(cur); + cur = []; + curSize = 128; + } + cur.push(event); + curSize += size; + } + groups.push(cur); + let from = fromRevision; + const out: DeltaBody[] = []; + for (const group of groups) { + const to = from + group.length; + out.push({ from_revision: from, to_revision: to, events: group }); + from = to; + } + return out; +} diff --git a/server/src/realtime/guards.ts b/server/src/realtime/guards.ts new file mode 100644 index 00000000..c0bdcf79 --- /dev/null +++ b/server/src/realtime/guards.ts @@ -0,0 +1,481 @@ +// Lightweight, hand-written runtime validation for the realtime protocol. +// Deliberately NOT ajv: proto/realtime/tools/validate.js is the fixture +// conformance tool (its own package, pinned ajv) and is the authority for +// "does this fixture match the schema". This module is the production-path +// guard the server actually runs on every inbound frame — it mirrors the +// same rules (proto/realtime/*.schema.json) by hand, deliberately narrow in +// scope (no generic JSON Schema engine as a production dependency). +// +// Per SPEC.md section 15, the envelope TOP LEVEL is permanently strict +// (unknown top-level field is always malformed) while `body` tolerates +// unknown fields in future minor versions — these guards enforce the exact +// v1.0.0 body shapes (matching the frozen schemas byte for byte, same as +// the fixtures expect), which is stricter than the runtime-tolerance clause +// permits for a hypothetical v1.1 peer. That's an accepted, documented +// trade: see docs/design/realtime-server.md. + +import { + MESSAGE_TYPES, + type AckBody, + type AnyEnvelope, + type AutomationState, + type CommandBody, + type ConfigEventBody, + type DeviceRole, + type ErrorBody, + type ErrorCode, + type HelloBody, + type InterestRenewBody, + type MessageEventBody, + type MessageType, + type MetricFrameBody, + type MetricSample, + type ResumeBody, + type SubscribeBody, + type TaskEventBody, + type UnsubscribeBody, +} from "./types.ts"; + +export const FRAME_MAX_BYTES = 64 * 1024; + +const ENVELOPE_ID_RE = /^[A-Za-z0-9_-]{1,64}$/; +const DEVICE_ID_RE = /^[A-Za-z0-9_-]{1,128}$/; +const METRIC_ID_RE = /^[a-z0-9_.-]{1,64}$/; + +export type ValidationResult = { ok: true; value: T } | { ok: false; code: ErrorCode; message: string }; + +const ok = (value: T): ValidationResult => ({ ok: true, value }); +const fail = (code: ErrorCode, message: string): ValidationResult => ({ ok: false, code, message }); + +export type ParseResult = + | { kind: "ok"; envelope: AnyEnvelope } + | { kind: "unknown_type"; type: string; id?: string } + | { kind: "error"; code: ErrorCode; message: string; inReplyTo?: string }; + +function isPlainObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +export function isUnixMsTimestamp(v: unknown): v is number { + return typeof v === "number" && Number.isInteger(v) && v >= 1_000_000_000_000 && v <= 9_999_999_999_999; +} + +function isFreeText(v: unknown): v is string { + return typeof v === "string" && v.length <= 2048; +} + +function isLabelText(v: unknown): v is string { + return typeof v === "string" && v.length <= 256; +} + +function isDeviceId(v: unknown): v is string { + return typeof v === "string" && DEVICE_ID_RE.test(v); +} + +function isEnvelopeId(v: unknown): v is string { + return typeof v === "string" && ENVELOPE_ID_RE.test(v); +} + +function hasOnlyKeys(obj: Record, allowed: readonly string[]): boolean { + return Object.keys(obj).every((k) => allowed.includes(k)); +} + +function validateDisplayHints(v: unknown): boolean { + if (v === undefined) return true; + if (!isPlainObject(v)) return false; + if (!hasOnlyKeys(v, ["icon", "tint", "template"])) return false; + if (v.icon !== undefined && !(typeof v.icon === "string" && v.icon.length <= 64)) return false; + if (v.tint !== undefined && !(typeof v.tint === "string" && v.tint.length <= 32)) return false; + if (v.template !== undefined && !(typeof v.template === "string" && v.template.length <= 32)) return false; + return true; +} + +function validateMetricSample(v: unknown): v is MetricSample { + if (!isPlainObject(v)) return false; + if (!hasOnlyKeys(v, ["metric_id", "value", "label", "ts", "display", "target", "min", "max", "alert_above", "alert_below"])) return false; + if (typeof v.metric_id !== "string" || !METRIC_ID_RE.test(v.metric_id)) return false; + if (typeof v.value !== "string" || v.value.length > 256) return false; + if (!isUnixMsTimestamp(v.ts)) return false; + if (v.label !== undefined && !isLabelText(v.label)) return false; + if (!validateDisplayHints(v.display)) return false; + for (const key of ["target", "min", "max", "alert_above", "alert_below"] as const) { + if (v[key] !== undefined && !(typeof v[key] === "string" && (v[key] as string).length <= 64)) return false; + } + return true; +} + +export function validateAutomationState(v: unknown): v is AutomationState { + if (!isPlainObject(v)) return false; + if (!hasOnlyKeys(v, ["automation_id", "name", "executor_kind", "schedule", "state", "last_run_at"])) return false; + if (typeof v.automation_id !== "string" || v.automation_id.length < 1 || v.automation_id.length > 128) return false; + if (typeof v.name !== "string" || v.name.length < 1 || v.name.length > 256) return false; + if (!["script", "agent", "hybrid"].includes(v.executor_kind as string)) return false; + const schedule = v.schedule; + if (!isPlainObject(schedule) || schedule.kind !== "interval" || typeof schedule.every_seconds !== "number" || schedule.every_seconds < 1) { + return false; + } + if (!["active", "paused"].includes(v.state as string)) return false; + if (v.last_run_at !== undefined && !isUnixMsTimestamp(v.last_run_at)) return false; + return true; +} + +// ---- per-type body validators ---- + +function validateHelloBody(body: Record): ValidationResult { + if (body.stage === "offer") { + if (!hasOnlyKeys(body, ["stage", "device_id", "role", "protocol_versions", "capabilities"])) { + return fail("malformed", "hello offer: unexpected field"); + } + if (!isDeviceId(body.device_id)) return fail("malformed", "hello offer: invalid device_id"); + if (body.role !== "source" && body.role !== "viewer") return fail("malformed", "hello offer: invalid role"); + if (!Array.isArray(body.protocol_versions) || body.protocol_versions.length < 1) { + return fail("malformed", "hello offer: protocol_versions must be non-empty"); + } + if (!body.protocol_versions.every((n) => Number.isInteger(n) && (n as number) >= 1)) { + return fail("malformed", "hello offer: invalid protocol_versions entry"); + } + if (new Set(body.protocol_versions).size !== body.protocol_versions.length) { + // Mirrors the schema's uniqueItems: true on protocol_versions. + return fail("malformed", "hello offer: protocol_versions must not contain duplicates"); + } + if (body.capabilities !== undefined) { + if (!Array.isArray(body.capabilities) || !body.capabilities.every((c) => typeof c === "string" && c.length <= 64)) { + return fail("malformed", "hello offer: invalid capabilities"); + } + } + return ok({ + stage: "offer", + device_id: body.device_id, + role: body.role, + protocol_versions: body.protocol_versions as number[], + capabilities: body.capabilities as string[] | undefined, + }); + } + if (body.stage === "accept") { + if (!hasOnlyKeys(body, ["stage", "protocol_version", "session_id", "heartbeat_interval_ms", "capabilities"])) { + return fail("malformed", "hello accept: unexpected field"); + } + if (!Number.isInteger(body.protocol_version) || (body.protocol_version as number) < 1) { + return fail("malformed", "hello accept: invalid protocol_version"); + } + if (typeof body.session_id !== "string" || body.session_id.length < 1 || body.session_id.length > 64) { + return fail("malformed", "hello accept: invalid session_id"); + } + if (!Number.isInteger(body.heartbeat_interval_ms) || (body.heartbeat_interval_ms as number) < 1000 || (body.heartbeat_interval_ms as number) > 300000) { + return fail("malformed", "hello accept: invalid heartbeat_interval_ms"); + } + return ok({ + stage: "accept", + protocol_version: body.protocol_version as number, + session_id: body.session_id, + heartbeat_interval_ms: body.heartbeat_interval_ms as number, + capabilities: body.capabilities as string[] | undefined, + }); + } + return fail("malformed", "hello: invalid stage"); +} + +function validateResumeBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["last_revision"])) return fail("malformed", "resume: unexpected field"); + if (!Number.isInteger(body.last_revision) || (body.last_revision as number) < 0) { + return fail("malformed", "resume: last_revision must be a non-negative integer"); + } + return ok({ last_revision: body.last_revision as number }); +} + +function validateAckBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["acked", "in_reply_to", "lease"])) return fail("malformed", "ack: unexpected field"); + if (body.acked === undefined && body.in_reply_to === undefined) { + return fail("malformed", "ack: at least one of acked or in_reply_to required"); + } + if (body.acked !== undefined) { + if (!Array.isArray(body.acked) || body.acked.length < 1) return fail("malformed", "ack: acked must be non-empty array"); + for (const pair of body.acked) { + if (!isPlainObject(pair) || !hasOnlyKeys(pair, ["device_id", "device_seq"])) return fail("malformed", "ack: invalid acked entry"); + if (!isDeviceId(pair.device_id)) return fail("malformed", "ack: invalid device_id in acked"); + if (!Number.isInteger(pair.device_seq) || (pair.device_seq as number) < 1) return fail("malformed", "ack: invalid device_seq in acked"); + } + } + if (body.in_reply_to !== undefined && !isEnvelopeId(body.in_reply_to)) return fail("malformed", "ack: invalid in_reply_to"); + if (body.lease !== undefined) { + if (!isPlainObject(body.lease) || !hasOnlyKeys(body.lease, ["expires_at"]) || !isUnixMsTimestamp(body.lease.expires_at)) { + return fail("malformed", "ack: invalid lease"); + } + } + return ok(body as unknown as AckBody); +} + +function validateTaskEventBody(body: Record): ValidationResult { + const allowed = ["device_id", "device_seq", "task_id", "kind", "occurred_at", "title", "percent", "step", "message", "display"]; + if (!hasOnlyKeys(body, allowed)) return fail("malformed", "task.event: unexpected field"); + if (!isDeviceId(body.device_id)) return fail("malformed", "task.event: invalid device_id"); + if (!Number.isInteger(body.device_seq) || (body.device_seq as number) < 1) return fail("malformed", "task.event: invalid device_seq"); + if (typeof body.task_id !== "string" || body.task_id.length < 1 || body.task_id.length > 256) return fail("malformed", "task.event: invalid task_id"); + if (!["started", "progress", "step", "done", "failed"].includes(body.kind as string)) return fail("malformed", "task.event: invalid kind"); + if (!isUnixMsTimestamp(body.occurred_at)) return fail("malformed", "task.event: invalid occurred_at"); + if (body.title !== undefined && !isFreeText(body.title)) return fail("malformed", "task.event: invalid title"); + if (body.step !== undefined && !isFreeText(body.step)) return fail("malformed", "task.event: invalid step"); + if (body.message !== undefined && !isFreeText(body.message)) return fail("malformed", "task.event: invalid message"); + if (body.percent !== undefined && !(Number.isInteger(body.percent) && (body.percent as number) >= 0 && (body.percent as number) <= 100)) { + return fail("malformed", "task.event: invalid percent"); + } + if (!validateDisplayHints(body.display)) return fail("malformed", "task.event: invalid display"); + if (body.kind === "progress" && body.percent === undefined) return fail("malformed", "task.event: progress requires percent"); + return ok(body as unknown as TaskEventBody); +} + +function validateMessageEventBody(body: Record): ValidationResult { + const allowed = ["device_id", "device_seq", "message_id", "level", "text", "occurred_at", "automation_id"]; + if (!hasOnlyKeys(body, allowed)) return fail("malformed", "message.event: unexpected field"); + if (!isDeviceId(body.device_id)) return fail("malformed", "message.event: invalid device_id"); + if (!Number.isInteger(body.device_seq) || (body.device_seq as number) < 1) return fail("malformed", "message.event: invalid device_seq"); + if (typeof body.message_id !== "string" || body.message_id.length < 1 || body.message_id.length > 128) { + return fail("malformed", "message.event: invalid message_id"); + } + if (!["info", "warn", "error"].includes(body.level as string)) return fail("malformed", "message.event: invalid level"); + if (!isFreeText(body.text)) return fail("malformed", "message.event: invalid text"); + if (!isUnixMsTimestamp(body.occurred_at)) return fail("malformed", "message.event: invalid occurred_at"); + if (body.automation_id !== undefined && !(typeof body.automation_id === "string" && body.automation_id.length <= 128)) { + return fail("malformed", "message.event: invalid automation_id"); + } + return ok(body as unknown as MessageEventBody); +} + +function validateMetricFrameBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["device_id", "metrics"])) return fail("malformed", "metric.frame: unexpected field"); + if (!isDeviceId(body.device_id)) return fail("malformed", "metric.frame: invalid device_id"); + if (!Array.isArray(body.metrics) || body.metrics.length < 1) return fail("malformed", "metric.frame: metrics must be non-empty array"); + if (body.metrics.length > 64) return fail("batch_too_large", "metric.frame: metrics exceeds 64 items"); + for (const m of body.metrics) { + if (!validateMetricSample(m)) return fail("malformed", "metric.frame: invalid metric sample"); + } + return ok(body as unknown as MetricFrameBody); +} + +function validateConfigEventBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["kind", "automation_id", "automation", "occurred_at"])) return fail("malformed", "config.event: unexpected field"); + if (!["automation.upserted", "automation.removed"].includes(body.kind as string)) return fail("malformed", "config.event: invalid kind"); + if (typeof body.automation_id !== "string" || body.automation_id.length < 1 || body.automation_id.length > 128) { + return fail("malformed", "config.event: invalid automation_id"); + } + if (!isUnixMsTimestamp(body.occurred_at)) return fail("malformed", "config.event: invalid occurred_at"); + if (body.kind === "automation.upserted" && !validateAutomationState(body.automation)) { + return fail("malformed", "config.event: automation.upserted requires a valid automation"); + } + return ok(body as unknown as ConfigEventBody); +} + +const SUBSCRIBE_TOPICS = ["task", "metric", "message"]; + +function validateSubscribeLikeBody(body: Record, label: string): ValidationResult { + if (!hasOnlyKeys(body, ["topics"])) return fail("malformed", `${label}: unexpected field`); + if (body.topics !== undefined) { + if (!Array.isArray(body.topics) || !body.topics.every((t) => SUBSCRIBE_TOPICS.includes(t as string))) { + return fail("malformed", `${label}: invalid topics`); + } + if (new Set(body.topics).size !== body.topics.length) { + // Mirrors the schema's uniqueItems: true on topics. + return fail("malformed", `${label}: topics must not contain duplicates`); + } + } + return ok(body as unknown as SubscribeBody); +} + +function validateUnsubscribeBody(body: Record): ValidationResult { + if (Object.keys(body).length > 0) return fail("malformed", "unsubscribe: body must be empty"); + return ok({} as UnsubscribeBody); +} + +function validateCommandBody(body: Record): ValidationResult { + const allowed = ["command_id", "origin", "issued_by_device_id", "action", "task_id", "automation_id", "target_device_id", "ttl_ms", "params"]; + if (!hasOnlyKeys(body, allowed)) return fail("malformed", "command: unexpected field"); + if (typeof body.command_id !== "string" || body.command_id.length < 1 || body.command_id.length > 64) { + return fail("malformed", "command: invalid command_id"); + } + if (body.origin !== "viewer" && body.origin !== "server") return fail("malformed", "command: invalid origin"); + if (!["pause", "resume", "stop", "run_now", "throttle", "resume_rate"].includes(body.action as string)) { + return fail("malformed", "command: invalid action"); + } + if (!Number.isInteger(body.ttl_ms) || (body.ttl_ms as number) < 1 || (body.ttl_ms as number) > 86_400_000) { + return fail("malformed", "command: invalid ttl_ms"); + } + if (body.issued_by_device_id !== undefined && !isDeviceId(body.issued_by_device_id)) { + return fail("malformed", "command: invalid issued_by_device_id"); + } + if (body.target_device_id !== undefined && !isDeviceId(body.target_device_id)) { + return fail("malformed", "command: invalid target_device_id"); + } + if (body.task_id !== undefined && !(typeof body.task_id === "string" && body.task_id.length >= 1 && body.task_id.length <= 256)) { + return fail("malformed", "command: invalid task_id"); + } + if (body.automation_id !== undefined && !(typeof body.automation_id === "string" && body.automation_id.length >= 1 && body.automation_id.length <= 128)) { + return fail("malformed", "command: invalid automation_id"); + } + if (body.params !== undefined && !isPlainObject(body.params)) return fail("malformed", "command: invalid params"); + + const action = body.action as string; + if (action === "pause" || action === "resume" || action === "stop") { + if (body.origin !== "viewer") return fail("malformed", "command: pause/resume/stop require origin viewer"); + if (body.issued_by_device_id === undefined) return fail("malformed", "command: pause/resume/stop require issued_by_device_id"); + if (body.task_id === undefined) return fail("malformed", "command: pause/resume/stop require task_id"); + if (body.automation_id !== undefined) return fail("malformed", "command: pause/resume/stop must not carry automation_id"); + } else if (action === "run_now") { + if (body.origin !== "viewer") return fail("malformed", "command: run_now requires origin viewer"); + if (body.issued_by_device_id === undefined) return fail("malformed", "command: run_now requires issued_by_device_id"); + if (body.automation_id === undefined) return fail("malformed", "command: run_now requires automation_id"); + if (body.task_id !== undefined) return fail("malformed", "command: run_now must not carry task_id"); + } else { + // throttle | resume_rate + if (body.origin !== "server") return fail("malformed", "command: throttle/resume_rate require origin server"); + if (body.task_id !== undefined || body.automation_id !== undefined) { + return fail("malformed", "command: throttle/resume_rate must not carry task_id or automation_id"); + } + } + return ok(body as unknown as CommandBody); +} + +const ERROR_CODES: readonly ErrorCode[] = [ + "version_unsupported", + "hello_required", + "unauthenticated", + "unauthorized", + "malformed", + "rate_limited", + "frame_too_large", + "batch_too_large", + "revision_unavailable", + "command_expired", + "sequence_gap", + "superseded", + "internal_error", +]; + +function validateErrorBody(body: Record): ValidationResult { + if (!hasOnlyKeys(body, ["code", "message", "in_reply_to", "retryable", "fatal"])) return fail("malformed", "error: unexpected field"); + if (!ERROR_CODES.includes(body.code as ErrorCode)) return fail("malformed", "error: invalid code"); + if (typeof body.message !== "string" || body.message.length > 500) return fail("malformed", "error: invalid message"); + if (typeof body.retryable !== "boolean") return fail("malformed", "error: retryable required"); + if (typeof body.fatal !== "boolean") return fail("malformed", "error: fatal required"); + if (body.in_reply_to !== undefined && !isEnvelopeId(body.in_reply_to)) return fail("malformed", "error: invalid in_reply_to"); + return ok(body as unknown as ErrorBody); +} + +function validateBody(type: MessageType, body: Record): ValidationResult { + switch (type) { + case "hello": + return validateHelloBody(body); + case "resume": + return validateResumeBody(body); + case "ack": + return validateAckBody(body); + case "task.event": + return validateTaskEventBody(body); + case "message.event": + return validateMessageEventBody(body); + case "metric.frame": + return validateMetricFrameBody(body); + case "config.event": + return validateConfigEventBody(body); + case "subscribe": + return validateSubscribeLikeBody(body, "subscribe"); + case "unsubscribe": + return validateUnsubscribeBody(body); + case "interest.renew": + return validateSubscribeLikeBody(body, "interest.renew") as ValidationResult; + case "command": + return validateCommandBody(body); + case "error": + return validateErrorBody(body); + case "snapshot": + case "delta": + // Server-only message types: any client-sent instance is rejected by + // the authorization layer (authorizeClientEnvelope), not here. We + // still validate shape for the server's own outbound round-trip test. + return ok(body); + default: + return fail("malformed", `unhandled type ${type as string}`); + } +} + +/** Parses one raw text frame. Returns "unknown_type" (never "malformed") + * for a `type` this version doesn't recognize (SPEC.md section 15) — the + * caller MUST ignore such frames rather than raising an error. */ +export function parseEnvelope(raw: string): ParseResult { + const byteLength = new TextEncoder().encode(raw).length; + if (byteLength > FRAME_MAX_BYTES) { + return { kind: "error", code: "frame_too_large", message: "frame exceeds 64 KiB" }; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { kind: "error", code: "malformed", message: "invalid JSON" }; + } + if (!isPlainObject(parsed)) return { kind: "error", code: "malformed", message: "envelope must be an object" }; + if (!hasOnlyKeys(parsed, ["type", "id", "ts", "body"])) { + return { kind: "error", code: "malformed", message: "envelope carries an unexpected top-level field" }; + } + const { type, id, ts, body } = parsed; + if (typeof type !== "string") return { kind: "error", code: "malformed", message: "type must be a string" }; + if (!isEnvelopeId(id)) return { kind: "error", code: "malformed", message: "invalid envelope id" }; + if (!isUnixMsTimestamp(ts)) return { kind: "error", code: "malformed", message: "invalid envelope ts" }; + if (!isPlainObject(body)) return { kind: "error", code: "malformed", message: "body must be an object" }; + if (!(MESSAGE_TYPES as readonly string[]).includes(type)) { + return { kind: "unknown_type", type, id }; + } + const result = validateBody(type as MessageType, body); + if (!result.ok) return { kind: "error", code: result.code, message: result.message, inReplyTo: id }; + return { kind: "ok", envelope: { type, id, ts, body: result.value } as AnyEnvelope }; +} + +/** Encodes an envelope back to its wire JSON string. Used by the round-trip + * fixture test (decode a valid fixture, encode it, decode again, compare). */ +export function encodeEnvelope(envelope: AnyEnvelope): string { + return JSON.stringify(envelope); +} + +// ---- SPEC.md section 10.1 authorization matrix ---- + +export type AuthorizeResult = { ok: true } | { ok: false; code: "unauthorized" | "hello_required" }; + +const authOk: AuthorizeResult = { ok: true }; +const authUnauthorized: AuthorizeResult = { ok: false, code: "unauthorized" }; +const authHelloRequired: AuthorizeResult = { ok: false, code: "hello_required" }; + +/** Validates a client-originated envelope's `type` against the sending + * connection's role, per SPEC.md section 10.1. Governs client-originated + * envelopes ONLY — server-originated `command` (throttle/resume_rate) is + * not subject to this matrix (SPEC.md section 8). */ +export function authorizeClientEnvelope(role: DeviceRole, envelope: AnyEnvelope): AuthorizeResult { + switch (envelope.type) { + case "hello": { + const body = envelope.body as HelloBody; + if (body.stage === "accept") return authHelloRequired; + return authOk; + } + case "resume": + case "subscribe": + case "unsubscribe": + case "interest.renew": + return role === "viewer" ? authOk : authUnauthorized; + case "task.event": + case "message.event": + case "metric.frame": + case "ack": + return role === "source" ? authOk : authUnauthorized; + case "command": { + const body = envelope.body as CommandBody; + if (role !== "viewer") return authUnauthorized; + if (body.origin !== "viewer") return authUnauthorized; + return authOk; + } + case "error": + return authOk; + case "snapshot": + case "delta": + case "config.event": + return authUnauthorized; + default: + return authUnauthorized; + } +} diff --git a/server/src/realtime/protocol.ts b/server/src/realtime/protocol.ts new file mode 100644 index 00000000..e0dfe662 --- /dev/null +++ b/server/src/realtime/protocol.ts @@ -0,0 +1,102 @@ +// Shared constants and envelope-construction helpers for the SpaceHub DO. +// Pure/no I/O so they are cheap to unit test directly. + +import type { + AnyEnvelope, + Envelope, + ErrorBody, + ErrorCode, + MessageType, +} from "./types.ts"; + +export const PROTOCOL_VERSION = 1; +export const SUPPORTED_PROTOCOL_VERSIONS = [1]; + +/** SPEC.md section 7: server MUST choose a lease duration between 30000 ms + * and 60000 ms inclusive; 45000 ms is RECOMMENDED as a default. */ +export const LEASE_MIN_MS = 30_000; +export const LEASE_MAX_MS = 60_000; +export const LEASE_DEFAULT_MS = 45_000; + +/** Heartbeat cadence advertised in hello{stage:accept}. Informative only + * (SPEC.md section 9.3); the server never depends on it for correctness. */ +export const HEARTBEAT_INTERVAL_MS = 30_000; + +/** Reliable-event retention window: how many trailing revisions the event + * log keeps for incremental `resume` catch-up before a viewer is forced + * onto a fresh `snapshot`. Documented normative choice (SPEC.md section + * 6.2 permits any retention policy as long as gaps fall back to snapshot). */ +export const EVENT_LOG_RETENTION_REVISIONS = 1000; + +/** Bounded message history window carried in `snapshot.body.messages` + * (SPEC.md section 6.4 recommends 200). */ +export const MESSAGE_WINDOW = 200; + +/** Soft ceiling used when packing snapshot/delta chunks, comfortably under + * the 64 KiB hard frame limit (SPEC.md section 11) to leave room for + * envelope/JSON overhead. */ +export const CHUNK_SOFT_MAX_BYTES = 56 * 1024; + +/** SPEC.md section 11: a connection MUST NOT emit more than 10 + * metric.frame envelopes per second. */ +export const METRIC_FRAME_RATE_PER_SEC = 10; + +/** SPEC.md section 12: 2 Hz ceiling per metric_id. */ +export const METRIC_SAMPLE_MIN_INTERVAL_MS = 500; + +/** Cardinality cap on `SpaceHub#metricsCache` (distinct `metric_id`s held + * at once, per space). Without this, a compromised or misbehaving source + * that rotates metric_ids without bound would grow DO memory unboundedly + * until the instance crashes — the existing per-frame size/rate limits + * (METRIC_FRAME_RATE_PER_SEC, the 64-samples-per-frame guard in + * guards.ts) bound how fast the cache can grow but not how big it can + * get. Beyond this cap, the least-recently-updated metric_id is evicted + * to make room (see SpaceHub#touchMetricCache); eviction is safe per + * SPEC.md section 6.2 (snapshot.metrics is a best-effort cache, evicted + * metrics are simply absent from the next snapshot). */ +export const METRIC_CACHE_MAX_METRICS = 256; + +export function newEnvelopeId(): string { + return crypto.randomUUID(); +} + +export function makeEnvelope(type: T, body: B, ts = Date.now()): Envelope { + return { type, id: newEnvelopeId(), ts, body }; +} + +export function makeError( + code: ErrorCode, + message: string, + opts: { retryable: boolean; fatal: boolean; inReplyTo?: string }, +): AnyEnvelope { + const body: ErrorBody = { + code, + message, + retryable: opts.retryable, + fatal: opts.fatal, + ...(opts.inReplyTo ? { in_reply_to: opts.inReplyTo } : {}), + }; + return makeEnvelope("error", body) as AnyEnvelope; +} + +/** Fixed retryable/fatal semantics per SPEC.md section 13's error code + * table, so call sites never have to repeat these pairings by hand. */ +export const ERROR_SEMANTICS: Record = { + version_unsupported: { retryable: false, fatal: true }, + hello_required: { retryable: true, fatal: true }, + unauthenticated: { retryable: false, fatal: true }, + unauthorized: { retryable: false, fatal: false }, + malformed: { retryable: true, fatal: false }, + rate_limited: { retryable: true, fatal: false }, + frame_too_large: { retryable: true, fatal: false }, + batch_too_large: { retryable: true, fatal: false }, + revision_unavailable: { retryable: true, fatal: false }, + command_expired: { retryable: false, fatal: false }, + sequence_gap: { retryable: false, fatal: false }, + superseded: { retryable: false, fatal: true }, + internal_error: { retryable: true, fatal: false }, +}; + +export function makeStandardError(code: ErrorCode, message: string, inReplyTo?: string): AnyEnvelope { + return makeError(code, message, { ...ERROR_SEMANTICS[code], inReplyTo }); +} diff --git a/server/src/realtime/rows.ts b/server/src/realtime/rows.ts new file mode 100644 index 00000000..253e37f6 --- /dev/null +++ b/server/src/realtime/rows.ts @@ -0,0 +1,83 @@ +// SQLite row shapes for SpaceHub storage, and pure mapping functions to/from +// the wire types in ./types.ts. Kept separate from space-hub.ts so the +// mapping logic is easy to read and (where pure) easy to unit test. + +import type { AutomationState, DeltaEventItem, MessageRecord, TaskState } from "./types.ts"; + +export interface TaskRow extends Record { + task_id: string; + device_id: string; + title: string | null; + state: string; + percent: number | null; + step: string | null; + message: string | null; + updated_at: number; + display: string | null; +} + +export function rowToTaskState(row: TaskRow): TaskState { + return { + task_id: row.task_id, + device_id: row.device_id, + state: row.state as TaskState["state"], + updated_at: row.updated_at, + ...(row.title !== null ? { title: row.title } : {}), + ...(row.percent !== null ? { percent: row.percent } : {}), + ...(row.step !== null ? { step: row.step } : {}), + ...(row.message !== null ? { message: row.message } : {}), + ...(row.display !== null ? { display: JSON.parse(row.display) } : {}), + }; +} + +export interface AutomationRow extends Record { + automation_id: string; + name: string; + executor_kind: string; + every_seconds: number; + state: string; + last_run_at: number | null; +} + +export function rowToAutomationState(row: AutomationRow): AutomationState { + return { + automation_id: row.automation_id, + name: row.name, + executor_kind: row.executor_kind as AutomationState["executor_kind"], + schedule: { kind: "interval", every_seconds: row.every_seconds }, + state: row.state as AutomationState["state"], + ...(row.last_run_at !== null ? { last_run_at: row.last_run_at } : {}), + }; +} + +export interface MessageRow extends Record { + message_id: string; + device_id: string; + level: string; + text: string; + occurred_at: number; + revision: number; +} + +export function rowToMessageRecord(row: MessageRow): MessageRecord { + return { + message_id: row.message_id, + device_id: row.device_id, + level: row.level as MessageRecord["level"], + text: row.text, + occurred_at: row.occurred_at, + }; +} + +export interface EventLogRow extends Record { + revision: number; + event_type: string; + device_id: string | null; + device_seq: number | null; + occurred_at: number; + payload: string; +} + +export function rowToDeltaEventItem(row: EventLogRow): DeltaEventItem { + return { event_type: row.event_type, event: JSON.parse(row.payload) } as DeltaEventItem; +} diff --git a/server/src/realtime/space-hub.ts b/server/src/realtime/space-hub.ts new file mode 100644 index 00000000..821f6554 --- /dev/null +++ b/server/src/realtime/space-hub.ts @@ -0,0 +1,1094 @@ +// SpaceHub: one Durable Object per Sitrep space, implementing the realtime +// synchronization protocol (proto/realtime/SPEC.md, frozen v1). SQLite +// backed, Hibernatable-WebSockets based — the DO holds no connection state +// in memory that isn't recoverable from storage + each connection's +// serialized attachment (see ./attachment.ts). +// +// Concurrency note read before editing: every handler below that must +// behave as "one durable transaction" (SPEC.md section 5.2 dedup+apply, +// section 5.5 config.event mint+revision) is written as a sequence of +// synchronous `this.ctx.storage.sql.exec` calls with NO `await` between +// them. Durable Object storage coalesces consecutive synchronous writes — +// and the JS isolate never interleaves another callback into a running +// synchronous function — so this is sufficient for atomicity without a +// separate transaction API. Do not introduce an `await` in the middle of +// one of these methods without re-establishing atomicity another way. +// +// The same single-threaded-synchronous-function argument is what satisfies +// SPEC.md section 6.2's "no other outbound envelope may interleave a +// chunked snapshot": sendSnapshotChunks/sendChainedDeltas build every frame +// up front and `ws.send()` them in a plain synchronous loop, so nothing +// else in this DO can run until the whole reply has gone out. + +import { DurableObject } from "cloudflare:workers"; +import { type ConnAttachment, isConnAttachment } from "./attachment.ts"; +import { chunkDeltaEvents, chunkSnapshot } from "./chunking.ts"; +import { authorizeClientEnvelope, parseEnvelope } from "./guards.ts"; +import { + ERROR_SEMANTICS, + HEARTBEAT_INTERVAL_MS, + LEASE_DEFAULT_MS, + METRIC_CACHE_MAX_METRICS, + METRIC_FRAME_RATE_PER_SEC, + EVENT_LOG_RETENTION_REVISIONS, + MESSAGE_WINDOW, + PROTOCOL_VERSION, + SUPPORTED_PROTOCOL_VERSIONS, + makeEnvelope, + makeStandardError, + newEnvelopeId, +} from "./protocol.ts"; +import { + rowToAutomationState, + rowToDeltaEventItem, + rowToMessageRecord, + rowToTaskState, + type AutomationRow, + type EventLogRow, + type MessageRow, + type TaskRow, +} from "./rows.ts"; +import type { + AnyEnvelope, + AutomationState, + CommandBody, + ConfigEventBody, + ConfigEventKind, + DeltaBody, + ErrorCode, + HelloOfferBody, + InterestRenewBody, + MessageEventBody, + MessageType, + MetricFrameBody, + MetricSample, + ResumeBody, + SubscribeBody, + TaskEventBody, +} from "./types.ts"; + +interface RateLimiterState { + frameTimestamps: number[]; +} + +/** Deterministic JSON with recursively sorted object keys, so two + * structurally-equal values always serialize identically regardless of + * property construction order. Used for idempotency fingerprints. */ +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`); + return `{${entries.join(",")}}`; + } + return JSON.stringify(value); +} + +export class SpaceHub extends DurableObject { + /** Current schema version this build expects. Bump when `migrate()`'s + * DDL block changes, alongside adding whatever new `CREATE TABLE/INDEX + * IF NOT EXISTS` statements the change needs. */ + private static readonly SCHEMA_VERSION = 1; + + /** Best-effort metric cache (SPEC.md section 6.2: snapshot.metrics is a + * convenience cache, allowed to be empty/stale, outside space_revision + * accounting) — deliberately in-memory only, never written to SQLite. + * Capped at METRIC_CACHE_MAX_METRICS distinct metric_ids with + * least-recently-updated eviction (see touchMetricCache) so a source + * rotating metric_ids without bound can't grow this without bound. Map + * iteration order is insertion order and JS re-inserts a key at the end + * on delete+set, which is exactly what touchMetricCache relies on to + * track recency. */ + private metricsCache = new Map(); + private rateLimiters = new WeakMap(); + private hotPathCounter = 0; + + /** Set true only when migrate() actually executed the DDL block during + * this construct, as opposed to finding the store already at + * SCHEMA_VERSION and returning without touching SQLite. Exists purely + * so tests can assert the 10+ `CREATE TABLE/INDEX IF NOT EXISTS` + * statements do NOT re-run on every hibernation wake (DO constructors + * re-run on wake; see the constructor's migrate() call and the + * space-hub.ts cost notes) — never read by production logic. */ + private ranMigrationDdl = false; + + /** Injectable for tests: decides whether the Nth hot-path event gets + * logged. Default samples at <=1%. Errors always log regardless of this + * (see logAlways) — only routine per-frame hot-path logging is sampled. */ + logSampler: (n: number) => boolean = (n) => n % 100 === 0; + + /** Every logAlways() (100%-sampled, security/error tier) event recorded + * by this DO instance, newest last. In-memory only, capped, and never + * persisted or read by production code — it exists purely so + * vitest-pool-workers tests can assert a log fired (e.g. on + * supersession) via `runInDurableObject(stub, (i) => (i as any). + * securityEventLog)`, since console output can't be captured across the + * workerd/vitest process boundary. */ + private securityEventLog: Array<{ event: string; data: Record }> = []; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + // Answers a bare "ping" text frame with "pong" entirely inside the + // runtime, without waking a hibernated DO or invoking webSocketMessage + // (SPEC.md section 9.3) — the whole point of the heartbeat being plain + // text rather than a JSON envelope. + this.ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair("ping", "pong")); + this.migrate(); + } + + /** Runs the schema DDL exactly once per store lifetime, not once per DO + * construct. DO constructors re-run on every hibernation wake (that's + * the whole point of hibernation — the JS instance is cheap to recreate, + * the SQLite store is what's durable), so unconditionally re-issuing + * 10+ `CREATE TABLE/INDEX IF NOT EXISTS` statements here would mean + * every wake pays for a full schema scan, defeating the sparse- + * connection cost goals documented in realtime-server.md. Instead: one + * lightweight version read, and the DDL block (plus the version write) + * only runs when the store is behind — including the fresh-DO case, + * where the version table itself doesn't exist yet. */ + private migrate(): void { + if (this.schemaVersion() >= SpaceHub.SCHEMA_VERSION) return; + + this.ranMigrationDdl = true; + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS _schema_migrations ( + id INTEGER PRIMARY KEY CHECK (id = 1), + version INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS space_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS event_log ( + revision INTEGER PRIMARY KEY, + event_type TEXT NOT NULL, + device_id TEXT, + device_seq INTEGER, + occurred_at INTEGER NOT NULL, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_event_log_device ON event_log(device_id, device_seq); + CREATE TABLE IF NOT EXISTS dedup ( + device_id TEXT NOT NULL, + device_seq INTEGER NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (device_id, device_seq) + ); + CREATE TABLE IF NOT EXISTS tasks ( + task_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + title TEXT, + state TEXT NOT NULL, + percent INTEGER, + step TEXT, + message TEXT, + updated_at INTEGER NOT NULL, + display TEXT + ); + CREATE TABLE IF NOT EXISTS messages ( + message_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + level TEXT NOT NULL, + text TEXT NOT NULL, + occurred_at INTEGER NOT NULL, + revision INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_messages_revision ON messages(revision); + CREATE TABLE IF NOT EXISTS automations ( + automation_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + executor_kind TEXT NOT NULL, + every_seconds INTEGER NOT NULL, + state TEXT NOT NULL, + last_run_at INTEGER + ); + CREATE TABLE IF NOT EXISTS leases ( + device_id TEXT PRIMARY KEY, + expires_at INTEGER NOT NULL, + topics TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS push_outbox ( + event_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS http_idempotency ( + idempotency_key TEXT PRIMARY KEY, + fingerprint TEXT NOT NULL, + revision INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS pending_commands ( + command_id TEXT PRIMARY KEY, + target_device_id TEXT, + origin_ts INTEGER NOT NULL, + ttl_ms INTEGER NOT NULL, + payload TEXT NOT NULL, + delivered INTEGER NOT NULL DEFAULT 0 + ); + `); + // Same synchronous, no-`await`-in-between transaction as the DDL above + // (see the class-level concurrency note) — the version write is + // atomic with the schema it describes. + this.ctx.storage.sql.exec( + `INSERT INTO _schema_migrations (id, version) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET version = excluded.version`, + SpaceHub.SCHEMA_VERSION, + ); + } + + /** One lightweight SELECT: current schema version, or 0 for a fresh DO + * (the `_schema_migrations` table itself doesn't exist yet, which throws + * rather than returning zero rows — SQLite has no "does this table + * exist" query result short of inspecting sqlite_master, and a bare + * SELECT-that-may-throw is cheaper and simpler than checking + * sqlite_master first on every call). */ + private schemaVersion(): number { + try { + const row = this.ctx.storage.sql.exec<{ version: number }>("SELECT version FROM _schema_migrations WHERE id = 1").toArray()[0]; + return row?.version ?? 0; + } catch (e) { + // A missing migrations table means this is a genuinely fresh DO — DDL + // below is IF NOT EXISTS, so re-running it is benign. Any other SQL + // error (corruption, I/O) must not be silently treated as "fresh": + // rethrow so it surfaces instead of being misdiagnosed. + if (String(e).includes("no such table")) return 0; + throw e; + } + } + + // ---- WebSocket entry point ---- + + async fetch(request: Request): Promise { + if ((request.headers.get("upgrade") ?? "").toLowerCase() !== "websocket") { + return new Response("expected websocket upgrade", { status: 426 }); + } + // Trust boundary: these headers are set by the Worker AFTER it already + // validated the st2 token (adapters/workers.ts) — never derived from + // anything the client sent inside the WebSocket connection itself. + const deviceId = request.headers.get("x-sitrep-device-id"); + const role = request.headers.get("x-sitrep-role"); + if (!deviceId || (role !== "source" && role !== "viewer")) { + return new Response("missing trusted identity", { status: 400 }); + } + + const pair = new WebSocketPair(); + const client = pair[0]; + const server = pair[1]; + this.ctx.acceptWebSocket(server, [deviceId]); + const attachment: ConnAttachment = { + deviceId, + role, + connectedAt: Date.now(), + helloDone: false, + subscribedThisConn: false, + deltaEligible: false, + }; + server.serializeAttachment(attachment); + return new Response(null, { status: 101, webSocket: client }); + } + + async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise { + // Top-level exception guard: an unexpected throw anywhere in a message + // handler must degrade to error{internal_error} (SPEC.md section 13: + // retryable, non-fatal, message free of implementation details), never + // kill the DO instance or silently swallow the frame. + try { + this.dispatchMessage(ws, message); + } catch (e) { + this.logAlways("unhandled_error", { message: String(e), stack: (e as Error)?.stack }); + try { + this.reply(ws, "internal_error", "unexpected server error", undefined); + } catch { + // The socket itself may be broken; nothing more to do. + } + } + } + + private dispatchMessage(ws: WebSocket, message: string | ArrayBuffer): void { + if (typeof message !== "string") return; // no binary frames in this protocol + if (message === "ping" || message === "pong") return; // handled by auto-response; defensive no-op + + const raw = ws.deserializeAttachment(); + if (!isConnAttachment(raw)) { + ws.close(1011, "missing connection identity"); + return; + } + const att = raw; + + if (!att.helloDone) { + this.handlePreHello(ws, att, message); + return; + } + + const parsed = parseEnvelope(message); + if (parsed.kind === "unknown_type") return; // SPEC.md section 15: ignore, never malformed + if (parsed.kind === "error") { + this.reply(ws, parsed.code, parsed.message, undefined); + return; + } + const envelope = parsed.envelope; + + // A client-sent hello{stage:"accept"} is a handshake violation at ANY + // point in the connection's life (SPEC.md section 9.1), not just before + // the first accept. + if (envelope.type === "hello") { + this.reply(ws, "hello_required", "stage:accept is server-only", envelope.id); + ws.close(1008, "hello_required"); + return; + } + + const authz = authorizeClientEnvelope(att.role, envelope); + if (!authz.ok) { + this.reply(ws, authz.code, `role ${att.role} may not send ${envelope.type}`, envelope.id); + if (ERROR_SEMANTICS[authz.code].fatal) ws.close(1008, authz.code); + return; + } + + this.logHotPath("frame", { type: envelope.type, device_id: att.deviceId }); + + switch (envelope.type) { + case "resume": + this.handleResume(ws, att, envelope.body as ResumeBody, envelope.id); + break; + case "subscribe": + this.handleSubscribe(ws, att, envelope.body as SubscribeBody, envelope.id); + break; + case "unsubscribe": + this.handleUnsubscribe(ws, att, envelope.id); + break; + case "interest.renew": + this.handleInterestRenew(ws, att, envelope.body as InterestRenewBody, envelope.id); + break; + case "task.event": + this.handleTaskEvent(ws, att, envelope.body as TaskEventBody, envelope.id); + break; + case "message.event": + this.handleMessageEvent(ws, att, envelope.body as MessageEventBody, envelope.id); + break; + case "metric.frame": + this.handleMetricFrame(ws, att, envelope.body as MetricFrameBody, envelope.id); + break; + case "command": + this.handleCommand(ws, att, envelope.body as CommandBody, envelope.ts, envelope.id); + break; + case "ack": + case "error": + // Optional/diagnostic client-sent frames: nothing to durably do. + break; + default: + this.reply(ws, "malformed", `unexpected type ${envelope.type}`, envelope.id); + } + } + + async webSocketClose(): Promise { + try { + // Interest leases and the event log are keyed by device/space, not by + // connection (SPEC.md section 7) — nothing to clean up on close beyond + // letting the socket go. The guard exists so any future cleanup added + // here degrades to a log line instead of an uncaught DO exception. + } catch (e) { + this.logAlways("unhandled_error", { message: String(e), phase: "webSocketClose" }); + } + } + + async webSocketError(_ws: WebSocket, error: unknown): Promise { + this.logAlways("ws_error", { message: String(error) }); + } + + // ---- HTTP control-plane entry point (config.event minting) ---- + + /** Mints (or replays, if idempotencyKey was already seen) a config.event + * for an automation upsert/removal. Single synchronous transaction, same + * discipline as applyReliableEvent — see the class-level comment. + * + * The idempotency key is bound to the request's CONTENT, not just its + * key string: the stored fingerprint is truly canonical JSON — + * recursively key-sorted (see canonicalJson), so it cannot drift if a + * caller's object-construction order ever changes — of + * (kind, automation_id, automation), compared verbatim on a replay. + * (Exact string comparison is strictly stronger than a hash and stays + * synchronous — crypto.subtle.digest is async and would break the + * single-transaction discipline.) A key reused for a DIFFERENT + * operation returns { conflict: true } and mints nothing, instead of + * silently replaying the first operation's result. + * + * Contract note: the fingerprint must cover only client-provided + * content. When the client omits automation_id on a create, the HTTP + * layer derives it deterministically from the Idempotency-Key BEFORE + * calling this method (see adapters/workers.ts deriveAutomationId), so + * an honest retry reconstructs the identical payload and replays with + * the originally-created automation_id rather than tripping a 409. */ + mintConfigEvent( + idempotencyKey: string | null, + payload: { kind: ConfigEventKind; automation_id: string; automation?: AutomationState }, + ): { revision: number; automation: AutomationState | null; conflict?: boolean } { + const fingerprint = canonicalJson({ + kind: payload.kind, + automation_id: payload.automation_id, + automation: payload.automation ?? null, + }); + if (idempotencyKey) { + const existing = this.ctx.storage.sql + .exec<{ revision: number; fingerprint: string }>( + "SELECT revision, fingerprint FROM http_idempotency WHERE idempotency_key = ?", + idempotencyKey, + ) + .toArray()[0]; + if (existing) { + if (existing.fingerprint !== fingerprint) { + return { revision: existing.revision, automation: null, conflict: true }; + } + return { revision: existing.revision, automation: this.readAutomation(payload.automation_id) }; + } + } + + const occurredAt = Date.now(); + const revision = this.getRevision() + 1; + const body: ConfigEventBody = { + kind: payload.kind, + automation_id: payload.automation_id, + automation: payload.automation, + occurred_at: occurredAt, + }; + const eventPayload = JSON.stringify(body); + + this.ctx.storage.sql.exec( + `INSERT INTO event_log (revision, event_type, device_id, device_seq, occurred_at, payload) VALUES (?, 'config.event', NULL, NULL, ?, ?)`, + revision, + occurredAt, + eventPayload, + ); + this.foldConfigEvent(body); + this.setRevision(revision); + if (idempotencyKey) { + this.ctx.storage.sql.exec( + `INSERT INTO http_idempotency (idempotency_key, fingerprint, revision, created_at) VALUES (?, ?, ?, ?)`, + idempotencyKey, + fingerprint, + revision, + occurredAt, + ); + // Bounded growth, cleaned lazily on the write path (no alarm): drop + // entries older than 24h, and cap the table at the most recent 500 + // rows. A retry arriving after both windows re-executes as a fresh + // request — acceptable, since config.event minting is idempotent at + // the state level (upsert/delete) even without the key. + this.ctx.storage.sql.exec(`DELETE FROM http_idempotency WHERE created_at < ?`, occurredAt - 24 * 3600 * 1000); + this.ctx.storage.sql.exec( + `DELETE FROM http_idempotency WHERE idempotency_key NOT IN ( + SELECT idempotency_key FROM http_idempotency ORDER BY created_at DESC LIMIT 500 + )`, + ); + } + this.pruneEventLog(revision); + + this.broadcastDelta({ + from_revision: revision - 1, + to_revision: revision, + events: [{ event_type: "config.event", event: body }], + }); + + return { revision, automation: this.readAutomation(payload.automation_id) }; + } + + automationsSnapshot(): AutomationState[] { + return this.ctx.storage.sql.exec("SELECT * FROM automations ORDER BY automation_id").toArray().map(rowToAutomationState); + } + + // ---- hello / handshake ---- + + private handlePreHello(ws: WebSocket, att: ConnAttachment, message: string): void { + const parsed = parseEnvelope(message); + const offerBody = + parsed.kind === "ok" && parsed.envelope.type === "hello" && (parsed.envelope.body as HelloOfferBody).stage === "offer" + ? (parsed.envelope.body as HelloOfferBody) + : null; + if (!offerBody) { + this.reply(ws, "hello_required", "hello{stage:offer} must be the first frame", undefined); + ws.close(1008, "hello_required"); + return; + } + + const intersection = offerBody.protocol_versions.filter((v) => SUPPORTED_PROTOCOL_VERSIONS.includes(v)); + if (intersection.length === 0) { + this.reply(ws, "version_unsupported", "no shared protocol version", parsed.kind === "ok" ? parsed.envelope.id : undefined); + ws.close(1008, "version_unsupported"); + return; + } + const negotiated = Math.max(...intersection); + + // SPEC.md section 9.4: the new connection completing hello supersedes + // any other open connection already authenticated for this device_id. + const newSessionId = crypto.randomUUID(); + for (const other of this.ctx.getWebSockets(att.deviceId)) { + if (other === ws) continue; + const otherAtt = other.deserializeAttachment(); + if (isConnAttachment(otherAtt) && otherAtt.helloDone) { + // Product invariant 5: superseded must never be silent — it is a + // security-relevant event (possible credential/session reuse), so it + // always logs at 100% via logAlways. Logged BEFORE the wire reply + // below: if `this.send(other, ...)` were to throw (e.g. a socket + // that's already gone), the security log must still have fired — + // it must never depend on the send succeeding. The reply itself + // stays byte-identical for the superseded peer in the success path. + this.logAlways("superseded", { + device_id: att.deviceId, + role: att.role, + superseded_session_id: otherAtt.sessionId, + superseding_session_id: newSessionId, + }); + this.send(other, "error", { + code: "superseded", + message: "device completed hello on a newer connection", + ...ERROR_SEMANTICS.superseded, + }); + other.close(1008, "superseded"); + } + } + + att.helloDone = true; + att.sessionId = newSessionId; + ws.serializeAttachment(att); + + this.send(ws, "hello", { + stage: "accept", + protocol_version: negotiated, + session_id: att.sessionId, + heartbeat_interval_ms: HEARTBEAT_INTERVAL_MS, + }); + + if (att.role === "source") { + // Interest-state sync on (re)connect. SPEC.md section 7's throttle/ + // resume_rate are pure edge triggers, so a source that was offline + // when the space's lease count crossed an edge would otherwise stay + // stuck at its last-known rate forever (e.g. throttled while viewers + // are actively watching). Immediately after the accept, tell this + // connection the CURRENT interest state. This adds no new message + // type or schema change — it reuses command{origin:server} with a + // fresh command_id — and is a v1.1 protocol clarification candidate + // (ruled by the protocol owner; see the handoff notes). + this.sendCurrentRateState(ws); + this.drainPendingCommands(ws, att.deviceId); + } + this.reconcileLeaseEdge(); + } + + // ---- resume / snapshot / delta ---- + + private handleResume(ws: WebSocket, att: ConnAttachment, body: ResumeBody, envelopeId: string): void { + if (!att.subscribedThisConn) { + this.reply(ws, "malformed", "resume must follow subscribe on the same connection", envelopeId); + return; + } + const N = body.last_revision; + const R = this.getRevision(); + + if (N > R) { + this.reply(ws, "revision_unavailable", `last_revision ${N} exceeds current revision ${R}`, envelopeId); + return; // deltaEligible stays false — no reply here confers eligibility + } + + if (N === 0) { + this.sendSnapshotChunks(ws, R); + } else if (N === R) { + this.send(ws, "delta", { from_revision: N, to_revision: N, events: [] }); + } else { + const minRetained = this.minRetainedRevision(); + if (minRetained !== null && minRetained <= N + 1) { + this.sendChainedDeltas(ws, N, this.eventLogRange(N, R)); + } else { + this.sendSnapshotChunks(ws, R); + } + } + + att.deltaEligible = true; + ws.serializeAttachment(att); + } + + private sendSnapshotChunks(ws: WebSocket, revision: number): void { + const arrays = this.buildSnapshotArrays(); + const chunks = chunkSnapshot(revision, arrays); + // No `await` in this loop: see the class-level concurrency note. This + // is what guarantees nothing else can interleave an envelope (SPEC.md + // section 6.2) between chunks, including on this same connection. + for (const chunk of chunks) this.send(ws, "snapshot", chunk); + } + + private sendChainedDeltas(ws: WebSocket, fromRevision: number, rows: EventLogRow[]): void { + const events = rows.map(rowToDeltaEventItem); + const deltas = chunkDeltaEvents(fromRevision, events); + for (const delta of deltas) this.send(ws, "delta", delta); + } + + private buildSnapshotArrays() { + const tasks = this.ctx.storage.sql.exec("SELECT * FROM tasks ORDER BY task_id").toArray().map(rowToTaskState); + const automations = this.ctx.storage.sql + .exec("SELECT * FROM automations ORDER BY automation_id") + .toArray() + .map(rowToAutomationState); + const messageRows = this.ctx.storage.sql + .exec("SELECT * FROM messages ORDER BY revision DESC LIMIT ?", MESSAGE_WINDOW) + .toArray(); + messageRows.reverse(); + const messages = messageRows.map(rowToMessageRecord); + const metrics = [...this.metricsCache.values()]; + return { tasks, metrics, messages, automations }; + } + + private minRetainedRevision(): number | null { + const row = this.ctx.storage.sql.exec<{ m: number | null }>("SELECT MIN(revision) as m FROM event_log").toArray()[0]; + return row?.m ?? null; + } + + private eventLogRange(fromExclusive: number, toInclusive: number): EventLogRow[] { + return this.ctx.storage.sql + .exec("SELECT * FROM event_log WHERE revision > ? AND revision <= ? ORDER BY revision", fromExclusive, toInclusive) + .toArray(); + } + + // ---- reliable event application (task.event / message.event) ---- + + private handleTaskEvent(ws: WebSocket, att: ConnAttachment, body: TaskEventBody, envelopeId: string): void { + if (body.device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "device_id does not match the authenticated identity", envelopeId); + return; + } + const { revision, duplicate } = this.applyReliableEvent("task.event", body.device_id, body.device_seq, body, body.occurred_at); + this.send(ws, "ack", { acked: [{ device_id: body.device_id, device_seq: body.device_seq }] }); + if (!duplicate) { + this.broadcastDelta({ from_revision: revision - 1, to_revision: revision, events: [{ event_type: "task.event", event: body }] }); + this.reconcileLeaseEdge(); + } + } + + private handleMessageEvent(ws: WebSocket, att: ConnAttachment, body: MessageEventBody, envelopeId: string): void { + if (body.device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "device_id does not match the authenticated identity", envelopeId); + return; + } + const { revision, duplicate } = this.applyReliableEvent("message.event", body.device_id, body.device_seq, body, body.occurred_at); + this.send(ws, "ack", { acked: [{ device_id: body.device_id, device_seq: body.device_seq }] }); + if (!duplicate) { + this.broadcastDelta({ from_revision: revision - 1, to_revision: revision, events: [{ event_type: "message.event", event: body }] }); + this.reconcileLeaseEdge(); + } + } + + private applyReliableEvent( + eventType: "task.event" | "message.event", + deviceId: string, + deviceSeq: number, + eventBody: TaskEventBody | MessageEventBody, + occurredAt: number, + ): { revision: number; duplicate: boolean } { + const existing = this.ctx.storage.sql + .exec<{ revision: number }>("SELECT revision FROM dedup WHERE device_id = ? AND device_seq = ?", deviceId, deviceSeq) + .toArray()[0]; + if (existing) { + // SPEC.md section 5.2: MUST NOT re-apply, MUST NOT bump revision, + // MUST still ack — the caller sends the ack regardless of `duplicate`. + return { revision: existing.revision, duplicate: true }; + } + + const revision = this.getRevision() + 1; + const payload = JSON.stringify(eventBody); + + this.ctx.storage.sql.exec( + `INSERT INTO event_log (revision, event_type, device_id, device_seq, occurred_at, payload) VALUES (?, ?, ?, ?, ?, ?)`, + revision, + eventType, + deviceId, + deviceSeq, + occurredAt, + payload, + ); + this.ctx.storage.sql.exec(`INSERT INTO dedup (device_id, device_seq, revision) VALUES (?, ?, ?)`, deviceId, deviceSeq, revision); + if (eventType === "task.event") this.foldTaskEvent(eventBody as TaskEventBody); + else this.foldMessageEvent(eventBody as MessageEventBody, revision); + this.setRevision(revision); + // event_id is its own idempotency key: this INSERT only ever runs on + // the non-duplicate branch, so a retried device_seq can never mint a + // second outbox row for the same logical event. + this.ctx.storage.sql.exec( + `INSERT INTO push_outbox (event_id, kind, payload, status, created_at) VALUES (?, ?, ?, 'pending', ?)`, + crypto.randomUUID(), + eventType, + payload, + Date.now(), + ); + this.pruneEventLog(revision); + + return { revision, duplicate: false }; + } + + private foldTaskEvent(body: TaskEventBody): void { + const prev = this.ctx.storage.sql.exec("SELECT * FROM tasks WHERE task_id = ?", body.task_id).toArray()[0]; + + const state = body.kind === "done" ? "done" : body.kind === "failed" ? "failed" : "running"; + + const title = body.title && body.title.length > 0 ? body.title : (prev?.title ?? null); + + let percent = prev?.percent ?? null; + if (state === "running" && body.percent !== undefined) percent = body.percent; + else if (body.kind === "done") percent = 100; + // failed: keep last value (no assignment) + + const step = body.kind === "done" || body.kind === "failed" ? null : (body.step ?? prev?.step ?? null); + const message = body.kind === "done" || body.kind === "failed" ? (body.message ?? null) : null; + const display = body.display ? JSON.stringify(body.display) : (prev?.display ?? null); + + this.ctx.storage.sql.exec( + `INSERT INTO tasks (task_id, device_id, title, state, percent, step, message, updated_at, display) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(task_id) DO UPDATE SET + device_id = excluded.device_id, title = excluded.title, state = excluded.state, + percent = excluded.percent, step = excluded.step, message = excluded.message, + updated_at = excluded.updated_at, display = excluded.display`, + body.task_id, + body.device_id, + title, + state, + percent, + step, + message, + body.occurred_at, + display, + ); + } + + private foldMessageEvent(body: MessageEventBody, revision: number): void { + this.ctx.storage.sql.exec( + `INSERT INTO messages (message_id, device_id, level, text, occurred_at, revision) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(message_id) DO NOTHING`, + body.message_id, + body.device_id, + body.level, + body.text, + body.occurred_at, + revision, + ); + // Bounded window (SPEC.md section 6.4, N=MESSAGE_WINDOW): keep only the + // most recent N rows. No-op while fewer than N rows exist (subquery + // returns NULL, so the DELETE matches nothing). + this.ctx.storage.sql.exec( + `DELETE FROM messages WHERE revision < ( + SELECT revision FROM messages ORDER BY revision DESC LIMIT 1 OFFSET ? + )`, + MESSAGE_WINDOW - 1, + ); + } + + private foldConfigEvent(body: ConfigEventBody): void { + if (body.kind === "automation.upserted" && body.automation) { + const a = body.automation; + this.ctx.storage.sql.exec( + `INSERT INTO automations (automation_id, name, executor_kind, every_seconds, state, last_run_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(automation_id) DO UPDATE SET + name = excluded.name, executor_kind = excluded.executor_kind, every_seconds = excluded.every_seconds, + state = excluded.state, last_run_at = excluded.last_run_at`, + a.automation_id, + a.name, + a.executor_kind, + a.schedule.every_seconds, + a.state, + a.last_run_at ?? null, + ); + } else if (body.kind === "automation.removed") { + this.ctx.storage.sql.exec(`DELETE FROM automations WHERE automation_id = ?`, body.automation_id); + } + } + + private readAutomation(id: string): AutomationState | null { + const row = this.ctx.storage.sql.exec("SELECT * FROM automations WHERE automation_id = ?", id).toArray()[0]; + return row ? rowToAutomationState(row) : null; + } + + private getRevision(): number { + const row = this.ctx.storage.sql.exec<{ value: string }>("SELECT value FROM space_meta WHERE key = 'revision'").toArray()[0]; + return row ? Number(row.value) : 0; + } + + private setRevision(n: number): void { + this.ctx.storage.sql.exec( + `INSERT INTO space_meta (key, value) VALUES ('revision', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + String(n), + ); + } + + private pruneEventLog(currentRevision: number): void { + const floor = currentRevision - EVENT_LOG_RETENTION_REVISIONS; + if (floor > 0) this.ctx.storage.sql.exec(`DELETE FROM event_log WHERE revision <= ?`, floor); + } + + private broadcastDelta(delta: DeltaBody): void { + for (const ws of this.ctx.getWebSockets()) { + const att = ws.deserializeAttachment(); + if (!isConnAttachment(att) || att.role !== "viewer" || !att.deltaEligible) continue; + if (!this.deviceHasActiveLease(att.deviceId)) continue; + this.send(ws, "delta", delta); + } + } + + // ---- metric.frame (best-effort, never persisted, never revisioned) ---- + + private handleMetricFrame(ws: WebSocket, att: ConnAttachment, body: MetricFrameBody, envelopeId: string): void { + if (body.device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "device_id does not match the authenticated identity", envelopeId); + return; + } + const limiter = this.rateLimiterFor(ws); + const now = Date.now(); + limiter.frameTimestamps = limiter.frameTimestamps.filter((t) => now - t < 1000); + if (limiter.frameTimestamps.length >= METRIC_FRAME_RATE_PER_SEC) { + this.reply(ws, "rate_limited", "metric.frame exceeded 10/s on this connection", envelopeId); + return; + } + limiter.frameTimestamps.push(now); + + const accepted: MetricSample[] = []; + for (const sample of body.metrics) { + // SPEC.md section 12: discard any sample whose ts <= the last-applied + // ts for that metric_id. The baseline is SPACE-level (metricsCache, + // keyed by metric_id alone), NOT per connection: staleness must be + // judged across connections and across devices, or a reconnecting or + // second source could replay an older sample past a fresher one. + // Same lifetime as the rest of metricsCache — reset after eviction + // is acceptable (the whole cache is best-effort, section 6.2). + const cached = this.metricsCache.get(sample.metric_id); + if (cached && sample.ts <= cached.ts) continue; + this.touchMetricCache(sample.metric_id, sample); + accepted.push(sample); + } + if (accepted.length > 0) this.broadcastMetricFrame({ device_id: body.device_id, metrics: accepted }); + // Deliberately no this.ctx.storage.sql.exec(...) anywhere in this + // method and no revision change — see the class-level cost notes and + // docs/design/realtime-server.md. + } + + /** Upserts one metric_id into metricsCache, enforcing + * METRIC_CACHE_MAX_METRICS with least-recently-updated eviction. Always + * deletes before re-setting (even for an existing key) so the entry + * moves to the end of Map iteration order — that's what makes "the + * first key in iteration order" equivalent to "the least recently + * updated key" below. */ + private touchMetricCache(metricId: string, sample: MetricSample): void { + this.metricsCache.delete(metricId); + if (this.metricsCache.size >= METRIC_CACHE_MAX_METRICS) { + const oldest = this.metricsCache.keys().next().value; + if (oldest !== undefined) this.metricsCache.delete(oldest); + } + this.metricsCache.set(metricId, sample); + } + + private broadcastMetricFrame(body: MetricFrameBody): void { + const now = Date.now(); + for (const ws of this.ctx.getWebSockets()) { + const att = ws.deserializeAttachment(); + if (!isConnAttachment(att) || att.role !== "viewer" || !att.helloDone) continue; + if (!this.deviceHasMetricInterest(att.deviceId, now)) continue; + this.send(ws, "metric.frame", body); + } + } + + private rateLimiterFor(ws: WebSocket): RateLimiterState { + let state = this.rateLimiters.get(ws); + if (!state) { + state = { frameTimestamps: [] }; + this.rateLimiters.set(ws, state); + } + return state; + } + + // ---- interest lease ---- + + private handleSubscribe(ws: WebSocket, att: ConnAttachment, body: SubscribeBody, envelopeId: string): void { + const expiresAt = this.upsertLease(att.deviceId, body.topics ?? []); + att.subscribedThisConn = true; + ws.serializeAttachment(att); + this.send(ws, "ack", { in_reply_to: envelopeId, lease: { expires_at: expiresAt } }); + this.reconcileLeaseEdge(); + } + + private handleUnsubscribe(ws: WebSocket, att: ConnAttachment, envelopeId: string): void { + this.ctx.storage.sql.exec(`DELETE FROM leases WHERE device_id = ?`, att.deviceId); + this.send(ws, "ack", { in_reply_to: envelopeId }); + this.reconcileLeaseEdge(); + } + + private handleInterestRenew(ws: WebSocket, att: ConnAttachment, body: InterestRenewBody, envelopeId: string): void { + // SPEC.md section 7: interest.renew always wholesale-replaces topics, + // and if no active lease exists it behaves exactly like a fresh + // subscribe — the upsert below does both without special-casing. + const expiresAt = this.upsertLease(att.deviceId, body.topics ?? []); + this.send(ws, "ack", { in_reply_to: envelopeId, lease: { expires_at: expiresAt } }); + this.reconcileLeaseEdge(); + } + + private upsertLease(deviceId: string, topics: string[]): number { + const expiresAt = Date.now() + LEASE_DEFAULT_MS; + this.ctx.storage.sql.exec( + `INSERT INTO leases (device_id, expires_at, topics) VALUES (?, ?, ?) + ON CONFLICT(device_id) DO UPDATE SET expires_at = excluded.expires_at, topics = excluded.topics`, + deviceId, + expiresAt, + JSON.stringify(topics), + ); + return expiresAt; + } + + private deviceHasActiveLease(deviceId: string): boolean { + const row = this.ctx.storage.sql.exec<{ expires_at: number }>("SELECT expires_at FROM leases WHERE device_id = ?", deviceId).toArray()[0]; + return !!row && row.expires_at > Date.now(); + } + + private deviceHasMetricInterest(deviceId: string, now: number): boolean { + const row = this.ctx.storage.sql + .exec<{ expires_at: number; topics: string }>("SELECT expires_at, topics FROM leases WHERE device_id = ?", deviceId) + .toArray()[0]; + if (!row || row.expires_at <= now) return false; + const topics: string[] = JSON.parse(row.topics); + return topics.length === 0 || topics.includes("metric"); + } + + /** Lazy edge-detection (SPEC.md section 7 permits lazy lease expiry): run + * after every lease mutation and every reliable-event application so a + * lapsed lease is never counted active, without any timer/alarm. */ + private reconcileLeaseEdge(): void { + const now = Date.now(); + const activeNow = this.ctx.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM leases WHERE expires_at > ?", now).toArray()[0].n; + const prevRow = this.ctx.storage.sql.exec<{ value: string }>("SELECT value FROM space_meta WHERE key = 'lease_active_count'").toArray()[0]; + const prev = prevRow ? Number(prevRow.value) : 0; + if (activeNow === prev) return; + this.ctx.storage.sql.exec( + `INSERT INTO space_meta (key, value) VALUES ('lease_active_count', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + String(activeNow), + ); + if (prev > 0 && activeNow === 0) this.broadcastServerCommand("throttle"); + else if (prev === 0 && activeNow > 0) this.broadcastServerCommand("resume_rate"); + } + + private broadcastServerCommand(action: "throttle" | "resume_rate"): void { + const body: CommandBody = { command_id: crypto.randomUUID(), origin: "server", action, ttl_ms: 60_000 }; + for (const ws of this.ctx.getWebSockets()) { + const att = ws.deserializeAttachment(); + if (!isConnAttachment(att) || att.role !== "source" || !att.helloDone) continue; + this.send(ws, "command", body); + } + } + + /** Unicast the space's CURRENT interest state to one freshly-connected + * source (see the call site in handlePreHello for why edges alone are + * not enough). Counts only unexpired leases, same rule as + * reconcileLeaseEdge. */ + private sendCurrentRateState(ws: WebSocket): void { + const active = this.ctx.storage.sql + .exec<{ n: number }>("SELECT COUNT(*) as n FROM leases WHERE expires_at > ?", Date.now()) + .toArray()[0].n; + const body: CommandBody = { + command_id: crypto.randomUUID(), + origin: "server", + action: active > 0 ? "resume_rate" : "throttle", + ttl_ms: 60_000, + }; + this.send(ws, "command", body); + } + + // ---- command relay (viewer-issued pause/resume/stop/run_now) ---- + + private handleCommand(ws: WebSocket, att: ConnAttachment, body: CommandBody, envelopeTs: number, envelopeId: string): void { + if (body.issued_by_device_id !== att.deviceId) { + this.reply(ws, "unauthorized", "issued_by_device_id must match the authenticated device", envelopeId); + return; + } + if (Date.now() > envelopeTs + body.ttl_ms) { + this.reply(ws, "command_expired", "command TTL elapsed before relay", envelopeId); + return; + } + + const targets = body.target_device_id ? this.ctx.getWebSockets(body.target_device_id) : this.ctx.getWebSockets(); + let delivered = false; + for (const target of targets) { + const targetAtt = target.deserializeAttachment(); + if (!isConnAttachment(targetAtt) || targetAtt.role !== "source" || !targetAtt.helloDone) continue; + if (body.target_device_id && targetAtt.deviceId !== body.target_device_id) continue; + // Relay rules (SPEC.md section 8): preserve the original ts and + // command_id, only the envelope id is fresh. + this.sendRaw(target, { type: "command", id: newEnvelopeId(), ts: envelopeTs, body }); + delivered = true; + } + if (!delivered) { + this.ctx.storage.sql.exec( + `INSERT INTO pending_commands (command_id, target_device_id, origin_ts, ttl_ms, payload, delivered) + VALUES (?, ?, ?, ?, ?, 0) ON CONFLICT(command_id) DO NOTHING`, + body.command_id, + body.target_device_id ?? null, + envelopeTs, + body.ttl_ms, + JSON.stringify(body), + ); + } + } + + /** Best-effort redelivery of commands that missed a disconnected source + * (SPEC.md section 14: command is "persisted... until delivered or + * expired"). Runs once a source completes hello on a fresh connection. */ + private drainPendingCommands(ws: WebSocket, deviceId: string): void { + const now = Date.now(); + const rows = this.ctx.storage.sql + .exec<{ command_id: string; origin_ts: number; ttl_ms: number; payload: string }>( + `SELECT command_id, origin_ts, ttl_ms, payload FROM pending_commands WHERE delivered = 0 AND (target_device_id = ? OR target_device_id IS NULL)`, + deviceId, + ) + .toArray(); + for (const row of rows) { + if (now > row.origin_ts + row.ttl_ms) { + this.ctx.storage.sql.exec(`DELETE FROM pending_commands WHERE command_id = ?`, row.command_id); + continue; + } + const body = JSON.parse(row.payload) as CommandBody; + this.sendRaw(ws, { type: "command", id: newEnvelopeId(), ts: row.origin_ts, body }); + this.ctx.storage.sql.exec(`UPDATE pending_commands SET delivered = 1 WHERE command_id = ?`, row.command_id); + } + } + + // ---- send helpers ---- + + private send(ws: WebSocket, type: MessageType, body: B): void { + ws.send(JSON.stringify(makeEnvelope(type, body))); + } + + private sendRaw(ws: WebSocket, envelope: AnyEnvelope): void { + ws.send(JSON.stringify(envelope)); + } + + private reply(ws: WebSocket, code: ErrorCode, message: string, inReplyTo: string | undefined): void { + this.sendRaw(ws, makeStandardError(code, message, inReplyTo)); + this.logAlways("protocol_error", { code, message }); + } + + private logHotPath(event: string, data: Record): void { + this.hotPathCounter++; + if (this.logSampler(this.hotPathCounter)) { + console.log(JSON.stringify({ level: "info", event, ...data, ts: Date.now() })); + } + } + + private logAlways(event: string, data: Record): void { + console.log(JSON.stringify({ level: "error", event, ...data, ts: Date.now() })); + this.securityEventLog.push({ event, data }); + // Bounded so a pathological run can't grow this without limit; recent + // history is all tests need, and this is never persisted or read by + // production logic. + if (this.securityEventLog.length > 200) this.securityEventLog.shift(); + } +} + +// Referenced only for the protocol_version constant it advertises via +// hello{stage:accept}; kept here so the export is exercised by typecheck. +export const SPACE_HUB_PROTOCOL_VERSION = PROTOCOL_VERSION; diff --git a/server/src/realtime/types.ts b/server/src/realtime/types.ts new file mode 100644 index 00000000..499caf1d --- /dev/null +++ b/server/src/realtime/types.ts @@ -0,0 +1,272 @@ +// Hand-written TypeScript mirror of proto/realtime/*.schema.json (protocol +// v1, frozen). This file is a DERIVED artifact: proto/realtime/ is the sole +// source of truth and every shape here must match its schemas field for +// field. Any semantic difference from proto/realtime/ is a bug in this file, +// never an intentional deviation — do not "improve" the protocol here. +// +// proto/realtime/ itself MUST NOT be edited as part of server work. + +export type DeviceRole = "source" | "viewer"; +export type MessageLevel = "info" | "warn" | "error"; +export type TaskKind = "started" | "progress" | "step" | "done" | "failed"; +export type TaskRunState = "running" | "done" | "failed"; +export type AutomationExecutorKind = "script" | "agent" | "hybrid"; +export type AutomationRunState = "active" | "paused"; +export type SubscribeTopic = "task" | "metric" | "message"; +export type CommandOrigin = "viewer" | "server"; +export type CommandAction = "pause" | "resume" | "stop" | "run_now" | "throttle" | "resume_rate"; +export type ConfigEventKind = "automation.upserted" | "automation.removed"; + +export type ErrorCode = + | "version_unsupported" + | "hello_required" + | "unauthenticated" + | "unauthorized" + | "malformed" + | "rate_limited" + | "frame_too_large" + | "batch_too_large" + | "revision_unavailable" + | "command_expired" + | "sequence_gap" + | "superseded" + | "internal_error"; + +export const MESSAGE_TYPES = [ + "hello", + "resume", + "snapshot", + "delta", + "ack", + "task.event", + "message.event", + "metric.frame", + "config.event", + "subscribe", + "unsubscribe", + "interest.renew", + "command", + "error", +] as const; + +export type MessageType = (typeof MESSAGE_TYPES)[number]; + +// ---- common.schema.json $defs ---- + +export interface DisplayHints { + icon?: string; + tint?: string; + template?: string; +} + +export interface TaskState { + task_id: string; + device_id: string; + title?: string; + state: TaskRunState; + percent?: number; + step?: string; + message?: string; + updated_at: number; + display?: DisplayHints; +} + +export interface MetricSample { + metric_id: string; + value: string; + label?: string; + ts: number; + display?: DisplayHints; + target?: string; + min?: string; + max?: string; + alert_above?: string; + alert_below?: string; +} + +export interface AutomationSchedule { + kind: "interval"; + every_seconds: number; +} + +export interface AutomationState { + automation_id: string; + name: string; + executor_kind: AutomationExecutorKind; + schedule: AutomationSchedule; + state: AutomationRunState; + last_run_at?: number; +} + +export interface MessageRecord { + message_id: string; + device_id: string; + level: MessageLevel; + text: string; + occurred_at: number; +} + +// ---- message bodies ---- + +export interface HelloOfferBody { + stage: "offer"; + device_id: string; + role: DeviceRole; + protocol_versions: number[]; + capabilities?: string[]; +} + +export interface HelloAcceptBody { + stage: "accept"; + protocol_version: number; + session_id: string; + heartbeat_interval_ms: number; + capabilities?: string[]; +} + +export type HelloBody = HelloOfferBody | HelloAcceptBody; + +export interface ResumeBody { + last_revision: number; +} + +export interface SnapshotBody { + revision: number; + part: number; + final: boolean; + tasks: TaskState[]; + metrics: MetricSample[]; + messages: MessageRecord[]; + automations: AutomationState[]; +} + +export interface TaskEventBody { + device_id: string; + device_seq: number; + task_id: string; + kind: TaskKind; + occurred_at: number; + title?: string; + percent?: number; + step?: string; + message?: string; + display?: DisplayHints; +} + +export interface MessageEventBody { + device_id: string; + device_seq: number; + message_id: string; + level: MessageLevel; + text: string; + occurred_at: number; + automation_id?: string; +} + +export interface ConfigEventBody { + kind: ConfigEventKind; + automation_id: string; + automation?: AutomationState; + occurred_at: number; +} + +export type DeltaEventItem = + | { event_type: "task.event"; event: TaskEventBody } + | { event_type: "message.event"; event: MessageEventBody } + | { event_type: "config.event"; event: ConfigEventBody }; + +export interface DeltaBody { + from_revision: number; + to_revision: number; + events: DeltaEventItem[]; +} + +export interface AckedPair { + device_id: string; + device_seq: number; +} + +export interface LeaseInfo { + expires_at: number; +} + +export interface AckBody { + acked?: AckedPair[]; + in_reply_to?: string; + lease?: LeaseInfo; +} + +export interface MetricFrameBody { + device_id: string; + metrics: MetricSample[]; +} + +export interface SubscribeBody { + topics?: SubscribeTopic[]; +} + +export type UnsubscribeBody = Record; + +export interface InterestRenewBody { + topics?: SubscribeTopic[]; +} + +export interface CommandBody { + command_id: string; + origin: CommandOrigin; + issued_by_device_id?: string; + action: CommandAction; + task_id?: string; + automation_id?: string; + target_device_id?: string; + ttl_ms: number; + params?: Record; +} + +export interface ErrorBody { + code: ErrorCode; + message: string; + in_reply_to?: string; + retryable: boolean; + fatal: boolean; +} + +// ---- envelope ---- + +export interface Envelope { + type: T; + id: string; + ts: number; + body: B; +} + +export type HelloEnvelope = Envelope<"hello", HelloBody>; +export type ResumeEnvelope = Envelope<"resume", ResumeBody>; +export type SnapshotEnvelope = Envelope<"snapshot", SnapshotBody>; +export type DeltaEnvelope = Envelope<"delta", DeltaBody>; +export type AckEnvelope = Envelope<"ack", AckBody>; +export type TaskEventEnvelope = Envelope<"task.event", TaskEventBody>; +export type MessageEventEnvelope = Envelope<"message.event", MessageEventBody>; +export type MetricFrameEnvelope = Envelope<"metric.frame", MetricFrameBody>; +export type ConfigEventEnvelope = Envelope<"config.event", ConfigEventBody>; +export type SubscribeEnvelope = Envelope<"subscribe", SubscribeBody>; +export type UnsubscribeEnvelope = Envelope<"unsubscribe", UnsubscribeBody>; +export type InterestRenewEnvelope = Envelope<"interest.renew", InterestRenewBody>; +export type CommandEnvelope = Envelope<"command", CommandBody>; +export type ErrorEnvelope = Envelope<"error", ErrorBody>; + +export type AnyEnvelope = + | HelloEnvelope + | ResumeEnvelope + | SnapshotEnvelope + | DeltaEnvelope + | AckEnvelope + | TaskEventEnvelope + | MessageEventEnvelope + | MetricFrameEnvelope + | ConfigEventEnvelope + | SubscribeEnvelope + | UnsubscribeEnvelope + | InterestRenewEnvelope + | CommandEnvelope + | ErrorEnvelope; diff --git a/server/test/app.test.ts b/server/test/app.test.ts index f28b370f..38876d55 100644 --- a/server/test/app.test.ts +++ b/server/test/app.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { createApp } from "../src/app.ts"; +import { createApp, parseRealtimeEnabledFlag } from "../src/app.ts"; import { SqliteStore } from "../src/sqlite-store.ts"; function testApp() { @@ -8,6 +8,36 @@ function testApp() { return createApp({ store: () => store }); } +test("v2 snapshot realtime_enabled defaults false when unwired", async () => { + const app = testApp(); + const snapshot = await (await app.request("/v2/snapshot")).json() as { realtime_enabled: boolean }; + assert.equal(snapshot.realtime_enabled, false); +}); + +test("v2 snapshot realtime_enabled reflects the configured var", async () => { + const store = new SqliteStore(":memory:"); + const app = createApp({ store: () => store, realtimeEnabled: () => true }); + const snapshot = await (await app.request("/v2/snapshot")).json() as { realtime_enabled: boolean }; + assert.equal(snapshot.realtime_enabled, true); +}); + +test("parseRealtimeEnabledFlag: only true/'true'/'1' enable, everything else disables", () => { + // Dashboard variable overrides always arrive as strings at runtime even + // though wrangler.jsonc declares REALTIME_ENABLED as boolean — a naive + // Boolean(value) would treat the string "false" as truthy. This parser + // must be a strict allow-list, not a truthiness check. + const enabling: Array = ["true", "TRUE ", " true", "1"]; + for (const value of enabling) { + assert.equal(parseRealtimeEnabledFlag(value), true, `expected ${JSON.stringify(value)} to enable`); + } + assert.equal(parseRealtimeEnabledFlag(true), true); + + const disabling: Array = [false, "false", "0", "", undefined, "yes", "TRUE_ISH", "truex"]; + for (const value of disabling) { + assert.equal(parseRealtimeEnabledFlag(value), false, `expected ${JSON.stringify(value)} to disable`); + } +}); + test("Node ingest returns success and records presence", async () => { const app = testApp(); const response = await app.request("/v2/ingest", { diff --git a/server/test/realtime/chunking.test.ts b/server/test/realtime/chunking.test.ts new file mode 100644 index 00000000..be43afcb --- /dev/null +++ b/server/test/realtime/chunking.test.ts @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { chunkDeltaEvents, chunkSnapshot } from "../../src/realtime/chunking.ts"; +import type { DeltaEventItem, TaskState } from "../../src/realtime/types.ts"; + +test("chunkSnapshot: empty space still yields exactly one final chunk", () => { + const chunks = chunkSnapshot(0, { tasks: [], metrics: [], messages: [], automations: [] }); + assert.equal(chunks.length, 1); + assert.deepEqual(chunks[0], { revision: 0, part: 1, final: true, tasks: [], metrics: [], messages: [], automations: [] }); +}); + +test("chunkSnapshot: splits large task list across multiple parts, same revision, sequential parts, final on last", () => { + const tasks: TaskState[] = Array.from({ length: 4000 }, (_, i) => ({ + task_id: `task-${i}`, + device_id: "mac-01", + state: "running", + updated_at: 1_700_000_000_000 + i, + title: "x".repeat(50), + })); + const chunks = chunkSnapshot(42, { tasks, metrics: [], messages: [], automations: [] }); + assert.ok(chunks.length > 1, "expected more than one chunk for a large task list"); + for (const [i, chunk] of chunks.entries()) { + assert.equal(chunk.revision, 42); + assert.equal(chunk.part, i + 1); + assert.equal(chunk.final, i === chunks.length - 1); + } + const total = chunks.flatMap((c) => c.tasks); + assert.deepEqual(total, tasks, "concatenating chunks must reproduce the full task list, in order"); +}); + +test("chunkDeltaEvents: chains from/to boundaries across chunks", () => { + const events: DeltaEventItem[] = Array.from({ length: 3000 }, (_, i) => ({ + event_type: "task.event", + event: { + device_id: "mac-01", + device_seq: i + 1, + task_id: `run-${i}`, + kind: "progress", + occurred_at: 1_700_000_000_000 + i, + percent: i % 100, + }, + })); + const deltas = chunkDeltaEvents(100, events); + assert.ok(deltas.length > 1); + assert.equal(deltas[0].from_revision, 100); + for (let i = 1; i < deltas.length; i++) { + assert.equal(deltas[i].from_revision, deltas[i - 1].to_revision, "chained deltas must connect exactly"); + } + assert.equal(deltas.at(-1)!.to_revision, 100 + events.length); + for (const d of deltas) assert.equal(d.to_revision - d.from_revision, d.events.length); +}); + +test("chunkDeltaEvents: empty range still yields one delta with matching from/to", () => { + const deltas = chunkDeltaEvents(5, []); + assert.deepEqual(deltas, [{ from_revision: 5, to_revision: 5, events: [] }]); +}); diff --git a/server/test/realtime/fixtures.test.ts b/server/test/realtime/fixtures.test.ts new file mode 100644 index 00000000..35439f89 --- /dev/null +++ b/server/test/realtime/fixtures.test.ts @@ -0,0 +1,72 @@ +// Fixture round-trip test (required coverage item #11): every fixture under +// proto/realtime/fixtures/valid/ and fixtures/scenarios/**/ must decode, +// re-encode, and decode again to the same value; every fixture under +// fixtures/invalid/ must be rejected by our hand-written guards — either by +// parseEnvelope (schema-shape rejection) or by authorizeClientEnvelope (the +// two role-wrapped fixtures that are only invalid given a sender role). +// +// This is a pure-logic test (no Workers runtime needed), so it runs under +// the plain Node test runner alongside the rest of test:unit. +import assert from "node:assert/strict"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import { authorizeClientEnvelope, encodeEnvelope, parseEnvelope } from "../../src/realtime/guards.ts"; +import type { AnyEnvelope, DeviceRole } from "../../src/realtime/types.ts"; + +const FIXTURES_ROOT = join(import.meta.dirname, "..", "..", "..", "proto", "realtime", "fixtures"); + +function listJsonFilesRecursive(dir: string): string[] { + const out: string[] = []; + for (const name of readdirSync(dir)) { + const full = join(dir, name); + if (statSync(full).isDirectory()) out.push(...listJsonFilesRecursive(full)); + else if (name.endsWith(".json")) out.push(full); + } + return out; +} + +const validFiles = [ + ...listJsonFilesRecursive(join(FIXTURES_ROOT, "valid")), + ...listJsonFilesRecursive(join(FIXTURES_ROOT, "scenarios")), +]; +const invalidFiles = listJsonFilesRecursive(join(FIXTURES_ROOT, "invalid")); + +test(`found valid fixtures (${validFiles.length}) and invalid fixtures (${invalidFiles.length})`, () => { + assert.ok(validFiles.length >= 20, "expected at least the 21 valid+scenario fixtures present at freeze time"); + assert.ok(invalidFiles.length >= 14, "expected at least the 15 invalid fixtures present at freeze time"); +}); + +for (const file of validFiles) { + test(`valid fixture decodes and round-trips: ${file.slice(FIXTURES_ROOT.length + 1)}`, () => { + const raw = readFileSync(file, "utf8"); + const first = parseEnvelope(raw); + assert.equal(first.kind, "ok", `expected ok, got ${JSON.stringify(first)}`); + if (first.kind !== "ok") return; + const reEncoded = encodeEnvelope(first.envelope); + const second = parseEnvelope(reEncoded); + assert.equal(second.kind, "ok"); + if (second.kind !== "ok") return; + assert.deepEqual(second.envelope, first.envelope, "round-trip must be lossless"); + }); +} + +for (const file of invalidFiles) { + test(`invalid fixture is rejected: ${file.slice(FIXTURES_ROOT.length + 1)}`, () => { + const raw = JSON.parse(readFileSync(file, "utf8")) as unknown; + const wrapper = raw as { sender_role?: DeviceRole; frame?: AnyEnvelope }; + if (wrapper.sender_role && wrapper.frame) { + // Role-dependent fixture (proto/realtime/SPEC.md section 16): valid + // shape, but only invalid when sent by the given role/authorization + // rule. Body should parse fine; authorization must reject it. + const parsed = parseEnvelope(JSON.stringify(wrapper.frame)); + assert.equal(parsed.kind, "ok", "wrapped fixture body should be schema-valid on its own"); + if (parsed.kind !== "ok") return; + const authz = authorizeClientEnvelope(wrapper.sender_role, parsed.envelope); + assert.equal(authz.ok, false, "expected authorization to reject this role/frame combination"); + } else { + const parsed = parseEnvelope(JSON.stringify(raw)); + assert.notEqual(parsed.kind, "ok", "expected this fixture to be rejected"); + } + }); +} diff --git a/server/test/workers/broadcast.workers.ts b/server/test/workers/broadcast.workers.ts new file mode 100644 index 00000000..88810988 --- /dev/null +++ b/server/test/workers/broadcast.workers.ts @@ -0,0 +1,204 @@ +// Required coverage #4 (metric.frame never writes SQLite / never bumps +// revision), #5 (one metric.frame reaches multiple viewers), #10 (chunked +// snapshot: same revision, sequential parts, final closes it, nothing else +// interleaves). +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; + +function tableCounts(state: any) { + const tables = ["event_log", "dedup", "tasks", "messages", "automations", "leases", "push_outbox", "space_meta"]; + const counts: Record = {}; + for (const t of tables) { + counts[t] = state.storage.sql.exec<{ n: number }>(`SELECT COUNT(*) as n FROM ${t}`).toArray()[0].n; + } + return counts; +} + +describe("metric.frame", () => { + it("never writes SQLite and never advances space_revision", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient, ["metric"]); + await resume(viewerClient, 0); + + const stub = env.SPACE_HUB.getByName(spaceId); + const before = await runInDurableObject(stub, async (_i: SpaceHub, state) => tableCounts(state)); + + sourceClient.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics: [{ metric_id: "cpu.load", value: "42", ts: Date.now() }] }, + }); + const frame = await viewerClient.recv(); + expect(frame.type).toBe("metric.frame"); + expect(frame.body.metrics[0].metric_id).toBe("cpu.load"); + + const after = await runInDurableObject(stub, async (_i: SpaceHub, state) => tableCounts(state)); + expect(after).toEqual(before); + + sourceClient.close(); + viewerClient.close(); + }); + + it("broadcasts one frame to every eligible viewer", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + const viewer2 = await inviteAndJoin("viewer"); + + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const v1 = await connect(viewer.token); + await helloOffer(v1, viewer.device_id, "viewer"); + await subscribe(v1, ["metric"]); + await resume(v1, 0); + + const v2 = await connect(viewer2.token); + await helloOffer(v2, viewer2.device_id, "viewer"); + await subscribe(v2, ["metric"]); + await resume(v2, 0); + + sourceClient.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics: [{ metric_id: "mem.used", value: "1024", ts: Date.now() }] }, + }); + + const [f1, f2] = await Promise.all([v1.recv(), v2.recv()]); + expect(f1.body.metrics[0].value).toBe("1024"); + expect(f2.body.metrics[0].value).toBe("1024"); + + sourceClient.close(); + v1.close(); + v2.close(); + }); + + it("staleness baseline is space-level: an older ts from a second source connection is dropped", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + const source2 = await inviteAndJoin("source"); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient, ["metric"]); + await resume(viewerClient, 0); + + const s1 = await connect(source.token); + await helloSource(s1, source.device_id); + const s2 = await connect(source2.token); + await helloSource(s2, source2.device_id); + + const t = Date.now(); + // Source 1 reports cpu.load at t+1000. + s1.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics: [{ metric_id: "cpu.load", value: "80", ts: t + 1000 }] }, + }); + const f1 = await viewerClient.recv(); + expect(f1.body.metrics[0].value).toBe("80"); + + // Source 2 (different device, different connection) reports the SAME + // metric_id with an OLDER ts — SPEC.md section 12's discard rule must + // apply across connections/devices, so the viewer sees nothing. + s2.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source2.device_id, metrics: [{ metric_id: "cpu.load", value: "10", ts: t + 500 }] }, + }); + await viewerClient.expectSilence(200); + + // A genuinely fresher sample from source 2 goes through. + s2.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source2.device_id, metrics: [{ metric_id: "cpu.load", value: "55", ts: t + 2000 }] }, + }); + const f2 = await viewerClient.recv(); + expect(f2.body.metrics[0].value).toBe("55"); + + s1.close(); + s2.close(); + viewerClient.close(); + }); +}); + +describe("chunked snapshot", () => { + it("splits a large snapshot into parts sharing one revision, sequential, final last, with nothing else interleaved", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + + // Seed enough state directly (fast + precise) to force multiple chunks + // well under the 64 KiB frame limit's soft chunk threshold. + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_i: SpaceHub, state) => { + const bigTitle = "x".repeat(2000); + for (let i = 0; i < 60; i++) { + state.storage.sql.exec( + `INSERT INTO tasks (task_id, device_id, title, state, percent, step, message, updated_at, display) + VALUES (?, ?, ?, 'running', 50, NULL, NULL, ?, NULL)`, + `run-${i}`, + source.device_id, + bigTitle, + Date.now(), + ); + } + state.storage.sql.exec( + "INSERT INTO space_meta (key, value) VALUES ('revision', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ); + // A single reliable event so a live delta exists to race against the + // chunk stream below. + state.storage.sql.exec( + `INSERT INTO event_log (revision, event_type, device_id, device_seq, occurred_at, payload) VALUES (1, 'task.event', ?, 1, ?, ?)`, + source.device_id, + Date.now(), + JSON.stringify({ device_id: source.device_id, device_seq: 1, task_id: "run-0", kind: "started", occurred_at: Date.now() }), + ); + state.storage.sql.exec(`INSERT INTO dedup (device_id, device_seq, revision) VALUES (?, 1, 1)`, source.device_id); + }); + + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient); + + viewerClient.send({ type: "resume", id: nextId(), ts: Date.now(), body: { last_revision: 0 } }); + + // Race a live event against the in-flight chunked snapshot: if the + // server ever interleaved it between chunks, the assertions below + // would see a non-"snapshot" envelope before `final`. + sourceClient.send({ + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 2, task_id: "run-1", kind: "done", occurred_at: Date.now() }, + }); + + let final = false; + let partCount = 0; + let sawRevision: number | null = null; + while (!final) { + const msg = await viewerClient.recv(); + expect(msg.type).toBe("snapshot"); + partCount += 1; + expect(msg.body.part).toBe(partCount); + if (sawRevision === null) sawRevision = msg.body.revision; + expect(msg.body.revision).toBe(sawRevision); + final = msg.body.final; + } + expect(partCount).toBeGreaterThan(1); + + sourceClient.close(); + viewerClient.close(); + }); +}); diff --git a/server/test/workers/config-event.workers.ts b/server/test/workers/config-event.workers.ts new file mode 100644 index 00000000..4f1f96d7 --- /dev/null +++ b/server/test/workers/config-event.workers.ts @@ -0,0 +1,188 @@ +// The /v3 HTTP automation control-plane companion: mints a config.event in +// the same durable transaction as the revision bump, and a retried request +// carrying the same Idempotency-Key must not mint twice (SPEC.md section +// 5.5). Also checks the automation reaches a subscribed viewer as a live +// delta, same as any other reliable event. +import { SELF } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, connect, helloOffer, resume, subscribe } from "./helpers"; + +const ORIGIN = "https://example.com"; + +describe("/v3/automations control plane", () => { + it("upserting twice with the same Idempotency-Key mints exactly one config.event", async () => { + const { ownerToken } = await bootstrapSpace(); + const payload = { + automation_id: "auto-1", + name: "disk watch", + executor_kind: "script", + schedule: { every_seconds: 60 }, + }; + const headers = { + authorization: `Bearer ${ownerToken}`, + "content-type": "application/json", + "idempotency-key": "req-abc", + }; + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: JSON.stringify(payload) }); + const body1 = (await res1.json()) as { revision: number }; + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: JSON.stringify(payload) }); + const body2 = (await res2.json()) as { revision: number }; + expect(body2.revision).toBe(body1.revision); + }); + + it("an upsert reaches a subscribed viewer as a live delta carrying a config.event", async () => { + const { viewer, ownerToken } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + await resume(client, 0); + + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: "auto-2", name: "log rotate", executor_kind: "script", schedule: { every_seconds: 30 } }), + }); + expect(res.status).toBe(200); + + const delta = await client.recv(); + expect(delta.type).toBe("delta"); + expect(delta.body.events[0].event_type).toBe("config.event"); + expect(delta.body.events[0].event.automation_id).toBe("auto-2"); + client.close(); + }); + + it("rejects an upsert from a viewer-role token (only owner/admin may create)", async () => { + const { viewer } = await bootstrapSpace(); + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${viewer.token}`, "content-type": "application/json" }, + body: JSON.stringify({ name: "x", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(res.status).toBe(403); + }); + + it("reusing an Idempotency-Key for a DIFFERENT operation returns 409 and mints nothing", async () => { + const { ownerToken } = await bootstrapSpace(); + const headers = { + authorization: `Bearer ${ownerToken}`, + "content-type": "application/json", + "idempotency-key": "req-reused", + }; + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ automation_id: "auto-x", name: "first op", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { revision: number }; + + // Same key, different content — must be rejected, NOT silently + // replayed as the first operation's result. + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ automation_id: "auto-y", name: "second op", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(res2.status).toBe(409); + + // Nothing was minted for the conflicting request: a fresh operation + // lands at exactly body1.revision + 1. + const res3 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: "auto-z", name: "third op", executor_kind: "script", schedule: { every_seconds: 30 } }), + }); + const body3 = (await res3.json()) as { revision: number }; + expect(body3.revision).toBe(body1.revision + 1); + }); + + it("retrying a create WITHOUT an explicit automation_id under the same key replays the same automation, not 409", async () => { + const { ownerToken } = await bootstrapSpace(); + const headers = { + authorization: `Bearer ${ownerToken}`, + "content-type": "application/json", + "idempotency-key": "req-serverid", + }; + // No automation_id in the body: the server picks one — deterministically + // from the Idempotency-Key, so a retry of a dropped response must + // replay the original result (same automation_id, same revision). + const payload = JSON.stringify({ name: "no explicit id", executor_kind: "script", schedule: { every_seconds: 60 } }); + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: payload }); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { revision: number; automation: { automation_id: string } }; + expect(body1.automation.automation_id.length).toBeGreaterThan(0); + + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations`, { method: "POST", headers, body: payload }); + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { revision: number; automation: { automation_id: string } }; + expect(body2.revision).toBe(body1.revision); + expect(body2.automation.automation_id).toBe(body1.automation.automation_id); + + // Same key with DIFFERENT client content still conflicts. + const res3 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ name: "changed content", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(res3.status).toBe(409); + }); + + it("role matrix: viewer may PATCH and DELETE but not POST", async () => { + const { viewer, ownerToken } = await bootstrapSpace(); + // Owner creates two automations for the viewer to act on. + for (const id of ["auto-p", "auto-d"]) { + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: id, name: id, executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(res.status).toBe(200); + } + const viewerHeaders = { authorization: `Bearer ${viewer.token}`, "content-type": "application/json" }; + + // Viewer PATCH (pause + reschedule) succeeds — intentional design: + // watcher schedules are editable from every device. + const patchRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-p`, { + method: "PATCH", + headers: viewerHeaders, + body: JSON.stringify({ state: "paused", schedule: { every_seconds: 120 } }), + }); + expect(patchRes.status).toBe(200); + const patched = (await patchRes.json()) as { automation: { state: string; schedule: { every_seconds: number } } }; + expect(patched.automation.state).toBe("paused"); + expect(patched.automation.schedule.every_seconds).toBe(120); + + // Viewer DELETE succeeds (aligned with /v2 canView's delete). + const deleteRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-d`, { method: "DELETE", headers: viewerHeaders }); + expect(deleteRes.status).toBe(200); + + // Viewer POST (creation) stays forbidden — creation is trusted-device + // only. + const postRes = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: viewerHeaders, + body: JSON.stringify({ name: "nope", executor_kind: "script", schedule: { every_seconds: 5 } }), + }); + expect(postRes.status).toBe(403); + }); + + it("DELETE of a nonexistent automation returns 404 without burning a revision", async () => { + const { ownerToken } = await bootstrapSpace(); + const headers = { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }; + const res1 = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers, + body: JSON.stringify({ automation_id: "auto-real", name: "real", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + const body1 = (await res1.json()) as { revision: number }; + + const missingRes = await SELF.fetch(`${ORIGIN}/v3/automations/no-such-automation`, { method: "DELETE", headers }); + expect(missingRes.status).toBe(404); + + // The 404 minted no config.event: the next real operation lands at + // exactly body1.revision + 1. + const res2 = await SELF.fetch(`${ORIGIN}/v3/automations/auto-real`, { method: "DELETE", headers }); + const body2 = (await res2.json()) as { revision: number }; + expect(body2.revision).toBe(body1.revision + 1); + }); +}); diff --git a/server/test/workers/env.d.ts b/server/test/workers/env.d.ts new file mode 100644 index 00000000..04397359 --- /dev/null +++ b/server/test/workers/env.d.ts @@ -0,0 +1,7 @@ +// Type-only ambient declaration for `cloudflare:test` in this directory. +// Not part of the root `tsc --noEmit` project (see tsconfig.json's +// "exclude") — vitest itself transpiles without type-checking, so this +// file exists purely for editor ergonomics when working on these tests. +declare module "cloudflare:test" { + interface ProvidedEnv extends Env {} +} diff --git a/server/test/workers/errors.workers.ts b/server/test/workers/errors.workers.ts new file mode 100644 index 00000000..54182ada --- /dev/null +++ b/server/test/workers/errors.workers.ts @@ -0,0 +1,44 @@ +// Top-level exception guard: an unexpected throw inside any message handler +// must degrade to error{internal_error} (retryable, non-fatal, no internal +// details leaked per SPEC.md section 13) with the connection left usable. +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, nextId, subscribe } from "./helpers"; + +describe("internal_error degradation", () => { + it("an injected handler exception yields error{internal_error} without closing the connection or leaking details", async () => { + const { spaceId, viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + + // Inject a fault into the live DO instance (same isolate as the open + // connection in vitest-pool-workers): the next subscribe throws. + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (instance: SpaceHub) => { + (instance as any).handleSubscribe = () => { + throw new Error("boom /internal/secret/path/space-hub.ts:123"); + }; + }); + + client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: {} }); + const err = await client.recv(); + expect(err.type).toBe("error"); + expect(err.body.code).toBe("internal_error"); + expect(err.body.retryable).toBe(true); + expect(err.body.fatal).toBe(false); + expect(err.body.message).not.toContain("boom"); + expect(err.body.message).not.toContain("space-hub"); + + // Remove the injected fault (restores the prototype method) and verify + // the same connection still works — internal_error is non-fatal. + await runInDurableObject(stub, async (instance: SpaceHub) => { + delete (instance as any).handleSubscribe; + }); + const ack = await subscribe(client); + expect(ack.type).toBe("ack"); + expect(ack.body.lease.expires_at).toBeGreaterThan(Date.now()); + + client.close(); + }); +}); diff --git a/server/test/workers/handshake.workers.ts b/server/test/workers/handshake.workers.ts new file mode 100644 index 00000000..1fd15adc --- /dev/null +++ b/server/test/workers/handshake.workers.ts @@ -0,0 +1,140 @@ +// Required coverage #8 (connection gating), #9 (supersede). +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, subscribe } from "./helpers"; + +describe("connection gating", () => { + it("any frame before hello offer gets hello_required and the connection closes", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + // Send subscribe before ever sending hello{stage:offer}. + client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: {} }); + const err = await client.recv(); + expect(err.type).toBe("error"); + expect(err.body.code).toBe("hello_required"); + expect(err.body.fatal).toBe(true); + await client.waitForClose(); + }); + + it("resume before subscribe on the same connection is rejected as malformed, not silently accepted", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + client.send({ type: "resume", id: nextId(), ts: Date.now(), body: { last_revision: 0 } }); + const err = await client.recv(); + expect(err.type).toBe("error"); + expect(err.body.code).toBe("malformed"); + expect(err.body.retryable).toBe(true); + expect(err.body.fatal).toBe(false); + client.close(); + }); + + it("no live delta reaches a connection before its own resume reply", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient); + // subscribe is acked, but the connection is not yet delta-eligible: a + // reliable event applied right now must produce nothing on this socket + // until resume's reply arrives. + // subscribe just fired the space's 0->1 lease edge, which notifies every + // connected source with command{resume_rate} — drain it before relying + // on ordered recv() below. + await sourceClient.recv(); + + sourceClient.send({ + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 1, task_id: "run-1", kind: "started", occurred_at: Date.now() }, + }); + const ack = await sourceClient.recv(); + expect(ack.type).toBe("ack"); + + await viewerClient.expectSilence(200); + + const reply = await resume(viewerClient, 0); + expect(reply.type).toBe("snapshot"); + sourceClient.close(); + viewerClient.close(); + }); + + it("supersede: an older connection for the same device is closed with `superseded`, not `throttle`", async () => { + const { source, viewer, spaceId } = await bootstrapSpace(); + + const first = await connect(source.token); + await helloSource(first, source.device_id); + + // An unrelated viewer subscribes throughout — if supersession ever + // spuriously toggled the lease count, this connection would see a + // stray throttle/resume_rate command it never asked for. + const bystander = await connect(viewer.token); + await helloOffer(bystander, viewer.device_id, "viewer"); + await subscribe(bystander); + // That subscribe just fired the space's 0->1 lease edge, notifying + // every connected source (including `first`) with resume_rate — drain + // it before asserting on `first`'s next message. + const resumeRate = await first.recv(); + expect(resumeRate.body.action).toBe("resume_rate"); + + const second = await connect(source.token); + await helloSource(second, source.device_id); + + const supersededErr = await first.recv(); + expect(supersededErr.type).toBe("error"); + expect(supersededErr.body.code).toBe("superseded"); + expect(supersededErr.body.fatal).toBe(true); + await first.waitForClose(); + + // Product invariant 5: supersession must never be silent — it's a + // security-relevant event (possible credential/session reuse) and must + // always produce a server-side log, independent of the wire error sent + // above. Console output can't be captured across the workerd/vitest + // process boundary, so inspect the DO's in-memory security event log + // directly via runInDurableObject (same pattern used in errors.workers.ts). + const stub = env.SPACE_HUB.getByName(spaceId); + const securityEvents = await runInDurableObject( + stub, + async (instance: SpaceHub) => (instance as any).securityEventLog as Array<{ event: string; data: Record }>, + ); + const supersededLog = securityEvents.find((e) => e.event === "superseded"); + expect(supersededLog).toBeDefined(); + expect(supersededLog?.data.device_id).toBe(source.device_id); + expect(supersededLog?.data.role).toBe("source"); + + // The new connection is fully functional. + second.send({ + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 1, task_id: "run-1", kind: "started", occurred_at: Date.now() }, + }); + const ack = await second.recv(); + expect(ack.type).toBe("ack"); + expect(ack.body.acked).toEqual([{ device_id: source.device_id, device_seq: 1 }]); + + await bystander.expectSilence(200); + + second.close(); + bystander.close(); + }); + + it("version_unsupported when the offer shares no protocol version with the server", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + client.send({ + type: "hello", + id: nextId(), + ts: Date.now(), + body: { stage: "offer", device_id: viewer.device_id, role: "viewer", protocol_versions: [99] }, + }); + const err = await client.recv(); + expect(err.body.code).toBe("version_unsupported"); + expect(err.body.fatal).toBe(true); + await client.waitForClose(); + }); +}); diff --git a/server/test/workers/helpers.ts b/server/test/workers/helpers.ts new file mode 100644 index 00000000..96b5bce8 --- /dev/null +++ b/server/test/workers/helpers.ts @@ -0,0 +1,226 @@ +// Shared helpers for the vitest-pool-workers realtime test suite. Not a +// test file itself (doesn't match vitest.config.ts's `*.workers.ts` +// include... actually it does match the extension but has no test()/ +// describe() calls, so it just runs as an empty, harmless module if vitest +// ever loads it directly). +import { SELF } from "cloudflare:test"; + +const ORIGIN = "https://example.com"; + +export interface JoinedDevice { + token: string; + device_id: string; + role: string; +} + +export interface Bootstrapped { + spaceId: string; + ownerToken: string; + source: JoinedDevice; + viewer: JoinedDevice; + inviteAndJoin: (role: "source" | "viewer") => Promise; +} + +/** Creates a fresh space (via the existing /v2 pairing flow) with one + * source device and one viewer device already joined, plus a helper to + * mint more devices of either role. */ +export async function bootstrapSpace(): Promise { + const createRes = await SELF.fetch(`${ORIGIN}/v2/spaces`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ platform: "test", name: "owner-mac" }), + }); + if (createRes.status !== 200) throw new Error(`space creation failed: ${createRes.status} ${await createRes.text()}`); + const { space_id, owner_token } = (await createRes.json()) as { space_id: string; owner_token: string }; + + const inviteAndJoin = async (role: "source" | "viewer"): Promise => { + const inviteRes = await SELF.fetch(`${ORIGIN}/v2/invites`, { + method: "POST", + headers: { authorization: `Bearer ${owner_token}`, "content-type": "application/json" }, + body: JSON.stringify({ role }), + }); + if (inviteRes.status !== 200) throw new Error(`invite failed: ${inviteRes.status} ${await inviteRes.text()}`); + const { code } = (await inviteRes.json()) as { code: string }; + const joinRes = await SELF.fetch(`${ORIGIN}/v2/join`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ code, space: space_id, name: `${role}-device`, platform: "test" }), + }); + if (joinRes.status !== 200) throw new Error(`join failed: ${joinRes.status} ${await joinRes.text()}`); + return (await joinRes.json()) as JoinedDevice; + }; + + const source = await inviteAndJoin("source"); + const viewer = await inviteAndJoin("viewer"); + return { spaceId: space_id, ownerToken: owner_token, source, viewer, inviteAndJoin }; +} + +/** Opens a /v3/realtime WebSocket authenticated with the given token. + * Returns the raw upgrade Response so callers can assert on failed + * upgrades (e.g. invalid token -> 401, no `webSocket`). */ +export async function upgrade(token: string | null): Promise { + const headers = new Headers({ upgrade: "websocket" }); + if (token !== null) headers.set("authorization", `Bearer ${token}`); + return SELF.fetch(`${ORIGIN}/v3/realtime`, { headers }); +} + +/** Thin promise-based wrapper around a Workers-runtime client WebSocket: + * queues incoming text frames so `recv()` can await the next one in order, + * exactly as a real client would process them. */ +export class WsClient { + readonly ws: WebSocket; + private queue: string[] = []; + private waiters: Array<(v: string) => void> = []; + private closed: { code: number; reason: string } | null = null; + private closeWaiters: Array<(v: { code: number; reason: string }) => void> = []; + + constructor(ws: WebSocket) { + this.ws = ws; + ws.accept(); + ws.addEventListener("message", (evt: MessageEvent) => { + const data = typeof evt.data === "string" ? evt.data : new TextDecoder().decode(evt.data as ArrayBuffer); + const waiter = this.waiters.shift(); + if (waiter) waiter(data); + else this.queue.push(data); + }); + ws.addEventListener("close", (evt: CloseEvent) => { + this.closed = { code: evt.code, reason: evt.reason }; + for (const w of this.closeWaiters.splice(0)) w(this.closed); + }); + } + + send(envelope: unknown): void { + this.ws.send(JSON.stringify(envelope)); + } + + sendRaw(text: string): void { + this.ws.send(text); + } + + recvRaw(timeoutMs = 10_000): Promise { + const queued = this.queue.shift(); + if (queued !== undefined) return Promise.resolve(queued); + return new Promise((resolve, reject) => { + const waiter = (v: string) => { + clearTimeout(timer); + resolve(v); + }; + const timer = setTimeout(() => { + // Critical: drop this waiter on timeout, or a message that arrives + // later (e.g. right after an expectSilence() timeout) would be + // handed to this already-settled promise and silently vanish + // instead of reaching the next real recv()/expectSilence() call. + const idx = this.waiters.indexOf(waiter); + if (idx !== -1) this.waiters.splice(idx, 1); + reject(new Error(`timed out waiting for a message after ${timeoutMs}ms`)); + }, timeoutMs); + this.waiters.push(waiter); + }); + } + + async recv(timeoutMs = 10_000): Promise { + return JSON.parse(await this.recvRaw(timeoutMs)); + } + + /** Resolves true if no message arrives within timeoutMs (used to assert + * silence — e.g. no live delta before a resume reply). */ + async expectSilence(timeoutMs = 200): Promise { + try { + const msg = await this.recvRaw(timeoutMs); + throw new Error(`expected silence but received: ${msg}`); + } catch (err) { + if (err instanceof Error && err.message.startsWith("timed out")) return true; + throw err; + } + } + + waitForClose(timeoutMs = 10_000): Promise<{ code: number; reason: string }> { + if (this.closed) return Promise.resolve(this.closed); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("timed out waiting for close")), timeoutMs); + this.closeWaiters.push((v) => { + clearTimeout(timer); + resolve(v); + }); + }); + } + + close(): void { + try { + this.ws.close(); + } catch { + // already closed + } + } +} + +export async function connect(token: string): Promise { + const res = await upgrade(token); + if (res.status !== 101 || !res.webSocket) { + throw new Error(`expected a 101 upgrade, got ${res.status}: ${await res.text().catch(() => "")}`); + } + return new WsClient(res.webSocket); +} + +let idCounter = 0; +export function nextId(): string { + idCounter += 1; + return `t${idCounter}`; +} + +export async function helloOffer(client: WsClient, deviceId: string, role: "source" | "viewer"): Promise { + client.send({ + type: "hello", + id: nextId(), + ts: Date.now(), + body: { stage: "offer", device_id: deviceId, role, protocol_versions: [1] }, + }); + return client.recv(); +} + +/** Completes a source's hello AND consumes the interest-state command the + * server sends every source immediately after its accept (throttle when no + * lease is active, resume_rate otherwise). Returns both so tests that care + * about the initial rate state can assert on it. */ +export async function helloSource(client: WsClient, deviceId: string): Promise<{ accept: any; rate: any }> { + const accept = await helloOffer(client, deviceId, "source"); + const rate = await client.recv(); + if (rate.type !== "command" || rate.body.origin !== "server") { + throw new Error(`expected the post-hello interest-state command, got: ${JSON.stringify(rate)}`); + } + return { accept, rate }; +} + +export async function subscribe(client: WsClient, topics?: Array<"task" | "metric" | "message">): Promise { + client.send({ type: "subscribe", id: nextId(), ts: Date.now(), body: topics ? { topics } : {} }); + return client.recv(); +} + +export async function unsubscribe(client: WsClient): Promise { + client.send({ type: "unsubscribe", id: nextId(), ts: Date.now(), body: {} }); + return client.recv(); +} + +export async function resume(client: WsClient, lastRevision: number): Promise { + client.send({ type: "resume", id: nextId(), ts: Date.now(), body: { last_revision: lastRevision } }); + return client.recv(); +} + +export async function sendTaskEvent( + client: WsClient, + deviceId: string, + deviceSeq: number, + overrides: Partial<{ task_id: string; kind: string; occurred_at: number; percent: number; title: string }> = {}, +): Promise { + const body = { + device_id: deviceId, + device_seq: deviceSeq, + task_id: overrides.task_id ?? "run-1", + kind: overrides.kind ?? "started", + occurred_at: overrides.occurred_at ?? Date.now(), + ...(overrides.percent !== undefined ? { percent: overrides.percent } : {}), + ...(overrides.title !== undefined ? { title: overrides.title } : {}), + }; + client.send({ type: "task.event", id: nextId(), ts: Date.now(), body }); + return client.recv(); +} diff --git a/server/test/workers/lease.workers.ts b/server/test/workers/lease.workers.ts new file mode 100644 index 00000000..304f34cc --- /dev/null +++ b/server/test/workers/lease.workers.ts @@ -0,0 +1,95 @@ +// Required coverage #6: the space's active-lease count's 1<->0 edges +// notify every connected source with command{throttle}/command{resume_rate} +// (SPEC.md section 7). We drive the 1->0 edge via explicit `unsubscribe` +// rather than waiting out the real 30-60s lease TTL — SPEC.md section 7 +// explicitly names unsubscribe as one of exactly two ways a lease ends, +// and both paths run through the same reconcileLeaseEdge() call. +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, connect, helloOffer, helloSource, subscribe, unsubscribe } from "./helpers"; + +describe("interest lease edges", () => { + it("0->1 sends resume_rate; 1->0 sends throttle; a later 0->1 sends resume_rate again", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const v1 = await connect(viewer.token); + await helloOffer(v1, viewer.device_id, "viewer"); + await subscribe(v1); // 0 -> 1 + + const resumeRate1 = await sourceClient.recv(); + expect(resumeRate1.type).toBe("command"); + expect(resumeRate1.body.origin).toBe("server"); + expect(resumeRate1.body.action).toBe("resume_rate"); + + await unsubscribe(v1); // 1 -> 0 (only lease in the space) + + const throttle = await sourceClient.recv(); + expect(throttle.type).toBe("command"); + expect(throttle.body.origin).toBe("server"); + expect(throttle.body.action).toBe("throttle"); + + const viewer2 = await inviteAndJoin("viewer"); + const v2 = await connect(viewer2.token); + await helloOffer(v2, viewer2.device_id, "viewer"); + await subscribe(v2); // 0 -> 1 again + + const resumeRate2 = await sourceClient.recv(); + expect(resumeRate2.type).toBe("command"); + expect(resumeRate2.body.action).toBe("resume_rate"); + + sourceClient.close(); + v1.close(); + v2.close(); + }); + + it("a second viewer subscribing while one is already active does not re-fire resume_rate", async () => { + const { source, viewer, inviteAndJoin } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const v1 = await connect(viewer.token); + await helloOffer(v1, viewer.device_id, "viewer"); + await subscribe(v1); // 0 -> 1 + await sourceClient.recv(); // resume_rate + + const viewer2 = await inviteAndJoin("viewer"); + const v2 = await connect(viewer2.token); + await helloOffer(v2, viewer2.device_id, "viewer"); + await subscribe(v2); // 1 -> 2, not an edge + + await sourceClient.expectSilence(200); + + sourceClient.close(); + v1.close(); + v2.close(); + }); + + it("a source receives the current interest state immediately after hello, so a missed edge cannot strand it", async () => { + const { source, viewer } = await bootstrapSpace(); + + // No lease exists yet: a connecting source is told to throttle. + const first = await connect(source.token); + const initial = await helloSource(first, source.device_id); + expect(initial.rate.type).toBe("command"); + expect(initial.rate.body.origin).toBe("server"); + expect(initial.rate.body.action).toBe("throttle"); + first.close(); + + // A viewer subscribes while the source is OFFLINE — the 0->1 + // resume_rate edge fires with nobody to hear it. + const v = await connect(viewer.token); + await helloOffer(v, viewer.device_id, "viewer"); + await subscribe(v); + + // The reconnecting source is immediately told the CURRENT state + // (resume_rate), not left stuck at its last-known throttle. + const second = await connect(source.token); + const after = await helloSource(second, source.device_id); + expect(after.rate.body.action).toBe("resume_rate"); + + second.close(); + v.close(); + }); +}); diff --git a/server/test/workers/metric-cache.workers.ts b/server/test/workers/metric-cache.workers.ts new file mode 100644 index 00000000..03e2686e --- /dev/null +++ b/server/test/workers/metric-cache.workers.ts @@ -0,0 +1,70 @@ +// Required coverage: metricsCache (SpaceHub#metricsCache) must not grow +// without bound. A compromised or misbehaving source rotating metric_ids +// past any real cardinality would otherwise grow DO memory until it +// crashes — see METRIC_CACHE_MAX_METRICS in src/realtime/protocol.ts and +// SpaceHub#touchMetricCache. +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { METRIC_CACHE_MAX_METRICS } from "../../src/realtime/protocol.ts"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, subscribe } from "./helpers"; + +describe("metricsCache cardinality cap", () => { + it("evicts least-recently-updated metric_ids beyond the cap, keeping the most recent", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient, ["metric"]); + + // Rotate far more distinct metric_ids than the cap, staying within the + // 64-samples-per-frame guard (guards.ts) and the 10 frames/s limiter + // (METRIC_FRAME_RATE_PER_SEC) by packing many ids per frame across a + // handful of frames rather of one-id-per-frame. + const perFrame = 64; + const frameCount = 6; // 384 distinct ids, comfortably over the 256 cap + const totalIds = perFrame * frameCount; + for (let f = 0; f < frameCount; f++) { + const metrics = Array.from({ length: perFrame }, (_, i) => { + const n = f * perFrame + i; + return { metric_id: `m${n}`, value: String(n), ts: Date.now() + n }; + }); + sourceClient.send({ + type: "metric.frame", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, metrics }, + }); + // Drain this frame's broadcast to the subscribed viewer before + // sending the next one, so the viewer's recv queue doesn't leak + // across iterations (not load-bearing for the assertion below, just + // hygiene for the WsClient helper). + await viewerClient.recv(); + } + + const stub = env.SPACE_HUB.getByName(spaceId); + const { size, ids } = await runInDurableObject(stub, async (instance: SpaceHub) => { + const cache = (instance as any).metricsCache as Map; + return { size: cache.size, ids: [...cache.keys()] }; + }); + + expect(size).toBe(METRIC_CACHE_MAX_METRICS); + expect(size).toBeLessThan(totalIds); + + // The most-recently-sent ids (the last frame's) must all have + // survived eviction... + for (let n = totalIds - perFrame; n < totalIds; n++) { + expect(ids).toContain(`m${n}`); + } + // ...while the earliest-sent ids (evicted as least-recently-updated) + // must be gone. + for (let n = 0; n < perFrame; n++) { + expect(ids).not.toContain(`m${n}`); + } + + sourceClient.close(); + viewerClient.close(); + }); +}); diff --git a/server/test/workers/migration.workers.ts b/server/test/workers/migration.workers.ts new file mode 100644 index 00000000..1b863c9b --- /dev/null +++ b/server/test/workers/migration.workers.ts @@ -0,0 +1,58 @@ +// Required coverage: DO constructors re-run on every hibernation wake +// (SpaceHub's constructor calls migrate()), so migrate() must not +// unconditionally re-issue its 10+ `CREATE TABLE/INDEX IF NOT EXISTS` +// statements every wake — only when the store's schema version is behind +// what this build expects. See the migrate()/schemaVersion() comments in +// src/realtime/space-hub.ts. +import { env, evictDurableObject, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace } from "./helpers"; + +describe("SpaceHub schema migration", () => { + it("runs the DDL block on a fresh DO (version table missing)", async () => { + const stub = env.SPACE_HUB.getByName(`fresh-${crypto.randomUUID()}`); + const ranDdl = await runInDurableObject(stub, async (instance: SpaceHub) => (instance as any).ranMigrationDdl as boolean); + expect(ranDdl).toBe(true); + + const version = await runInDurableObject(stub, async (_instance: SpaceHub, state) => + state.storage.sql.exec<{ version: number }>("SELECT version FROM _schema_migrations WHERE id = 1").toArray()[0]?.version, + ); + expect(version).toBe(1); + + // The rest of the schema landed too, not just the version table. + const tableExists = await runInDurableObject(stub, async (_instance: SpaceHub, state) => + state.storage.sql + .exec<{ n: number }>("SELECT COUNT(*) as n FROM sqlite_master WHERE type='table' AND name='event_log'") + .toArray()[0].n, + ); + expect(tableExists).toBe(1); + }); + + it("does not re-run the DDL block on a second construct against an already-migrated store", async () => { + // bootstrapSpace() drives a real construct through the app's /v2 + // routes (registry -> UserStore -> SpaceHub stub), exercising the + // exact path a hibernation wake takes in production. + const { spaceId } = await bootstrapSpace(); + const stub = env.SPACE_HUB.getByName(spaceId); + + // bootstrapSpace() only touches UserStore over HTTP — nothing has + // addressed this space's SpaceHub yet, so it isn't "running" for + // evictDurableObject's purposes until something does. + await runInDurableObject(stub, async () => {}); + + // Force a fresh DO *instance* (constructor re-runs) while preserving + // durable SQLite storage — this is precisely what a hibernation wake + // does, and the primitive vitest-pool-workers provides to simulate it. + await evictDurableObject(stub); + + const ranDdl = await runInDurableObject(stub, async (instance: SpaceHub) => (instance as any).ranMigrationDdl as boolean); + expect(ranDdl).toBe(false); + + // Evict and re-check again for good measure: every subsequent wake + // against an up-to-date store must skip the DDL, not just the first. + await evictDurableObject(stub); + const ranDdlAgain = await runInDurableObject(stub, async (instance: SpaceHub) => (instance as any).ranMigrationDdl as boolean); + expect(ranDdlAgain).toBe(false); + }); +}); diff --git a/server/test/workers/realtime-gate.workers.ts b/server/test/workers/realtime-gate.workers.ts new file mode 100644 index 00000000..b9466c08 --- /dev/null +++ b/server/test/workers/realtime-gate.workers.ts @@ -0,0 +1,94 @@ +// P0-1 (server half): REALTIME_ENABLED=false must actually gate /v3/realtime +// and its HTTP control-plane companion, /v3/automations, BEFORE any +// SpaceHub Durable Object is touched — otherwise a daemon with its local +// flag on keeps writing to SpaceHub while viewers read UserStore (dual +// authority). The rest of the suite runs with the test pool's +// REALTIME_ENABLED override (see vitest.config.ts) on, so this file flips +// `env.REALTIME_ENABLED` back to `false` per-test to exercise the gate. +import { env, listDurableObjectIds } from "cloudflare:test"; +import { describe, expect, it, afterEach } from "vitest"; +import { SELF } from "cloudflare:test"; +import { bootstrapSpace, upgrade } from "./helpers"; + +const ORIGIN = "https://example.com"; + +describe("realtime kill switch", () => { + afterEach(() => { + // Restore the pool-wide override so later files/tests aren't affected + // by this file's per-test flips. + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = true; + }); + + it("flag off: /v3/realtime returns 403 realtime_disabled and wakes no SpaceHub", async () => { + const { viewer } = await bootstrapSpace(); + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + const res = await upgrade(viewer.token); + expect(res.status).toBe(403); + expect(res.webSocket).toBeNull(); + expect(await res.json()).toEqual({ error: "realtime_disabled" }); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("flag off: /v3/automations POST returns 403 realtime_disabled and wakes no SpaceHub", async () => { + const { ownerToken } = await bootstrapSpace(); + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + const res = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ name: "disk watch", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(res.status).toBe(403); + expect(await res.json()).toEqual({ error: "realtime_disabled" }); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("flag off: /v3/automations PATCH and DELETE also return 403 realtime_disabled", async () => { + const { ownerToken } = await bootstrapSpace(); + // create it first, while the flag is still on for this test. + const createRes = await SELF.fetch(`${ORIGIN}/v3/automations`, { + method: "POST", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ automation_id: "auto-gate", name: "disk watch", executor_kind: "script", schedule: { every_seconds: 60 } }), + }); + expect(createRes.status).toBe(200); + + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + const patchRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-gate`, { + method: "PATCH", + headers: { authorization: `Bearer ${ownerToken}`, "content-type": "application/json" }, + body: JSON.stringify({ state: "paused" }), + }); + expect(patchRes.status).toBe(403); + expect(await patchRes.json()).toEqual({ error: "realtime_disabled" }); + + const deleteRes = await SELF.fetch(`${ORIGIN}/v3/automations/auto-gate`, { + method: "DELETE", + headers: { authorization: `Bearer ${ownerToken}` }, + }); + expect(deleteRes.status).toBe(403); + expect(await deleteRes.json()).toEqual({ error: "realtime_disabled" }); + }); + + it("flag on: /v3/realtime upgrade still works (explicit regression check)", async () => { + const { source } = await bootstrapSpace(); + const res = await upgrade(source.token); + expect(res.status).toBe(101); + expect(res.webSocket).not.toBeNull(); + res.webSocket?.accept(); + res.webSocket?.close(); + }); + + it("flag off: an unauthenticated request still gets 401, not 403 (no flag-state leak)", async () => { + (env as unknown as { REALTIME_ENABLED: boolean }).REALTIME_ENABLED = false; + + const res = await upgrade(null); + expect(res.status).toBe(401); + const badToken = await upgrade("not-a-real-token"); + expect(badToken.status).toBe(401); + }); +}); diff --git a/server/test/workers/reliability.workers.ts b/server/test/workers/reliability.workers.ts new file mode 100644 index 00000000..6c9b0945 --- /dev/null +++ b/server/test/workers/reliability.workers.ts @@ -0,0 +1,205 @@ +// Required coverage #2 (dedup), #3 (revision gap/current/unavailable), +// #12 (push_outbox idempotency), and #1 (attachment survives +// reconstruction — see the "recovery" test below and its comment). +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import type { SpaceHub } from "../../src/realtime/space-hub.ts"; +import { bootstrapSpace, connect, helloOffer, helloSource, nextId, resume, sendTaskEvent, subscribe } from "./helpers"; + +describe("reliable event dedup and revision accounting", () => { + it("a duplicate device_seq re-acks without applying twice or bumping revision again", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + + const viewerClient = await connect(viewer.token); + await helloOffer(viewerClient, viewer.device_id, "viewer"); + await subscribe(viewerClient); + await resume(viewerClient, 0); // now delta-eligible + // subscribe's 0->1 lease edge notified the source with resume_rate; + // drain it before relying on ordered recv() on sourceClient below. + await sourceClient.recv(); + + const envelope = { + type: "task.event", + id: nextId(), + ts: Date.now(), + body: { device_id: source.device_id, device_seq: 1, task_id: "run-1", kind: "started", occurred_at: Date.now() }, + }; + sourceClient.send(envelope); + const ack1 = await sourceClient.recv(); + expect(ack1.body.acked).toEqual([{ device_id: source.device_id, device_seq: 1 }]); + const delta1 = await viewerClient.recv(); + expect(delta1.type).toBe("delta"); + expect(delta1.body.to_revision).toBe(1); + + // Resend the identical device_seq in a fresh envelope (new id/ts), as a + // real source would after a dropped ack (SPEC.md section 5.4). + sourceClient.send({ ...envelope, id: nextId(), ts: Date.now() }); + const ack2 = await sourceClient.recv(); + expect(ack2.body.acked).toEqual([{ device_id: source.device_id, device_seq: 1 }]); + + // No second delta was broadcast for the retry. + await viewerClient.expectSilence(200); + + // And the space's revision only advanced once. + const current = await resume(viewerClient, 1); + expect(current.type).toBe("delta"); + expect(current.body.from_revision).toBe(1); + expect(current.body.to_revision).toBe(1); + expect(current.body.events).toEqual([]); + + sourceClient.close(); + viewerClient.close(); + }); + + it("event_id in push_outbox is idempotent across a duplicate device_seq retry", async () => { + const { spaceId, source } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + await sendTaskEvent(sourceClient, source.device_id, 1); + await sendTaskEvent(sourceClient, source.device_id, 1); // duplicate + sourceClient.close(); + + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_instance: SpaceHub, state) => { + const outbox = state.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM push_outbox").toArray()[0]; + expect(outbox.n).toBe(1); + const events = state.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM event_log").toArray()[0]; + expect(events.n).toBe(1); + const dedup = state.storage.sql.exec<{ n: number }>("SELECT COUNT(*) as n FROM dedup").toArray()[0]; + expect(dedup.n).toBe(1); + }); + }); +}); + +describe("resume decision table", () => { + it("last_revision 0 on a fresh space yields an (empty) snapshot", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 0); + expect(reply.type).toBe("snapshot"); + expect(reply.body.revision).toBe(0); + expect(reply.body.final).toBe(true); + expect(reply.body.tasks).toEqual([]); + client.close(); + }); + + it("last_revision == current revision yields the explicit empty delta", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + await sendTaskEvent(sourceClient, source.device_id, 1); + + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 1); + expect(reply.type).toBe("delta"); + expect(reply.body.from_revision).toBe(1); + expect(reply.body.to_revision).toBe(1); + expect(reply.body.events).toEqual([]); + sourceClient.close(); + client.close(); + }); + + it("last_revision > current revision yields revision_unavailable and no eligibility", async () => { + const { viewer } = await bootstrapSpace(); + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 999); + expect(reply.type).toBe("error"); + expect(reply.body.code).toBe("revision_unavailable"); + expect(reply.body.retryable).toBe(true); + client.close(); + }); + + it("0 < last_revision < current, fully retained, yields a catch-up delta (not a snapshot)", async () => { + const { source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + await sendTaskEvent(sourceClient, source.device_id, 1, { kind: "started" }); + await sendTaskEvent(sourceClient, source.device_id, 2, { kind: "progress", percent: 50 }); + await sendTaskEvent(sourceClient, source.device_id, 3, { kind: "done" }); + + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 1); + expect(reply.type).toBe("delta"); + expect(reply.body.from_revision).toBe(1); + expect(reply.body.to_revision).toBe(3); + expect(reply.body.events.length).toBe(2); + sourceClient.close(); + client.close(); + }); + + it("a range whose retention was lost falls back to a snapshot instead of an error", async () => { + const { spaceId, source, viewer } = await bootstrapSpace(); + const sourceClient = await connect(source.token); + await helloSource(sourceClient, source.device_id); + await sendTaskEvent(sourceClient, source.device_id, 1); + await sendTaskEvent(sourceClient, source.device_id, 2); + await sendTaskEvent(sourceClient, source.device_id, 3); + sourceClient.close(); + + // Simulate the retention window having moved past revision 1 (the real + // window is 1000 revisions; forcing it directly keeps this test fast + // and precise rather than sending 1000+ real events). + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_instance: SpaceHub, state) => { + state.storage.sql.exec("DELETE FROM event_log WHERE revision <= 2"); + }); + + const client = await connect(viewer.token); + await helloOffer(client, viewer.device_id, "viewer"); + await subscribe(client); + const reply = await resume(client, 1); // wants (1,3], but revision 2 is gone + expect(reply.type).toBe("snapshot"); + expect(reply.body.revision).toBe(3); + client.close(); + }); +}); + +describe("connection identity recovery", () => { + // A true forced eviction/hibernation cycle isn't triggerable from the + // outside in vitest-pool-workers. What we can and do verify: (a) the + // DO's only source of connection identity is the per-connection + // WebSocket attachment (never an in-memory map keyed by connection — + // reviewed in space-hub.ts, every handler calls + // ws.deserializeAttachment() fresh), and (b) all durable facts (revision, + // dedup, folded task state) live in SQLite and are correctly rebuilt by a + // brand new connection/stub reference with no shared JS state, which is + // exactly what a reconstructed DO instance would see after eviction. + it("a fresh connection for the same device, and a fresh stub reference, both observe identical durable state", async () => { + const { spaceId, source } = await bootstrapSpace(); + const first = await connect(source.token); + await helloSource(first, source.device_id); + await sendTaskEvent(first, source.device_id, 1, { kind: "started", title: "build" }); + first.close(); + + // A brand new WebSocket connection (fresh attachment, no memory of the + // previous one) still authenticates correctly and can continue the + // device's device_seq sequence — proving identity isn't cached + // anywhere but the attachment + storage. + const second = await connect(source.token); + const { accept } = await helloSource(second, source.device_id); + expect(accept.body.stage).toBe("accept"); + await sendTaskEvent(second, source.device_id, 2, { kind: "done" }); + second.close(); + + // A fresh DurableObjectStub reference (not reusing any object from + // above) reads the exact same durable state straight from storage. + const stub = env.SPACE_HUB.getByName(spaceId); + await runInDurableObject(stub, async (_instance: SpaceHub, state) => { + const revision = state.storage.sql.exec<{ value: string }>("SELECT value FROM space_meta WHERE key='revision'").toArray()[0]; + expect(Number(revision.value)).toBe(2); + const task = state.storage.sql.exec<{ state: string; title: string }>("SELECT state, title FROM tasks WHERE task_id='run-1'").toArray()[0]; + expect(task.state).toBe("done"); + expect(task.title).toBe("build"); + }); + }); +}); diff --git a/server/test/workers/token-gate.workers.ts b/server/test/workers/token-gate.workers.ts new file mode 100644 index 00000000..cc0ca4d8 --- /dev/null +++ b/server/test/workers/token-gate.workers.ts @@ -0,0 +1,48 @@ +// Required coverage #7: an invalid token gets 401 at the Worker layer +// without ever instantiating a SpaceHub Durable Object, so a reconnect +// storm of bad/garbage tokens cannot spin up empty spaces. +import { env, listDurableObjectIds } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { bootstrapSpace, upgrade } from "./helpers"; + +describe("token gate", () => { + it("rejects a malformed token with 401 and creates no SpaceHub", async () => { + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + const res = await upgrade("not-a-real-token"); + expect(res.status).toBe(401); + expect(res.webSocket).toBeNull(); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("rejects a missing Authorization header with 401 and creates no SpaceHub", async () => { + const res = await upgrade(null); + expect(res.status).toBe(401); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("rejects a well-formed but unresolvable st2 token with 401 and still creates no SpaceHub", async () => { + // Right shape (matches TOKEN_RE), but no device was ever issued this + // secret — resolveToken() legitimately touches UserStore (pre-existing + // behavior), but must never reach SpaceHub. + const res = await upgrade("st2_zzzzzzzzzz_" + "0".repeat(48)); + expect(res.status).toBe(401); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("a reconnect storm of invalid tokens across many spaces never creates a single SpaceHub", async () => { + const attempts = Array.from({ length: 25 }, (_, i) => upgrade(`st2_space${i}_` + "a".repeat(48))); + const results = await Promise.all(attempts); + for (const res of results) expect(res.status).toBe(401); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(0); + }); + + it("a valid token succeeds and creates exactly one SpaceHub for that space", async () => { + const { source } = await bootstrapSpace(); + const res = await upgrade(source.token); + expect(res.status).toBe(101); + expect(res.webSocket).not.toBeNull(); + res.webSocket?.accept(); + res.webSocket?.close(); + expect((await listDurableObjectIds(env.SPACE_HUB)).length).toBe(1); + }); +}); diff --git a/server/tsconfig.json b/server/tsconfig.json index b3c01ce6..6bfe3944 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -9,5 +9,6 @@ "types": ["node"], "allowImportingTsExtensions": true }, - "include": ["worker-configuration.d.ts", "src/**/*.ts", "test/**/*.ts"] + "include": ["worker-configuration.d.ts", "src/**/*.ts", "test/**/*.ts"], + "exclude": ["test/workers/**/*"] } diff --git a/server/vitest.config.ts b/server/vitest.config.ts new file mode 100644 index 00000000..867cd2f2 --- /dev/null +++ b/server/vitest.config.ts @@ -0,0 +1,21 @@ +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["test/workers/**/*.workers.ts"], + }, + plugins: [ + cloudflareTest({ + wrangler: { configPath: "./wrangler.jsonc" }, + // wrangler.jsonc's REALTIME_ENABLED stays "false" as the production + // kill-switch default (flipped per-environment at deploy time), but + // the /v3/realtime protocol suite needs it on to exercise the + // handshake/broadcast/reliability paths — override it here for the + // test pool only. Tests covering the disabled path flip it back to + // `false` per-test via `env.REALTIME_ENABLED` (see + // realtime-gate.workers.ts). + miniflare: { bindings: { REALTIME_ENABLED: true } }, + }), + ], +}); diff --git a/server/worker-configuration.d.ts b/server/worker-configuration.d.ts index 19ed6da5..0c912432 100644 --- a/server/worker-configuration.d.ts +++ b/server/worker-configuration.d.ts @@ -1,16 +1,18 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 73e87c808e9c6c4997497b2a096a4dab) +// Generated by Wrangler by running `wrangler types` (hash: 0eac4d6591f9e4c5dd04df5e8b07aced) // Runtime types generated with workerd@1.20260714.1 2026-07-18 nodejs_compat interface __BaseEnv_Env { INVITE_DIR: KVNamespace; APNS_BUNDLE_ID: "dev.sitrep.app"; APNS_HOST: "api.sandbox.push.apple.com"; + REALTIME_ENABLED: false; USER_STORE: DurableObjectNamespace; + SPACE_HUB: DurableObjectNamespace; } declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/adapters/workers"); - durableNamespaces: "UserStore"; + durableNamespaces: "UserStore" | "SpaceHub"; } interface Env extends __BaseEnv_Env {} } @@ -19,7 +21,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } // Begin runtime types diff --git a/server/wrangler.jsonc b/server/wrangler.jsonc index 6d9a5fba..5260d245 100644 --- a/server/wrangler.jsonc +++ b/server/wrangler.jsonc @@ -4,15 +4,40 @@ "main": "src/adapters/workers.ts", "compatibility_date": "2026-07-18", "compatibility_flags": ["nodejs_compat"], - "observability": { "enabled": true }, + // Observability (Fix B, pre-merge review): "enabled": true alone only + // turns telemetry on — it does not bound invocation-log volume, so the + // documented cost model (docs/design/realtime-server.md) wasn't real. + // logAlways() security events (e.g. "superseded") must never be + // head-sampled away, so `logs.head_sampling_rate` stays at 1.0 (its + // default) and the ≤1% in-code `logSampler` in SpaceHub is what actually + // bounds hot-path console.log volume. `invocation_logs: false` is what + // bounds the OTHER, uncappable log source — the platform's automatic + // per-request invocation log, emitted once per webSocketMessage + // regardless of what the handler logs. Traces are diagnostic-only (no + // logAlways equivalent), so they're head-sampled at 1%. + "observability": { + "enabled": true, + "logs": { "enabled": true, "head_sampling_rate": 1, "invocation_logs": false }, + "traces": { "enabled": true, "head_sampling_rate": 0.01 } + }, "routes": [{ "pattern": "sitrep.quintinshaw.com", "custom_domain": true }], "kv_namespaces": [{ "binding": "INVITE_DIR", "id": "490111734d5246cc949f0eec1c0ca0f3" }], "vars": { "APNS_BUNDLE_ID": "dev.sitrep.app", - "APNS_HOST": "api.sandbox.push.apple.com" + "APNS_HOST": "api.sandbox.push.apple.com", + // Server-side kill switch for /v3/realtime auto-connect (Fix A, pre-merge + // review). Absent/false means disabled; flip to true per-environment to + // enable. No per-space granularity in v1. + "REALTIME_ENABLED": false }, "durable_objects": { - "bindings": [{ "name": "USER_STORE", "class_name": "UserStore" }] + "bindings": [ + { "name": "USER_STORE", "class_name": "UserStore" }, + { "name": "SPACE_HUB", "class_name": "SpaceHub" } + ] }, - "migrations": [{ "tag": "v1", "new_sqlite_classes": ["UserStore"] }] + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["UserStore"] }, + { "tag": "v2", "new_sqlite_classes": ["SpaceHub"] } + ] }