From 7e18bbcfdc0b6783898a3530518739e5cfcdfc90 Mon Sep 17 00:00:00 2001 From: Michael Yeack Date: Fri, 29 May 2026 20:09:00 -0700 Subject: [PATCH] Add localhost web console to BlackbookServer (KPI dashboard + safe actions) BlackbookServer was headless (HTTP API + menu-bar popover); there was no way to see operational health at a glance. Adds a web console. - Second NWListener bound to 127.0.0.1:8766 (requiredLocalEndpoint). The Cloudflare tunnel maps only 8765, so the console is never internet- exposed and needs no password. Reuses the existing receiveRequest/ sendResponse plumbing; cancelled on stop()/scheduleRestart(); a loopback remote-endpoint check adds defense-in-depth. - Routes (no header auth): GET / -> bundled console.html; GET /api/stats -> KPI JSON; POST /api/imessage/{backfill,toggle} -> safe actions. - /api/stats aggregates: sync health (latest heartbeat per device), backups (count/bytes/per-user), requests (access-log status histogram + recent tail), data (per-model fetchCount + iMessage stats). - console.html: single self-contained dark dashboard, polls every 5s, Sync-Last-30-Days + toggle buttons. Auto-bundled via the source glob. - ServerStatusModel passes the iMessage service into BackupServer; ServerMenuView gains an "Open Web Console" button. Localhost-only by design; safe actions only (no delete/restart). Verified: GET / 200, /api/stats full JSON; LAN:8766 refused; tunnel /api/stats 401. BlackbookServer is local-deploy (not TestFlight); rebuild/reinstall per docs/test-scenarios/TEST_SCENARIOS.md. Co-Authored-By: Claude Opus 4.8 --- Blackbook.xcodeproj/project.pbxproj | 4 + BlackbookServer/App/BackupServer.swift | 248 ++++++++++++++++++- BlackbookServer/App/ServerMenuView.swift | 5 + BlackbookServer/App/ServerStatusModel.swift | 2 +- BlackbookServer/Resources/console.html | 249 ++++++++++++++++++++ docs/test-scenarios/TEST_SCENARIOS.md | 29 +++ 6 files changed, 535 insertions(+), 2 deletions(-) create mode 100644 BlackbookServer/Resources/console.html diff --git a/Blackbook.xcodeproj/project.pbxproj b/Blackbook.xcodeproj/project.pbxproj index 6946edb..6fc178e 100644 --- a/Blackbook.xcodeproj/project.pbxproj +++ b/Blackbook.xcodeproj/project.pbxproj @@ -11,6 +11,7 @@ 0143CB904AAF1E6B53CCBAD1 /* FeatureGatingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4DCA116B785721C8942FD325 /* FeatureGatingTests.swift */; }; 03915555188215A47C2FFBAF /* DuplicateScanView.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED64DF1DD19F6ADC3991B97A /* DuplicateScanView.swift */; }; 047D658F5CE553E21B993DF3 /* InteractionLogView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 970CC06BF6BA7D37D1C593DC /* InteractionLogView.swift */; }; + 07E34CFDC7C149FAED57F53B /* console.html in Resources */ = {isa = PBXBuildFile; fileRef = 4FC80EA436D4FD929A77680C /* console.html */; }; 09159275BC5943BE485081DD /* GoogleCalendarService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2382E25FE6BDF339DA44560D /* GoogleCalendarService.swift */; }; 0A8A779BFC33094AB6F3B121 /* Tag.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33F7E74C7FFD8B03E32C1F7B /* Tag.swift */; }; 113034766F531D2FF71921B2 /* LocalSyncProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61B8831D66944BBBD80A6E6E /* LocalSyncProtocol.swift */; }; @@ -207,6 +208,7 @@ 4C58C1200FB5DE674799040E /* AIInsightsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AIInsightsView.swift; sourceTree = ""; }; 4DCA116B785721C8942FD325 /* FeatureGatingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeatureGatingTests.swift; sourceTree = ""; }; 4E45E53C6F0C7F7A6808F6EE /* LocalSyncServer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalSyncServer.swift; sourceTree = ""; }; + 4FC80EA436D4FD929A77680C /* console.html */ = {isa = PBXFileReference; lastKnownFileType = text.html; path = console.html; sourceTree = ""; }; 530F5990544546A519FE2390 /* IMessageSyncService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IMessageSyncService.swift; sourceTree = ""; }; 535A0087C38A75EF251D28DF /* DeviceIdentity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceIdentity.swift; sourceTree = ""; }; 56EBAC598F8585225B1F09E4 /* RelationshipScoreEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelationshipScoreEngine.swift; sourceTree = ""; }; @@ -577,6 +579,7 @@ isa = PBXGroup; children = ( A80CFB6F6BA1BDFBDE29CBD4 /* Assets.xcassets */, + 4FC80EA436D4FD929A77680C /* console.html */, ); path = Resources; sourceTree = ""; @@ -753,6 +756,7 @@ buildActionMask = 2147483647; files = ( 77D2B599AE22B4597BEBEEF8 /* Assets.xcassets in Resources */, + 07E34CFDC7C149FAED57F53B /* console.html in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/BlackbookServer/App/BackupServer.swift b/BlackbookServer/App/BackupServer.swift index 266f26f..f26b06f 100644 --- a/BlackbookServer/App/BackupServer.swift +++ b/BlackbookServer/App/BackupServer.swift @@ -20,8 +20,13 @@ private struct HTTPRequest { final class BackupServer: @unchecked Sendable { private let password: String private let container: ModelContainer? + /// iMessage reader, used by the localhost console's safe actions (backfill/toggle) and stats. + private let imessage: IMessageSyncService? private let queue = DispatchQueue(label: "com.blackbookdevelopment.backupserver") private var listener: NWListener? + /// Loopback-only listener serving the web console on `defaultConsolePort`. Never exposed via + /// the Cloudflare tunnel (which forwards only the sync port), so it needs no password. + private var consoleListener: NWListener? private var bonjourService: NetService? private(set) var isRunning = false private(set) var port: UInt16 = 0 @@ -31,6 +36,7 @@ final class BackupServer: @unchecked Sendable { private let maxRestartDelay: TimeInterval = 30.0 static let defaultPort: UInt16 = 8765 + static let defaultConsolePort: UInt16 = 8766 static let bonjourType = "_blackbook-sync._tcp" static let passwordHeader = "X-Sync-Password" static let userEmailHeader = "X-User-Email" @@ -46,9 +52,10 @@ final class BackupServer: @unchecked Sendable { /// Loaded once at init from the epoch file next to the master store; persists across restarts. private let serverEpoch: String - init(password: String, container: ModelContainer? = nil) { + init(password: String, container: ModelContainer? = nil, imessage: IMessageSyncService? = nil) { self.password = password self.container = container + self.imessage = imessage self.serverEpoch = ServerModelContainer.currentEpoch() } @@ -69,6 +76,7 @@ final class BackupServer: @unchecked Sendable { self?.isRunning = true self?.restartDelay = 1.0 self?.publishBonjour(port: Int(self?.port ?? 0)) + self?.startConsole() logger.info("Backup server listening on port \(self?.port ?? 0)") case .failed(let error): logger.error("Listener failed: \(error.localizedDescription)") @@ -96,18 +104,256 @@ final class BackupServer: @unchecked Sendable { isStopping = true listener?.cancel() listener = nil + consoleListener?.cancel() + consoleListener = nil bonjourService?.stop() bonjourService = nil isRunning = false port = 0 } + // MARK: - Web Console (loopback only) + + /// Starts the localhost-only console listener on `defaultConsolePort`. Bound to 127.0.0.1 via + /// `requiredLocalEndpoint` so it's unreachable from the LAN or the Cloudflare tunnel (which maps + /// only the sync port). Idempotent — a no-op if already listening. + private func startConsole() { + guard consoleListener == nil else { return } + let params = NWParameters.tcp + params.allowLocalEndpointReuse = true + params.requiredLocalEndpoint = NWEndpoint.hostPort( + host: NWEndpoint.Host("127.0.0.1"), + port: NWEndpoint.Port(rawValue: Self.defaultConsolePort)! + ) + do { + let cl = try NWListener(using: params) + self.consoleListener = cl + cl.stateUpdateHandler = { state in + switch state { + case .ready: + logger.info("Console listening on 127.0.0.1:\(Self.defaultConsolePort)") + case .failed(let error): + logger.error("Console listener failed: \(error.localizedDescription)") + default: + break + } + } + cl.newConnectionHandler = { [weak self] conn in + self?.handleConsole(connection: conn) + } + cl.start(queue: queue) + } catch { + logger.error("Failed to start console listener: \(error.localizedDescription)") + } + } + + // MARK: - Console Request Handling + + private func handleConsole(connection: NWConnection) { + // Defense-in-depth on top of the loopback-only bind: only serve loopback peers. + guard Self.isLoopback(connection.endpoint) else { + connection.cancel() + return + } + connection.start(queue: queue) + receiveRequest(connection: connection, accumulated: Data()) { [weak self] request in + guard let self else { return } + let response = self.handleConsoleRoute(request) + self.sendResponse(response, on: connection) { + connection.cancel() + } + } + } + + private static func isLoopback(_ endpoint: NWEndpoint) -> Bool { + guard case let .hostPort(host, _) = endpoint else { return false } + switch host { + case .ipv4(let addr): return addr.isLoopback + case .ipv6(let addr): return addr.isLoopback + case .name(let name, _): return name == "localhost" || name == "127.0.0.1" || name == "::1" + @unknown default: return false + } + } + + private func handleConsoleRoute(_ request: HTTPRequest?) -> (status: Int, headers: [String: String], body: Data?) { + guard let request else { return (400, [:], "Bad Request".data(using: .utf8)) } + switch (request.method, request.path) { + case ("GET", "/"), ("GET", "/console"): + return consoleHTMLResponse() + case ("GET", "/api/stats"): + return consoleJSON(buildConsoleStats()) + case ("POST", "/api/imessage/backfill"): + let days = consoleBodyInt(request.body, key: "days") ?? 30 + if let imessage { Task { @MainActor in await imessage.backfill(daysBack: days) } } + return consoleJSON(["ok": true, "days": days], status: 202) + case ("POST", "/api/imessage/toggle"): + let enabled = consoleBodyBool(request.body, key: "enabled") ?? false + if let imessage { Task { @MainActor in imessage.isEnabled = enabled } } + return consoleJSON(["ok": true, "enabled": enabled]) + default: + return (404, [:], "Not Found".data(using: .utf8)) + } + } + + private func consoleHTMLResponse() -> (status: Int, headers: [String: String], body: Data?) { + guard let url = Bundle.main.url(forResource: "console", withExtension: "html"), + let data = try? Data(contentsOf: url) else { + return (500, [:], "console.html missing from bundle".data(using: .utf8)) + } + return (200, ["Content-Type": "text/html; charset=utf-8"], data) + } + + private func consoleJSON(_ object: Any, status: Int = 200) -> (status: Int, headers: [String: String], body: Data?) { + guard let data = try? JSONSerialization.data(withJSONObject: object) else { return (500, [:], nil) } + return (status, ["Content-Type": "application/json"], data) + } + + private func consoleBodyInt(_ body: Data?, key: String) -> Int? { + guard let body, let obj = try? JSONSerialization.jsonObject(with: body) as? [String: Any] else { return nil } + return obj[key] as? Int + } + + private func consoleBodyBool(_ body: Data?, key: String) -> Bool? { + guard let body, let obj = try? JSONSerialization.jsonObject(with: body) as? [String: Any] else { return nil } + return obj[key] as? Bool + } + + // MARK: - Console Stats + + private func buildConsoleStats() -> [String: Any] { + [ + "generatedAt": ISO8601DateFormatter().string(from: Date()), + "server": [ + "running": isRunning, + "port": Int(port), + "consolePort": Int(Self.defaultConsolePort), + "epoch": serverEpoch + ], + "syncHealth": consoleSyncHealth(), + "backups": consoleBackups(), + "requests": consoleRequests(), + "data": consoleData() + ] + } + + private func consoleSyncHealth() -> [[String: Any]] { + let fm = FileManager.default + let days = consoleRecentDayStems(2) // oldest first → today's lines overwrite + var latestByDevice: [String: [String: Any]] = [:] + guard let userDirs = try? fm.contentsOfDirectory(at: remoteBackupsDirectory, includingPropertiesForKeys: nil) else { return [] } + for userDir in userDirs where userDir.hasDirectoryPath { + let logsDir = userDir.appendingPathComponent("Logs", isDirectory: true) + for day in days { + let url = logsDir.appendingPathComponent("heartbeats-\(day).jsonl") + for line in consoleReadLines(url) { + guard let obj = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any] else { continue } + let device = (obj["device"] as? String) ?? (obj["platform"] as? String) ?? "unknown" + let key = "\((obj["email"] as? String) ?? "")|\(device)" + latestByDevice[key] = obj + } + } + } + return latestByDevice.values.map { obj in + [ + "email": obj["email"] as? String ?? "", + "device": obj["device"] as? String ?? "", + "platform": obj["platform"] as? String ?? "", + "appVersion": obj["appVersion"] as? String ?? "", + "status": obj["status"] as? String ?? "", + "pushPending": obj["pushPending"] as? Int ?? 0, + "lastSyncDate": obj["lastSyncDate"] as? String ?? "", + "sentAt": obj["sentAt"] as? String ?? "", + "receivedAt": obj["receivedAt"] as? String ?? "", + "durationMs": obj["durationMs"] as? Int ?? 0, + "error": obj["error"] as? String ?? "" + ] + }.sorted { ($0["receivedAt"] as? String ?? "") > ($1["receivedAt"] as? String ?? "") } + } + + private func consoleBackups() -> [String: Any] { + let fm = FileManager.default + var perUser: [[String: Any]] = [] + var total = 0 + if let userDirs = try? fm.contentsOfDirectory(at: remoteBackupsDirectory, includingPropertiesForKeys: nil) { + for userDir in userDirs where userDir.hasDirectoryPath { + let backups = ((try? fm.contentsOfDirectory(at: userDir, includingPropertiesForKeys: [.contentModificationDateKey])) ?? []) + .filter { $0.hasDirectoryPath && $0.lastPathComponent != "Logs" } + let latest = backups.compactMap { try? $0.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate }.max() + total += backups.count + perUser.append([ + "email": userDir.lastPathComponent, + "count": backups.count, + "latest": latest.map { ISO8601DateFormatter().string(from: $0) } ?? "" + ]) + } + } + return ["totalCount": total, "totalBytes": diskUsageBytes, "perUser": perUser] + } + + private func consoleRequests() -> [String: Any] { + let day = Self.accessDateFormatter.string(from: Date()) + let url = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + .appendingPathComponent("Blackbook/Logs/_access/access-\(day).jsonl") + let lines = consoleReadLines(url) + var byStatus: [String: Int] = [:] + for line in lines { + guard let obj = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any] else { continue } + let status = obj["status"] as? Int ?? 0 + byStatus[status >= 500 ? "5xx" : "\(status)", default: 0] += 1 + } + let recent = lines.suffix(50).compactMap { try? JSONSerialization.jsonObject(with: Data($0.utf8)) as? [String: Any] } + return ["total": lines.count, "byStatus": byStatus, "recent": Array(recent.reversed())] + } + + private func consoleData() -> [String: Any] { + var counts: [String: Int] = [:] + if let container { + let ctx = ModelContext(container) + counts["Contact"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Interaction"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Note"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Tag"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Group"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Location"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["ContactRelationship"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Reminder"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["Activity"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + counts["RejectedCalendarEvent"] = (try? ctx.fetchCount(FetchDescriptor())) ?? 0 + } + var im: [String: Any] = ["available": imessage != nil] + if let imessage { + im["isRunning"] = imessage.isRunning + im["isEnabled"] = imessage.isEnabled + im["messagesProcessed"] = imessage.messagesProcessed + im["isBackfilling"] = imessage.isBackfilling + im["lastSyncDate"] = imessage.lastSyncDate.map { ISO8601DateFormatter().string(from: $0) } ?? "" + im["unmatchedHandles"] = imessage.unmatchedHandlesLastPoll + if let err = imessage.syncError { im["error"] = err } + } + return ["recordCounts": counts, "imessage": im] + } + + /// Day stems (UTC `yyyy-MM-dd`) for the last `n` days, oldest first. + private func consoleRecentDayStems(_ n: Int) -> [String] { + (0.. [String] { + guard let text = try? String(contentsOf: url, encoding: .utf8) else { return [] } + return text.split(separator: "\n").map(String.init).filter { !$0.isEmpty } + } + // MARK: - Restart private func scheduleRestart() { guard !isStopping else { return } listener?.cancel() listener = nil + consoleListener?.cancel() + consoleListener = nil bonjourService?.stop() bonjourService = nil isRunning = false diff --git a/BlackbookServer/App/ServerMenuView.swift b/BlackbookServer/App/ServerMenuView.swift index 97a7311..60828da 100644 --- a/BlackbookServer/App/ServerMenuView.swift +++ b/BlackbookServer/App/ServerMenuView.swift @@ -68,6 +68,11 @@ struct ServerMenuView: View { Button("Stop Server") { model.stopServer() } + Button("Open Web Console") { + if let url = URL(string: "http://127.0.0.1:\(BackupServer.defaultConsolePort)") { + NSWorkspace.shared.open(url) + } + } } else { Button("Start Server") { model.startServer() diff --git a/BlackbookServer/App/ServerStatusModel.swift b/BlackbookServer/App/ServerStatusModel.swift index 23fc10a..d764e9a 100644 --- a/BlackbookServer/App/ServerStatusModel.swift +++ b/BlackbookServer/App/ServerStatusModel.swift @@ -68,7 +68,7 @@ final class ServerStatusModel { func startServer() { guard !email.isEmpty else { return } let password = BackupServer.derivePassword(from: email) - let srv = BackupServer(password: password, container: modelContainer) + let srv = BackupServer(password: password, container: modelContainer, imessage: imessage) srv.start() server = srv diff --git a/BlackbookServer/Resources/console.html b/BlackbookServer/Resources/console.html new file mode 100644 index 0000000..da179c9 --- /dev/null +++ b/BlackbookServer/Resources/console.html @@ -0,0 +1,249 @@ + + + + + +Blackbook Server Console + + + +
+ +

Blackbook Server

+ + + connecting… +
+
+
+

Sync Health

+
+
+ +
+

Backups & Storage

+
+
+
+ +
+

Data & iMessage

+
+
+
+ +
+

Requests (today)

+
+
+
+
+
+
+
+ + + + diff --git a/docs/test-scenarios/TEST_SCENARIOS.md b/docs/test-scenarios/TEST_SCENARIOS.md index 9c152e2..9b085cb 100644 --- a/docs/test-scenarios/TEST_SCENARIOS.md +++ b/docs/test-scenarios/TEST_SCENARIOS.md @@ -361,3 +361,32 @@ cp -R /tmp/blackbook-server-build/Build/Products/Release/BlackbookServer.app /Ap open /Applications/BlackbookServer.app ``` Then re-confirm Full Disk Access for "Blackbook Server" and toggle "Log iMessages" on. + +## 2026-05-30 — BlackbookServer web console (localhost KPI dashboard + safe actions) + +**Context:** Added a localhost-only web console to BlackbookServer for at-a-glance operational health. A second `NWListener` bound to `127.0.0.1:8766` (the Cloudflare tunnel maps only 8765, so the console is never internet-exposed and needs no password). Serves `console.html` + `GET /api/stats` (sync health, backups/storage, request/error log, record counts, iMessage stats) and two safe POST actions (`/api/imessage/backfill`, `/api/imessage/toggle`). + +### Open it +- Menu-bar server.rack icon → **Open Web Console** (visible when the server is running), or browse to `http://127.0.0.1:8766`. + +### Scenario 1 — Dashboard loads with live data +1. Open the console. Header shows a green dot + `port 8765 · console 8766 · epoch …`. +2. Four sections populate and auto-refresh every 5 s: Sync Health (per-device heartbeats), Backups & Storage, Data & iMessage (record counts + iMessage panel), Requests (today's status histogram + recent tail). + +### Scenario 2 — Safe actions +1. **Sync Last 30 Days** button → toast "Backfill started"; iMsgs-Logged KPI rises (requires FDA granted to Blackbook Server + Messages present). +2. **Enable/Disable iMessage Logging** button flips `isRunning` in the Data panel on the next refresh. + +### Scenario 3 — Security: NOT exposed off-loopback (the important check) +- `curl http://$(ipconfig getifaddr en0):8766/` → connection refused (status 000). Loopback-only bind confirmed. +- `curl https://sync.libersecretorum.com/api/stats` → 401 (hits the 8765 sync server's auth gate; the console route doesn't exist there). The console is not tunneled. +- Defense-in-depth: the console connection handler also rejects any non-loopback remote endpoint. + +### Verification done +- BlackbookServer builds clean (Debug + Release); `console.html` confirmed bundled in `Contents/Resources/`. +- `GET /` → 200 text/html 12049 bytes; `GET /api/stats` → full JSON (Contacts 1288, backups 5/3.8 MB, 2 sync devices, 148 requests today). Verified with curl + a strict Python HTTP client. +- LAN:8766 refused; tunnel /api/stats → 401. Backfill endpoint → 202. +- 13 Swift Testing tests still pass. + +### Note +The console only runs while the sync server is running (it starts/stops with the main listener). The main Blackbook app is unaffected — these are BlackbookServer-only changes, deployed locally (not TestFlight).