From 66dbd621be4cd42c25b16e314a73313359d63a4a Mon Sep 17 00:00:00 2001 From: Paul Armbruster Date: Sun, 26 Jul 2026 18:19:50 +0200 Subject: [PATCH 1/5] feat(local-api): add Settings toggle + docs for the on-device Local API The loopback-only Local API (POST /v1/transcribe, /v1/postprocess) already exists but was enable-only via a hidden "LocalAPIEnabled" default and was undocumented. This surfaces and documents it: - SettingsStore.localAPIEnabled (reuses the existing "LocalAPIEnabled" key) - LocalAPIServer.refresh() to start/stop the server live when toggled - a "Local API (on-device agents)" toggle in Settings (loopback-only, off by default) - README section + docs/local-api.md documenting endpoints & security Lets other apps/agents on the same Mac reuse FluidVoice's transcription instead of loading their own ASR model. Co-Authored-By: Claude Opus 4.8 --- README.md | 22 +++++ Sources/Fluid/Persistence/SettingsStore.swift | 10 ++ .../Services/LocalAPI/LocalAPIServer.swift | 11 +++ Sources/Fluid/UI/SettingsView.swift | 14 +++ docs/local-api.md | 98 +++++++++++++++++++ 5 files changed, 155 insertions(+) create mode 100644 docs/local-api.md diff --git a/README.md b/README.md index 2daad1f8..0f7e934c 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,28 @@ Whisper supports up to 99 languages, depending on the model size you choose. --- +## Local API (for on-device agents) + +FluidVoice can expose a **loopback-only** HTTP API so other apps and AI agents running **on the same Mac** can reuse its transcription and post-processing — instead of bundling and loading their own ASR model. One model stays warm in FluidVoice; agents get FluidVoice-quality transcription with no extra memory footprint. + +**Enable it:** `Settings → Local API (on-device agents)`. It is **off by default** and only ever binds to `127.0.0.1` (non-loopback connections are rejected at the listener). + +- Base URL: `http://127.0.0.1:47733` (port configurable via the `LocalAPIPort` preference) +- `GET /v1/health` — liveness check +- `POST /v1/transcribe` — transcribe an audio file → `{ text, confidence, sampleCount, provider }` +- `POST /v1/postprocess` — run text post-processing/enhancement + +```bash +curl -s http://127.0.0.1:47733/v1/transcribe \ + -H 'Content-Type: application/json' \ + -d '{"path": "/absolute/path/to/audio.ogg"}' +# → {"text":"…","confidence":0.97,"sampleCount":…,"provider":"Parakeet TDT v3 (Multilingual)"} +``` + +Audio can also be sent inline as `{"audioBase64": "…", "filename": "clip.wav"}` (max request 25 MB). Full reference: [docs/local-api.md](docs/local-api.md). + +--- + ## Requirements - macOS 15.0 (Sequoia) or later diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index e786893f..91ebfa65 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -1799,6 +1799,15 @@ final class SettingsStore: ObservableObject { set { self.defaults.set(newValue, forKey: Keys.copyTranscriptionToClipboard) } } + /// When enabled, the loopback-only Local API server (`LocalAPIServer`) is started so + /// on-device agents can reuse FluidVoice's transcription (POST /v1/transcribe) instead + /// of shipping their own ASR model. Off by default. Uses the same "LocalAPIEnabled" + /// key that `LocalAPI.Configuration` reads. + var localAPIEnabled: Bool { + get { self.defaults.bool(forKey: Keys.localAPIEnabled) } + set { self.defaults.set(newValue, forKey: Keys.localAPIEnabled) } + } + var preferredInputDeviceUID: String? { get { self.defaults.string(forKey: Keys.preferredInputDeviceUID) } set { self.defaults.set(newValue, forKey: Keys.preferredInputDeviceUID) } @@ -4915,6 +4924,7 @@ private extension SettingsStore { static let experimentalDirectAudioCaptureForcedOff = "ExperimentalDirectAudioCaptureForcedOff" static let directAudioCaptureConsecutiveFailures = "DirectAudioCaptureConsecutiveFailures" static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard" + static let localAPIEnabled = "LocalAPIEnabled" static let textInsertionMode = "TextInsertionMode" static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled" static let betaReleasesEnabled = "BetaReleasesEnabled" diff --git a/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift b/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift index 2da5eb1a..61d79321 100644 --- a/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift +++ b/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift @@ -79,6 +79,17 @@ final class LocalAPIServer { self.listener = nil } + /// Start or stop the server to match the current `LocalAPIEnabled` setting. + /// Call this after the user toggles the Local API preference so the change takes + /// effect immediately, without restarting the app. + func refresh() { + if LocalAPI.Configuration.current.enabled { + self.start() + } else { + self.stop() + } + } + private func handleState(_ state: NWListener.State) { switch state { case .ready: diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 115ebc21..4767a85d 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -861,6 +861,20 @@ struct SettingsView: View { } Divider().opacity(0.2) + self.optionToggleRow( + title: "Local API (on-device agents)", + description: "Expose a loopback-only HTTP API on 127.0.0.1 (default port 47733) so local agents/apps on this Mac can reuse FluidVoice's transcription (POST /v1/transcribe). Off by default; never leaves this machine.", + isOn: Binding( + get: { SettingsStore.shared.localAPIEnabled }, + set: { newValue in + SettingsStore.shared.localAPIEnabled = newValue + LocalAPIServer.shared.refresh() + } + ), + allowsDescriptionWrapping: true + ) + Divider().opacity(0.2) + HStack(alignment: .center) { VStack(alignment: .leading, spacing: 2) { Text("Text Insertion Mode") diff --git a/docs/local-api.md b/docs/local-api.md new file mode 100644 index 00000000..9f698251 --- /dev/null +++ b/docs/local-api.md @@ -0,0 +1,98 @@ +# Local API + +FluidVoice ships a small, **loopback-only** HTTP API that lets other apps and AI agents +running **on the same Mac** reuse its speech-to-text and text post-processing. This avoids +each agent bundling and loading its own ASR model — FluidVoice keeps one model warm and +serves transcription requests locally. + +## Enabling + +The API is **off by default**. Turn it on in `Settings → Local API (on-device agents)`. + +You can also toggle it via `defaults`: + +```bash +defaults write com.FluidApp.app LocalAPIEnabled -bool true +# optional custom port (default 47733): +defaults write com.FluidApp.app LocalAPIPort -int 47733 +``` + +## Security + +- The server binds to `127.0.0.1` only and **rejects any non-loopback connection** at the + listener level, so it is reachable only from processes on this Mac. +- It is opt-in and off by default. +- Maximum request body size is 25 MB. + +## Endpoints + +Base URL: `http://127.0.0.1:47733` + +### `GET /v1/health` + +Liveness probe. Returns `200 OK` when the server is running. + +### `POST /v1/transcribe` + +Transcribe an audio file with the currently selected speech model (reuses the already-loaded +model — no second instance). + +Request body (JSON), one of: + +```jsonc +{ "path": "/absolute/path/to/audio.ogg" } // transcribe a file by path +{ "audioBase64": "", "filename": "clip.wav" } // or inline audio (extension hint) +``` + +Supported inputs include WAV, MP3, M4A, OGG and other common formats (decoded internally). + +Response: + +```json +{ + "text": "the transcribed text", + "confidence": 0.97, + "sampleCount": 1402136, + "provider": "Parakeet TDT v3 (Multilingual)" +} +``` + +### `POST /v1/postprocess` + +Run FluidVoice's text post-processing/enhancement on a string. + +```json +{ "text": "raw text to clean up" } +``` + +Response: + +```json +{ "text": "cleaned text", "provider": "…", "model": "…" } +``` + +## Example integrations + +Shell: + +```bash +curl -s http://127.0.0.1:47733/v1/transcribe \ + -H 'Content-Type: application/json' \ + -d '{"path": "'"$PWD"'/memo.ogg"}' | jq -r .text +``` + +Python: + +```python +import requests +r = requests.post("http://127.0.0.1:47733/v1/transcribe", + json={"path": "/tmp/memo.ogg"}, timeout=90) +print(r.json()["text"]) +``` + +## Notes + +- If the selected model is not resident yet, the first request loads it; subsequent requests + are fast while it stays warm. +- Other endpoints exist for history and the custom dictionary (`GET /v1/history`, + `GET|POST /v1/dictionary/*`); this document focuses on the ones most useful to agents. From 0e68a5d5322e9ee30b11489c92d5c4cd060570eb Mon Sep 17 00:00:00 2001 From: Paul Armbruster Date: Sun, 26 Jul 2026 22:00:29 +0200 Subject: [PATCH 2/5] fix(local-api): address review feedback (listener race, wording, audio-cap docs) - LocalAPIServer: ignore state callbacks from a superseded listener (guard self.listener === listener, weak capture) so a rapid off->on toggle can no longer orphan/cancel the newly-installed listener (Codex P2 / Greptile P1). - Qualify the privacy claim: only the HTTP listener and /v1/transcribe are on-device; /v1/postprocess follows the configured provider (may be remote) (toggle text, README, docs). - Describe the accept-time loopback filtering accurately instead of claiming it binds 127.0.0.1. - Document the 300s (5 min) audio-duration cap alongside the 25 MB body cap. Co-Authored-By: Claude Opus 4.8 --- README.md | 4 ++-- Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift | 7 ++++++- Sources/Fluid/UI/SettingsView.swift | 2 +- docs/local-api.md | 12 +++++++++--- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0f7e934c..95db6448 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ Whisper supports up to 99 languages, depending on the model size you choose. FluidVoice can expose a **loopback-only** HTTP API so other apps and AI agents running **on the same Mac** can reuse its transcription and post-processing — instead of bundling and loading their own ASR model. One model stays warm in FluidVoice; agents get FluidVoice-quality transcription with no extra memory footprint. -**Enable it:** `Settings → Local API (on-device agents)`. It is **off by default** and only ever binds to `127.0.0.1` (non-loopback connections are rejected at the listener). +**Enable it:** `Settings → Local API (on-device agents)`. It is **off by default** and **refuses all non-loopback connections** — any non-local peer is rejected at accept time, before it can send a request — so the API is usable only from processes on this Mac. - Base URL: `http://127.0.0.1:47733` (port configurable via the `LocalAPIPort` preference) - `GET /v1/health` — liveness check @@ -191,7 +191,7 @@ curl -s http://127.0.0.1:47733/v1/transcribe \ # → {"text":"…","confidence":0.97,"sampleCount":…,"provider":"Parakeet TDT v3 (Multilingual)"} ``` -Audio can also be sent inline as `{"audioBase64": "…", "filename": "clip.wav"}` (max request 25 MB). Full reference: [docs/local-api.md](docs/local-api.md). +Audio can also be sent inline as `{"audioBase64": "…", "filename": "clip.wav"}`. Limits: **25 MB** request body and **300 s (5 min)** of audio per transcription (independent caps). Note: `/v1/transcribe` runs on-device, but `/v1/postprocess` uses your configured AI provider (Settings → AI Enhancement), which may be remote. Full reference: [docs/local-api.md](docs/local-api.md). --- diff --git a/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift b/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift index 61d79321..752e1e88 100644 --- a/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift +++ b/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift @@ -56,8 +56,13 @@ final class LocalAPIServer { handler.start(on: self.queue) } } - listener.stateUpdateHandler = { state in + listener.stateUpdateHandler = { [weak self, weak listener] state in Task { @MainActor in + // Ignore callbacks from a superseded listener. A rapid off→on toggle + // (now user-reachable via refresh()) can install a new listener before + // an old one's async .cancelled/.failed callback reaches the main actor; + // without this guard that stale callback would clear/cancel the new one. + guard let self, let listener, self.listener === listener else { return } self.handleState(state) } } diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 4767a85d..6a6dcae4 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -863,7 +863,7 @@ struct SettingsView: View { self.optionToggleRow( title: "Local API (on-device agents)", - description: "Expose a loopback-only HTTP API on 127.0.0.1 (default port 47733) so local agents/apps on this Mac can reuse FluidVoice's transcription (POST /v1/transcribe). Off by default; never leaves this machine.", + description: "Loopback-only HTTP API (127.0.0.1, default port 47733) so on-device agents can reuse transcription (POST /v1/transcribe). Off by default; non-loopback connections are refused. Note: /v1/postprocess runs your configured AI provider, which may be remote.", isOn: Binding( get: { SettingsStore.shared.localAPIEnabled }, set: { newValue in diff --git a/docs/local-api.md b/docs/local-api.md index 9f698251..ba2eadf0 100644 --- a/docs/local-api.md +++ b/docs/local-api.md @@ -19,10 +19,16 @@ defaults write com.FluidApp.app LocalAPIPort -int 47733 ## Security -- The server binds to `127.0.0.1` only and **rejects any non-loopback connection** at the - listener level, so it is reachable only from processes on this Mac. +- The listener **refuses all non-loopback connections** — any non-local peer is rejected at + accept time, before it can send a request — so the API is usable only from processes on this Mac. - It is opt-in and off by default. -- Maximum request body size is 25 MB. +- Limits: request body **25 MB**; transcription audio **300 s (5 min)** per request (longer audio is + rejected). These caps are independent — compressed audio can satisfy the byte limit yet still hit the + duration limit. +- **Locality caveat:** `/v1/transcribe` runs fully on-device. `/v1/postprocess` runs your configured + post-processing provider (Settings → AI Enhancement); if that provider is a remote API + (OpenAI/Groq/Anthropic/…), the submitted text is sent there. Only the HTTP listener and transcription + are guaranteed local. ## Endpoints From d8ef83f38cdb4d071d0c870a31b0a3918b64d9a7 Mon Sep 17 00:00:00 2001 From: Paul Armbruster Date: Sun, 26 Jul 2026 22:30:15 +0200 Subject: [PATCH 3/5] fix(local-api): move the Settings toggle out of the accessibility-gated section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The toggle sat next to 'Copy to Clipboard', which lives inside the `if accessibilityEnabled` block, so it was hidden until the user granted Accessibility — unrelated to a loopback HTTP API. Moved it to the always-visible 'App Settings' section, and split the long description into description + footnote so it no longer truncates on one line. Co-Authored-By: Claude Opus 4.8 --- Sources/Fluid/UI/SettingsView.swift | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 6a6dcae4..41f70e5c 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -277,6 +277,21 @@ struct SettingsView: View { ) Divider().opacity(0.2) + // Local API (on-device agents) + self.settingsToggleRow( + title: "Local API (on-device agents)", + description: "Loopback-only HTTP API for on-device agents to reuse transcription (POST /v1/transcribe).", + footnote: "127.0.0.1, default port 47733. Off by default; non-loopback connections are refused. /v1/postprocess runs your configured AI provider, which may be remote.", + isOn: Binding( + get: { SettingsStore.shared.localAPIEnabled }, + set: { newValue in + SettingsStore.shared.localAPIEnabled = newValue + LocalAPIServer.shared.refresh() + } + ) + ) + Divider().opacity(0.2) + // Accent Color VStack(alignment: .leading, spacing: 6) { HStack(alignment: .center) { @@ -861,20 +876,6 @@ struct SettingsView: View { } Divider().opacity(0.2) - self.optionToggleRow( - title: "Local API (on-device agents)", - description: "Loopback-only HTTP API (127.0.0.1, default port 47733) so on-device agents can reuse transcription (POST /v1/transcribe). Off by default; non-loopback connections are refused. Note: /v1/postprocess runs your configured AI provider, which may be remote.", - isOn: Binding( - get: { SettingsStore.shared.localAPIEnabled }, - set: { newValue in - SettingsStore.shared.localAPIEnabled = newValue - LocalAPIServer.shared.refresh() - } - ), - allowsDescriptionWrapping: true - ) - Divider().opacity(0.2) - HStack(alignment: .center) { VStack(alignment: .leading, spacing: 2) { Text("Text Insertion Mode") From e88f57419bb28fa130cd5adf78edc5b0d8da8364 Mon Sep 17 00:00:00 2001 From: Paul Armbruster Date: Sun, 26 Jul 2026 22:32:12 +0200 Subject: [PATCH 4/5] fix(local-api): shorten the toggle footnote so it doesn't truncate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit settingsToggleRow renders description and footnote on single truncating lines; the previous footnote overflowed. Trimmed it to the essentials — the full contract (non-loopback filtering, limits, post-processing locality) lives in README + docs/local-api.md. Co-Authored-By: Claude Opus 4.8 --- Sources/Fluid/UI/SettingsView.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 41f70e5c..e2941687 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -281,7 +281,7 @@ struct SettingsView: View { self.settingsToggleRow( title: "Local API (on-device agents)", description: "Loopback-only HTTP API for on-device agents to reuse transcription (POST /v1/transcribe).", - footnote: "127.0.0.1, default port 47733. Off by default; non-loopback connections are refused. /v1/postprocess runs your configured AI provider, which may be remote.", + footnote: "127.0.0.1:47733, off by default. /v1/postprocess may use a remote AI provider.", isOn: Binding( get: { SettingsStore.shared.localAPIEnabled }, set: { newValue in From cec22e7eb06f35c71ce4e03a3a1b0cba451c715a Mon Sep 17 00:00:00 2001 From: Paul Armbruster Date: Mon, 27 Jul 2026 22:35:54 +0200 Subject: [PATCH 5/5] fix(local-api): drop connections queued before disable + doc corrections - LocalAPIServer: the deferred @MainActor task now also checks the server is still running (listener != nil) before creating a handler, so a connection accepted just before stop()/toggle-off is dropped instead of handled. - docs: note that enabling via `defaults` needs an app relaunch (the Settings toggle applies live via refresh()); clarify the 300s audio cap rejects file-path input but truncates inline audioBase64 input. Co-Authored-By: Claude Opus 4.8 --- Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift | 5 ++++- docs/local-api.md | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift b/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift index 752e1e88..d6af7a10 100644 --- a/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift +++ b/Sources/Fluid/Services/LocalAPI/LocalAPIServer.swift @@ -40,7 +40,10 @@ final class LocalAPIServer { } Task { @MainActor [weak self] in - guard let self else { + // Drop connections accepted just before the server was disabled: stop() + // clears `listener`, so a task still queued here must not spin up a handler + // after the API was turned off (e.g. a live toggle-off via refresh()). + guard let self, self.listener != nil else { connection.cancel() return } diff --git a/docs/local-api.md b/docs/local-api.md index ba2eadf0..550a297e 100644 --- a/docs/local-api.md +++ b/docs/local-api.md @@ -17,14 +17,19 @@ defaults write com.FluidApp.app LocalAPIEnabled -bool true defaults write com.FluidApp.app LocalAPIPort -int 47733 ``` +The Settings toggle starts/stops the server immediately. Setting the preference with `defaults` +while FluidVoice is running only writes the value — **relaunch FluidVoice** (or use the toggle) for +it to take effect, since the server is otherwise started only at app launch. + ## Security - The listener **refuses all non-loopback connections** — any non-local peer is rejected at accept time, before it can send a request — so the API is usable only from processes on this Mac. - It is opt-in and off by default. -- Limits: request body **25 MB**; transcription audio **300 s (5 min)** per request (longer audio is - rejected). These caps are independent — compressed audio can satisfy the byte limit yet still hit the - duration limit. +- Limits: request body **25 MB**; transcription audio **300 s (5 min)** per request. For file-`path` + input, audio longer than the limit is **rejected**; for inline `audioBase64` input it is currently + **truncated to the first 300 s**. These caps are independent — compressed audio can satisfy the byte + limit yet still hit the duration limit. - **Locality caveat:** `/v1/transcribe` runs fully on-device. `/v1/postprocess` runs your configured post-processing provider (Settings → AI Enhancement); if that provider is a remote API (OpenAI/Groq/Anthropic/…), the submitted text is sent there. Only the HTTP listener and transcription