From 12a6a08d52881f5ca65afa3028960d18490731c3 Mon Sep 17 00:00:00 2001 From: spelech <28486500+spelech@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:30:36 +0000 Subject: [PATCH 1/2] feat(audio): add managed TTS engine service with OpenAI speech compatibility Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- Endpoints/DiscoveryEndpoints.cs | 11 +- Endpoints/EngineEndpoints.cs | 61 +++++++++ Endpoints/ModelProxyEndpoints.cs | 55 ++++++++ .../Models/AppSettings.cs | 5 +- .../ViewModels/SettingsViewModel.cs | 71 +++++++++- .../Views/Controls/SettingsTabControl.axaml | 30 +++++ .../AppSettingsTests.cs | 8 +- .../ServerEndpointsTests.cs | 28 ++++ .../ServicesAndEngineManagerTests.cs | 23 ++++ .../SettingsViewModelTests.cs | 44 +++++++ .../ToolDiscoveryServiceTests.cs | 34 +++++ Services/AiEngineManager.cs | 121 ++++++++++++++++++ Services/IAiEngineManager.cs | 3 + Services/IToolDiscoveryService.cs | 7 +- Services/ToolDiscoveryService.cs | 107 +++++++++++++++- 15 files changed, 600 insertions(+), 8 deletions(-) diff --git a/Endpoints/DiscoveryEndpoints.cs b/Endpoints/DiscoveryEndpoints.cs index 0b2eb32..9372190 100644 --- a/Endpoints/DiscoveryEndpoints.cs +++ b/Endpoints/DiscoveryEndpoints.cs @@ -49,7 +49,11 @@ public static void MapDiscoveryEndpoints(this WebApplication app) ComfyModelsPath = string.IsNullOrWhiteSpace(currentSettings.ComfyModelsPath) && !string.IsNullOrWhiteSpace(detected.ComfyUi.ModelsDirectory) ? detected.ComfyUi.ModelsDirectory - : currentSettings.ComfyModelsPath + : currentSettings.ComfyModelsPath, + + AudioEngineExecutablePath = string.IsNullOrWhiteSpace(currentSettings.AudioEngineExecutablePath) && !string.IsNullOrWhiteSpace(detected.AudioEngine?.ExecutablePath) + ? detected.AudioEngine.ExecutablePath + : currentSettings.AudioEngineExecutablePath }; settingsService.SaveSettings(updatedSettings); @@ -121,6 +125,11 @@ public static void MapDiscoveryEndpoints(this WebApplication app) results[nameof(request.OllamaExecutablePath)] = discoveryService.ValidatePath(request.OllamaExecutablePath, PathTargetType.Executable); } + if (request.AudioEngineExecutablePath != null) + { + results[nameof(request.AudioEngineExecutablePath)] = discoveryService.ValidatePath(request.AudioEngineExecutablePath, PathTargetType.Executable); + } + var allValid = results.Values.All(r => r.IsValid); return Results.Ok(new ValidatePathsResponse(results, allValid)); }); diff --git a/Endpoints/EngineEndpoints.cs b/Endpoints/EngineEndpoints.cs index bc3ceec..541111f 100644 --- a/Endpoints/EngineEndpoints.cs +++ b/Endpoints/EngineEndpoints.cs @@ -91,5 +91,66 @@ public static void MapEngineEndpoints(this WebApplication app) await orchestrator.FreeComfyUiVramAsync(); return Results.Ok(new { message = "ComfyUI VRAM freed" }); }); + + app.MapPost("/api/audio/start", async (IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => + { + var logger = loggerFactory.CreateLogger("EngineEndpoints"); + var settings = settingsService.LoadSettings(); + var execPath = string.IsNullOrWhiteSpace(settings.AudioEngineExecutablePath) ? @"C:\AI\Kokoro-FastAPI\main.py" : settings.AudioEngineExecutablePath; + + if (!execPath.TrimStart().StartsWith("docker", StringComparison.OrdinalIgnoreCase)) + { + if (!Program.IsSafePath(execPath) || !System.IO.File.Exists(Program.ResolvePath(execPath, @"C:\AI\Kokoro-FastAPI\main.py"))) + { + return Results.BadRequest(new { message = $"Invalid or unsafe executable path: {execPath}" }); + } + } + + var success = await engineManager.StartAudioEngineAsync(execPath, logger); + if (success) + { + return Results.Ok(new { message = "Audio Engine Started", pid = engineManager.AudioProcess?.Id }); + } + return Results.Problem("Failed to start Audio Engine process"); + }); + + app.MapPost("/api/audio/stop", async (IAiEngineManager engineManager, ILoggerFactory loggerFactory) => + { + var logger = loggerFactory.CreateLogger("EngineEndpoints"); + await engineManager.StopAudioEngineAsync(logger); + return Results.Ok(new { message = "Audio Engine Stopped" }); + }); + + app.MapGet("/api/audio/voices", async (ISettingsService settingsService, System.Net.Http.IHttpClientFactory clientFactory) => + { + var settings = settingsService.LoadSettings(); + var baseUrl = (string.IsNullOrWhiteSpace(settings.AudioEngineUrl) ? "http://127.0.0.1:8880" : settings.AudioEngineUrl).TrimEnd('/'); + + try + { + var client = clientFactory.CreateClient(); + using var cts = new System.Threading.CancellationTokenSource(TimeSpan.FromSeconds(3)); + + var response = await client.GetAsync($"{baseUrl}/v1/audio/voices", cts.Token); + if (!response.IsSuccessStatusCode) + { + response = await client.GetAsync($"{baseUrl}/voices", cts.Token); + } + + if (response.IsSuccessStatusCode) + { + var json = await response.Content.ReadAsStringAsync(cts.Token); + return Results.Content(json, "application/json"); + } + } + catch { } + + var defaultVoices = new[] + { + "af_heart", "af_bella", "af_nicole", "af_sarah", "af_sky", + "am_adam", "am_michael", "bf_emma", "bf_isabella", "bm_george", "bm_fable" + }; + return Results.Ok(new { voices = defaultVoices, preferred = settings.PreferredAudioVoice }); + }); } } diff --git a/Endpoints/ModelProxyEndpoints.cs b/Endpoints/ModelProxyEndpoints.cs index 3d81015..31aa063 100644 --- a/Endpoints/ModelProxyEndpoints.cs +++ b/Endpoints/ModelProxyEndpoints.cs @@ -1,6 +1,7 @@ using System.Net.Http.Headers; using System.Text.Json; using System.Text.Json.Nodes; +using LocalLLMServerManager.Services; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; @@ -141,5 +142,59 @@ public static void MapModelProxyEndpoints(this WebApplication app) return Results.Problem(ex.Message); } }); + + app.MapPost("/v1/audio/speech", async (HttpContext context, ISettingsService settingsService, IHttpClientFactory clientFactory) => + { + try + { + var settings = settingsService.LoadSettings(); + var baseUrl = (string.IsNullOrWhiteSpace(settings.AudioEngineUrl) ? "http://127.0.0.1:8880" : settings.AudioEngineUrl).TrimEnd('/'); + var targetUrl = $"{baseUrl}/v1/audio/speech"; + + using var reader = new StreamReader(context.Request.Body); + var requestBodyStr = await reader.ReadToEndAsync(); + + string outgoingJson = requestBodyStr; + if (!string.IsNullOrWhiteSpace(requestBodyStr)) + { + try + { + using var doc = JsonDocument.Parse(requestBodyStr); + var root = doc.RootElement; + var hasVoice = root.TryGetProperty("voice", out var voiceProp) && !string.IsNullOrWhiteSpace(voiceProp.GetString()); + + if (!hasVoice) + { + var dict = JsonSerializer.Deserialize>(requestBodyStr) ?? new Dictionary(); + dict["voice"] = string.IsNullOrWhiteSpace(settings.PreferredAudioVoice) ? "af_heart" : settings.PreferredAudioVoice; + outgoingJson = JsonSerializer.Serialize(dict); + } + } + catch { } + } + + var http = clientFactory.CreateClient(); + using var targetReq = new HttpRequestMessage(HttpMethod.Post, targetUrl); + targetReq.Content = new StringContent(outgoingJson, System.Text.Encoding.UTF8, "application/json"); + + var targetResponse = await http.SendAsync(targetReq, HttpCompletionOption.ResponseHeadersRead, context.RequestAborted); + + context.Response.StatusCode = (int)targetResponse.StatusCode; + var contentType = targetResponse.Content.Headers.ContentType?.ToString() ?? "audio/mpeg"; + context.Response.ContentType = contentType; + + await using var responseStream = await targetResponse.Content.ReadAsStreamAsync(context.RequestAborted); + await responseStream.CopyToAsync(context.Response.Body, context.RequestAborted); + } + catch (Exception ex) + { + if (!context.Response.HasStarted) + { + context.Response.StatusCode = StatusCodes.Status502BadGateway; + context.Response.ContentType = "application/json"; + await context.Response.WriteAsync(JsonSerializer.Serialize(new { error = ex.Message })); + } + } + }); } } diff --git a/LocalLLMServerManager.Shared/Models/AppSettings.cs b/LocalLLMServerManager.Shared/Models/AppSettings.cs index e0f506d..ce47d1b 100644 --- a/LocalLLMServerManager.Shared/Models/AppSettings.cs +++ b/LocalLLMServerManager.Shared/Models/AppSettings.cs @@ -16,6 +16,9 @@ public record AppSettings( string PublishOutputPath = "C:\\LocalLLMServerManager", string ComfyModelsPath = "", string LanAccessUrl = "http://127.0.0.1:5246", - string SelectedThemeStyle = "semi" + string SelectedThemeStyle = "semi", + string AudioEngineExecutablePath = "", + string AudioEngineUrl = "http://127.0.0.1:8880", + string PreferredAudioVoice = "af_heart" ); diff --git a/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs b/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs index 48c1274..a5064e6 100644 --- a/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs +++ b/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs @@ -26,6 +26,9 @@ public partial class SettingsViewModel : ObservableObject [ObservableProperty] private string _selectedThemeStyle = "semi"; [ObservableProperty] private string _serviceName = "LocalLLMServerManager"; [ObservableProperty] private string _publishOutputPath = "C:\\LocalLLMServerManager"; + [ObservableProperty] private string _audioEngineExecutablePath = ""; + [ObservableProperty] private string _audioEngineUrl = "http://127.0.0.1:8880"; + [ObservableProperty] private string _preferredAudioVoice = "af_heart"; [ObservableProperty] private IStorageProvider? _storageProvider; [ObservableProperty] private bool _isAutoDetecting; @@ -38,6 +41,7 @@ public partial class SettingsViewModel : ObservableObject [ObservableProperty] private string _comfyModelsStatus = "⚠️ Missing"; [ObservableProperty] private string _threeDModelsStatus = "⚠️ Missing"; [ObservableProperty] private string _workflowsStatus = "⚠️ Missing"; + [ObservableProperty] private string _audioEngineExecutableStatus = "⚠️ Missing"; public string OllamaExecutableStatus => OllamaStatus; @@ -69,6 +73,7 @@ public SettingsViewModel(IThemeService themeService) partial void OnWorkflowsPathChanged(string value) => WorkflowsStatus = EvaluateDirectoryStatus(value); partial void OnComfyUiExecutablePathChanged(string value) => ComfyUiExecutableStatus = EvaluateExecutableStatus(value); partial void OnForgeExecutablePathChanged(string value) => ForgeExecutableStatus = EvaluateExecutableStatus(value); + partial void OnAudioEngineExecutablePathChanged(string value) => AudioEngineExecutableStatus = EvaluateExecutableStatus(value); partial void OnOllamaExecutablePathChanged(string value) { OllamaStatus = EvaluateExecutableStatus(value); @@ -104,6 +109,7 @@ public void RefreshAllStatuses() ComfyModelsStatus = EvaluateDirectoryStatus(ComfyModelsPath); ThreeDModelsStatus = EvaluateDirectoryStatus(ThreeDModelsPath); WorkflowsStatus = EvaluateDirectoryStatus(WorkflowsPath); + AudioEngineExecutableStatus = EvaluateExecutableStatus(AudioEngineExecutablePath); OnPropertyChanged(nameof(OllamaExecutableStatus)); } @@ -265,6 +271,19 @@ public async Task AutoDetectToolsAsync(string apiBase, HttpClient http) } } + if (root.TryGetProperty("audioEngine", out var audio)) + { + if (audio.TryGetProperty("executablePath", out var aExe) && aExe.ValueKind == JsonValueKind.String) + { + var val = aExe.GetString(); + if (!string.IsNullOrWhiteSpace(val) && string.IsNullOrWhiteSpace(AudioEngineExecutablePath)) + { + AudioEngineExecutablePath = val; + AudioEngineExecutableStatus = "🔍 Auto-Discovered"; + } + } + } + ToastService.Instance.Show("Auto-detection complete.", ToastType.Success); } else @@ -352,6 +371,50 @@ public async Task BrowseWorkflowsAsync(IStorageProvider? provider = null) } } + [RelayCommand] + public async Task BrowseAudioEngineExecutableAsync(IStorageProvider? provider = null) + { + var path = await PickFileAsync(provider, "Select Audio Engine Executable or Script", new[] { "*.py", "*.bat", "*.cmd", "*.exe", "*.sh" }); + if (!string.IsNullOrWhiteSpace(path)) + { + AudioEngineExecutablePath = path; + } + } + + [RelayCommand] + public async Task TestVoiceSynthesizerAsync() + { + await TestVoiceSynthesizerAsync("http://127.0.0.1:5246", new HttpClient()); + } + + public async Task TestVoiceSynthesizerAsync(string apiBase, HttpClient http) + { + try + { + var payload = new + { + model = "kokoro", + input = "Local LLM Server Manager audio text-to-speech engine test.", + voice = string.IsNullOrWhiteSpace(PreferredAudioVoice) ? "af_heart" : PreferredAudioVoice, + response_format = "mp3" + }; + var content = new StringContent(JsonSerializer.Serialize(payload), System.Text.Encoding.UTF8, "application/json"); + var response = await http.PostAsync($"{apiBase}/v1/audio/speech", content); + if (response.IsSuccessStatusCode) + { + ToastService.Instance.Show("TTS Synthesis test succeeded!", ToastType.Success); + } + else + { + ToastService.Instance.Show($"TTS Synthesis test returned status code {(int)response.StatusCode}", ToastType.Warning); + } + } + catch + { + ToastService.Instance.Show("Failed to test TTS Voice Synthesizer.", ToastType.Error); + } + } + private async Task PickFileAsync(IStorageProvider? explicitProvider, string title, string[] patterns) { var provider = explicitProvider ?? StorageProvider; @@ -431,6 +494,9 @@ public async Task LoadSettingsAsync(string apiBase, HttpClient http) SelectedThemeStyle = settings.SelectedThemeStyle ?? "semi"; ServiceName = settings.ServiceName ?? "LocalLLMServerManager"; PublishOutputPath = settings.PublishOutputPath ?? "C:\\LocalLLMServerManager"; + AudioEngineExecutablePath = settings.AudioEngineExecutablePath ?? ""; + AudioEngineUrl = settings.AudioEngineUrl ?? "http://127.0.0.1:8880"; + PreferredAudioVoice = settings.PreferredAudioVoice ?? "af_heart"; RefreshAllStatuses(); } @@ -462,7 +528,10 @@ public async Task SaveSettingsAsync(string apiBase, HttpClient http) PublishOutputPath: this.PublishOutputPath, ComfyModelsPath: this.ComfyModelsPath, LanAccessUrl: this.LanAccessUrl, - SelectedThemeStyle: this.SelectedThemeStyle + SelectedThemeStyle: this.SelectedThemeStyle, + AudioEngineExecutablePath: this.AudioEngineExecutablePath, + AudioEngineUrl: this.AudioEngineUrl, + PreferredAudioVoice: this.PreferredAudioVoice ); var content = new StringContent( diff --git a/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml b/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml index eb8b3bc..a8227af 100644 --- a/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml +++ b/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml @@ -103,6 +103,36 @@ + + + + + + + + + + +