diff --git a/docs/architecture/system-audio-capture.md b/docs/architecture/system-audio-capture.md index e29f31a..ba08634 100644 --- a/docs/architecture/system-audio-capture.md +++ b/docs/architecture/system-audio-capture.md @@ -129,3 +129,19 @@ To measure a real recording: `dotnet run --project src/PrimeDictate.Tools.Channe (left = microphone, right = system audio). It correlates the two channels' loudness and prints the lag; it needs the microphone to have heard the speakers (speakers, not headphones). Below correlation 0.3 the number is meaningless. + +## Self-contained skew probe + +`dotnet run --project src/PrimeDictate.Tools.ChannelSkew -- --probe [--save out.wav]` opens the same combined +lease as the app, plays a known 3.2 s noise-burst train (low level, 0.8 s fade-in) through the default output, and finds the train in each +channel with a matched filter (`ReferenceLocator`). It prints the lag of the microphone behind system audio, plus +correlation and peak/sidelobe for each side, and says RESULT: unusable when either side did not hear the train. +The matched filter finds a faint copy in room noise, so it works where envelope correlation of a recording did not. +The lag includes the speaker path (output device latency, ~3 ms/m of air). Linux check with a null sink whose +suspend-on-idle module is unloaded: +14 ms (the miniaudio mic's known ~20 ms). Not yet run on Windows. + +The probe refuses to play when the default output looks like headphones or a headset (Windows endpoint form +factor, or a name such as headset/Arctis/AirPods/Bluetooth; Linux `device.form_factor` or name) and tells the +user to switch to speakers; `--allow-headphones` overrides. First Windows runs, both on a wireless headset: +loopback found the train ~75 ms after play(), system stream opened 41 to 65 ms after the mic, no padding; the +headset mic heard its own earcup leak, giving +149 ms, which is not a capture offset, so no fixed correction is applied. diff --git a/src/PrimeDictate.Core/Audio/ReferenceLocator.cs b/src/PrimeDictate.Core/Audio/ReferenceLocator.cs new file mode 100644 index 0000000..ae75fd4 --- /dev/null +++ b/src/PrimeDictate.Core/Audio/ReferenceLocator.cs @@ -0,0 +1,146 @@ +using System.Numerics; + +namespace PrimeDictate.Core.Audio; + +/// Where a known signal was found in a recording. +/// Recording sample at which the reference starts. +/// Normalized correlation at that position, 0 to 1. +/// Peak divided by the median correlation elsewhere. Above about 6 the match is unambiguous. +public sealed record ReferenceMatch(long SampleIndex, double Correlation, double PeakToSidelobe) +{ + public bool IsReliable => this.Correlation >= 0.05 && this.PeakToSidelobe >= 6; +} + +/// +/// Finds a known test signal in a recording with a matched filter (FFT cross-correlation). Unlike +/// it needs no clean echo: a faint copy of a noise train is still found. +/// +public static class ReferenceLocator +{ + public static ReferenceMatch Locate(ReadOnlySpan recording, ReadOnlySpan reference) + { + if (reference.Length == 0 || recording.Length < reference.Length) + { + return new ReferenceMatch(0, 0, 0); + } + + var size = 1; + while (size < recording.Length + reference.Length) + { + size <<= 1; + } + + var a = new Complex[size]; + var b = new Complex[size]; + for (var i = 0; i < recording.Length; i++) + { + a[i] = recording[i]; + } + + double refEnergy = 0; + for (var i = 0; i < reference.Length; i++) + { + b[i] = reference[i]; + refEnergy += reference[i] * (double)reference[i]; + } + + Fft(a, false); + Fft(b, false); + for (var i = 0; i < size; i++) + { + a[i] *= Complex.Conjugate(b[i]); + } + + Fft(a, true); + + var prefix = new double[recording.Length + 1]; + for (var i = 0; i < recording.Length; i++) + { + prefix[i + 1] = prefix[i] + (recording[i] * (double)recording[i]); + } + + var positions = recording.Length - reference.Length + 1; + var ncc = new double[positions]; + var best = 0; + for (var k = 0; k < positions; k++) + { + var energy = prefix[k + reference.Length] - prefix[k]; + var norm = Math.Sqrt(energy * refEnergy); + ncc[k] = norm > 1e-12 ? Math.Abs(a[k].Real) / norm : 0; + if (ncc[k] > ncc[best]) + { + best = k; + } + } + + var sorted = (double[])ncc.Clone(); + Array.Sort(sorted); + var median = sorted[sorted.Length / 2]; + var psr = Math.Min(999, ncc[best] / Math.Max(median, 1e-3)); + return new ReferenceMatch(best, ncc[best], psr); + } + + /// A repeatable noise-burst train: bursts of white noise with silence between. + public static float[] NoiseBurstTrain(int sampleRate, int bursts = 8, int burstMs = 60, int gapMs = 340, int seed = 12345) + { + var burst = sampleRate * burstMs / 1000; + var gap = sampleRate * gapMs / 1000; + var rng = new Random(seed); + var x = new float[bursts * (burst + gap)]; + for (var n = 0; n < bursts; n++) + { + for (var i = 0; i < burst; i++) + { + var fade = Math.Min(1f, Math.Min(i, burst - 1 - i) / (sampleRate * 0.003f)); // 3 ms edges avoid a pop + x[(n * (burst + gap)) + i] = ((float)(rng.NextDouble() * 2 - 1)) * fade; + } + } + + return x; + } + + private static void Fft(Complex[] x, bool inverse) + { + var n = x.Length; + for (int i = 1, j = 0; i < n; i++) + { + var bit = n >> 1; + for (; (j & bit) != 0; bit >>= 1) + { + j ^= bit; + } + + j ^= bit; + if (i < j) + { + (x[i], x[j]) = (x[j], x[i]); + } + } + + for (var len = 2; len <= n; len <<= 1) + { + var angle = 2 * Math.PI / len * (inverse ? 1 : -1); + var wl = new Complex(Math.Cos(angle), Math.Sin(angle)); + for (var i = 0; i < n; i += len) + { + var w = Complex.One; + for (var k = 0; k < len / 2; k++) + { + var u = x[i + k]; + var v = x[i + k + (len / 2)] * w; + x[i + k] = u + v; + x[i + k + (len / 2)] = u - v; + w *= wl; + } + } + } + + if (inverse) + { + for (var i = 0; i < n; i++) + { + x[i] /= n; + } + } + } +} diff --git a/src/PrimeDictate.Tools.ChannelSkew/OutputGuard.cs b/src/PrimeDictate.Tools.ChannelSkew/OutputGuard.cs new file mode 100644 index 0000000..91e90f6 --- /dev/null +++ b/src/PrimeDictate.Tools.ChannelSkew/OutputGuard.cs @@ -0,0 +1,78 @@ +using System.Diagnostics; +using System.Text.RegularExpressions; + +/// Detects a default output that is a headset or headphones, where a test signal would go straight into someone's ears. +internal static partial class OutputGuard +{ + // Windows PKEY_AudioEndpoint_FormFactor: 1 speakers, 2 line level, 3 headphones, 5 headset, 7 hands-free/handset. + private static readonly int[] HeadFormFactors = [3, 5, 6, 7]; + + public static bool LooksLikeHeadphones(string? name, int? windowsFormFactor, string? pulseFormFactor) + { + if (windowsFormFactor is { } f && HeadFormFactors.Contains(f)) + { + return true; + } + + if (pulseFormFactor is "headphone" or "headset" or "handsfree" or "hands-free" or "hifi" or "portable") + { + return true; + } + + return name is not null && HeadNames().IsMatch(name); + } + + /// Returns a description of the default output when it looks like headphones, otherwise null. + public static string? DetectHeadphones() + { + try + { + if (OperatingSystem.IsWindows()) + { + using var enumerator = new NAudio.CoreAudioApi.MMDeviceEnumerator(); + using var device = enumerator.GetDefaultAudioEndpoint(NAudio.CoreAudioApi.DataFlow.Render, NAudio.CoreAudioApi.Role.Multimedia); + int? form = null; + var key = new NAudio.CoreAudioApi.PropertyKey(new Guid("1da5d803-d492-4edd-8c23-e0c0ffee7f0e"), 0); + if (device.Properties.Contains(key) && device.Properties[key].Value is uint u) + { + form = (int)u; + } + + return LooksLikeHeadphones(device.FriendlyName, form, null) ? $"{device.FriendlyName} (form factor {form?.ToString() ?? "unknown"})" : null; + } + + if (OperatingSystem.IsLinux()) + { + var name = Run("pactl", "get-default-sink")?.Trim(); + var info = Run("pactl", "list sinks") ?? string.Empty; + var block = info.Split("Sink #", StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(b => name is not null && b.Contains($"Name: {name}")); + var form = block is null ? null : PulseForm().Match(block) is { Success: true } m ? m.Groups[1].Value : null; + var desc = block is null ? null : PulseDescription().Match(block) is { Success: true } d ? d.Groups[1].Value : null; + return LooksLikeHeadphones($"{desc} {name}", null, form) ? $"{desc ?? name} (form factor {form ?? "unknown"})" : null; + } + } + catch (Exception) + { + // Could not tell. The level is low regardless, so carry on. + } + + return null; + } + + private static string? Run(string file, string arguments) + { + using var p = Process.Start(new ProcessStartInfo(file, arguments) { RedirectStandardOutput = true, UseShellExecute = false, Environment = { ["LC_ALL"] = "C" } }); + var text = p?.StandardOutput.ReadToEnd(); + p?.WaitForExit(); + return text; + } + + [GeneratedRegex(@"headphone|headset|arctis|airpod|earbud|earphone|\bbuds\b|hands-?free|bluetooth|\bbt\b|wh-1000|quietcomfort|hyperx cloud|corsair void", RegexOptions.IgnoreCase)] + private static partial Regex HeadNames(); + + [GeneratedRegex(@"device\.form_factor = ""([^""]+)""")] + private static partial Regex PulseForm(); + + [GeneratedRegex(@"Description: (.+)")] + private static partial Regex PulseDescription(); +} diff --git a/src/PrimeDictate.Tools.ChannelSkew/PrimeDictate.Tools.ChannelSkew.csproj b/src/PrimeDictate.Tools.ChannelSkew/PrimeDictate.Tools.ChannelSkew.csproj index 5330d6c..e11bcb6 100644 --- a/src/PrimeDictate.Tools.ChannelSkew/PrimeDictate.Tools.ChannelSkew.csproj +++ b/src/PrimeDictate.Tools.ChannelSkew/PrimeDictate.Tools.ChannelSkew.csproj @@ -10,6 +10,7 @@ + diff --git a/src/PrimeDictate.Tools.ChannelSkew/Probe.cs b/src/PrimeDictate.Tools.ChannelSkew/Probe.cs new file mode 100644 index 0000000..17ed03b --- /dev/null +++ b/src/PrimeDictate.Tools.ChannelSkew/Probe.cs @@ -0,0 +1,129 @@ +using System.Diagnostics; +using PrimeDictate.Core.Audio; +using PrimeDictate.Core.Providers; +using PrimeDictate.Platforms.Audio; + +/// +/// Self-contained skew measurement: opens the same combined lease the app uses, plays a known noise-burst +/// train through the default output, and finds that train in each channel with a matched filter. +/// +internal static class Probe +{ + private const int PlayRate = 48_000; + + /// Peak level of the test signal. Deliberately low: it is noise, and it may be next to someone's ears. + private const float PlayLevel = 0.15f; + private const int FadeInMs = 800; + + public static async Task RunAsync(string? saveWavPath, bool allowHeadphones) + { + var headphones = OutputGuard.DetectHeadphones(); + if (headphones is not null && !allowHeadphones) + { + Console.Error.WriteLine($"Refusing to play: the default output looks like headphones or a headset: {headphones}."); + Console.Error.WriteLine("The mic cannot hear those, and the noise goes into your ears. Switch the Windows default output to speakers and rerun. (--allow-headphones overrides; the level is low and fades in.)"); + return 3; + } + + var system = SystemAudioSources.TryCreate(out var reason); + if (system is null) + { + Console.Error.WriteLine($"System audio capture unavailable: {reason}"); + return 2; + } + + MiniAudioCaptureSource mic; + try + { + mic = new MiniAudioCaptureSource(); + } + catch (AudioSourceException ex) + { + Console.Error.WriteLine($"Microphone unavailable: {ex.Message}"); + return 2; + } + + var reference = ReferenceLocator.NoiseBurstTrain(16_000); + for (var i = 0; i < reference.Length; i++) + { + reference[i] *= PlayLevel * Math.Min(1f, i / (16_000f * FadeInMs / 1000)); // the exact signal that is played, so the matched filter still fits + } + + var trainSeconds = reference.Length / 16_000.0; + var left = new List(); + var right = new List(); + var lease = await new CombinedAudioSource(mic, system).OpenAsync(null, default); + await using var _ = lease; + Console.WriteLine($"Microphone: {lease.DeviceName}"); + Console.WriteLine($"Opened combined lease at {lease.Format.SampleRate} Hz, {lease.Format.Channels} ch. Playing a {trainSeconds:0.0} s noise train through the default output; at a low level with a fade-in. Keep the mic within about a metre of the speakers."); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1.5 + trainSeconds + 2.5)); + var clock = Stopwatch.StartNew(); + long playCallSample = -1; + var playTask = Task.Run(async () => + { + await Task.Delay(1500); + playCallSample = left.Count; + await TonePlayer.PlayAsync(reference, 16_000, PlayRate); + }); + + try + { + await foreach (var frame in lease.ReadFramesAsync(cts.Token)) + { + var s = frame.Samples.Span; + for (var i = 0; i + 1 < s.Length; i += 2) + { + left.Add(s[i]); + right.Add(s[i + 1]); + } + } + } + catch (OperationCanceledException) + { + } + + await playTask; + var lc = lease as ICombinedAudioCaptureLease; + var micArr = left.ToArray(); + var sysArr = right.ToArray(); + Console.WriteLine($"Captured {micArr.Length / 16_000.0:0.0} s. Padded silence: mic {lc?.MicrophonePaddedSamples ?? 0} samples, system {lc?.SystemAudioPaddedSamples ?? 0} samples. System opened {lc?.SystemOpenedAfterMicrophone.TotalMilliseconds:0} ms after mic."); + if (saveWavPath is not null) + { + using var writer = new WavFileWriter(saveWavPath, 16_000, 2); + var inter = new float[micArr.Length * 2]; + for (var i = 0; i < micArr.Length; i++) + { + inter[2 * i] = micArr[i]; + inter[(2 * i) + 1] = sysArr[i]; + } + + writer.Write(inter); + Console.WriteLine($"Saved {saveWavPath}"); + } + + var micMatch = ReferenceLocator.Locate(micArr, reference); + var sysMatch = ReferenceLocator.Locate(sysArr, reference); + Report("system (loopback)", sysMatch, playCallSample); + Report("microphone ", micMatch, playCallSample); + if (!sysMatch.IsReliable) + { + Console.WriteLine("RESULT: unusable. The loopback did not capture the test train; check that the default output device is the one playing sound."); + return 1; + } + + if (!micMatch.IsReliable) + { + Console.WriteLine("RESULT: unusable. The microphone did not hear the test train. Turn the speakers up, move the mic closer, or use a cable from output to line-in."); + return 1; + } + + var lagMs = (micMatch.SampleIndex - sysMatch.SampleIndex) * 1000.0 / 16_000; + Console.WriteLine($"RESULT: microphone is {lagMs:+0.0;-0.0;0.0} ms after system audio (positive = mic later)."); + Console.WriteLine("This includes the speaker path: output device latency (Bluetooth can add 100 to 300 ms) plus ~3 ms per metre of air. Wired speakers and a mic within a metre should show under about 60 ms if capture is aligned."); + return 0; + } + + private static void Report(string name, ReferenceMatch m, long playCallSample) => + Console.WriteLine($" {name}: train found at {m.SampleIndex / 16.0:0} ms (play was called at {playCallSample / 16.0:0} ms), correlation {m.Correlation:0.00}, peak/sidelobe {m.PeakToSidelobe:0.0}{(m.IsReliable ? "" : " <- not reliable")}"); +} diff --git a/src/PrimeDictate.Tools.ChannelSkew/Program.cs b/src/PrimeDictate.Tools.ChannelSkew/Program.cs index 078b1e6..6e89b06 100644 --- a/src/PrimeDictate.Tools.ChannelSkew/Program.cs +++ b/src/PrimeDictate.Tools.ChannelSkew/Program.cs @@ -2,9 +2,16 @@ // Usage: ChannelSkew // Left channel = microphone, right channel = system audio. Works when the microphone heard the speakers. +if (args.Length >= 1 && args[0] == "--probe") +{ + // ChannelSkew --probe [--save recording.wav]: self-contained measurement, no app needed. + var save = args.Length >= 3 && args[1] == "--save" ? args[2] : null; + return await Probe.RunAsync(save, args.Contains("--allow-headphones")); +} + if (args.Length != 1 || !File.Exists(args[0])) { - Console.Error.WriteLine("Usage: ChannelSkew "); + Console.Error.WriteLine("Usage: ChannelSkew | ChannelSkew --probe [--save out.wav]"); return 2; } diff --git a/src/PrimeDictate.Tools.ChannelSkew/TonePlayer.cs b/src/PrimeDictate.Tools.ChannelSkew/TonePlayer.cs new file mode 100644 index 0000000..ead0811 --- /dev/null +++ b/src/PrimeDictate.Tools.ChannelSkew/TonePlayer.cs @@ -0,0 +1,65 @@ +using System.Diagnostics; +using NAudio.Wave; + +/// Plays a mono float signal on the default output device: WASAPI on Windows, paplay elsewhere. +internal static class TonePlayer +{ + public static async Task PlayAsync(float[] mono, int sourceRate, int playRate) + { + var up = new float[(long)mono.Length * playRate / sourceRate]; + for (var i = 0; i < up.Length; i++) + { + var pos = (double)i * sourceRate / playRate; + var i0 = (int)pos; + var i1 = Math.Min(i0 + 1, mono.Length - 1); + var f = (float)(pos - i0); + up[i] = (mono[i0] * (1 - f)) + (mono[i1] * f); + } + + if (OperatingSystem.IsWindows()) + { + var bytes = new byte[up.Length * 2 * sizeof(float)]; + for (var i = 0; i < up.Length; i++) + { + var b = BitConverter.GetBytes(up[i]); + b.CopyTo(bytes, i * 8); + b.CopyTo(bytes, (i * 8) + 4); + } + + using var stream = new RawSourceWaveStream(new MemoryStream(bytes), WaveFormat.CreateIeeeFloatWaveFormat(playRate, 2)); + using var output = new WasapiOut(NAudio.CoreAudioApi.AudioClientShareMode.Shared, 50); + output.Init(stream); + output.Play(); + while (output.PlaybackState == PlaybackState.Playing) + { + await Task.Delay(20); + } + + return; + } + + var path = Path.Combine(Path.GetTempPath(), $"pd-probe-{Environment.ProcessId}.wav"); + try + { + using (var w = new WaveFileWriter(path, new WaveFormat(playRate, 16, 2))) + { + foreach (var v in up) + { + var s = (short)(Math.Clamp(v, -1f, 1f) * short.MaxValue); + w.WriteByte((byte)(s & 0xFF)); + w.WriteByte((byte)((s >> 8) & 0xFF)); + w.WriteByte((byte)(s & 0xFF)); + w.WriteByte((byte)((s >> 8) & 0xFF)); + } + } + + using var p = Process.Start(new ProcessStartInfo("paplay", $"\"{path}\"") { UseShellExecute = false }) + ?? throw new InvalidOperationException("Could not start paplay."); + await p.WaitForExitAsync(); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/PrimeDictate.Core.Tests/ReferenceLocatorTests.cs b/tests/PrimeDictate.Core.Tests/ReferenceLocatorTests.cs new file mode 100644 index 0000000..1a15645 --- /dev/null +++ b/tests/PrimeDictate.Core.Tests/ReferenceLocatorTests.cs @@ -0,0 +1,44 @@ +using PrimeDictate.Core.Audio; + +namespace PrimeDictate.Core.Tests; + +public sealed class ReferenceLocatorTests +{ + [Theory] + [InlineData(0, 1.0f)] + [InlineData(4321, 1.0f)] + [InlineData(20000, 0.02f)] + public void FindsTrainAtKnownOffsetEvenWhenFaintAndNoisy(int offset, float gain) + { + var train = ReferenceLocator.NoiseBurstTrain(16_000); + var rng = new Random(7); + var recording = new float[train.Length + 40_000]; + for (var i = 0; i < recording.Length; i++) + { + recording[i] = (float)(rng.NextDouble() - 0.5) * 0.02f; // room noise + } + + for (var i = 0; i < train.Length; i++) + { + recording[offset + i] += train[i] * gain; + } + + var match = ReferenceLocator.Locate(recording, train); + Assert.Equal(offset, match.SampleIndex); + Assert.True(match.IsReliable, $"corr {match.Correlation}, psr {match.PeakToSidelobe}"); + } + + [Fact] + public void ReportsUnreliableWhenTrainIsAbsent() + { + var train = ReferenceLocator.NoiseBurstTrain(16_000); + var rng = new Random(9); + var recording = new float[train.Length + 10_000]; + for (var i = 0; i < recording.Length; i++) + { + recording[i] = (float)(rng.NextDouble() - 0.5); + } + + Assert.False(ReferenceLocator.Locate(recording, train).IsReliable); + } +}