From a4d5c7207d8dcfa0aaa2469ee328a020bd8450d1 Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Mon, 8 Apr 2024 13:16:35 +0530 Subject: [PATCH 01/10] Support System audio loopback using Screen Capture Kit API --- CMakeLists.txt | 17 + include/portaudio.h | 3 +- .../pa_mac_screencapturekit.m | 494 ++++++++++++++++++ src/os/unix/pa_unix_hostapis.c | 5 + 4 files changed, 518 insertions(+), 1 deletion(-) create mode 100644 src/hostapi/screencapturekit/pa_mac_screencapturekit.m diff --git a/CMakeLists.txt b/CMakeLists.txt index 77e5388b5..7cee3f138 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -330,6 +330,23 @@ elseif(UNIX) set(PKGCONFIG_LDFLAGS_PRIVATE "${PKGCONFIG_LDFLAGS_PRIVATE} -framework CoreAudio -framework AudioToolbox -framework AudioUnit -framework CoreFoundation -framework CoreServices") + if(CMAKE_SYSTEM_VERSION VERSION_GREATER_EQUAL "13.0") + # Add the ScreenCaptureKit framework + find_library(SCREENCAPTUREKIT_FRAMEWORK ScreenCaptureKit) + if(SCREENCAPTUREKIT_FRAMEWORK) + target_sources(PortAudio PRIVATE + src/hostapi/screencapturekit/pa_mac_screencapturekit.m + ) + target_link_libraries(PortAudio + PRIVATE + -Wl,-framework,ScreenCaptureKit + -Wl,-framework,CoreMedia + ) + target_compile_definitions(PortAudio PUBLIC PA_USE_SCREENCAPTUREKIT=1) + set(PKGCONFIG_CFLAGS "${PKGCONFIG_CFLAGS} -DPA_USE_SCREENCAPTUREKIT=1") + set(PKGCONFIG_LDFLAGS_PRIVATE "${PKGCONFIG_LDFLAGS_PRIVATE} -framework ScreenCaptureKit -framework CoreMedia") + endif() + endif() else() # Some BSDs have a reimplementation of alsalib, so do not explicitly check for Linux. find_package(ALSA) diff --git a/include/portaudio.h b/include/portaudio.h index deb210eb1..7ebe9b708 100644 --- a/include/portaudio.h +++ b/include/portaudio.h @@ -291,7 +291,8 @@ typedef enum PaHostApiTypeId paAudioScienceHPI=14, paAudioIO=15, paPulseAudio=16, - paSndio=17 + paSndio=17, + paScreenCaptureKit=18, } PaHostApiTypeId; diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m new file mode 100644 index 000000000..8b638d4d8 --- /dev/null +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -0,0 +1,494 @@ +/* + * Portable Audio I/O Library Screen Capture Kit implementation + * Copyright (c) 2006-2010 David Viens + * Copyright (c) 2010-2023 Dmitry Kostjuchenko + * + * Based on the Open Source API proposed by Ross Bencina + * Copyright (c) 1999-2019 Ross Bencina, Phil Burk + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * The text above constitutes the entire PortAudio license; however, + * the PortAudio community also makes the following non-binding requests: + * + * Any person wishing to distribute modifications to the Software is + * requested to send the modifications to the original developer so that + * they can be incorporated into the canonical version. It is also + * requested that these non-binding requests be included along with the + * license above. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +@interface ScreenCaptureKitStreamOutput : NSObject +@property(nonatomic, assign) PaUtilRingBuffer *ringBuffer; +@end + +typedef struct PaScreenCaptureKitHostApiRepresentation +{ + PaUtilHostApiRepresentation inheritedHostApiRep; + PaUtilStreamInterface blockingStreamInterface; +} PaScreenCaptureKitHostApiRepresentation; + +typedef struct PaScreenCaptureKitStream +{ + PaUtilStreamRepresentation streamRepresentation; + SCStream *audioStream; + ScreenCaptureKitStreamOutput *streamOutput; + PaUtilRingBuffer ringBuffer; + BOOL isStopped; + int sampleRate; +} PaScreenCaptureKitStream; + +@implementation ScreenCaptureKitStreamOutput + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type +{ + if (type == SCStreamOutputTypeAudio) + { + // Get the audio buffer list from the sample buffer + AudioBufferList *bufferList = NULL; + size_t bufferListSize = 0; + OSStatus status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, &bufferListSize, NULL, + 0, NULL, NULL, 0, NULL); + + if (status != noErr) + { + PA_DEBUG(("Failed to get audio buffer list size\n")); + goto exit; + } + + bufferList = (AudioBufferList *)PaUtil_AllocateZeroInitializedMemory(bufferListSize); + if (bufferList == NULL) + { + PA_DEBUG(("Failed to allocate memory for audio buffer list\n")); + goto exit; + } + + CMBlockBufferRef blockBuffer = NULL; + + status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, NULL, bufferList, bufferListSize, + NULL, NULL, 0, &blockBuffer); + + if (status != noErr) + { + PA_DEBUG(("Failed to get audio buffer list status=%d\n", status)); + goto exit; + } + // Get the format description from the sample buffer + CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer); + const AudioStreamBasicDescription *asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription); + if (asbd == NULL) + { + PA_DEBUG(("Failed to get audio stream basic description\n")); + goto exit; + } + + // Check the sample format + if (asbd->mFormatID == kAudioFormatLinearPCM && (asbd->mFormatFlags & kAudioFormatFlagIsFloat)) + { + // Samples are in float format + for (int i = 0; i < bufferList->mNumberBuffers; i++) + { + PaUtil_WriteRingBuffer(self.ringBuffer, bufferList->mBuffers[i].mData, + bufferList->mBuffers[i].mDataByteSize / asbd->mBytesPerFrame); + } + } + else + { + PA_DEBUG(("Unsupported audio format\n")); + } + + exit: + if (blockBuffer) + CFRelease(blockBuffer); + + if (bufferList) + PaUtil_FreeMemory(bufferList); + + return; + } +} +@end + +// PortAudio host API stream read function +static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) +{ + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + // TODO : Need to implement exit on StopStream, Closestream or similar + while (PaUtil_GetRingBufferReadAvailable(&stream->ringBuffer) < frames) + { + // Sleep for 10ms + usleep(10000); + } + + ring_buffer_size_t read = PaUtil_ReadRingBuffer(&stream->ringBuffer, buffer, frames); + + return paNoError; +} + +// PortAudio host API is format supported function +static PaError IsFormatSupported(struct PaUtilHostApiRepresentation *hostApi, const PaStreamParameters *inputParameters, + const PaStreamParameters *outputParameters, double sampleRate) +{ + if (inputParameters == NULL) + { + return paInvalidDevice; + } + // Check if the input parameters are supported + if (inputParameters != NULL) + { + // Check if the number of channels is supported + if (inputParameters->channelCount > hostApi->deviceInfos[0]->maxInputChannels) + { + return paInvalidChannelCount; + } + + // Check if the sample format is supported + if (inputParameters->sampleFormat != paFloat32) + { + return paSampleFormatNotSupported; + } + } + + // Check if the output parameters are supported + if (outputParameters != NULL) + { + return paInvalidDevice; + } + + if (sampleRate == 0) + { + return paInvalidSampleRate; + } + + return paFormatIsSupported; +} + +// PortAudio host API stream open function +static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream **s, + const PaStreamParameters *inputParameters, const PaStreamParameters *outputParameters, + double sampleRate, unsigned long framesPerBuffer, PaStreamFlags streamFlags, + PaStreamCallback *streamCallback, void *userData) +{ + PaError result = IsFormatSupported(hostApi, inputParameters, outputParameters, sampleRate); + if (result != paFormatIsSupported) + return result; + if (streamCallback != NULL) + return paCanNotWriteToACallbackStream; + PaScreenCaptureKitHostApiRepresentation *paSck = (PaScreenCaptureKitHostApiRepresentation *)hostApi; + PaScreenCaptureKitStream *stream = NULL; + result = paNoError; + __block NSArray *displays = nil; + if ((stream = (PaScreenCaptureKitStream *)PaUtil_AllocateZeroInitializedMemory(sizeof(PaScreenCaptureKitStream))) == + NULL) + { + result = paInsufficientMemory; + goto error; + } + + dispatch_group_t handlerGroup = dispatch_group_create(); + + dispatch_group_enter(handlerGroup); + + [SCShareableContent + getShareableContentWithCompletionHandler:^(SCShareableContent *shareableContent, NSError *error) { + if (error) + { + // Handle the error + PA_DEBUG(("Failed to retrieve shareable content: %s\n", [error.localizedDescription UTF8String])); + } + else + { + // Store the array of available displays in the global variable + displays = [shareableContent.displays retain]; + } + + // Leave the dispatch group + dispatch_group_leave(handlerGroup); + }]; + + dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + + // Check the size of displays array + if (!displays || displays.count < 1) + { + PA_DEBUG(("No displays found\n")); + result = paInternalError; + goto error; + } + + SCDisplay *firstDisplay = displays[0]; + + NSArray *exclude_windows = nil; + SCContentFilter *contentFilter = [[SCContentFilter alloc] initWithDisplay:firstDisplay + excludingWindows:exclude_windows]; + if (!contentFilter) + { + PA_DEBUG(("Failed to create content filter\n")); + result = paInternalError; + goto error; + } + // Create a screen capture configuration + SCStreamConfiguration *streamConfig = [[SCStreamConfiguration alloc] init]; + if (!streamConfig) + { + PA_DEBUG(("Failed to create stream configuration\n")); + result = paInternalError; + goto error; + } + streamConfig.capturesAudio = YES; + streamConfig.sampleRate = stream->sampleRate = sampleRate; + streamConfig.channelCount = inputParameters->channelCount; + + // Create an audio capture session + stream->audioStream = [[SCStream alloc] initWithFilter:contentFilter configuration:streamConfig delegate:nil]; + if (!stream->audioStream) + { + PA_DEBUG(("Failed to create audio stream\n")); + result = paInternalError; + goto error; + } + + stream->streamOutput = [[ScreenCaptureKitStreamOutput alloc] init]; + if (!stream->streamOutput) + { + PA_DEBUG(("Failed to create stream output\n")); + result = paInternalError; + goto error; + } + NSError *error = nil; + + BOOL success = [stream->audioStream addStreamOutput:stream->streamOutput + type:SCStreamOutputTypeAudio + sampleHandlerQueue:nil + error:&error]; + if (!success || error) + { + PA_DEBUG(("Failed to add stream output: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + goto error; + } + + success = [stream->audioStream addStreamOutput:stream->streamOutput + type:SCStreamOutputTypeScreen + sampleHandlerQueue:nil + error:&error]; + if (!success || error) + { + PA_DEBUG(("Failed to add stream output: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + goto error; + } + + PaUtil_InitializeStreamRepresentation(&stream->streamRepresentation, &paSck->blockingStreamInterface, NULL, NULL); + stream->isStopped = TRUE; + + // Around 1 second of ringbuffer (allocated to nearest power of 2) + int audioBufferSize = 1 << (int)ceil(log2(sampleRate)); + void *data = PaUtil_AllocateZeroInitializedMemory(audioBufferSize * sizeof(float)); + if (!data) + { + PA_DEBUG(("Failed to allocate memory for ringbuffer")); + result = paInternalError; + goto error; + } + PaUtil_InitializeRingBuffer(&stream->ringBuffer, sizeof(float), audioBufferSize, data); + stream->streamOutput.ringBuffer = &stream->ringBuffer; + + [displays release]; + *s = (PaStream *)stream; + return paNoError; +error: + if (stream) + { + PaUtil_FreeMemory(stream); + } + if (displays) + [displays release]; + return result; +} + +// PortAudio host API stream close function +static PaError CloseStream(PaStream *s) +{ + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + PaUtil_FreeMemory(stream->ringBuffer.buffer); + PaUtil_FreeMemory(stream); + return paNoError; +} + +static PaError IsStreamStopped(PaStream *s) +{ + return ((PaScreenCaptureKitStream *)s)->isStopped; +} + +static PaError IsStreamActive(PaStream *s) +{ + return !IsStreamStopped(s); +} + +static PaError StartStream(PaStream *s) +{ + if (IsStreamActive(s)) + return paStreamIsNotStopped; + __block PaError result = paNoError; + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + dispatch_group_t handlerGroup = dispatch_group_create(); + dispatch_group_enter(handlerGroup); + + // Start the audio capture session + [stream->audioStream startCaptureWithCompletionHandler:^(NSError *error) { + if (error) + { + PA_DEBUG(("Failed to start audio capture: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + } + dispatch_group_leave(handlerGroup); + }]; + dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + if (result == paNoError) + { + stream->isStopped = FALSE; + } + + return result; +} + +// PortAudio host API stream stop function +static PaError StopStream(PaStream *s) +{ + if (IsStreamStopped(s)) + return paStreamIsStopped; + __block PaError result = paNoError; + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + dispatch_group_t handlerGroup = dispatch_group_create(); + dispatch_group_enter(handlerGroup); + // Stop the audio capture session + [stream->audioStream stopCaptureWithCompletionHandler:^(NSError *error) { + if (error) + { + PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + return; + } + dispatch_group_leave(handlerGroup); + }]; + stream->isStopped = TRUE; + dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + + return result; +} + +// PortAudio host API stream abort function +static PaError AbortStream(PaStream *s) +{ + return StopStream(s); +} + +static void Terminate(struct PaUtilHostApiRepresentation *hostApi) +{ + // Free the host API representation + if (hostApi != NULL) + { + if (hostApi->deviceInfos[0] != NULL) + PaUtil_FreeMemory(hostApi->deviceInfos[0]); + if (hostApi->deviceInfos != NULL) + PaUtil_FreeMemory(hostApi->deviceInfos); + PaUtil_FreeMemory(hostApi); + } + + return; +} + +static PaTime GetStreamTime(PaStream *s) +{ + /* suppress unused variable warnings */ + (void)s; + + return PaUtil_GetTime(); +} + +PaError PaMacScreenCapture_Initialize(PaUtilHostApiRepresentation **hostApi, PaHostApiIndex hostApiIndex) +{ + PaError result = paNoError; + PaScreenCaptureKitHostApiRepresentation *paSck = NULL; + + paSck = (PaScreenCaptureKitHostApiRepresentation *)PaUtil_AllocateZeroInitializedMemory( + sizeof(PaScreenCaptureKitHostApiRepresentation)); + if (paSck == NULL) + { + result = paInsufficientMemory; + goto error; + } + + *hostApi = &paSck->inheritedHostApiRep; + // Set the host API function pointers + (*hostApi)->info.structVersion = 1; + (*hostApi)->info.type = paScreenCaptureKit; + (*hostApi)->info.name = "Mac ScreenCaptureKit"; + (*hostApi)->info.deviceCount = 1; + (*hostApi)->info.defaultInputDevice = 0; + (*hostApi)->info.defaultOutputDevice = paNoDevice; + (*hostApi)->Terminate = Terminate; + (*hostApi)->OpenStream = OpenStream; + (*hostApi)->IsFormatSupported = IsFormatSupported; + + (*hostApi)->deviceInfos = PaUtil_AllocateZeroInitializedMemory(sizeof(PaDeviceInfo *)); + if ((*hostApi)->deviceInfos == NULL) + { + result = paInsufficientMemory; + goto error; + } + + (*hostApi)->deviceInfos[0] = PaUtil_AllocateZeroInitializedMemory(sizeof(PaDeviceInfo)); + if ((*hostApi)->deviceInfos[0] == NULL) + { + result = paInsufficientMemory; + goto error; + } + + (*hostApi)->deviceInfos[0]->structVersion = 2; + (*hostApi)->deviceInfos[0]->hostApi = hostApiIndex; + (*hostApi)->deviceInfos[0]->name = "System Audio [Loopback]"; + (*hostApi)->deviceInfos[0]->maxInputChannels = 1; + (*hostApi)->deviceInfos[0]->defaultSampleRate = 44100; + + PaUtil_InitializeStreamInterface(&paSck->blockingStreamInterface, CloseStream, StartStream, StopStream, AbortStream, + IsStreamStopped, IsStreamActive, GetStreamTime, PaUtil_DummyGetCpuLoad, ReadStream, + PaUtil_DummyWrite, PaUtil_DummyGetReadAvailable, PaUtil_DummyGetWriteAvailable); + + return result; + +error: + if (paSck != NULL) + Terminate((PaUtilHostApiRepresentation *)paSck); + + return result; +} \ No newline at end of file diff --git a/src/os/unix/pa_unix_hostapis.c b/src/os/unix/pa_unix_hostapis.c index 1fc022f30..04eedac13 100644 --- a/src/os/unix/pa_unix_hostapis.c +++ b/src/os/unix/pa_unix_hostapis.c @@ -53,6 +53,7 @@ PaError PaSGI_Initialize( PaUtilHostApiRepresentation **hostApi, PaHostApiIndex /* Linux AudioScience HPI */ PaError PaAsiHpi_Initialize( PaUtilHostApiRepresentation **hostApi, PaHostApiIndex index ); PaError PaMacCore_Initialize( PaUtilHostApiRepresentation **hostApi, PaHostApiIndex index ); +PaError PaMacScreenCapture_Initialize( PaUtilHostApiRepresentation **hostApi, PaHostApiIndex index ); PaError PaSkeleton_Initialize( PaUtilHostApiRepresentation **hostApi, PaHostApiIndex index ); /** Note that on Linux, ALSA is placed before OSS so that the former is preferred over the latter. @@ -110,6 +111,10 @@ PaUtilHostApiInitializer *paHostApiInitializers[] = PaMacCore_Initialize, #endif +#if PA_USE_SCREENCAPTUREKIT + PaMacScreenCapture_Initialize, +#endif + #if PA_USE_PULSEAUDIO PaPulseAudio_Initialize, #endif From 28acd77d2e2be91413f29a846909a87645c48139 Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Sun, 12 May 2024 17:04:48 +0530 Subject: [PATCH 02/10] ScreenCaptureKit: Add support for callback APIs --- .../pa_mac_screencapturekit.m | 120 ++++++++++++++---- 1 file changed, 93 insertions(+), 27 deletions(-) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 8b638d4d8..f5bc0dcd2 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -54,6 +55,7 @@ @interface ScreenCaptureKitStreamOutput : NSObject { PaUtilHostApiRepresentation inheritedHostApiRep; PaUtilStreamInterface blockingStreamInterface; + PaUtilStreamInterface callbackStreamInterface; } PaScreenCaptureKitHostApiRepresentation; typedef struct PaScreenCaptureKitStream @@ -64,6 +66,10 @@ @interface ScreenCaptureKitStreamOutput : NSObject PaUtilRingBuffer ringBuffer; BOOL isStopped; int sampleRate; + PaStreamCallback *streamCallback; + void *userData; + unsigned long framesPerBuffer; + pthread_t callbackThreadId; } PaScreenCaptureKitStream; @implementation ScreenCaptureKitStreamOutput @@ -137,6 +143,27 @@ - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampl } @end +static PaError StopStreamInternal(PaStream *s) +{ + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + __block PaError result = paNoError; + dispatch_group_t handlerGroup = dispatch_group_create(); + dispatch_group_enter(handlerGroup); + // Stop the audio capture session + [stream->audioStream stopCaptureWithCompletionHandler:^(NSError *error) { + if (error) + { + PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + return; + } + dispatch_group_leave(handlerGroup); + }]; + stream->isStopped = TRUE; + dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + return result; +} + // PortAudio host API stream read function static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) { @@ -153,6 +180,35 @@ static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) return paNoError; } +static void *StreamProcessingThread(void *userData) +{ + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)userData; + while (!stream->isStopped) + { + // Wait until enough data is available in the ring buffer + if (PaUtil_GetRingBufferReadAvailable(&stream->ringBuffer) >= stream->framesPerBuffer) + { + float buffer[stream->framesPerBuffer]; + // Copy the data to the buffer + ring_buffer_size_t read = PaUtil_ReadRingBuffer(&stream->ringBuffer, buffer, stream->framesPerBuffer); + + const PaStreamCallbackTimeInfo timeInfo = {0, 0, 0}; // TODO : Fill the timestamps + PaStreamCallbackFlags statusFlags = 0; // TODO : Determine underflow/overflow flags as needed + + // Call the user callback + PaStreamCallbackResult callbackResult = stream->streamCallback(buffer, NULL, + stream->framesPerBuffer, &timeInfo, statusFlags, stream->userData); + + if (callbackResult != paContinue) + { + StopStreamInternal((PaStream *)stream); + } + } + usleep(1000); + } + return NULL; +} + // PortAudio host API is format supported function static PaError IsFormatSupported(struct PaUtilHostApiRepresentation *hostApi, const PaStreamParameters *inputParameters, const PaStreamParameters *outputParameters, double sampleRate) @@ -200,8 +256,6 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream PaError result = IsFormatSupported(hostApi, inputParameters, outputParameters, sampleRate); if (result != paFormatIsSupported) return result; - if (streamCallback != NULL) - return paCanNotWriteToACallbackStream; PaScreenCaptureKitHostApiRepresentation *paSck = (PaScreenCaptureKitHostApiRepresentation *)hostApi; PaScreenCaptureKitStream *stream = NULL; result = paNoError; @@ -307,7 +361,19 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream goto error; } - PaUtil_InitializeStreamRepresentation(&stream->streamRepresentation, &paSck->blockingStreamInterface, NULL, NULL); + if (streamCallback) + { + PaUtil_InitializeStreamRepresentation(&stream->streamRepresentation, &paSck->callbackStreamInterface, + streamCallback, userData); + stream->streamCallback = streamCallback; + stream->userData = userData; + stream->framesPerBuffer = framesPerBuffer; + } + else + { + PaUtil_InitializeStreamRepresentation(&stream->streamRepresentation, &paSck->blockingStreamInterface, NULL, + NULL); + } stream->isStopped = TRUE; // Around 1 second of ringbuffer (allocated to nearest power of 2) @@ -335,15 +401,6 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream return result; } -// PortAudio host API stream close function -static PaError CloseStream(PaStream *s) -{ - PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; - PaUtil_FreeMemory(stream->ringBuffer.buffer); - PaUtil_FreeMemory(stream); - return paNoError; -} - static PaError IsStreamStopped(PaStream *s) { return ((PaScreenCaptureKitStream *)s)->isStopped; @@ -360,6 +417,7 @@ static PaError StartStream(PaStream *s) return paStreamIsNotStopped; __block PaError result = paNoError; PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + dispatch_group_t handlerGroup = dispatch_group_create(); dispatch_group_enter(handlerGroup); @@ -377,6 +435,14 @@ static PaError StartStream(PaStream *s) { stream->isStopped = FALSE; } + if (stream->streamCallback != NULL) + { + int result = pthread_create(&stream->callbackThreadId, NULL, StreamProcessingThread, stream); + if (result != 0) { + PA_DEBUG(("Failed to create audio processing thread\n")); + return paUnanticipatedHostError; + } + } return result; } @@ -386,22 +452,9 @@ static PaError StopStream(PaStream *s) { if (IsStreamStopped(s)) return paStreamIsStopped; - __block PaError result = paNoError; PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; - dispatch_group_t handlerGroup = dispatch_group_create(); - dispatch_group_enter(handlerGroup); - // Stop the audio capture session - [stream->audioStream stopCaptureWithCompletionHandler:^(NSError *error) { - if (error) - { - PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); - result = paInternalError; - return; - } - dispatch_group_leave(handlerGroup); - }]; - stream->isStopped = TRUE; - dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + PaError result = StopStreamInternal(s); + pthread_join(stream->callbackThreadId, NULL); return result; } @@ -412,6 +465,16 @@ static PaError AbortStream(PaStream *s) return StopStream(s); } +// PortAudio host API stream close function +static PaError CloseStream(PaStream *s) +{ + StopStream(s); + PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + PaUtil_FreeMemory(stream->ringBuffer.buffer); + PaUtil_FreeMemory(stream); + return paNoError; +} + static void Terminate(struct PaUtilHostApiRepresentation *hostApi) { // Free the host API representation @@ -484,6 +547,9 @@ PaError PaMacScreenCapture_Initialize(PaUtilHostApiRepresentation **hostApi, PaH IsStreamStopped, IsStreamActive, GetStreamTime, PaUtil_DummyGetCpuLoad, ReadStream, PaUtil_DummyWrite, PaUtil_DummyGetReadAvailable, PaUtil_DummyGetWriteAvailable); + PaUtil_InitializeStreamInterface(&paSck->callbackStreamInterface, CloseStream, StartStream, StopStream, AbortStream, + IsStreamStopped, IsStreamActive, GetStreamTime, PaUtil_DummyGetCpuLoad, PaUtil_DummyRead, + PaUtil_DummyWrite, PaUtil_DummyGetReadAvailable, PaUtil_DummyGetWriteAvailable); return result; error: From ffcc4f6c1f6ed98ea4906eeb6792da592b43ce95 Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Mon, 20 May 2024 21:44:09 +0530 Subject: [PATCH 03/10] Minor fixes/improvements - Changed the sample rate supported to only 48000. Other sampling rates seem to have some issue. - Reduced the sleep time for lower latency - Reduced the buffer size for lower latency --- .../screencapturekit/pa_mac_screencapturekit.m | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index f5bc0dcd2..3a3b077d0 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -72,6 +72,8 @@ @interface ScreenCaptureKitStreamOutput : NSObject pthread_t callbackThreadId; } PaScreenCaptureKitStream; +#define SAMPLE_RATE 48000 + @implementation ScreenCaptureKitStreamOutput - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type @@ -171,8 +173,8 @@ static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) // TODO : Need to implement exit on StopStream, Closestream or similar while (PaUtil_GetRingBufferReadAvailable(&stream->ringBuffer) < frames) { - // Sleep for 10ms - usleep(10000); + // Sleep for 1ms + usleep(1000); } ring_buffer_size_t read = PaUtil_ReadRingBuffer(&stream->ringBuffer, buffer, frames); @@ -239,7 +241,7 @@ static PaError IsFormatSupported(struct PaUtilHostApiRepresentation *hostApi, co return paInvalidDevice; } - if (sampleRate == 0) + if (sampleRate != SAMPLE_RATE) { return paInvalidSampleRate; } @@ -376,8 +378,8 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream } stream->isStopped = TRUE; - // Around 1 second of ringbuffer (allocated to nearest power of 2) - int audioBufferSize = 1 << (int)ceil(log2(sampleRate)); + // Around 100 ms of ringbuffer (allocated to nearest power of 2) + int audioBufferSize = 1 << (int)ceil(log2(sampleRate / 10)); void *data = PaUtil_AllocateZeroInitializedMemory(audioBufferSize * sizeof(float)); if (!data) { @@ -541,7 +543,7 @@ PaError PaMacScreenCapture_Initialize(PaUtilHostApiRepresentation **hostApi, PaH (*hostApi)->deviceInfos[0]->hostApi = hostApiIndex; (*hostApi)->deviceInfos[0]->name = "System Audio [Loopback]"; (*hostApi)->deviceInfos[0]->maxInputChannels = 1; - (*hostApi)->deviceInfos[0]->defaultSampleRate = 44100; + (*hostApi)->deviceInfos[0]->defaultSampleRate = SAMPLE_RATE; PaUtil_InitializeStreamInterface(&paSck->blockingStreamInterface, CloseStream, StartStream, StopStream, AbortStream, IsStreamStopped, IsStreamActive, GetStreamTime, PaUtil_DummyGetCpuLoad, ReadStream, From cade2ce5afa42e549eeea0a869c69a98d45250be Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Wed, 5 Jun 2024 00:47:54 +0530 Subject: [PATCH 04/10] Fix the hang issue in ReadStream --- .../pa_mac_screencapturekit.m | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 3a3b077d0..8f3eddb12 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -157,7 +157,6 @@ static PaError StopStreamInternal(PaStream *s) { PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); result = paInternalError; - return; } dispatch_group_leave(handlerGroup); }]; @@ -170,11 +169,22 @@ static PaError StopStreamInternal(PaStream *s) static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) { PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + // Sleep for 1ms + const int sleepDurationMs = 1; + // Set a timeout of 1 second + the time it takes to read the frames + const unsigned long timeoutMs = 1000 + ((frames * 1000) / stream->sampleRate); + const int maxIterations = timeoutMs / sleepDurationMs; + int numIterations = 0; // TODO : Need to implement exit on StopStream, Closestream or similar - while (PaUtil_GetRingBufferReadAvailable(&stream->ringBuffer) < frames) + while (PaUtil_GetRingBufferReadAvailable(&stream->ringBuffer) < frames && + numIterations++ < maxIterations) { - // Sleep for 1ms - usleep(1000); + usleep(sleepDurationMs * 1000); + } + + if (PaUtil_GetRingBufferReadAvailable(&stream->ringBuffer) < frames) + { + return paTimedOut; } ring_buffer_size_t read = PaUtil_ReadRingBuffer(&stream->ringBuffer, buffer, frames); @@ -433,10 +443,10 @@ static PaError StartStream(PaStream *s) dispatch_group_leave(handlerGroup); }]; dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); - if (result == paNoError) - { - stream->isStopped = FALSE; - } + if (result != paNoError) + return result; + + stream->isStopped = FALSE; if (stream->streamCallback != NULL) { int result = pthread_create(&stream->callbackThreadId, NULL, StreamProcessingThread, stream); From fed7a393279a8ae111210e7f69da967eb1caf94d Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Tue, 11 Jun 2024 23:21:13 +0530 Subject: [PATCH 05/10] Screen Capture Kit : Handle stream stopped error --- .../pa_mac_screencapturekit.m | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 8f3eddb12..69a3d99a3 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -47,6 +47,13 @@ #include #include +typedef struct PaScreenCaptureKitStream PaScreenCaptureKitStream; +static PaError StopStreamInternal(PaStream *s); + +@interface ScreenCaptureDelegate : NSObject +@property (nonatomic, assign) PaScreenCaptureKitStream *stream; +@end + @interface ScreenCaptureKitStreamOutput : NSObject @property(nonatomic, assign) PaUtilRingBuffer *ringBuffer; @end @@ -63,6 +70,7 @@ @interface ScreenCaptureKitStreamOutput : NSObject PaUtilStreamRepresentation streamRepresentation; SCStream *audioStream; ScreenCaptureKitStreamOutput *streamOutput; + ScreenCaptureDelegate *delegate; PaUtilRingBuffer ringBuffer; BOOL isStopped; int sampleRate; @@ -74,6 +82,23 @@ @interface ScreenCaptureKitStreamOutput : NSObject #define SAMPLE_RATE 48000 +@implementation ScreenCaptureDelegate + +- (void)outputVideoEffectDidStartForStream:(SCStream *)stream { +} + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + fprintf(stderr, "Stream encountered an error\n"); + if (self.stream) { + StopStreamInternal((PaStream *)self.stream); + } +} + +- (void)outputVideoEffectDidStopForStream:(SCStream *)stream { +} + +@end + @implementation ScreenCaptureKitStreamOutput - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type @@ -169,6 +194,8 @@ static PaError StopStreamInternal(PaStream *s) static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) { PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + if (stream->isStopped) + return paStreamIsStopped; // Sleep for 1ms const int sleepDurationMs = 1; // Set a timeout of 1 second + the time it takes to read the frames @@ -333,8 +360,11 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream streamConfig.sampleRate = stream->sampleRate = sampleRate; streamConfig.channelCount = inputParameters->channelCount; + stream->delegate = [[ScreenCaptureDelegate alloc] init]; // Create an instance of the delegate + stream->delegate.stream = stream; + // Create an audio capture session - stream->audioStream = [[SCStream alloc] initWithFilter:contentFilter configuration:streamConfig delegate:nil]; + stream->audioStream = [[SCStream alloc] initWithFilter:contentFilter configuration:streamConfig delegate:stream->delegate]; if (!stream->audioStream) { PA_DEBUG(("Failed to create audio stream\n")); @@ -401,13 +431,22 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream stream->streamOutput.ringBuffer = &stream->ringBuffer; [displays release]; + [contentFilter release]; *s = (PaStream *)stream; return paNoError; error: if (stream) { + if (stream->audioStream) + [stream->audioStream release]; + if (stream->streamOutput) + [stream->streamOutput release]; + if (stream->delegate) + [stream->delegate release]; PaUtil_FreeMemory(stream); } + if (contentFilter) + [contentFilter release]; if (displays) [displays release]; return result; @@ -482,6 +521,9 @@ static PaError CloseStream(PaStream *s) { StopStream(s); PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + [stream->audioStream release]; + [stream->streamOutput release]; + [stream->delegate release]; PaUtil_FreeMemory(stream->ringBuffer.buffer); PaUtil_FreeMemory(stream); return paNoError; From 8b7348617f7b5b594f358cf2f184083c1aa1fd07 Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Sun, 8 Sep 2024 12:04:50 +0530 Subject: [PATCH 06/10] Handle race condition in StopStreamInternal function --- .../pa_mac_screencapturekit.m | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 69a3d99a3..08baa0611 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -78,6 +78,7 @@ @interface ScreenCaptureKitStreamOutput : NSObject void *userData; unsigned long framesPerBuffer; pthread_t callbackThreadId; + pthread_mutex_t stopMutex; } PaScreenCaptureKitStream; #define SAMPLE_RATE 48000 @@ -173,6 +174,13 @@ - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampl static PaError StopStreamInternal(PaStream *s) { PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + pthread_mutex_lock(&stream->stopMutex); + + if (stream->isStopped) { + pthread_mutex_unlock(&stream->stopMutex); + return paNoError; + } + __block PaError result = paNoError; dispatch_group_t handlerGroup = dispatch_group_create(); dispatch_group_enter(handlerGroup); @@ -187,6 +195,7 @@ static PaError StopStreamInternal(PaStream *s) }]; stream->isStopped = TRUE; dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + pthread_mutex_unlock(&stream->stopMutex); return result; } @@ -297,6 +306,7 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream return result; PaScreenCaptureKitHostApiRepresentation *paSck = (PaScreenCaptureKitHostApiRepresentation *)hostApi; PaScreenCaptureKitStream *stream = NULL; + bool mutexCreated = false; result = paNoError; __block NSArray *displays = nil; if ((stream = (PaScreenCaptureKitStream *)PaUtil_AllocateZeroInitializedMemory(sizeof(PaScreenCaptureKitStream))) == @@ -363,6 +373,14 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream stream->delegate = [[ScreenCaptureDelegate alloc] init]; // Create an instance of the delegate stream->delegate.stream = stream; + if (pthread_mutex_init(&stream->stopMutex, NULL) != 0) + { + PA_DEBUG(("Failed to initialize stop mutex\n")); + result = paInternalError; + goto error; + } + mutexCreated = true; + // Create an audio capture session stream->audioStream = [[SCStream alloc] initWithFilter:contentFilter configuration:streamConfig delegate:stream->delegate]; if (!stream->audioStream) @@ -437,6 +455,8 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream error: if (stream) { + if (mutexCreated) + pthread_mutex_destroy(&stream->stopMutex); if (stream->audioStream) [stream->audioStream release]; if (stream->streamOutput) @@ -521,6 +541,7 @@ static PaError CloseStream(PaStream *s) { StopStream(s); PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; + pthread_mutex_destroy(&stream->stopMutex); [stream->audioStream release]; [stream->streamOutput release]; [stream->delegate release]; From 75255e3b11cc12fd387191bee35e8c6f30aad17f Mon Sep 17 00:00:00 2001 From: Galkon Date: Wed, 11 Sep 2024 15:35:56 -0600 Subject: [PATCH 07/10] attempt to fix deadlock in stopping stream --- .gitignore | 5 ++- .../pa_mac_screencapturekit.m | 34 +++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 345708350..b33c654c2 100644 --- a/.gitignore +++ b/.gitignore @@ -45,7 +45,7 @@ autom4te.cache/* # Visual Studio 2017/2019 state folder .vs/ -# Visual Studio CMake build output folder +# Visual Studio CMake build output folder out/ # Visual Studio CMake settings file @@ -54,5 +54,8 @@ CMakeSettings.json # VSCode .vscode* +# JetBrains +.idea/ + # Common build directories of users and VSCode build* diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 08baa0611..07b75fcdf 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -89,9 +89,16 @@ - (void)outputVideoEffectDidStartForStream:(SCStream *)stream { } - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { - fprintf(stderr, "Stream encountered an error\n"); + fprintf(stderr, "Stream encountered an error: %s\n", [[error localizedDescription] UTF8String]); + if (self.stream) { - StopStreamInternal((PaStream *)self.stream); + // Mark the stream as having encountered an error to avoid re-entrance issues + self.stream->isStopped = TRUE; + + // Dispatch a block to perform the stop operation after returning from this delegate method + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + StopStreamInternal((PaStream *)self.stream); + }); } } @@ -184,21 +191,28 @@ static PaError StopStreamInternal(PaStream *s) __block PaError result = paNoError; dispatch_group_t handlerGroup = dispatch_group_create(); dispatch_group_enter(handlerGroup); + // Stop the audio capture session [stream->audioStream stopCaptureWithCompletionHandler:^(NSError *error) { - if (error) - { - PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); - result = paInternalError; - } - dispatch_group_leave(handlerGroup); + if (error) { + PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + } + dispatch_group_leave(handlerGroup); }]; + stream->isStopped = TRUE; - dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + + if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)) != 0) { + PA_DEBUG(("Timeout occurred while waiting for audio capture to stop.\n")); + result = paInternalError; + } + pthread_mutex_unlock(&stream->stopMutex); return result; } + // PortAudio host API stream read function static PaError ReadStream(PaStream *s, void *buffer, unsigned long frames) { @@ -632,4 +646,4 @@ PaError PaMacScreenCapture_Initialize(PaUtilHostApiRepresentation **hostApi, PaH Terminate((PaUtilHostApiRepresentation *)paSck); return result; -} \ No newline at end of file +} From ac9c78fbeef42c44348ea61355b80613c09db6a8 Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Thu, 12 Sep 2024 09:27:41 +0530 Subject: [PATCH 08/10] Continuation in our attempt to fix the StopStream hang issue --- src/hostapi/screencapturekit/pa_mac_screencapturekit.m | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 07b75fcdf..128c45e80 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -72,7 +72,7 @@ @interface ScreenCaptureKitStreamOutput : NSObject ScreenCaptureKitStreamOutput *streamOutput; ScreenCaptureDelegate *delegate; PaUtilRingBuffer ringBuffer; - BOOL isStopped; + BOOL volatile isStopped; int sampleRate; PaStreamCallback *streamCallback; void *userData; @@ -92,13 +92,10 @@ - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { fprintf(stderr, "Stream encountered an error: %s\n", [[error localizedDescription] UTF8String]); if (self.stream) { + pthread_mutex_lock(&self.stream->stopMutex); // Mark the stream as having encountered an error to avoid re-entrance issues self.stream->isStopped = TRUE; - - // Dispatch a block to perform the stop operation after returning from this delegate method - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - StopStreamInternal((PaStream *)self.stream); - }); + pthread_mutex_unlock(&self.stream->stopMutex); } } From 7349eb70737413ebe17d47b6668af6deb3fa23f3 Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Fri, 11 Oct 2024 09:13:34 +0530 Subject: [PATCH 09/10] Add timeouts to waits in OpenStream and StartStream --- .../screencapturekit/pa_mac_screencapturekit.m | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 128c45e80..5ad58d9bb 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -200,9 +200,9 @@ static PaError StopStreamInternal(PaStream *s) stream->isStopped = TRUE; - if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)) != 0) { + if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC)) != 0) { PA_DEBUG(("Timeout occurred while waiting for audio capture to stop.\n")); - result = paInternalError; + result = paTimedOut; } pthread_mutex_unlock(&stream->stopMutex); @@ -348,7 +348,11 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream dispatch_group_leave(handlerGroup); }]; - dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC)) != 0) { + PA_DEBUG(("Timeout occurred while waiting for async operation.\n")); + result = paTimedOut; + goto error; + } // Check the size of displays array if (!displays || displays.count < 1) @@ -512,7 +516,12 @@ static PaError StartStream(PaStream *s) } dispatch_group_leave(handlerGroup); }]; - dispatch_group_wait(handlerGroup, DISPATCH_TIME_FOREVER); + + if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC)) != 0) { + PA_DEBUG(("Timeout occurred while waiting for async operation.\n")); + return paTimedOut; + } + if (result != paNoError) return result; From 9596bd003568c5c8d53750240bc7023d7f2c8ade Mon Sep 17 00:00:00 2001 From: Karthick Jeyapal Date: Sat, 12 Oct 2024 15:08:22 +0530 Subject: [PATCH 10/10] Replace dispatch_group_wait with CFRunLoopRunInMode to avoid blocking waits in main thread --- CMakeLists.txt | 3 +- .../pa_mac_screencapturekit.m | 74 ++++++++++--------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7cee3f138..6e5c63b14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,10 +341,11 @@ elseif(UNIX) PRIVATE -Wl,-framework,ScreenCaptureKit -Wl,-framework,CoreMedia + -Wl,-framework,Foundation ) target_compile_definitions(PortAudio PUBLIC PA_USE_SCREENCAPTUREKIT=1) set(PKGCONFIG_CFLAGS "${PKGCONFIG_CFLAGS} -DPA_USE_SCREENCAPTUREKIT=1") - set(PKGCONFIG_LDFLAGS_PRIVATE "${PKGCONFIG_LDFLAGS_PRIVATE} -framework ScreenCaptureKit -framework CoreMedia") + set(PKGCONFIG_LDFLAGS_PRIVATE "${PKGCONFIG_LDFLAGS_PRIVATE} -framework ScreenCaptureKit -framework CoreMedia -framework Foundation") endif() endif() else() diff --git a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m index 5ad58d9bb..670cf29f3 100644 --- a/src/hostapi/screencapturekit/pa_mac_screencapturekit.m +++ b/src/hostapi/screencapturekit/pa_mac_screencapturekit.m @@ -185,28 +185,17 @@ static PaError StopStreamInternal(PaStream *s) return paNoError; } - __block PaError result = paNoError; - dispatch_group_t handlerGroup = dispatch_group_create(); - dispatch_group_enter(handlerGroup); - - // Stop the audio capture session [stream->audioStream stopCaptureWithCompletionHandler:^(NSError *error) { if (error) { + // This error can be ignored. It usually means that the stream has already been stopped. PA_DEBUG(("Failed to stop audio capture: %s\n", [[error localizedDescription] UTF8String])); - result = paInternalError; } - dispatch_group_leave(handlerGroup); }]; stream->isStopped = TRUE; - if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC)) != 0) { - PA_DEBUG(("Timeout occurred while waiting for audio capture to stop.\n")); - result = paTimedOut; - } - pthread_mutex_unlock(&stream->stopMutex); - return result; + return paNoError; } @@ -320,6 +309,7 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream bool mutexCreated = false; result = paNoError; __block NSArray *displays = nil; + __block bool gotShareableContent = false; if ((stream = (PaScreenCaptureKitStream *)PaUtil_AllocateZeroInitializedMemory(sizeof(PaScreenCaptureKitStream))) == NULL) { @@ -327,10 +317,6 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream goto error; } - dispatch_group_t handlerGroup = dispatch_group_create(); - - dispatch_group_enter(handlerGroup); - [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *shareableContent, NSError *error) { if (error) @@ -344,12 +330,23 @@ static PaError OpenStream(struct PaUtilHostApiRepresentation *hostApi, PaStream displays = [shareableContent.displays retain]; } - // Leave the dispatch group - dispatch_group_leave(handlerGroup); + gotShareableContent = true; }]; - if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC)) != 0) { - PA_DEBUG(("Timeout occurred while waiting for async operation.\n")); + // Yield to the run loop until the async operation completes or times out + CFTimeInterval timeout = 10.0; // Timeout in seconds + CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent(); + while (!gotShareableContent && (CFAbsoluteTimeGetCurrent() - startTime) < timeout) { + if ([NSThread isMainThread]) { + CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, true); + } else { + usleep(100 * 1000); + } + } + + if (!gotShareableContent) + { + PA_DEBUG(("Timeout occurred while waiting for displays\n")); result = paTimedOut; goto error; } @@ -502,29 +499,38 @@ static PaError StartStream(PaStream *s) if (IsStreamActive(s)) return paStreamIsNotStopped; __block PaError result = paNoError; + __block bool isCaptureStarted = false; PaScreenCaptureKitStream *stream = (PaScreenCaptureKitStream *)s; - dispatch_group_t handlerGroup = dispatch_group_create(); - dispatch_group_enter(handlerGroup); - - // Start the audio capture session [stream->audioStream startCaptureWithCompletionHandler:^(NSError *error) { - if (error) - { - PA_DEBUG(("Failed to start audio capture: %s\n", [[error localizedDescription] UTF8String])); - result = paInternalError; - } - dispatch_group_leave(handlerGroup); + if (error) + { + PA_DEBUG(("Failed to start audio capture: %s\n", [[error localizedDescription] UTF8String])); + result = paInternalError; + } + isCaptureStarted = true; }]; - if (dispatch_group_wait(handlerGroup, dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC)) != 0) { - PA_DEBUG(("Timeout occurred while waiting for async operation.\n")); - return paTimedOut; + // Yield to the run loop until the async operation completes or times out + CFTimeInterval timeout = 10.0; // Timeout in seconds + CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent(); + while (!isCaptureStarted && (CFAbsoluteTimeGetCurrent() - startTime) < timeout) { + if ([NSThread isMainThread]) { + CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, true); + } else { + usleep(100 * 1000); + } } if (result != paNoError) return result; + if (!isCaptureStarted) + { + PA_DEBUG(("Timeout occurred while waiting for capture to start\n")); + return paTimedOut; + } + stream->isStopped = FALSE; if (stream->streamCallback != NULL) {