From 918b0c5d752e06c9510f6e00066b5be479145e97 Mon Sep 17 00:00:00 2001 From: Matt Bargar Date: Fri, 6 Nov 2026 22:36:52 -0500 Subject: [PATCH] Add per-stream native microphone capture on iOS and tvOS Add explicit launch or resume with microphone, permission and route gates, capture status and teardown on inactivity, interruption or disconnect. Encode through bundled Opus and send using common-c native transport. --- Limelight/Limelight-Info.plist | 2 + Limelight/Stream/Connection.h | 1 + Limelight/Stream/Connection.m | 83 +++++++++- Limelight/Stream/ConnectionCallbacks.h | 7 + Limelight/Stream/NativeAppleMic.h | 9 ++ Limelight/Stream/NativeAppleMic.m | 143 ++++++++++++++++++ Limelight/Stream/NativeMicEncoder.c | 33 ++++ Limelight/Stream/NativeMicEncoder.h | 20 +++ Limelight/Stream/StreamConfiguration.h | 1 + Limelight/Stream/StreamManager.h | 1 + Limelight/Stream/StreamManager.m | 7 + .../ViewControllers/MainFrameViewController.m | 36 +++++ .../StreamFrameViewController.m | 41 ++++- Moonlight TV/Info.plist | 2 + Moonlight.xcodeproj/project.pbxproj | 26 ++++ README.md | 21 +++ moonlight-common/moonlight-common-c | 2 +- .../project.pbxproj | 6 + 18 files changed, 437 insertions(+), 4 deletions(-) create mode 100644 Limelight/Stream/NativeAppleMic.h create mode 100644 Limelight/Stream/NativeAppleMic.m create mode 100644 Limelight/Stream/NativeMicEncoder.c create mode 100644 Limelight/Stream/NativeMicEncoder.h diff --git a/Limelight/Limelight-Info.plist b/Limelight/Limelight-Info.plist index 963c433d5..8f01f1fac 100644 --- a/Limelight/Limelight-Info.plist +++ b/Limelight/Limelight-Info.plist @@ -2,6 +2,8 @@ + NSMicrophoneUsageDescription + Send your microphone to the NVIDIA GameStream host when you choose to stream with microphone forwarding. CADisableMinimumFrameDuration CADisableMinimumFrameDurationOnPhone diff --git a/Limelight/Stream/Connection.h b/Limelight/Stream/Connection.h index dc155419c..224d141c8 100644 --- a/Limelight/Stream/Connection.h +++ b/Limelight/Stream/Connection.h @@ -27,6 +27,7 @@ typedef struct { -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer*)myRenderer connectionCallbacks:(id)callbacks; -(void) terminate; +-(void) stopMicrophone; -(void) main; -(BOOL) getVideoStats:(video_stats_t*)stats; -(NSString*) getActiveCodecName; diff --git a/Limelight/Stream/Connection.m b/Limelight/Stream/Connection.m index 33361fc12..ab4ed296c 100644 --- a/Limelight/Stream/Connection.m +++ b/Limelight/Stream/Connection.m @@ -8,6 +8,8 @@ #import "Connection.h" #import "Utils.h" +#import "NativeAppleMic.h" +#import #import @@ -18,6 +20,9 @@ #include "opus_multistream.h" @implementation Connection { + StreamConfiguration* _micConfig; + __weak id _microphoneCallbacks; + NativeAppleMic* _nativeMicrophone; SERVER_INFORMATION _serverInfo; STREAM_CONFIGURATION _streamConfig; CONNECTION_LISTENER_CALLBACKS _clCallbacks; @@ -371,8 +376,54 @@ void ClSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g, uint8_t [_callbacks setControllerLed:controllerNumber r:r g:g b:b]; } +-(void) stopMicrophone +{ + [self stopMicrophoneWithStatus:NativeMicrophoneStatusOff]; +} + +-(void) microphoneUnavailable +{ + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_micConfig.requestNativeMicrophone) { + [self stopMicrophoneWithStatus:NativeMicrophoneStatusUnavailable]; + } + }); +} + +-(void) stopMicrophoneWithStatus:(NativeMicrophoneStatus)status +{ + _micConfig.requestNativeMicrophone = NO; + dispatch_async(dispatch_get_main_queue(), ^{ + [self->_microphoneCallbacks microphoneStatusChanged:status]; + }); + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ + [initLock lock]; + [self->_nativeMicrophone stop]; + self->_nativeMicrophone = nil; + [initLock unlock]; + }); +} + +-(void) microphoneInterrupted:(NSNotification*)notification +{ + if ([notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue] == + AVAudioSessionInterruptionTypeBegan) { + [self stopMicrophone]; + } +} + +-(void) microphoneRouteChanged:(NSNotification*)notification +{ + NSUInteger reason = [notification.userInfo[AVAudioSessionRouteChangeReasonKey] unsignedIntegerValue]; + if (reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable || + reason == AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory) { + [self microphoneUnavailable]; + } +} + -(void) terminate { + _micConfig.requestNativeMicrophone = NO; // Interrupt any action blocking LiStartConnection(). This is // thread-safe and done outside initLock on purpose, since we // won't be able to acquire it if LiStartConnection is in @@ -384,6 +435,8 @@ -(void) terminate // blocking on the caller's thread waiting to acquire initLock. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ [initLock lock]; + [self->_nativeMicrophone stop]; + self->_nativeMicrophone = nil; LiStopConnection(); [initLock unlock]; }); @@ -435,6 +488,14 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer renderer = myRenderer; _callbacks = callbacks; + _micConfig = config; + _microphoneCallbacks = callbacks; + if (config.requestNativeMicrophone) { + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(microphoneInterrupted:) + name:AVAudioSessionInterruptionNotification object:nil]; + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(microphoneRouteChanged:) + name:AVAudioSessionRouteChangeNotification object:nil]; + } LiInitializeStreamConfiguration(&_streamConfig); _streamConfig.width = config.width; _streamConfig.height = config.height; @@ -498,13 +559,33 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer -(void) main { [initLock lock]; - LiStartConnection(&_serverInfo, + _streamConfig.requestNativeMic = _micConfig.requestNativeMicrophone; + int result = LiStartConnection(&_serverInfo, &_streamConfig, &_clCallbacks, &_drCallbacks, &_arCallbacks, NULL, 0, NULL, 0); + if (result == 0 && _micConfig.requestNativeMicrophone) { + _nativeMicrophone = [[NativeAppleMic alloc] init]; + __weak Connection* weakSelf = self; + StreamConfiguration* config = _micConfig; + if (![_nativeMicrophone startWhileAllowed:^BOOL { + return config.requestNativeMicrophone; + } onFailure:^{ + [weakSelf microphoneUnavailable]; + }]) { + _nativeMicrophone = nil; + [self microphoneUnavailable]; + } else { + dispatch_async(dispatch_get_main_queue(), ^{ + if (config.requestNativeMicrophone) { + [self->_microphoneCallbacks microphoneStatusChanged:NativeMicrophoneStatusActive]; + } + }); + } + } [initLock unlock]; } diff --git a/Limelight/Stream/ConnectionCallbacks.h b/Limelight/Stream/ConnectionCallbacks.h index f9c13b4e5..718b92489 100644 --- a/Limelight/Stream/ConnectionCallbacks.h +++ b/Limelight/Stream/ConnectionCallbacks.h @@ -6,7 +6,14 @@ // Copyright © 2020 Moonlight Game Streaming Project. All rights reserved. // +typedef NS_ENUM(NSInteger, NativeMicrophoneStatus) { + NativeMicrophoneStatusActive, + NativeMicrophoneStatusUnavailable, + NativeMicrophoneStatusOff, +}; + @protocol ConnectionCallbacks +- (void)microphoneStatusChanged:(NativeMicrophoneStatus)status; - (void) connectionStarted; - (void) connectionTerminated:(int)errorCode; diff --git a/Limelight/Stream/NativeAppleMic.h b/Limelight/Stream/NativeAppleMic.h new file mode 100644 index 000000000..d398ceeee --- /dev/null +++ b/Limelight/Stream/NativeAppleMic.h @@ -0,0 +1,9 @@ +#import + +@interface NativeAppleMic : NSObject + +// Call start and stop from the connection's serialized lifecycle. +- (BOOL)startWhileAllowed:(BOOL (^)(void))allowed onFailure:(void (^)(void))failure; +- (void)stop; + +@end diff --git a/Limelight/Stream/NativeAppleMic.m b/Limelight/Stream/NativeAppleMic.m new file mode 100644 index 000000000..04831e268 --- /dev/null +++ b/Limelight/Stream/NativeAppleMic.m @@ -0,0 +1,143 @@ +#import "NativeAppleMic.h" +#import +#import +#include +#include "Limelight.h" +#include "NativeMicEncoder.h" + +@implementation NativeAppleMic { + AudioQueueRef _queue; + dispatch_queue_t _captureQueue; + NATIVE_MIC_SENDER* _sender; + OpusEncoder* _encoder; + int16_t _frame[NATIVE_MIC_FRAME_SAMPLES]; + size_t _filled; + atomic_bool _active; + BOOL _duplex; +} + +- (BOOL)startWhileAllowed:(BOOL (^)(void))allowed onFailure:(void (^)(void))failure +{ + atomic_init(&_active, false); + if (@available(iOS 17.0, tvOS 17.0, *)) { + if (AVAudioApplication.sharedInstance.recordPermission != AVAudioApplicationRecordPermissionGranted || + !allowed()) { + return NO; + } + } else { + return NO; + } + + _sender = LiCreateNativeMicSender(); + if (!_sender) { + return NO; + } + _encoder = NativeMicEncoderCreate(); + AVAudioSession* session = AVAudioSession.sharedInstance; + if (!_encoder || ![session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVoiceChat + options:AVAudioSessionCategoryOptionMixWithOthers error:NULL]) { + [self stop]; + return NO; + } + _duplex = YES; + if (!session.isInputAvailable || !allowed()) { + [self stop]; + return NO; + } + + AudioStreamBasicDescription format = { + .mSampleRate = NATIVE_MIC_SAMPLE_RATE, + .mFormatID = kAudioFormatLinearPCM, + .mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, + .mBytesPerPacket = sizeof(int16_t), + .mFramesPerPacket = 1, + .mBytesPerFrame = sizeof(int16_t), + .mChannelsPerFrame = 1, + .mBitsPerChannel = 16, + }; + _captureQueue = dispatch_queue_create("org.moonlight.native-mic", DISPATCH_QUEUE_SERIAL); + __weak NativeAppleMic* weakSelf = self; + OSStatus error = AudioQueueNewInputWithDispatchQueue(&_queue, &format, 0, _captureQueue, + ^(AudioQueueRef queue, AudioQueueBufferRef buffer, const AudioTimeStamp* time, + UInt32 packetCount, const AudioStreamPacketDescription* descriptions) { + (void)time; + (void)packetCount; + (void)descriptions; + NativeAppleMic* mic = weakSelf; + if (!mic || !atomic_load(&mic->_active) || !allowed()) { + return; + } + if (![mic sendBuffer:buffer whileAllowed:allowed] || + AudioQueueEnqueueBuffer(queue, buffer, 0, NULL) != noErr) { + atomic_store(&mic->_active, false); + failure(); + } + }); + for (int i = 0; error == noErr && i < 4; i++) { + AudioQueueBufferRef buffer; + error = AudioQueueAllocateBuffer(_queue, sizeof(_frame), &buffer); + if (error == noErr) { + error = AudioQueueEnqueueBuffer(_queue, buffer, 0, NULL); + } + } + if (error == noErr && allowed()) { + atomic_store(&_active, true); + if (AudioQueueStart(_queue, NULL) == noErr) { + return YES; + } + } + [self stop]; + return NO; +} + +- (BOOL)sendBuffer:(AudioQueueBufferRef)buffer whileAllowed:(BOOL (^)(void))allowed +{ + if (buffer->mAudioDataByteSize > buffer->mAudioDataBytesCapacity || + buffer->mAudioDataByteSize % sizeof(int16_t) != 0) { + return NO; + } + const int16_t* pcm = buffer->mAudioData; + for (size_t i = 0; i < buffer->mAudioDataByteSize / sizeof(*pcm); i++) { + _frame[_filled++] = pcm[i]; + if (_filled == NATIVE_MIC_FRAME_SAMPLES) { + unsigned char packet[NATIVE_MIC_FRAME_BYTES]; + if (!atomic_load(&_active) || !allowed() || + NativeMicEncode(_encoder, _frame, _filled, packet, sizeof(packet)) != OPUS_OK || + LiSendNativeMicFrame(_sender, packet, sizeof(packet)) != 0) { + return NO; + } + _filled = 0; + } + } + return YES; +} + +- (void)stop +{ + atomic_store(&_active, false); + if (_queue) { + AudioQueueStop(_queue, true); + AudioQueueDispose(_queue, true); + _queue = NULL; + } + if (_captureQueue) { + dispatch_sync(_captureQueue, ^{}); + _captureQueue = nil; + } + if (_sender) { + LiDestroyNativeMicSender(_sender); + _sender = NULL; + } + if (_encoder) { + opus_encoder_destroy(_encoder); + _encoder = NULL; + } + if (_duplex) { + [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback + withOptions:AVAudioSessionCategoryOptionMixWithOthers error:NULL]; + _duplex = NO; + } +} + +@end diff --git a/Limelight/Stream/NativeMicEncoder.c b/Limelight/Stream/NativeMicEncoder.c new file mode 100644 index 000000000..f9257a48f --- /dev/null +++ b/Limelight/Stream/NativeMicEncoder.c @@ -0,0 +1,33 @@ +#include "NativeMicEncoder.h" + +OpusEncoder* NativeMicEncoderCreate(void) +{ + int error; + OpusEncoder* encoder = opus_encoder_create(NATIVE_MIC_SAMPLE_RATE, 1, + OPUS_APPLICATION_RESTRICTED_LOWDELAY, &error); + if (encoder == NULL) { + return NULL; + } + if (opus_encoder_ctl(encoder, OPUS_SET_BITRATE(32000)) != OPUS_OK || + opus_encoder_ctl(encoder, OPUS_SET_VBR(0)) != OPUS_OK || + opus_encoder_ctl(encoder, OPUS_SET_DTX(0)) != OPUS_OK) { + opus_encoder_destroy(encoder); + return NULL; + } + return encoder; +} + +int NativeMicEncode(OpusEncoder* encoder, const int16_t* pcm, size_t samples, + unsigned char* packet, size_t capacity) +{ + if (encoder == NULL || pcm == NULL || packet == NULL || + samples != NATIVE_MIC_FRAME_SAMPLES || capacity < NATIVE_MIC_FRAME_BYTES) { + return OPUS_BAD_ARG; + } + int bytes = opus_encode(encoder, pcm, NATIVE_MIC_FRAME_SAMPLES, packet, + NATIVE_MIC_FRAME_BYTES); + if (bytes < 0) { + return bytes; + } + return bytes == NATIVE_MIC_FRAME_BYTES ? OPUS_OK : OPUS_INTERNAL_ERROR; +} diff --git a/Limelight/Stream/NativeMicEncoder.h b/Limelight/Stream/NativeMicEncoder.h new file mode 100644 index 000000000..99992b364 --- /dev/null +++ b/Limelight/Stream/NativeMicEncoder.h @@ -0,0 +1,20 @@ +#ifndef NATIVE_MIC_ENCODER_H +#define NATIVE_MIC_ENCODER_H + +#include +#include +#include + +enum { + NATIVE_MIC_SAMPLE_RATE = 48000, + NATIVE_MIC_FRAME_SAMPLES = 240, + NATIVE_MIC_FRAME_BYTES = 20 +}; + +// Serialize encoder use and destruction. Ownership may transfer before encoding. +OpusEncoder* NativeMicEncoderCreate(void); +// Returns zero for a complete packet and a negative Opus error otherwise. +int NativeMicEncode(OpusEncoder* encoder, const int16_t* pcm, size_t samples, + unsigned char* packet, size_t capacity); + +#endif diff --git a/Limelight/Stream/StreamConfiguration.h b/Limelight/Stream/StreamConfiguration.h index e912b5788..99ef3f879 100644 --- a/Limelight/Stream/StreamConfiguration.h +++ b/Limelight/Stream/StreamConfiguration.h @@ -25,6 +25,7 @@ @property int gamepadMask; @property BOOL optimizeGameSettings; @property BOOL playAudioOnPC; +@property BOOL requestNativeMicrophone; @property BOOL swapABXYButtons; @property int audioConfiguration; @property int supportedVideoFormats; diff --git a/Limelight/Stream/StreamManager.h b/Limelight/Stream/StreamManager.h index 277d6fe43..597dee903 100644 --- a/Limelight/Stream/StreamManager.h +++ b/Limelight/Stream/StreamManager.h @@ -14,6 +14,7 @@ - (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id)callback; - (void) stopStream; +- (void) stopMicrophone; - (NSString*) getStatsOverlayText; diff --git a/Limelight/Stream/StreamManager.m b/Limelight/Stream/StreamManager.m index e776feef0..d1db869e9 100644 --- a/Limelight/Stream/StreamManager.m +++ b/Limelight/Stream/StreamManager.m @@ -106,8 +106,15 @@ - (void)main { }); } +- (void) stopMicrophone +{ + _config.requestNativeMicrophone = NO; + [_connection stopMicrophone]; +} + - (void) stopStream { + _config.requestNativeMicrophone = NO; [_connection terminate]; } diff --git a/Limelight/ViewControllers/MainFrameViewController.m b/Limelight/ViewControllers/MainFrameViewController.m index 169bb13ba..b65fa1647 100644 --- a/Limelight/ViewControllers/MainFrameViewController.m +++ b/Limelight/ViewControllers/MainFrameViewController.m @@ -21,6 +21,7 @@ #import "AppListResponse.h" #import "ServerInfoResponse.h" #import "StreamFrameViewController.h" +#import #import "LoadingFrameViewController.h" #import "ComputerScrollView.h" #import "TemporaryApp.h" @@ -46,6 +47,7 @@ @implementation MainFrameViewController { DiscoveryManager* _discMan; AppAssetManager* _appManager; StreamConfiguration* _streamConfig; + NSUInteger _microphoneLaunchGeneration; UIAlertController* _pairAlert; LoadingFrameViewController* _loadingFrame; UIScrollView* hostScrollView; @@ -602,6 +604,7 @@ - (void) addHostClicked { } - (void) prepareToStreamApp:(TemporaryApp *)app { + ++_microphoneLaunchGeneration; _streamConfig = [[StreamConfiguration alloc] init]; _streamConfig.host = app.host.activeAddress; _streamConfig.httpsPort = app.host.httpsPort; @@ -751,6 +754,39 @@ - (void)appLongClicked:(TemporaryApp *)app view:(UIView *)view { [self performSegueWithIdentifier:@"createStreamFrame" sender:nil]; }]]; + if (@available(iOS 17.0, tvOS 17.0, *)) { + TemporaryApp* microphoneApp = currentApp ?: app; + NSString* microphoneAction = [NSString stringWithFormat:currentApp ? @"Resume %@ with Microphone" : + @"Launch %@ with Microphone", + microphoneApp.name]; + [alertController addAction:[UIAlertAction actionWithTitle:microphoneAction + style:UIAlertActionStyleDefault handler:^(UIAlertAction* action) { + NSUInteger generation = ++self->_microphoneLaunchGeneration; + [AVAudioApplication requestRecordPermissionWithCompletionHandler:^(BOOL granted) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (!self.view.window || self->_selectedHost != app.host || + UIApplication.sharedApplication.applicationState != UIApplicationStateActive || + self->_microphoneLaunchGeneration != generation) { + return; + } + if (!granted) { + UIAlertController* permissionAlert = [UIAlertController + alertControllerWithTitle:@"Microphone unavailable" + message:@"Allow microphone access in Settings to stream with your microphone." + preferredStyle:UIAlertControllerStyleAlert]; + [permissionAlert addAction:[UIAlertAction actionWithTitle:@"OK" + style:UIAlertActionStyleCancel handler:nil]]; + [[self activeViewController] presentViewController:permissionAlert animated:YES completion:nil]; + return; + } + [self prepareToStreamApp:microphoneApp]; + self->_streamConfig.requestNativeMicrophone = YES; + [self performSegueWithIdentifier:@"createStreamFrame" sender:nil]; + }); + }]; + }]]; + } + if (currentApp != nil) { [alertController addAction:[UIAlertAction actionWithTitle: [app.id isEqualToString:currentApp.id] ? @"Quit App" : @"Quit Running App and Start" style:UIAlertActionStyleDestructive handler:^(UIAlertAction* action){ diff --git a/Limelight/ViewControllers/StreamFrameViewController.m b/Limelight/ViewControllers/StreamFrameViewController.m index a4c8f3448..68884d60c 100644 --- a/Limelight/ViewControllers/StreamFrameViewController.m +++ b/Limelight/ViewControllers/StreamFrameViewController.m @@ -42,6 +42,7 @@ @implementation StreamFrameViewController { UITextView *_overlayView; UILabel *_stageLabel; UILabel *_tipLabel; + UILabel *_microphoneLabel; UIActivityIndicatorView *_spinner; StreamView *_streamView; UIScrollView *_scrollView; @@ -154,8 +155,6 @@ - (void)viewDidLoad _streamMan = [[StreamManager alloc] initWithConfig:self.streamConfig renderView:_streamView connectionCallbacks:self]; - NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; - [opQueue addOperation:_streamMan]; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(applicationWillResignActive:) @@ -209,6 +208,27 @@ - (void)viewDidLoad [self.view addSubview:_stageLabel]; [self.view addSubview:_spinner]; [self.view addSubview:_tipLabel]; + if (self.streamConfig.requestNativeMicrophone) { + _microphoneLabel = [[UILabel alloc] init]; + _microphoneLabel.translatesAutoresizingMaskIntoConstraints = NO; + _microphoneLabel.userInteractionEnabled = NO; + _microphoneLabel.textAlignment = NSTextAlignmentCenter; + _microphoneLabel.textColor = UIColor.whiteColor; + _microphoneLabel.backgroundColor = [UIColor colorWithWhite:0 alpha:0.8]; + _microphoneLabel.font = [UIFont preferredFontForTextStyle:UIFontTextStyleFootnote]; + _microphoneLabel.numberOfLines = 0; + _microphoneLabel.text = @"Starting microphone…"; + [self.view addSubview:_microphoneLabel]; + UILayoutGuide* safeArea = self.view.safeAreaLayoutGuide; + [NSLayoutConstraint activateConstraints:@[ + [_microphoneLabel.centerXAnchor constraintEqualToAnchor:safeArea.centerXAnchor], + [_microphoneLabel.topAnchor constraintEqualToAnchor:safeArea.topAnchor constant:8], + [_microphoneLabel.leadingAnchor constraintGreaterThanOrEqualToAnchor:safeArea.leadingAnchor constant:8], + [_microphoneLabel.trailingAnchor constraintLessThanOrEqualToAnchor:safeArea.trailingAnchor constant:-8], + ]]; + } + NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; + [opQueue addOperation:_streamMan]; } - (UIView *)viewForZoomingInScrollView:(UIScrollView *)scrollView { @@ -316,7 +336,23 @@ - (void) returnToMainFrame { } // This will fire if the user opens control center or gets a low battery message +- (void)microphoneStatusChanged:(NativeMicrophoneStatus)status { + switch (status) { + case NativeMicrophoneStatusActive: + _microphoneLabel.text = @" Microphone on · Leave stream to stop "; + break; + case NativeMicrophoneStatusUnavailable: + _microphoneLabel.text = @" Microphone unavailable · Stream continues "; + break; + case NativeMicrophoneStatusOff: + _microphoneLabel.text = @" Microphone off "; + break; + } +} + - (void)applicationWillResignActive:(NSNotification *)notification { + [_streamMan stopMicrophone]; + [self microphoneStatusChanged:NativeMicrophoneStatusOff]; if (_inactivityTimer != nil) { [_inactivityTimer invalidate]; } @@ -390,6 +426,7 @@ - (void) connectionStarted { } - (void)connectionTerminated:(int)errorCode { + [_streamMan stopMicrophone]; Log(LOG_I, @"Connection terminated: %d", errorCode); unsigned int portFlags = LiGetPortFlagsFromTerminationErrorCode(errorCode); diff --git a/Moonlight TV/Info.plist b/Moonlight TV/Info.plist index c1f0beebf..e9be5387d 100644 --- a/Moonlight TV/Info.plist +++ b/Moonlight TV/Info.plist @@ -2,6 +2,8 @@ + NSMicrophoneUsageDescription + Send your microphone to the NVIDIA GameStream host when you choose to stream with microphone forwarding. CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleDisplayName diff --git a/Moonlight.xcodeproj/project.pbxproj b/Moonlight.xcodeproj/project.pbxproj index e686ec129..c4d4ddb9a 100644 --- a/Moonlight.xcodeproj/project.pbxproj +++ b/Moonlight.xcodeproj/project.pbxproj @@ -49,6 +49,8 @@ FB1A67622132419A00507771 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FB1A67612132419A00507771 /* Assets.xcassets */; }; FB1A67A12132458C00507771 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = FB1A67642132419A00507771 /* main.m */; }; FB1A67A3213245BD00507771 /* Connection.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461719F646E200339C8A /* Connection.m */; }; + C00536ED94C34138BBC376FB /* NativeMicEncoder.c in Sources */ = {isa = PBXBuildFile; fileRef = FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */; }; + 504AB7156B7643B292C10299 /* NativeAppleMic.m in Sources */ = {isa = PBXBuildFile; fileRef = AE40E7E0868344F395471576 /* NativeAppleMic.m */; }; FB1A67A5213245BD00507771 /* StreamConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461919F646E200339C8A /* StreamConfiguration.m */; }; FB1A67A7213245BD00507771 /* StreamManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461B19F646E200339C8A /* StreamManager.m */; }; FB1A67A9213245BD00507771 /* VideoDecoderRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */; }; @@ -81,6 +83,8 @@ FB1A67E32132498A00507771 /* Limelight.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = FB290D0519B2C406004C83CF /* Limelight.xcdatamodeld */; }; FB1A67E521324A1F00507771 /* CoreData.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FB1A67E421324A1F00507771 /* CoreData.framework */; }; FB1A67E821324DE300507771 /* libopus.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FB8946EA19F6AFB800339C8A /* libopus.a */; }; + 82C1A86EEA314CA9B2848884 /* AudioToolbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 638BD6868E254A4980037CEB /* AudioToolbox.framework */; }; + B462FCC5A0214D1ABC3FF981 /* AVFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 9A9B4337320D4B9493FB09EF /* AVFoundation.framework */; }; FB1A67EA21324DF300507771 /* libxml2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = FB1A67E921324DF300507771 /* libxml2.tbd */; }; FB1A6819213284FB00507771 /* UIComputerView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86DF19F7A837001C18A8 /* UIComputerView.m */; }; FB1A681A213284FB00507771 /* UIAppView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86E519F82297001C18A8 /* UIAppView.m */; }; @@ -115,6 +119,8 @@ FB89462D19F646E200339C8A /* MDNSManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461219F646E200339C8A /* MDNSManager.m */; }; FB89462E19F646E200339C8A /* PairManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461419F646E200339C8A /* PairManager.m */; }; FB89462F19F646E200339C8A /* Connection.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461719F646E200339C8A /* Connection.m */; }; + 6F2A12808AF84F8B8F011376 /* NativeMicEncoder.c in Sources */ = {isa = PBXBuildFile; fileRef = FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */; }; + 63B66D53AA0647A39CBEF921 /* NativeAppleMic.m in Sources */ = {isa = PBXBuildFile; fileRef = AE40E7E0868344F395471576 /* NativeAppleMic.m */; }; FB89463019F646E200339C8A /* StreamConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461919F646E200339C8A /* StreamConfiguration.m */; }; FB89463119F646E200339C8A /* StreamManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461B19F646E200339C8A /* StreamManager.m */; }; FB89463219F646E200339C8A /* VideoDecoderRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */; }; @@ -123,6 +129,8 @@ FB89463619F646E200339C8A /* StreamFrameViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462719F646E200339C8A /* StreamFrameViewController.m */; }; FB89463819F6473800339C8A /* Launch Screen.xib in Resources */ = {isa = PBXBuildFile; fileRef = FB89463719F6473800339C8A /* Launch Screen.xib */; }; FB8946ED19F6AFE800339C8A /* libopus.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FB8946EA19F6AFB800339C8A /* libopus.a */; }; + 7F2311C24E6042CF9D57BCF7 /* AudioToolbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 638BD6868E254A4980037CEB /* AudioToolbox.framework */; }; + C25BDE0324F746169F7C629D /* AVFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 9A9B4337320D4B9493FB09EF /* AVFoundation.framework */; }; FB9AFD281A7C84ED00872C98 /* HttpResponse.m in Sources */ = {isa = PBXBuildFile; fileRef = FB9AFD271A7C84ED00872C98 /* HttpResponse.m */; }; FB9AFD321A7D867C00872C98 /* AppAssetRetriever.m in Sources */ = {isa = PBXBuildFile; fileRef = FB9AFD311A7D867C00872C98 /* AppAssetRetriever.m */; }; FB9AFD371A7E02DB00872C98 /* HttpRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = FB9AFD361A7E02DB00872C98 /* HttpRequest.m */; }; @@ -275,6 +283,10 @@ FB89461419F646E200339C8A /* PairManager.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PairManager.m; sourceTree = ""; }; FB89461619F646E200339C8A /* Connection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Connection.h; sourceTree = ""; }; FB89461719F646E200339C8A /* Connection.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Connection.m; sourceTree = ""; }; + 2D0E9120706F440FB232FE0C /* NativeMicEncoder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NativeMicEncoder.h; sourceTree = ""; }; + 0366C723CB7A4512AB8DD129 /* NativeAppleMic.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NativeAppleMic.h; sourceTree = ""; }; + FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = NativeMicEncoder.c; sourceTree = ""; }; + AE40E7E0868344F395471576 /* NativeAppleMic.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NativeAppleMic.m; sourceTree = ""; }; FB89461819F646E200339C8A /* StreamConfiguration.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamConfiguration.h; sourceTree = ""; }; FB89461919F646E200339C8A /* StreamConfiguration.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = StreamConfiguration.m; sourceTree = ""; }; FB89461A19F646E200339C8A /* StreamManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamManager.h; sourceTree = ""; }; @@ -293,6 +305,8 @@ FB8946E719F6AFB800339C8A /* opus_multistream.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = opus_multistream.h; sourceTree = ""; }; FB8946E819F6AFB800339C8A /* opus_types.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = opus_types.h; sourceTree = ""; }; FB8946EA19F6AFB800339C8A /* libopus.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libopus.a; sourceTree = ""; }; + 638BD6868E254A4980037CEB /* AudioToolbox.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AudioToolbox.framework; path = System/Library/Frameworks/AudioToolbox.framework; sourceTree = SDKROOT; }; + 9A9B4337320D4B9493FB09EF /* AVFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AVFoundation.framework; path = System/Library/Frameworks/AVFoundation.framework; sourceTree = SDKROOT; }; FB9AFD261A7C84ED00872C98 /* HttpResponse.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HttpResponse.h; sourceTree = ""; }; FB9AFD271A7C84ED00872C98 /* HttpResponse.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HttpResponse.m; sourceTree = ""; }; FB9AFD301A7D867C00872C98 /* AppAssetRetriever.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AppAssetRetriever.h; sourceTree = ""; }; @@ -334,6 +348,8 @@ F74BEFA42C1A80E400224667 /* OpenSSL in Frameworks */, FB1A67EA21324DF300507771 /* libxml2.tbd in Frameworks */, FB1A67E821324DE300507771 /* libopus.a in Frameworks */, + 82C1A86EEA314CA9B2848884 /* AudioToolbox.framework in Frameworks */, + B462FCC5A0214D1ABC3FF981 /* AVFoundation.framework in Frameworks */, 98181BEB2791278F00E43572 /* libSDL2.a in Frameworks */, FB1A67E521324A1F00507771 /* CoreData.framework in Frameworks */, 98B9CE6D27B2144B00B473C4 /* AVKit.framework in Frameworks */, @@ -348,6 +364,8 @@ 9890CF6B203B7EE1006C4B06 /* libxml2.tbd in Frameworks */, 98CFB82F1CAD481B0048EF74 /* libmoonlight-common.a in Frameworks */, FB8946ED19F6AFE800339C8A /* libopus.a in Frameworks */, + 7F2311C24E6042CF9D57BCF7 /* AudioToolbox.framework in Frameworks */, + C25BDE0324F746169F7C629D /* AVFoundation.framework in Frameworks */, 98882A082AF60F7400C5A11C /* libavutil.a in Frameworks */, FB290CF419B2C406004C83CF /* CoreGraphics.framework in Frameworks */, FB290CF819B2C406004C83CF /* CoreData.framework in Frameworks */, @@ -544,6 +562,10 @@ children = ( FB89461619F646E200339C8A /* Connection.h */, FB89461719F646E200339C8A /* Connection.m */, + 2D0E9120706F440FB232FE0C /* NativeMicEncoder.h */, + 0366C723CB7A4512AB8DD129 /* NativeAppleMic.h */, + FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */, + AE40E7E0868344F395471576 /* NativeAppleMic.m */, FB89461819F646E200339C8A /* StreamConfiguration.h */, FB89461919F646E200339C8A /* StreamConfiguration.m */, FB89461A19F646E200339C8A /* StreamManager.h */, @@ -874,6 +896,8 @@ FB1A67AF213245C500507771 /* IdManager.m in Sources */, 988FCD42293B091B003050E2 /* KeyboardInputField.m in Sources */, FB1A67A3213245BD00507771 /* Connection.m in Sources */, + C00536ED94C34138BBC376FB /* NativeMicEncoder.c in Sources */, + 504AB7156B7643B292C10299 /* NativeAppleMic.m in Sources */, FB1A67A5213245BD00507771 /* StreamConfiguration.m in Sources */, FB1A67A7213245BD00507771 /* StreamManager.m in Sources */, FB1A67A9213245BD00507771 /* VideoDecoderRenderer.m in Sources */, @@ -895,6 +919,8 @@ FB89463419F646E200339C8A /* Utils.m in Sources */, FBDE86E619F82297001C18A8 /* UIAppView.m in Sources */, FB89462F19F646E200339C8A /* Connection.m in Sources */, + 6F2A12808AF84F8B8F011376 /* NativeMicEncoder.c in Sources */, + 63B66D53AA0647A39CBEF921 /* NativeAppleMic.m in Sources */, FB9AFD321A7D867C00872C98 /* AppAssetRetriever.m in Sources */, FB4678FF1A565DAC00377732 /* WakeOnLanManager.m in Sources */, FB53E1431BE5DC4400CD6ECE /* IdManager.m in Sources */, diff --git a/README.md b/README.md index 4b2cf9987..885268bda 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,24 @@ Check out [the Moonlight wiki](https://github.com/moonlight-stream/moonlight-doc * In the "Team" dropdown, select your name. If your name doesn't appear, you may need to sign into Xcode with your Apple account. * Change the "Bundle Identifier" to something different. You can add your name or some random letters to make it unique. * Now you can select your Apple device in the top bar as a target and click the Play button to run. + +## Native GameStream microphone + +On iOS/iPadOS 17 or later and tvOS 17 or later, open an app tile's launch +options and choose **Launch [app] with Microphone** or **Resume [app] with +Microphone**. Grant recording permission when prompted. Apple TV requires +a supported Bluetooth or Continuity input that the OS makes available. + +The choice applies to one stream. An on-screen label reports capture +status. Leaving the stream, interruption, inactivity or loss of the input +route stops forwarding. Start a new explicitly opted-in stream to resume +capture. + +Use the [compatible Windows NVIDIA GameStream host setup](https://github.com/Bargs/native-gamestream-microphone-host) +and a trusted private IPv4 network or secured tunnel. The native legacy +CBC mode does not authenticate packets. The shared transport is reviewed +in [common-c PR #173](https://github.com/moonlight-stream/moonlight-common-c/pull/173). + +Both arm64 device targets passed source and link checks with resource +phases omitted. Standard app packaging and on-device permission, route +and live host delivery need verification. diff --git a/moonlight-common/moonlight-common-c b/moonlight-common/moonlight-common-c index f900dd476..a9107cf1f 160000 --- a/moonlight-common/moonlight-common-c +++ b/moonlight-common/moonlight-common-c @@ -1 +1 @@ -Subproject commit f900dd4767759c7b9d0e93bcea666b55c69ea62f +Subproject commit a9107cf1f54e50be328573f9a79de31d2f3fdcad diff --git a/moonlight-common/moonlight-common.xcodeproj/project.pbxproj b/moonlight-common/moonlight-common.xcodeproj/project.pbxproj index 4c5699382..67c5fee8f 100644 --- a/moonlight-common/moonlight-common.xcodeproj/project.pbxproj +++ b/moonlight-common/moonlight-common.xcodeproj/project.pbxproj @@ -30,6 +30,7 @@ 98AB2E4D1CAD425A0089BB98 /* SdpGenerator.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3B1CAD423B0089BB98 /* SdpGenerator.c */; }; 98AB2E4E1CAD425A0089BB98 /* VideoDepacketizer.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */; }; 98AB2E4F1CAD425A0089BB98 /* VideoStream.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */; }; + D836F126BEB34CD7B04E3E9B /* NativeMicStream.c in Sources */ = {isa = PBXBuildFile; fileRef = F11F0AD711634D1B80B488FC /* NativeMicStream.c */; }; 98AB2E501CAD427A0089BB98 /* callbacks.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF01CAD422B0089BB98 /* callbacks.c */; }; 98AB2E511CAD427A0089BB98 /* compress.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF31CAD422B0089BB98 /* compress.c */; }; 98AB2E521CAD427A0089BB98 /* host.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E001CAD422B0089BB98 /* host.c */; }; @@ -64,6 +65,7 @@ FB1A680721324F4900507771 /* SdpGenerator.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3B1CAD423B0089BB98 /* SdpGenerator.c */; }; FB1A680821324F4900507771 /* VideoDepacketizer.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */; }; FB1A680921324F4900507771 /* VideoStream.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */; }; + 250B5B5384F14AD28BEFD7D2 /* NativeMicStream.c in Sources */ = {isa = PBXBuildFile; fileRef = F11F0AD711634D1B80B488FC /* NativeMicStream.c */; }; FB1A680A21324F4900507771 /* callbacks.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF01CAD422B0089BB98 /* callbacks.c */; }; FB1A680B21324F4900507771 /* compress.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF31CAD422B0089BB98 /* compress.c */; }; FB1A680C21324F4900507771 /* host.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E001CAD422B0089BB98 /* host.c */; }; @@ -155,6 +157,7 @@ 98AB2E3C1CAD423B0089BB98 /* Video.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Video.h; sourceTree = ""; }; 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = VideoDepacketizer.c; sourceTree = ""; }; 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = VideoStream.c; sourceTree = ""; }; + F11F0AD711634D1B80B488FC /* NativeMicStream.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = NativeMicStream.c; sourceTree = ""; }; 98B8C2B5262BC1220027F659 /* PlatformCrypto.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = PlatformCrypto.c; sourceTree = ""; }; 98B8C2BA262BC12F0027F659 /* PlatformCrypto.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PlatformCrypto.h; sourceTree = ""; }; 98C1FC11218404030023FDDF /* SimpleStun.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = SimpleStun.c; sourceTree = ""; }; @@ -310,6 +313,7 @@ 98AB2E3C1CAD423B0089BB98 /* Video.h */, 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */, 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */, + F11F0AD711634D1B80B488FC /* NativeMicStream.c */, ); name = src; path = "moonlight-common-c/src"; @@ -440,6 +444,7 @@ 98F145382667D847004018E3 /* RtpVideoQueue.c in Sources */, FB1A680821324F4900507771 /* VideoDepacketizer.c in Sources */, FB1A680921324F4900507771 /* VideoStream.c in Sources */, + 250B5B5384F14AD28BEFD7D2 /* NativeMicStream.c in Sources */, 98B8C2B7262BC1220027F659 /* PlatformCrypto.c in Sources */, FB1A680A21324F4900507771 /* callbacks.c in Sources */, FB1A680B21324F4900507771 /* compress.c in Sources */, @@ -480,6 +485,7 @@ 98F145372667D847004018E3 /* RtpVideoQueue.c in Sources */, 98AB2E4E1CAD425A0089BB98 /* VideoDepacketizer.c in Sources */, 98AB2E4F1CAD425A0089BB98 /* VideoStream.c in Sources */, + D836F126BEB34CD7B04E3E9B /* NativeMicStream.c in Sources */, 98B8C2B6262BC1220027F659 /* PlatformCrypto.c in Sources */, 98AB2E501CAD427A0089BB98 /* callbacks.c in Sources */, 98AB2E511CAD427A0089BB98 /* compress.c in Sources */,