diff --git a/Limelight/Limelight-Info.plist b/Limelight/Limelight-Info.plist index 963c433d5..8f01f1fac 100644 --- a/Limelight/Limelight-Info.plist +++ b/Limelight/Limelight-Info.plist @@ -2,6 +2,8 @@ + NSMicrophoneUsageDescription + Send your microphone to the NVIDIA GameStream host when you choose to stream with microphone forwarding. CADisableMinimumFrameDuration CADisableMinimumFrameDurationOnPhone diff --git a/Limelight/Stream/Connection.h b/Limelight/Stream/Connection.h index dc155419c..224d141c8 100644 --- a/Limelight/Stream/Connection.h +++ b/Limelight/Stream/Connection.h @@ -27,6 +27,7 @@ typedef struct { -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer*)myRenderer connectionCallbacks:(id)callbacks; -(void) terminate; +-(void) stopMicrophone; -(void) main; -(BOOL) getVideoStats:(video_stats_t*)stats; -(NSString*) getActiveCodecName; diff --git a/Limelight/Stream/Connection.m b/Limelight/Stream/Connection.m index 33361fc12..ab4ed296c 100644 --- a/Limelight/Stream/Connection.m +++ b/Limelight/Stream/Connection.m @@ -8,6 +8,8 @@ #import "Connection.h" #import "Utils.h" +#import "NativeAppleMic.h" +#import #import @@ -18,6 +20,9 @@ #include "opus_multistream.h" @implementation Connection { + StreamConfiguration* _micConfig; + __weak id _microphoneCallbacks; + NativeAppleMic* _nativeMicrophone; SERVER_INFORMATION _serverInfo; STREAM_CONFIGURATION _streamConfig; CONNECTION_LISTENER_CALLBACKS _clCallbacks; @@ -371,8 +376,54 @@ void ClSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g, uint8_t [_callbacks setControllerLed:controllerNumber r:r g:g b:b]; } +-(void) stopMicrophone +{ + [self stopMicrophoneWithStatus:NativeMicrophoneStatusOff]; +} + +-(void) microphoneUnavailable +{ + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_micConfig.requestNativeMicrophone) { + [self stopMicrophoneWithStatus:NativeMicrophoneStatusUnavailable]; + } + }); +} + +-(void) stopMicrophoneWithStatus:(NativeMicrophoneStatus)status +{ + _micConfig.requestNativeMicrophone = NO; + dispatch_async(dispatch_get_main_queue(), ^{ + [self->_microphoneCallbacks microphoneStatusChanged:status]; + }); + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ + [initLock lock]; + [self->_nativeMicrophone stop]; + self->_nativeMicrophone = nil; + [initLock unlock]; + }); +} + +-(void) microphoneInterrupted:(NSNotification*)notification +{ + if ([notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue] == + AVAudioSessionInterruptionTypeBegan) { + [self stopMicrophone]; + } +} + +-(void) microphoneRouteChanged:(NSNotification*)notification +{ + NSUInteger reason = [notification.userInfo[AVAudioSessionRouteChangeReasonKey] unsignedIntegerValue]; + if (reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable || + reason == AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory) { + [self microphoneUnavailable]; + } +} + -(void) terminate { + _micConfig.requestNativeMicrophone = NO; // Interrupt any action blocking LiStartConnection(). This is // thread-safe and done outside initLock on purpose, since we // won't be able to acquire it if LiStartConnection is in @@ -384,6 +435,8 @@ -(void) terminate // blocking on the caller's thread waiting to acquire initLock. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ [initLock lock]; + [self->_nativeMicrophone stop]; + self->_nativeMicrophone = nil; LiStopConnection(); [initLock unlock]; }); @@ -435,6 +488,14 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer renderer = myRenderer; _callbacks = callbacks; + _micConfig = config; + _microphoneCallbacks = callbacks; + if (config.requestNativeMicrophone) { + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(microphoneInterrupted:) + name:AVAudioSessionInterruptionNotification object:nil]; + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(microphoneRouteChanged:) + name:AVAudioSessionRouteChangeNotification object:nil]; + } LiInitializeStreamConfiguration(&_streamConfig); _streamConfig.width = config.width; _streamConfig.height = config.height; @@ -498,13 +559,33 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer -(void) main { [initLock lock]; - LiStartConnection(&_serverInfo, + _streamConfig.requestNativeMic = _micConfig.requestNativeMicrophone; + int result = LiStartConnection(&_serverInfo, &_streamConfig, &_clCallbacks, &_drCallbacks, &_arCallbacks, NULL, 0, NULL, 0); + if (result == 0 && _micConfig.requestNativeMicrophone) { + _nativeMicrophone = [[NativeAppleMic alloc] init]; + __weak Connection* weakSelf = self; + StreamConfiguration* config = _micConfig; + if (![_nativeMicrophone startWhileAllowed:^BOOL { + return config.requestNativeMicrophone; + } onFailure:^{ + [weakSelf microphoneUnavailable]; + }]) { + _nativeMicrophone = nil; + [self microphoneUnavailable]; + } else { + dispatch_async(dispatch_get_main_queue(), ^{ + if (config.requestNativeMicrophone) { + [self->_microphoneCallbacks microphoneStatusChanged:NativeMicrophoneStatusActive]; + } + }); + } + } [initLock unlock]; } diff --git a/Limelight/Stream/ConnectionCallbacks.h b/Limelight/Stream/ConnectionCallbacks.h index f9c13b4e5..718b92489 100644 --- a/Limelight/Stream/ConnectionCallbacks.h +++ b/Limelight/Stream/ConnectionCallbacks.h @@ -6,7 +6,14 @@ // Copyright © 2020 Moonlight Game Streaming Project. All rights reserved. // +typedef NS_ENUM(NSInteger, NativeMicrophoneStatus) { + NativeMicrophoneStatusActive, + NativeMicrophoneStatusUnavailable, + NativeMicrophoneStatusOff, +}; + @protocol ConnectionCallbacks +- (void)microphoneStatusChanged:(NativeMicrophoneStatus)status; - (void) connectionStarted; - (void) connectionTerminated:(int)errorCode; diff --git a/Limelight/Stream/NativeAppleMic.h b/Limelight/Stream/NativeAppleMic.h new file mode 100644 index 000000000..d398ceeee --- /dev/null +++ b/Limelight/Stream/NativeAppleMic.h @@ -0,0 +1,9 @@ +#import + +@interface NativeAppleMic : NSObject + +// Call start and stop from the connection's serialized lifecycle. +- (BOOL)startWhileAllowed:(BOOL (^)(void))allowed onFailure:(void (^)(void))failure; +- (void)stop; + +@end diff --git a/Limelight/Stream/NativeAppleMic.m b/Limelight/Stream/NativeAppleMic.m new file mode 100644 index 000000000..04831e268 --- /dev/null +++ b/Limelight/Stream/NativeAppleMic.m @@ -0,0 +1,143 @@ +#import "NativeAppleMic.h" +#import +#import +#include +#include "Limelight.h" +#include "NativeMicEncoder.h" + +@implementation NativeAppleMic { + AudioQueueRef _queue; + dispatch_queue_t _captureQueue; + NATIVE_MIC_SENDER* _sender; + OpusEncoder* _encoder; + int16_t _frame[NATIVE_MIC_FRAME_SAMPLES]; + size_t _filled; + atomic_bool _active; + BOOL _duplex; +} + +- (BOOL)startWhileAllowed:(BOOL (^)(void))allowed onFailure:(void (^)(void))failure +{ + atomic_init(&_active, false); + if (@available(iOS 17.0, tvOS 17.0, *)) { + if (AVAudioApplication.sharedInstance.recordPermission != AVAudioApplicationRecordPermissionGranted || + !allowed()) { + return NO; + } + } else { + return NO; + } + + _sender = LiCreateNativeMicSender(); + if (!_sender) { + return NO; + } + _encoder = NativeMicEncoderCreate(); + AVAudioSession* session = AVAudioSession.sharedInstance; + if (!_encoder || ![session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVoiceChat + options:AVAudioSessionCategoryOptionMixWithOthers error:NULL]) { + [self stop]; + return NO; + } + _duplex = YES; + if (!session.isInputAvailable || !allowed()) { + [self stop]; + return NO; + } + + AudioStreamBasicDescription format = { + .mSampleRate = NATIVE_MIC_SAMPLE_RATE, + .mFormatID = kAudioFormatLinearPCM, + .mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, + .mBytesPerPacket = sizeof(int16_t), + .mFramesPerPacket = 1, + .mBytesPerFrame = sizeof(int16_t), + .mChannelsPerFrame = 1, + .mBitsPerChannel = 16, + }; + _captureQueue = dispatch_queue_create("org.moonlight.native-mic", DISPATCH_QUEUE_SERIAL); + __weak NativeAppleMic* weakSelf = self; + OSStatus error = AudioQueueNewInputWithDispatchQueue(&_queue, &format, 0, _captureQueue, + ^(AudioQueueRef queue, AudioQueueBufferRef buffer, const AudioTimeStamp* time, + UInt32 packetCount, const AudioStreamPacketDescription* descriptions) { + (void)time; + (void)packetCount; + (void)descriptions; + NativeAppleMic* mic = weakSelf; + if (!mic || !atomic_load(&mic->_active) || !allowed()) { + return; + } + if (![mic sendBuffer:buffer whileAllowed:allowed] || + AudioQueueEnqueueBuffer(queue, buffer, 0, NULL) != noErr) { + atomic_store(&mic->_active, false); + failure(); + } + }); + for (int i = 0; error == noErr && i < 4; i++) { + AudioQueueBufferRef buffer; + error = AudioQueueAllocateBuffer(_queue, sizeof(_frame), &buffer); + if (error == noErr) { + error = AudioQueueEnqueueBuffer(_queue, buffer, 0, NULL); + } + } + if (error == noErr && allowed()) { + atomic_store(&_active, true); + if (AudioQueueStart(_queue, NULL) == noErr) { + return YES; + } + } + [self stop]; + return NO; +} + +- (BOOL)sendBuffer:(AudioQueueBufferRef)buffer whileAllowed:(BOOL (^)(void))allowed +{ + if (buffer->mAudioDataByteSize > buffer->mAudioDataBytesCapacity || + buffer->mAudioDataByteSize % sizeof(int16_t) != 0) { + return NO; + } + const int16_t* pcm = buffer->mAudioData; + for (size_t i = 0; i < buffer->mAudioDataByteSize / sizeof(*pcm); i++) { + _frame[_filled++] = pcm[i]; + if (_filled == NATIVE_MIC_FRAME_SAMPLES) { + unsigned char packet[NATIVE_MIC_FRAME_BYTES]; + if (!atomic_load(&_active) || !allowed() || + NativeMicEncode(_encoder, _frame, _filled, packet, sizeof(packet)) != OPUS_OK || + LiSendNativeMicFrame(_sender, packet, sizeof(packet)) != 0) { + return NO; + } + _filled = 0; + } + } + return YES; +} + +- (void)stop +{ + atomic_store(&_active, false); + if (_queue) { + AudioQueueStop(_queue, true); + AudioQueueDispose(_queue, true); + _queue = NULL; + } + if (_captureQueue) { + dispatch_sync(_captureQueue, ^{}); + _captureQueue = nil; + } + if (_sender) { + LiDestroyNativeMicSender(_sender); + _sender = NULL; + } + if (_encoder) { + opus_encoder_destroy(_encoder); + _encoder = NULL; + } + if (_duplex) { + [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback + withOptions:AVAudioSessionCategoryOptionMixWithOthers error:NULL]; + _duplex = NO; + } +} + +@end diff --git a/Limelight/Stream/NativeMicEncoder.c b/Limelight/Stream/NativeMicEncoder.c new file mode 100644 index 000000000..f9257a48f --- /dev/null +++ b/Limelight/Stream/NativeMicEncoder.c @@ -0,0 +1,33 @@ +#include "NativeMicEncoder.h" + +OpusEncoder* NativeMicEncoderCreate(void) +{ + int error; + OpusEncoder* encoder = opus_encoder_create(NATIVE_MIC_SAMPLE_RATE, 1, + OPUS_APPLICATION_RESTRICTED_LOWDELAY, &error); + if (encoder == NULL) { + return NULL; + } + if (opus_encoder_ctl(encoder, OPUS_SET_BITRATE(32000)) != OPUS_OK || + opus_encoder_ctl(encoder, OPUS_SET_VBR(0)) != OPUS_OK || + opus_encoder_ctl(encoder, OPUS_SET_DTX(0)) != OPUS_OK) { + opus_encoder_destroy(encoder); + return NULL; + } + return encoder; +} + +int NativeMicEncode(OpusEncoder* encoder, const int16_t* pcm, size_t samples, + unsigned char* packet, size_t capacity) +{ + if (encoder == NULL || pcm == NULL || packet == NULL || + samples != NATIVE_MIC_FRAME_SAMPLES || capacity < NATIVE_MIC_FRAME_BYTES) { + return OPUS_BAD_ARG; + } + int bytes = opus_encode(encoder, pcm, NATIVE_MIC_FRAME_SAMPLES, packet, + NATIVE_MIC_FRAME_BYTES); + if (bytes < 0) { + return bytes; + } + return bytes == NATIVE_MIC_FRAME_BYTES ? OPUS_OK : OPUS_INTERNAL_ERROR; +} diff --git a/Limelight/Stream/NativeMicEncoder.h b/Limelight/Stream/NativeMicEncoder.h new file mode 100644 index 000000000..99992b364 --- /dev/null +++ b/Limelight/Stream/NativeMicEncoder.h @@ -0,0 +1,20 @@ +#ifndef NATIVE_MIC_ENCODER_H +#define NATIVE_MIC_ENCODER_H + +#include +#include +#include + +enum { + NATIVE_MIC_SAMPLE_RATE = 48000, + NATIVE_MIC_FRAME_SAMPLES = 240, + NATIVE_MIC_FRAME_BYTES = 20 +}; + +// Serialize encoder use and destruction. Ownership may transfer before encoding. +OpusEncoder* NativeMicEncoderCreate(void); +// Returns zero for a complete packet and a negative Opus error otherwise. +int NativeMicEncode(OpusEncoder* encoder, const int16_t* pcm, size_t samples, + unsigned char* packet, size_t capacity); + +#endif diff --git a/Limelight/Stream/StreamConfiguration.h b/Limelight/Stream/StreamConfiguration.h index e912b5788..99ef3f879 100644 --- a/Limelight/Stream/StreamConfiguration.h +++ b/Limelight/Stream/StreamConfiguration.h @@ -25,6 +25,7 @@ @property int gamepadMask; @property BOOL optimizeGameSettings; @property BOOL playAudioOnPC; +@property BOOL requestNativeMicrophone; @property BOOL swapABXYButtons; @property int audioConfiguration; @property int supportedVideoFormats; diff --git a/Limelight/Stream/StreamManager.h b/Limelight/Stream/StreamManager.h index 277d6fe43..597dee903 100644 --- a/Limelight/Stream/StreamManager.h +++ b/Limelight/Stream/StreamManager.h @@ -14,6 +14,7 @@ - (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id)callback; - (void) stopStream; +- (void) stopMicrophone; - (NSString*) getStatsOverlayText; diff --git a/Limelight/Stream/StreamManager.m b/Limelight/Stream/StreamManager.m index e776feef0..d1db869e9 100644 --- a/Limelight/Stream/StreamManager.m +++ b/Limelight/Stream/StreamManager.m @@ -106,8 +106,15 @@ - (void)main { }); } +- (void) stopMicrophone +{ + _config.requestNativeMicrophone = NO; + [_connection stopMicrophone]; +} + - (void) stopStream { + _config.requestNativeMicrophone = NO; [_connection terminate]; } diff --git a/Limelight/ViewControllers/MainFrameViewController.m b/Limelight/ViewControllers/MainFrameViewController.m index 169bb13ba..b65fa1647 100644 --- a/Limelight/ViewControllers/MainFrameViewController.m +++ b/Limelight/ViewControllers/MainFrameViewController.m @@ -21,6 +21,7 @@ #import "AppListResponse.h" #import "ServerInfoResponse.h" #import "StreamFrameViewController.h" +#import #import "LoadingFrameViewController.h" #import "ComputerScrollView.h" #import "TemporaryApp.h" @@ -46,6 +47,7 @@ @implementation MainFrameViewController { DiscoveryManager* _discMan; AppAssetManager* _appManager; StreamConfiguration* _streamConfig; + NSUInteger _microphoneLaunchGeneration; UIAlertController* _pairAlert; LoadingFrameViewController* _loadingFrame; UIScrollView* hostScrollView; @@ -602,6 +604,7 @@ - (void) addHostClicked { } - (void) prepareToStreamApp:(TemporaryApp *)app { + ++_microphoneLaunchGeneration; _streamConfig = [[StreamConfiguration alloc] init]; _streamConfig.host = app.host.activeAddress; _streamConfig.httpsPort = app.host.httpsPort; @@ -751,6 +754,39 @@ - (void)appLongClicked:(TemporaryApp *)app view:(UIView *)view { [self performSegueWithIdentifier:@"createStreamFrame" sender:nil]; }]]; + if (@available(iOS 17.0, tvOS 17.0, *)) { + TemporaryApp* microphoneApp = currentApp ?: app; + NSString* microphoneAction = [NSString stringWithFormat:currentApp ? @"Resume %@ with Microphone" : + @"Launch %@ with Microphone", + microphoneApp.name]; + [alertController addAction:[UIAlertAction actionWithTitle:microphoneAction + style:UIAlertActionStyleDefault handler:^(UIAlertAction* action) { + NSUInteger generation = ++self->_microphoneLaunchGeneration; + [AVAudioApplication requestRecordPermissionWithCompletionHandler:^(BOOL granted) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (!self.view.window || self->_selectedHost != app.host || + UIApplication.sharedApplication.applicationState != UIApplicationStateActive || + self->_microphoneLaunchGeneration != generation) { + return; + } + if (!granted) { + UIAlertController* permissionAlert = [UIAlertController + alertControllerWithTitle:@"Microphone unavailable" + message:@"Allow microphone access in Settings to stream with your microphone." + preferredStyle:UIAlertControllerStyleAlert]; + [permissionAlert addAction:[UIAlertAction actionWithTitle:@"OK" + style:UIAlertActionStyleCancel handler:nil]]; + [[self activeViewController] presentViewController:permissionAlert animated:YES completion:nil]; + return; + } + [self prepareToStreamApp:microphoneApp]; + self->_streamConfig.requestNativeMicrophone = YES; + [self performSegueWithIdentifier:@"createStreamFrame" sender:nil]; + }); + }]; + }]]; + } + if (currentApp != nil) { [alertController addAction:[UIAlertAction actionWithTitle: [app.id isEqualToString:currentApp.id] ? @"Quit App" : @"Quit Running App and Start" style:UIAlertActionStyleDestructive handler:^(UIAlertAction* action){ diff --git a/Limelight/ViewControllers/StreamFrameViewController.m b/Limelight/ViewControllers/StreamFrameViewController.m index a4c8f3448..68884d60c 100644 --- a/Limelight/ViewControllers/StreamFrameViewController.m +++ b/Limelight/ViewControllers/StreamFrameViewController.m @@ -42,6 +42,7 @@ @implementation StreamFrameViewController { UITextView *_overlayView; UILabel *_stageLabel; UILabel *_tipLabel; + UILabel *_microphoneLabel; UIActivityIndicatorView *_spinner; StreamView *_streamView; UIScrollView *_scrollView; @@ -154,8 +155,6 @@ - (void)viewDidLoad _streamMan = [[StreamManager alloc] initWithConfig:self.streamConfig renderView:_streamView connectionCallbacks:self]; - NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; - [opQueue addOperation:_streamMan]; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(applicationWillResignActive:) @@ -209,6 +208,27 @@ - (void)viewDidLoad [self.view addSubview:_stageLabel]; [self.view addSubview:_spinner]; [self.view addSubview:_tipLabel]; + if (self.streamConfig.requestNativeMicrophone) { + _microphoneLabel = [[UILabel alloc] init]; + _microphoneLabel.translatesAutoresizingMaskIntoConstraints = NO; + _microphoneLabel.userInteractionEnabled = NO; + _microphoneLabel.textAlignment = NSTextAlignmentCenter; + _microphoneLabel.textColor = UIColor.whiteColor; + _microphoneLabel.backgroundColor = [UIColor colorWithWhite:0 alpha:0.8]; + _microphoneLabel.font = [UIFont preferredFontForTextStyle:UIFontTextStyleFootnote]; + _microphoneLabel.numberOfLines = 0; + _microphoneLabel.text = @"Starting microphone…"; + [self.view addSubview:_microphoneLabel]; + UILayoutGuide* safeArea = self.view.safeAreaLayoutGuide; + [NSLayoutConstraint activateConstraints:@[ + [_microphoneLabel.centerXAnchor constraintEqualToAnchor:safeArea.centerXAnchor], + [_microphoneLabel.topAnchor constraintEqualToAnchor:safeArea.topAnchor constant:8], + [_microphoneLabel.leadingAnchor constraintGreaterThanOrEqualToAnchor:safeArea.leadingAnchor constant:8], + [_microphoneLabel.trailingAnchor constraintLessThanOrEqualToAnchor:safeArea.trailingAnchor constant:-8], + ]]; + } + NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; + [opQueue addOperation:_streamMan]; } - (UIView *)viewForZoomingInScrollView:(UIScrollView *)scrollView { @@ -316,7 +336,23 @@ - (void) returnToMainFrame { } // This will fire if the user opens control center or gets a low battery message +- (void)microphoneStatusChanged:(NativeMicrophoneStatus)status { + switch (status) { + case NativeMicrophoneStatusActive: + _microphoneLabel.text = @" Microphone on · Leave stream to stop "; + break; + case NativeMicrophoneStatusUnavailable: + _microphoneLabel.text = @" Microphone unavailable · Stream continues "; + break; + case NativeMicrophoneStatusOff: + _microphoneLabel.text = @" Microphone off "; + break; + } +} + - (void)applicationWillResignActive:(NSNotification *)notification { + [_streamMan stopMicrophone]; + [self microphoneStatusChanged:NativeMicrophoneStatusOff]; if (_inactivityTimer != nil) { [_inactivityTimer invalidate]; } @@ -390,6 +426,7 @@ - (void) connectionStarted { } - (void)connectionTerminated:(int)errorCode { + [_streamMan stopMicrophone]; Log(LOG_I, @"Connection terminated: %d", errorCode); unsigned int portFlags = LiGetPortFlagsFromTerminationErrorCode(errorCode); diff --git a/Moonlight TV/Info.plist b/Moonlight TV/Info.plist index c1f0beebf..e9be5387d 100644 --- a/Moonlight TV/Info.plist +++ b/Moonlight TV/Info.plist @@ -2,6 +2,8 @@ + NSMicrophoneUsageDescription + Send your microphone to the NVIDIA GameStream host when you choose to stream with microphone forwarding. CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleDisplayName diff --git a/Moonlight.xcodeproj/project.pbxproj b/Moonlight.xcodeproj/project.pbxproj index e686ec129..c4d4ddb9a 100644 --- a/Moonlight.xcodeproj/project.pbxproj +++ b/Moonlight.xcodeproj/project.pbxproj @@ -49,6 +49,8 @@ FB1A67622132419A00507771 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = FB1A67612132419A00507771 /* Assets.xcassets */; }; FB1A67A12132458C00507771 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = FB1A67642132419A00507771 /* main.m */; }; FB1A67A3213245BD00507771 /* Connection.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461719F646E200339C8A /* Connection.m */; }; + C00536ED94C34138BBC376FB /* NativeMicEncoder.c in Sources */ = {isa = PBXBuildFile; fileRef = FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */; }; + 504AB7156B7643B292C10299 /* NativeAppleMic.m in Sources */ = {isa = PBXBuildFile; fileRef = AE40E7E0868344F395471576 /* NativeAppleMic.m */; }; FB1A67A5213245BD00507771 /* StreamConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461919F646E200339C8A /* StreamConfiguration.m */; }; FB1A67A7213245BD00507771 /* StreamManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461B19F646E200339C8A /* StreamManager.m */; }; FB1A67A9213245BD00507771 /* VideoDecoderRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */; }; @@ -81,6 +83,8 @@ FB1A67E32132498A00507771 /* Limelight.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = FB290D0519B2C406004C83CF /* Limelight.xcdatamodeld */; }; FB1A67E521324A1F00507771 /* CoreData.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FB1A67E421324A1F00507771 /* CoreData.framework */; }; FB1A67E821324DE300507771 /* libopus.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FB8946EA19F6AFB800339C8A /* libopus.a */; }; + 82C1A86EEA314CA9B2848884 /* AudioToolbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 638BD6868E254A4980037CEB /* AudioToolbox.framework */; }; + B462FCC5A0214D1ABC3FF981 /* AVFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 9A9B4337320D4B9493FB09EF /* AVFoundation.framework */; }; FB1A67EA21324DF300507771 /* libxml2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = FB1A67E921324DF300507771 /* libxml2.tbd */; }; FB1A6819213284FB00507771 /* UIComputerView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86DF19F7A837001C18A8 /* UIComputerView.m */; }; FB1A681A213284FB00507771 /* UIAppView.m in Sources */ = {isa = PBXBuildFile; fileRef = FBDE86E519F82297001C18A8 /* UIAppView.m */; }; @@ -115,6 +119,8 @@ FB89462D19F646E200339C8A /* MDNSManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461219F646E200339C8A /* MDNSManager.m */; }; FB89462E19F646E200339C8A /* PairManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461419F646E200339C8A /* PairManager.m */; }; FB89462F19F646E200339C8A /* Connection.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461719F646E200339C8A /* Connection.m */; }; + 6F2A12808AF84F8B8F011376 /* NativeMicEncoder.c in Sources */ = {isa = PBXBuildFile; fileRef = FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */; }; + 63B66D53AA0647A39CBEF921 /* NativeAppleMic.m in Sources */ = {isa = PBXBuildFile; fileRef = AE40E7E0868344F395471576 /* NativeAppleMic.m */; }; FB89463019F646E200339C8A /* StreamConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461919F646E200339C8A /* StreamConfiguration.m */; }; FB89463119F646E200339C8A /* StreamManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461B19F646E200339C8A /* StreamManager.m */; }; FB89463219F646E200339C8A /* VideoDecoderRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */; }; @@ -123,6 +129,8 @@ FB89463619F646E200339C8A /* StreamFrameViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462719F646E200339C8A /* StreamFrameViewController.m */; }; FB89463819F6473800339C8A /* Launch Screen.xib in Resources */ = {isa = PBXBuildFile; fileRef = FB89463719F6473800339C8A /* Launch Screen.xib */; }; FB8946ED19F6AFE800339C8A /* libopus.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FB8946EA19F6AFB800339C8A /* libopus.a */; }; + 7F2311C24E6042CF9D57BCF7 /* AudioToolbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 638BD6868E254A4980037CEB /* AudioToolbox.framework */; }; + C25BDE0324F746169F7C629D /* AVFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 9A9B4337320D4B9493FB09EF /* AVFoundation.framework */; }; FB9AFD281A7C84ED00872C98 /* HttpResponse.m in Sources */ = {isa = PBXBuildFile; fileRef = FB9AFD271A7C84ED00872C98 /* HttpResponse.m */; }; FB9AFD321A7D867C00872C98 /* AppAssetRetriever.m in Sources */ = {isa = PBXBuildFile; fileRef = FB9AFD311A7D867C00872C98 /* AppAssetRetriever.m */; }; FB9AFD371A7E02DB00872C98 /* HttpRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = FB9AFD361A7E02DB00872C98 /* HttpRequest.m */; }; @@ -275,6 +283,10 @@ FB89461419F646E200339C8A /* PairManager.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PairManager.m; sourceTree = ""; }; FB89461619F646E200339C8A /* Connection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Connection.h; sourceTree = ""; }; FB89461719F646E200339C8A /* Connection.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Connection.m; sourceTree = ""; }; + 2D0E9120706F440FB232FE0C /* NativeMicEncoder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NativeMicEncoder.h; sourceTree = ""; }; + 0366C723CB7A4512AB8DD129 /* NativeAppleMic.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NativeAppleMic.h; sourceTree = ""; }; + FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = NativeMicEncoder.c; sourceTree = ""; }; + AE40E7E0868344F395471576 /* NativeAppleMic.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NativeAppleMic.m; sourceTree = ""; }; FB89461819F646E200339C8A /* StreamConfiguration.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamConfiguration.h; sourceTree = ""; }; FB89461919F646E200339C8A /* StreamConfiguration.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = StreamConfiguration.m; sourceTree = ""; }; FB89461A19F646E200339C8A /* StreamManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StreamManager.h; sourceTree = ""; }; @@ -293,6 +305,8 @@ FB8946E719F6AFB800339C8A /* opus_multistream.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = opus_multistream.h; sourceTree = ""; }; FB8946E819F6AFB800339C8A /* opus_types.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = opus_types.h; sourceTree = ""; }; FB8946EA19F6AFB800339C8A /* libopus.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libopus.a; sourceTree = ""; }; + 638BD6868E254A4980037CEB /* AudioToolbox.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AudioToolbox.framework; path = System/Library/Frameworks/AudioToolbox.framework; sourceTree = SDKROOT; }; + 9A9B4337320D4B9493FB09EF /* AVFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AVFoundation.framework; path = System/Library/Frameworks/AVFoundation.framework; sourceTree = SDKROOT; }; FB9AFD261A7C84ED00872C98 /* HttpResponse.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HttpResponse.h; sourceTree = ""; }; FB9AFD271A7C84ED00872C98 /* HttpResponse.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = HttpResponse.m; sourceTree = ""; }; FB9AFD301A7D867C00872C98 /* AppAssetRetriever.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AppAssetRetriever.h; sourceTree = ""; }; @@ -334,6 +348,8 @@ F74BEFA42C1A80E400224667 /* OpenSSL in Frameworks */, FB1A67EA21324DF300507771 /* libxml2.tbd in Frameworks */, FB1A67E821324DE300507771 /* libopus.a in Frameworks */, + 82C1A86EEA314CA9B2848884 /* AudioToolbox.framework in Frameworks */, + B462FCC5A0214D1ABC3FF981 /* AVFoundation.framework in Frameworks */, 98181BEB2791278F00E43572 /* libSDL2.a in Frameworks */, FB1A67E521324A1F00507771 /* CoreData.framework in Frameworks */, 98B9CE6D27B2144B00B473C4 /* AVKit.framework in Frameworks */, @@ -348,6 +364,8 @@ 9890CF6B203B7EE1006C4B06 /* libxml2.tbd in Frameworks */, 98CFB82F1CAD481B0048EF74 /* libmoonlight-common.a in Frameworks */, FB8946ED19F6AFE800339C8A /* libopus.a in Frameworks */, + 7F2311C24E6042CF9D57BCF7 /* AudioToolbox.framework in Frameworks */, + C25BDE0324F746169F7C629D /* AVFoundation.framework in Frameworks */, 98882A082AF60F7400C5A11C /* libavutil.a in Frameworks */, FB290CF419B2C406004C83CF /* CoreGraphics.framework in Frameworks */, FB290CF819B2C406004C83CF /* CoreData.framework in Frameworks */, @@ -544,6 +562,10 @@ children = ( FB89461619F646E200339C8A /* Connection.h */, FB89461719F646E200339C8A /* Connection.m */, + 2D0E9120706F440FB232FE0C /* NativeMicEncoder.h */, + 0366C723CB7A4512AB8DD129 /* NativeAppleMic.h */, + FBF5ADD0A7D94DE785F24794 /* NativeMicEncoder.c */, + AE40E7E0868344F395471576 /* NativeAppleMic.m */, FB89461819F646E200339C8A /* StreamConfiguration.h */, FB89461919F646E200339C8A /* StreamConfiguration.m */, FB89461A19F646E200339C8A /* StreamManager.h */, @@ -874,6 +896,8 @@ FB1A67AF213245C500507771 /* IdManager.m in Sources */, 988FCD42293B091B003050E2 /* KeyboardInputField.m in Sources */, FB1A67A3213245BD00507771 /* Connection.m in Sources */, + C00536ED94C34138BBC376FB /* NativeMicEncoder.c in Sources */, + 504AB7156B7643B292C10299 /* NativeAppleMic.m in Sources */, FB1A67A5213245BD00507771 /* StreamConfiguration.m in Sources */, FB1A67A7213245BD00507771 /* StreamManager.m in Sources */, FB1A67A9213245BD00507771 /* VideoDecoderRenderer.m in Sources */, @@ -895,6 +919,8 @@ FB89463419F646E200339C8A /* Utils.m in Sources */, FBDE86E619F82297001C18A8 /* UIAppView.m in Sources */, FB89462F19F646E200339C8A /* Connection.m in Sources */, + 6F2A12808AF84F8B8F011376 /* NativeMicEncoder.c in Sources */, + 63B66D53AA0647A39CBEF921 /* NativeAppleMic.m in Sources */, FB9AFD321A7D867C00872C98 /* AppAssetRetriever.m in Sources */, FB4678FF1A565DAC00377732 /* WakeOnLanManager.m in Sources */, FB53E1431BE5DC4400CD6ECE /* IdManager.m in Sources */, diff --git a/README.md b/README.md index 4b2cf9987..885268bda 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,24 @@ Check out [the Moonlight wiki](https://github.com/moonlight-stream/moonlight-doc * In the "Team" dropdown, select your name. If your name doesn't appear, you may need to sign into Xcode with your Apple account. * Change the "Bundle Identifier" to something different. You can add your name or some random letters to make it unique. * Now you can select your Apple device in the top bar as a target and click the Play button to run. + +## Native GameStream microphone + +On iOS/iPadOS 17 or later and tvOS 17 or later, open an app tile's launch +options and choose **Launch [app] with Microphone** or **Resume [app] with +Microphone**. Grant recording permission when prompted. Apple TV requires +a supported Bluetooth or Continuity input that the OS makes available. + +The choice applies to one stream. An on-screen label reports capture +status. Leaving the stream, interruption, inactivity or loss of the input +route stops forwarding. Start a new explicitly opted-in stream to resume +capture. + +Use the [compatible Windows NVIDIA GameStream host setup](https://github.com/Bargs/native-gamestream-microphone-host) +and a trusted private IPv4 network or secured tunnel. The native legacy +CBC mode does not authenticate packets. The shared transport is reviewed +in [common-c PR #173](https://github.com/moonlight-stream/moonlight-common-c/pull/173). + +Both arm64 device targets passed source and link checks with resource +phases omitted. Standard app packaging and on-device permission, route +and live host delivery need verification. diff --git a/moonlight-common/moonlight-common-c b/moonlight-common/moonlight-common-c index f900dd476..a9107cf1f 160000 --- a/moonlight-common/moonlight-common-c +++ b/moonlight-common/moonlight-common-c @@ -1 +1 @@ -Subproject commit f900dd4767759c7b9d0e93bcea666b55c69ea62f +Subproject commit a9107cf1f54e50be328573f9a79de31d2f3fdcad diff --git a/moonlight-common/moonlight-common.xcodeproj/project.pbxproj b/moonlight-common/moonlight-common.xcodeproj/project.pbxproj index 4c5699382..67c5fee8f 100644 --- a/moonlight-common/moonlight-common.xcodeproj/project.pbxproj +++ b/moonlight-common/moonlight-common.xcodeproj/project.pbxproj @@ -30,6 +30,7 @@ 98AB2E4D1CAD425A0089BB98 /* SdpGenerator.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3B1CAD423B0089BB98 /* SdpGenerator.c */; }; 98AB2E4E1CAD425A0089BB98 /* VideoDepacketizer.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */; }; 98AB2E4F1CAD425A0089BB98 /* VideoStream.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */; }; + D836F126BEB34CD7B04E3E9B /* NativeMicStream.c in Sources */ = {isa = PBXBuildFile; fileRef = F11F0AD711634D1B80B488FC /* NativeMicStream.c */; }; 98AB2E501CAD427A0089BB98 /* callbacks.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF01CAD422B0089BB98 /* callbacks.c */; }; 98AB2E511CAD427A0089BB98 /* compress.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF31CAD422B0089BB98 /* compress.c */; }; 98AB2E521CAD427A0089BB98 /* host.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E001CAD422B0089BB98 /* host.c */; }; @@ -64,6 +65,7 @@ FB1A680721324F4900507771 /* SdpGenerator.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3B1CAD423B0089BB98 /* SdpGenerator.c */; }; FB1A680821324F4900507771 /* VideoDepacketizer.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */; }; FB1A680921324F4900507771 /* VideoStream.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */; }; + 250B5B5384F14AD28BEFD7D2 /* NativeMicStream.c in Sources */ = {isa = PBXBuildFile; fileRef = F11F0AD711634D1B80B488FC /* NativeMicStream.c */; }; FB1A680A21324F4900507771 /* callbacks.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF01CAD422B0089BB98 /* callbacks.c */; }; FB1A680B21324F4900507771 /* compress.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2DF31CAD422B0089BB98 /* compress.c */; }; FB1A680C21324F4900507771 /* host.c in Sources */ = {isa = PBXBuildFile; fileRef = 98AB2E001CAD422B0089BB98 /* host.c */; }; @@ -155,6 +157,7 @@ 98AB2E3C1CAD423B0089BB98 /* Video.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Video.h; sourceTree = ""; }; 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = VideoDepacketizer.c; sourceTree = ""; }; 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = VideoStream.c; sourceTree = ""; }; + F11F0AD711634D1B80B488FC /* NativeMicStream.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = NativeMicStream.c; sourceTree = ""; }; 98B8C2B5262BC1220027F659 /* PlatformCrypto.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = PlatformCrypto.c; sourceTree = ""; }; 98B8C2BA262BC12F0027F659 /* PlatformCrypto.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PlatformCrypto.h; sourceTree = ""; }; 98C1FC11218404030023FDDF /* SimpleStun.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = SimpleStun.c; sourceTree = ""; }; @@ -310,6 +313,7 @@ 98AB2E3C1CAD423B0089BB98 /* Video.h */, 98AB2E3D1CAD423B0089BB98 /* VideoDepacketizer.c */, 98AB2E3E1CAD423B0089BB98 /* VideoStream.c */, + F11F0AD711634D1B80B488FC /* NativeMicStream.c */, ); name = src; path = "moonlight-common-c/src"; @@ -440,6 +444,7 @@ 98F145382667D847004018E3 /* RtpVideoQueue.c in Sources */, FB1A680821324F4900507771 /* VideoDepacketizer.c in Sources */, FB1A680921324F4900507771 /* VideoStream.c in Sources */, + 250B5B5384F14AD28BEFD7D2 /* NativeMicStream.c in Sources */, 98B8C2B7262BC1220027F659 /* PlatformCrypto.c in Sources */, FB1A680A21324F4900507771 /* callbacks.c in Sources */, FB1A680B21324F4900507771 /* compress.c in Sources */, @@ -480,6 +485,7 @@ 98F145372667D847004018E3 /* RtpVideoQueue.c in Sources */, 98AB2E4E1CAD425A0089BB98 /* VideoDepacketizer.c in Sources */, 98AB2E4F1CAD425A0089BB98 /* VideoStream.c in Sources */, + D836F126BEB34CD7B04E3E9B /* NativeMicStream.c in Sources */, 98B8C2B6262BC1220027F659 /* PlatformCrypto.c in Sources */, 98AB2E501CAD427A0089BB98 /* callbacks.c in Sources */, 98AB2E511CAD427A0089BB98 /* compress.c in Sources */,