Add per-stream native microphone capture on iOS and tvOS

Limelight/Limelight-Info.plist

@@ -2,6 +2,8 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>Send your microphone to the NVIDIA GameStream host when you choose to stream with microphone forwarding.</string>
 	<key>CADisableMinimumFrameDuration</key>
 	<true/>
 	<key>CADisableMinimumFrameDurationOnPhone</key>

Limelight/Stream/Connection.h

@@ -27,6 +27,7 @@ typedef struct {
 
 -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer*)myRenderer connectionCallbacks:(id<ConnectionCallbacks>)callbacks;
 -(void) terminate;
+-(void) stopMicrophone;
 -(void) main;
 -(BOOL) getVideoStats:(video_stats_t*)stats;
 -(NSString*) getActiveCodecName;

Limelight/Stream/Connection.m

@@ -8,6 +8,8 @@
 
 #import "Connection.h"
 #import "Utils.h"
+#import "NativeAppleMic.h"
+#import <AVFoundation/AVFoundation.h>
 
 #import <VideoToolbox/VideoToolbox.h>
 
@@ -18,6 +20,9 @@
 #include "opus_multistream.h"
 
 @implementation Connection {
+    StreamConfiguration* _micConfig;
+    __weak id<ConnectionCallbacks> _microphoneCallbacks;
+    NativeAppleMic* _nativeMicrophone;
     SERVER_INFORMATION _serverInfo;
     STREAM_CONFIGURATION _streamConfig;
     CONNECTION_LISTENER_CALLBACKS _clCallbacks;
@@ -371,8 +376,54 @@ void ClSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g, uint8_t
     [_callbacks setControllerLed:controllerNumber r:r g:g b:b];
 }
 
+-(void) stopMicrophone
+{
+    [self stopMicrophoneWithStatus:NativeMicrophoneStatusOff];
+}
+
+-(void) microphoneUnavailable
+{
+    dispatch_async(dispatch_get_main_queue(), ^{
+        if (self->_micConfig.requestNativeMicrophone) {
+            [self stopMicrophoneWithStatus:NativeMicrophoneStatusUnavailable];
+        }
+    });
+}
+
+-(void) stopMicrophoneWithStatus:(NativeMicrophoneStatus)status
+{
+    _micConfig.requestNativeMicrophone = NO;
+    dispatch_async(dispatch_get_main_queue(), ^{
+        [self->_microphoneCallbacks microphoneStatusChanged:status];
+    });
+    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
+        [initLock lock];
+        [self->_nativeMicrophone stop];
+        self->_nativeMicrophone = nil;
+        [initLock unlock];
+    });
+}
+
+-(void) microphoneInterrupted:(NSNotification*)notification
+{
+    if ([notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue] ==
+            AVAudioSessionInterruptionTypeBegan) {
+        [self stopMicrophone];
+    }
+}
+
+-(void) microphoneRouteChanged:(NSNotification*)notification
+{
+    NSUInteger reason = [notification.userInfo[AVAudioSessionRouteChangeReasonKey] unsignedIntegerValue];
+    if (reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable ||
+        reason == AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory) {
+        [self microphoneUnavailable];
+    }
+}
+
 -(void) terminate
 {
+    _micConfig.requestNativeMicrophone = NO;
     // Interrupt any action blocking LiStartConnection(). This is
     // thread-safe and done outside initLock on purpose, since we
     // won't be able to acquire it if LiStartConnection is in
@@ -384,6 +435,8 @@ -(void) terminate
     // blocking on the caller's thread waiting to acquire initLock.
     dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
         [initLock lock];
+        [self->_nativeMicrophone stop];
+        self->_nativeMicrophone = nil;
         LiStopConnection();
         [initLock unlock];
     });
@@ -435,6 +488,14 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer
     renderer = myRenderer;
     _callbacks = callbacks;
 
+    _micConfig = config;
+    _microphoneCallbacks = callbacks;
+    if (config.requestNativeMicrophone) {
+        [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(microphoneInterrupted:)
+                                                   name:AVAudioSessionInterruptionNotification object:nil];
+        [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(microphoneRouteChanged:)
+                                                   name:AVAudioSessionRouteChangeNotification object:nil];
+    }
     LiInitializeStreamConfiguration(&_streamConfig);
     _streamConfig.width = config.width;
     _streamConfig.height = config.height;
@@ -498,13 +559,33 @@ -(id) initWithConfig:(StreamConfiguration*)config renderer:(VideoDecoderRenderer
 -(void) main
 {
     [initLock lock];
-    LiStartConnection(&_serverInfo,
+    _streamConfig.requestNativeMic = _micConfig.requestNativeMicrophone;
+    int result = LiStartConnection(&_serverInfo,
                       &_streamConfig,
                       &_clCallbacks,
                       &_drCallbacks,
                       &_arCallbacks,
                       NULL, 0,
                       NULL, 0);
+    if (result == 0 && _micConfig.requestNativeMicrophone) {
+        _nativeMicrophone = [[NativeAppleMic alloc] init];
+        __weak Connection* weakSelf = self;
+        StreamConfiguration* config = _micConfig;
+        if (![_nativeMicrophone startWhileAllowed:^BOOL {
+            return config.requestNativeMicrophone;
+        } onFailure:^{
+            [weakSelf microphoneUnavailable];
+        }]) {
+            _nativeMicrophone = nil;
+            [self microphoneUnavailable];
+        } else {
+            dispatch_async(dispatch_get_main_queue(), ^{
+                if (config.requestNativeMicrophone) {
+                    [self->_microphoneCallbacks microphoneStatusChanged:NativeMicrophoneStatusActive];
+                }
+            });
+        }
+    }
     [initLock unlock];
 }
 

Limelight/Stream/ConnectionCallbacks.h

@@ -6,7 +6,14 @@
 //  Copyright © 2020 Moonlight Game Streaming Project. All rights reserved.
 //
 
+typedef NS_ENUM(NSInteger, NativeMicrophoneStatus) {
+    NativeMicrophoneStatusActive,
+    NativeMicrophoneStatusUnavailable,
+    NativeMicrophoneStatusOff,
+};
+
 @protocol ConnectionCallbacks <NSObject>
+- (void)microphoneStatusChanged:(NativeMicrophoneStatus)status;
 
 - (void) connectionStarted;
 - (void) connectionTerminated:(int)errorCode;

Limelight/Stream/NativeAppleMic.h

@@ -0,0 +1,9 @@
+#import <Foundation/Foundation.h>
+
+@interface NativeAppleMic : NSObject
+
+// Call start and stop from the connection's serialized lifecycle.
+- (BOOL)startWhileAllowed:(BOOL (^)(void))allowed onFailure:(void (^)(void))failure;
+- (void)stop;
+
+@end

Limelight/Stream/NativeAppleMic.m

@@ -0,0 +1,143 @@
+#import "NativeAppleMic.h"
+#import <AVFoundation/AVFoundation.h>
+#import <AudioToolbox/AudioToolbox.h>
+#include <stdatomic.h>
+#include "Limelight.h"
+#include "NativeMicEncoder.h"
+
+@implementation NativeAppleMic {
+    AudioQueueRef _queue;
+    dispatch_queue_t _captureQueue;
+    NATIVE_MIC_SENDER* _sender;
+    OpusEncoder* _encoder;
+    int16_t _frame[NATIVE_MIC_FRAME_SAMPLES];
+    size_t _filled;
+    atomic_bool _active;
+    BOOL _duplex;
+}
+
+- (BOOL)startWhileAllowed:(BOOL (^)(void))allowed onFailure:(void (^)(void))failure
+{
+    atomic_init(&_active, false);
+    if (@available(iOS 17.0, tvOS 17.0, *)) {
+        if (AVAudioApplication.sharedInstance.recordPermission != AVAudioApplicationRecordPermissionGranted ||
+            !allowed()) {
+            return NO;
+        }
+    } else {
+        return NO;
+    }
+
+    _sender = LiCreateNativeMicSender();
+    if (!_sender) {
+        return NO;
+    }
+    _encoder = NativeMicEncoderCreate();
+    AVAudioSession* session = AVAudioSession.sharedInstance;
+    if (!_encoder || ![session setCategory:AVAudioSessionCategoryPlayAndRecord
+                                      mode:AVAudioSessionModeVoiceChat
+                                   options:AVAudioSessionCategoryOptionMixWithOthers error:NULL]) {
+        [self stop];
+        return NO;
+    }
+    _duplex = YES;
+    if (!session.isInputAvailable || !allowed()) {
+        [self stop];
+        return NO;
+    }
+
+    AudioStreamBasicDescription format = {
+        .mSampleRate = NATIVE_MIC_SAMPLE_RATE,
+        .mFormatID = kAudioFormatLinearPCM,
+        .mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked,
+        .mBytesPerPacket = sizeof(int16_t),
+        .mFramesPerPacket = 1,
+        .mBytesPerFrame = sizeof(int16_t),
+        .mChannelsPerFrame = 1,
+        .mBitsPerChannel = 16,
+    };
+    _captureQueue = dispatch_queue_create("org.moonlight.native-mic", DISPATCH_QUEUE_SERIAL);
+    __weak NativeAppleMic* weakSelf = self;
+    OSStatus error = AudioQueueNewInputWithDispatchQueue(&_queue, &format, 0, _captureQueue,
+        ^(AudioQueueRef queue, AudioQueueBufferRef buffer, const AudioTimeStamp* time,
+          UInt32 packetCount, const AudioStreamPacketDescription* descriptions) {
+            (void)time;
+            (void)packetCount;
+            (void)descriptions;
+            NativeAppleMic* mic = weakSelf;
+            if (!mic || !atomic_load(&mic->_active) || !allowed()) {
+                return;
+            }
+            if (![mic sendBuffer:buffer whileAllowed:allowed] ||
+                AudioQueueEnqueueBuffer(queue, buffer, 0, NULL) != noErr) {
+                atomic_store(&mic->_active, false);
+                failure();
+            }
+        });
+    for (int i = 0; error == noErr && i < 4; i++) {
+        AudioQueueBufferRef buffer;
+        error = AudioQueueAllocateBuffer(_queue, sizeof(_frame), &buffer);
+        if (error == noErr) {
+            error = AudioQueueEnqueueBuffer(_queue, buffer, 0, NULL);
+        }
+    }
+    if (error == noErr && allowed()) {
+        atomic_store(&_active, true);
+        if (AudioQueueStart(_queue, NULL) == noErr) {
+            return YES;
+        }
+    }
+    [self stop];
+    return NO;
+}
+
+- (BOOL)sendBuffer:(AudioQueueBufferRef)buffer whileAllowed:(BOOL (^)(void))allowed
+{
+    if (buffer->mAudioDataByteSize > buffer->mAudioDataBytesCapacity ||
+        buffer->mAudioDataByteSize % sizeof(int16_t) != 0) {
+        return NO;
+    }
+    const int16_t* pcm = buffer->mAudioData;
+    for (size_t i = 0; i < buffer->mAudioDataByteSize / sizeof(*pcm); i++) {
+        _frame[_filled++] = pcm[i];
+        if (_filled == NATIVE_MIC_FRAME_SAMPLES) {
+            unsigned char packet[NATIVE_MIC_FRAME_BYTES];
+            if (!atomic_load(&_active) || !allowed() ||
+                NativeMicEncode(_encoder, _frame, _filled, packet, sizeof(packet)) != OPUS_OK ||
+                LiSendNativeMicFrame(_sender, packet, sizeof(packet)) != 0) {
+                return NO;
+            }
+            _filled = 0;
+        }
+    }
+    return YES;
+}
+
+- (void)stop
+{
+    atomic_store(&_active, false);
+    if (_queue) {
+        AudioQueueStop(_queue, true);
+        AudioQueueDispose(_queue, true);
+        _queue = NULL;
+    }
+    if (_captureQueue) {
+        dispatch_sync(_captureQueue, ^{});
+        _captureQueue = nil;
+    }
+    if (_sender) {
+        LiDestroyNativeMicSender(_sender);
+        _sender = NULL;
+    }
+    if (_encoder) {
+        opus_encoder_destroy(_encoder);
+        _encoder = NULL;
+    }
+    if (_duplex) {
+        [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback
+                                       withOptions:AVAudioSessionCategoryOptionMixWithOthers error:NULL];
+        _duplex = NO;
+    }
+}
+
+@end

Limelight/Stream/NativeMicEncoder.c

@@ -0,0 +1,33 @@
+#include "NativeMicEncoder.h"
+
+OpusEncoder* NativeMicEncoderCreate(void)
+{
+    int error;
+    OpusEncoder* encoder = opus_encoder_create(NATIVE_MIC_SAMPLE_RATE, 1,
+                                               OPUS_APPLICATION_RESTRICTED_LOWDELAY, &error);
+    if (encoder == NULL) {
+        return NULL;
+    }
+    if (opus_encoder_ctl(encoder, OPUS_SET_BITRATE(32000)) != OPUS_OK ||
+        opus_encoder_ctl(encoder, OPUS_SET_VBR(0)) != OPUS_OK ||
+        opus_encoder_ctl(encoder, OPUS_SET_DTX(0)) != OPUS_OK) {
+        opus_encoder_destroy(encoder);
+        return NULL;
+    }
+    return encoder;
+}
+
+int NativeMicEncode(OpusEncoder* encoder, const int16_t* pcm, size_t samples,
+                    unsigned char* packet, size_t capacity)
+{
+    if (encoder == NULL || pcm == NULL || packet == NULL ||
+        samples != NATIVE_MIC_FRAME_SAMPLES || capacity < NATIVE_MIC_FRAME_BYTES) {
+        return OPUS_BAD_ARG;
+    }
+    int bytes = opus_encode(encoder, pcm, NATIVE_MIC_FRAME_SAMPLES, packet,
+                            NATIVE_MIC_FRAME_BYTES);
+    if (bytes < 0) {
+        return bytes;
+    }
+    return bytes == NATIVE_MIC_FRAME_BYTES ? OPUS_OK : OPUS_INTERNAL_ERROR;
+}

Limelight/Stream/NativeMicEncoder.h

@@ -0,0 +1,20 @@
+#ifndef NATIVE_MIC_ENCODER_H
+#define NATIVE_MIC_ENCODER_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <opus.h>
+
+enum {
+    NATIVE_MIC_SAMPLE_RATE = 48000,
+    NATIVE_MIC_FRAME_SAMPLES = 240,
+    NATIVE_MIC_FRAME_BYTES = 20
+};
+
+// Serialize encoder use and destruction. Ownership may transfer before encoding.
+OpusEncoder* NativeMicEncoderCreate(void);
+// Returns zero for a complete packet and a negative Opus error otherwise.
+int NativeMicEncode(OpusEncoder* encoder, const int16_t* pcm, size_t samples,
+                    unsigned char* packet, size_t capacity);
+
+#endif

Limelight/Stream/StreamConfiguration.h

@@ -25,6 +25,7 @@
 @property int gamepadMask;
 @property BOOL optimizeGameSettings;
 @property BOOL playAudioOnPC;
+@property BOOL requestNativeMicrophone;
 @property BOOL swapABXYButtons;
 @property int audioConfiguration;
 @property int supportedVideoFormats;

Limelight/Stream/StreamManager.h

@@ -14,6 +14,7 @@
 - (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id<ConnectionCallbacks>)callback;
 
 - (void) stopStream;
+- (void) stopMicrophone;
 
 - (NSString*) getStatsOverlayText;