Forward native GameStream microphone input on SDL-capable builds

CMakeLists.txt

@@ -172,6 +172,10 @@ if (SOFTWARE_FOUND)
     target_sources(moonlight PRIVATE ./src/video/sdl.c ./src/audio/sdl.c ./src/input/sdl.c)
     target_include_directories(moonlight PRIVATE ${SDL_INCLUDE_DIRS})
     target_link_libraries(moonlight ${SDL_LIBRARIES})
+    if(NOT SDL_VERSION VERSION_LESS "2.0.5")
+      list(APPEND MOONLIGHT_DEFINITIONS HAVE_NATIVE_MIC)
+      target_sources(moonlight PRIVATE ./src/audio/native-mic.c)
+    endif()
   endif()
   if(X11_FOUND)
     list(APPEND MOONLIGHT_DEFINITIONS HAVE_X11)

README.md

@@ -25,3 +25,23 @@ Bugs can be reported to the [issue tracker](https://github.com/moonlight-stream/
 1. Fork us
 2. Write code
 3. Send Pull Requests
+
+## Native GameStream microphone
+
+On builds with SDL 2.0.5 or later and a working default SDL recording device,
+pass `-microphone` explicitly for the stream:
+
+```sh
+moonlight stream -microphone HOST
+```
+
+The option applies to this invocation only. Configuration files cannot enable
+it. View-only mode disables forwarding. The client reports when forwarding
+starts or encounters an error. End the stream to stop microphone capture.
+A device or negotiation failure leaves ordinary streaming available.
+
+Use an NVIDIA GameStream host with the compatible
+[Windows host setup](https://github.com/Bargs/native-gamestream-microphone-host)
+on a trusted private IPv4 network or secured tunnel. The native legacy CBC mode
+has no packet authentication. IPv6 and Sunshine microphone forwarding are
+outside this implementation. Device and live host delivery still need testing.

libgamestream/CMakeLists.txt

@@ -13,7 +13,10 @@ aux_source_directory(../third_party/h264bitstream GAMESTREAM_SRC_LIST)
 
 aux_source_directory(../third_party/moonlight-common-c/enet MOONLIGHT_COMMON_SRC_LIST)
 aux_source_directory(../third_party/moonlight-common-c/src MOONLIGHT_COMMON_SRC_LIST)
-aux_source_directory(../third_party/moonlight-common-c/reedsolomon MOONLIGHT_COMMON_SRC_LIST)
+list(APPEND MOONLIGHT_COMMON_SRC_LIST
+  ../third_party/moonlight-common-c/nanors/rs.c
+  ../third_party/moonlight-common-c/nanors/deps/obl/oblas_common.c
+  ../third_party/moonlight-common-c/nanors/deps/obl/oblas_lite.c)
 
 add_library(moonlight-common SHARED ${MOONLIGHT_COMMON_SRC_LIST})
 
@@ -24,7 +27,13 @@ set_target_properties(gamestream PROPERTIES SOVERSION ${SO_VERSION} VERSION ${PR
 set_target_properties(moonlight-common PROPERTIES SOVERSION ${SO_VERSION} VERSION ${PROJECT_VERSION})
 
 target_include_directories(gamestream PRIVATE ../third_party/moonlight-common-c/src ../third_party/h264bitstream ${AVAHI_INCLUDE_DIRS} ${LibUUID_INCLUDE_DIRS})
-target_include_directories(moonlight-common PRIVATE ../third_party/moonlight-common-c/reedsolomon ../third_party/moonlight-common-c/enet/include)
+set_property(TARGET moonlight-common PROPERTY C_STANDARD 11)
+target_compile_definitions(moonlight-common PRIVATE HAS_SOCKLEN_T)
+target_include_directories(moonlight-common PRIVATE
+  ../third_party/moonlight-common-c/enet/include
+  ../third_party/moonlight-common-c/nanors
+  ../third_party/moonlight-common-c/nanors/deps
+  ../third_party/moonlight-common-c/nanors/deps/obl)
 target_link_libraries(gamestream ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES} ${EXPAT_LIBRARIES} ${AVAHI_LIBRARIES} ${LibUUID_LIBRARIES})
 
 target_link_libraries(gamestream ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})

src/audio/native-mic.c

@@ -0,0 +1,122 @@
+#include "native-mic.h"
+
+#include <Limelight.h>
+#include <SDL.h>
+#include <opus.h>
+#include <stdbool.h>
+#include <stdio.h>
+
+#define FRAME_SAMPLES 240
+#define FRAME_BYTES (FRAME_SAMPLES * sizeof(opus_int16))
+#define ENCODED_BYTES 20
+
+static SDL_AudioDeviceID device;
+static SDL_Thread* worker;
+static SDL_atomic_t running;
+static NATIVE_MIC_SENDER* sender;
+static OpusEncoder* encoder;
+static bool audio_initialized;
+
+static int SDLCALL capture_thread(void* unused) {
+  opus_int16 pcm[FRAME_SAMPLES];
+  unsigned char encoded[ENCODED_BYTES];
+
+  while (SDL_AtomicGet(&running)) {
+    if (SDL_GetAudioDeviceStatus(device) != SDL_AUDIO_PLAYING) {
+      fprintf(stderr, "Microphone device disconnected\n");
+      break;
+    }
+
+    Uint32 available = SDL_GetQueuedAudioSize(device);
+    if (available < FRAME_BYTES) {
+      SDL_Delay(1);
+      continue;
+    }
+    // Drop queued audio after a scheduling stall to keep latency bounded.
+    if (available > FRAME_BYTES * 20) {
+      SDL_ClearQueuedAudio(device);
+      continue;
+    }
+
+    if (SDL_DequeueAudio(device, pcm, FRAME_BYTES) != FRAME_BYTES ||
+        opus_encode(encoder, pcm, FRAME_SAMPLES, encoded, sizeof(encoded)) != ENCODED_BYTES ||
+        LiSendNativeMicFrame(sender, encoded, sizeof(encoded)) != 0) {
+      fprintf(stderr, "Microphone forwarding stopped after a capture or transport error\n");
+      break;
+    }
+  }
+
+  SDL_PauseAudioDevice(device, 1);
+  SDL_AtomicSet(&running, 0);
+  return 0;
+}
+
+void native_mic_stop(void) {
+  SDL_AtomicSet(&running, 0);
+  if (worker != NULL) {
+    SDL_WaitThread(worker, NULL);
+    worker = NULL;
+  }
+  if (device != 0) {
+    SDL_CloseAudioDevice(device);
+    device = 0;
+  }
+  if (encoder != NULL) {
+    opus_encoder_destroy(encoder);
+    encoder = NULL;
+  }
+  LiDestroyNativeMicSender(sender);
+  sender = NULL;
+  if (audio_initialized) {
+    SDL_QuitSubSystem(SDL_INIT_AUDIO);
+    audio_initialized = false;
+  }
+}
+
+void native_mic_start(void) {
+  sender = LiCreateNativeMicSender();
+  if (sender == NULL) {
+    fprintf(stderr, "Microphone unavailable: encrypted native GameStream endpoint on private IPv4 required\n");
+    return;
+  }
+
+  int error;
+  encoder = opus_encoder_create(48000, 1, OPUS_APPLICATION_RESTRICTED_LOWDELAY, &error);
+  if (error != OPUS_OK || encoder == NULL ||
+      opus_encoder_ctl(encoder, OPUS_SET_BITRATE(32000)) != OPUS_OK ||
+      opus_encoder_ctl(encoder, OPUS_SET_VBR(0)) != OPUS_OK ||
+      opus_encoder_ctl(encoder, OPUS_SET_DTX(0)) != OPUS_OK) {
+    fprintf(stderr, "Microphone encoder could not start\n");
+    native_mic_stop();
+    return;
+  }
+
+  if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) {
+    fprintf(stderr, "Microphone audio initialization failed: %s\n", SDL_GetError());
+    native_mic_stop();
+    return;
+  }
+  audio_initialized = true;
+
+  SDL_AudioSpec requested = {0};
+  requested.freq = 48000;
+  requested.format = AUDIO_S16SYS;
+  requested.channels = 1;
+  requested.samples = 256;
+  device = SDL_OpenAudioDevice(NULL, 1, &requested, NULL, 0);
+  if (device == 0) {
+    fprintf(stderr, "Default microphone could not open: %s\n", SDL_GetError());
+    native_mic_stop();
+    return;
+  }
+
+  SDL_AtomicSet(&running, 1);
+  SDL_PauseAudioDevice(device, 0);
+  worker = SDL_CreateThread(capture_thread, "native-mic", NULL);
+  if (worker == NULL) {
+    fprintf(stderr, "Microphone worker could not start: %s\n", SDL_GetError());
+    native_mic_stop();
+    return;
+  }
+  fprintf(stderr, "Microphone forwarding ON. End the stream to stop forwarding.\n");
+}

src/audio/native-mic.h

@@ -0,0 +1,5 @@
+#pragma once
+
+// Call only after a successful, explicitly opted-in GameStream connection.
+void native_mic_start(void);
+void native_mic_stop(void);

src/config.c

@@ -44,7 +44,12 @@
 
 bool inputAdded = false;
 
+enum { OPT_MICROPHONE = 1000 };
+
 static struct option long_options[] = {
+#ifdef HAVE_NATIVE_MIC
+  {"microphone", no_argument, NULL, OPT_MICROPHONE},
+#endif
   {"720", no_argument, NULL, 'a'},
   {"1080", no_argument, NULL, 'b'},
   {"4k", no_argument, NULL, '0'},
@@ -133,6 +138,11 @@ char* get_path(char* name, char* extra_data_dirs) {
 
 static void parse_argument(int c, char* value, PCONFIGURATION config) {
   switch (c) {
+#ifdef HAVE_NATIVE_MIC
+  case OPT_MICROPHONE:
+    config->stream.requestNativeMic = true;
+    break;
+#endif
   case 'a':
     config->stream.width = 1280;
     config->stream.height = 720;
@@ -292,6 +302,9 @@ bool config_file_parse(char* filename, PCONFIGURATION config) {
       } else {
         for (int i=0;long_options[i].name != NULL;i++) {
           if (strcmp(long_options[i].name, key) == 0) {
+            // Microphone consent is accepted only from the command line.
+            if (long_options[i].val == OPT_MICROPHONE)
+              break;
             if (long_options[i].has_arg == required_argument)
               parse_argument(long_options[i].val, value, config);
             else if (strcmp("true", value) == 0)

src/main.c

@@ -25,6 +25,9 @@
 #include "sdl.h"
 
 #include "audio/audio.h"
+#ifdef HAVE_NATIVE_MIC
+#include "audio/native-mic.h"
+#endif
 #include "video/video.h"
 
 #include "input/mapping.h"
@@ -143,12 +146,28 @@ static void stream(PSERVER_DATA server, PCONFIGURATION config, enum platform sys
     loop_init();
 
   platform_start(system);
-  LiStartConnection(&server->serverInfo, &config->stream, &connection_callbacks, platform_get_video(system), platform_get_audio(system, config->audio_device), NULL, drFlags, config->audio_device, 0);
+#ifdef HAVE_NATIVE_MIC
+  if (config->viewonly)
+    config->stream.requestNativeMic = false;
+#endif
+  ret = LiStartConnection(&server->serverInfo, &config->stream, &connection_callbacks, platform_get_video(system), platform_get_audio(system, config->audio_device), NULL, drFlags, config->audio_device, 0);
+  if (ret != 0) {
+    platform_stop(system);
+    return;
+  }
+
+#ifdef HAVE_NATIVE_MIC
+  if (config->stream.requestNativeMic)
+    native_mic_start();
+#endif
 
   if (IS_EMBEDDED(system)) {
     if (!config->viewonly)
       evdev_start();
     loop_main();
+#ifdef HAVE_NATIVE_MIC
+    native_mic_stop();
+#endif
     if (!config->viewonly)
       evdev_stop();
   }
@@ -157,6 +176,9 @@ static void stream(PSERVER_DATA server, PCONFIGURATION config, enum platform sys
     sdl_loop();
   #endif
 
+#ifdef HAVE_NATIVE_MIC
+  native_mic_stop();
+#endif
   LiStopConnection();
 
   if (config->quitappafter) {
@@ -206,6 +228,9 @@ static void help() {
   printf("\t-remote <yes/no/auto>\t\t\tEnable optimizations for WAN streaming (default auto)\n");
   printf("\t-app <app>\t\tName of app to stream\n");
   printf("\t-nosops\t\t\tDon't allow GFE to modify game settings\n");
+#ifdef HAVE_NATIVE_MIC
+  printf("\t-microphone\t\tForward the default microphone for this stream only\n");
+#endif
   printf("\t-localaudio\t\tPlay audio locally on the host computer\n");
   printf("\t-surround <5.1/7.1>\t\tStream 5.1 or 7.1 surround sound\n");
   printf("\t-keydir <directory>\tLoad encryption keys from directory\n");

src/sdl.c

@@ -20,6 +20,9 @@
 #ifdef HAVE_SDL
 
 #include "sdl.h"
+#ifdef HAVE_NATIVE_MIC
+#include "audio/native-mic.h"
+#endif
 #include "input/sdl.h"
 
 #include <Limelight.h>
@@ -113,6 +116,9 @@ void sdl_loop() {
     }
   }
 
+#ifdef HAVE_NATIVE_MIC
+  native_mic_stop();
+#endif
   SDL_DestroyWindow(window);
   SDL_Quit();
 }