Forward native GameStream microphone audio on Android

README.md

@@ -33,3 +33,21 @@ You can follow development on our [Discord server](https://moonlight-stream.org/
 
 Moonlight is the work of students at [Case Western](http://case.edu) and was
 started as a project at [MHacks](http://mhacks.org).
+
+## Native GameStream microphone
+
+On Android 6 or later, enable **Forward microphone to NVIDIA GameStream
+for this run** in streaming settings and grant microphone permission.
+This setting starts off for each app process. Capture starts after successful
+native GameStream negotiation, shows a stream indicator, and stops when the
+activity pauses or the connection ends. Reconnect to enable it again after
+leaving the foreground.
+
+Use a compatible [Windows NVIDIA GameStream host](https://github.com/Bargs/native-gamestream-microphone-host)
+on a trusted private IPv4 network or separately secured tunnel. The legacy
+CBC mode does not authenticate microphone packets. This path is distinct
+from Sunshine microphone extensions.
+
+The arm64 non-root debug app builds with SDK 37 and NDK 29. Physical
+microphone, permission, route changes, pause and native host delivery
+still need device verification.

app/src/main/AndroidManifest.xml

@@ -2,6 +2,8 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
 
     <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-feature android:name="android.hardware.microphone" android:required="false" />
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
     <uses-permission android:name="android.permission.VIBRATE" />
     <uses-permission android:name="android.permission.WAKE_LOCK" />

app/src/main/java/com/limelight/Game.java

@@ -478,6 +478,7 @@ public void notifyCrash(Exception e) {
                 .setAudioConfiguration(prefConfig.audioConfiguration)
                 .setColorSpace(decoderRenderer.getPreferredColorSpace())
                 .setColorRange(decoderRenderer.getPreferredColorRange())
+                .setNativeMicrophone(prefConfig.nativeMicrophone)
                 .setPersistGamepadsAfterDisconnect(!prefConfig.multiController)
                 .build();
 
@@ -1062,6 +1063,10 @@ protected void onDestroy() {
 
     @Override
     protected void onPause() {
+        if (conn != null) {
+            conn.stopMicrophone();
+        }
+
         if (isFinishing()) {
             // Stop any further input device notifications before we lose focus (and pointer capture)
             if (controllerHandler != null) {
@@ -2460,6 +2465,12 @@ public void run() {
         });
     }
 
+    @Override
+    public void microphoneStateChanged(boolean active) {
+        runOnUiThread(() -> findViewById(R.id.microphoneIndicator)
+                .setVisibility(active ? View.VISIBLE : View.GONE));
+    }
+
     @Override
     public void displayTransientMessage(final String message) {
         if (!prefConfig.disableWarnings) {

app/src/main/java/com/limelight/nvstream/NvConnection.java

@@ -39,6 +39,8 @@
 import com.limelight.nvstream.http.PairingManager;
 import com.limelight.nvstream.input.MouseButtonPacket;
 import com.limelight.nvstream.jni.MoonBridge;
+import com.limelight.nvstream.jni.NativeMicCapture;
+import com.limelight.R;
 
 public class NvConnection {
     // Context parameters
@@ -48,6 +50,19 @@ public class NvConnection {
     private static Semaphore connectionAllowed = new Semaphore(1);
     private final boolean isMonkey;
     private final Context appContext;
+    private final Object microphoneLock = new Object();
+    private boolean microphoneAllowed = true;
+    private NativeMicCapture microphone;
+
+    public void stopMicrophone() {
+        synchronized (microphoneLock) {
+            microphoneAllowed = false;
+            if (microphone != null) {
+                microphone.close();
+                microphone = null;
+            }
+        }
+    }
 
     public NvConnection(Context appContext, ComputerDetails.AddressTuple host, int httpsPort, String uniqueId, StreamConfiguration config, LimelightCryptoProvider cryptoProvider, X509Certificate serverCert)
     {
@@ -87,6 +102,7 @@ private static int generateRiKeyId() {
     }
 
     public void stop() {
+        stopMicrophone();
         // Interrupt any pending connection. This is thread-safe.
         MoonBridge.interruptConnection();
 
@@ -425,6 +441,11 @@ public void run() {
                 // we must not invoke that functionality in parallel.
                 synchronized (MoonBridge.class) {
                     MoonBridge.setupBridge(videoDecoderRenderer, audioRenderer, connectionListener);
+                    boolean requestMicrophone;
+                    synchronized (microphoneLock) {
+                        requestMicrophone = microphoneAllowed && context.streamConfig.getNativeMicrophone() &&
+                                NativeMicCapture.hasPermission(appContext);
+                    }
                     int ret = MoonBridge.startConnection(context.serverAddress.address,
                             context.serverAppVersion, context.serverGfeVersion, context.rtspSessionUrl,
                             context.serverCodecModeSupport,
@@ -437,14 +458,23 @@ public void run() {
                             context.riKey.getEncoded(), ib.array(),
                             context.videoCapabilities,
                             context.streamConfig.getColorSpace(),
-                            context.streamConfig.getColorRange());
+                            context.streamConfig.getColorRange(), requestMicrophone);
                     if (ret != 0) {
                         // LiStartConnection() failed, so the caller is not expected
                         // to stop the connection themselves. We need to release their
                         // semaphore count for them.
                         connectionAllowed.release();
                         return;
                     }
+                    synchronized (microphoneLock) {
+                        if (microphoneAllowed && requestMicrophone) {
+                            microphone = NativeMicCapture.start(appContext, connectionListener::microphoneStateChanged);
+                            if (microphone == null) {
+                                connectionListener.displayTransientMessage(
+                                        appContext.getString(R.string.native_microphone_unavailable));
+                            }
+                        }
+                    }
                 }
             }
         }).start();

app/src/main/java/com/limelight/nvstream/NvConnectionListener.java

@@ -8,6 +8,7 @@ public interface NvConnectionListener {
     void connectionStarted();
     void connectionTerminated(int errorCode);
     void connectionStatusUpdate(int connectionStatus);
+    default void microphoneStateChanged(boolean active) {}
 
     void displayMessage(String message);
     void displayTransientMessage(String message);

app/src/main/java/com/limelight/nvstream/StreamConfiguration.java

@@ -26,6 +26,11 @@ public class StreamConfiguration {
     private int attachedGamepadMask;
     private int encryptionFlags;
     private int colorRange;
+    private boolean nativeMicrophone;
+
+    public boolean getNativeMicrophone() {
+        return nativeMicrophone;
+    }
     private int colorSpace;
     private boolean persistGamepadsAfterDisconnect;
 
@@ -118,6 +123,11 @@ public StreamConfiguration.Builder setSupportedVideoFormats(int supportedVideoFo
             return this;
         }
 
+        public StreamConfiguration.Builder setNativeMicrophone(boolean enabled) {
+            config.nativeMicrophone = enabled;
+            return this;
+        }
+
         public StreamConfiguration.Builder setColorRange(int colorRange) {
             config.colorRange = colorRange;
             return this;

app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java

@@ -348,7 +348,7 @@ public static native int startConnection(String address, String appVersion, Stri
                                               int clientRefreshRateX100,
                                               byte[] riAesKey, byte[] riAesIv,
                                               int videoCapabilities,
-                                              int colorSpace, int colorRange);
+                                              int colorSpace, int colorRange, boolean requestNativeMicrophone);
 
     public static native void stopConnection();
 

app/src/main/java/com/limelight/nvstream/jni/NativeMicCapture.java

@@ -0,0 +1,144 @@
+package com.limelight.nvstream.jni;
+
+import android.Manifest;
+import android.content.Context;
+import android.content.pm.PackageManager;
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.MediaRecorder;
+import android.os.Build;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/** Owned by a single opted-in connection. Close before native connection teardown. */
+public final class NativeMicCapture implements AutoCloseable {
+    private static final int SAMPLE_RATE = 48000;
+    private static final int FRAME_SAMPLES = 240;
+
+    static {
+        System.loadLibrary("moonlight-core");
+    }
+
+    private final AtomicBoolean stopping = new AtomicBoolean();
+    private final Thread worker;
+    public interface Listener {
+        void onMicrophoneStateChanged(boolean active);
+    }
+
+    private NativeMicCapture(AudioRecord input, long handle, Listener listener) {
+        worker = new Thread(() -> capture(input, handle, listener), "Moonlight microphone");
+        worker.start();
+    }
+
+    public static boolean hasPermission(Context context) {
+        return Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
+                context.checkSelfPermission(Manifest.permission.RECORD_AUDIO)
+                        == PackageManager.PERMISSION_GRANTED;
+    }
+
+    /** Call after successful native connection start, while foreground and opted in. */
+    public static NativeMicCapture start(Context context, Listener listener) {
+        if (!hasPermission(context)) {
+            return null;
+        }
+        long handle = create();
+        if (handle == 0) {
+            return null;
+        }
+
+        AudioRecord input = null;
+        boolean handedOff = false;
+        try {
+            int minimum = AudioRecord.getMinBufferSize(SAMPLE_RATE,
+                    AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
+            if (minimum <= 0) {
+                return null;
+            }
+            input = new AudioRecord(MediaRecorder.AudioSource.MIC, SAMPLE_RATE,
+                    AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT,
+                    Math.max(minimum, FRAME_SAMPLES * Short.BYTES * 4));
+            if (input.getState() != AudioRecord.STATE_INITIALIZED) {
+                return null;
+            }
+            NativeMicCapture capture = new NativeMicCapture(input, handle, listener);
+            handedOff = true;
+            return capture;
+        } catch (IllegalArgumentException | SecurityException e) {
+            return null;
+        } finally {
+            if (!handedOff) {
+                try {
+                    if (input != null) {
+                        input.release();
+                    }
+                } finally {
+                    destroy(handle);
+                }
+            }
+        }
+    }
+
+    private void capture(AudioRecord input, long handle, Listener listener) {
+        try {
+            short[] frame = new short[FRAME_SAMPLES];
+            int filled = 0;
+            if (stopping.get()) {
+                return;
+            }
+            input.startRecording();
+            if (input.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
+                return;
+            }
+            listener.onMicrophoneStateChanged(true);
+            while (!stopping.get()) {
+                int read = input.read(frame, filled, FRAME_SAMPLES - filled,
+                        AudioRecord.READ_NON_BLOCKING);
+                if (read < 0) {
+                    return;
+                }
+                if (read == 0) {
+                    Thread.sleep(2);
+                    continue;
+                }
+                filled += read;
+                if (filled == FRAME_SAMPLES) {
+                    if (stopping.get() || send(handle, frame) != 0) {
+                        return;
+                    }
+                    filled = 0;
+                }
+            }
+        } catch (IllegalStateException | SecurityException | InterruptedException e) {
+            // Device loss, permission change or owner shutdown ends this capture.
+        } finally {
+            try {
+                input.release();
+            } finally {
+                destroy(handle);
+                listener.onMicrophoneStateChanged(false);
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        stopping.set(true);
+        worker.interrupt();
+        boolean interrupted = false;
+        for (;;) {
+            try {
+                worker.join();
+                break;
+            } catch (InterruptedException e) {
+                interrupted = true;
+            }
+        }
+        if (interrupted) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    private static native long create();
+    private static native int send(long handle, short[] frame);
+    private static native void destroy(long handle);
+}

app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java

@@ -127,6 +127,16 @@ public enum AnalogStickForScrolling {
     public FormatOption videoFormat;
     public int deadzonePercentage;
     public int oscOpacity;
+    private static volatile boolean nativeMicrophoneForRun;
+    public boolean nativeMicrophone;
+
+    static boolean isNativeMicrophoneEnabledForRun() {
+        return nativeMicrophoneForRun;
+    }
+
+    static void setNativeMicrophoneEnabledForRun(boolean enabled) {
+        nativeMicrophoneForRun = enabled;
+    }
     public boolean stretchVideo, enableSops, playHostAudio, disableWarnings;
     public String language;
     public boolean smallIconMode, multiController, usbDriver, flipFaceButtons;
@@ -574,6 +584,7 @@ else if (audioConfig.equals("51")) {
         config.disableWarnings = prefs.getBoolean(DISABLE_TOASTS_PREF_STRING, DEFAULT_DISABLE_TOASTS);
         config.enableSops = prefs.getBoolean(SOPS_PREF_STRING, DEFAULT_SOPS);
         config.stretchVideo = prefs.getBoolean(STRETCH_PREF_STRING, DEFAULT_STRETCH);
+        config.nativeMicrophone = isNativeMicrophoneEnabledForRun();
         config.playHostAudio = prefs.getBoolean(HOST_AUDIO_PREF_STRING, DEFAULT_HOST_AUDIO);
         config.smallIconMode = prefs.getBoolean(SMALL_ICONS_PREF_STRING, getDefaultSmallMode(context));
         config.multiController = prefs.getBoolean(MULTI_CONTROLLER_PREF_STRING, DEFAULT_MULTI_CONTROLLER);

app/src/main/java/com/limelight/preferences/StreamSettings.java

@@ -1,5 +1,6 @@
 package com.limelight.preferences;
 
+import android.Manifest;
 import android.content.Context;
 import android.content.Intent;
 import android.content.SharedPreferences;
@@ -122,6 +123,20 @@ public void onBackPressed() {
     }
 
     public static class SettingsFragment extends PreferenceFragment {
+        private static final int NATIVE_MIC_PERMISSION = 211;
+
+        @Override
+        public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) {
+            super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+            if (requestCode == NATIVE_MIC_PERMISSION && grantResults.length == 1 &&
+                    grantResults[0] == PackageManager.PERMISSION_GRANTED && isAdded()) {
+                CheckBoxPreference microphone = (CheckBoxPreference) findPreference("checkbox_native_microphone");
+                if (microphone != null) {
+                    PreferenceConfiguration.setNativeMicrophoneEnabledForRun(true);
+                    microphone.setChecked(true);
+                }
+            }
+        }
         private int nativeResolutionStartIndex = Integer.MAX_VALUE;
         private boolean nativeFramerateShown = false;
 
@@ -276,6 +291,21 @@ public void onCreate(Bundle savedInstanceState) {
 
             addPreferencesFromResource(R.xml.preferences);
             PreferenceScreen screen = getPreferenceScreen();
+            CheckBoxPreference microphone = (CheckBoxPreference) findPreference("checkbox_native_microphone");
+            microphone.setChecked(PreferenceConfiguration.isNativeMicrophoneEnabledForRun());
+            if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) {
+                microphone.setEnabled(false);
+            } else {
+                microphone.setOnPreferenceChangeListener((preference, value) -> {
+                    if (Boolean.TRUE.equals(value) &&
+                            getActivity().checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
+                        requestPermissions(new String[]{Manifest.permission.RECORD_AUDIO}, NATIVE_MIC_PERMISSION);
+                        return false;
+                    }
+                    PreferenceConfiguration.setNativeMicrophoneEnabledForRun(Boolean.TRUE.equals(value));
+                    return true;
+                });
+            }
 
             // hide on-screen controls category on non touch screen devices
             if (!getActivity().getPackageManager().hasSystemFeature(PackageManager.FEATURE_TOUCHSCREEN)) {