From 3244ca8041437fa19146d062dac4c90ec2162423 Mon Sep 17 00:00:00 2001 From: Janyger Date: Wed, 15 Oct 2025 21:28:32 +0200 Subject: [PATCH 01/22] Improvements 3d --- app/src/main/java/com/limelight/Game.java | 31 +- app/src/main/java/com/limelight/GameMenu.java | 3 + .../video/MediaCodecDecoderRenderer.java | 36 ++- .../limelight/preferences/StreamSettings.java | 12 +- .../utils/ReflectivePaddingInt8Minimal.java | 2 +- .../com/limelight/utils/ServerHelper.java | 3 +- .../java/com/limelight/utils/ShaderUtils.java | 124 ++------ .../com/limelight/utils/Stereo3DRenderer.java | 301 ++++++++++-------- app/src/main/res/xml/preferences.xml | 16 +- 9 files changed, 266 insertions(+), 262 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 992fc00a71..a413577c51 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -398,15 +398,10 @@ protected void onCreate(Bundle savedInstanceState) { boolean shouldInvertDecoderResolution = false; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M - && onExternelDisplay - && prefConfig.renderMode == 0 // For 3D we want to maintain configured resolution - ) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && onExternelDisplay) { Display.Mode currentMode = currentDisplay.getMode(); - displayWidth = currentMode.getPhysicalWidth(); - displayHeight = currentMode.getPhysicalHeight(); - prefConfig.width = displayWidth; - prefConfig.height = displayHeight; + displayWidth = prefConfig.width; + displayHeight = prefConfig.height; prefConfig.fps = currentMode.getRefreshRate(); prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; prefConfig.enableFloatingButton = false; @@ -668,7 +663,8 @@ public void notifyCrash(Exception e) { willStreamHdr, shouldInvertDecoderResolution, glPrefs.glRenderer, - this); + this, + currentDisplay); // --- Force tight thresholds (prefConfig.forceTightThresholds) --- try { @@ -748,7 +744,12 @@ public void notifyCrash(Exception e) { // Set to the optimal mode for streaming float displayRefreshRate = prepareDisplayForRendering(currentDisplay); - LimeLog.info("Display refresh rate: "+displayRefreshRate); + + // Set WindowAttributes is not working on external screens and received fps were wrong + // leading to weird stream connection with barely 500kbs + if (isOnExternalDisplay() && Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + displayRefreshRate = currentDisplay.getMode().getRefreshRate(); + } // If the user requested frame pacing using a capped FPS, we will need to change our // desired FPS setting here in accordance with the active display refresh rate. @@ -775,6 +776,10 @@ public void notifyCrash(Exception e) { if (prefConfig.framePacingWarpFactor > 0) { chosenFrameRate *= prefConfig.framePacingWarpFactor; } + //As external displays might return 60.004 fps, it seems to cause low quality stream + if(isOnExternalDisplay()) { + chosenFrameRate = (int) chosenFrameRate; + } StreamConfiguration config = new StreamConfiguration.Builder() .setResolution( @@ -927,6 +932,7 @@ public void notifyCrash(Exception e) { try { java.lang.reflect.Method m = SurfaceView.class.getMethod("setFrameRate", float.class, int.class); + LimeLog.info("MYLOG TARGETTEST " + targetFps + " " + displayHz); m.invoke(streamSurfaceView, Math.min(targetFps, displayHz), compat); } catch (Throwable ignored) {} } @@ -1142,7 +1148,7 @@ public void toggleVirtualController(){ } private void setPreferredOrientationForActivity() { - Display display = getActiveDisplay(Game.this, prefConfig); + Display display = getActiveDisplay(Game.this); // For semi-square displays, we use more complex logic to determine which orientation to use (if any) if (PreferenceConfiguration.isSquarishScreen(display)) { @@ -1429,7 +1435,7 @@ private boolean shouldIgnoreInsetsForResolution(int width, int height) { } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display display = getActiveDisplay(Game.this, prefConfig); + Display display = getActiveDisplay(Game.this); for (Display.Mode candidate : display.getSupportedModes()) { // Ignore insets if this is an exact match for the display resolution if ((width == candidate.getPhysicalWidth() && height == candidate.getPhysicalHeight()) || @@ -1623,6 +1629,7 @@ else if (!isRefreshRateGoodMatch(candidate.getRefreshRate())) { LimeLog.info("surfaceChanged-->"+(double)displayWidth / (double)displayHeight); LimeLog.info("scaleMode-->"+prefConfig.videoScaleMode); } + // streamContainer.setAsIs(true); // Set the desired refresh rate that will get passed into setFrameRate() later desiredRefreshRate = displayRefreshRate; diff --git a/app/src/main/java/com/limelight/GameMenu.java b/app/src/main/java/com/limelight/GameMenu.java index 63f3532206..7d09789018 100755 --- a/app/src/main/java/com/limelight/GameMenu.java +++ b/app/src/main/java/com/limelight/GameMenu.java @@ -19,6 +19,7 @@ import com.limelight.preferences.PreferenceConfiguration; import com.limelight.utils.KeyConfigHelper; import com.limelight.utils.KeyMapper; +import com.limelight.utils.Stereo3DRenderer; import java.lang.reflect.Field; import java.util.ArrayList; @@ -106,6 +107,8 @@ private void run(MenuOption option) { } private void showMenuDialog(String title, MenuOption[] options) { + + Stereo3DRenderer.isDebugMode = !Stereo3DRenderer.isDebugMode; int themeResId = game.getApplicationInfo().theme; Context themedContext = new ContextThemeWrapper(dialogScreenContext, themeResId); diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index b131684e2b..3b8ef1bb1d 100755 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -1,5 +1,6 @@ package com.limelight.binding.video; +import static com.limelight.utils.ServerHelper.getActiveDisplay; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -8,11 +9,9 @@ import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; - import org.jcodec.codecs.h264.H264Utils; import org.jcodec.codecs.h264.io.model.SeqParameterSet; import org.jcodec.codecs.h264.io.model.VUIParameters; - import com.limelight.BuildConfig; import com.limelight.LimeLog; import com.limelight.R; @@ -21,14 +20,12 @@ import com.limelight.preferences.PreferenceConfiguration; import com.limelight.utils.Stereo3DRenderer; import com.limelight.utils.TrafficStatsHelper; - import android.annotation.SuppressLint; import android.util.LongSparseArray; import android.annotation.TargetApi; import android.app.Activity; import android.content.Context; import android.media.MediaCodec; -import android.os.Bundle; import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.media.MediaCodec.BufferInfo; @@ -41,6 +38,7 @@ import android.os.SystemClock; import android.util.Range; import android.view.Choreographer; +import android.view.Display; import android.view.Surface; public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements Choreographer.FrameCallback { @@ -120,6 +118,7 @@ private void updateDecodeLatencyStats(long presentationTimeUs) { private ByteBuffer nextInputBuffer; private Context context; + private Display display; private Activity activity; private MediaCodec videoDecoder; private Thread rendererThread; @@ -365,9 +364,9 @@ public void setRenderTarget(Surface renderTarget) { public MediaCodecDecoderRenderer(Activity activity, PreferenceConfiguration prefs, CrashListener crashListener, int consecutiveCrashCount, boolean meteredData, boolean requestedHdr, boolean invertResolution, - String glRenderer, PerfOverlayListener perfListener) { + String glRenderer, PerfOverlayListener perfListener, Display display) { //dumpDecoders(); - + this.display = display; this.context = activity; this.activity = activity; this.prefs = prefs; @@ -604,7 +603,7 @@ else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { } } - LimeLog.info("Configuring with format: "+format); + LimeLog.info("Configuring with format: "+format + " " + (renderTarget != null && renderTarget instanceof Surface)); videoDecoder.configure(format, renderTarget, null, 0); @@ -796,7 +795,14 @@ public void onFrameRendered(MediaCodec mediaCodec, long presentationTimeUs, long @Override public int setup(int format, int width, int height, int redrawRate) { - this.targetFps = (redrawRate > 0 ? redrawRate : 60); + // External displayes return a redrawRate of zero, so default 60 was wrong. + int fpsTarget = redrawRate; + if(display == null && fpsTarget <= 0) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + fpsTarget = (int) getActiveDisplay(activity).getMode().getRefreshRate(); + } + } + this.targetFps = fpsTarget; this.initialWidth = invertResolution ? height : width; this.initialHeight = invertResolution ? width : height; this.videoFormat = format; @@ -1073,7 +1079,10 @@ public void doFrame(long frameTimeNanos) { } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - frameTimeNanos -= activity.getWindowManager().getDefaultDisplay().getAppVsyncOffsetNanos(); + if(display == null) { + display = getActiveDisplay(activity); + } + frameTimeNanos -= display.getAppVsyncOffsetNanos(); } // Don't render unless a new frame is due. This prevents microstutter when streaming @@ -1164,11 +1173,14 @@ public void run() { long vsyncPeriodNs; float displayHz = 60f; try { - if (Build.VERSION.SDK_INT >= 17 && context != null) { - android.view.Display d = ((android.view.WindowManager) context.getSystemService(android.content.Context.WINDOW_SERVICE)).getDefaultDisplay(); - if (d != null) displayHz = d.getRefreshRate(); + if (Build.VERSION.SDK_INT >= 17 && activity != null) { + if(display == null) { + display = getActiveDisplay(activity); + } + if (display != null) displayHz = display.getRefreshRate(); } } catch (Throwable ignored) {} + if (displayHz <= 0f) displayHz = 60f; vsyncPeriodNs = (long) (1_000_000_000L / displayHz); diff --git a/app/src/main/java/com/limelight/preferences/StreamSettings.java b/app/src/main/java/com/limelight/preferences/StreamSettings.java index 5b2d20de13..38175463fa 100755 --- a/app/src/main/java/com/limelight/preferences/StreamSettings.java +++ b/app/src/main/java/com/limelight/preferences/StreamSettings.java @@ -76,7 +76,7 @@ public class StreamSettings extends AppCompatActivity { void reloadSettings() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display.Mode mode = getActiveDisplay(StreamSettings.this, previousPrefs).getMode(); + Display.Mode mode = getActiveDisplay(StreamSettings.this).getMode(); previousDisplayPixelCount = mode.getPhysicalWidth() * mode.getPhysicalHeight(); } prefsFragment = new SettingsFragment(PreferenceConfiguration.readPreferences( @@ -125,7 +125,7 @@ public void onConfigurationChanged(Configuration newConfig) { super.onConfigurationChanged(newConfig); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display.Mode mode = getActiveDisplay(StreamSettings.this, previousPrefs).getMode(); + Display.Mode mode = getActiveDisplay(StreamSettings.this).getMode(); // If the display's physical pixel count has changed, we consider that it's a new display // and we should reload our settings (which include display-dependent values). @@ -439,7 +439,7 @@ else if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O || } } } - + // Check custom refresh rate String customRefreshRateStr = prevPrefConfig.customRefreshRate; if (customRefreshRateStr != null && !customRefreshRateStr.isEmpty()) { @@ -904,7 +904,7 @@ public boolean onPreferenceClick(@NonNull Preference preference) { try { int width = Integer.parseInt(resolutionSegments[0]); int height = Integer.parseInt(resolutionSegments[1]); - + if (width <= 0 || height <= 0) { Toast.makeText(getActivity(), getString(R.string.pref_error_occurred), Toast.LENGTH_SHORT).show(); return false; @@ -934,14 +934,14 @@ public boolean onPreferenceClick(@NonNull Preference preference) { Toast.makeText(getActivity(), getString(R.string.pref_enter_value_0_9999), Toast.LENGTH_SHORT).show(); return false; } - + try { float refreshRate = Float.parseFloat(value); if (refreshRate <= 0) { Toast.makeText(getActivity(), getString(R.string.pref_enter_value_0_9999), Toast.LENGTH_SHORT).show(); return false; } - + // Format to max 3 decimal places String formattedValue = String.format("%.3f", refreshRate); // Remove trailing zeros diff --git a/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java b/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java index 9e21784616..05f25edbe8 100644 --- a/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java +++ b/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java @@ -13,7 +13,7 @@ public class ReflectivePaddingInt8Minimal { */ public static void applyReflectedPadding(ByteBuffer buffer) { final int size = 256; - final int band = (int)(size * 0.15); // obere/untere 20% + final int band = (int)(size * 0.1); // obere/untere 20% final int featherPx = 12; final int blurKsize = 7; diff --git a/app/src/main/java/com/limelight/utils/ServerHelper.java b/app/src/main/java/com/limelight/utils/ServerHelper.java index 2f474fb7ee..0e308986e5 100755 --- a/app/src/main/java/com/limelight/utils/ServerHelper.java +++ b/app/src/main/java/com/limelight/utils/ServerHelper.java @@ -61,8 +61,9 @@ public static Intent createAppShortcutIntent(Activity parent, ComputerDetails co i.setAction(Intent.ACTION_DEFAULT); return i; } - public static Display getActiveDisplay(Context context, PreferenceConfiguration prefs) { + public static Display getActiveDisplay(Context context) { Display secondary = getSecondaryDisplay(context); + PreferenceConfiguration prefs = PreferenceConfiguration.readPreferences(context); if (secondary != null && (prefs.enableFullExDisplay)) { return secondary; } else { diff --git a/app/src/main/java/com/limelight/utils/ShaderUtils.java b/app/src/main/java/com/limelight/utils/ShaderUtils.java index ffb115d1b9..f0ea8dbef0 100644 --- a/app/src/main/java/com/limelight/utils/ShaderUtils.java +++ b/app/src/main/java/com/limelight/utils/ShaderUtils.java @@ -22,31 +22,38 @@ public class ShaderUtils { "uniform bool u_debugMode;\n" + "\n" + "void main() {\n" + - " float depth = texture2D(s_DepthTexture, v_TexCoord).r;\n" + + " float depth = texture2D(s_DepthTexture, v_TexCoord - vec2(abs(u_parallax / 2.0), -0.007)).r;\n" + "\n" + - " // Remap depth into symmetric range around convergence\n" + - " float depthDiff;\n" + - " if (depth < u_convergence) {\n" + - " depthDiff = (depth - u_convergence) / u_convergence; // [-1,0]\n" + - " } else {\n" + - " depthDiff = (depth - u_convergence) / (1.0 - u_convergence); // [0,1]\n" + - " }\n" + + " const float zone_radius = 0.70; // Breite der neutralen Zone um Konvergenz\n" + + "\n" + + " float depthDiff = depth - u_convergence;\n" + + " // clamp to ensure total width = 1 unit, sliding around convergence\n" + + " depthDiff = clamp(depthDiff, -u_convergence, 1.0 - u_convergence);\n" + + "\n" + + " float dist_from_convergence = abs(depth - u_convergence);\n" + + " float fade_multiplier = smoothstep(0.0, zone_radius, dist_from_convergence);\n" + "\n" + " float parallax_magnitude = abs(u_parallax);\n" + " float ai_shift = parallax_magnitude * depthDiff;\n" + "\n" + - " // --- Dynamische Vignette ---\n" + + " // --- Dynamische Vignette, an Konvergenz angepasst ---\n" + " float edgeWidth = 0.01;\n" + " float depthLeft = texture2D(s_DepthTexture, vec2(edgeWidth, 0.5)).r;\n" + " float depthRight = texture2D(s_DepthTexture, vec2(1.0 - edgeWidth, 0.5)).r;\n" + - " float ai_shift_left = u_parallax * (depthLeft - 0.5);\n" + - " float ai_shift_right = u_parallax * (depthRight - 0.5);\n" + + "\n" + + " // Korrektur: Bezug auf u_convergence statt 0.5\n" + + " float ai_shift_left = u_parallax * (depthLeft - u_convergence);\n" + + " float ai_shift_right = u_parallax * (depthRight - u_convergence);\n" + " float maxEdgeShift = max(abs(ai_shift_left), abs(ai_shift_right));\n" + + "\n" + " bool isLeftEye = (u_parallax < 0.0);\n" + - " float isLeftEyeIndicator = isLeftEye ? -1.0 : 1.0;\n" + - " float vignette_start = mix(0.7, 1.0, clamp(maxEdgeShift / 0.5, 0.0, 1.0));\n" + - " const float vignette_end = 1.0;\n" + + " float isLeftEyeIndicator = isLeftEye ? 1.0 : -1.0;\n" + "\n" + + " // Vignette leicht an Konvergenz koppeln\n" + + " float vignette_bias = (u_convergence - 0.5) * 0.3; // ±0.15 Anpassung\n" + + " float vignette_start = mix(0.7 - vignette_bias, 1.0 - vignette_bias,\n" + + " clamp(maxEdgeShift / 0.5, 0.0, 1.0));\n" + + " const float vignette_end = 1.0;\n" + " if ((depth - u_convergence) < 0.0) {\n" + " ai_shift *= isLeftEye ? u_shift : (1.0-u_shift);\n" + " } else {\n" + @@ -66,7 +73,6 @@ public class ShaderUtils { " float artifactBlendFactor = (1.0 - smoothstep(0.1, 1.0, shiftMagnitude)) * 0.005;\n" + " vec4 finalColor = mix(shiftedColor, originalColor, artifactBlendFactor);\n" + "\n" + - " // ---------------- Rot/Blau Debug -----------------\n" + " if (u_debugMode) {\n" + " float debugDepth = final_shift;\n" + " vec3 debugTint = vec3(0.0);\n" + @@ -82,6 +88,7 @@ public class ShaderUtils { + /** * An optimized, single-pass Gaussian blur shader that works as a drop-in replacement. * It achieves better performance by taking fewer texture samples over the same blur radius. @@ -96,9 +103,12 @@ public class ShaderUtils { "uniform float u_parallax;\n" + "void main() {\n" + - "float blurRadius = 60.0 * u_parallax;\n" + - "float blurStep = 2.0 / u_parallax;\n" + - "float sigma = 50.0 * u_parallax;\n" + + " float minRadius = 10.0;\n" + + " float minSigma = 5.0;\n" + + "\n" + + " float blurRadius = max(minRadius, 60.0 * u_parallax);\n" + + " float sigma = max(minSigma, 50.0 * u_parallax);\n" + + " float blurStep = 1.0; // immer in 1-Pixel-Schritten\n" + " vec4 sum = vec4(0.0);\n" + " float weightSum = 0.0;\n" + @@ -113,84 +123,6 @@ public class ShaderUtils { " gl_FragColor = sum / weightSum;\n" + "}\n"; - public static final String SIMPLE_VERTEX_SHADER = - "attribute vec4 a_Position;\n" + - "attribute vec2 a_TexCoord;\n" + - "varying vec2 v_TexCoord;\n" + - "void main() {\n" + - " gl_Position = a_Position;\n" + - " v_TexCoord = a_TexCoord;\n" + - "}\n"; - - public static final String EDGE_AWARE_VERTEX_SHADER = - "attribute vec4 a_Position;\n" + - "attribute vec2 a_TexCoord;\n" + - "varying vec2 v_TexCoord;\n" + - "void main(){\n" + - " v_TexCoord = a_TexCoord;\n" + - " gl_Position = a_Position;\n" + - "}\n"; - - public static final String EDGE_AWARE_DEPTH_BLUR_SHADER = - "precision mediump float;\n" + - "varying vec2 v_TexCoord;\n" + - "uniform sampler2D uDepthMap;\n" + - "uniform vec2 u_texelSize;\n" + - "uniform bool u_debugMode;\n" + - "uniform float u_parallax;\n" + - "\n" + - "void main(){\n" + - "float parallaxFactor = clamp(u_parallax, 0.0, 1.0);\n" + - "int radius = int(30.0 * parallaxFactor);\n"+ - "float sharpness = 1.0 * (1.1 - parallaxFactor); \n" + - "float holeThreshold = 20.0 * parallaxFactor; \n" + - " float centerDepth = texture2D(uDepthMap, v_TexCoord).r;\n" + - " vec4 sum = vec4(0.0);\n" + - " float weightSum = 0.0;\n" + - "\n" + - " // ---------------- Horizontal Blur ----------------\n" + - " for(int i=-radius;i<=radius;i++){\n" + - " vec2 offsetUV = v_TexCoord + vec2(float(i)*u_texelSize.x,0.0);\n" + - " float sampleDepth = texture2D(uDepthMap, offsetUV).r;\n" + - " float w = exp(-abs(sampleDepth-centerDepth)*sharpness);\n" + - " sum += texture2D(uDepthMap, offsetUV) * w;\n" + - " weightSum += w;\n" + - " }\n" + - " vec4 blurred = sum/weightSum;\n" + - "\n" + - " // ---------------- Hole Filling ----------------\n" + - " if(abs(blurred.r - centerDepth) > holeThreshold){\n" + - " vec4 fill = vec4(0.0);\n" + - " float fillWeight = 0.0;\n" + - " vec2 offs[4];\n" + - " offs[0] = vec2(u_texelSize.x,0.0);\n" + - " offs[1] = vec2(-u_texelSize.x,0.0);\n" + - " offs[2] = vec2(0.0,u_texelSize.y);\n" + - " offs[3] = vec2(0.0,-u_texelSize.y);\n" + - " for(int k=0;k<4;k++){\n" + - " float d = texture2D(uDepthMap, v_TexCoord+offs[k]).r;\n" + - " if(abs(d-centerDepth)0.0){ blurred = mix(blurred, fill/fillWeight, 0.7); }\n" + - " }\n" + - "\n" + - " // ---------------- Debug Rot/Blau ----------------\n" + - " if(u_debugMode){\n" + - " vec3 debugTint = vec3(0.0);\n" + - " float diff = blurred.r - centerDepth;\n" + - " if(diff > 0.0) debugTint.r = diff*50.0;\n" + - " else debugTint.b = -diff*50.0;\n" + - " blurred.rgb += debugTint;\n" + - " }\n" + - "\n" + - " gl_FragColor = blurred;\n" + - "}\n"; - - - public static final String SIMPLE_FRAGMENT_SHADER = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index b815b48153..0024278874 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -6,7 +6,6 @@ import android.opengl.GLES20; import android.opengl.GLES30; import android.opengl.GLSurfaceView; -import android.os.Build; import android.util.Log; import android.view.Surface; @@ -33,8 +32,12 @@ import java.nio.FloatBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -122,11 +125,10 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private BlockingQueue freeSmoothedBuffers; private BlockingQueue inferenceInputQueue = new ArrayBlockingQueue<>(1); private PreferenceConfiguration prefConfig; - private ByteBuffer previousPixelBuffer; private Surface videoSurface; private SurfaceTexture videoSurfaceTexture; - private float ON_DRAW_CHANGE_TRESHOLD = 2.0f; + private float ON_DRAW_CHANGE_TRESHOLD = 2.5f; public interface OnSurfaceReadyListener { @@ -210,7 +212,6 @@ public void onSurfaceDestroyed() { }); if (filledOutputBuffers != null) filledOutputBuffers.clear(); - previousPixelBuffer = null; currentlyRenderingMap = null; prefConfig = null; drawDelay = 0.0f; @@ -241,7 +242,7 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { depthMapTextureId = createEmptyTexture(modelInputWidth, modelInputHeight); - simple3dProgram = createProgram(ShaderUtils.SIMPLE_VERTEX_SHADER, ShaderUtils.SIMPLE_FRAGMENT_SHADER); + simple3dProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.SIMPLE_FRAGMENT_SHADER); bilateralBlurProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.OPTIMIZED_SINGLE_PASS_GAUSSIAN_BLUR_SHADER); dibr3dProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.FRAGMENT_SHADER_3D); @@ -259,7 +260,6 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { } int pboSize = modelInputWidth * modelInputHeight * 4; - previousPixelBuffer = ByteBuffer.allocateDirect(pboSize).order(ByteOrder.nativeOrder()); previousFrameForComparison = ByteBuffer.allocateDirect(pboSize).order(ByteOrder.nativeOrder()); int inputPixelSize = modelInputWidth * modelInputHeight * 4; freeInputBuffers = new ArrayBlockingQueue<>(NUM_INPUT_BUFFERS); @@ -298,7 +298,7 @@ private void initializeIntermediateFbo() { } private float getParallax() { - return prefConfig.parallax_depth * 0.7f; + return prefConfig.parallax_depth * 0.2f; } private void applyTwoPassGaussianBlur() { @@ -349,7 +349,7 @@ private void drawBothEyes(int dualBubble3dProgram, float convergence, float shif int viewWidth = glSurfaceView.getWidth(); int viewHeight = glSurfaceView.getHeight(); - float parallax = getParallax() * 0.06f; + float parallax = getParallax() * 0.2f; GLES20.glViewport(0, 0, viewWidth / 2, viewHeight); drawEye(dualBubble3dProgram, -parallax, convergence, shift); @@ -602,34 +602,6 @@ public static void convertRgbaToRgb(ByteBuffer rgbaBuffer, ByteBuffer rgbBuffer, } } - public static double calculateAverageDifferenceOCV(ByteBuffer buffer1, ByteBuffer buffer2, int width, int height) { - if (buffer1 == null || buffer2 == null) { - return 1; - } - - Mat mat1 = null; - Mat mat2 = null; - Mat diffMat = null; - try { - mat1 = new Mat(height, width, CvType.CV_8UC1, buffer1); - mat2 = new Mat(height, width, CvType.CV_8UC1, buffer2); - diffMat = new Mat(); - Core.absdiff(mat1, mat2, diffMat); - Scalar meanDifference = Core.mean(diffMat); - return meanDifference.val[0] / 255.0; - } finally { - if (mat1 != null) { - mat1.release(); - } - if (mat2 != null) { - mat2.release(); - } - if (diffMat != null) { - diffMat.release(); - } - } - } - private void initializePBOs() { PBO_SIZE = modelInputWidth * modelInputHeight * 4; @@ -657,35 +629,6 @@ private Boolean readPixelsForAI(ByteBuffer destinationBuffer) { return true; } - private boolean readPixelsForAI_Async(ByteBuffer destinationBuffer) { - GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboHandle); - GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - drawQuad(simple3dProgram, 1.0f, 0.0f); - int writeIndex = pboIndex; - int readIndex = (pboIndex + 1) % 2; - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[writeIndex]); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - GLES30.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES30.GL_RGBA, GLES30.GL_UNSIGNED_BYTE, 0); - } - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[readIndex]); - ByteBuffer mappedBuffer = (ByteBuffer) GLES30.glMapBufferRange( - GLES30.GL_PIXEL_PACK_BUFFER, 0, PBO_SIZE, GLES30.GL_MAP_READ_BIT); - boolean success = false; - if (mappedBuffer != null) { - destinationBuffer.rewind(); - mappedBuffer.rewind(); - destinationBuffer.put(mappedBuffer); - GLES30.glUnmapBuffer(GLES30.GL_PIXEL_PACK_BUFFER); - success = true; - } - - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, 0); - GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); - - pboIndex = readIndex; - return success; - } - private void initializeTfLite() { Interpreter.Options options = new Interpreter.Options(); @@ -859,6 +802,104 @@ private int createProgram(String vertex, String fragment) { return program; } + private double computeColorSimilarity(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { + if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { + return 0.0; // komplett unterschiedlich + } + + // ByteBuffers zurücksetzen + newPixelBuffer.rewind(); + oldPixelBuffer.rewind(); + + Mat mat1 = null, mat2 = null; + Mat matBGR1 = null, matBGR2 = null; + Mat histB1 = null, histG1 = null, histR1 = null; + Mat histB2 = null, histG2 = null, histR2 = null; + + List bgrPlanes1 = null; + List bgrPlanes2 = null; + try { + // Mats aus ByteBuffer + mat1 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, newPixelBuffer); + mat2 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, oldPixelBuffer); + + // RGBA -> BGR konvertieren + matBGR1 = new Mat(); + matBGR2 = new Mat(); + Imgproc.cvtColor(mat1, matBGR1, Imgproc.COLOR_RGBA2BGR); + Imgproc.cvtColor(mat2, matBGR2, Imgproc.COLOR_RGBA2BGR); + + // Core.split vorbereiten + bgrPlanes1 = new ArrayList<>(); + bgrPlanes1.add(new Mat()); + bgrPlanes1.add(new Mat()); + bgrPlanes1.add(new Mat()); + Core.split(matBGR1, bgrPlanes1); + + bgrPlanes2 = new ArrayList<>(); + bgrPlanes2.add(new Mat()); + bgrPlanes2.add(new Mat()); + bgrPlanes2.add(new Mat()); + Core.split(matBGR2, bgrPlanes2); + + // Histogramme + histB1 = new Mat(); + histG1 = new Mat(); + histR1 = new Mat(); + histB2 = new Mat(); + histG2 = new Mat(); + histR2 = new Mat(); + + int histSize = 16; // grobe Farbanalyse + float[] range = {0f, 256f}; + MatOfFloat histRange = new MatOfFloat(range); + + // Histogramme berechnen + Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(0)), new MatOfInt(0), new Mat(), histB1, new MatOfInt(histSize), histRange); + Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(1)), new MatOfInt(0), new Mat(), histG1, new MatOfInt(histSize), histRange); + Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(2)), new MatOfInt(0), new Mat(), histR1, new MatOfInt(histSize), histRange); + + Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(0)), new MatOfInt(0), new Mat(), histB2, new MatOfInt(histSize), histRange); + Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(1)), new MatOfInt(0), new Mat(), histG2, new MatOfInt(histSize), histRange); + Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(2)), new MatOfInt(0), new Mat(), histR2, new MatOfInt(histSize), histRange); + + // Normalisieren + Core.normalize(histB1, histB1, 0, 1, Core.NORM_MINMAX); + Core.normalize(histG1, histG1, 0, 1, Core.NORM_MINMAX); + Core.normalize(histR1, histR1, 0, 1, Core.NORM_MINMAX); + + Core.normalize(histB2, histB2, 0, 1, Core.NORM_MINMAX); + Core.normalize(histG2, histG2, 0, 1, Core.NORM_MINMAX); + Core.normalize(histR2, histR2, 0, 1, Core.NORM_MINMAX); + + // Histogramm-Korrelation pro Kanal + double corrB = Imgproc.compareHist(histB1, histB2, Imgproc.HISTCMP_CORREL); + double corrG = Imgproc.compareHist(histG1, histG2, Imgproc.HISTCMP_CORREL); + double corrR = Imgproc.compareHist(histR1, histR2, Imgproc.HISTCMP_CORREL); + + return 1f - Math.max(0, (corrB + corrG + corrR) / 3.0); + + } finally { + // Alle Mats freigeben + if (mat1 != null) mat1.release(); + if (mat2 != null) mat2.release(); + if (matBGR1 != null) matBGR1.release(); + if (matBGR2 != null) matBGR2.release(); + if (histB1 != null) histB1.release(); + if (histG1 != null) histG1.release(); + if (histR1 != null) histR1.release(); + if (histB2 != null) histB2.release(); + if (histG2 != null) histG2.release(); + if (histR2 != null) histR2.release(); + for (Mat m : new Mat[]{bgrPlanes1.get(0), bgrPlanes1.get(1), bgrPlanes1.get(2)}) + m.release(); + for (Mat m : new Mat[]{bgrPlanes2.get(0), bgrPlanes2.get(1), bgrPlanes2.get(2)}) + m.release(); + } + } + + + private double hasFrameChangedSignificantlyOCV(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { return 1.0; // maximal unterschiedliche Frames @@ -1034,7 +1075,7 @@ public void run() { if (pixelBuffer != null) { freeInputBuffers.offer(pixelBuffer); } - Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + " ms " + filledOutputBuffers.remainingCapacity() + " " + waitTimeText + " ms" + "aitime: " + aitimeText); + Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + " ms " + filledOutputBuffers.remainingCapacity() + " " + waitTimeText + " ms " + "aitime: " + aitimeText); } } isAiRunning.set(false); @@ -1043,13 +1084,11 @@ public void run() { private class AiResultHandling implements Runnable { - private static final double IMAGE_DIFFERENCE_MULTIPLIER = 100.0; - private static final double MAX_SMOOTHING_FACTOR = 1; - - private static final double MIN_SMOOTHING_FACTOR = 0.005; + private static final double DEPTH_DIFF_THRESHOLD = 0.1; // large jump threshold + private static final double DEPTH_DIFF_AVERAGE_THRESHOLD = 0.05; // large jump threshold + private static final double MAX_SMOOTHING = 1.0; // full adoption + private static final double MIN_SMOOTHING = 0.0; // ignore - // --- Member Fields --- - private final byte[] processedDataArray = new byte[modelInputWidth * modelInputHeight]; private Mat previousSmoothedMat; private boolean isFirstFrame = true; @@ -1060,90 +1099,100 @@ public void run() { while (!Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); - long waitTime = System.nanoTime(); Mat rawMat = null; - Mat processedMat = null; + Mat rawFloat = null; try { result = filledOutputBuffers.take(); resultBuffer = freeSmoothedBuffers.take(); - waitTime = System.nanoTime(); + // Take latest intermediate frame if multiple available InferenceResult intermediate; while ((intermediate = filledOutputBuffers.poll()) != null) { freeInputBuffers.offer(result.pixelBuffer); freeOutputBuffers.offer(result.rawDepthBuffer); result = intermediate; } - ByteBuffer rawDepthBuffer = result.rawDepthBuffer; - ByteBuffer currentPixelBuffer = result.pixelBuffer; - - currentPixelBuffer.rewind(); - double imageDifference = hasFrameChangedSignificantlyOCV(currentPixelBuffer, previousPixelBuffer) * IMAGE_DIFFERENCE_MULTIPLIER; - rawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, rawDepthBuffer); - processedMat = new Mat(); - Core.normalize(rawMat, processedMat, 0, 255, Core.NORM_MINMAX); + rawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, result.rawDepthBuffer); + rawFloat = new Mat(); + rawMat.convertTo(rawFloat, CvType.CV_32F); // keep un-normalized if (isFirstFrame) { - previousSmoothedMat = processedMat.clone(); + previousSmoothedMat = rawFloat.clone(); isFirstFrame = false; } - double smoothing = (imageDifference * 10) / (threeDFps * 3); - smoothing = Math.min(smoothing, MAX_SMOOTHING_FACTOR); - smoothing = Math.max(smoothing, MIN_SMOOTHING_FACTOR); - Mat diff = new Mat(); - Core.absdiff(processedMat, previousSmoothedMat, diff); - Core.MinMaxLocResult mmr = Core.minMaxLoc(diff); - double thresholdValue = Math.max(1, mmr.maxVal * ((1.0 - smoothing)) * 0.1); - Mat validMask = new Mat(); - Imgproc.threshold(diff, validMask, thresholdValue, 255, Imgproc.THRESH_BINARY_INV); - processedMat.copyTo(previousSmoothedMat, validMask); - Mat blended = new Mat(); - Core.addWeighted(processedMat, smoothing, previousSmoothedMat, 1.0 - smoothing, 0.0, blended); - Mat inverseMask = new Mat(); - Core.bitwise_not(validMask, inverseMask); - blended.copyTo(previousSmoothedMat, inverseMask); - diff.release(); - validMask.release(); - inverseMask.release(); - blended.release(); - previousSmoothedMat.get(0, 0, processedDataArray); - rawDepthBuffer.rewind(); - rawDepthBuffer.put(processedDataArray); - - rawDepthBuffer.rewind(); - resultBuffer.rewind(); - resultBuffer.put(rawDepthBuffer); - - rawDepthBuffer.rewind(); - resultBuffer.rewind(); + // --- Calculate robust depth difference --- + Mat diffMat = new Mat(); + Core.absdiff(rawFloat, previousSmoothedMat, diffMat); + + // Mean difference (global) + Scalar sumDiff = Core.sumElems(diffMat); + double meanDiff = sumDiff.val[0] / (diffMat.rows() * diffMat.cols() * 255.0); + + // Standard deviation (local fluctuations) + Mat diffFloat = new Mat(); + diffMat.convertTo(diffFloat, CvType.CV_32F, 1.0 / 255.0); + Scalar meanVal = Core.mean(diffFloat); + Mat meanMat = new Mat(diffFloat.size(), CvType.CV_32F, new Scalar(meanVal.val[0])); + Mat varianceMat = new Mat(); + Core.subtract(diffFloat, meanMat, varianceMat); + Core.multiply(varianceMat, varianceMat, varianceMat); + double stdDev = Math.sqrt(Core.sumElems(varianceMat).val[0] / (diffFloat.rows() * diffFloat.cols())); + + double depthMapDifference = Math.max(meanDiff, stdDev); + + // --- Determine smoothing factor --- + double smoothing; + if (depthMapDifference > DEPTH_DIFF_THRESHOLD) { + smoothing = MAX_SMOOTHING; // fully adopt large changes + } else if(depthMapDifference > 0.01){ + smoothing = depthMapDifference; // proportional to difference + smoothing = Math.max(MIN_SMOOTHING, Math.min(MAX_SMOOTHING, smoothing)); + } else { + // Prevents mini pixel shifts reducing sharpness on still images + smoothing = 0; + } + + // --- Apply smoothing --- + Imgproc.accumulateWeighted(rawFloat, previousSmoothedMat, smoothing); + + // --- Normalize for shader output --- + Mat normalizedForShader = new Mat(); + Core.MinMaxLocResult mmr = Core.minMaxLoc(previousSmoothedMat); + Core.subtract(previousSmoothedMat, new Scalar(mmr.minVal), normalizedForShader); + Core.divide(normalizedForShader, new Scalar(mmr.maxVal - mmr.minVal + 1e-6), normalizedForShader); + + Mat outputMat = new Mat(); + normalizedForShader.convertTo(outputMat, CvType.CV_8U, 255.0); + outputMat.get(0, 0, resultBuffer.array()); + latestDepthMap.set(resultBuffer); - previousPixelBuffer.rewind(); - previousPixelBuffer.put(currentPixelBuffer); + // --- Cleanup --- + diffMat.release(); + diffFloat.release(); + meanMat.release(); + varianceMat.release(); + normalizedForShader.release(); + outputMat.release(); + } catch (Exception e) { LimeLog.severe("AI exception " + e.getMessage()); } finally { - if (rawMat != null) { - rawMat.release(); - } - if (processedMat != null) { - processedMat.release(); - } - if (resultBuffer != null) { - freeSmoothedBuffers.offer(resultBuffer); - } + if (rawMat != null) rawMat.release(); + if (rawFloat != null) rawFloat.release(); + if (resultBuffer != null) freeSmoothedBuffers.offer(resultBuffer); if (result != null) { freeInputBuffers.offer(result.pixelBuffer); freeOutputBuffers.offer(result.rawDepthBuffer); } long duration = (System.nanoTime() - startTime) / 1_000_000; - long waitTimeText = (waitTime - startTime) / 1_000_000; - Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + " ms" + " " + freeOutputBuffers.remainingCapacity() + " " + waitTimeText + " ms "); + Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + " ms"); } } isAiResultHandlingRunning.set(false); } } + } \ No newline at end of file diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 5b009c891f..8d8c713a2c 100755 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -58,12 +58,12 @@ android:summary="@string/summary_frame_pacing" android:title="@string/title_frame_pacing" app:iconSpaceReserved="false" /> - + - + - + Date: Wed, 15 Oct 2025 21:32:24 +0200 Subject: [PATCH 02/22] Improvements 3d --- app/src/main/java/com/limelight/Game.java | 2 -- app/src/main/java/com/limelight/GameMenu.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index a413577c51..4b13378b2e 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -932,7 +932,6 @@ public void notifyCrash(Exception e) { try { java.lang.reflect.Method m = SurfaceView.class.getMethod("setFrameRate", float.class, int.class); - LimeLog.info("MYLOG TARGETTEST " + targetFps + " " + displayHz); m.invoke(streamSurfaceView, Math.min(targetFps, displayHz), compat); } catch (Throwable ignored) {} } @@ -1629,7 +1628,6 @@ else if (!isRefreshRateGoodMatch(candidate.getRefreshRate())) { LimeLog.info("surfaceChanged-->"+(double)displayWidth / (double)displayHeight); LimeLog.info("scaleMode-->"+prefConfig.videoScaleMode); } - // streamContainer.setAsIs(true); // Set the desired refresh rate that will get passed into setFrameRate() later desiredRefreshRate = displayRefreshRate; diff --git a/app/src/main/java/com/limelight/GameMenu.java b/app/src/main/java/com/limelight/GameMenu.java index 7d09789018..63f3532206 100755 --- a/app/src/main/java/com/limelight/GameMenu.java +++ b/app/src/main/java/com/limelight/GameMenu.java @@ -19,7 +19,6 @@ import com.limelight.preferences.PreferenceConfiguration; import com.limelight.utils.KeyConfigHelper; import com.limelight.utils.KeyMapper; -import com.limelight.utils.Stereo3DRenderer; import java.lang.reflect.Field; import java.util.ArrayList; @@ -107,8 +106,6 @@ private void run(MenuOption option) { } private void showMenuDialog(String title, MenuOption[] options) { - - Stereo3DRenderer.isDebugMode = !Stereo3DRenderer.isDebugMode; int themeResId = game.getApplicationInfo().theme; Context themedContext = new ContextThemeWrapper(dialogScreenContext, themeResId); From 8ac4647000a2c7fbe765e33e4420ea2e8196675c Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 17 Oct 2025 22:15:59 +0200 Subject: [PATCH 03/22] Improvements 3d - dilute --- .../video/MediaCodecDecoderRenderer.java | 4 +- .../java/com/limelight/utils/ShaderUtils.java | 34 ++++- .../com/limelight/utils/Stereo3DRenderer.java | 143 ++++++++++++++++-- app/src/main/res/values-ru/strings.xml | 12 -- 4 files changed, 163 insertions(+), 30 deletions(-) diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 3b8ef1bb1d..a52ef463d6 100755 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -603,7 +603,7 @@ else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { } } - LimeLog.info("Configuring with format: "+format + " " + (renderTarget != null && renderTarget instanceof Surface)); + LimeLog.info("Configuring with format: "+format); videoDecoder.configure(format, renderTarget, null, 0); @@ -795,7 +795,7 @@ public void onFrameRendered(MediaCodec mediaCodec, long presentationTimeUs, long @Override public int setup(int format, int width, int height, int redrawRate) { - // External displayes return a redrawRate of zero, so default 60 was wrong. + // External displayes occasionally return a redrawRate of zero, so default 60 was wrong. int fpsTarget = redrawRate; if(display == null && fpsTarget <= 0) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { diff --git a/app/src/main/java/com/limelight/utils/ShaderUtils.java b/app/src/main/java/com/limelight/utils/ShaderUtils.java index f0ea8dbef0..d0fa711f08 100644 --- a/app/src/main/java/com/limelight/utils/ShaderUtils.java +++ b/app/src/main/java/com/limelight/utils/ShaderUtils.java @@ -22,7 +22,10 @@ public class ShaderUtils { "uniform bool u_debugMode;\n" + "\n" + "void main() {\n" + - " float depth = texture2D(s_DepthTexture, v_TexCoord - vec2(abs(u_parallax / 2.0), -0.007)).r;\n" + + " vec2 depthTexCoord = vec2(v_TexCoord.x, 1.0 - v_TexCoord.y);\n" + + " // Wende deinen bestehenden Offset auf die korrigierte Koordinate an.\n" + + " depthTexCoord -= vec2(abs(u_parallax / 2.0), -0.007);\n" + + " float depth = texture2D(s_DepthTexture, depthTexCoord).r;\n" + "\n" + " const float zone_radius = 0.70; // Breite der neutralen Zone um Konvergenz\n" + "\n" + @@ -84,11 +87,30 @@ public class ShaderUtils { " gl_FragColor = finalColor;\n" + "}\n"; - - - - - + public static final String FRAGMENT_SHADER_SEPARABLE_DILATE = + "precision mediump float;\n" + + "varying vec2 v_TexCoord;\n" + + "uniform sampler2D s_InputTexture;\n" + + "uniform vec2 u_texelSize;\n" + + "uniform int u_radius;\n" + + // NEU: Die Richtung (z.B. (1.0, 0.0) für horizontal) + "uniform vec2 u_direction;\n" + + "\n" + + "void main() {\n" + + " if (u_radius <= 0) {\n" + + " gl_FragColor = texture2D(s_InputTexture, v_TexCoord);\n" + + " return;\n" + + " }\n" + + "\n" + + " float maxDepth = texture2D(s_InputTexture, v_TexCoord).r;\n" + + "\n" + + " // Loop in one direction only\n" + + " for (int i = -u_radius; i <= u_radius; i++) {\n" + + " vec2 offset = u_direction * float(i) * u_texelSize;\n" + + " maxDepth = max(maxDepth, texture2D(s_InputTexture, v_TexCoord + offset).r);\n" + + " }\n" + + " gl_FragColor = vec4(vec3(maxDepth), 1.0);\n" + + "}\n"; /** * An optimized, single-pass Gaussian blur shader that works as a drop-in replacement. * It achieves better performance by taking fewer texture samples over the same blur radius. diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index 0024278874..4c13bcb724 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -1,5 +1,8 @@ package com.limelight.utils; +import static com.limelight.utils.ShaderUtils.FRAGMENT_SHADER_SEPARABLE_DILATE; +import static com.limelight.utils.ShaderUtils.VERTEX_SHADER; + import android.content.Context; import android.content.res.AssetFileDescriptor; import android.graphics.SurfaceTexture; @@ -55,6 +58,7 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65; private static final float[] QUAD_VERTICES = {-1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f}; private static final float[] TEXTURE_VERTICES = {0.0f, 1.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 0.0f}; + private final String AI_MODEL = "midas-midas-v2-w8a8.tflite"; private final int modelInputHeight = 256; private final int modelInputWidth = 256; @@ -62,7 +66,11 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private final int NUM_INPUT_BUFFERS = 10; private final int NUM_SMOOTHED_BUFFERS = 3; private final int[] pboHandles = new int[2]; - private int pboIndex = 0; + + private int mDilationProgram; + // Deine bestehenden Member-Variablen + private int intermediateDilutionFboHandle; + private int intermediateDilutionTextureId; public static boolean isMovieMode = true; private int PBO_SIZE = modelInputWidth * modelInputHeight * 4; @@ -242,12 +250,17 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { depthMapTextureId = createEmptyTexture(modelInputWidth, modelInputHeight); - simple3dProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.SIMPLE_FRAGMENT_SHADER); - bilateralBlurProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.OPTIMIZED_SINGLE_PASS_GAUSSIAN_BLUR_SHADER); - dibr3dProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.FRAGMENT_SHADER_3D); + simple3dProgram = createProgram(VERTEX_SHADER, ShaderUtils.SIMPLE_FRAGMENT_SHADER); + bilateralBlurProgram = createProgram(VERTEX_SHADER, ShaderUtils.OPTIMIZED_SINGLE_PASS_GAUSSIAN_BLUR_SHADER); + dibr3dProgram = createProgram(VERTEX_SHADER, ShaderUtils.FRAGMENT_SHADER_3D); + mDilationProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER_SEPARABLE_DILATE); + if (mDilationProgram == 0) { + throw new RuntimeException("Konnte Dilation-Shader-Programm nicht erstellen."); + } initializeFilterFbo(); initializeIntermediateFbo(); + initializeDilationFbo(); initializeTfLite(); initializeFbo(); initBuffer(); @@ -297,10 +310,95 @@ private void initializeIntermediateFbo() { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } + private void initializeDilationFbo() { + // Create the texture to store the dilation result + intermediateDilutionTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); + + // Create the framebuffer object (FBO) + int[] fbos = new int[1]; + GLES20.glGenFramebuffers(1, fbos, 0); + intermediateDilutionFboHandle = fbos[0]; + + // Bind the FBO and attach the texture to it + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId, 0); + + // Check if the FBO was created successfully + if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE) { + LimeLog.warning("Dilation Framebuffer is not complete."); + } + + // Unbind the FBO to restore the default state + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + private float getParallax() { return prefConfig.parallax_depth * 0.2f; } + /** + * Wendet einen performanten, zweistufigen Dilation-Filter an. + * Dieses Verfahren ist bei großen Radien deutlich schneller als ein einstufiger Filter. + * Liest von 'depthMapTextureId', schreibt das Zwischenergebnis nach 'intermediateDilutionFboHandle' + * und das Endergebnis nach 'intermediateFboHandle'. + */ + private void applyTwoPassDilation() { + // Das NEUE, separable Dilation-Shader-Programm aktivieren + GLES20.glUseProgram(mDilationProgram); // Stelle sicher, dass du diese Variable hast + + // Handles für Attribute und Uniforms holen (sollten als Member-Variablen gecached sein) + int posHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_Position"); + int texHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_TexCoord"); + int inputTextureHandle = GLES20.glGetUniformLocation(mDilationProgram, "s_InputTexture"); + int texelSizeHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_texelSize"); + int radiusHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_radius"); + int directionHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_direction"); + + // Vertex-Daten verbinden (mit den KORREKTEN, nicht-gespiegelten Koordinaten) + GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glEnableVertexAttribArray(posHandle); + GLES20.glEnableVertexAttribArray(texHandle); + + // --- 1. DURCHGANG: HORIZONTAL --- + // Ziel ist der erste Zwischenspeicher (`intermediateDilutionFboHandle`). + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); + GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); + + // Input ist die rohe, originale Tiefenkarte. + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); + + // Setze alle Uniforms für den horizontalen Durchgang. + GLES20.glUniform1i(inputTextureHandle, 0); + GLES20.glUniform1i(radiusHandle, 15); // Dein gewünschter, großer Radius. + GLES20.glUniform2f(texelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); + GLES20.glUniform2f(directionHandle, 1.0f, 0.0f); // Richtung: Horizontal (X-Achse) + + // Führe den ersten Shader-Durchgang aus. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + + + // --- 2. DURCHGANG: VERTIKAL --- + // Ziel ist der zweite Zwischenspeicher (`intermediateFboHandle`), + // aus dem der Gauß-Filter später lesen wird. + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateFboHandle); + // Viewport muss nicht neu gesetzt werden, wenn die Größe gleich bleibt. + + // Input ist jetzt das Ergebnis des ersten Durchgangs. + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); + + // Die meisten Uniforms bleiben gleich, wir ändern nur die Richtung. + GLES20.glUniform2f(directionHandle, 0.0f, 1.0f); // Richtung: Vertikal (Y-Achse) + + // Führe den zweiten Shader-Durchgang aus. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + + // WICHTIG: Der Framebuffer (`intermediateFboHandle`) bleibt für den + // nachfolgenden Gauß-Filter gebunden. Er wird erst am Ende der + // gesamten Filterkette (in onDrawFrame) auf 0 zurückgesetzt. + } private void applyTwoPassGaussianBlur() { int blurProgram = bilateralBlurProgram; @@ -326,7 +424,7 @@ private void applyTwoPassGaussianBlur() { GLES20.glUniform2f(directionHandle, 1.0f, 0.0f); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); GLES20.glUniform1i(inputTextureHandle, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); @@ -484,6 +582,7 @@ public void onDrawFrame(GL10 gl) { uploadLatestDepthMapToGpu(currentlyRenderingMap); } GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + applyTwoPassDilation(); applyTwoPassGaussianBlur(); drawWithShader(); long endTime = System.nanoTime(); @@ -780,11 +879,36 @@ private int loadShader(int type, String shaderCode) { } private int createProgram(String vertex, String fragment) { - int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertex); - if (vertexShader == 0) return 0; - int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragment); - if (fragmentShader == 0) return 0; + // --- VERTEX SHADER COMPILATION --- + int vertexShader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER); + GLES20.glShaderSource(vertexShader, vertex); + GLES20.glCompileShader(vertexShader); + + // --- NEUES LOGGING HINZUGEFÜGT --- + int[] compiled = new int[1]; + GLES20.glGetShaderiv(vertexShader, GLES20.GL_COMPILE_STATUS, compiled, 0); + if (compiled[0] == 0) { + LimeLog.severe("Could not compile vertex shader:"); + LimeLog.severe(GLES20.glGetShaderInfoLog(vertexShader)); + GLES20.glDeleteShader(vertexShader); + return 0; + } + + // --- FRAGMENT SHADER COMPILATION --- + int fragmentShader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER); + GLES20.glShaderSource(fragmentShader, fragment); + GLES20.glCompileShader(fragmentShader); + + // --- NEUES LOGGING HINZUGEFÜGT --- + GLES20.glGetShaderiv(fragmentShader, GLES20.GL_COMPILE_STATUS, compiled, 0); + if (compiled[0] == 0) { + LimeLog.severe("Could not compile fragment shader:"); + LimeLog.severe(GLES20.glGetShaderInfoLog(fragmentShader)); + GLES20.glDeleteShader(fragmentShader); + return 0; + } + // --- PROGRAM LINKING (DEIN BESTEHENDER CODE) --- int program = GLES20.glCreateProgram(); if (program != 0) { GLES20.glAttachShader(program, vertexShader); @@ -801,7 +925,6 @@ private int createProgram(String vertex, String fragment) { } return program; } - private double computeColorSimilarity(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { return 0.0; // komplett unterschiedlich diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 2eb455f4f7..aef010af5a 100755 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -652,18 +652,6 @@ %1$dмс %1$.2fмс Уменьшает задержку за счёт пропуска кадров в очереди - Графики производительности - График задержки сети - График времени декодирования - График FPS - Включить графики производительности - Вывод графиков производительности в игровой панели - График: задержка сети - График средней задержки сети (мс) - График: время декодирования - График времени декодирования кадра (мс) - График: FPS - График входящих/отрендереных кадров в секунду Монитор производительности LFR (эксперементально) Агрессивная вертикальная синхронизация (эксперементально) From ca3122d910b8f842d3e71c7567d9ecd13176b4d4 Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 17 Oct 2025 22:24:48 +0200 Subject: [PATCH 04/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/utils/ShaderUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/com/limelight/utils/ShaderUtils.java b/app/src/main/java/com/limelight/utils/ShaderUtils.java index d0fa711f08..a2c0591240 100644 --- a/app/src/main/java/com/limelight/utils/ShaderUtils.java +++ b/app/src/main/java/com/limelight/utils/ShaderUtils.java @@ -24,7 +24,7 @@ public class ShaderUtils { "void main() {\n" + " vec2 depthTexCoord = vec2(v_TexCoord.x, 1.0 - v_TexCoord.y);\n" + " // Wende deinen bestehenden Offset auf die korrigierte Koordinate an.\n" + - " depthTexCoord -= vec2(abs(u_parallax / 2.0), -0.007);\n" + + " depthTexCoord -= vec2(abs(u_parallax / 2.0), 0.04);\n" + " float depth = texture2D(s_DepthTexture, depthTexCoord).r;\n" + "\n" + " const float zone_radius = 0.70; // Breite der neutralen Zone um Konvergenz\n" + From 0e957c6a86c640cf819c0c031c73af3516794721 Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 17 Oct 2025 22:39:43 +0200 Subject: [PATCH 05/22] Improvements 3d - dilute --- .../com/limelight/utils/Stereo3DRenderer.java | 438 +++++------------- 1 file changed, 119 insertions(+), 319 deletions(-) diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index 4c13bcb724..8de281365d 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -38,9 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedList; import java.util.List; -import java.util.Queue; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -58,7 +56,6 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65; private static final float[] QUAD_VERTICES = {-1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f}; private static final float[] TEXTURE_VERTICES = {0.0f, 1.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 0.0f}; - private final String AI_MODEL = "midas-midas-v2-w8a8.tflite"; private final int modelInputHeight = 256; private final int modelInputWidth = 256; @@ -67,11 +64,6 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private final int NUM_SMOOTHED_BUFFERS = 3; private final int[] pboHandles = new int[2]; - private int mDilationProgram; - // Deine bestehenden Member-Variablen - private int intermediateDilutionFboHandle; - private int intermediateDilutionTextureId; - public static boolean isMovieMode = true; private int PBO_SIZE = modelInputWidth * modelInputHeight * 4; @@ -101,19 +93,28 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private final AtomicBoolean isAiRunning = new AtomicBoolean(false); // OpenGL Handles + private int mDilationProgram; private int bilateralBlurProgram; private int depthMapTextureId; private int dibr3dProgram; - - private final AtomicReference latestDepthMap = new AtomicReference<>(null); + private int simple3dProgram; + private int videoTextureId; private int fboHandle; private int fboTextureId; private int filterFboHandle; private int filteredDepthMapTextureId; private int intermediateFboHandle; private int intermediateTextureId; - private int simple3dProgram; - private int videoTextureId; + private int intermediateDilutionFboHandle; + private int intermediateDilutionTextureId; + + // --- VORGELADENE SHADER-LOCATIONS FÜR PERFORMANCE --- + private int mDilationPosHandle, mDilationTexHandle, mDilationInputTextureHandle, + mDilationTexelSizeHandle, mDilationRadiusHandle, mDilationDirectionHandle; + private int mGaussPosHandle, mGaussTexHandle, mGaussInputTextureHandle, + mGaussTexelSizeHandle, mGaussDirectionHandle, mGaussParallaxHandle; + private int mDibrPosHandle, mDibrTexHandle, mDibrColorTexHandle, mDibrDepthTexHandle, + mDibrParallaxHandle, mDibrConvergenceHandle, mDibrShiftHandle, mDibrDebugModeHandle; // AI & TFLite Variables private GpuDelegate gpuDelegate; @@ -135,10 +136,9 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private PreferenceConfiguration prefConfig; private Surface videoSurface; private SurfaceTexture videoSurfaceTexture; - + private final AtomicReference latestDepthMap = new AtomicReference<>(null); private float ON_DRAW_CHANGE_TRESHOLD = 2.5f; - public interface OnSurfaceReadyListener { void onStereo3DSurfaceReady(Surface surface); } @@ -205,17 +205,19 @@ public void onSurfaceDestroyed() { GLES20.glDeleteProgram(simple3dProgram); GLES20.glDeleteProgram(bilateralBlurProgram); GLES20.glDeleteProgram(dibr3dProgram); + GLES20.glDeleteProgram(mDilationProgram); int[] textures = { videoTextureId, depthMapTextureId, filteredDepthMapTextureId, fboTextureId, - intermediateTextureId + intermediateTextureId, + intermediateDilutionTextureId }; GLES20.glDeleteTextures(textures.length, textures, 0); - int[] fbos = {fboHandle, intermediateFboHandle, filterFboHandle}; + int[] fbos = {fboHandle, intermediateFboHandle, filterFboHandle, intermediateDilutionFboHandle}; GLES20.glDeleteFramebuffers(fbos.length, fbos, 0); }); @@ -258,6 +260,8 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { throw new RuntimeException("Konnte Dilation-Shader-Programm nicht erstellen."); } + getShaderLocations(); + initializeFilterFbo(); initializeIntermediateFbo(); initializeDilationFbo(); @@ -297,6 +301,31 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { isActive = true; } + private void getShaderLocations() { + mDilationPosHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_Position"); + mDilationTexHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_TexCoord"); + mDilationInputTextureHandle = GLES20.glGetUniformLocation(mDilationProgram, "s_InputTexture"); + mDilationTexelSizeHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_texelSize"); + mDilationRadiusHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_radius"); + mDilationDirectionHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_direction"); + + mGaussPosHandle = GLES20.glGetAttribLocation(bilateralBlurProgram, "a_Position"); + mGaussTexHandle = GLES20.glGetAttribLocation(bilateralBlurProgram, "a_TexCoord"); + mGaussInputTextureHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "s_InputTexture"); + mGaussTexelSizeHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "u_texelSize"); + mGaussDirectionHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "u_blurDirection"); + mGaussParallaxHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "u_parallax"); + + mDibrPosHandle = GLES20.glGetAttribLocation(dibr3dProgram, "a_Position"); + mDibrTexHandle = GLES20.glGetAttribLocation(dibr3dProgram, "a_TexCoord"); + mDibrColorTexHandle = GLES20.glGetUniformLocation(dibr3dProgram, "s_ColorTexture"); + mDibrDepthTexHandle = GLES20.glGetUniformLocation(dibr3dProgram, "s_DepthTexture"); + mDibrParallaxHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_parallax"); + mDibrConvergenceHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_convergence"); + mDibrShiftHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_shift"); + mDibrDebugModeHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_debugMode"); + } + private void initializeIntermediateFbo() { intermediateTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); int[] fbos = new int[1]; @@ -311,24 +340,15 @@ private void initializeIntermediateFbo() { } private void initializeDilationFbo() { - // Create the texture to store the dilation result intermediateDilutionTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); - - // Create the framebuffer object (FBO) int[] fbos = new int[1]; GLES20.glGenFramebuffers(1, fbos, 0); intermediateDilutionFboHandle = fbos[0]; - - // Bind the FBO and attach the texture to it GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId, 0); - - // Check if the FBO was created successfully if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE) { LimeLog.warning("Dilation Framebuffer is not complete."); } - - // Unbind the FBO to restore the default state GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } @@ -336,117 +356,73 @@ private float getParallax() { return prefConfig.parallax_depth * 0.2f; } - /** - * Wendet einen performanten, zweistufigen Dilation-Filter an. - * Dieses Verfahren ist bei großen Radien deutlich schneller als ein einstufiger Filter. - * Liest von 'depthMapTextureId', schreibt das Zwischenergebnis nach 'intermediateDilutionFboHandle' - * und das Endergebnis nach 'intermediateFboHandle'. - */ private void applyTwoPassDilation() { - // Das NEUE, separable Dilation-Shader-Programm aktivieren - GLES20.glUseProgram(mDilationProgram); // Stelle sicher, dass du diese Variable hast - - // Handles für Attribute und Uniforms holen (sollten als Member-Variablen gecached sein) - int posHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_Position"); - int texHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_TexCoord"); - int inputTextureHandle = GLES20.glGetUniformLocation(mDilationProgram, "s_InputTexture"); - int texelSizeHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_texelSize"); - int radiusHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_radius"); - int directionHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_direction"); - - // Vertex-Daten verbinden (mit den KORREKTEN, nicht-gespiegelten Koordinaten) - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); - GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); - GLES20.glEnableVertexAttribArray(posHandle); - GLES20.glEnableVertexAttribArray(texHandle); + GLES20.glUseProgram(mDilationProgram); + + GLES20.glVertexAttribPointer(mDilationPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mDilationTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glEnableVertexAttribArray(mDilationPosHandle); + GLES20.glEnableVertexAttribArray(mDilationTexHandle); // --- 1. DURCHGANG: HORIZONTAL --- - // Ziel ist der erste Zwischenspeicher (`intermediateDilutionFboHandle`). GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - - // Input ist die rohe, originale Tiefenkarte. GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); - // Setze alle Uniforms für den horizontalen Durchgang. - GLES20.glUniform1i(inputTextureHandle, 0); - GLES20.glUniform1i(radiusHandle, 15); // Dein gewünschter, großer Radius. - GLES20.glUniform2f(texelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); - GLES20.glUniform2f(directionHandle, 1.0f, 0.0f); // Richtung: Horizontal (X-Achse) + GLES20.glUniform1i(mDilationInputTextureHandle, 0); + GLES20.glUniform1i(mDilationRadiusHandle, 15); + GLES20.glUniform2f(mDilationTexelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); + GLES20.glUniform2f(mDilationDirectionHandle, 1.0f, 0.0f); - // Führe den ersten Shader-Durchgang aus. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); - // --- 2. DURCHGANG: VERTIKAL --- - // Ziel ist der zweite Zwischenspeicher (`intermediateFboHandle`), - // aus dem der Gauß-Filter später lesen wird. GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateFboHandle); - // Viewport muss nicht neu gesetzt werden, wenn die Größe gleich bleibt. - - // Input ist jetzt das Ergebnis des ersten Durchgangs. GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); + GLES20.glUniform2f(mDilationDirectionHandle, 0.0f, 1.0f); - // Die meisten Uniforms bleiben gleich, wir ändern nur die Richtung. - GLES20.glUniform2f(directionHandle, 0.0f, 1.0f); // Richtung: Vertikal (Y-Achse) - - // Führe den zweiten Shader-Durchgang aus. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); - - // WICHTIG: Der Framebuffer (`intermediateFboHandle`) bleibt für den - // nachfolgenden Gauß-Filter gebunden. Er wird erst am Ende der - // gesamten Filterkette (in onDrawFrame) auf 0 zurückgesetzt. } + private void applyTwoPassGaussianBlur() { + // WIR VERWENDEN HIER ABSICHTLICH WIEDER DIE LOKALEN VARIABLEN ZUM TESTEN int blurProgram = bilateralBlurProgram; GLES20.glUseProgram(blurProgram); - int posHandle = GLES20.glGetAttribLocation(blurProgram, "a_Position"); - int texHandle = GLES20.glGetAttribLocation(blurProgram, "a_TexCoord"); - int inputTextureHandle = GLES20.glGetUniformLocation(blurProgram, "s_InputTexture"); - int texelSizeHandle = GLES20.glGetUniformLocation(blurProgram, "u_texelSize"); - int directionHandle = GLES20.glGetUniformLocation(blurProgram, "u_blurDirection"); - int parallaxHandle = GLES20.glGetUniformLocation(blurProgram, "u_parallax"); - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); - GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); - GLES20.glEnableVertexAttribArray(posHandle); - GLES20.glEnableVertexAttribArray(texHandle); - GLES20.glUniform1f(parallaxHandle, getParallax()); + GLES20.glVertexAttribPointer(mGaussPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mGaussTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glEnableVertexAttribArray(mGaussPosHandle); + GLES20.glEnableVertexAttribArray(mGaussTexHandle); + GLES20.glUniform1f(mGaussParallaxHandle, getParallax()); - GLES20.glUniform2f(texelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); + GLES20.glUniform2f(mGaussTexelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateFboHandle); GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - - GLES20.glUniform2f(directionHandle, 1.0f, 0.0f); - + GLES20.glUniform2f(mGaussDirectionHandle, 1.0f, 0.0f); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); - GLES20.glUniform1i(inputTextureHandle, 0); - + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); // Korrekter Input von Dilation + GLES20.glUniform1i(mGaussInputTextureHandle, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, filterFboHandle); GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - - GLES20.glUniform2f(directionHandle, 0.0f, 1.0f); - + GLES20.glUniform2f(mGaussDirectionHandle, 0.0f, 1.0f); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateTextureId); - GLES20.glUniform1i(inputTextureHandle, 0); - + GLES20.glUniform1i(mGaussInputTextureHandle, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } + private void drawBothEyes(int dualBubble3dProgram, float convergence, float shift) { int viewWidth = glSurfaceView.getWidth(); int viewHeight = glSurfaceView.getHeight(); - float parallax = getParallax() * 0.2f; GLES20.glViewport(0, 0, viewWidth / 2, viewHeight); @@ -458,32 +434,24 @@ private void drawBothEyes(int dualBubble3dProgram, float convergence, float shif private void drawEye(int program, float parallax, float convergence, float shift) { GLES20.glUseProgram(program); - int posHandle = GLES20.glGetAttribLocation(program, "a_Position"); - int texHandle = GLES20.glGetAttribLocation(program, "a_TexCoord"); - int colorTexHandle = GLES20.glGetUniformLocation(program, "s_ColorTexture"); - int depthTexHandle = GLES20.glGetUniformLocation(program, "s_DepthTexture"); - int parallaxHandle = GLES20.glGetUniformLocation(program, "u_parallax"); - int convergenceHandle = GLES20.glGetUniformLocation(program, "u_convergence"); - int shiftHandle = GLES20.glGetUniformLocation(program, "u_shift"); - int debugModeHandle = GLES20.glGetUniformLocation(program, "u_debugMode"); - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); - GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); - GLES20.glEnableVertexAttribArray(posHandle); - GLES20.glEnableVertexAttribArray(texHandle); + GLES20.glVertexAttribPointer(mDibrPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mDibrTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glEnableVertexAttribArray(mDibrPosHandle); + GLES20.glEnableVertexAttribArray(mDibrTexHandle); - GLES20.glUniform1i(debugModeHandle, isDebugMode ? 1 : 0); + GLES20.glUniform1i(mDibrDebugModeHandle, isDebugMode ? 1 : 0); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, videoTextureId); - GLES20.glUniform1i(colorTexHandle, 0); + GLES20.glUniform1i(mDibrColorTexHandle, 0); GLES20.glActiveTexture(GLES20.GL_TEXTURE1); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, filteredDepthMapTextureId); - GLES20.glUniform1i(depthTexHandle, 1); - GLES20.glUniform1f(parallaxHandle, parallax); - GLES20.glUniform1f(convergenceHandle, convergence); - GLES20.glUniform1f(shiftHandle, shift); + GLES20.glUniform1i(mDibrDepthTexHandle, 1); + GLES20.glUniform1f(mDibrParallaxHandle, parallax); + GLES20.glUniform1f(mDibrConvergenceHandle, convergence); + GLES20.glUniform1f(mDibrShiftHandle, shift); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); } @@ -536,8 +504,6 @@ public void onDrawFrame(GL10 gl) { freeSmoothedBuffers.offer(currentlyRenderingMap); } - long startTimeAi = System.nanoTime(); - long endTimeAi = System.nanoTime(); if (tflite != null) { if (block || !isMovieMode) { ByteBuffer pixelBufferForAI = freeInputBuffers.poll(); @@ -564,6 +530,7 @@ public void onDrawFrame(GL10 gl) { try { Thread.sleep(1); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } } else { @@ -573,8 +540,6 @@ public void onDrawFrame(GL10 gl) { block = false; currentlyRenderingMap = newMap; depthMapResultCount++; - endTimeAi = System.nanoTime(); - Log.d("Stereo3DRenderer", "DepthMap OutputSpeed " + (endTimeAi - startTimeAi) / 1_000_000 + " ms"); } } @@ -626,9 +591,7 @@ private ByteBuffer createFlatDepthMap() { int mapSize = modelInputWidth * modelInputHeight; byte[] flatData = new byte[mapSize]; Arrays.fill(flatData, (byte) 128); - ByteBuffer flatMap = ByteBuffer.allocateDirect(mapSize).order(ByteOrder.nativeOrder()); - flatMap.put(flatData); flatMap.rewind(); return flatMap; @@ -644,30 +607,24 @@ private void uploadLatestDepthMapToGpu(ByteBuffer depthMap) { private void drawQuad(int program, float scale, float offset) { GLES20.glUseProgram(program); - int posHandle = GLES20.glGetAttribLocation(program, "a_Position"); int texHandle = GLES20.glGetAttribLocation(program, "a_TexCoord"); int offsetHandle = GLES20.glGetUniformLocation(program, "u_xOffset"); int scaleHandle = GLES20.glGetUniformLocation(program, "u_xScale"); - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); GLES20.glEnableVertexAttribArray(posHandle); GLES20.glEnableVertexAttribArray(texHandle); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, videoTextureId); - if (scaleHandle != -1) GLES20.glUniform1f(scaleHandle, scale); if (offsetHandle != -1) GLES20.glUniform1f(offsetHandle, offset); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); } private static class InferenceResult { final ByteBuffer pixelBuffer; final ByteBuffer rawDepthBuffer; - InferenceResult(ByteBuffer pixelBuffer, ByteBuffer rawDepthBuffer) { this.pixelBuffer = pixelBuffer; this.rawDepthBuffer = rawDepthBuffer; @@ -677,7 +634,6 @@ private static class InferenceResult { private static class RenderResult { final ByteBuffer pixelBuffer; final double imageDifference; - RenderResult(ByteBuffer pixelBuffer, double imageDifference) { this.pixelBuffer = pixelBuffer; this.imageDifference = imageDifference; @@ -692,26 +648,18 @@ public static void convertRgbaToRgb(ByteBuffer rgbaBuffer, ByteBuffer rgbBuffer, rgbMat = new Mat(height, width, CvType.CV_8UC3, rgbBuffer); Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); } finally { - if (rgbaMat != null) { - rgbaMat.release(); - } - if (rgbMat != null) { - rgbMat.release(); - } + if (rgbaMat != null) rgbaMat.release(); + if (rgbMat != null) rgbMat.release(); } } private void initializePBOs() { PBO_SIZE = modelInputWidth * modelInputHeight * 4; - GLES30.glGenBuffers(2, pboHandles, 0); - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[0]); GLES30.glBufferData(GLES30.GL_PIXEL_PACK_BUFFER, PBO_SIZE, null, GLES30.GL_DYNAMIC_READ); - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[1]); GLES30.glBufferData(GLES30.GL_PIXEL_PACK_BUFFER, PBO_SIZE, null, GLES30.GL_DYNAMIC_READ); - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, 0); } @@ -720,17 +668,13 @@ private Boolean readPixelsForAI(ByteBuffer destinationBuffer) { GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); drawQuad(simple3dProgram, 1.0f, 0.0f); destinationBuffer.rewind(); - GLES20.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, destinationBuffer); - GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); - return true; } private void initializeTfLite() { Interpreter.Options options = new Interpreter.Options(); - try { GpuDelegate.Options gpuOptions = new GpuDelegate.Options(); gpuOptions.setQuantizedModelsAllowed(true); @@ -743,7 +687,7 @@ private void initializeTfLite() { tflite = new Interpreter(loadModelFile(context, AI_MODEL), options); } catch (Exception e) { LimeLog.info("GPU Delegate nicht verfügbar: " + e.getMessage()); - gpuDelegate.close(); + if (gpuDelegate != null) gpuDelegate.close(); try { nnApiDelegate = new NnApiDelegate(); options.addDelegate(nnApiDelegate); @@ -752,10 +696,10 @@ private void initializeTfLite() { renderer = "NNAPI"; } catch (Exception exception) { LimeLog.info("NNAPI Delegate nicht verfügbar: " + e.getMessage()); - nnApiDelegate.close(); + if (nnApiDelegate != null) nnApiDelegate.close(); try { LimeLog.info("Fallback: CPU"); - tflite = new Interpreter(loadModelFile(context, AI_MODEL), options); + tflite = new Interpreter(loadModelFile(context, AI_MODEL), new Interpreter.Options()); renderer = "CPU"; } catch (Exception ex) { reinitializeTfLiteOnCpu(); @@ -773,7 +717,6 @@ private void reinitializeTfLiteOnCpu() { gpuDelegate.close(); gpuDelegate = null; } - try { Interpreter.Options options = new Interpreter.Options(); options.setUseNNAPI(true); @@ -863,28 +806,11 @@ private int createRgbaTexture(int width, int height) { return textureId; } - private int loadShader(int type, String shaderCode) { - int shader = GLES20.glCreateShader(type); - GLES20.glShaderSource(shader, shaderCode); - GLES20.glCompileShader(shader); - int[] compiled = new int[1]; - GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); - if (compiled[0] == 0) { - LimeLog.severe("Could not compile shader " + type + ":"); - LimeLog.severe(GLES20.glGetShaderInfoLog(shader)); - GLES20.glDeleteShader(shader); - shader = 0; - } - return shader; - } - private int createProgram(String vertex, String fragment) { - // --- VERTEX SHADER COMPILATION --- int vertexShader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER); GLES20.glShaderSource(vertexShader, vertex); GLES20.glCompileShader(vertexShader); - // --- NEUES LOGGING HINZUGEFÜGT --- int[] compiled = new int[1]; GLES20.glGetShaderiv(vertexShader, GLES20.GL_COMPILE_STATUS, compiled, 0); if (compiled[0] == 0) { @@ -894,12 +820,10 @@ private int createProgram(String vertex, String fragment) { return 0; } - // --- FRAGMENT SHADER COMPILATION --- int fragmentShader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER); GLES20.glShaderSource(fragmentShader, fragment); GLES20.glCompileShader(fragmentShader); - // --- NEUES LOGGING HINZUGEFÜGT --- GLES20.glGetShaderiv(fragmentShader, GLES20.GL_COMPILE_STATUS, compiled, 0); if (compiled[0] == 0) { LimeLog.severe("Could not compile fragment shader:"); @@ -908,7 +832,6 @@ private int createProgram(String vertex, String fragment) { return 0; } - // --- PROGRAM LINKING (DEIN BESTEHENDER CODE) --- int program = GLES20.glCreateProgram(); if (program != 0) { GLES20.glAttachShader(program, vertexShader); @@ -925,85 +848,45 @@ private int createProgram(String vertex, String fragment) { } return program; } + private double computeColorSimilarity(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { - return 0.0; // komplett unterschiedlich + return 0.0; } - - // ByteBuffers zurücksetzen newPixelBuffer.rewind(); oldPixelBuffer.rewind(); - - Mat mat1 = null, mat2 = null; - Mat matBGR1 = null, matBGR2 = null; - Mat histB1 = null, histG1 = null, histR1 = null; - Mat histB2 = null, histG2 = null, histR2 = null; - - List bgrPlanes1 = null; - List bgrPlanes2 = null; + Mat mat1 = null, mat2 = null, matBGR1 = null, matBGR2 = null, histB1 = null, histG1 = null, histR1 = null, histB2 = null, histG2 = null, histR2 = null; + List bgrPlanes1 = new ArrayList<>(), bgrPlanes2 = new ArrayList<>(); try { - // Mats aus ByteBuffer mat1 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, newPixelBuffer); mat2 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, oldPixelBuffer); - - // RGBA -> BGR konvertieren matBGR1 = new Mat(); matBGR2 = new Mat(); Imgproc.cvtColor(mat1, matBGR1, Imgproc.COLOR_RGBA2BGR); Imgproc.cvtColor(mat2, matBGR2, Imgproc.COLOR_RGBA2BGR); - - // Core.split vorbereiten - bgrPlanes1 = new ArrayList<>(); - bgrPlanes1.add(new Mat()); - bgrPlanes1.add(new Mat()); - bgrPlanes1.add(new Mat()); Core.split(matBGR1, bgrPlanes1); - - bgrPlanes2 = new ArrayList<>(); - bgrPlanes2.add(new Mat()); - bgrPlanes2.add(new Mat()); - bgrPlanes2.add(new Mat()); Core.split(matBGR2, bgrPlanes2); - - // Histogramme - histB1 = new Mat(); - histG1 = new Mat(); - histR1 = new Mat(); - histB2 = new Mat(); - histG2 = new Mat(); - histR2 = new Mat(); - - int histSize = 16; // grobe Farbanalyse - float[] range = {0f, 256f}; - MatOfFloat histRange = new MatOfFloat(range); - - // Histogramme berechnen + histB1 = new Mat(); histG1 = new Mat(); histR1 = new Mat(); + histB2 = new Mat(); histG2 = new Mat(); histR2 = new Mat(); + int histSize = 16; + MatOfFloat histRange = new MatOfFloat(0f, 256f); Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(0)), new MatOfInt(0), new Mat(), histB1, new MatOfInt(histSize), histRange); Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(1)), new MatOfInt(0), new Mat(), histG1, new MatOfInt(histSize), histRange); Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(2)), new MatOfInt(0), new Mat(), histR1, new MatOfInt(histSize), histRange); - Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(0)), new MatOfInt(0), new Mat(), histB2, new MatOfInt(histSize), histRange); Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(1)), new MatOfInt(0), new Mat(), histG2, new MatOfInt(histSize), histRange); Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(2)), new MatOfInt(0), new Mat(), histR2, new MatOfInt(histSize), histRange); - - // Normalisieren Core.normalize(histB1, histB1, 0, 1, Core.NORM_MINMAX); Core.normalize(histG1, histG1, 0, 1, Core.NORM_MINMAX); Core.normalize(histR1, histR1, 0, 1, Core.NORM_MINMAX); - Core.normalize(histB2, histB2, 0, 1, Core.NORM_MINMAX); Core.normalize(histG2, histG2, 0, 1, Core.NORM_MINMAX); Core.normalize(histR2, histR2, 0, 1, Core.NORM_MINMAX); - - // Histogramm-Korrelation pro Kanal double corrB = Imgproc.compareHist(histB1, histB2, Imgproc.HISTCMP_CORREL); double corrG = Imgproc.compareHist(histG1, histG2, Imgproc.HISTCMP_CORREL); double corrR = Imgproc.compareHist(histR1, histR2, Imgproc.HISTCMP_CORREL); - return 1f - Math.max(0, (corrB + corrG + corrR) / 3.0); - } finally { - // Alle Mats freigeben if (mat1 != null) mat1.release(); if (mat2 != null) mat2.release(); if (matBGR1 != null) matBGR1.release(); @@ -1014,142 +897,92 @@ private double computeColorSimilarity(ByteBuffer newPixelBuffer, ByteBuffer oldP if (histB2 != null) histB2.release(); if (histG2 != null) histG2.release(); if (histR2 != null) histR2.release(); - for (Mat m : new Mat[]{bgrPlanes1.get(0), bgrPlanes1.get(1), bgrPlanes1.get(2)}) - m.release(); - for (Mat m : new Mat[]{bgrPlanes2.get(0), bgrPlanes2.get(1), bgrPlanes2.get(2)}) - m.release(); + for (Mat m : bgrPlanes1) if (m != null) m.release(); + for (Mat m : bgrPlanes2) if (m != null) m.release(); } } - - private double hasFrameChangedSignificantlyOCV(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { - return 1.0; // maximal unterschiedliche Frames + return 1.0; } - - Mat mat1 = null, mat2 = null; - Mat gray1 = null, gray2 = null; - Mat edges1 = null, edges2 = null; - Mat histGray1 = null, histGray2 = null; - Mat histEdge1 = null, histEdge2 = null; - + Mat mat1 = null, mat2 = null, gray1 = null, gray2 = null, edges1 = null, edges2 = null, + histGray1 = null, histGray2 = null, histEdge1 = null, histEdge2 = null; try { mat1 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, newPixelBuffer); mat2 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, oldPixelBuffer); - - // Graustufen gray1 = new Mat(); gray2 = new Mat(); Imgproc.cvtColor(mat1, gray1, Imgproc.COLOR_RGBA2GRAY); Imgproc.cvtColor(mat2, gray2, Imgproc.COLOR_RGBA2GRAY); - - // Kanten (Sobel) edges1 = new Mat(); edges2 = new Mat(); - Mat gradX1 = new Mat(), gradY1 = new Mat(); - Mat gradX2 = new Mat(), gradY2 = new Mat(); + Mat gradX1 = new Mat(), gradY1 = new Mat(), gradX2 = new Mat(), gradY2 = new Mat(); Imgproc.Sobel(gray1, gradX1, CvType.CV_16S, 1, 0); Imgproc.Sobel(gray1, gradY1, CvType.CV_16S, 0, 1); Core.convertScaleAbs(gradX1, gradX1); Core.convertScaleAbs(gradY1, gradY1); Core.addWeighted(gradX1, 0.5, gradY1, 0.5, 0, edges1); - Imgproc.Sobel(gray2, gradX2, CvType.CV_16S, 1, 0); Imgproc.Sobel(gray2, gradY2, CvType.CV_16S, 0, 1); Core.convertScaleAbs(gradX2, gradX2); Core.convertScaleAbs(gradY2, gradY2); Core.addWeighted(gradX2, 0.5, gradY2, 0.5, 0, edges2); - - gradX1.release(); - gradY1.release(); - gradX2.release(); - gradY2.release(); - - // Histogramme Graustufen - histGray1 = new Mat(); - histGray2 = new Mat(); + gradX1.release(); gradY1.release(); gradX2.release(); gradY2.release(); + histGray1 = new Mat(); histGray2 = new Mat(); Imgproc.calcHist(Collections.singletonList(gray1), new MatOfInt(0), new Mat(), histGray1, new MatOfInt(256), new MatOfFloat(0f, 256f)); Imgproc.calcHist(Collections.singletonList(gray2), new MatOfInt(0), new Mat(), histGray2, new MatOfInt(256), new MatOfFloat(0f, 256f)); - - // Histogramme Kanten - histEdge1 = new Mat(); - histEdge2 = new Mat(); + histEdge1 = new Mat(); histEdge2 = new Mat(); Imgproc.calcHist(Collections.singletonList(edges1), new MatOfInt(0), new Mat(), histEdge1, new MatOfInt(256), new MatOfFloat(0f, 256f)); Imgproc.calcHist(Collections.singletonList(edges2), new MatOfInt(0), new Mat(), histEdge2, new MatOfInt(256), new MatOfFloat(0f, 256f)); - - // Vergleich: Graustufen + Kanten double grayDiff = 1.0 - Imgproc.compareHist(histGray1, histGray2, Imgproc.HISTCMP_CORREL); double edgeDiff = 1.0 - Imgproc.compareHist(histEdge1, histEdge2, Imgproc.HISTCMP_CORREL); - - // Kombiniere beide Differenzen (Gewichtung kann angepasst werden) - double combinedDiff = 0.5 * grayDiff + 0.5 * edgeDiff; - return combinedDiff; - + return 0.5 * grayDiff + 0.5 * edgeDiff; } finally { - if (mat1 != null) mat1.release(); - if (mat2 != null) mat2.release(); - if (gray1 != null) gray1.release(); - if (gray2 != null) gray2.release(); - if (edges1 != null) edges1.release(); - if (edges2 != null) edges2.release(); - if (histGray1 != null) histGray1.release(); - if (histGray2 != null) histGray2.release(); - if (histEdge1 != null) histEdge1.release(); - if (histEdge2 != null) histEdge2.release(); + if (mat1 != null) mat1.release(); if (mat2 != null) mat2.release(); + if (gray1 != null) gray1.release(); if (gray2 != null) gray2.release(); + if (edges1 != null) edges1.release(); if (edges2 != null) edges2.release(); + if (histGray1 != null) histGray1.release(); if (histGray2 != null) histGray2.release(); + if (histEdge1 != null) histEdge1.release(); if (histEdge2 != null) histEdge2.release(); } } - private double hasSceneChangedFast(ByteBuffer currentFrame, ByteBuffer previousFrame) { if (currentFrame == null || previousFrame == null || currentFrame.capacity() != previousFrame.capacity()) { return 0.0; } - currentFrame.rewind(); previousFrame.rewind(); - long totalDifference = 0; int pixelsSampled = 0; - final int PIXEL_STRIDE = 4; final int PIXEL_SAMPLE_RATE = 32; final int ROW_SAMPLE_RATE = 32; final int SAMPLE_STRIDE = PIXEL_STRIDE * PIXEL_SAMPLE_RATE; final int ROW_STRIDE = modelInputWidth * PIXEL_STRIDE * ROW_SAMPLE_RATE; - for (int row = 0; row < currentFrame.capacity(); row += ROW_STRIDE) { for (int col = 0; col < modelInputWidth * PIXEL_STRIDE; col += SAMPLE_STRIDE) { int index = row + col; if (index + 2 >= currentFrame.capacity()) break; - totalDifference += Math.abs((currentFrame.get(index) & 0xFF) - (previousFrame.get(index) & 0xFF)); totalDifference += Math.abs((currentFrame.get(index + 1) & 0xFF) - (previousFrame.get(index + 1) & 0xFF)); totalDifference += Math.abs((currentFrame.get(index + 2) & 0xFF) - (previousFrame.get(index + 2) & 0xFF)); pixelsSampled++; } } - if (pixelsSampled == 0) return 0.0; - - double averageDifference = (double) totalDifference / pixelsSampled; - - return averageDifference; + return (double) totalDifference / pixelsSampled; } private class AiTask implements Runnable { - private ByteBuffer previousRawMap = null; - @Override public void run() { ByteBuffer pixelBuffer = null; double difference = 0.0f; while (!Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); - long waitTime = System.nanoTime(); - long aiTime = System.nanoTime(); - long aiTime_end = System.nanoTime(); + long waitTime, aiTime, aiTime_end; try { if (tflite == null) return; RenderResult result = inferenceInputQueue.take(); @@ -1158,13 +991,10 @@ public void run() { ByteBuffer outputBuffer = freeOutputBuffers.take(); waitTime = System.nanoTime(); outputBuffer.rewind(); - if (difference > ON_DRAW_CHANGE_TRESHOLD || previousRawMap == null) { tfliteInputBuffer.rewind(); pixelBuffer.rewind(); - convertRgbaToRgb(pixelBuffer, tfliteInputBuffer, modelInputWidth, modelInputHeight); - aiTime = System.nanoTime(); ReflectivePaddingInt8Minimal.applyReflectedPadding(tfliteInputBuffer); tflite.run(tfliteInputBuffer, outputBuffer); @@ -1193,12 +1023,10 @@ public void run() { gpuDelegateFailed.set(true); } finally { long duration = (System.nanoTime() - startTime) / 1_000_000; - long waitTimeText = (waitTime - startTime) / 1_000_000; - long aitimeText = (aiTime_end - aiTime) / 1_000_000; if (pixelBuffer != null) { freeInputBuffers.offer(pixelBuffer); } - Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + " ms " + filledOutputBuffers.remainingCapacity() + " " + waitTimeText + " ms " + "aitime: " + aitimeText); + Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + " ms"); } } isAiRunning.set(false); @@ -1206,12 +1034,9 @@ public void run() { } private class AiResultHandling implements Runnable { - - private static final double DEPTH_DIFF_THRESHOLD = 0.1; // large jump threshold - private static final double DEPTH_DIFF_AVERAGE_THRESHOLD = 0.05; // large jump threshold - private static final double MAX_SMOOTHING = 1.0; // full adoption - private static final double MIN_SMOOTHING = 0.0; // ignore - + private static final double DEPTH_DIFF_THRESHOLD = 0.1; + private static final double MAX_SMOOTHING = 1.0; + private static final double MIN_SMOOTHING = 0.0; private Mat previousSmoothedMat; private boolean isFirstFrame = true; @@ -1219,7 +1044,6 @@ private class AiResultHandling implements Runnable { public void run() { ByteBuffer resultBuffer = createFlatDepthMap(); InferenceResult result = null; - while (!Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); Mat rawMat = null; @@ -1227,33 +1051,23 @@ public void run() { try { result = filledOutputBuffers.take(); resultBuffer = freeSmoothedBuffers.take(); - - // Take latest intermediate frame if multiple available InferenceResult intermediate; while ((intermediate = filledOutputBuffers.poll()) != null) { freeInputBuffers.offer(result.pixelBuffer); freeOutputBuffers.offer(result.rawDepthBuffer); result = intermediate; } - rawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, result.rawDepthBuffer); rawFloat = new Mat(); - rawMat.convertTo(rawFloat, CvType.CV_32F); // keep un-normalized - + rawMat.convertTo(rawFloat, CvType.CV_32F); if (isFirstFrame) { previousSmoothedMat = rawFloat.clone(); isFirstFrame = false; } - - // --- Calculate robust depth difference --- Mat diffMat = new Mat(); Core.absdiff(rawFloat, previousSmoothedMat, diffMat); - - // Mean difference (global) Scalar sumDiff = Core.sumElems(diffMat); double meanDiff = sumDiff.val[0] / (diffMat.rows() * diffMat.cols() * 255.0); - - // Standard deviation (local fluctuations) Mat diffFloat = new Mat(); diffMat.convertTo(diffFloat, CvType.CV_32F, 1.0 / 255.0); Scalar meanVal = Core.mean(diffFloat); @@ -1262,44 +1076,31 @@ public void run() { Core.subtract(diffFloat, meanMat, varianceMat); Core.multiply(varianceMat, varianceMat, varianceMat); double stdDev = Math.sqrt(Core.sumElems(varianceMat).val[0] / (diffFloat.rows() * diffFloat.cols())); - double depthMapDifference = Math.max(meanDiff, stdDev); - - // --- Determine smoothing factor --- double smoothing; if (depthMapDifference > DEPTH_DIFF_THRESHOLD) { - smoothing = MAX_SMOOTHING; // fully adopt large changes - } else if(depthMapDifference > 0.01){ - smoothing = depthMapDifference; // proportional to difference + smoothing = MAX_SMOOTHING; + } else if (depthMapDifference > 0.01) { + smoothing = depthMapDifference; smoothing = Math.max(MIN_SMOOTHING, Math.min(MAX_SMOOTHING, smoothing)); } else { - // Prevents mini pixel shifts reducing sharpness on still images smoothing = 0; } - - // --- Apply smoothing --- Imgproc.accumulateWeighted(rawFloat, previousSmoothedMat, smoothing); - - // --- Normalize for shader output --- Mat normalizedForShader = new Mat(); Core.MinMaxLocResult mmr = Core.minMaxLoc(previousSmoothedMat); Core.subtract(previousSmoothedMat, new Scalar(mmr.minVal), normalizedForShader); Core.divide(normalizedForShader, new Scalar(mmr.maxVal - mmr.minVal + 1e-6), normalizedForShader); - Mat outputMat = new Mat(); normalizedForShader.convertTo(outputMat, CvType.CV_8U, 255.0); outputMat.get(0, 0, resultBuffer.array()); - latestDepthMap.set(resultBuffer); - - // --- Cleanup --- diffMat.release(); diffFloat.release(); meanMat.release(); varianceMat.release(); normalizedForShader.release(); outputMat.release(); - } catch (Exception e) { LimeLog.severe("AI exception " + e.getMessage()); } finally { @@ -1317,5 +1118,4 @@ public void run() { isAiResultHandlingRunning.set(false); } } - } \ No newline at end of file From 225af2161b94eff224c2d01c575743f357bf4e75 Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 17 Oct 2025 23:15:06 +0200 Subject: [PATCH 06/22] Improvements 3d - dilute --- .../java/com/limelight/utils/ShaderUtils.java | 2 +- .../com/limelight/utils/Stereo3DRenderer.java | 165 +++++++++++++----- 2 files changed, 123 insertions(+), 44 deletions(-) diff --git a/app/src/main/java/com/limelight/utils/ShaderUtils.java b/app/src/main/java/com/limelight/utils/ShaderUtils.java index a2c0591240..8727e87982 100644 --- a/app/src/main/java/com/limelight/utils/ShaderUtils.java +++ b/app/src/main/java/com/limelight/utils/ShaderUtils.java @@ -24,7 +24,7 @@ public class ShaderUtils { "void main() {\n" + " vec2 depthTexCoord = vec2(v_TexCoord.x, 1.0 - v_TexCoord.y);\n" + " // Wende deinen bestehenden Offset auf die korrigierte Koordinate an.\n" + - " depthTexCoord -= vec2(abs(u_parallax / 2.0), 0.04);\n" + + " depthTexCoord -= vec2(abs(u_parallax / 2.0), 0);\n" + " float depth = texture2D(s_DepthTexture, depthTexCoord).r;\n" + "\n" + " const float zone_radius = 0.70; // Breite der neutralen Zone um Konvergenz\n" + diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index 8de281365d..624c7c3100 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -1037,85 +1037,164 @@ private class AiResultHandling implements Runnable { private static final double DEPTH_DIFF_THRESHOLD = 0.1; private static final double MAX_SMOOTHING = 1.0; private static final double MIN_SMOOTHING = 0.0; - private Mat previousSmoothedMat; + private Mat previousSmoothedMat; // Bleibt Member, hält Zustand über Frames private boolean isFirstFrame = true; + // --- Wiederverwendbare Mat-Objekte für diesen Thread --- + private Mat reusableRawMat = null; // Wird nur als Header verwendet + private Mat reusableRawFloat = new Mat(); + private Mat reusableDiffMat = new Mat(); + private Mat reusableDiffFloat = new Mat(); + private Mat reusableMeanMat = new Mat(); + private Mat reusableVarianceMat = new Mat(); + private Mat reusableNormalizedForShader = new Mat(); + private Mat reusableOutputMat = new Mat(); + @Override public void run() { - ByteBuffer resultBuffer = createFlatDepthMap(); + ByteBuffer resultBuffer = createFlatDepthMap(); // Initialer flacher Puffer InferenceResult result = null; - while (!Thread.currentThread().isInterrupted()) { + + // Markierung, ob der Thread noch laufen soll + while (isAiResultHandlingRunning.get() && !Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); - Mat rawMat = null; - Mat rawFloat = null; try { - result = filledOutputBuffers.take(); - resultBuffer = freeSmoothedBuffers.take(); + result = filledOutputBuffers.take(); // Blockiert, bis Ergebnis da ist + resultBuffer = freeSmoothedBuffers.take(); // Blockiert, bis Puffer frei ist + + // Verarbeite nur das letzte verfügbare Ergebnis InferenceResult intermediate; while ((intermediate = filledOutputBuffers.poll()) != null) { - freeInputBuffers.offer(result.pixelBuffer); + freeInputBuffers.offer(result.pixelBuffer); // Gib alte Buffer zurück freeOutputBuffers.offer(result.rawDepthBuffer); - result = intermediate; + result = intermediate; // Behalte das Neueste } - rawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, result.rawDepthBuffer); - rawFloat = new Mat(); - rawMat.convertTo(rawFloat, CvType.CV_32F); + + // --- Verwende wiederverwendbare Mats --- + // Erstelle nur den Header neu, zeigt auf den Buffer, keine Datenkopie + reusableRawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, result.rawDepthBuffer); + reusableRawMat.convertTo(reusableRawFloat, CvType.CV_32F); + if (isFirstFrame) { - previousSmoothedMat = rawFloat.clone(); + // Initialisiere previousSmoothedMat sicher + if (previousSmoothedMat == null) { + previousSmoothedMat = new Mat(); + } + reusableRawFloat.copyTo(previousSmoothedMat); isFirstFrame = false; + } else if (previousSmoothedMat == null || previousSmoothedMat.empty()) { + // Fallback, falls Initialisierung fehlschlug + if (previousSmoothedMat == null) previousSmoothedMat = new Mat(); + reusableRawFloat.copyTo(previousSmoothedMat); + LimeLog.warning("previousSmoothedMat war null/leer, neu initialisiert in Schleife."); + } + + // --- Differenzberechnung --- + Core.absdiff(reusableRawFloat, previousSmoothedMat, reusableDiffMat); + Scalar sumDiff = Core.sumElems(reusableDiffMat); + // Vermeide Division durch Null, falls Mat leer ist + double totalPixels = reusableDiffMat.total(); + double meanDiff = (totalPixels > 0) ? sumDiff.val[0] / (totalPixels * 255.0) : 0.0; + + reusableDiffMat.convertTo(reusableDiffFloat, CvType.CV_32F, 1.0 / 255.0); + Scalar meanVal = Core.mean(reusableDiffFloat); + + // Stelle sicher, dass meanMat die korrekte Größe/Typ hat, bevor setTo verwendet wird + if (reusableMeanMat.empty() || !reusableMeanMat.size().equals(reusableDiffFloat.size()) || reusableMeanMat.type() != reusableDiffFloat.type()) { + reusableMeanMat.create(reusableDiffFloat.size(), reusableDiffFloat.type()); } - Mat diffMat = new Mat(); - Core.absdiff(rawFloat, previousSmoothedMat, diffMat); - Scalar sumDiff = Core.sumElems(diffMat); - double meanDiff = sumDiff.val[0] / (diffMat.rows() * diffMat.cols() * 255.0); - Mat diffFloat = new Mat(); - diffMat.convertTo(diffFloat, CvType.CV_32F, 1.0 / 255.0); - Scalar meanVal = Core.mean(diffFloat); - Mat meanMat = new Mat(diffFloat.size(), CvType.CV_32F, new Scalar(meanVal.val[0])); - Mat varianceMat = new Mat(); - Core.subtract(diffFloat, meanMat, varianceMat); - Core.multiply(varianceMat, varianceMat, varianceMat); - double stdDev = Math.sqrt(Core.sumElems(varianceMat).val[0] / (diffFloat.rows() * diffFloat.cols())); + reusableMeanMat.setTo(new Scalar(meanVal.val[0])); + + Core.subtract(reusableDiffFloat, reusableMeanMat, reusableVarianceMat); // reusableVarianceMat wird überschrieben + Core.multiply(reusableVarianceMat, reusableVarianceMat, reusableVarianceMat); // In-place quadrieren + double varianceSum = Core.sumElems(reusableVarianceMat).val[0]; + double stdDev = (totalPixels > 0) ? Math.sqrt(varianceSum / totalPixels) : 0.0; + double depthMapDifference = Math.max(meanDiff, stdDev); + + // --- Glättungsfaktor --- double smoothing; if (depthMapDifference > DEPTH_DIFF_THRESHOLD) { smoothing = MAX_SMOOTHING; } else if (depthMapDifference > 0.01) { - smoothing = depthMapDifference; + smoothing = depthMapDifference * 2.0; // Evtl. Faktor anpassen smoothing = Math.max(MIN_SMOOTHING, Math.min(MAX_SMOOTHING, smoothing)); } else { smoothing = 0; } - Imgproc.accumulateWeighted(rawFloat, previousSmoothedMat, smoothing); - Mat normalizedForShader = new Mat(); + + // --- Glättung anwenden --- + Imgproc.accumulateWeighted(reusableRawFloat, previousSmoothedMat, smoothing); + + // --- Normalisieren für Ausgabe --- Core.MinMaxLocResult mmr = Core.minMaxLoc(previousSmoothedMat); - Core.subtract(previousSmoothedMat, new Scalar(mmr.minVal), normalizedForShader); - Core.divide(normalizedForShader, new Scalar(mmr.maxVal - mmr.minVal + 1e-6), normalizedForShader); - Mat outputMat = new Mat(); - normalizedForShader.convertTo(outputMat, CvType.CV_8U, 255.0); - outputMat.get(0, 0, resultBuffer.array()); + double range = mmr.maxVal - mmr.minVal; + if (range < 1e-6) range = 1e-6; // Schutz vor Division durch Null + + Core.subtract(previousSmoothedMat, new Scalar(mmr.minVal), reusableNormalizedForShader); + Core.divide(reusableNormalizedForShader, new Scalar(range), reusableNormalizedForShader); + + reusableNormalizedForShader.convertTo(reusableOutputMat, CvType.CV_8U, 255.0); + + // --- Ergebnis in ByteBuffer kopieren --- + if (reusableOutputMat.isContinuous() && resultBuffer.hasArray()) { + reusableOutputMat.get(0, 0, resultBuffer.array()); + // Wichtig: Limit muss evtl. angepasst werden, wenn Puffer größer ist + resultBuffer.limit(reusableOutputMat.rows() * reusableOutputMat.cols() * (int)reusableOutputMat.elemSize()); + } else { + // Langsamerer Fallback, falls nicht kontinuierlich oder kein Array hat + int bufferSize = modelInputWidth * modelInputHeight; + byte[] data = new byte[bufferSize]; + reusableOutputMat.get(0, 0, data); + resultBuffer.put(data); + } + resultBuffer.rewind(); // Puffer für den Konsumenten vorbereiten + latestDepthMap.set(resultBuffer); - diffMat.release(); - diffFloat.release(); - meanMat.release(); - varianceMat.release(); - normalizedForShader.release(); - outputMat.release(); + resultBuffer = null; // Besitz wurde an latestDepthMap übergeben + + } catch (InterruptedException e) { + LimeLog.warning("AiResultHandling interrupted."); + Thread.currentThread().interrupt(); // Interrupt-Status wiederherstellen + break; // Schleife verlassen } catch (Exception e) { - LimeLog.severe("AI exception " + e.getMessage()); + LimeLog.severe("AI result handling exception: " + e.getMessage()); + // Hier könnte man überlegen, ob isFirstFrame zurückgesetzt werden soll } finally { - if (rawMat != null) rawMat.release(); - if (rawFloat != null) rawFloat.release(); + // Gib nur die Buffer zurück, deren Besitz nicht übertragen wurde if (resultBuffer != null) freeSmoothedBuffers.offer(resultBuffer); if (result != null) { freeInputBuffers.offer(result.pixelBuffer); freeOutputBuffers.offer(result.rawDepthBuffer); + result = null; // Referenz löschen } + // Die wiederverwendeten Mat-Objekte werden NICHT hier freigegeben + long duration = (System.nanoTime() - startTime) / 1_000_000; Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + " ms"); } + } // Ende while-Schleife + + // --- Aufräumen, wenn der Thread endet --- + releaseMat(previousSmoothedMat); previousSmoothedMat = null; + // releaseMat(reusableRawMat); // Ist nur ein Header, muss nicht freigegeben werden + releaseMat(reusableRawFloat); + releaseMat(reusableDiffMat); + releaseMat(reusableDiffFloat); + releaseMat(reusableMeanMat); + releaseMat(reusableVarianceMat); + releaseMat(reusableNormalizedForShader); + releaseMat(reusableOutputMat); + + isAiResultHandlingRunning.set(false); // Signal setzen, dass der Thread beendet ist + LimeLog.info("AiResultHandling finished."); + } // Ende run() + + // Hilfsmethode zum sicheren Freigeben von Mats + private void releaseMat(Mat mat) { + if (mat != null && !mat.empty()) { + mat.release(); } - isAiResultHandlingRunning.set(false); } } } \ No newline at end of file From ec4540af16543c03ce4d2bf592309e053c97f85f Mon Sep 17 00:00:00 2001 From: Janyger Date: Sat, 18 Oct 2025 21:30:49 +0200 Subject: [PATCH 07/22] Improvements 3d - dilute --- .../java/com/limelight/utils/ShaderUtils.java | 2 +- .../com/limelight/utils/Stereo3DRenderer.java | 52 ++++++++++++------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/app/src/main/java/com/limelight/utils/ShaderUtils.java b/app/src/main/java/com/limelight/utils/ShaderUtils.java index 8727e87982..219b0f3c41 100644 --- a/app/src/main/java/com/limelight/utils/ShaderUtils.java +++ b/app/src/main/java/com/limelight/utils/ShaderUtils.java @@ -22,7 +22,7 @@ public class ShaderUtils { "uniform bool u_debugMode;\n" + "\n" + "void main() {\n" + - " vec2 depthTexCoord = vec2(v_TexCoord.x, 1.0 - v_TexCoord.y);\n" + + " vec2 depthTexCoord = v_TexCoord;\n" + " // Wende deinen bestehenden Offset auf die korrigierte Koordinate an.\n" + " depthTexCoord -= vec2(abs(u_parallax / 2.0), 0);\n" + " float depth = texture2D(s_DepthTexture, depthTexCoord).r;\n" + diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index 624c7c3100..11acc7f343 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -107,6 +107,8 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private int intermediateTextureId; private int intermediateDilutionFboHandle; private int intermediateDilutionTextureId; + private int gaussIntermediateFboHandle; + private int gaussIntermediateTextureId; // --- VORGELADENE SHADER-LOCATIONS FÜR PERFORMANCE --- private int mDilationPosHandle, mDilationTexHandle, mDilationInputTextureHandle, @@ -213,7 +215,8 @@ public void onSurfaceDestroyed() { filteredDepthMapTextureId, fboTextureId, intermediateTextureId, - intermediateDilutionTextureId + intermediateDilutionTextureId, + }; GLES20.glDeleteTextures(textures.length, textures, 0); @@ -235,6 +238,19 @@ public Surface getVideoSurface() { return videoSurface; } + private void initializeGaussIntermediateFbo() { + gaussIntermediateTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); // Oder passende Textur erstellen + int[] fbos = new int[1]; + GLES20.glGenFramebuffers(1, fbos, 0); + gaussIntermediateFboHandle = fbos[0]; + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, gaussIntermediateFboHandle); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, gaussIntermediateTextureId, 0); + if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE) { + LimeLog.warning("Gauss Intermediate Framebuffer is not complete."); + } + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + @Override public void onFrameAvailable(SurfaceTexture surfaceTexture) { synchronized (frameLock) { @@ -265,6 +281,7 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { initializeFilterFbo(); initializeIntermediateFbo(); initializeDilationFbo(); + initializeGaussIntermediateFbo(); initializeTfLite(); initializeFbo(); initBuffer(); @@ -353,41 +370,40 @@ private void initializeDilationFbo() { } private float getParallax() { - return prefConfig.parallax_depth * 0.2f; + return prefConfig.parallax_depth * 0.25f; } + /** + * Wendet einen performanten, zweistufigen Dilation-Filter korrekt an. + * Liest von 'depthMapTextureId', schreibt das Zwischenergebnis nach 'intermediateDilutionFboHandle' (Tex A) + * und das Endergebnis nach 'intermediateFboHandle' (Tex B). + */ private void applyTwoPassDilation() { GLES20.glUseProgram(mDilationProgram); - - GLES20.glVertexAttribPointer(mDilationPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); - GLES20.glVertexAttribPointer(mDilationTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); GLES20.glEnableVertexAttribArray(mDilationPosHandle); GLES20.glEnableVertexAttribArray(mDilationTexHandle); - - // --- 1. DURCHGANG: HORIZONTAL --- + GLES20.glVertexAttribPointer(mDilationPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mDilationTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glUniform1i(mDilationInputTextureHandle, 0); + GLES20.glUniform2f(mDilationTexelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); - - GLES20.glUniform1i(mDilationInputTextureHandle, 0); - GLES20.glUniform1i(mDilationRadiusHandle, 15); - GLES20.glUniform2f(mDilationTexelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); + GLES20.glUniform1i(mDilationRadiusHandle, 5); GLES20.glUniform2f(mDilationDirectionHandle, 1.0f, 0.0f); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); - - // --- 2. DURCHGANG: VERTIKAL --- - GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateFboHandle); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, gaussIntermediateFboHandle); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); + GLES20.glUniform1i(mDilationRadiusHandle, 5); GLES20.glUniform2f(mDilationDirectionHandle, 0.0f, 1.0f); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLES20.glDisableVertexAttribArray(mDilationPosHandle); + GLES20.glDisableVertexAttribArray(mDilationTexHandle); } private void applyTwoPassGaussianBlur() { - // WIR VERWENDEN HIER ABSICHTLICH WIEDER DIE LOKALEN VARIABLEN ZUM TESTEN int blurProgram = bilateralBlurProgram; GLES20.glUseProgram(blurProgram); @@ -404,7 +420,7 @@ private void applyTwoPassGaussianBlur() { GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); GLES20.glUniform2f(mGaussDirectionHandle, 1.0f, 0.0f); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); // Korrekter Input von Dilation + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, gaussIntermediateTextureId); GLES20.glUniform1i(mGaussInputTextureHandle, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); From 03b2e3bb5d818b3c103b1bab31cee6f8a6285a29 Mon Sep 17 00:00:00 2001 From: Janyger Date: Sat, 18 Oct 2025 22:09:05 +0200 Subject: [PATCH 08/22] Improvements 3d - dilute --- .../utils/ReflectivePaddingInt8Minimal.java | 79 ------------------- .../java/com/limelight/utils/ShaderUtils.java | 9 +++ .../com/limelight/utils/Stereo3DRenderer.java | 25 ++++-- 3 files changed, 29 insertions(+), 84 deletions(-) delete mode 100644 app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java diff --git a/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java b/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java deleted file mode 100644 index 05f25edbe8..0000000000 --- a/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.limelight.utils; - -import org.opencv.core.*; -import org.opencv.imgproc.Imgproc; - -import java.nio.ByteBuffer; - -public class ReflectivePaddingInt8Minimal { - - /** - * In-place Reflected Padding + Feather + Blur auf ByteBuffer (INT8 RGB) - * Minimaler Speicher: kein alpha3 Merge, nur 2 temporäre Mats - */ - public static void applyReflectedPadding(ByteBuffer buffer) { - final int size = 256; - final int band = (int)(size * 0.1); // obere/untere 20% - final int featherPx = 12; - final int blurKsize = 7; - - // --- ByteBuffer -> Mat (CV_8UC3) --- - buffer.rewind(); - byte[] arr = new byte[size * size * 3]; - buffer.get(arr); - Mat mat = new Mat(size, size, CvType.CV_8UC3); - mat.put(0, 0, arr); - - // --- Top-Band --- - Mat topBand = mat.submat(band, 2*band, 0, size); - Mat tmp = new Mat(); - Core.flip(topBand, tmp, 0); - Imgproc.GaussianBlur(tmp, tmp, new Size(blurKsize, blurKsize), 0); - blendInt8Minimal(mat.submat(0, band, 0, size), tmp, band, featherPx); - tmp.release(); - topBand.release(); - - // --- Bottom-Band --- - Mat botBand = mat.submat(size - 2*band, size - band, 0, size); - tmp = new Mat(); - Core.flip(botBand, tmp, 0); - Imgproc.GaussianBlur(tmp, tmp, new Size(blurKsize, blurKsize), 0); - blendInt8Minimal(mat.submat(size - band, size, 0, size), tmp, band, featherPx); - tmp.release(); - botBand.release(); - - // --- Mat -> ByteBuffer zurück --- - Core.flip(mat, mat, 0); - mat.get(0,0,arr); - buffer.rewind(); - buffer.put(arr); - buffer.rewind(); - mat.release(); - } - - /** - * INT8 Alpha-Blend ohne Merge: dst = (alpha*padded + (255-alpha)*dst)/255 - * alpha linear von 0-255 über band Pixel - */ - private static void blendInt8Minimal(Mat dst, Mat padded, int band, int featherPx) { - int width = dst.cols(); - int channels = dst.channels(); - byte[] dstRow = new byte[width * channels]; - byte[] padRow = new byte[width * channels]; - - for(int y=0; y Date: Mon, 20 Oct 2025 10:43:14 +0200 Subject: [PATCH 09/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/Game.java | 8 +- .../video/MediaCodecDecoderRenderer.java | 9 +- .../preferences/PreferenceConfiguration.java | 5 +- .../com/limelight/utils/Stereo3DRenderer.java | 283 ++++++++---------- app/src/main/res/values/strings.xml | 2 + app/src/main/res/xml/preferences.xml | 20 +- 6 files changed, 152 insertions(+), 175 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 4b13378b2e..f6229c56b3 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -400,9 +400,15 @@ protected void onCreate(Bundle savedInstanceState) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && onExternelDisplay) { Display.Mode currentMode = currentDisplay.getMode(); + if(prefConfig.externalScreenAutoConfig && prefConfig.renderMode == 0) { + displayWidth = currentMode.getPhysicalWidth(); + displayHeight = currentMode.getPhysicalHeight(); + prefConfig.width = displayWidth; + prefConfig.height = displayHeight; + prefConfig.fps = (int) currentMode.getRefreshRate(); + } displayWidth = prefConfig.width; displayHeight = prefConfig.height; - prefConfig.fps = currentMode.getRefreshRate(); prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; prefConfig.enableFloatingButton = false; prefConfig.showOverlayZoomToggleButton = false; diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index a52ef463d6..1f528df69f 100755 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -1826,9 +1826,9 @@ public int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int dec sb.append("\t"); sb.append(context.getString(R.string.perf_overlay_lite_packet_loss) + ": "); sb.append(context.getString(R.string.perf_overlay_lite_netdrops,(float)lastTwo.framesLost / lastTwo.totalFrames * 100)); - sb.append("\t FPS："); - sb.append(context.getString(R.string.perf_overlay_lite_fps, fps.totalFps)); if(Stereo3DRenderer.isActive) { + sb.append("\t FPS："); + sb.append(context.getString(R.string.perf_overlay_lite_fps, Stereo3DRenderer.fps)); sb.append(" "); sb.append(context.getString(R.string.perf_overlay_ai_fps)); sb.append(" "); @@ -1839,10 +1839,13 @@ public int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int dec sb.append(Stereo3DRenderer.renderer); sb.append(" "); sb.append(context.getString(R.string.perf_overlay_drawdelay, Stereo3DRenderer.drawDelay)); + } else { + sb.append("\t FPS："); + sb.append(context.getString(R.string.perf_overlay_lite_fps, fps.totalFps)); } }else{ if(Stereo3DRenderer.isActive) { - sb.append(context.getString(R.string.perf_overlay_streamdetails, initialWidth + "x" + initialHeight, fps.totalFps)); + sb.append(context.getString(R.string.perf_overlay_streamdetails, initialWidth + "x" + initialHeight, Stereo3DRenderer.fps)); sb.append('\n'); sb.append(" "); sb.append(context.getString(R.string.perf_overlay_ai_fps)); diff --git a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java index 96249d87ac..96528b5b3b 100755 --- a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java +++ b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java @@ -117,6 +117,8 @@ public enum AnalogStickForScrolling { private static final String CHECKBOX_SHOW_OVERLAY_ZOOM_TOGGLE_BUTTON = "checkbox_show_overlay_zoom_toggle_button"; + private static final String CHECKBOX_ENABLE_FULLEXDISPLAY_AUTO = "checkbox_enable_fullexdisplay_auto"; + //竖屏模式 private static final String CHECKBOX_AUTO_ORIENTATION = "checkbox_auto_orientation"; //屏幕特殊按键 @@ -282,7 +284,7 @@ public enum AnalogStickForScrolling { public boolean enableBackMenu; public boolean enableFloatingButton; public boolean showOverlayZoomToggleButton; - + public boolean externalScreenAutoConfig; //Invert video width/height public boolean autoInvertVideoResolution; public int resolutionScaleFactor; @@ -938,6 +940,7 @@ else if (audioConfig.equals("51")) { config.enableBackMenu = prefs.getBoolean(CHECKBOX_ENABLE_QUIT_DIALOG,true); config.enableFloatingButton = prefs.getBoolean(CHECKBOX_ENABLE_FLOATING_BUTTON,DEFAULT_ENABLE_FLOATING_BUTTON); config.showOverlayZoomToggleButton = prefs.getBoolean(CHECKBOX_SHOW_OVERLAY_ZOOM_TOGGLE_BUTTON, DEFAULT_SHOW_OVERLAY_TOGGLE_BUTTON); + config.externalScreenAutoConfig = prefs.getBoolean(CHECKBOX_ENABLE_FULLEXDISPLAY_AUTO, true); config.autoOrientation = prefs.getBoolean(CHECKBOX_AUTO_ORIENTATION,false); config.autoInvertVideoResolution = prefs.getBoolean(AUTO_INVERT_VIDEO_RESOLUTION_PREF_STRING, DEFAULT_AUTO_INVERT_VIDEO_RESOLUTION); config.resolutionScaleFactor = prefs.getInt(RESOLUTION_SCALE_FACTOR_PREF_STRING, DEFAULT_RESOLUTION_SCALE_FACTOR); diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index c176aaaa40..11b9a1faa2 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -293,9 +293,9 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { freeSmoothedBuffers.offer(ByteBuffer.allocateDirect(mapSize).order(ByteOrder.nativeOrder())); } - int pboSize = modelInputWidth * modelInputHeight * 4; + int pboSize = modelInputWidth * modelInputHeight * 3; previousFrameForComparison = ByteBuffer.allocateDirect(pboSize).order(ByteOrder.nativeOrder()); - int inputPixelSize = modelInputWidth * modelInputHeight * 4; + int inputPixelSize = modelInputWidth * modelInputHeight * 3; freeInputBuffers = new ArrayBlockingQueue<>(NUM_INPUT_BUFFERS); inferenceInputQueue = new ArrayBlockingQueue<>(1); for (int i = 0; i < NUM_INPUT_BUFFERS; i++) { @@ -390,13 +390,13 @@ private void applyTwoPassDilation() { GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); - GLES20.glUniform1i(mDilationRadiusHandle, 5); + GLES20.glUniform1i(mDilationRadiusHandle, 7); GLES20.glUniform2f(mDilationDirectionHandle, 1.0f, 0.0f); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, gaussIntermediateFboHandle); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); - GLES20.glUniform1i(mDilationRadiusHandle, 5); + GLES20.glUniform1i(mDilationRadiusHandle, 7); GLES20.glUniform2f(mDilationDirectionHandle, 0.0f, 1.0f); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glDisableVertexAttribArray(mDilationPosHandle); @@ -498,13 +498,9 @@ public void onDrawFrame(GL10 gl) { synchronized (frameLock) { if (!frameAvailable.get()) { - if (!isMovieMode) { - glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); - } else { glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); return; - } - } else if (isMovieMode) { + } else if (isMovieMode) { block = true; } frameAvailable.set(false); @@ -530,8 +526,7 @@ public void onDrawFrame(GL10 gl) { pixelBufferForAI.rewind(); previousFrameForComparison.rewind(); previousFrameForComparison.put(pixelBufferForAI); - - if (inferenceInputQueue.offer(new RenderResult(pixelBufferForAI, difference))) { + if (inferenceInputQueue.offer(new RenderResult(pixelBufferForAI, difference, nextFrameId++))) { Log.d("AiTask", "Success: The AI will now process this buffer."); } else { freeInputBuffers.offer(pixelBufferForAI); @@ -578,7 +573,7 @@ public void onDrawFrame(GL10 gl) { drawDelay = ((float) totalDrawTime / fps / 1000000000f); } totalDrawTime = 0; - fps = calcFps; + fps = (calcFps +1); calcFps = 0; depthMapResultCount = 0; threeDFps = calcThreeDFps; @@ -647,38 +642,20 @@ private static class InferenceResult { } } - private static class RenderResult { - final ByteBuffer pixelBuffer; - final double imageDifference; - RenderResult(ByteBuffer pixelBuffer, double imageDifference) { - this.pixelBuffer = pixelBuffer; - this.imageDifference = imageDifference; - } - } + private int nextFrameId = 0; - public static void convertRgbaToRgb(ByteBuffer rgbaBuffer, ByteBuffer rgbBuffer, int width, int height) { - Mat rgbaMat = null; - Mat rgbMat = null; - try { - rgbaMat = new Mat(height, width, CvType.CV_8UC4, rgbaBuffer); - rgbMat = new Mat(height, width, CvType.CV_8UC3, rgbBuffer); - Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); - } finally { - if (rgbaMat != null) rgbaMat.release(); - if (rgbMat != null) rgbMat.release(); - } - } + public class RenderResult { + public final ByteBuffer pixelBuffer; + public final double imageDifference; + public final int frameId; // new field - public static void fastRgbaToRgb(ByteBuffer rgba, ByteBuffer rgb, int width, int height) { - for (int i = 0; i < width * height; i++) { - rgb.put(rgba.get()); // R - rgb.put(rgba.get()); // G - rgb.put(rgba.get()); // B - rgba.get(); // skip A + public RenderResult(ByteBuffer pixelBuffer, double imageDifference, int frameId) { + this.pixelBuffer = pixelBuffer; + this.imageDifference = imageDifference; + this.frameId = frameId; } } - private void initializePBOs() { PBO_SIZE = modelInputWidth * modelInputHeight * 4; GLES30.glGenBuffers(2, pboHandles, 0); @@ -694,7 +671,7 @@ private Boolean readPixelsForAI(ByteBuffer destinationBuffer) { GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); drawQuad(simple3dProgram, 1.0f, 0.0f); destinationBuffer.rewind(); - GLES20.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, destinationBuffer); + GLES20.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES20.GL_RGB, GLES20.GL_UNSIGNED_BYTE, destinationBuffer); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); return true; } @@ -769,7 +746,7 @@ public void onSurfaceChanged(GL10 gl, int width, int height) { } private void initializeFbo() { - fboTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); + fboTextureId = createRgbTexture(modelInputWidth, modelInputHeight); int[] fbos = new int[1]; GLES20.glGenFramebuffers(1, fbos, 0); fboHandle = fbos[0]; @@ -781,6 +758,29 @@ private void initializeFbo() { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } + private int createRgbTexture(int width, int height) { + int[] textures = new int[1]; + GLES20.glGenTextures(1, textures, 0); + int textureId = textures[0]; + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, + 0, + GLES20.GL_RGB, // <- RGB statt RGBA + width, + height, + 0, + GLES20.GL_RGB, // <- RGB statt RGBA + GLES20.GL_UNSIGNED_BYTE, + null + ); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + return textureId; + } + private void initializeFilterFbo() { filteredDepthMapTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); int[] fbos = new int[1]; @@ -875,157 +875,99 @@ private int createProgram(String vertex, String fragment) { return program; } - private double computeColorSimilarity(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { - if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { - return 0.0; - } - newPixelBuffer.rewind(); - oldPixelBuffer.rewind(); - Mat mat1 = null, mat2 = null, matBGR1 = null, matBGR2 = null, histB1 = null, histG1 = null, histR1 = null, histB2 = null, histG2 = null, histR2 = null; - List bgrPlanes1 = new ArrayList<>(), bgrPlanes2 = new ArrayList<>(); - try { - mat1 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, newPixelBuffer); - mat2 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, oldPixelBuffer); - matBGR1 = new Mat(); - matBGR2 = new Mat(); - Imgproc.cvtColor(mat1, matBGR1, Imgproc.COLOR_RGBA2BGR); - Imgproc.cvtColor(mat2, matBGR2, Imgproc.COLOR_RGBA2BGR); - Core.split(matBGR1, bgrPlanes1); - Core.split(matBGR2, bgrPlanes2); - histB1 = new Mat(); histG1 = new Mat(); histR1 = new Mat(); - histB2 = new Mat(); histG2 = new Mat(); histR2 = new Mat(); - int histSize = 16; - MatOfFloat histRange = new MatOfFloat(0f, 256f); - Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(0)), new MatOfInt(0), new Mat(), histB1, new MatOfInt(histSize), histRange); - Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(1)), new MatOfInt(0), new Mat(), histG1, new MatOfInt(histSize), histRange); - Imgproc.calcHist(Collections.singletonList(bgrPlanes1.get(2)), new MatOfInt(0), new Mat(), histR1, new MatOfInt(histSize), histRange); - Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(0)), new MatOfInt(0), new Mat(), histB2, new MatOfInt(histSize), histRange); - Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(1)), new MatOfInt(0), new Mat(), histG2, new MatOfInt(histSize), histRange); - Imgproc.calcHist(Collections.singletonList(bgrPlanes2.get(2)), new MatOfInt(0), new Mat(), histR2, new MatOfInt(histSize), histRange); - Core.normalize(histB1, histB1, 0, 1, Core.NORM_MINMAX); - Core.normalize(histG1, histG1, 0, 1, Core.NORM_MINMAX); - Core.normalize(histR1, histR1, 0, 1, Core.NORM_MINMAX); - Core.normalize(histB2, histB2, 0, 1, Core.NORM_MINMAX); - Core.normalize(histG2, histG2, 0, 1, Core.NORM_MINMAX); - Core.normalize(histR2, histR2, 0, 1, Core.NORM_MINMAX); - double corrB = Imgproc.compareHist(histB1, histB2, Imgproc.HISTCMP_CORREL); - double corrG = Imgproc.compareHist(histG1, histG2, Imgproc.HISTCMP_CORREL); - double corrR = Imgproc.compareHist(histR1, histR2, Imgproc.HISTCMP_CORREL); - return 1f - Math.max(0, (corrB + corrG + corrR) / 3.0); - } finally { - if (mat1 != null) mat1.release(); - if (mat2 != null) mat2.release(); - if (matBGR1 != null) matBGR1.release(); - if (matBGR2 != null) matBGR2.release(); - if (histB1 != null) histB1.release(); - if (histG1 != null) histG1.release(); - if (histR1 != null) histR1.release(); - if (histB2 != null) histB2.release(); - if (histG2 != null) histG2.release(); - if (histR2 != null) histR2.release(); - for (Mat m : bgrPlanes1) if (m != null) m.release(); - for (Mat m : bgrPlanes2) if (m != null) m.release(); - } - } - - private double hasFrameChangedSignificantlyOCV(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { - if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { - return 1.0; - } - Mat mat1 = null, mat2 = null, gray1 = null, gray2 = null, edges1 = null, edges2 = null, - histGray1 = null, histGray2 = null, histEdge1 = null, histEdge2 = null; - try { - mat1 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, newPixelBuffer); - mat2 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, oldPixelBuffer); - gray1 = new Mat(); - gray2 = new Mat(); - Imgproc.cvtColor(mat1, gray1, Imgproc.COLOR_RGBA2GRAY); - Imgproc.cvtColor(mat2, gray2, Imgproc.COLOR_RGBA2GRAY); - edges1 = new Mat(); - edges2 = new Mat(); - Mat gradX1 = new Mat(), gradY1 = new Mat(), gradX2 = new Mat(), gradY2 = new Mat(); - Imgproc.Sobel(gray1, gradX1, CvType.CV_16S, 1, 0); - Imgproc.Sobel(gray1, gradY1, CvType.CV_16S, 0, 1); - Core.convertScaleAbs(gradX1, gradX1); - Core.convertScaleAbs(gradY1, gradY1); - Core.addWeighted(gradX1, 0.5, gradY1, 0.5, 0, edges1); - Imgproc.Sobel(gray2, gradX2, CvType.CV_16S, 1, 0); - Imgproc.Sobel(gray2, gradY2, CvType.CV_16S, 0, 1); - Core.convertScaleAbs(gradX2, gradX2); - Core.convertScaleAbs(gradY2, gradY2); - Core.addWeighted(gradX2, 0.5, gradY2, 0.5, 0, edges2); - gradX1.release(); gradY1.release(); gradX2.release(); gradY2.release(); - histGray1 = new Mat(); histGray2 = new Mat(); - Imgproc.calcHist(Collections.singletonList(gray1), new MatOfInt(0), new Mat(), histGray1, new MatOfInt(256), new MatOfFloat(0f, 256f)); - Imgproc.calcHist(Collections.singletonList(gray2), new MatOfInt(0), new Mat(), histGray2, new MatOfInt(256), new MatOfFloat(0f, 256f)); - histEdge1 = new Mat(); histEdge2 = new Mat(); - Imgproc.calcHist(Collections.singletonList(edges1), new MatOfInt(0), new Mat(), histEdge1, new MatOfInt(256), new MatOfFloat(0f, 256f)); - Imgproc.calcHist(Collections.singletonList(edges2), new MatOfInt(0), new Mat(), histEdge2, new MatOfInt(256), new MatOfFloat(0f, 256f)); - double grayDiff = 1.0 - Imgproc.compareHist(histGray1, histGray2, Imgproc.HISTCMP_CORREL); - double edgeDiff = 1.0 - Imgproc.compareHist(histEdge1, histEdge2, Imgproc.HISTCMP_CORREL); - return 0.5 * grayDiff + 0.5 * edgeDiff; - } finally { - if (mat1 != null) mat1.release(); if (mat2 != null) mat2.release(); - if (gray1 != null) gray1.release(); if (gray2 != null) gray2.release(); - if (edges1 != null) edges1.release(); if (edges2 != null) edges2.release(); - if (histGray1 != null) histGray1.release(); if (histGray2 != null) histGray2.release(); - if (histEdge1 != null) histEdge1.release(); if (histEdge2 != null) histEdge2.release(); - } - } - private double hasSceneChangedFast(ByteBuffer currentFrame, ByteBuffer previousFrame) { if (currentFrame == null || previousFrame == null || currentFrame.capacity() != previousFrame.capacity()) { return 0.0; } + currentFrame.rewind(); previousFrame.rewind(); + long totalDifference = 0; int pixelsSampled = 0; - final int PIXEL_STRIDE = 4; - final int PIXEL_SAMPLE_RATE = 32; - final int ROW_SAMPLE_RATE = 32; + + final int PIXEL_STRIDE = 3; // RGB = 3 bytes per pixel + final int PIXEL_SAMPLE_RATE = 32; // horizontal sampling + final int ROW_SAMPLE_RATE = 32; // vertical sampling final int SAMPLE_STRIDE = PIXEL_STRIDE * PIXEL_SAMPLE_RATE; final int ROW_STRIDE = modelInputWidth * PIXEL_STRIDE * ROW_SAMPLE_RATE; - for (int row = 0; row < currentFrame.capacity(); row += ROW_STRIDE) { + final int frameCapacity = currentFrame.capacity(); + + for (int row = 0; row < frameCapacity; row += ROW_STRIDE) { for (int col = 0; col < modelInputWidth * PIXEL_STRIDE; col += SAMPLE_STRIDE) { int index = row + col; - if (index + 2 >= currentFrame.capacity()) break; - totalDifference += Math.abs((currentFrame.get(index) & 0xFF) - (previousFrame.get(index) & 0xFF)); - totalDifference += Math.abs((currentFrame.get(index + 1) & 0xFF) - (previousFrame.get(index + 1) & 0xFF)); - totalDifference += Math.abs((currentFrame.get(index + 2) & 0xFF) - (previousFrame.get(index + 2) & 0xFF)); + if (index + 2 >= frameCapacity) break; + + int rDiff = Math.abs((currentFrame.get(index) & 0xFF) - (previousFrame.get(index) & 0xFF)); + int gDiff = Math.abs((currentFrame.get(index + 1) & 0xFF) - (previousFrame.get(index + 1) & 0xFF)); + int bDiff = Math.abs((currentFrame.get(index + 2) & 0xFF) - (previousFrame.get(index + 2) & 0xFF)); + + totalDifference += rDiff + gDiff + bDiff; pixelsSampled++; } } + if (pixelsSampled == 0) return 0.0; return (double) totalDifference / pixelsSampled; } + private class AiTask implements Runnable { private ByteBuffer previousRawMap = null; + private int lastFrameId = -1; // Track last processed frame ID + @Override public void run() { ByteBuffer pixelBuffer = null; double difference = 0.0f; + while (!Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); long waitTime = System.nanoTime(); + long waitTime_end = System.nanoTime(); long aiTime = System.nanoTime(); long aiTime_end = System.nanoTime(); + try { if (tflite == null) return; + + // --- Take latest frame from queue --- RenderResult result = inferenceInputQueue.take(); + waitTime_end = System.nanoTime(); pixelBuffer = result.pixelBuffer; difference = result.imageDifference; + int currentFrameId = result.frameId; + + // Skip if this frame was already processed + if (currentFrameId == lastFrameId) { + // Reuse previous map + ByteBuffer outputBuffer = freeOutputBuffers.take(); + outputBuffer.clear(); + previousRawMap.rewind(); + outputBuffer.put(previousRawMap); + outputBuffer.rewind(); + filledOutputBuffers.put(new InferenceResult(pixelBuffer, outputBuffer)); + pixelBuffer = null; + Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: lastFrameId"); + continue; + } + + lastFrameId = currentFrameId; + + // --- Process new frame --- ByteBuffer outputBuffer = freeOutputBuffers.take(); - waitTime = System.nanoTime(); outputBuffer.rewind(); + if (difference > ON_DRAW_CHANGE_TRESHOLD || previousRawMap == null) { tfliteInputBuffer.rewind(); pixelBuffer.rewind(); - convertRgbaToRgb(pixelBuffer, tfliteInputBuffer, modelInputWidth, modelInputHeight); + tfliteInputBuffer.put(pixelBuffer); + tfliteInputBuffer.rewind(); aiTime = System.nanoTime(); tflite.run(tfliteInputBuffer, outputBuffer); - aiTime_end = System.currentTimeMillis(); + aiTime_end = System.nanoTime(); + + // Store result for reuse if (previousRawMap == null) { previousRawMap = ByteBuffer.allocateDirect(outputBuffer.capacity()); } @@ -1034,17 +976,18 @@ public void run() { previousRawMap.put(outputBuffer); previousRawMap.rewind(); } else { + // No significant change: reuse previous map outputBuffer.clear(); previousRawMap.rewind(); outputBuffer.put(previousRawMap); outputBuffer.rewind(); } calcThreeDFps++; - aiTime_end = System.nanoTime(); filledOutputBuffers.put(new InferenceResult(pixelBuffer, outputBuffer)); pixelBuffer = null; + } catch (InterruptedException e) { - LimeLog.severe("AI inference failed: " + e.getMessage()); + LimeLog.severe("AI inference interrupted: " + e.getMessage()); Thread.currentThread().interrupt(); } catch (Exception e) { LimeLog.severe("AI inference failed: " + e.getMessage()); @@ -1053,17 +996,23 @@ public void run() { if (pixelBuffer != null) { freeInputBuffers.offer(pixelBuffer); } + long duration = (System.nanoTime() - startTime) / 1_000_000; - long waitTimeText = (waitTime - startTime) / 1_000_000; + long waitTimeText = (waitTime_end - waitTime) / 1_000_000; long aitimeText = (aiTime_end - aiTime) / 1_000_000; - long restTimeText = ((System.nanoTime() - startTime) - (aiTime_end - aiTime)) / 1_000_000; - Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + " ms waitTime: " + waitTimeText + " ms " + "aitime: " + aitimeText + " otherTime: " +restTimeText); + long restTimeText = ((System.nanoTime() - startTime) - (aiTime_end - aiTime)- (waitTime_end - waitTime)) / 1_000_000; + + Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + + " ms waitTime: " + waitTimeText + + " ms aitime: " + aitimeText + + " otherTime: " + restTimeText); } } isAiRunning.set(false); } } + private class AiResultHandling implements Runnable { private static final double DEPTH_DIFF_THRESHOLD = 0.1; private static final double MAX_SMOOTHING = 1.0; @@ -1083,16 +1032,19 @@ private class AiResultHandling implements Runnable { @Override public void run() { - ByteBuffer resultBuffer = createFlatDepthMap(); // Initialer flacher Puffer + ByteBuffer resultBuffer = createFlatDepthMap(); InferenceResult result = null; - // Markierung, ob der Thread noch laufen soll while (isAiResultHandlingRunning.get() && !Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); + long waitTime = System.nanoTime(); + long waitTime_end = System.nanoTime(); + long aiTime = System.nanoTime(); + long aiTime_end = System.nanoTime(); try { result = filledOutputBuffers.take(); // Blockiert, bis Ergebnis da ist resultBuffer = freeSmoothedBuffers.take(); // Blockiert, bis Puffer frei ist - + waitTime_end = System.nanoTime(); // Verarbeite nur das letzte verfügbare Ergebnis InferenceResult intermediate; while ((intermediate = filledOutputBuffers.poll()) != null) { @@ -1120,6 +1072,7 @@ public void run() { LimeLog.warning("previousSmoothedMat war null/leer, neu initialisiert in Schleife."); } + aiTime = System.nanoTime(); // --- Differenzberechnung --- Core.absdiff(reusableRawFloat, previousSmoothedMat, reusableDiffMat); Scalar sumDiff = Core.sumElems(reusableDiffMat); @@ -1167,18 +1120,17 @@ public void run() { reusableNormalizedForShader.convertTo(reusableOutputMat, CvType.CV_8U, 255.0); - // --- Ergebnis in ByteBuffer kopieren --- if (reusableOutputMat.isContinuous() && resultBuffer.hasArray()) { reusableOutputMat.get(0, 0, resultBuffer.array()); // Wichtig: Limit muss evtl. angepasst werden, wenn Puffer größer ist resultBuffer.limit(reusableOutputMat.rows() * reusableOutputMat.cols() * (int)reusableOutputMat.elemSize()); } else { - // Langsamerer Fallback, falls nicht kontinuierlich oder kein Array hat int bufferSize = modelInputWidth * modelInputHeight; byte[] data = new byte[bufferSize]; reusableOutputMat.get(0, 0, data); resultBuffer.put(data); } + aiTime_end = System.nanoTime(); resultBuffer.rewind(); // Puffer für den Konsumenten vorbereiten latestDepthMap.set(resultBuffer); @@ -1199,10 +1151,15 @@ public void run() { freeOutputBuffers.offer(result.rawDepthBuffer); result = null; // Referenz löschen } - // Die wiederverwendeten Mat-Objekte werden NICHT hier freigegeben - long duration = (System.nanoTime() - startTime) / 1_000_000; - Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + " ms"); + long waitTimeText = (waitTime_end - waitTime) / 1_000_000; + long aitimeText = (aiTime_end - aiTime) / 1_000_000; + long restTimeText = ((System.nanoTime() - startTime) - (aiTime_end - aiTime)- (waitTime_end - waitTime)) / 1_000_000; + + Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + + " ms waitTime: " + waitTimeText + + " ms calc time: " + aitimeText + + " otherTime: " + restTimeText); } } // Ende while-Schleife diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 95f64c92a5..ccdde68ed0 100755 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -503,6 +503,8 @@ Fully External Display Mode (Beta) Stream display on the monitor, virtual buttons and performance information and other controls on the phone screen. Allows full immersive secondary screen mode with automatically matched refresh rate and resolution. Manual settings won\'t apply in this mode. + External Display - auto configuration + Resolution, FrameRate and Fit will be automatically picked by externals display actively supported connection. Display in Top Center Streaming picture will be aligned to the top of the screen instead of centered when resolution is not native. Touch Screen Sensitivity diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 8d8c713a2c..01498e1241 100755 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -163,6 +163,19 @@ android:summary="@string/summary_checkbox_enforce_display_mode" android:title="@string/title_checkbox_enforce_display_mode" app:iconSpaceReserved="false" /> + + - - Date: Mon, 20 Oct 2025 11:06:13 +0200 Subject: [PATCH 10/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/Game.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index f6229c56b3..e1d8fd527c 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -697,12 +697,12 @@ public void notifyCrash(Exception e) { decoderRenderer.setPreferLowerDelaysTimeoutUs(500); // 0.5 ms prefConfig.framePacing = PreferenceConfiguration.FRAME_PACING_BALANCED; LimeLog.info("PreferLowerDelays: preferLowerDelays=true, timeout=500us, pacing=BALANCED"); - } else { - // Balanced default + } else if(prefConfig.framePacing == PreferenceConfiguration.FRAME_PACING_BALANCED && !isOnExternalDisplay()) { decoderRenderer.setPreferLowerDelays(false); decoderRenderer.setPreferLowerDelaysTimeoutUs(2000); // 2 ms - prefConfig.framePacing = PreferenceConfiguration.FRAME_PACING_BALANCED; LimeLog.info("Balanced: preferLowerDelays=false, timeout=2000us, pacing=BALANCED"); + } else { + LimeLog.info("No balance mode selected or on external screen (LFR not working)"); } } catch (Throwable ignored) {} From 90e3d7622eff42685a2df6bd6e055594b2e84406 Mon Sep 17 00:00:00 2001 From: Janyger Date: Mon, 20 Oct 2025 11:12:47 +0200 Subject: [PATCH 11/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/Game.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index e1d8fd527c..92a7fb1568 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -406,6 +406,13 @@ protected void onCreate(Bundle savedInstanceState) { prefConfig.width = displayWidth; prefConfig.height = displayHeight; prefConfig.fps = (int) currentMode.getRefreshRate(); + } else if(prefConfig.externalScreenAutoConfig) { + // For 3d auto config would be half the width + displayWidth = currentMode.getPhysicalWidth() / 2; + displayHeight = currentMode.getPhysicalHeight(); + prefConfig.width = displayWidth; + prefConfig.height = displayHeight; + prefConfig.fps = (int) currentMode.getRefreshRate(); } displayWidth = prefConfig.width; displayHeight = prefConfig.height; From 3eb7594f8169557ee1c1d68aafb197d65cf458e9 Mon Sep 17 00:00:00 2001 From: Janyger Date: Mon, 20 Oct 2025 11:14:12 +0200 Subject: [PATCH 12/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/Game.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 92a7fb1568..af1fe57330 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -400,15 +400,13 @@ protected void onCreate(Bundle savedInstanceState) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && onExternelDisplay) { Display.Mode currentMode = currentDisplay.getMode(); - if(prefConfig.externalScreenAutoConfig && prefConfig.renderMode == 0) { - displayWidth = currentMode.getPhysicalWidth(); - displayHeight = currentMode.getPhysicalHeight(); - prefConfig.width = displayWidth; - prefConfig.height = displayHeight; - prefConfig.fps = (int) currentMode.getRefreshRate(); - } else if(prefConfig.externalScreenAutoConfig) { - // For 3d auto config would be half the width - displayWidth = currentMode.getPhysicalWidth() / 2; + if(prefConfig.externalScreenAutoConfig) { + if(prefConfig.renderMode == 0) { + displayWidth = currentMode.getPhysicalWidth(); + } else { + // For 3d auto config would be half the width + displayWidth = currentMode.getPhysicalWidth() / 2; + } displayHeight = currentMode.getPhysicalHeight(); prefConfig.width = displayWidth; prefConfig.height = displayHeight; From 3af8fec616bf6668241cbbf8c9cb210e471a35cb Mon Sep 17 00:00:00 2001 From: Janyger Date: Mon, 20 Oct 2025 12:35:59 +0200 Subject: [PATCH 13/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/Game.java | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index af1fe57330..9d1c15d33b 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -404,8 +404,24 @@ protected void onCreate(Bundle savedInstanceState) { if(prefConfig.renderMode == 0) { displayWidth = currentMode.getPhysicalWidth(); } else { - // For 3d auto config would be half the width - displayWidth = currentMode.getPhysicalWidth() / 2; + float ratio = (float) currentMode.getPhysicalWidth() / (float) currentMode.getPhysicalHeight(); + + // A 32:9 aspect ratio is 3.555... + final float SBS_3D_ASPECT_RATIO = 32.0f / 9.0f; + + // Use a small tolerance for floating-point comparison + final float EPSILON = 0.01f; + // User can keep render mode 3 in its setting without the need to switch + // so plug in glasses and turn on 3d should trigger it otherwise 2dmode + if (Math.abs(ratio - SBS_3D_ASPECT_RATIO) < EPSILON) { + // This is a 32:9 SbS 3D mode (like 3840x1080). + // We set the displayWidth to be for a single eye (1920). + displayWidth = currentMode.getPhysicalWidth() / 2; + } else { + // This is a standard 16:9, 4:3, etc. mode. Use the full width. + displayWidth = currentMode.getPhysicalWidth(); + prefConfig.renderMode = 0; + } } displayHeight = currentMode.getPhysicalHeight(); prefConfig.width = displayWidth; From 683bb86ff73dbe20a32cc5c8ee17a328a1484ac5 Mon Sep 17 00:00:00 2001 From: Janyger Date: Mon, 20 Oct 2025 12:37:07 +0200 Subject: [PATCH 14/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/utils/Stereo3DRenderer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index 11b9a1faa2..59e3fbcfca 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -370,7 +370,7 @@ private void initializeDilationFbo() { } private float getParallax() { - return prefConfig.parallax_depth * 0.25f; + return prefConfig.parallax_depth * 0.18f; } /** From 652257e60af17d24d48e4f31697f9302345b0b3f Mon Sep 17 00:00:00 2001 From: Janyger Date: Mon, 20 Oct 2025 12:45:45 +0200 Subject: [PATCH 15/22] Improvements 3d - dilute --- app/src/main/java/com/limelight/Game.java | 3 +++ app/src/main/res/values/strings.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 9d1c15d33b..aedb0c47f8 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -439,6 +439,9 @@ protected void onCreate(Bundle savedInstanceState) { } else { if (prefConfig.renderMode != 0) { prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; + if(prefConfig.externalScreenAutoConfig) { + prefConfig.renderMode = 0; + } } if (prefConfig.autoOrientation) { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index ccdde68ed0..0740d3b9c9 100755 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -504,7 +504,7 @@ Stream display on the monitor, virtual buttons and performance information and other controls on the phone screen. Allows full immersive secondary screen mode with automatically matched refresh rate and resolution. Manual settings won\'t apply in this mode. External Display - auto configuration - Resolution, FrameRate and Fit will be automatically picked by externals display actively supported connection. + Resolution, FrameRate, Fit and RenderMode will be automatically picked by externals display actively supported connection. If glasses or monitor are in 2d mode, renderMode 3d will be ignored. Display in Top Center Streaming picture will be aligned to the top of the screen instead of centered when resolution is not native. Touch Screen Sensitivity From 5693e7eed452a337d3c12ffd73cfac3b92b77b1c Mon Sep 17 00:00:00 2001 From: Janyger Date: Wed, 22 Oct 2025 12:24:46 +0200 Subject: [PATCH 16/22] Auto Configuration of resolution and fps --- app/src/main/java/com/limelight/Game.java | 84 +++++++++-------- .../preferences/PreferenceConfiguration.java | 4 - .../limelight/preferences/StreamSettings.java | 13 ++- .../com/limelight/utils/DisplayUtils.java | 94 +++++++++++++++++++ .../com/limelight/utils/Stereo3DRenderer.java | 2 +- app/src/main/res/values/arrays.xml | 4 + app/src/main/res/values/strings.xml | 4 +- app/src/main/res/xml/preferences.xml | 7 -- 8 files changed, 159 insertions(+), 53 deletions(-) create mode 100644 app/src/main/java/com/limelight/utils/DisplayUtils.java diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index aedb0c47f8..7fdc8846ba 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -3,6 +3,7 @@ import static com.limelight.StartExternalDisplayControlReceiver.requestFocusToExternalDisplayControl; import static com.limelight.binding.input.KeyboardTranslator.getModifier; +import static com.limelight.utils.DisplayUtils.getDisplayInfo; import static com.limelight.utils.ExternalDisplayControlActivity.SECONDARY_SCREEN_NOTIFICATION_ID; import static com.limelight.utils.ExternalDisplayControlActivity.closeExternalDisplayControl; import static com.limelight.utils.ServerHelper.getActiveDisplay; @@ -44,6 +45,7 @@ import com.limelight.ui.GameGestures; import com.limelight.ui.StreamContainer; import com.limelight.utils.Dialog; +import com.limelight.utils.DisplayUtils; import com.limelight.utils.ExternalDisplayControlActivity; import com.limelight.utils.MouseModeOption; import com.limelight.utils.PanZoomHandler; @@ -395,39 +397,10 @@ protected void onCreate(Bundle savedInstanceState) { } onExternelDisplay = currentDisplay.getDisplayId() != Display.DEFAULT_DISPLAY; - boolean shouldInvertDecoderResolution = false; + prepareResolutionAndFps(currentDisplay); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && onExternelDisplay) { - Display.Mode currentMode = currentDisplay.getMode(); - if(prefConfig.externalScreenAutoConfig) { - if(prefConfig.renderMode == 0) { - displayWidth = currentMode.getPhysicalWidth(); - } else { - float ratio = (float) currentMode.getPhysicalWidth() / (float) currentMode.getPhysicalHeight(); - - // A 32:9 aspect ratio is 3.555... - final float SBS_3D_ASPECT_RATIO = 32.0f / 9.0f; - - // Use a small tolerance for floating-point comparison - final float EPSILON = 0.01f; - // User can keep render mode 3 in its setting without the need to switch - // so plug in glasses and turn on 3d should trigger it otherwise 2dmode - if (Math.abs(ratio - SBS_3D_ASPECT_RATIO) < EPSILON) { - // This is a 32:9 SbS 3D mode (like 3840x1080). - // We set the displayWidth to be for a single eye (1920). - displayWidth = currentMode.getPhysicalWidth() / 2; - } else { - // This is a standard 16:9, 4:3, etc. mode. Use the full width. - displayWidth = currentMode.getPhysicalWidth(); - prefConfig.renderMode = 0; - } - } - displayHeight = currentMode.getPhysicalHeight(); - prefConfig.width = displayWidth; - prefConfig.height = displayHeight; - prefConfig.fps = (int) currentMode.getRefreshRate(); - } + if (onExternelDisplay) { displayWidth = prefConfig.width; displayHeight = prefConfig.height; prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; @@ -437,13 +410,6 @@ protected void onCreate(Bundle savedInstanceState) { currentOrientation = Configuration.ORIENTATION_LANDSCAPE; setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_USER_LANDSCAPE); } else { - if (prefConfig.renderMode != 0) { - prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; - if(prefConfig.externalScreenAutoConfig) { - prefConfig.renderMode = 0; - } - } - if (prefConfig.autoOrientation) { currentOrientation = getResources().getConfiguration().orientation; } else { @@ -969,6 +935,48 @@ public void notifyCrash(Exception e) { } catch (Throwable ignored) {} } + private void prepareResolutionAndFps(Display display) { + DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(display); + + if(isMatchDisplayFPS()) { + prefConfig.fps = (int) displayInfo.refreshRate; + } + if(prefConfig.renderMode != 0) { // 3D Mode selected + float ratio = (float) displayInfo.width / (float) displayInfo.height; + + // A 32:9 aspect ratio is 3.555... + final float SBS_3D_ASPECT_RATIO = 32.0f / 9.0f; + + // Use a small tolerance for floating-point comparison + final float EPSILON = 0.01f; + // User can keep render mode 3 in its setting without the need to switch + // so plug in glasses and turn on 3d should trigger it otherwise 2dmode + if (Math.abs(ratio - SBS_3D_ASPECT_RATIO) < EPSILON) { + // This is a 32:9 SbS 3D mode (like 3840x1080). + // We set the displayWidth to be for a single eye (1920). + if(isMatchDisplayResolution()) { + prefConfig.width = displayInfo.width / 2; + prefConfig.height = displayInfo.height; + } + } else { + // This is a standard 16:9, 4:3, etc. mode. No 3d needed + prefConfig.renderMode = 0; + } + } + if(isMatchDisplayResolution() && prefConfig.renderMode == 0) { + prefConfig.width = displayInfo.width; + prefConfig.height = displayInfo.height; + } + } + + private boolean isMatchDisplayResolution() { + return prefConfig.width == 0 && prefConfig.height == 0; + } + + private boolean isMatchDisplayFPS() { + return prefConfig.fps == 0; + } + @SuppressLint("ClickableViewAccessibility") private void setupOverlayToggleButton() { if (overlayToggleButton != null) { diff --git a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java index 96528b5b3b..4501d54dd4 100755 --- a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java +++ b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java @@ -117,8 +117,6 @@ public enum AnalogStickForScrolling { private static final String CHECKBOX_SHOW_OVERLAY_ZOOM_TOGGLE_BUTTON = "checkbox_show_overlay_zoom_toggle_button"; - private static final String CHECKBOX_ENABLE_FULLEXDISPLAY_AUTO = "checkbox_enable_fullexdisplay_auto"; - //竖屏模式 private static final String CHECKBOX_AUTO_ORIENTATION = "checkbox_auto_orientation"; //屏幕特殊按键 @@ -284,7 +282,6 @@ public enum AnalogStickForScrolling { public boolean enableBackMenu; public boolean enableFloatingButton; public boolean showOverlayZoomToggleButton; - public boolean externalScreenAutoConfig; //Invert video width/height public boolean autoInvertVideoResolution; public int resolutionScaleFactor; @@ -940,7 +937,6 @@ else if (audioConfig.equals("51")) { config.enableBackMenu = prefs.getBoolean(CHECKBOX_ENABLE_QUIT_DIALOG,true); config.enableFloatingButton = prefs.getBoolean(CHECKBOX_ENABLE_FLOATING_BUTTON,DEFAULT_ENABLE_FLOATING_BUTTON); config.showOverlayZoomToggleButton = prefs.getBoolean(CHECKBOX_SHOW_OVERLAY_ZOOM_TOGGLE_BUTTON, DEFAULT_SHOW_OVERLAY_TOGGLE_BUTTON); - config.externalScreenAutoConfig = prefs.getBoolean(CHECKBOX_ENABLE_FULLEXDISPLAY_AUTO, true); config.autoOrientation = prefs.getBoolean(CHECKBOX_AUTO_ORIENTATION,false); config.autoInvertVideoResolution = prefs.getBoolean(AUTO_INVERT_VIDEO_RESOLUTION_PREF_STRING, DEFAULT_AUTO_INVERT_VIDEO_RESOLUTION); config.resolutionScaleFactor = prefs.getInt(RESOLUTION_SCALE_FACTOR_PREF_STRING, DEFAULT_RESOLUTION_SCALE_FACTOR); diff --git a/app/src/main/java/com/limelight/preferences/StreamSettings.java b/app/src/main/java/com/limelight/preferences/StreamSettings.java index 38175463fa..e3e5a28878 100755 --- a/app/src/main/java/com/limelight/preferences/StreamSettings.java +++ b/app/src/main/java/com/limelight/preferences/StreamSettings.java @@ -1,5 +1,7 @@ package com.limelight.preferences; +import static com.limelight.preferences.PreferenceConfiguration.DEFAULT_FPS; +import static com.limelight.utils.DisplayUtils.getDisplayInfo; import static com.limelight.utils.ServerHelper.getActiveDisplay; import android.content.Context; @@ -53,6 +55,7 @@ import com.limelight.binding.input.virtual_controller.keyboard.KeyBoardControllerConfigurationLoader; import com.limelight.binding.video.MediaCodecHelper; import com.limelight.utils.Dialog; +import com.limelight.utils.DisplayUtils; import com.limelight.utils.FileUriUtils; import com.limelight.utils.PerformanceDataTracker; import com.limelight.utils.UiHelper; @@ -306,9 +309,17 @@ private void resetBitrateToDefault(SharedPreferences prefs, String res, String f res = prefs.getString(PreferenceConfiguration.RESOLUTION_PREF_STRING, PreferenceConfiguration.DEFAULT_RESOLUTION); } if (fps == null) { - fps = prefs.getString(PreferenceConfiguration.FPS_PREF_STRING, PreferenceConfiguration.DEFAULT_FPS); + fps = prefs.getString(PreferenceConfiguration.FPS_PREF_STRING, DEFAULT_FPS); } + // If MatchDisplayRes/FPS is selected we need the actual values + DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(getActiveDisplay(getContext())); + if(res.equals("0x0")) { + res = displayInfo.width + "x" +displayInfo.height; + } + if(fps.equals("0")) { + fps = displayInfo.refreshRate + ""; + } prefs.edit() .putInt(PreferenceConfiguration.BITRATE_PREF_STRING, PreferenceConfiguration.getDefaultBitrate(res, fps)) diff --git a/app/src/main/java/com/limelight/utils/DisplayUtils.java b/app/src/main/java/com/limelight/utils/DisplayUtils.java new file mode 100644 index 0000000000..03237b8580 --- /dev/null +++ b/app/src/main/java/com/limelight/utils/DisplayUtils.java @@ -0,0 +1,94 @@ +package com.limelight.utils; + +import android.graphics.Point; +import android.os.Build; +import android.view.Display; +import android.annotation.TargetApi; // Optional, for clarity + +/** + * Utility class for display information. + */ +public class DisplayUtils { + + /** + * Simple data class to hold display information. + */ + public static class DisplayInfo { + public final int width; // Guaranteed landscape width + public final int height; // Guaranteed landscape height + public final float refreshRate; + + public DisplayInfo(int width, int height, float refreshRate) { + this.width = width; + this.height = height; + this.refreshRate = refreshRate; + } + + @Override + public String toString() { + return "DisplayInfo{" + + "width=" + width + + ", height=" + height + + ", refreshRate=" + refreshRate + + '}'; + } + } + + /** + * Gets the current display's physical dimensions (guaranteed landscape) and refresh rate. + * Handles different Android API levels. + * + * @param display The Display object to query. + * @return A DisplayInfo object containing width, height, and refresh rate, + * or null if the display object is null. + */ + public static DisplayInfo getDisplayInfo(Display display) { + int oneAxeLength = 0; + int secondAxeLength = 0; + float displayRefreshRate = 0f; // Initialize with a default + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + // API 23+ (Marshmallow) - Use Display.Mode + Display.Mode currentMode = display.getMode(); + if (currentMode != null) { + oneAxeLength = currentMode.getPhysicalWidth(); + secondAxeLength = currentMode.getPhysicalHeight(); + displayRefreshRate = currentMode.getRefreshRate(); + } else { + // Fallback if getMode() surprisingly returns null + getLegacyDisplayInfo(display, sizePoint); + oneAxeLength = sizePoint.x; + secondAxeLength = sizePoint.y; + displayRefreshRate = display.getRefreshRate(); // Still try to get refresh rate + } + } else { + // API < 23 (pre-Marshmallow) - Use deprecated methods + getLegacyDisplayInfo(display, sizePoint); + oneAxeLength = sizePoint.x; + secondAxeLength = sizePoint.y; + displayRefreshRate = display.getRefreshRate(); + } + + // Ensure landscape dimensions regardless of current orientation + int physicalWidth = Math.max(oneAxeLength, secondAxeLength); + int physicalHeight = Math.min(oneAxeLength, secondAxeLength); + + return new DisplayInfo(physicalWidth, physicalHeight, displayRefreshRate); + } + + // Helper for older APIs (using a static Point to avoid allocations in loops if called often) + private static final Point sizePoint = new Point(); + private static synchronized void getLegacyDisplayInfo(Display display, Point outSize) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { + // getRealSize() is API 17+ + display.getRealSize(outSize); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB_MR2) { + // getSize() is API 13+ (slightly less accurate, might exclude nav bar) + display.getSize(outSize); + } else { + // Very old fallback (even less accurate) + outSize.x = display.getWidth(); // Deprecated in API 13 + outSize.y = display.getHeight(); // Deprecated in API 13 + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index 59e3fbcfca..af3f4410a2 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -370,7 +370,7 @@ private void initializeDilationFbo() { } private float getParallax() { - return prefConfig.parallax_depth * 0.18f; + return prefConfig.parallax_depth * 0.10f; } /** diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index 763ed88197..8cd1700d91 100755 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -1,6 +1,7 @@ + @string/resolution_auto @string/resolution_360p @string/resolution_480p @string/resolution_720p @@ -11,6 +12,7 @@ + 0x0 640x360 854x480 1280x720 @@ -20,12 +22,14 @@ + @string/fps_auto @string/fps_30 @string/fps_60 @string/fps_90 @string/fps_120 + 0 30 60 90 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 0740d3b9c9..b0c908382c 100755 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -331,6 +331,7 @@ Gamepad type may be changed due to motion sensor emulation + Match Display Resolution 360p 480p 720p @@ -338,6 +339,7 @@ 1440p 4K + Match Display FPS 30 FPS 60 FPS 90 FPS @@ -503,8 +505,6 @@ Fully External Display Mode (Beta) Stream display on the monitor, virtual buttons and performance information and other controls on the phone screen. Allows full immersive secondary screen mode with automatically matched refresh rate and resolution. Manual settings won\'t apply in this mode. - External Display - auto configuration - Resolution, FrameRate, Fit and RenderMode will be automatically picked by externals display actively supported connection. If glasses or monitor are in 2d mode, renderMode 3d will be ignored. Display in Top Center Streaming picture will be aligned to the top of the screen instead of centered when resolution is not native. Touch Screen Sensitivity diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 01498e1241..39015d2a4c 100755 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -169,13 +169,6 @@ android:summary="@string/summary_fullexdisplay_mode" android:title="@string/title_fullexdisplay_mode" app:iconSpaceReserved="false" /> - Date: Thu, 23 Oct 2025 19:53:29 +0200 Subject: [PATCH 17/22] Auto Configuration of resolution and fps --- app/src/main/java/com/limelight/Game.java | 11 ++++-- .../preferences/PreferenceConfiguration.java | 23 +++++++++--- .../limelight/preferences/StreamSettings.java | 35 +++++++++++-------- app/src/main/res/values/strings.xml | 3 +- app/src/main/res/xml/preferences.xml | 2 +- 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 7fdc8846ba..807824da8b 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -398,7 +398,7 @@ protected void onCreate(Bundle savedInstanceState) { onExternelDisplay = currentDisplay.getDisplayId() != Display.DEFAULT_DISPLAY; boolean shouldInvertDecoderResolution = false; - prepareResolutionAndFps(currentDisplay); + matchSettings(currentDisplay); if (onExternelDisplay) { displayWidth = prefConfig.width; @@ -935,7 +935,7 @@ public void notifyCrash(Exception e) { } catch (Throwable ignored) {} } - private void prepareResolutionAndFps(Display display) { + private void matchSettings(Display display) { DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(display); if(isMatchDisplayFPS()) { @@ -967,12 +967,17 @@ private void prepareResolutionAndFps(Display display) { prefConfig.width = displayInfo.width; prefConfig.height = displayInfo.height; } + if(isMatchBitrate()) { + prefConfig.bitrate = PreferenceConfiguration.getDefaultBitrate(prefConfig.width+"x"+prefConfig.height, ((int) prefConfig.fps) +"", this); + } } private boolean isMatchDisplayResolution() { return prefConfig.width == 0 && prefConfig.height == 0; } - + private boolean isMatchBitrate() { + return prefConfig.bitrate == 0; + } private boolean isMatchDisplayFPS() { return prefConfig.fps == 0; } diff --git a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java index 4501d54dd4..de657aeda7 100755 --- a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java +++ b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java @@ -1,5 +1,8 @@ package com.limelight.preferences; +import static com.limelight.utils.DisplayUtils.getDisplayInfo; +import static com.limelight.utils.ServerHelper.getActiveDisplay; + import android.content.Context; import android.content.SharedPreferences; import android.content.pm.PackageManager; @@ -8,6 +11,7 @@ import com.limelight.nvstream.jni.MoonBridge; import com.limelight.profiles.ProfilesManager; +import com.limelight.utils.DisplayUtils; public class PreferenceConfiguration { @@ -42,6 +46,8 @@ public enum AnalogStickForScrolling { static final String RESOLUTION_PREF_STRING = "list_resolution"; static final String FPS_PREF_STRING = "list_fps"; static final String BITRATE_PREF_STRING = "seekbar_bitrate_kbps"; + + static final String BITRATE_RECOMMENDATION_STRING = "bitrate_recommendation"; private static final String BITRATE_PREF_OLD_STRING = "seekbar_bitrate"; private static final String METERED_BITRATE_PREF_STRING = "seekbar_metered_bitrate_kbps"; private static final String ENABLE_ULTRA_LOW_LATENCY_PREF_STRING = "checkbox_ultra_low_latency"; @@ -491,7 +497,16 @@ private static String getResolutionString(int width, int height) { } } - public static int getDefaultBitrate(String resString, String fpsString) { + public static int getDefaultBitrate(String resString, String fpsString, Context context) { + + // If MatchDisplayRes/FPS is selected we need the actual values + DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(getActiveDisplay(context)); + if(resString.equals("0x0")) { + resString = displayInfo.width + "x" +displayInfo.height; + } + if(fpsString.equals("0")) { + fpsString = displayInfo.refreshRate + ""; + } int width = getWidthFromResolutionString(resString); int height = getHeightFromResolutionString(resString); int fps = Math.round(Float.parseFloat(fpsString)); @@ -578,7 +593,7 @@ public static int getDefaultBitrate(Context context) { SharedPreferences prefs = ProfilesManager.getInstance().getOverlayingSharedPreferences(context); return getDefaultBitrate( prefs.getString(RESOLUTION_PREF_STRING, DEFAULT_RESOLUTION), - prefs.getString(FPS_PREF_STRING, DEFAULT_FPS)); + prefs.getString(FPS_PREF_STRING, DEFAULT_FPS), context); } private static FormatOption getVideoFormatValue(Context context) { @@ -684,6 +699,7 @@ public static void resetStreamingSettings(Context context) { SharedPreferences prefs = ProfilesManager.getInstance().getOverlayingSharedPreferences(context); prefs.edit() .remove(BITRATE_PREF_STRING) + .remove(BITRATE_RECOMMENDATION_STRING) .remove(BITRATE_PREF_OLD_STRING) .remove(LEGACY_RES_FPS_PREF_STRING) .remove(RESOLUTION_PREF_STRING) @@ -831,9 +847,6 @@ else if (str.equals("4K60")) { // This must happen after the preferences migration to ensure the preferences are populated config.bitrate = prefs.getInt(BITRATE_PREF_STRING, prefs.getInt(BITRATE_PREF_OLD_STRING, 0) * 1000); - if (config.bitrate == 0) { - config.bitrate = getDefaultBitrate(context); - } config.meteredBitrate = prefs.getInt((METERED_BITRATE_PREF_STRING), 0); if (config.meteredBitrate == 0) { diff --git a/app/src/main/java/com/limelight/preferences/StreamSettings.java b/app/src/main/java/com/limelight/preferences/StreamSettings.java index e3e5a28878..5cdad7c02e 100755 --- a/app/src/main/java/com/limelight/preferences/StreamSettings.java +++ b/app/src/main/java/com/limelight/preferences/StreamSettings.java @@ -1,6 +1,8 @@ package com.limelight.preferences; +import static com.limelight.preferences.PreferenceConfiguration.BITRATE_RECOMMENDATION_STRING; import static com.limelight.preferences.PreferenceConfiguration.DEFAULT_FPS; +import static com.limelight.preferences.PreferenceConfiguration.getDefaultBitrate; import static com.limelight.utils.DisplayUtils.getDisplayInfo; import static com.limelight.utils.ServerHelper.getActiveDisplay; @@ -304,25 +306,16 @@ private void removeValue(String preferenceKey, String value, Runnable onMatched) pref.setEntryValues(entryValues); } - private void resetBitrateToDefault(SharedPreferences prefs, String res, String fps) { + private void recalculateRecommendedBitrate(SharedPreferences prefs, String res, String fps) { if (res == null) { res = prefs.getString(PreferenceConfiguration.RESOLUTION_PREF_STRING, PreferenceConfiguration.DEFAULT_RESOLUTION); } if (fps == null) { fps = prefs.getString(PreferenceConfiguration.FPS_PREF_STRING, DEFAULT_FPS); } - - // If MatchDisplayRes/FPS is selected we need the actual values - DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(getActiveDisplay(getContext())); - if(res.equals("0x0")) { - res = displayInfo.width + "x" +displayInfo.height; - } - if(fps.equals("0")) { - fps = displayInfo.refreshRate + ""; - } prefs.edit() - .putInt(PreferenceConfiguration.BITRATE_PREF_STRING, - PreferenceConfiguration.getDefaultBitrate(res, fps)) + .putInt(PreferenceConfiguration.BITRATE_RECOMMENDATION_STRING, + PreferenceConfiguration.getDefaultBitrate(res, fps, getContext())) .apply(); } @@ -707,12 +700,24 @@ public boolean onPreferenceChange(Preference preference, Object newValue) { } // Write the new bitrate value - resetBitrateToDefault(prefs, valueStr, null); + recalculateRecommendedBitrate(prefs, valueStr, null); // Allow the original preference change to take place return true; } }); + findPreference(PreferenceConfiguration.BITRATE_PREF_STRING).setOnPreferenceClickListener(new Preference.OnPreferenceClickListener() { + + @Override + public boolean onPreferenceClick(@NonNull Preference preference) { + int recommendationBitrate = getPrefs().getInt(BITRATE_RECOMMENDATION_STRING, 0); + if(recommendationBitrate != 0) { + Toast.makeText(getContext(), getString(R.string.bitrate_recommendation_toast, recommendationBitrate / 1000), Toast.LENGTH_LONG).show(); + } + return true; + } + } + ); findPreference(PreferenceConfiguration.FPS_PREF_STRING).setOnPreferenceChangeListener(new Preference.OnPreferenceChangeListener() { @Override public boolean onPreferenceChange(Preference preference, Object newValue) { @@ -729,7 +734,7 @@ public boolean onPreferenceChange(Preference preference, Object newValue) { } // Write the new bitrate value - resetBitrateToDefault(prefs, null, valueStr); + recalculateRecommendedBitrate(prefs, null, valueStr); // Allow the original preference change to take place return true; @@ -975,7 +980,7 @@ private void removeEntryFromListAndSetValue(String resolutionPrefString, String public void run() { SharedPreferences prefs = getPrefs(); setValue(resolutionPrefString, nextDefault); - resetBitrateToDefault(prefs, null, null); + recalculateRecommendedBitrate(prefs, null, null); } }); } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index b0c908382c..c0a74348e3 100755 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -192,7 +192,8 @@ Video frame rate Increase for a smoother video stream. Decrease for better performance on lower end devices. Video bitrate - Increase for better image quality. Decrease to improve performance on slower connections. + Setting based recommendation: %1$s or 0 Mbps (Automatic) + NEW: 0 for automatically calculated based on fps/resolution on stream start. Increase for better image quality. Decrease to improve performance on slower connections. Video bitrate on metered networks High bitrates consume more data, lower bitrates improve smoothness in metered networks.\nSet 0 for ¼ of the above bitrate settings. Mbps diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 39015d2a4c..f90870896d 100755 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -35,7 +35,7 @@ app:iconSpaceReserved="false" seekbar:divisor="1000" seekbar:keyStep="1000" - seekbar:min="500" + seekbar:min="0" seekbar:step="500" /> Date: Thu, 23 Oct 2025 21:00:22 +0200 Subject: [PATCH 18/22] Allow size reduction with panZoom on external screens to improve the viewing experience if the FOV is too big --- app/src/main/java/com/limelight/Game.java | 20 ++++++++++--------- .../com/limelight/utils/PanZoomHandler.java | 10 ++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 807824da8b..145cff1756 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -4038,16 +4038,18 @@ public boolean isZoomModeEnabled() { return isPanZoomMode; } public void toggleZoomMode() { - this.isPanZoomMode = !this.isPanZoomMode; - if (this.isPanZoomMode) { - Toast.makeText(this, getString(R.string.pan_zoom_mode_enabled), Toast.LENGTH_SHORT).show(); - } else { - Toast.makeText(this, getString(R.string.pan_zoom_mode_disabled), Toast.LENGTH_SHORT).show(); - } - updateZoomButtonAppearance(); + if(prefConfig.renderMode == 0) { + this.isPanZoomMode = !this.isPanZoomMode; + if (this.isPanZoomMode) { + Toast.makeText(this, getString(R.string.pan_zoom_mode_enabled), Toast.LENGTH_SHORT).show(); + } else { + Toast.makeText(this, getString(R.string.pan_zoom_mode_disabled), Toast.LENGTH_SHORT).show(); + } + updateZoomButtonAppearance(); - if (ExternalDisplayControlActivity.instance != null) { - ExternalDisplayControlActivity.instance.toggleZoomMode(false); + if (ExternalDisplayControlActivity.instance != null) { + ExternalDisplayControlActivity.instance.toggleZoomMode(false); + } } } diff --git a/app/src/main/java/com/limelight/utils/PanZoomHandler.java b/app/src/main/java/com/limelight/utils/PanZoomHandler.java index d0d3384ac0..b51209ee58 100644 --- a/app/src/main/java/com/limelight/utils/PanZoomHandler.java +++ b/app/src/main/java/com/limelight/utils/PanZoomHandler.java @@ -62,12 +62,10 @@ private void constrainToBounds() { } if (parentHeight >= childHeight) { - if (isTopMode) { - childY = 0; - } else { - childY = (parentHeight - childHeight) / 2; - } + // ALWAYS center it vertically when it's smaller than the parent + childY = (parentHeight - childHeight) / 2; } else { + // This handles panning when the view is larger than the parent float boundaryY = parentHeight - childHeight; childY = Math.max(boundaryY, Math.min(childY, 0)); } @@ -112,7 +110,7 @@ private class ScaleListener extends ScaleGestureDetector.SimpleOnScaleGestureLis @Override public boolean onScale(ScaleGestureDetector detector) { float newScaleFactor = scaleFactor * detector.getScaleFactor(); - newScaleFactor = Math.max(1, Math.min(newScaleFactor, MAX_SCALE)); // Apply minimum scale + newScaleFactor = Math.max(0.5f, Math.min(newScaleFactor, MAX_SCALE)); // Apply minimum scale // Calculate pivot point float focusX = detector.getFocusX(); From a918585602b697f955cc6df141a233df855d3263 Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 24 Oct 2025 13:01:02 +0200 Subject: [PATCH 19/22] Added MultiScreen Support for 2 internal screens. --- app/src/main/java/com/limelight/Game.java | 19 +- .../com/limelight/ShortcutTrampoline.java | 2 - .../StartExternalDisplayControlReceiver.java | 4 +- .../video/MediaCodecDecoderRenderer.java | 8 +- .../preferences/PreferenceConfiguration.java | 4 +- .../limelight/preferences/StreamSettings.java | 9 +- .../com/limelight/utils/DisplayUtils.java | 653 ++++++++++++++++-- .../utils/ExternalDisplayControlActivity.java | 29 +- .../com/limelight/utils/ServerHelper.java | 46 +- 9 files changed, 648 insertions(+), 126 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 145cff1756..c1c05ecca8 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -4,10 +4,10 @@ import static com.limelight.StartExternalDisplayControlReceiver.requestFocusToExternalDisplayControl; import static com.limelight.binding.input.KeyboardTranslator.getModifier; import static com.limelight.utils.DisplayUtils.getDisplayInfo; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; +import static com.limelight.utils.DisplayUtils.hasSecondaryDisplay; import static com.limelight.utils.ExternalDisplayControlActivity.SECONDARY_SCREEN_NOTIFICATION_ID; import static com.limelight.utils.ExternalDisplayControlActivity.closeExternalDisplayControl; -import static com.limelight.utils.ServerHelper.getActiveDisplay; -import static com.limelight.utils.ServerHelper.getSecondaryDisplay; import com.limelight.binding.PlatformBinding; import com.limelight.binding.audio.AndroidAudioRenderer; @@ -387,16 +387,19 @@ protected void onCreate(Bundle savedInstanceState) { Display currentDisplay = null; + int displayId = -2; if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - int displayId = getIntent().getIntExtra(EXTRA_DISPLAY_ID, Display.DEFAULT_DISPLAY); - currentDisplay = getSystemService(DisplayManager.class).getDisplay(displayId); + displayId = getIntent().getIntExtra(EXTRA_DISPLAY_ID, displayId); + if(displayId != -2) { + currentDisplay = getSystemService(DisplayManager.class).getDisplay(displayId); + } } if (currentDisplay == null) { currentDisplay = getWindowManager().getDefaultDisplay(); } - onExternelDisplay = currentDisplay.getDisplayId() != Display.DEFAULT_DISPLAY; + onExternelDisplay = displayId != -2; boolean shouldInvertDecoderResolution = false; matchSettings(currentDisplay); @@ -1065,7 +1068,7 @@ public void onDisplayAdded(int displayId) { @Override public void onDisplayRemoved(int displayId) { - if (getSecondaryDisplay(getBaseContext()) == null) { + if (hasSecondaryDisplay(getBaseContext())) { handleDisplayRemoved(); finish(); } @@ -1190,7 +1193,7 @@ public void toggleVirtualController(){ } private void setPreferredOrientationForActivity() { - Display display = getActiveDisplay(Game.this); + Display display = getGameStreamDisplay(Game.this); // For semi-square displays, we use more complex logic to determine which orientation to use (if any) if (PreferenceConfiguration.isSquarishScreen(display)) { @@ -1477,7 +1480,7 @@ private boolean shouldIgnoreInsetsForResolution(int width, int height) { } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display display = getActiveDisplay(Game.this); + Display display = getGameStreamDisplay(Game.this); for (Display.Mode candidate : display.getSupportedModes()) { // Ignore insets if this is an exact match for the display resolution if ((width == candidate.getPhysicalWidth() && height == candidate.getPhysicalHeight()) || diff --git a/app/src/main/java/com/limelight/ShortcutTrampoline.java b/app/src/main/java/com/limelight/ShortcutTrampoline.java index e7cd2cfefa..f543d6b6b9 100755 --- a/app/src/main/java/com/limelight/ShortcutTrampoline.java +++ b/app/src/main/java/com/limelight/ShortcutTrampoline.java @@ -1,7 +1,5 @@ package com.limelight; -import static com.limelight.utils.ServerHelper.getSecondaryDisplay; - import android.app.Activity; import android.app.Service; import android.content.ComponentName; diff --git a/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java b/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java index 4c030fa817..de7e1ab1a9 100644 --- a/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java +++ b/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java @@ -1,5 +1,7 @@ package com.limelight; +import static com.limelight.utils.DisplayUtils.getControlsDisplay; + import android.app.ActivityManager; import android.app.ActivityOptions; import android.content.BroadcastReceiver; @@ -30,7 +32,7 @@ public static void requestFocusToExternalDisplayControl(Context context) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { Intent intentTouchpad = new Intent(context, ExternalDisplayControlActivity.class); intentTouchpad.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_CLEAR_TOP); - Bundle optionsDefault = ActivityOptions.makeBasic().setLaunchDisplayId(Display.DEFAULT_DISPLAY).toBundle(); + Bundle optionsDefault = ActivityOptions.makeBasic().setLaunchDisplayId(getControlsDisplay(context).getDisplayId()).toBundle(); context.startActivity(intentTouchpad, optionsDefault); } } diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 1f528df69f..74466cc5b1 100755 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -1,6 +1,6 @@ package com.limelight.binding.video; -import static com.limelight.utils.ServerHelper.getActiveDisplay; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -799,7 +799,7 @@ public int setup(int format, int width, int height, int redrawRate) { int fpsTarget = redrawRate; if(display == null && fpsTarget <= 0) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - fpsTarget = (int) getActiveDisplay(activity).getMode().getRefreshRate(); + fpsTarget = (int) getGameStreamDisplay(activity).getMode().getRefreshRate(); } } this.targetFps = fpsTarget; @@ -1080,7 +1080,7 @@ public void doFrame(long frameTimeNanos) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { if(display == null) { - display = getActiveDisplay(activity); + display = getGameStreamDisplay(activity); } frameTimeNanos -= display.getAppVsyncOffsetNanos(); } @@ -1175,7 +1175,7 @@ public void run() { try { if (Build.VERSION.SDK_INT >= 17 && activity != null) { if(display == null) { - display = getActiveDisplay(activity); + display = getGameStreamDisplay(activity); } if (display != null) displayHz = display.getRefreshRate(); } diff --git a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java index de657aeda7..5261dc1d0c 100755 --- a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java +++ b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java @@ -1,7 +1,7 @@ package com.limelight.preferences; import static com.limelight.utils.DisplayUtils.getDisplayInfo; -import static com.limelight.utils.ServerHelper.getActiveDisplay; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import android.content.Context; import android.content.SharedPreferences; @@ -500,7 +500,7 @@ private static String getResolutionString(int width, int height) { public static int getDefaultBitrate(String resString, String fpsString, Context context) { // If MatchDisplayRes/FPS is selected we need the actual values - DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(getActiveDisplay(context)); + DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(getGameStreamDisplay(context)); if(resString.equals("0x0")) { resString = displayInfo.width + "x" +displayInfo.height; } diff --git a/app/src/main/java/com/limelight/preferences/StreamSettings.java b/app/src/main/java/com/limelight/preferences/StreamSettings.java index 5cdad7c02e..3871b25e64 100755 --- a/app/src/main/java/com/limelight/preferences/StreamSettings.java +++ b/app/src/main/java/com/limelight/preferences/StreamSettings.java @@ -2,9 +2,7 @@ import static com.limelight.preferences.PreferenceConfiguration.BITRATE_RECOMMENDATION_STRING; import static com.limelight.preferences.PreferenceConfiguration.DEFAULT_FPS; -import static com.limelight.preferences.PreferenceConfiguration.getDefaultBitrate; -import static com.limelight.utils.DisplayUtils.getDisplayInfo; -import static com.limelight.utils.ServerHelper.getActiveDisplay; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import android.content.Context; import android.content.Intent; @@ -57,7 +55,6 @@ import com.limelight.binding.input.virtual_controller.keyboard.KeyBoardControllerConfigurationLoader; import com.limelight.binding.video.MediaCodecHelper; import com.limelight.utils.Dialog; -import com.limelight.utils.DisplayUtils; import com.limelight.utils.FileUriUtils; import com.limelight.utils.PerformanceDataTracker; import com.limelight.utils.UiHelper; @@ -81,7 +78,7 @@ public class StreamSettings extends AppCompatActivity { void reloadSettings() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display.Mode mode = getActiveDisplay(StreamSettings.this).getMode(); + Display.Mode mode = getGameStreamDisplay(StreamSettings.this).getMode(); previousDisplayPixelCount = mode.getPhysicalWidth() * mode.getPhysicalHeight(); } prefsFragment = new SettingsFragment(PreferenceConfiguration.readPreferences( @@ -130,7 +127,7 @@ public void onConfigurationChanged(Configuration newConfig) { super.onConfigurationChanged(newConfig); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display.Mode mode = getActiveDisplay(StreamSettings.this).getMode(); + Display.Mode mode = getGameStreamDisplay(StreamSettings.this).getMode(); // If the display's physical pixel count has changed, we consider that it's a new display // and we should reload our settings (which include display-dependent values). diff --git a/app/src/main/java/com/limelight/utils/DisplayUtils.java b/app/src/main/java/com/limelight/utils/DisplayUtils.java index 03237b8580..73797d021f 100644 --- a/app/src/main/java/com/limelight/utils/DisplayUtils.java +++ b/app/src/main/java/com/limelight/utils/DisplayUtils.java @@ -1,94 +1,629 @@ package com.limelight.utils; +import android.app.AlertDialog; +import android.content.ClipData; +import android.content.ClipboardManager; +import android.content.Context; import android.graphics.Point; +import android.hardware.display.DisplayManager; import android.os.Build; +import android.os.Handler; +import android.os.Looper; import android.view.Display; -import android.annotation.TargetApi; // Optional, for clarity +import android.annotation.TargetApi; +import android.widget.Toast; + +import com.limelight.LimeLog; +import com.limelight.preferences.PreferenceConfiguration; + +import java.util.ArrayList; +import java.util.List; -/** - * Utility class for display information. - */ public class DisplayUtils { - /** - * Simple data class to hold display information. - */ + private static AlertDialog openDialog = null; + public static class DisplayInfo { - public final int width; // Guaranteed landscape width - public final int height; // Guaranteed landscape height + public final int width; + public final int height; public final float refreshRate; + public final long totalPixels; public DisplayInfo(int width, int height, float refreshRate) { - this.width = width; - this.height = height; + this.width = Math.max(width, height); + this.height = Math.min(width, height); this.refreshRate = refreshRate; + this.totalPixels = (long)this.width * this.height; } @Override public String toString() { - return "DisplayInfo{" + - "width=" + width + - ", height=" + height + - ", refreshRate=" + refreshRate + - '}'; + return String.format("%dx%d @ %.1f Hz", width, height, refreshRate); } } - /** - * Gets the current display's physical dimensions (guaranteed landscape) and refresh rate. - * Handles different Android API levels. - * - * @param display The Display object to query. - * @return A DisplayInfo object containing width, height, and refresh rate, - * or null if the display object is null. - */ public static DisplayInfo getDisplayInfo(Display display) { - int oneAxeLength = 0; - int secondAxeLength = 0; - float displayRefreshRate = 0f; // Initialize with a default - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - // API 23+ (Marshmallow) - Use Display.Mode - Display.Mode currentMode = display.getMode(); - if (currentMode != null) { - oneAxeLength = currentMode.getPhysicalWidth(); - secondAxeLength = currentMode.getPhysicalHeight(); - displayRefreshRate = currentMode.getRefreshRate(); + if (display == null) { + LimeLog.warning("getDisplayInfo called with null display."); + return null; + } + + int axeOneLength = 0; + int axeTwoLength = 0; + float displayRefreshRate = 0f; + + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + Display.Mode currentMode = display.getMode(); + if (currentMode != null) { + axeOneLength = currentMode.getPhysicalWidth(); + axeTwoLength = currentMode.getPhysicalHeight(); + displayRefreshRate = currentMode.getRefreshRate(); + } else { + LimeLog.warning("display.getMode() returned null on API " + Build.VERSION.SDK_INT + ". Falling back to legacy methods."); + getLegacyDisplayInfo(display, sizePoint); + axeOneLength = sizePoint.x; + axeTwoLength = sizePoint.y; + displayRefreshRate = display.getRefreshRate(); + } } else { - // Fallback if getMode() surprisingly returns null getLegacyDisplayInfo(display, sizePoint); - oneAxeLength = sizePoint.x; - secondAxeLength = sizePoint.y; - displayRefreshRate = display.getRefreshRate(); // Still try to get refresh rate + axeOneLength = sizePoint.x; + axeTwoLength = sizePoint.y; + displayRefreshRate = display.getRefreshRate(); + } + } catch (Exception e) { + LimeLog.severe("Error getting display info for display ID " + display.getDisplayId() + ": " + e.getMessage()); + return null; + } + + + if (axeOneLength <= 0 || axeTwoLength <= 0) { + LimeLog.warning("Retrieved invalid dimensions (" + axeOneLength + "x" + axeTwoLength + ") for display ID " + display.getDisplayId()); + if (sizePoint.x <= 0 || sizePoint.y <= 0) { + try { + axeOneLength = display.getWidth(); + axeTwoLength = display.getHeight(); + } catch (Exception ignored) {} + } else { + axeOneLength = sizePoint.x; + axeTwoLength = sizePoint.y; + } + if (axeOneLength <= 0 || axeTwoLength <= 0) { + LimeLog.severe("Could not retrieve valid dimensions for display ID " + display.getDisplayId()); + return null; } - } else { - // API < 23 (pre-Marshmallow) - Use deprecated methods - getLegacyDisplayInfo(display, sizePoint); - oneAxeLength = sizePoint.x; - secondAxeLength = sizePoint.y; - displayRefreshRate = display.getRefreshRate(); } - // Ensure landscape dimensions regardless of current orientation - int physicalWidth = Math.max(oneAxeLength, secondAxeLength); - int physicalHeight = Math.min(oneAxeLength, secondAxeLength); + int physicalWidth = Math.max(axeOneLength, axeTwoLength); + int physicalHeight = Math.min(axeOneLength, axeTwoLength); return new DisplayInfo(physicalWidth, physicalHeight, displayRefreshRate); } - - // Helper for older APIs (using a static Point to avoid allocations in loops if called often) private static final Point sizePoint = new Point(); + private static synchronized void getLegacyDisplayInfo(Display display, Point outSize) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { - // getRealSize() is API 17+ - display.getRealSize(outSize); - } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB_MR2) { - // getSize() is API 13+ (slightly less accurate, might exclude nav bar) - display.getSize(outSize); + outSize.set(0, 0); + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { + display.getRealSize(outSize); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB_MR2) { + display.getSize(outSize); + } else { + outSize.x = display.getWidth(); + outSize.y = display.getHeight(); + } + } catch (Exception e) { + LimeLog.severe("Exception in getLegacyDisplayInfo: " + e.getMessage()); + outSize.set(0, 0); + } + } + + private static class CategorizedDisplays { + Display mainDefaultDisplay = null; + Display externalPresentationDisplay = null; + Display secondaryInternalDisplay = null; + } + + private static CategorizedDisplays findAndCategorizeDisplays(Context context) { + CategorizedDisplays info = new CategorizedDisplays(); + DisplayManager displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + + if (displayManager == null) { + LimeLog.warning("DisplayManager service not found. Attempting fallback via WindowManager."); + try { + android.view.WindowManager wm = (android.view.WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + if (wm != null) info.mainDefaultDisplay = wm.getDefaultDisplay(); + else LimeLog.severe("WindowManager service also not found."); + } catch (Exception e) { + LimeLog.severe("Could not get default display via WindowManager: " + e.toString()); + } + if (info.mainDefaultDisplay == null) LimeLog.severe("FATAL: Could not obtain any reference to the main display."); + return info; + } + + Display[] displays = {}; + try { + displays = displayManager.getDisplays(); + } catch (Exception e) { + LimeLog.severe("Error getting displays from DisplayManager: " + e.toString()); + } + + + for (Display display : displays) { + if (display == null) continue; + + if (display.getDisplayId() == Display.DEFAULT_DISPLAY) { + info.mainDefaultDisplay = display; + continue; + } + + if ((display.getFlags() & Display.FLAG_PRESENTATION) != 0) { + if (info.externalPresentationDisplay == null) { + info.externalPresentationDisplay = display; + LimeLog.info("Found external presentation display: " + display.getName() + " (ID: " + display.getDisplayId() + ")"); + } else { + LimeLog.info("Ignoring additional external presentation display: " + display.getName()); + } + continue; + } + + if (info.secondaryInternalDisplay == null) { + info.secondaryInternalDisplay = display; + LimeLog.info("Found secondary internal display: " + display.getName() + " (ID: " + display.getDisplayId() + ")"); + } else { + LimeLog.info("Ignoring additional secondary internal display: " + display.getName()); + } + } + + if (info.mainDefaultDisplay == null) { + try { + info.mainDefaultDisplay = displayManager.getDisplay(Display.DEFAULT_DISPLAY); + LimeLog.warning("Main display (ID 0) not found in displays list, using getDisplay(DEFAULT_DISPLAY)."); + } catch (Exception e) { + LimeLog.severe("FATAL: Could not get display for DEFAULT_DISPLAY ID: " + e.toString()); + if (displays.length > 0 && displays[0] != null) info.mainDefaultDisplay = displays[0]; + } + } + if (info.mainDefaultDisplay == null) { + LimeLog.severe("FATAL: Could not obtain any valid reference to the main display after all fallbacks."); + } + // showCategorizationDialog(context, info); + return info; + } + + // --- getGameStreamDisplay - Contains Full Logic --- + public static Display getGameStreamDisplay(Context context) { + ensureDialogShown(context); // Ensure dialog appears once + CategorizedDisplays info = findAndCategorizeDisplays(context); // Get current displays + PreferenceConfiguration prefs = PreferenceConfiguration.readPreferences(context); + + Display defaultDisplay = info.mainDefaultDisplay; + // Essential fallback + if (defaultDisplay == null) { + LimeLog.severe("Cannot determine game stream display: mainDefaultDisplay is null. Using OS default."); + try { + DisplayManager dm = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + Display fallbackDisplay = (dm != null) ? dm.getDisplay(Display.DEFAULT_DISPLAY) : null; + if (fallbackDisplay == null) { LimeLog.severe("OS default display is also null!"); } + return fallbackDisplay; + } catch (Exception e) { + LimeLog.severe("Error getting OS default display in fallback: " + e.toString()); + return null; + } + } + + // --- Logic Flow --- + boolean treatAsInternal = false; + if (prefs.enableFullExDisplay) { + if (info.externalPresentationDisplay != null) { + boolean potentiallyMismatchedIDs = false; + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { // API 34 + try { + String deviceManufacturer = Build.MANUFACTURER; + android.hardware.display.DeviceProductInfo productInfo = info.externalPresentationDisplay.getDeviceProductInfo(); + String displayManufacturerId = (productInfo != null) ? productInfo.getManufacturerPnpId() : null; + + if (deviceManufacturer != null && displayManufacturerId != null && + displayManufacturerId.toLowerCase().contains(deviceManufacturer.toLowerCase())) + { + LimeLog.info("External display manufacturer ID matches device. Treating as potentially internal."); + treatAsInternal = true; + } else { + LimeLog.info("External display manufacturer ID does NOT match device or info unavailable."); + potentiallyMismatchedIDs = true; + } + } catch (Exception e) { + LimeLog.severe("Error comparing manufacturer IDs:" + e); + potentiallyMismatchedIDs = true; + } + } else { + LimeLog.info("Manufacturer ID check skipped (Requires API 34+)."); + potentiallyMismatchedIDs = true; + } + + if (!treatAsInternal) { + DisplayInfo mainInfo = getDisplayInfo(defaultDisplay); + DisplayInfo externalInfo = getDisplayInfo(info.externalPresentationDisplay); + if (potentiallyMismatchedIDs && + mainInfo != null && externalInfo != null && + // externalInfo.refreshRate < mainInfo.refreshRate - 1.0f && // Refresh rate check removed + externalInfo.totalPixels < mainInfo.totalPixels && + !isCommonMonitorAspectRatio(externalInfo)) + { + LimeLog.info("External display (ID " + info.externalPresentationDisplay.getDisplayId() + ") meets heuristics (smaller, non-monitor ratio). Manufacturer check failed/skipped. Assuming it's secondary internal."); + treatAsInternal = true; + } else { + LimeLog.info("enableFullExDisplay ON: Using true external presentation display (ID: " + info.externalPresentationDisplay.getDisplayId() + "). Heuristics did not apply or failed."); + return info.externalPresentationDisplay; + } + } + } + + Display effectiveSecondary = treatAsInternal ? info.externalPresentationDisplay : info.secondaryInternalDisplay; + + if (effectiveSecondary != null) { + DisplayInfo mainInfo = getDisplayInfo(defaultDisplay); + DisplayInfo secondaryInfo = getDisplayInfo(effectiveSecondary); + Display largerScreen = defaultDisplay; + + if (mainInfo != null && secondaryInfo != null) { + if (secondaryInfo.totalPixels > mainInfo.totalPixels) { + largerScreen = effectiveSecondary; + } + } else { + LimeLog.warning("enableFullExDisplay ON: Could not compare internal/effective-secondary displays; assuming default (ID 0) is larger."); + } + LimeLog.info("enableFullExDisplay ON (treating as internal screens): Using LARGER display (ID: " + largerScreen.getDisplayId() + ") for game stream."); + return largerScreen; + } + else { + LimeLog.info("enableFullExDisplay ON: Using main default display (ID: " + defaultDisplay.getDisplayId() + ") for game stream (only internal screen)."); + return defaultDisplay; + } + } + + if (info.secondaryInternalDisplay != null) { + DisplayInfo mainInfo = getDisplayInfo(defaultDisplay); + DisplayInfo secondaryInfo = getDisplayInfo(info.secondaryInternalDisplay); + Display largerInternal = defaultDisplay; + + if (mainInfo != null && secondaryInfo != null) { + if (secondaryInfo.totalPixels > mainInfo.totalPixels) { + largerInternal = info.secondaryInternalDisplay; + } + } else { + LimeLog.warning("Default OFF: Could not compare internal displays; assuming default (ID 0) is larger."); + } + LimeLog.info("Default OFF: Using LARGER internal display (ID: " + largerInternal.getDisplayId() + ") for game stream."); + return largerInternal; + } + + LimeLog.info("Default OFF: Using main default display (ID: " + defaultDisplay.getDisplayId() + ") for game stream (single screen)."); + return defaultDisplay; + } + + + private static boolean isCommonMonitorAspectRatio(DisplayInfo info) { + if (info == null || info.height <= 0) return false; + + float ratio = (float) info.width / (float) info.height; + final float EPSILON = 0.05f; + + final float RATIO_16_9 = 16.0f / 9.0f; + final float RATIO_16_10 = 16.0f / 10.0f; + final float RATIO_21_9 = 21.0f / 9.0f; + final float RATIO_32_9 = 32.0f / 9.0f; + final float RATIO_4_3 = 4.0f / 3.0f; + + boolean isMonitorRatio = Math.abs(ratio - RATIO_16_9) < EPSILON || + Math.abs(ratio - RATIO_16_10) < EPSILON || + Math.abs(ratio - RATIO_21_9) < EPSILON || + Math.abs(ratio - RATIO_32_9) < EPSILON || + Math.abs(ratio - RATIO_4_3) < EPSILON; + + LimeLog.info("Aspect ratio check: " + info.width + "x" + info.height + " -> ratio=" + String.format("%.3f", ratio) + ", isMonitorRatio=" + isMonitorRatio); + return isMonitorRatio; + } + + + // --- getControlsDisplay - Contains Full Logic --- + public static Display getControlsDisplay(Context context) { + ensureDialogShown(context); // Ensure dialog appears once + CategorizedDisplays info = findAndCategorizeDisplays(context); // Get current displays + // --- Determine where the game WILL be displayed --- + Display gameDisplay = getGameStreamDisplay(context); // Call the public method directly + // --- End Game Display Determination --- + + Display defaultDisplay = info.mainDefaultDisplay; + // Essential Fallbacks + if (defaultDisplay == null || gameDisplay == null) { + LimeLog.severe("Cannot determine controls display: mainDefaultDisplay or gameDisplay is null. Using OS default."); + try { + DisplayManager dm = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + Display fallbackDisplay = (dm != null) ? dm.getDisplay(Display.DEFAULT_DISPLAY) : null; + if (fallbackDisplay == null) { LimeLog.severe("OS default display is also null!"); } + return fallbackDisplay; + } catch (Exception e) { + LimeLog.severe("Error getting OS default display in controls fallback: " + e.toString()); + return null; + } + } + + // --- Identify potential candidates for the controls display --- + List controlCandidates = new ArrayList<>(); + if (info.mainDefaultDisplay != null && info.mainDefaultDisplay.getDisplayId() != gameDisplay.getDisplayId()) { + controlCandidates.add(info.mainDefaultDisplay); + } + if (info.secondaryInternalDisplay != null && info.secondaryInternalDisplay.getDisplayId() != gameDisplay.getDisplayId()) { + controlCandidates.add(info.secondaryInternalDisplay); + } + if (info.externalPresentationDisplay != null && info.externalPresentationDisplay.getDisplayId() != gameDisplay.getDisplayId()) { + controlCandidates.add(info.externalPresentationDisplay); + } + + List validControlCandidates = new ArrayList<>(); + for (Display candidate : controlCandidates) { + DisplayInfo candidateInfo = getDisplayInfo(candidate); + if (candidateInfo != null) { + validControlCandidates.add(candidate); + } else { + LimeLog.warning("Control candidate display (ID: " + ((candidate != null) ? candidate.getDisplayId() : "null") + ") is below minimum size or info unavailable. Ignoring."); + } + } + + // --- Select the best control display --- + Display selectedControlsDisplay = null; + if (validControlCandidates.size() == 1) { + selectedControlsDisplay = validControlCandidates.get(0); + LimeLog.info("Using the only valid secondary display (ID: " + selectedControlsDisplay.getDisplayId() + ") for controls."); + } else if (validControlCandidates.size() > 1) { + Display smallestValid = null; + long smallestPixels = Long.MAX_VALUE; + for (Display validCandidate : validControlCandidates) { + DisplayInfo validInfo = getDisplayInfo(validCandidate); + if (validInfo != null && validInfo.totalPixels < smallestPixels) { + smallestPixels = validInfo.totalPixels; + smallestValid = validCandidate; + } + } + selectedControlsDisplay = smallestValid; + if (selectedControlsDisplay != null) { + LimeLog.info("Multiple valid secondary displays found. Using the smallest (ID: " + selectedControlsDisplay.getDisplayId() + ") for controls."); + } else { + LimeLog.warning("Could not determine smallest among valid control candidates. Falling back to overlay."); + selectedControlsDisplay = gameDisplay; + } + } else { + LimeLog.info("No valid secondary display found for controls. Using game display (ID: " + gameDisplay.getDisplayId() + ") for overlay controls."); + selectedControlsDisplay = gameDisplay; + } + + return selectedControlsDisplay; + } + + + private static String getDisplayDetailsString(Display display) { + if (display == null) { + return "null display object"; + } + StringBuilder details = new StringBuilder(); + DisplayInfo di = getDisplayInfo(display); + + details.append("ID: ").append(display.getDisplayId()); + details.append(", Name: ").append(display.getName()); + details.append(", Res: ").append(di != null ? di.toString() : "N/A"); // Use DisplayInfo.toString() + + try { + int flags = display.getFlags(); + List flagNames = new ArrayList<>(); + if ((flags & Display.FLAG_PRIVATE) != 0) flagNames.add("PRIVATE"); + if ((flags & Display.FLAG_PRESENTATION) != 0) flagNames.add("PRESENTATION"); + if ((flags & Display.FLAG_SECURE) != 0) flagNames.add("SECURE"); + if ((flags & Display.FLAG_SUPPORTS_PROTECTED_BUFFERS) != 0) flagNames.add("PROTECTED"); + if ((flags & Display.FLAG_ROUND) != 0) flagNames.add("ROUND"); + + details.append(", Flags: "); + if (flagNames.isEmpty()) { + details.append("None"); + } else { + details.append("[").append(String.join("|", flagNames)).append("]"); + } + details.append(" (").append(flags).append(")"); + } catch (Exception e) { + details.append(", Flags: Error reading flags (").append(e.getMessage()).append(")"); + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { // API 34 + try { + android.hardware.display.DeviceProductInfo productInfo = display.getDeviceProductInfo(); + if (productInfo != null) { + details.append(", Product: ["); + details.append("Name: ").append(productInfo.getName()); + String manufId = productInfo.getManufacturerPnpId(); + if (manufId != null && !manufId.isEmpty()) { + details.append(", ManufId: ").append(manufId); + } + String prodId = productInfo.getProductId(); + if (prodId != null && !prodId.isEmpty()) { + details.append(", ProdId: ").append(prodId); + } + details.append("]"); + } else { + details.append(", ProductInfo: null"); + } + } catch (NoSuchMethodError e) { + LimeLog.warning("getProductInfo method not found on API 34+ device?"); + details.append(", ProductInfo: Not Available (API Error)"); + } catch (Exception e) { + LimeLog.severe("Error getting product info: " + e.getMessage()); + details.append(", ProductInfo: Error reading info"); + } + } else { + details.append(", ProductInfo: Not Available (API < 34)"); + } + + return details.toString(); + } + + + private static void showCategorizationDialog(Context context, CategorizedDisplays info) { + if (context == null) { + LimeLog.severe("Cannot show display dialog: Context is null."); + return; + } + + if(openDialog != null && openDialog.isShowing()) { + try { openDialog.dismiss(); } catch (Exception e) { LimeLog.warning("Error dismissing previous dialog: " + e.getMessage()); } + openDialog = null; + } + + final StringBuilder messageBuilder = new StringBuilder(); + boolean displayFound = false; + + messageBuilder.append("--- Displays Found ---\n"); + + if (info.mainDefaultDisplay != null) { + messageBuilder.append("Main (Default): \n ").append(getDisplayDetailsString(info.mainDefaultDisplay)).append("\n\n"); + displayFound = true; + } else { + messageBuilder.append("Main (Default): Not Found!\n\n"); + } + if (info.externalPresentationDisplay != null) { + messageBuilder.append("External (Presentation Flag): \n ").append(getDisplayDetailsString(info.externalPresentationDisplay)).append("\n\n"); + displayFound = true; + } else { + messageBuilder.append("External (Presentation Flag): None\n\n"); + } + if (info.secondaryInternalDisplay != null) { + messageBuilder.append("Secondary Internal (No Pres. Flag): \n ").append(getDisplayDetailsString(info.secondaryInternalDisplay)).append("\n\n"); + displayFound = true; } else { - // Very old fallback (even less accurate) - outSize.x = display.getWidth(); // Deprecated in API 13 - outSize.y = display.getHeight(); // Deprecated in API 13 + messageBuilder.append("Secondary Internal (No Pres. Flag): None\n\n"); } + + // --- Determine final selections for the dialog --- + // Need to replicate the logic *briefly* here to show the result + Display determinedGameDisplay = null; + Display determinedControlsDisplay = null; + try { + // Replicate getGameStreamDisplay logic (simplified, without dialog call) + PreferenceConfiguration tempPrefs = PreferenceConfiguration.readPreferences(context); + Display tempDefault = info.mainDefaultDisplay; + if (tempDefault != null) { + boolean tempTreatAsInternal = false; + if (tempPrefs.enableFullExDisplay && info.externalPresentationDisplay != null) { + // Simplified heuristic check for dialog display purpose + boolean tempPotentiallyMismatched = true; // Assume mismatch for dialog unless proven otherwise by API 34+ check (not replicated here for brevity) + DisplayInfo tempMainInfo = getDisplayInfo(tempDefault); + DisplayInfo tempExternalInfo = getDisplayInfo(info.externalPresentationDisplay); + if (tempPotentiallyMismatched && tempMainInfo != null && tempExternalInfo != null && + tempExternalInfo.totalPixels < tempMainInfo.totalPixels && !isCommonMonitorAspectRatio(tempExternalInfo)) { + tempTreatAsInternal = true; + } else { + determinedGameDisplay = info.externalPresentationDisplay; // Assume external if heuristics fail/don't apply + } + } + + if (determinedGameDisplay == null) { // If not assigned external + Display tempEffectiveSecondary = tempTreatAsInternal ? info.externalPresentationDisplay : info.secondaryInternalDisplay; + if (tempEffectiveSecondary != null) { + DisplayInfo tempMainInfo = getDisplayInfo(tempDefault); + DisplayInfo tempSecondaryInfo = getDisplayInfo(tempEffectiveSecondary); + determinedGameDisplay = tempDefault; + if (tempMainInfo != null && tempSecondaryInfo != null && tempSecondaryInfo.totalPixels > tempMainInfo.totalPixels) { + determinedGameDisplay = tempEffectiveSecondary; + } + } else { + determinedGameDisplay = tempDefault; + } + } + } + // Replicate getControlsDisplay logic (simplified) + if (determinedGameDisplay != null && tempDefault != null) { + boolean tempGameIsTrulyExternal = info.externalPresentationDisplay != null && determinedGameDisplay.getDisplayId() == info.externalPresentationDisplay.getDisplayId(); + // Additional check needed here to ensure it wasn't treated as internal + // For simplicity in dialog, we might omit perfect replication + if (tempGameIsTrulyExternal) { + determinedControlsDisplay = (info.secondaryInternalDisplay != null) ? + ((getDisplayInfo(info.secondaryInternalDisplay).totalPixels < getDisplayInfo(tempDefault).totalPixels) ? info.secondaryInternalDisplay : tempDefault) + : tempDefault; // Simplified: picks smaller of internals, or default + // Missing MIN_SIZE check here for dialog brevity + } else if (info.secondaryInternalDisplay != null) { + determinedControlsDisplay = (determinedGameDisplay.getDisplayId() == tempDefault.getDisplayId()) ? info.secondaryInternalDisplay : tempDefault; + } else { + determinedControlsDisplay = determinedGameDisplay; // Overlay + } + } + + } catch (Exception e) { + LimeLog.severe("Error determining selections for dialog: " + e.getMessage()); + } + + + messageBuilder.append("--- Final Selection ---\n"); + messageBuilder.append("Game Stream Display: \n "); + messageBuilder.append(determinedGameDisplay != null ? getDisplayDetailsString(determinedGameDisplay) : "ERROR (null)").append("\n\n"); + messageBuilder.append("Controls Display: \n "); + messageBuilder.append(determinedControlsDisplay != null ? getDisplayDetailsString(determinedControlsDisplay) : "ERROR (null)").append("\n"); + + + final String message = displayFound ? messageBuilder.toString().trim() : "Error: No displays were categorized."; + final String title = "Display Selection Info"; + + new Handler(Looper.getMainLooper()).post(() -> { + try { + if (context instanceof android.app.Activity && ((android.app.Activity) context).isFinishing()) { + LimeLog.warning("Activity is finishing, cannot show display dialog."); + return; + } + + openDialog = new AlertDialog.Builder(context) + .setTitle(title) + .setMessage(message) + .setPositiveButton(android.R.string.ok, (dialog, which) -> { + dialog.dismiss(); + openDialog = null; + }) + .setNeutralButton("Copy Info", (dialog, which) -> { + ClipboardManager clipboard = (ClipboardManager) context.getSystemService(Context.CLIPBOARD_SERVICE); + if (clipboard != null) { + ClipData clip = ClipData.newPlainText("Display Info", message); + clipboard.setPrimaryClip(clip); + Toast.makeText(context, "Display info copied to clipboard", Toast.LENGTH_SHORT).show(); + } else { + Toast.makeText(context, "Failed to access clipboard", Toast.LENGTH_SHORT).show(); + } + }) + .setCancelable(false) + .show(); + } catch (Exception e) { + LimeLog.severe("Failed to show display selection dialog: " + e.getMessage()); + openDialog = null; + } + }); } + + private static volatile boolean dialogShown = false; + private static void ensureDialogShown(Context context) { + if (!dialogShown) { + synchronized (DisplayUtils.class) { + if (!dialogShown) { + CategorizedDisplays info = findAndCategorizeDisplays(context); // This now calls the dialog + // Selections are determined inside the dialog show logic for display + dialogShown = true; + } + } + } + } + + + public static boolean hasSecondaryDisplay(Context context) { + ensureDialogShown(context); // Ensure dialog shows if not already + CategorizedDisplays info = findAndCategorizeDisplays(context); // Find displays again + boolean hasSecondary = info.externalPresentationDisplay != null || info.secondaryInternalDisplay != null; + return hasSecondary; + } + } \ No newline at end of file diff --git a/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java b/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java index 5bbb0a30b0..78c20f7780 100644 --- a/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java +++ b/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java @@ -1,7 +1,7 @@ package com.limelight.utils; import static com.limelight.StartExternalDisplayControlReceiver.requestFocusToGameActivity; -import static com.limelight.utils.ServerHelper.getSecondaryDisplay; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import android.Manifest; import android.annotation.SuppressLint; @@ -69,6 +69,8 @@ public class ExternalDisplayControlActivity extends AppCompatActivity implements private boolean isKeyboardVisible = false; + private boolean shouldManageBrightness = false; + private final Handler handler = new Handler(Looper.getMainLooper()); private int failCount = 0; private Runnable dimScreenRunnable; @@ -122,15 +124,15 @@ protected void onCreate(Bundle savedInstanceState) { if (gameIntent == null) { finish(); } else { - Display secondaryDisplay = getSecondaryDisplay(this); - if (secondaryDisplay != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Display gameStreamDisplay = getGameStreamDisplay(this); + if (gameStreamDisplay != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { ActivityOptions options = ActivityOptions.makeBasic(); - options.setLaunchDisplayId(secondaryDisplay.getDisplayId()); + options.setLaunchDisplayId(gameStreamDisplay.getDisplayId()); Toast.makeText(this, getString(R.string.external_display_info, - secondaryDisplay.getMode().getPhysicalWidth(), - secondaryDisplay.getMode().getPhysicalHeight(), - secondaryDisplay.getMode().getRefreshRate()), + gameStreamDisplay.getMode().getPhysicalWidth(), + gameStreamDisplay.getMode().getPhysicalHeight(), + gameStreamDisplay.getMode().getRefreshRate()), Toast.LENGTH_LONG).show(); startActivity(gameIntent, options.toBundle()); @@ -175,11 +177,17 @@ private void initViews() { }); } + Display controlsDisplay = DisplayUtils.getControlsDisplay(this); + shouldManageBrightness = (controlsDisplay != null && controlsDisplay.getDisplayId() == Display.DEFAULT_DISPLAY); + LimeLog.info("Brightness management " + (shouldManageBrightness ? "enabled" : "disabled") + " for this display (ID: " + ((controlsDisplay != null) ? controlsDisplay.getDisplayId() : "null") + ")"); + initializeComponents(); createProgrammaticUI(); checkNotificationPermission(); initTouchEventHandling(); - setupInactivityTimeoutForBrightness(); + if (shouldManageBrightness) { + setupInactivityTimeoutForBrightness(); + } requestFocusToGameActivity(false); } @@ -224,12 +232,14 @@ public void onKeyboardControllerVisibilityChange(boolean visible) { @SuppressLint("ClickableViewAccessibility") private void setupInactivityTimeoutForBrightness() { + if (!shouldManageBrightness) return; // Save the original brightness WindowManager.LayoutParams layout = getWindow().getAttributes(); originalBrightness = layout.screenBrightness; // Runnable to dim screen dimScreenRunnable = () -> { + if (!shouldManageBrightness) return; WindowManager.LayoutParams l = getWindow().getAttributes(); l.screenBrightness = 0.0f; getWindow().setAttributes(l); @@ -255,6 +265,7 @@ private void updateKeyboardVisibility(boolean visible) { } private void restoreBrightnessIfNeeded() { + if (!shouldManageBrightness) return; WindowManager.LayoutParams l = getWindow().getAttributes(); if (l.screenBrightness == 0.0f) { l.screenBrightness = originalBrightness; @@ -263,12 +274,14 @@ private void restoreBrightnessIfNeeded() { } private void handleUserActivity() { + if (!shouldManageBrightness) return; // Restore brightness if dimmed restoreBrightnessIfNeeded(); resetInactivityTimer(); } private void resetInactivityTimer() { + if (!shouldManageBrightness) return; handler.removeCallbacks(dimScreenRunnable); if (!isKeyboardVisible) { handler.postDelayed(dimScreenRunnable, INACTIVITY_TIMEOUT_MS); diff --git a/app/src/main/java/com/limelight/utils/ServerHelper.java b/app/src/main/java/com/limelight/utils/ServerHelper.java index 0e308986e5..0e8d8e3a51 100755 --- a/app/src/main/java/com/limelight/utils/ServerHelper.java +++ b/app/src/main/java/com/limelight/utils/ServerHelper.java @@ -1,5 +1,8 @@ package com.limelight.utils; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; +import static com.limelight.utils.DisplayUtils.hasSecondaryDisplay; + import android.app.Activity; import android.content.Context; import android.content.Intent; @@ -61,35 +64,6 @@ public static Intent createAppShortcutIntent(Activity parent, ComputerDetails co i.setAction(Intent.ACTION_DEFAULT); return i; } - public static Display getActiveDisplay(Context context) { - Display secondary = getSecondaryDisplay(context); - PreferenceConfiguration prefs = PreferenceConfiguration.readPreferences(context); - if (secondary != null && (prefs.enableFullExDisplay)) { - return secondary; - } else { - return ((DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE)).getDisplay(Display.DEFAULT_DISPLAY); - } - } - - public static Display getSecondaryDisplay(Context context) { - DisplayManager displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); - Display display = null; - Display[] displays = displayManager.getDisplays(); - int mainDisplayId = Display.DEFAULT_DISPLAY; - int secondaryDisplayId = -1; - for (Display displayVariant : displays) { - LimeLog.info(displayVariant.toString()); - if (displayVariant.getDisplayId() != mainDisplayId) { - secondaryDisplayId = displayVariant.getDisplayId(); - break; - } - } - - if (secondaryDisplayId != -1) { - display = displayManager.getDisplay(secondaryDisplayId); - } - return display; - } public static Intent createStartIntent(Activity parent, NvApp app, ComputerDetails computer, ComputerManagerService.ComputerManagerBinder managerBinder, @@ -97,8 +71,8 @@ public static Intent createStartIntent(Activity parent, NvApp app, ComputerDetai Intent gameIntent = null; PreferenceConfiguration prefConfig = PreferenceConfiguration.readPreferences(parent); // Try to add secondary DisplayContext if supported and connected - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && prefConfig.enableFullExDisplay && getSecondaryDisplay(parent) != null) { - Context displayContext = parent.createDisplayContext(getSecondaryDisplay(parent)); // use secondary display + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && prefConfig.enableFullExDisplay && hasSecondaryDisplay(parent)) { + Context displayContext = parent.createDisplayContext(getGameStreamDisplay(parent)); // use secondary display gameIntent = new Intent(displayContext, Game.class); gameIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); } @@ -124,11 +98,11 @@ public static Intent createStartIntent(Activity parent, NvApp app, ComputerDetai e.printStackTrace(); } - if (prefConfig.enableFullExDisplay) { - Display secondaryDisplay = getSecondaryDisplay(parent); - if (secondaryDisplay != null) { - int secondaryDisplayId = secondaryDisplay.getDisplayId(); - gameIntent.putExtra(Game.EXTRA_DISPLAY_ID, secondaryDisplayId); + if (prefConfig.enableFullExDisplay && hasSecondaryDisplay(parent)) { + Display gameStreamDisplay = getGameStreamDisplay(parent); + if (gameStreamDisplay != null) { + int gameStreamDisplayId = gameStreamDisplay.getDisplayId(); + gameIntent.putExtra(Game.EXTRA_DISPLAY_ID, gameStreamDisplayId); Intent touchpadIntent = new Intent(parent, ExternalDisplayControlActivity.class); touchpadIntent.putExtra(ExternalDisplayControlActivity.EXTRA_LAUNCH_INTENT, gameIntent); return touchpadIntent; From d6dd37940d19d8a8b960bab641c2e33718862c88 Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 24 Oct 2025 13:11:37 +0200 Subject: [PATCH 20/22] minor fix used intent extra for displayId, not needed anymore --- app/src/main/java/com/limelight/Game.java | 19 ++++++++----------- .../com/limelight/utils/ServerHelper.java | 2 -- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index c1c05ecca8..06b0cd72e5 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -268,7 +268,6 @@ public void onServiceDisconnected(ComponentName componentName) { public static final String EXTRA_SERVER_CERT = "ServerCert"; public static final String EXTRA_VDISPLAY = "VirtualDisplay"; public static final String EXTRA_SERVER_COMMANDS = "ServerCommands"; - public static final String EXTRA_DISPLAY_ID = "DisplayID"; public static final String CLIPBOARD_IDENTIFIER = "ArtemisStreaming"; @@ -386,20 +385,18 @@ protected void onCreate(Bundle savedInstanceState) { getResources().getString(R.string.conn_establishing_msg), true); - Display currentDisplay = null; - int displayId = -2; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - displayId = getIntent().getIntExtra(EXTRA_DISPLAY_ID, displayId); - if(displayId != -2) { - currentDisplay = getSystemService(DisplayManager.class).getDisplay(displayId); - } - } + Display currentDisplay = DisplayUtils.getGameStreamDisplay(this); if (currentDisplay == null) { - currentDisplay = getWindowManager().getDefaultDisplay(); + LimeLog.severe("FATAL: getGameStreamDisplay returned null! Cannot continue."); + // Show an error to the user and finish + Toast.makeText(this, "Critical Error: Could not determine target display.", Toast.LENGTH_LONG).show(); + finish(); + return; // Important: Stop further execution in onCreate } - onExternelDisplay = displayId != -2; + onExternelDisplay = (currentDisplay.getDisplayId() != Display.DEFAULT_DISPLAY); + boolean shouldInvertDecoderResolution = false; matchSettings(currentDisplay); diff --git a/app/src/main/java/com/limelight/utils/ServerHelper.java b/app/src/main/java/com/limelight/utils/ServerHelper.java index 0e8d8e3a51..565ae7b5d5 100755 --- a/app/src/main/java/com/limelight/utils/ServerHelper.java +++ b/app/src/main/java/com/limelight/utils/ServerHelper.java @@ -101,8 +101,6 @@ public static Intent createStartIntent(Activity parent, NvApp app, ComputerDetai if (prefConfig.enableFullExDisplay && hasSecondaryDisplay(parent)) { Display gameStreamDisplay = getGameStreamDisplay(parent); if (gameStreamDisplay != null) { - int gameStreamDisplayId = gameStreamDisplay.getDisplayId(); - gameIntent.putExtra(Game.EXTRA_DISPLAY_ID, gameStreamDisplayId); Intent touchpadIntent = new Intent(parent, ExternalDisplayControlActivity.class); touchpadIntent.putExtra(ExternalDisplayControlActivity.EXTRA_LAUNCH_INTENT, gameIntent); return touchpadIntent; From 79ac9111ccc1b2fd31e25fc5516a7966bfede070 Mon Sep 17 00:00:00 2001 From: Janyger Date: Fri, 24 Oct 2025 13:18:33 +0200 Subject: [PATCH 21/22] minor fix used intent extra for displayId, not needed anymore --- app/src/main/java/com/limelight/Game.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 06b0cd72e5..ebe7059802 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -1065,7 +1065,7 @@ public void onDisplayAdded(int displayId) { @Override public void onDisplayRemoved(int displayId) { - if (hasSecondaryDisplay(getBaseContext())) { + if (onExternelDisplay) { handleDisplayRemoved(); finish(); } From 9ce05290d31d652128027f11c069060979a9da72 Mon Sep 17 00:00:00 2001 From: Janyger Date: Sun, 26 Oct 2025 19:34:38 +0100 Subject: [PATCH 22/22] minor fix used intent extra for displayId, not needed anymore --- app/src/main/java/com/limelight/utils/PanZoomHandler.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/limelight/utils/PanZoomHandler.java b/app/src/main/java/com/limelight/utils/PanZoomHandler.java index b51209ee58..c00dd7c421 100644 --- a/app/src/main/java/com/limelight/utils/PanZoomHandler.java +++ b/app/src/main/java/com/limelight/utils/PanZoomHandler.java @@ -62,10 +62,12 @@ private void constrainToBounds() { } if (parentHeight >= childHeight) { - // ALWAYS center it vertically when it's smaller than the parent - childY = (parentHeight - childHeight) / 2; + if (isTopMode) { + childY = 0; + } else { + childY = (parentHeight - childHeight) / 2; + } } else { - // This handles panning when the view is larger than the parent float boundaryY = parentHeight - childHeight; childY = Math.max(boundaryY, Math.min(childY, 0)); }