Handle OpenGL rendering on frontend

This substantially improves performance when using the OpenGL renderer

app/CMakeLists.txt

@@ -49,6 +49,7 @@ add_library(
         src/main/cpp/UriFileHandler.cpp
         src/main/cpp/JniEnvHandler.cpp
         src/main/cpp/MelonDSAndroidCameraHandler.cpp
+        src/main/cpp/AndroidFrameRenderedCallback.cpp
         src/main/cpp/AndroidRACallback.cpp
         src/main/cpp/RAAchievementMapper.cpp
 )

app/proguard-rules.pro

@@ -42,6 +42,7 @@
 }
 -keep interface me.magnum.melonds.common.camera.DSiCameraSource { *; }
 -keep interface me.magnum.melonds.common.RetroAchievementsCallback { *; }
+-keep interface me.magnum.melonds.ui.emulator.EmulatorFrameRenderedListener { *; }
 
 # Migration fields. These rules are required for migrations to work properly
 -keep,allowobfuscation class me.magnum.melonds.migrations.legacy.** { *; }

app/src/main/cpp/AndroidFrameRenderedCallback.cpp

@@ -0,0 +1,16 @@
+#include "AndroidFrameRenderedCallback.h"
+
+AndroidFrameRenderedCallback::AndroidFrameRenderedCallback(JniEnvHandler* jniEnvHandler, jobject androidFrameRenderedListener)
+{
+    this->jniEnvHandler = jniEnvHandler;
+    this->androidFrameRenderedListener = androidFrameRenderedListener;
+}
+
+void AndroidFrameRenderedCallback::onFrameRendered(long syncFence, int textureId)
+{
+    JNIEnv* env = this->jniEnvHandler->getCurrentThreadEnv();
+
+    jclass listenerClass = env->GetObjectClass(this->androidFrameRenderedListener);
+    jmethodID onFrameRenderedMethod = env->GetMethodID(listenerClass, "onFrameRendered", "(JI)V");
+    env->CallVoidMethod(this->androidFrameRenderedListener, onFrameRenderedMethod, syncFence, textureId);
+}

app/src/main/cpp/AndroidFrameRenderedCallback.h

@@ -0,0 +1,20 @@
+#ifndef ANDROIDFRAMERENDEREDCALLBACK_H
+#define ANDROIDFRAMERENDEREDCALLBACK_H
+
+#include "JniEnvHandler.h"
+#include <FrameRenderedCallback.h>
+#include <jni.h>
+
+class AndroidFrameRenderedCallback : public FrameRenderedCallback
+{
+private:
+    JniEnvHandler* jniEnvHandler;
+    jobject androidFrameRenderedListener;
+
+public:
+    AndroidFrameRenderedCallback(JniEnvHandler* jniEnvHandler, jobject androidFrameRenderedListener);
+    void onFrameRendered(long syncFence, int textureId);
+};
+
+
+#endif //ANDROIDFRAMERENDEREDCALLBACK_H

app/src/main/cpp/MelonDSAndroidJNI.cpp

@@ -12,6 +12,7 @@
 #include "UriFileHandler.h"
 #include "JniEnvHandler.h"
 #include "AndroidRACallback.h"
+#include "AndroidFrameRenderedCallback.h"
 #include "MelonDSAndroidInterface.h"
 #include "MelonDSAndroidConfiguration.h"
 #include "MelonDSAndroidCameraHandler.h"
@@ -46,29 +47,32 @@ bool isFastForwardEnabled = false;
 jobject globalAssetManager;
 jobject globalCameraManager;
 jobject androidRaCallback;
+jobject androidFrameRenderListener;
 MelonDSAndroidCameraHandler* androidCameraHandler;
 AndroidRACallback* raCallback;
+AndroidFrameRenderedCallback* frameRenderedCallback;
 
 extern "C"
 {
 JNIEXPORT void JNICALL
-Java_me_magnum_melonds_MelonEmulator_setupEmulator(JNIEnv* env, jobject thiz, jobject emulatorConfiguration, jobject javaAssetManager, jobject cameraManager, jobject retroAchievementsCallback, jobject textureBuffer)
+Java_me_magnum_melonds_MelonEmulator_setupEmulator(JNIEnv* env, jobject thiz, jobject emulatorConfiguration, jobject javaAssetManager, jobject cameraManager, jobject retroAchievementsCallback, jobject frameRenderListener, jobject screenshotBuffer, jlong glContext)
 {
     MelonDSAndroid::EmulatorConfiguration finalEmulatorConfiguration = MelonDSAndroidConfiguration::buildEmulatorConfiguration(env, emulatorConfiguration);
     fastForwardSpeedMultiplier = finalEmulatorConfiguration.fastForwardSpeedMultiplier;
 
     globalAssetManager = env->NewGlobalRef(javaAssetManager);
     globalCameraManager = env->NewGlobalRef(cameraManager);
     androidRaCallback = env->NewGlobalRef(retroAchievementsCallback);
+    androidFrameRenderListener = env->NewGlobalRef(frameRenderListener);
 
     AAssetManager* assetManager = AAssetManager_fromJava(env, globalAssetManager);
     androidCameraHandler = new MelonDSAndroidCameraHandler(jniEnvHandler, globalCameraManager);
     raCallback = new AndroidRACallback(jniEnvHandler, androidRaCallback);
-
-    u32* textureBufferPointer = (u32*) env->GetDirectBufferAddress(textureBuffer);
+    frameRenderedCallback = new AndroidFrameRenderedCallback(jniEnvHandler, androidFrameRenderListener);
+    u32* screenshotBufferPointer = (u32*) env->GetDirectBufferAddress(screenshotBuffer);
 
     MelonDSAndroid::setConfiguration(finalEmulatorConfiguration);
-    MelonDSAndroid::setup(assetManager, androidCameraHandler, raCallback, textureBufferPointer, true);
+    MelonDSAndroid::setup(assetManager, androidCameraHandler, raCallback, frameRenderedCallback, screenshotBufferPointer, glContext, true);
     paused = false;
 }
 
@@ -415,13 +419,16 @@ Java_me_magnum_melonds_MelonEmulator_stopEmulation(JNIEnv* env, jobject thiz)
     env->DeleteGlobalRef(globalAssetManager);
     env->DeleteGlobalRef(globalCameraManager);
     env->DeleteGlobalRef(androidRaCallback);
+    env->DeleteGlobalRef(androidFrameRenderListener);
 
     globalAssetManager = nullptr;
     globalCameraManager = nullptr;
     androidRaCallback = nullptr;
+    androidFrameRenderListener = nullptr;
 
     delete androidCameraHandler;
     delete raCallback;
+    delete frameRenderedCallback;
 }
 
 JNIEXPORT void JNICALL
@@ -462,12 +469,11 @@ Java_me_magnum_melonds_MelonEmulator_setFastForwardEnabled(JNIEnv* env, jobject
 }
 
 JNIEXPORT void JNICALL
-Java_me_magnum_melonds_MelonEmulator_updateEmulatorConfiguration(JNIEnv* env, jobject thiz, jobject emulatorConfiguration, jobject frameBuffer)
+Java_me_magnum_melonds_MelonEmulator_updateEmulatorConfiguration(JNIEnv* env, jobject thiz, jobject emulatorConfiguration)
 {
     MelonDSAndroid::EmulatorConfiguration newConfiguration = MelonDSAndroidConfiguration::buildEmulatorConfiguration(env, emulatorConfiguration);
-    u32* frameBufferPointer = (u32*) env->GetDirectBufferAddress(frameBuffer);
 
-    MelonDSAndroid::updateEmulatorConfiguration(newConfiguration, frameBufferPointer);
+    MelonDSAndroid::updateEmulatorConfiguration(newConfiguration);
     fastForwardSpeedMultiplier = newConfiguration.fastForwardSpeedMultiplier;
 
     if (isFastForwardEnabled) {

app/src/main/java/me/magnum/melonds/MelonEmulator.kt

@@ -8,6 +8,7 @@ import me.magnum.melonds.domain.model.EmulatorConfiguration
 import me.magnum.melonds.domain.model.Input
 import me.magnum.melonds.domain.model.retroachievements.RASimpleAchievement
 import me.magnum.melonds.common.RetroAchievementsCallback
+import me.magnum.melonds.ui.emulator.EmulatorFrameRenderedListener
 import me.magnum.melonds.ui.emulator.rewind.model.RewindSaveState
 import me.magnum.melonds.ui.emulator.rewind.model.RewindWindow
 import java.nio.ByteBuffer
@@ -43,7 +44,15 @@ object MelonEmulator {
         MEMORY_EXPANSION,
     }
 
-	external fun setupEmulator(emulatorConfiguration: EmulatorConfiguration, assetManager: AssetManager?, dsiCameraSource: DSiCameraSource?, retroAchievementsCallback: RetroAchievementsCallback, textureBuffer: ByteBuffer)
+	external fun setupEmulator(
+        emulatorConfiguration: EmulatorConfiguration,
+        assetManager: AssetManager?,
+        dsiCameraSource: DSiCameraSource?,
+        retroAchievementsCallback: RetroAchievementsCallback,
+        frameRenderedListener: EmulatorFrameRenderedListener,
+        screenshotBuffer: ByteBuffer,
+        glContext: Long,
+    )
 
     external fun setupCheats(cheats: Array<Cheat>)
 
@@ -119,5 +128,5 @@ object MelonEmulator {
 
     external fun setFastForwardEnabled(enabled: Boolean)
 
-    external fun updateEmulatorConfiguration(emulatorConfiguration: EmulatorConfiguration, frameBuffer: ByteBuffer)
+    external fun updateEmulatorConfiguration(emulatorConfiguration: EmulatorConfiguration)
 }

app/src/main/java/me/magnum/melonds/common/opengl/Shader.kt

@@ -2,7 +2,10 @@ package me.magnum.melonds.common.opengl
 
 import android.opengl.GLES20
 
-class Shader(private val programId: Int) {
+class Shader(
+    private val programId: Int,
+    val textureFiltering: Int,
+) {
     val uniformMvp: Int
     val attribUv: Int
     val attribPos: Int

app/src/main/java/me/magnum/melonds/common/opengl/ShaderFactory.kt

@@ -4,10 +4,16 @@ import android.opengl.GLES20
 
 object ShaderFactory {
     fun createShaderProgram(source: ShaderProgramSource): Shader {
-        return createShaderProgram(source.vertexShaderSource, source.fragmentShaderSource)
+        val shaderProgram = createShaderProgram(source.vertexShaderSource, source.fragmentShaderSource)
+        val textureFilter = when (source.textureFiltering) {
+            ShaderProgramSource.TextureFiltering.NEAREST -> GLES20.GL_NEAREST
+            ShaderProgramSource.TextureFiltering.LINEAR -> GLES20.GL_LINEAR
+        }
+
+        return Shader(shaderProgram, textureFilter)
     }
 
-    private fun createShaderProgram(vertexShader: String, fragmentShader: String): Shader {
+    private fun createShaderProgram(vertexShader: String, fragmentShader: String): Int {
         val program = GLES20.glCreateProgram()
         GLES20.glAttachShader(program, createShader(GLES20.GL_VERTEX_SHADER, vertexShader))
         GLES20.glAttachShader(program, createShader(GLES20.GL_FRAGMENT_SHADER, fragmentShader))
@@ -19,7 +25,7 @@ object ShaderFactory {
             System.err.println(GLES20.glGetProgramInfoLog(program))
         }
 
-        return Shader(program)
+        return program
     }
 
     private fun createShader(shaderType: Int, code: String): Int {

app/src/main/java/me/magnum/melonds/common/runtime/ScreenshotFrameBufferProvider.kt

@@ -0,0 +1,53 @@
+package me.magnum.melonds.common.runtime
+
+import android.graphics.Bitmap
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+
+class ScreenshotFrameBufferProvider {
+
+    companion object {
+        private const val SCREEN_WIDTH = 256
+        private const val SCREEN_HEIGHT = 384
+    }
+
+    private var screenshotBuffer: ByteBuffer? = null
+
+    fun frameBuffer(): ByteBuffer {
+        return ensureBufferIsReady()
+    }
+
+    fun getScreenshot(): Bitmap {
+        val frameBuffer = ensureBufferIsReady()
+
+        return Bitmap.createBitmap(SCREEN_WIDTH, SCREEN_HEIGHT, Bitmap.Config.ARGB_8888).apply {
+            for (x in 0 until SCREEN_WIDTH) {
+                for (y in 0 until SCREEN_HEIGHT) {
+                    val pixelPosition = (y * SCREEN_WIDTH + x) * 4
+                    // There's no need to do a manual pixel format conversion. Since getInt() uses the buffer's byte order, which is little endian, it will automatically
+                    // convert the internal BGRA format into the ARGB format, which is what we need to build the bitmap
+                    val argbPixel = frameBuffer.getInt(pixelPosition)
+                    setPixel(x, y, argbPixel)
+                }
+            }
+        }
+    }
+
+    fun clearBuffer() {
+        screenshotBuffer?.let { buffer ->
+            buffer.position(0)
+            repeat(buffer.capacity() / 4) {
+                buffer.putInt(0xFF000000.toInt())
+            }
+        }
+    }
+
+    private fun ensureBufferIsReady(): ByteBuffer {
+        if (screenshotBuffer != null) {
+            return screenshotBuffer!!
+        }
+
+        screenshotBuffer = ByteBuffer.allocateDirect(SCREEN_WIDTH * SCREEN_HEIGHT * 4).order(ByteOrder.nativeOrder())
+        return screenshotBuffer!!
+    }
+}

app/src/main/java/me/magnum/melonds/di/EmulatorRuntimeModule.kt

@@ -14,7 +14,7 @@ import me.magnum.melonds.common.PermissionHandler
 import me.magnum.melonds.common.camera.BlackDSiCameraSource
 import me.magnum.melonds.common.camera.DSiCameraSource
 import me.magnum.melonds.common.romprocessors.RomFileProcessorFactory
-import me.magnum.melonds.common.runtime.FrameBufferProvider
+import me.magnum.melonds.common.runtime.ScreenshotFrameBufferProvider
 import me.magnum.melonds.common.uridelegates.UriHandler
 import me.magnum.melonds.domain.model.camera.DSiCameraSourceType
 import me.magnum.melonds.domain.repositories.SettingsRepository
@@ -40,8 +40,8 @@ object EmulatorRuntimeModule {
 
     @Provides
     @ActivityRetainedScoped
-    fun provideFrameBufferProvider(): FrameBufferProvider {
-        return FrameBufferProvider()
+    fun provideFrameBufferProvider(): ScreenshotFrameBufferProvider {
+        return ScreenshotFrameBufferProvider()
     }
 
     @Provides
@@ -113,7 +113,7 @@ object EmulatorRuntimeModule {
         @ApplicationContext context: Context,
         settingsRepository: SettingsRepository,
         sramProvider: SramProvider,
-        frameBufferProvider: FrameBufferProvider,
+        screenshotFrameBufferProvider: ScreenshotFrameBufferProvider,
         romFileProcessorFactory: RomFileProcessorFactory,
         permissionHandler: PermissionHandler,
         cameraManagerMultiplexer: DSiCameraSourceMultiplexer,
@@ -122,7 +122,7 @@ object EmulatorRuntimeModule {
             context,
             settingsRepository,
             sramProvider,
-            frameBufferProvider,
+            screenshotFrameBufferProvider,
             romFileProcessorFactory,
             permissionHandler,
             cameraManagerMultiplexer,

app/src/main/java/me/magnum/melonds/domain/model/RendererConfiguration.kt

@@ -2,16 +2,11 @@ package me.magnum.melonds.domain.model
 
 data class RendererConfiguration(
     val renderer: VideoRenderer,
-    private val internalVideoFiltering: VideoFiltering,
+    val videoFiltering: VideoFiltering,
     val threadedRendering: Boolean,
     private val internalResolutionScaling: Int,
 ) {
 
-    val videoFiltering get() = when (renderer) {
-        VideoRenderer.SOFTWARE -> internalVideoFiltering
-        VideoRenderer.OPENGL -> VideoFiltering.NONE
-    }
-
     val resolutionScaling get() = when (renderer) {
         VideoRenderer.SOFTWARE -> 1
         VideoRenderer.OPENGL -> internalResolutionScaling

app/src/main/java/me/magnum/melonds/domain/model/render/FrameRenderEvent.kt

@@ -0,0 +1,6 @@
+package me.magnum.melonds.domain.model.render
+
+data class FrameRenderEvent(
+    val glSyncFence: Long,
+    val textureId: Int,
+)

app/src/main/java/me/magnum/melonds/domain/services/EmulatorManager.kt

@@ -7,15 +7,18 @@ import me.magnum.melonds.domain.model.ConsoleType
 import me.magnum.melonds.domain.model.rom.Rom
 import me.magnum.melonds.domain.model.emulator.FirmwareLaunchResult
 import me.magnum.melonds.domain.model.emulator.RomLaunchResult
+import me.magnum.melonds.domain.model.render.FrameRenderEvent
 import me.magnum.melonds.domain.model.retroachievements.GameAchievementData
 import me.magnum.melonds.domain.model.retroachievements.RAEvent
 import me.magnum.melonds.ui.emulator.rewind.model.RewindSaveState
 import me.magnum.melonds.ui.emulator.rewind.model.RewindWindow
 
 interface EmulatorManager {
-    suspend fun loadRom(rom: Rom, cheats: List<Cheat>): RomLaunchResult
+    val frameRenderedEvent: Flow<FrameRenderEvent>
 
-    suspend fun loadFirmware(consoleType: ConsoleType): FirmwareLaunchResult
+    suspend fun loadRom(rom: Rom, cheats: List<Cheat>, glContext: Long): RomLaunchResult
+
+    suspend fun loadFirmware(consoleType: ConsoleType, glContext: Long): FirmwareLaunchResult
 
     suspend fun updateRomEmulatorConfiguration(rom: Rom)
 

app/src/main/java/me/magnum/melonds/impl/emulator/AndroidEmulatorManager.kt

@@ -13,20 +13,21 @@ import me.magnum.melonds.MelonEmulator
 import me.magnum.melonds.common.PermissionHandler
 import me.magnum.melonds.common.RetroAchievementsCallback
 import me.magnum.melonds.common.romprocessors.RomFileProcessorFactory
-import me.magnum.melonds.common.runtime.FrameBufferProvider
+import me.magnum.melonds.common.runtime.ScreenshotFrameBufferProvider
 import me.magnum.melonds.domain.model.Cheat
 import me.magnum.melonds.domain.model.ConsoleType
 import me.magnum.melonds.domain.model.EmulatorConfiguration
 import me.magnum.melonds.domain.model.MicSource
-import me.magnum.melonds.domain.model.rom.Rom
-import me.magnum.melonds.domain.model.rom.config.RuntimeConsoleType
-import me.magnum.melonds.domain.model.rom.config.RuntimeEnum
 import me.magnum.melonds.domain.model.emulator.FirmwareLaunchResult
 import me.magnum.melonds.domain.model.emulator.RomLaunchResult
+import me.magnum.melonds.domain.model.render.FrameRenderEvent
 import me.magnum.melonds.domain.model.retroachievements.GameAchievementData
 import me.magnum.melonds.domain.model.retroachievements.RAEvent
 import me.magnum.melonds.domain.model.retroachievements.RASimpleAchievement
+import me.magnum.melonds.domain.model.rom.Rom
 import me.magnum.melonds.domain.model.rom.config.RomGbaSlotConfig
+import me.magnum.melonds.domain.model.rom.config.RuntimeConsoleType
+import me.magnum.melonds.domain.model.rom.config.RuntimeEnum
 import me.magnum.melonds.domain.repositories.SettingsRepository
 import me.magnum.melonds.domain.services.EmulatorManager
 import me.magnum.melonds.impl.camera.DSiCameraSourceMultiplexer
@@ -38,22 +39,25 @@ class AndroidEmulatorManager(
     private val context: Context,
     private val settingsRepository: SettingsRepository,
     private val sramProvider: SramProvider,
-    private val frameBufferProvider: FrameBufferProvider,
+    private val screenshotFrameBufferProvider: ScreenshotFrameBufferProvider,
     private val romFileProcessorFactory: RomFileProcessorFactory,
     private val permissionHandler: PermissionHandler,
     private val cameraManager: DSiCameraSourceMultiplexer,
 ) : EmulatorManager {
 
     private val achievementsSharedFlow = MutableSharedFlow<RAEvent>(replay = 0, extraBufferCapacity = Int.MAX_VALUE)
 
+    private val _frameRenderedEvent = MutableSharedFlow<FrameRenderEvent>(replay = 0, extraBufferCapacity = 1)
+    override val frameRenderedEvent = _frameRenderedEvent.asSharedFlow()
+
     private val loadedAchievements = mutableListOf<RASimpleAchievement>()
 
-    override suspend fun loadRom(rom: Rom, cheats: List<Cheat>): RomLaunchResult {
+    override suspend fun loadRom(rom: Rom, cheats: List<Cheat>, glContext: Long): RomLaunchResult {
         return withContext(Dispatchers.IO) {
             val fileRomProcessor = romFileProcessorFactory.getFileRomProcessorForDocument(rom.uri)
             val romUri = fileRomProcessor?.getRealRomUri(rom)?.await() ?: throw RomLoadException("Unsupported ROM file extension")
 
-            setupEmulator(getRomEmulatorConfiguration(rom))
+            setupEmulator(getRomEmulatorConfiguration(rom), glContext)
 
             val sram = try {
                 sramProvider.getSramForRom(rom)
@@ -87,9 +91,9 @@ class AndroidEmulatorManager(
         }
     }
 
-    override suspend fun loadFirmware(consoleType: ConsoleType): FirmwareLaunchResult {
+    override suspend fun loadFirmware(consoleType: ConsoleType, glContext: Long): FirmwareLaunchResult {
         return withContext(Dispatchers.IO) {
-            setupEmulator(getFirmwareEmulatorConfiguration(consoleType))
+            setupEmulator(getFirmwareEmulatorConfiguration(consoleType), glContext)
             val result = MelonEmulator.bootFirmware()
             if (result != MelonEmulator.FirmwareLoadResult.SUCCESS) {
                 cameraManager.stopCurrentCameraSource()
@@ -103,14 +107,12 @@ class AndroidEmulatorManager(
 
     override suspend fun updateRomEmulatorConfiguration(rom: Rom) {
         val configuration = getRomEmulatorConfiguration(rom)
-        frameBufferProvider.setRendererConfiguration(configuration.rendererConfiguration)
-        MelonEmulator.updateEmulatorConfiguration(configuration, frameBufferProvider.frameBuffer())
+        MelonEmulator.updateEmulatorConfiguration(configuration)
     }
 
     override suspend fun updateFirmwareEmulatorConfiguration(consoleType: ConsoleType) {
         val configuration = getFirmwareEmulatorConfiguration(consoleType)
-        frameBufferProvider.setRendererConfiguration(configuration.rendererConfiguration)
-        MelonEmulator.updateEmulatorConfiguration(configuration, frameBufferProvider.frameBuffer())
+        MelonEmulator.updateEmulatorConfiguration(configuration)
     }
 
     override suspend fun getRewindWindow(): RewindWindow {
@@ -181,14 +183,12 @@ class AndroidEmulatorManager(
         return achievementsSharedFlow.asSharedFlow()
     }
 
-    private fun setupEmulator(emulatorConfiguration: EmulatorConfiguration) {
-        frameBufferProvider.setRendererConfiguration(emulatorConfiguration.rendererConfiguration)
-
+    private fun setupEmulator(emulatorConfiguration: EmulatorConfiguration, glContext: Long) {
         MelonEmulator.setupEmulator(
-            emulatorConfiguration,
-            context.assets,
-            cameraManager,
-            object : RetroAchievementsCallback {
+            emulatorConfiguration = emulatorConfiguration,
+            assetManager = context.assets,
+            dsiCameraSource = cameraManager,
+            retroAchievementsCallback = object : RetroAchievementsCallback {
                 override fun onAchievementPrimed(achievementId: Long) {
                     achievementsSharedFlow.tryEmit(RAEvent.OnAchievementPrimed(achievementId))
                 }
@@ -201,7 +201,11 @@ class AndroidEmulatorManager(
                     achievementsSharedFlow.tryEmit(RAEvent.OnAchievementUnPrimed(achievementId))
                 }
             },
-            frameBufferProvider.frameBuffer()
+            frameRenderedListener = { glFenceSync, textureId ->
+                _frameRenderedEvent.tryEmit(FrameRenderEvent(glFenceSync, textureId))
+            },
+            screenshotBuffer = screenshotFrameBufferProvider.frameBuffer(),
+            glContext = glContext,
         )
     }
 

app/src/main/java/me/magnum/melonds/ui/emulator/DSRenderer.kt

@@ -2,30 +2,33 @@ package me.magnum.melonds.ui.emulator
 
 import android.content.Context
 import android.graphics.BitmapFactory
+import android.opengl.EGL14
 import android.opengl.GLES20
+import android.opengl.GLES30
 import android.opengl.GLSurfaceView
 import android.opengl.GLUtils
 import android.opengl.Matrix
 import me.magnum.melonds.common.opengl.Shader
 import me.magnum.melonds.common.opengl.ShaderFactory
 import me.magnum.melonds.common.opengl.ShaderProgramSource
-import me.magnum.melonds.common.runtime.FrameBufferProvider
-import me.magnum.melonds.domain.model.layout.BackgroundMode
 import me.magnum.melonds.domain.model.Rect
 import me.magnum.melonds.domain.model.RuntimeBackground
 import me.magnum.melonds.domain.model.VideoFiltering
+import me.magnum.melonds.domain.model.layout.BackgroundMode
+import me.magnum.melonds.domain.model.render.FrameRenderEvent
 import me.magnum.melonds.ui.emulator.model.RuntimeRendererConfiguration
 import me.magnum.melonds.utils.BitmapUtils
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
 import java.nio.FloatBuffer
+import java.util.LinkedList
 import javax.microedition.khronos.egl.EGLConfig
 import javax.microedition.khronos.opengles.GL10
 import kotlin.math.roundToInt
 
 class DSRenderer(
-    private val frameBufferProvider: FrameBufferProvider,
     private val context: Context,
+    private val onGlContextReady: (glContext: Long) -> Unit,
 ) : GLSurfaceView.Renderer {
     companion object {
         private const val SCREEN_WIDTH = 256
@@ -43,13 +46,13 @@ class DSRenderer(
         )
     }
 
+    private val renderEventQueue = LinkedList<FrameRenderEvent>()
     private var rendererConfiguration: RuntimeRendererConfiguration? = null
     private var mustUpdateConfiguration = false
     private var isBackgroundPositionDirty = false
     private var isBackgroundLoaded = false
 
     private var backgroundTexture = 0
-    private var mainTexture = 0
 
     private var screenShader: Shader? = null
     private lateinit var backgroundShader: Shader
@@ -98,6 +101,12 @@ class DSRenderer(
         }
     }
 
+    fun prepareNextFrame(frameRenderEvent: FrameRenderEvent) {
+        synchronized(renderEventQueue) {
+            renderEventQueue.add(frameRenderEvent)
+        }
+    }
+
     private fun screenXToViewportX(x: Int): Float {
         return (x / this.width) * 2f - 1f
     }
@@ -108,16 +117,13 @@ class DSRenderer(
 
     override fun onSurfaceCreated(gl: GL10, config: EGLConfig) {
         GLES20.glClearColor(0f, 0f, 0f, 1f)
+        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
         GLES20.glDisable(GLES20.GL_CULL_FACE)
 
         // Setup textures
-        val textures = IntArray(2)
-        GLES20.glGenTextures(2, textures, 0)
-        mainTexture = textures[0]
-        backgroundTexture = textures[1]
-        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mainTexture)
-        GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE.toFloat())
-        GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE.toFloat())
+        val textures = IntArray(1)
+        GLES20.glGenTextures(1, textures, 0)
+        backgroundTexture = textures[0]
 
         GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, backgroundTexture)
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE.toFloat())
@@ -137,6 +143,7 @@ class DSRenderer(
         Matrix.multiplyMM(mvpMatrix, 0, projectionMatrix, 0, viewMatrix, 0)
 
         applyRendererConfiguration()
+        onGlContextReady(EGL14.eglGetCurrentContext().nativeHandle)
     }
 
     private fun applyRendererConfiguration() {
@@ -162,9 +169,11 @@ class DSRenderer(
         //   3                         5
         // Texture is vertically flipped
 
+        // The texture will have 2 lines between the screens. Take that into account when computing UVs
+        val lineRelativeSize = 1 / (SCREEN_HEIGHT + 1).toFloat()
         topScreenRect?.let {
             uvs.add(0f)
-            uvs.add(0.5f)
+            uvs.add(0.5f - lineRelativeSize)
 
             uvs.add(0f)
             uvs.add(0f)
@@ -173,13 +182,13 @@ class DSRenderer(
             uvs.add(0f)
 
             uvs.add(0f)
-            uvs.add(0.5f)
+            uvs.add(0.5f - lineRelativeSize)
 
             uvs.add(1f)
             uvs.add(0f)
 
             uvs.add(1f)
-            uvs.add(0.5f)
+            uvs.add(0.5f - lineRelativeSize)
 
             coords.add(screenXToViewportX(it.x))
             coords.add(screenYToViewportY(it.y + it.height))
@@ -204,16 +213,16 @@ class DSRenderer(
             uvs.add(1f)
 
             uvs.add(0f)
-            uvs.add(0.5f)
+            uvs.add(0.5f + lineRelativeSize)
 
             uvs.add(1f)
-            uvs.add(0.5f)
+            uvs.add(0.5f + lineRelativeSize)
 
             uvs.add(0f)
             uvs.add(1f)
 
             uvs.add(1f)
-            uvs.add(0.5f)
+            uvs.add(0.5f + lineRelativeSize)
 
             uvs.add(1f)
             uvs.add(1f)
@@ -254,15 +263,6 @@ class DSRenderer(
 
         val shaderSource = FILTERING_SHADER_MAP[rendererConfiguration?.videoFiltering ?: VideoFiltering.NONE] ?: throw Exception("Invalid video filtering")
         screenShader = ShaderFactory.createShaderProgram(shaderSource)
-
-        val textureFilter = when (shaderSource.textureFiltering) {
-            ShaderProgramSource.TextureFiltering.NEAREST -> GLES20.GL_NEAREST
-            ShaderProgramSource.TextureFiltering.LINEAR -> GLES20.GL_LINEAR
-        }
-
-        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mainTexture)
-        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, textureFilter)
-        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, textureFilter)
     }
 
     override fun onSurfaceChanged(gl: GL10, width: Int, height: Int) {
@@ -282,12 +282,20 @@ class DSRenderer(
             mustUpdateConfiguration = false
         }
 
-        if (!frameBufferProvider.isFrameBufferReady()) {
-            return
+        val currentGlFenceSync: Long
+        val currentTextureId: Int
+        synchronized(renderEventQueue) {
+            val renderEvent = renderEventQueue.removeLastOrNull() ?: return
+            currentGlFenceSync = renderEvent.glSyncFence
+            currentTextureId = renderEvent.textureId
+
+            while (renderEventQueue.isNotEmpty()) {
+                // Discard old events
+                val discardedEvent = renderEventQueue.removeLast()
+                GLES30.glDeleteSync(discardedEvent.glSyncFence)
+            }
         }
 
-        val frameBuffer = frameBufferProvider.frameBuffer()
-
         GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
 
         synchronized(backgroundLock) {
@@ -296,18 +304,23 @@ class DSRenderer(
 
         posBuffer.position(0)
         uvBuffer.position(0)
-        frameBuffer.position(0)
 
         val indices = posBuffer.capacity() / 2
-        screenShader?.let {
-            it.use()
+        screenShader?.let { shader ->
+            shader.use()
+
+            GLES30.glWaitSync(currentGlFenceSync, GLES30.GL_SYNC_FLUSH_COMMANDS_BIT, 100_000_000)
+            GLES30.glDeleteSync(currentGlFenceSync)
+
             GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
-            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mainTexture)
-            GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, internalWidth, internalHeight, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, frameBuffer)
-            GLES20.glUniformMatrix4fv(it.uniformMvp, 1, false, mvpMatrix, 0)
-            GLES20.glVertexAttribPointer(it.attribPos, 2, GLES20.GL_FLOAT, false, 0, posBuffer)
-            GLES20.glVertexAttribPointer(it.attribUv, 2, GLES20.GL_FLOAT, false, 0, uvBuffer)
-            GLES20.glUniform1i(it.uniformTex, 0)
+            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, currentTextureId)
+            GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, shader.textureFiltering)
+            GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, shader.textureFiltering)
+
+            GLES20.glUniformMatrix4fv(shader.uniformMvp, 1, false, mvpMatrix, 0)
+            GLES20.glVertexAttribPointer(shader.attribPos, 2, GLES20.GL_FLOAT, false, 0, posBuffer)
+            GLES20.glVertexAttribPointer(shader.attribUv, 2, GLES20.GL_FLOAT, false, 0, uvBuffer)
+            GLES20.glUniform1i(shader.uniformTex, 0)
             GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, indices)
         }
     }

app/src/main/java/me/magnum/melonds/ui/emulator/EmulatorActivity.kt

@@ -4,6 +4,7 @@ import android.content.Context
 import android.content.Intent
 import android.content.res.Configuration
 import android.net.Uri
+import android.opengl.GLSurfaceView
 import android.os.Build
 import android.os.Bundle
 import android.view.KeyEvent
@@ -58,23 +59,25 @@ import androidx.window.layout.WindowInfoTracker
 import com.squareup.picasso.Picasso
 import dagger.hilt.android.AndroidEntryPoint
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.collectLatest
+import kotlinx.coroutines.flow.filterNotNull
+import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.merge
 import kotlinx.coroutines.launch
 import me.magnum.melonds.MelonEmulator
 import me.magnum.melonds.R
 import me.magnum.melonds.common.PermissionHandler
-import me.magnum.melonds.common.runtime.FrameBufferProvider
 import me.magnum.melonds.databinding.ActivityEmulatorBinding
 import me.magnum.melonds.domain.model.ConsoleType
 import me.magnum.melonds.domain.model.FpsCounterPosition
-import me.magnum.melonds.domain.model.layout.LayoutComponent
 import me.magnum.melonds.domain.model.Rect
-import me.magnum.melonds.domain.model.ui.Orientation
-import me.magnum.melonds.domain.model.rom.Rom
 import me.magnum.melonds.domain.model.SaveStateSlot
+import me.magnum.melonds.domain.model.layout.LayoutComponent
 import me.magnum.melonds.domain.model.layout.ScreenFold
+import me.magnum.melonds.domain.model.rom.Rom
+import me.magnum.melonds.domain.model.ui.Orientation
 import me.magnum.melonds.domain.repositories.SettingsRepository
 import me.magnum.melonds.extensions.insetsControllerCompat
 import me.magnum.melonds.extensions.parcelable
@@ -139,15 +142,13 @@ class EmulatorActivity : AppCompatActivity() {
     @Inject
     lateinit var picasso: Picasso
 
-    @Inject
-    lateinit var frameBufferProvider: FrameBufferProvider
-
     @Inject
     lateinit var permissionHandler: PermissionHandler
 
     @Inject
     lateinit var lifecycleOwnerProvider: LifecycleOwnerProvider
 
+    private val currentOpenGlContext = MutableStateFlow<Long?>(null)
     private lateinit var dsRenderer: DSRenderer
     private lateinit var melonTouchHandler: MelonTouchHandler
     private lateinit var nativeInputListener: INativeInputListener
@@ -238,13 +239,17 @@ class EmulatorActivity : AppCompatActivity() {
         onBackPressedDispatcher.addCallback(backPressedCallback)
 
         melonTouchHandler = MelonTouchHandler()
-        dsRenderer = DSRenderer(frameBufferProvider, this)
+        dsRenderer = DSRenderer(
+            context = this,
+            onGlContextReady = {
+                currentOpenGlContext.value = it
+            }
+        )
         binding.surfaceMain.apply {
             setEGLContextClientVersion(2)
             preserveEGLContextOnPause = true
-            /*setEGLConfigChooser(8, 8, 8, 8, 0, 0)
-            holder.setFormat(PixelFormat.RGBA_8888)*/
             setRenderer(dsRenderer)
+            renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY
         }
 
         binding.textFps.visibility = View.INVISIBLE
@@ -393,6 +398,14 @@ class EmulatorActivity : AppCompatActivity() {
             }
         }
 
+        lifecycleScope.launch {
+            lifecycle.repeatOnLifecycle(Lifecycle.State.CREATED) {
+                viewModel.frameRenderEvent.collect {
+                    dsRenderer.prepareNextFrame(it)
+                    binding.surfaceMain.requestRender()
+                }
+            }
+        }
         lifecycleScope.launch {
             lifecycle.repeatOnLifecycle(Lifecycle.State.CREATED) {
                 viewModel.runtimeLayout.collectLatest {
@@ -589,29 +602,33 @@ class EmulatorActivity : AppCompatActivity() {
         val extras = intent?.extras
         val bootFirmwareOnly = extras?.getBoolean(KEY_BOOT_FIRMWARE_ONLY) ?: false
 
-        disableScreenTimeOut()
-        if (bootFirmwareOnly) {
-            val consoleTypeParameter = extras?.getInt(KEY_BOOT_FIRMWARE_CONSOLE, -1)
-            if (consoleTypeParameter == null || consoleTypeParameter == -1) {
-                throw RuntimeException("No console type specified")
-            }
+        lifecycleScope.launch {
+            val glContext = currentOpenGlContext.filterNotNull().first()
 
-            val firmwareConsoleType = ConsoleType.entries[consoleTypeParameter]
-            viewModel.loadFirmware(firmwareConsoleType)
-        } else {
-            val romParcelable = extras?.parcelable(KEY_ROM) as RomParcelable?
+            disableScreenTimeOut()
+            if (bootFirmwareOnly) {
+                val consoleTypeParameter = extras?.getInt(KEY_BOOT_FIRMWARE_CONSOLE, -1)
+                if (consoleTypeParameter == null || consoleTypeParameter == -1) {
+                    throw RuntimeException("No console type specified")
+                }
 
-            if (romParcelable?.rom != null) {
-                viewModel.loadRom(romParcelable.rom)
+                val firmwareConsoleType = ConsoleType.entries[consoleTypeParameter]
+                viewModel.loadFirmware(firmwareConsoleType, glContext)
             } else {
-                if (extras?.containsKey(KEY_PATH) == true) {
-                    val romPath = extras.getString(KEY_PATH)!!
-                    viewModel.loadRom(romPath)
-                } else if (extras?.containsKey(KEY_URI) == true) {
-                    val romUri = extras.getString(KEY_URI)!!
-                    viewModel.loadRom(romUri.toUri())
+                val romParcelable = extras?.parcelable(KEY_ROM) as RomParcelable?
+
+                if (romParcelable?.rom != null) {
+                    viewModel.loadRom(romParcelable.rom, glContext)
                 } else {
-                    throw RuntimeException("No ROM was specified")
+                    if (extras?.containsKey(KEY_PATH) == true) {
+                        val romPath = extras.getString(KEY_PATH)!!
+                        viewModel.loadRom(romPath, glContext)
+                    } else if (extras?.containsKey(KEY_URI) == true) {
+                        val romUri = extras.getString(KEY_URI)!!
+                        viewModel.loadRom(romUri.toUri(), glContext)
+                    } else {
+                        throw RuntimeException("No ROM was specified")
+                    }
                 }
             }
         }

app/src/main/java/me/magnum/melonds/ui/emulator/EmulatorFrameRenderedListener.kt

@@ -0,0 +1,5 @@
+package me.magnum.melonds.ui.emulator
+
+fun interface EmulatorFrameRenderedListener {
+    fun onFrameRendered(glFenceSync: Long, textureId: Int)
+}

app/src/main/java/me/magnum/melonds/ui/emulator/EmulatorViewModel.kt

@@ -38,7 +38,7 @@ import kotlinx.coroutines.rx2.awaitSingleOrNull
 import kotlinx.coroutines.rx2.rxMaybe
 import me.magnum.melonds.MelonEmulator
 import me.magnum.melonds.common.romprocessors.RomFileProcessorFactory
-import me.magnum.melonds.common.runtime.FrameBufferProvider
+import me.magnum.melonds.common.runtime.ScreenshotFrameBufferProvider
 import me.magnum.melonds.domain.model.Cheat
 import me.magnum.melonds.domain.model.ConsoleType
 import me.magnum.melonds.domain.model.FpsCounterPosition
@@ -96,7 +96,7 @@ class EmulatorViewModel @Inject constructor(
     private val layoutsRepository: LayoutsRepository,
     private val backgroundsRepository: BackgroundRepository,
     private val saveStatesRepository: SaveStatesRepository,
-    private val frameBufferProvider: FrameBufferProvider,
+    private val screenshotFrameBufferProvider: ScreenshotFrameBufferProvider,
     private val uiLayoutProvider: UILayoutProvider,
     private val emulatorManager: EmulatorManager,
     private val emulatorSession: EmulatorSession,
@@ -110,6 +110,8 @@ class EmulatorViewModel @Inject constructor(
 
     private val _layout = MutableStateFlow<LayoutConfiguration?>(null)
 
+    val frameRenderEvent = emulatorManager.frameRenderedEvent
+
     private val _runtimeLayout = MutableStateFlow<RuntimeInputLayoutConfiguration?>(null)
     val runtimeLayout = _runtimeLayout.asStateFlow()
 
@@ -142,44 +144,44 @@ class EmulatorViewModel @Inject constructor(
         }
     }
 
-    fun loadRom(rom: Rom) {
+    fun loadRom(rom: Rom, glContext: Long) {
         viewModelScope.launch {
             resetEmulatorState(EmulatorState.LoadingRom)
             sessionCoroutineScope.launch {
-                launchRom(rom)
+                launchRom(rom, glContext)
             }
         }
     }
 
-    fun loadRom(romUri: Uri) {
+    fun loadRom(romUri: Uri, glContext: Long) {
         viewModelScope.launch {
             resetEmulatorState(EmulatorState.LoadingRom)
             sessionCoroutineScope.launch {
                 val rom = getRomAtUri(romUri).awaitSingleOrNull()
                 if (rom != null) {
-                    launchRom(rom)
+                    launchRom(rom, glContext)
                 } else {
                     _emulatorState.value = EmulatorState.RomNotFoundError(romUri.toString())
                 }
             }
         }
     }
 
-    fun loadRom(romPath: String) {
+    fun loadRom(romPath: String, glContext: Long) {
         viewModelScope.launch {
             resetEmulatorState(EmulatorState.LoadingRom)
             sessionCoroutineScope.launch {
                 val rom = getRomAtPath(romPath).awaitSingleOrNull()
                 if (rom != null) {
-                    launchRom(rom)
+                    launchRom(rom, glContext)
                 } else {
                     _emulatorState.value = EmulatorState.RomNotFoundError(romPath)
                 }
             }
         }
     }
 
-    private suspend fun launchRom(rom: Rom) = coroutineScope {
+    private suspend fun launchRom(rom: Rom, glContext: Long) = coroutineScope {
         startEmulatorSession(EmulatorSession.SessionType.RomSession(rom))
         startObservingBackground()
         startObservingRuntimeInputLayoutConfiguration()
@@ -189,7 +191,7 @@ class EmulatorViewModel @Inject constructor(
         startRetroAchievementsSession(rom)
 
         val cheats = getRomInfo(rom)?.let { getRomEnabledCheats(it) } ?: emptyList()
-        val result = emulatorManager.loadRom(rom, cheats)
+        val result = emulatorManager.loadRom(rom, cheats, glContext)
         when (result) {
             is RomLaunchResult.LaunchFailedSramProblem,
             is RomLaunchResult.LaunchFailed -> {
@@ -205,7 +207,7 @@ class EmulatorViewModel @Inject constructor(
         }
     }
 
-    fun loadFirmware(consoleType: ConsoleType) {
+    fun loadFirmware(consoleType: ConsoleType, glContext: Long) {
         viewModelScope.launch {
             resetEmulatorState(EmulatorState.LoadingFirmware)
             startEmulatorSession(EmulatorSession.SessionType.FirmwareSession(consoleType))
@@ -215,7 +217,7 @@ class EmulatorViewModel @Inject constructor(
                 startObservingRendererConfiguration()
                 startObservingLayoutForFirmware()
 
-                val result = emulatorManager.loadFirmware(consoleType)
+                val result = emulatorManager.loadFirmware(consoleType, glContext)
                 when (result) {
                     is FirmwareLaunchResult.LaunchFailed -> {
                         _emulatorState.value = EmulatorState.FirmwareLoadError(result.reason)
@@ -313,7 +315,7 @@ class EmulatorViewModel @Inject constructor(
 
     fun stopEmulator() {
         emulatorManager.stopEmulator()
-        frameBufferProvider.clearFrameBuffer()
+        screenshotFrameBufferProvider.clearBuffer()
     }
 
     fun onPauseMenuOptionSelected(option: PauseMenuOption) {
@@ -469,7 +471,7 @@ class EmulatorViewModel @Inject constructor(
     private suspend fun saveRomState(rom: Rom, slot: SaveStateSlot): Boolean {
         val slotUri = saveStatesRepository.getRomSaveStateUri(rom, slot)
         return if (emulatorManager.saveState(slotUri)) {
-            val screenshot = frameBufferProvider.getScreenshot()
+            val screenshot = screenshotFrameBufferProvider.getScreenshot()
             saveStatesRepository.setRomSaveStateScreenshot(rom, slot, screenshot)
             true
         } else {

app/src/main/java/me/magnum/melonds/ui/settings/fragments/VideoPreferencesFragment.kt

@@ -36,7 +36,6 @@ class VideoPreferencesFragment : PreferenceFragmentCompat(), PreferenceFragmentT
         setPreferencesFromResource(R.xml.pref_video, rootKey)
 
         softwareRendererPreferences.apply {
-            add(findPreference("video_filtering")!!)
             add(findPreference("enable_threaded_rendering")!!)
         }
 

app/src/main/res/xml/pref_video.xml

@@ -21,6 +21,13 @@
             android:entryValues="@array/video_internal_resolution_values"
             android:defaultValue="1"/>
 
+    <SwitchPreference
+            android:key="enable_threaded_rendering"
+            android:title="@string/threaded_rendering"
+            android:summary="@string/threaded_rendering_summary"
+            app:iconSpaceReserved="false"
+            android:defaultValue="true" />
+
     <ListPreference
             android:key="video_filtering"
             android:title="@string/filter"
@@ -30,13 +37,6 @@
             android:entryValues="@array/video_filtering_values"
             android:defaultValue="linear"/>
 
-    <SwitchPreference
-            android:key="enable_threaded_rendering"
-            android:title="@string/threaded_rendering"
-            android:summary="@string/threaded_rendering_summary"
-            app:iconSpaceReserved="false"
-            android:defaultValue="true" />
-
     <ListPreference
             android:key="fps_counter_position"
             android:title="@string/fps_counter_position"