diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index cb1deceb96..23ed09a2ce 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -282,6 +282,11 @@ if (ENABLE_WEB_SERVICE OR ENABLE_UPDATE_CHECKER) AddJsonPackage(cpp-jwt) endif() +# libgav1 (AV1 video decoding where FFmpeg has no libdav1d: Android) +if (ANDROID) + add_subdirectory(libgav1) +endif() + # FFMpeg if (YUZU_USE_EXTERNAL_FFMPEG OR YUZU_USE_BUNDLED_FFMPEG) add_subdirectory(ffmpeg) diff --git a/externals/libgav1/CMakeLists.txt b/externals/libgav1/CMakeLists.txt new file mode 100644 index 0000000000..f5e0739b57 --- /dev/null +++ b/externals/libgav1/CMakeLists.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# libgav1 decodes the AV1 movies of Switch 2 titles (nx2 video decoder) where the bundled FFmpeg +# has no libdav1d (Android's package); with libdav1d, FFmpeg decodes them. + +include(FetchContent) + +set(LIBGAV1_ENABLE_TESTS OFF) +set(LIBGAV1_ENABLE_EXAMPLES OFF) +# std::mutex in its thread pool: no Abseil (it is only needed for absl::Mutex, tests and examples) +set(LIBGAV1_THREADPOOL_USE_STD_MUTEX 1) +# 8 and 10-bit streams only, which leaves the 12-bit DSP code out +set(LIBGAV1_MAX_BITDEPTH 10) +FetchContent_Declare(libgav1 + GIT_REPOSITORY https://chromium.googlesource.com/codecs/libgav1 + GIT_TAG c1deec657b32b911920c78e078cfd089faa77200 +) +FetchContent_MakeAvailable(libgav1) + +set_property(GLOBAL APPEND PROPERTY CPM_PACKAGE_NAMES libgav1) +set_property(GLOBAL APPEND PROPERTY CPM_PACKAGE_URLS https://chromium.googlesource.com/codecs/libgav1) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 515914f9a5..3eb2d26b1d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -153,6 +153,10 @@ else() $<$:-pipe> # Disable RTTI (C++ only) $<$:-fno-rtti>) + else() + # clang-cl: like the MSVC branch, or windows.h (included by vulkan.h) defines min/max + # macros that break std::min/std::max calls + add_compile_definitions(NOMINMAX) endif() if (ENABLE_WERROR) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt index 77222884c6..c7956847cf 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt @@ -106,9 +106,17 @@ object NativeLibrary { if (DocumentsTree.isNativePath(path)) { YuzuApplication.documentsTree!!.getParentDirectory(path) } else { - path + FileUtil.getParentUri(path) ?: path } + /** + * Children of a content URI directory, four strings each: "d" or "f", size (-1 if unknown), + * display name and document URI. Native code finds files next to a loose ExeFS with it. + */ + @Keep + @JvmStatic + fun listChildren(path: String): Array = FileUtil.listChildren(path) + @Keep @JvmStatic fun getFilename(path: String): String = diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt index 6c33fc7812..187621f971 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt @@ -26,6 +26,13 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_FORCE_MAX_CLOCK("force_max_clock"), RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"), RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"), + RENDERER_REDUCE_TEXTURE_RESOLUTION("reduce_texture_resolution"), + RENDERER_RELEASE_GPU_ONLY_MEMORY("release_gpu_only_memory"), + RENDERER_NX2_COMMAND_LISTS("nx2_command_lists"), + RENDERER_MKW_30FPS("mkw_30fps"), + RENDERER_MKW_60FPS("mkw_60fps"), + RENDERER_MKW_LOD_DISTANCE("mkw_lod_distance"), + RENDERER_MKW_SHADOW_CASCADES("mkw_shadow_cascades"), RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"), ENABLE_BUFFER_HISTORY("enable_buffer_history"), USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"), diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt index f968a0ae18..85863bbd32 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt @@ -778,6 +778,55 @@ abstract class SettingsItem( descriptionId = R.string.renderer_asynchronous_shaders_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_REDUCE_TEXTURE_RESOLUTION, + titleId = R.string.renderer_reduce_texture_resolution, + descriptionId = R.string.renderer_reduce_texture_resolution_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_RELEASE_GPU_ONLY_MEMORY, + titleId = R.string.renderer_release_gpu_only_memory, + descriptionId = R.string.renderer_release_gpu_only_memory_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_NX2_COMMAND_LISTS, + titleId = R.string.renderer_nx2_command_lists, + descriptionId = R.string.renderer_nx2_command_lists_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_MKW_30FPS, + titleId = R.string.renderer_mkw_30fps, + descriptionId = R.string.renderer_mkw_30fps_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_MKW_60FPS, + titleId = R.string.renderer_mkw_60fps, + descriptionId = R.string.renderer_mkw_60fps_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_MKW_LOD_DISTANCE, + titleId = R.string.renderer_mkw_lod_distance, + descriptionId = R.string.renderer_mkw_lod_distance_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_MKW_SHADOW_CASCADES, + titleId = R.string.renderer_mkw_shadow_cascades, + descriptionId = R.string.renderer_mkw_shadow_cascades_description + ) + ) put( SingleChoiceSetting( IntSetting.FAST_GPU_TIME, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt index 99b45cf3f3..bbbddb4887 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt @@ -558,6 +558,13 @@ class SettingsFragmentPresenter( add(BooleanSetting.FIX_BLOOM_EFFECTS.key) add(BooleanSetting.EMULATE_BGR565.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key) + add(BooleanSetting.RENDERER_REDUCE_TEXTURE_RESOLUTION.key) + add(BooleanSetting.RENDERER_RELEASE_GPU_ONLY_MEMORY.key) + add(BooleanSetting.RENDERER_NX2_COMMAND_LISTS.key) + add(BooleanSetting.RENDERER_MKW_30FPS.key) + add(BooleanSetting.RENDERER_MKW_60FPS.key) + add(BooleanSetting.RENDERER_MKW_LOD_DISTANCE.key) + add(BooleanSetting.RENDERER_MKW_SHADOW_CASCADES.key) add(IntSetting.ANDROID_PIPELINE_WORKERS.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key) add(SettingsItem.GPU_UNSWIZZLE_COMBINED) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt index 9db255d70c..dd1e22850f 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt @@ -239,7 +239,9 @@ class EmulationFragment : Fragment(), SurfaceHolder.Callback { handleEmuReadyIntent(intent) return } else if (intentUri != null) { - intentGame = if (Game.extensions.contains(FileUtil.getExtension(intentUri))) { + intentGame = if (Game.extensions.contains(FileUtil.getExtension(intentUri)) || + FileUtil.getFilename(intentUri) == "main" // a loose ExeFS + ) { GameHelper.getGame(requireActivity().intent.data!!, false) } else { null diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/HomeSettingsFragment.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/HomeSettingsFragment.kt index 2b4cfc5ca8..16db01eb4a 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/HomeSettingsFragment.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/HomeSettingsFragment.kt @@ -258,6 +258,14 @@ class HomeSettingsFragment : Fragment() { } ) ) + add( + HomeSetting( + R.string.load_file, + R.string.load_file_description, + R.drawable.ic_folder_open, + { mainActivity.loadGameFile.launch(arrayOf("*/*")) } + ) + ) add( HomeSetting( R.string.verify_installed_content, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/Game.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/Game.kt index f5f0cda0e9..6e32478cd5 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/Game.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/Game.kt @@ -100,7 +100,7 @@ class Game( companion object { val extensions: Set = HashSet( - listOf("xci", "nsp", "nca", "nro", "nsz", "xcz") + listOf("xci", "nsp", "nca", "nro", "nsz", "xcz", "nx2", "nup") ) } } diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/ui/main/MainActivity.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/ui/main/MainActivity.kt index 1da275ab87..5f15548cab 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/ui/main/MainActivity.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/ui/main/MainActivity.kt @@ -33,6 +33,8 @@ import org.yuzu.yuzu_emu.dialogs.NetPlayDialog import org.yuzu.yuzu_emu.features.settings.model.Settings import org.yuzu.yuzu_emu.fragments.AddGameFolderDialogFragment import org.yuzu.yuzu_emu.fragments.MessageDialogFragment +import org.yuzu.yuzu_emu.fragments.ProgressDialogFragment +import org.yuzu.yuzu_emu.HomeNavigationDirections import org.yuzu.yuzu_emu.model.AddonViewModel import org.yuzu.yuzu_emu.model.DriverViewModel import org.yuzu.yuzu_emu.model.GamesViewModel @@ -405,6 +407,87 @@ class MainActivity : AppCompatActivity(), ThemeProvider { } } + // "Load file": boots a picked file without adding it to the library. A loose ExeFS main also + // needs its folder (main.npdm, sdk, subsdk*, romfs/), which picking one file does not grant: + // it boots when a granted tree holds it, else the folder is asked for. + val loadGameFile = + registerForActivityResult(ActivityResultContracts.OpenDocument()) { result -> + if (result != null) { + processLoadGameFile(result) + } + } + + val loadLooseExeFsFolder = + registerForActivityResult(ActivityResultContracts.OpenDocumentTree()) { result -> + if (result != null) { + processLooseExeFsFolder(result) + } + } + + private fun processLoadGameFile(result: Uri) { + val filename = FileUtil.getFilename(result) + if (filename != "main" && filename != "main.npdm") { + try { + contentResolver.takePersistableUriPermission( + result, + Intent.FLAG_GRANT_READ_URI_PERMISSION + ) + } catch (e: SecurityException) { + // Not persistable: the grant still lasts while the app runs + } + bootGameUri { result } + return + } + + val treeUri = FileUtil.getTreeDocumentUri(result) + if (treeUri == null) { + MessageDialogFragment.newInstance( + this, + titleId = R.string.loose_exefs_folder_access, + descriptionId = R.string.loose_exefs_folder_access_description, + positiveAction = { loadLooseExeFsFolder.launch(null) } + ).show(supportFragmentManager, MessageDialogFragment.TAG) + return + } + bootGameUri { + FileUtil.getParentUri(treeUri.toString())?.let { + GameHelper.findLooseExeFsMain(FileUtil.listFiles(it.toUri())) + } + } + } + + private fun processLooseExeFsFolder(result: Uri) { + contentResolver.takePersistableUriPermission( + result, + Intent.FLAG_GRANT_READ_URI_PERMISSION + ) + bootGameUri { GameHelper.findLooseExeFsMain(FileUtil.listFiles(result)) } + } + + // Identifies the game off the main thread (a loose ExeFS indexes its whole RomFS through the + // document provider, which takes a while) and boots it. + private fun bootGameUri(findGameUri: () -> Uri?) { + ProgressDialogFragment.newInstance(this, R.string.loading_game) { _, _ -> + val gameUri = findGameUri() + ?: return@newInstance MessageDialogFragment.newInstance( + titleId = R.string.load_file_failed, + descriptionId = R.string.loose_exefs_not_found + ) + val game = GameHelper.getGame(gameUri, false) + ?: return@newInstance MessageDialogFragment.newInstance( + titleId = R.string.load_file_failed, + descriptionId = R.string.load_file_invalid + ) + withContext(Dispatchers.Main) { + val navHostFragment = supportFragmentManager + .findFragmentById(R.id.fragment_container) as NavHostFragment + navHostFragment.navController.navigate( + HomeNavigationDirections.actionGlobalEmulationActivity(game, true) + ) + } + }.show(supportFragmentManager, ProgressDialogFragment.TAG) + } + fun processGamesDir(result: Uri, calledFromGameFragment: Boolean = false) { contentResolver.takePersistableUriPermission( result, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/FileUtil.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/FileUtil.kt index 1deba1aade..f42e74bbb0 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/FileUtil.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/FileUtil.kt @@ -8,6 +8,7 @@ package org.yuzu.yuzu_emu.utils import android.database.Cursor import android.net.Uri +import android.os.Build import android.provider.DocumentsContract import androidx.documentfile.provider.DocumentFile import java.io.BufferedInputStream @@ -146,6 +147,123 @@ object FileUtil { return results.toTypedArray() } + /** + * Children of a directory in a granted tree, flattened for native code: four strings per + * child, "d" or "f", size in bytes (-1 if the provider does not report it), display name and + * document URI. + * @param path Native content uri path of a tree or a directory inside one + */ + fun listChildren(path: String): Array { + val uri = path.toUri() + if (!DocumentsContract.isTreeUri(uri)) { + return emptyArray() + } + val columns = arrayOf( + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + DocumentsContract.Document.COLUMN_DISPLAY_NAME, + DocumentsContract.Document.COLUMN_MIME_TYPE, + DocumentsContract.Document.COLUMN_SIZE + ) + var c: Cursor? = null + val results = mutableListOf() + try { + val docId = if (isRootTreeUri(uri)) { + DocumentsContract.getTreeDocumentId(uri) + } else { + DocumentsContract.getDocumentId(uri) + } + val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, docId) + c = context.contentResolver.query(childrenUri, columns, null, null, null) + while (c!!.moveToNext()) { + val isDirectory = c.getString(2) == DocumentsContract.Document.MIME_TYPE_DIR + results.add(if (isDirectory) "d" else "f") + results.add(if (c.isNull(3)) "-1" else c.getLong(3).toString()) + results.add(c.getString(1)) + results.add( + DocumentsContract.buildDocumentUriUsingTree(uri, c.getString(0)).toString() + ) + } + } catch (e: Exception) { + Log.error("[FileUtil]: Cannot list children of $path: " + e.message) + } finally { + closeQuietly(c) + } + return results.toTypedArray() + } + + /** + * Parent directory of a document inside a granted tree, or null at the tree's root and for + * documents outside a tree. + * @param path Native content uri path + */ + fun getParentUri(path: String): String? { + val uri = path.toUri() + if (!DocumentsContract.isTreeUri(uri) || isRootTreeUri(uri)) { + return null + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + try { + // The tree root's ID first, the document's own ID last. + val documentIds = + DocumentsContract.findDocumentPath(context.contentResolver, uri)?.path + if (documentIds != null && documentIds.size >= 2) { + return DocumentsContract.buildDocumentUriUsingTree( + uri, + documentIds[documentIds.size - 2] + ).toString() + } + } catch (e: Exception) { + Log.warning("[FileUtil]: findDocumentPath failed for $path: " + e.message) + } + } + + // Providers that do not implement findDocumentPath: path-like IDs (external storage) + val treeId = DocumentsContract.getTreeDocumentId(uri) + val docId = DocumentsContract.getDocumentId(uri) + if (docId != treeId && docId.startsWith(treeId) && docId.contains('/')) { + val parentId = docId.substringBeforeLast('/') + if (parentId.length >= treeId.length) { + return DocumentsContract.buildDocumentUriUsingTree(uri, parentId).toString() + } + } + return null + } + + /** + * The same document addressed through a tree the app holds a persisted grant for, so its + * neighbours can be read too (a loose ExeFS needs main.npdm, sdk, subsdk* and romfs/ next + * to main). Null when no granted tree contains it. + * @param documentUri Document picked with ACTION_OPEN_DOCUMENT + */ + fun getTreeDocumentUri(documentUri: Uri): Uri? { + if (DocumentsContract.isTreeUri(documentUri)) { + return documentUri + } + val docId = try { + DocumentsContract.getDocumentId(documentUri) + } catch (e: Exception) { + return null + } + for (permission in context.contentResolver.persistedUriPermissions) { + val treeUri = permission.uri + if (!permission.isReadPermission || treeUri.authority != documentUri.authority || + !DocumentsContract.isTreeUri(treeUri) + ) { + continue + } + val treeId = try { + DocumentsContract.getTreeDocumentId(treeUri) + } catch (e: Exception) { + continue + } + val prefix = if (treeId.endsWith('/') || treeId.endsWith(':')) treeId else "$treeId/" + if (docId == treeId || docId.startsWith(prefix)) { + return DocumentsContract.buildDocumentUriUsingTree(treeUri, docId) + } + } + return null + } + /** * Check whether given path exists. * @param path Native content uri path diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/GameHelper.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/GameHelper.kt index bb2337a310..8ecc6f124a 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/GameHelper.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/utils/GameHelper.kt @@ -62,13 +62,19 @@ object GameHelper { if (isValid) { val scanDepth = if (gameDir.deepScan) 3 else 1 - addGamesRecursive( - games, - gamesByProgramId, - FileUtil.listFiles(gameDirUri), - scanDepth, - mountedContainerUris - ) + val gameDirFiles = FileUtil.listFiles(gameDirUri) + val looseMain = findLooseExeFsMain(gameDirFiles) + if (looseMain != null) { + addLooseExeFsGame(games, gamesByProgramId, looseMain) + } else { + addGamesRecursive( + games, + gamesByProgramId, + gameDirFiles, + scanDepth, + mountedContainerUris + ) + } } else { badDirs.add(index) } @@ -109,7 +115,7 @@ object GameHelper { // File extensions considered as external content, buuut should // be done better imo. - private val externalContentExtensions = setOf("nsp", "xci", "nsz", "xcz") + private val externalContentExtensions = setOf("nsp", "xci", "nsz", "xcz", "nup") private fun scanContentContainersRecursive( files: Array, @@ -149,13 +155,21 @@ object GameHelper { files.forEach { if (it.isDirectory) { - addGamesRecursive( - games, - gamesByProgramId, - FileUtil.listFiles(it.uri), - depth - 1, - mountedContainerUris - ) + val children = FileUtil.listFiles(it.uri) + // A loose ExeFS dump (how Switch 2 titles boot) is a game, not a folder to scan. + // Checked at every depth, shallow scans too: main is one or two levels down. + val looseMain = findLooseExeFsMain(children) + if (looseMain != null) { + addLooseExeFsGame(games, gamesByProgramId, looseMain) + } else { + addGamesRecursive( + games, + gamesByProgramId, + children, + depth - 1, + mountedContainerUris + ) + } } else { val extension = FileUtil.getExtension(it.uri).lowercase() val filePath = it.uri.toString() @@ -190,6 +204,48 @@ object GameHelper { } } + /** + * main of a loose ExeFS dump: main and main.npdm side by side in [files], or in their + * exefs/ subdirectory. Null for anything else. + */ + fun findLooseExeFsMain(files: Array): Uri? { + fun mainOf(entries: Array): Uri? { + val main = entries.firstOrNull { !it.isDirectory && it.filename == "main" } + ?: return null + val hasNpdm = entries.any { !it.isDirectory && it.filename == "main.npdm" } + return if (hasNpdm) main.uri else null + } + + mainOf(files)?.let { return it } + val exefs = files.firstOrNull { + it.isDirectory && it.filename.equals("exefs", ignoreCase = true) + } ?: return null + return mainOf(FileUtil.listFiles(exefs.uri)) + } + + private fun addLooseExeFsGame( + games: MutableList, + gamesByProgramId: MutableMap, + mainUri: Uri + ) { + val game = getGame(mainUri, true, false) ?: return + games.add(game) + if (game.programId != "0") { + gamesByProgramId[game.programId] = game + } + } + + // Name of a loose ExeFS dump's folder: the one holding exefs/, else the one holding main. + private fun looseExeFsDumpName(mainPath: String): String? { + val dir = FileUtil.getParentUri(mainPath) ?: return null + val dirName = FileUtil.getFilename(dir.toUri()) + if (!dirName.equals("exefs", ignoreCase = true)) { + return dirName + } + val titleDir = FileUtil.getParentUri(dir) ?: return dirName + return FileUtil.getFilename(titleDir.toUri()) + } + private fun mountExternalContentDirectories(mountedContainerUris: MutableSet) { val uniqueExternalContentDirs = linkedSetOf() NativeConfig.getExternalContentDirs().forEach { externalDir -> @@ -272,15 +328,18 @@ object GameHelper { var name = GameMetadata.getTitle(filePath) - // If the game's title field is empty, use the filename. + // If the game's title field is empty, use the filename (for a loose ExeFS, its folder's). if (name.isEmpty()) { name = FileUtil.getFilename(uri) + if (name == "main") { + name = looseExeFsDumpName(filePath) ?: name + } } var programId = GameMetadata.getProgramId(filePath) // If the game's ID field is empty, use the filename without extension. if (programId.isEmpty()) { - programId = name.substring(0, name.lastIndexOf(".")) + programId = name.substringBeforeLast(".") } val newGame = Game( diff --git a/src/android/app/src/main/jni/emu_window/emu_window.cpp b/src/android/app/src/main/jni/emu_window/emu_window.cpp index 8dfea40693..1890f5d2f8 100644 --- a/src/android/app/src/main/jni/emu_window/emu_window.cpp +++ b/src/android/app/src/main/jni/emu_window/emu_window.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "common/android/id_cache.h" @@ -180,8 +181,16 @@ void EmuWindow_Android::UpdateFrameRateHint() { return; } + // EDEN_ANDROID_FRAME_RATE_HINT=0 sends no preference, = a fixed one. The observed rate + // feeds back: once a game ran its menus at 30 fps, a hint of 30 has the display schedule us + // at 30, so nothing above 30 is ever observed again (Mario Kart World stayed at exactly 30.0 + // in races its GPU ran at 40-45 fps). + // Read each time: the surface exists before a game's env.txt is applied. + const char* const forced_value = std::getenv("EDEN_ANDROID_FRAME_RATE_HINT"); + const float forced_hint = forced_value == nullptr ? -1.0f : std::strtof(forced_value, nullptr); const auto now = Clock::now(); - const float frame_rate_hint = GetFrameRateHint(); + const float frame_rate_hint = + forced_hint >= 0.0f ? QuantizeFrameRateHint(forced_hint) : GetFrameRateHint(); if (std::fabs(frame_rate_hint - m_last_frame_rate_hint) < 0.01f) { m_pending_frame_rate_hint = frame_rate_hint; m_pending_frame_rate_hint_votes = 0; diff --git a/src/android/app/src/main/jni/game_metadata.cpp b/src/android/app/src/main/jni/game_metadata.cpp index 9b6cd44e9f..dbe788011e 100644 --- a/src/android/app/src/main/jni/game_metadata.cpp +++ b/src/android/app/src/main/jni/game_metadata.cpp @@ -76,8 +76,7 @@ jboolean Java_org_yuzu_yuzu_1emu_utils_GameMetadata_getIsValid(JNIEnv* env, jobj auto const file_type = loader->GetFileType(); if (file_type == Loader::FileType::Unknown || file_type == Loader::FileType::Error) return false; - if ((file_type == Loader::FileType::NSP || file_type == Loader::FileType::XCI || - file_type == Loader::FileType::NSZ || file_type == Loader::FileType::XCZ) && + if (Loader::IsContainerType(file_type) && !Loader::IsBootableGameContainer(file, file_type)) return false; u64 program_id = 0; diff --git a/src/android/app/src/main/res/values/strings.xml b/src/android/app/src/main/res/values/strings.xml index 6304ef256b..656584f290 100644 --- a/src/android/app/src/main/res/values/strings.xml +++ b/src/android/app/src/main/res/values/strings.xml @@ -247,6 +247,14 @@ No files were found or no game directory has been selected yet. Manage game folders Allows Eden to populate the games list + Load file + Boot a game file directly, including a packed Switch 2 dump (.nx2) or its main file + Loading game + Could not load the file + This file is not a game Eden can boot. + No Switch 2 dump found here: the folder needs main and main.npdm, directly or in an exefs folder. + Folder access needed + A Switch 2 dump boots from its folder: main needs main.npdm, the sdk files and romfs next to it, and picking a single file only grants access to that file. Select the dump\'s folder (the one holding exefs and romfs) next. Skip selecting games folder? Games won\'t be displayed in the Games list if a folder isn\'t selected. https://yuzu-mirror.github.io/help/quickstart/#dumping-games @@ -601,6 +609,20 @@ Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path Use asynchronous shaders Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches. + Reduce texture resolution + Keeps large compressed textures at half resolution, using about a quarter of their video memory. Helps games that run out of memory, such as Switch 2 titles. Textures look slightly blurrier up close. Takes effect when the game starts. + Release GPU-only memory (experimental) + Switch 2 command list fixes + Upstream\'s fixes for Switch 2 command lists the game rewrites after submitting. Without them games such as Ocarina of Time can freeze. Mario Kart World runs without them unless its own custom settings turn them on: on an Odin 3 they cost ~45 ms of GPU-thread waiting per second there and caused flicker. Takes effect when a game starts. + Mario Kart World: 30 FPS mode + Forces the game to use its native 30 FPS mode at normal game speed. Takes effect after restarting the game. + Mario Kart World: 60 FPS mode + Always renders at 60 fps. Slower hardware runs the game in slow motion. Takes effect after restarting the game; the 30 FPS mode takes precedence. + Mario Kart World: shorter LOD distance + Objects switch to lower detail at half the distance (the game\'s default ratio instead of 2.0). Fewer, lighter draws. Takes effect after restarting the game. + Mario Kart World: fewer shadow cascades + Drops the farthest dynamic shadow cascade (160-250 m). Takes effect after restarting the game. + Frees the RAM copy of Switch 2 GPU-only memory once its textures are on the GPU, saving RAM. Experimental: may cause missing or corrupted textures. GPU Unswizzle Settings Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality. Enable GPU Unswizzle diff --git a/src/common/fs/fs_android.cpp b/src/common/fs/fs_android.cpp index 9a80532227..480b23d4b4 100644 --- a/src/common/fs/fs_android.cpp +++ b/src/common/fs/fs_android.cpp @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include + #include "common/android/android_common.h" #include "common/android/id_cache.h" #include "common/assert.h" @@ -22,6 +24,8 @@ void RegisterCallbacks(JNIEnv* env, jclass clazz) { s_file_exists = env->GetStaticMethodID(native_library, "exists", "(Ljava/lang/String;)Z"); s_open_content_uri = env->GetStaticMethodID(native_library, "openContentUri", "(Ljava/lang/String;Ljava/lang/String;)I"); + s_list_children = env->GetStaticMethodID(native_library, "listChildren", + "(Ljava/lang/String;)[Ljava/lang/String;"); } void UnRegisterCallbacks() { @@ -33,6 +37,7 @@ void UnRegisterCallbacks() { s_file_exists = nullptr; s_open_content_uri = nullptr; + s_list_children = nullptr; } bool IsContentUri(const std::string& path) { @@ -95,7 +100,7 @@ bool Exists(const std::string& filepath) { std::string GetParentDirectory(const std::string& filepath) { if (s_get_parent_directory == nullptr) { - return 0; + return {}; } auto env = Common::Android::GetEnvForThread(); jstring j_return = static_cast(env->CallStaticObjectMethod( @@ -108,7 +113,7 @@ std::string GetParentDirectory(const std::string& filepath) { std::string GetFilename(const std::string& filepath) { if (s_get_filename == nullptr) { - return 0; + return {}; } auto env = Common::Android::GetEnvForThread(); jstring j_return = static_cast(env->CallStaticObjectMethod( @@ -119,4 +124,40 @@ std::string GetFilename(const std::string& filepath) { return Common::Android::GetJString(env, j_return); } +std::vector ListChildren(const std::string& dir_path) { + if (s_list_children == nullptr) { + return {}; + } + auto env = Common::Android::GetEnvForThread(); + jstring j_path = Common::Android::ToJString(env, dir_path); + auto j_entries = static_cast( + env->CallStaticObjectMethod(native_library, s_list_children, j_path)); + env->DeleteLocalRef(j_path); + if (!j_entries) { + return {}; + } + + // Four strings per child: "d" or "f", size, display name, document URI. A big directory + // makes thousands of local references, so each one is dropped once read. + const auto element = [&](jsize index) { + auto j_value = static_cast(env->GetObjectArrayElement(j_entries, index)); + std::string value = j_value ? Common::Android::GetJString(env, j_value) : std::string{}; + env->DeleteLocalRef(j_value); + return value; + }; + const jsize count = env->GetArrayLength(j_entries); + std::vector entries; + entries.reserve(static_cast(count / 4)); + for (jsize i = 0; i + 3 < count; i += 4) { + ContentEntry entry; + entry.is_directory = element(i) == "d"; + entry.size = std::strtoll(element(i + 1).c_str(), nullptr, 10); + entry.name = element(i + 2); + entry.uri = element(i + 3); + entries.push_back(std::move(entry)); + } + env->DeleteLocalRef(j_entries); + return entries; +} + } // namespace Common::FS::Android diff --git a/src/common/fs/fs_android.h b/src/common/fs/fs_android.h index b33f4beb8e..889256a20f 100644 --- a/src/common/fs/fs_android.h +++ b/src/common/fs/fs_android.h @@ -18,6 +18,15 @@ static jmethodID s_get_size; static jmethodID s_is_directory; static jmethodID s_file_exists; static jmethodID s_open_content_uri; +static jmethodID s_list_children; + +// A child of a content URI directory, as listed by the document provider. +struct ContentEntry { + std::string name; + std::string uri; + bool is_directory; + std::int64_t size; // -1 when the provider does not report it +}; enum class OpenMode { Read, @@ -42,5 +51,6 @@ bool IsDirectory(const std::string& filepath); bool Exists(const std::string& filepath); std::string GetParentDirectory(const std::string& filepath); std::string GetFilename(const std::string& filepath); +std::vector ListChildren(const std::string& dir_path); } // namespace Common::FS::Android diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 2ef1982cbc..cb2713f15b 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -24,6 +24,7 @@ #include "common/scope_exit.h" #if defined(__linux__) +#include #include #elif defined(__APPLE__) #include @@ -244,6 +245,8 @@ class HostMemory::Impl { u8* backing_base{}; u8* virtual_base{}; + void ReleaseBackingRegion(size_t, size_t) const {} + private: /// Release all resources in the object void Release() { @@ -668,6 +671,27 @@ class HostMemory::Impl { u8* virtual_base{reinterpret_cast(MAP_FAILED)}; u8* virtual_map_base{reinterpret_cast(MAP_FAILED)}; + void ReleaseBackingRegion(size_t physical_offset, size_t length) const { + // Punching holes frees the memfd's pages for every mapping of it; without a memfd the + // backing is private anonymous memory, which MADV_DONTNEED frees. + constexpr size_t page = 4096; + const size_t begin = (physical_offset + page - 1) & ~(page - 1); + const size_t end = (physical_offset + length) & ~(page - 1); + if (end <= begin) { + return; + } +#ifdef __linux__ + if (fd > 0) { + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, static_cast(begin), + static_cast(end - begin)); + return; + } +#endif + { + madvise(backing_base + begin, end - begin, MADV_DONTNEED); + } + } + private: /// Release all resources in the object void Release() { @@ -896,6 +920,12 @@ void HostMemory::ForgetContents() { } } +void HostMemory::ReleaseBackingRegion(size_t physical_offset, size_t length) const { + if (impl) { + impl->ReleaseBackingRegion(physical_offset, length); + } +} + void HostMemory::EnableDirectMappedAddress() { #if !(defined(__OPENORBIS__) || defined(__managarm__)) if (impl) { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 84315091c5..7865da6e29 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -71,6 +71,10 @@ class HostMemory { /// A new kernel on this memory: everything a previous one used may hold data. void ForgetContents(); + /// Gives the backing pages of [physical_offset, +length) (whole pages inside it) back to the + /// system: they read as zeros until written again. Does nothing where unsupported. + void ReleaseBackingRegion(size_t physical_offset, size_t length) const; + [[nodiscard]] u8* BackingBasePointer() noexcept { return backing_base; } diff --git a/src/common/settings.h b/src/common/settings.h index 9138bcdcf5..d02dfef07d 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -364,6 +364,12 @@ struct Values { Specialization::Default, true, true}; + // Upstream's command list series (copies of lists at submit, and of lists the GPU writes). + // On Android, Mario Kart World runs without it unless its own custom settings turn it on + // (nvdrv.cpp): on an Odin 3 it cost ~45 ms of GPU-thread waiting a second there and caused + // flicker. + SwitchableSetting nx2_command_lists{linkage, true, "nx2_command_lists", + Category::RendererHacks}; SwitchableSetting fsr_sharpening_slider{linkage, #ifdef __ANDROID__ 0, @@ -609,8 +615,14 @@ struct Values { // Mario Kart World: select a fixed or dynamic in-game frame-rate mode (loader/nso.cpp). SwitchableSetting mkw_30fps{linkage, false, "mkw_30fps", Category::RendererAdvanced}; + SwitchableSetting mkw_60fps{linkage, false, "mkw_60fps", Category::RendererAdvanced}; SwitchableSetting mkw_dynamic_fps{linkage, false, "mkw_dynamic_fps", Category::RendererAdvanced}; + // Mario Kart World: shorter LOD distances and fewer shadow cascades (loader/nso.cpp). + SwitchableSetting mkw_lod_distance{linkage, false, "mkw_lod_distance", + Category::RendererAdvanced}; + SwitchableSetting mkw_shadow_cascades{linkage, false, "mkw_shadow_cascades", + Category::RendererAdvanced}; SwitchableSetting use_reactive_flushing{linkage, #ifdef __ANDROID__ @@ -672,8 +684,24 @@ struct Values { SwitchableSetting emulate_bgr565{linkage, false, "emulate_bgr565", Category::RendererHacks}; - SwitchableSetting use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders", + // On by default on Android (pipelines building on the GPU thread stall a phone for 100s of + // ms); desktop keeps it off. + SwitchableSetting use_asynchronous_shaders{linkage, +#ifdef __ANDROID__ + true, +#else + false, +#endif + "use_asynchronous_shaders", Category::RendererHacks}; + /// Large compressed textures keep their mip levels from 1 on (half resolution): less video + /// memory, slightly blurrier textures up close. + SwitchableSetting reduce_texture_resolution{linkage, false, "reduce_texture_resolution", + Category::RendererHacks}; + /// GPU-only guest memory (Switch 2) a texture was uploaded from is released afterwards, the + /// host image holding the data: less RAM, experimental. + SwitchableSetting release_gpu_only_memory{linkage, false, "release_gpu_only_memory", + Category::RendererHacks}; SwitchableSetting gpu_unswizzle_texture_size{linkage, GpuUnswizzleSize::Large, diff --git a/src/common/slot_vector.h b/src/common/slot_vector.h index 8e1c06e35a..e9eb757427 100644 --- a/src/common/slot_vector.h +++ b/src/common/slot_vector.h @@ -124,9 +124,16 @@ class SlotVector { } template - [[nodiscard]] SlotId insert(Args&&... args) noexcept { + [[nodiscard]] SlotId insert(Args&&... args) { const u32 index = FreeValueIndex(); - new (&values[index].object) T(std::forward(args)...); + try { + new (&values[index].object) T(std::forward(args)...); + } catch (...) { + // A constructor may throw (a texture the GPU cannot allocate): keep the slot free and + // let the caller handle it, instead of terminating at a noexcept boundary. + free_list.push_back(index); + throw; + } SetStorageBit(index); return SlotId{index}; @@ -138,6 +145,12 @@ class SlotVector { ResetStorageBit(id.Value()); } + /// Returns true when id refers to a slot that currently holds an object. + [[nodiscard]] bool contains(SlotId id) const noexcept { + return id && id.Value() / 64 < stored_bitset.size() && + ((stored_bitset[id.Value() / 64] >> (id.Value() % 64)) & 1) != 0; + } + [[nodiscard]] Iterator begin() noexcept { const auto it = std::ranges::find_if(stored_bitset, [](u64 value) { return value != 0; }); if (it == stored_bitset.end()) { diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1ff9f271d2..507a5e6561 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -124,6 +124,8 @@ add_library(core STATIC file_sys/kernel_executable.h file_sys/nca_metadata.cpp file_sys/nca_metadata.h + file_sys/nx2_package.cpp + file_sys/nx2_package.h file_sys/partition_filesystem.cpp file_sys/partition_filesystem.h file_sys/patch_manager.cpp diff --git a/src/core/core.cpp b/src/core/core.cpp index 6e82eeecc7..ebe3f51cc6 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -11,6 +13,7 @@ #include "common/cpu_features.h" #include "audio_core/audio_core.h" #include "common/fs/fs.h" +#include "common/fs/path_util.h" #include "common/logging.h" #include "common/adpf.h" #include "common/settings.h" @@ -69,6 +72,7 @@ #include "hid_core/hid_core.h" #include "network/network.h" #include "video_core/host1x/host1x.h" +#include "video_core/nx2_profile.h" #include "video_core/renderer_base.h" #include "video_core/video_core.h" @@ -109,7 +113,17 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, } if (Common::FS::IsDir(path)) { - return vfs->OpenFile(path + "/main", FileSys::OpenMode::Read); + // A loose ExeFS: main in the directory, else in its exefs/ subdirectory. Looked up through + // the directory, since a content URI on Android cannot have "/main" appended. + const auto dir = vfs->OpenDirectory(path, FileSys::OpenMode::Read); + if (dir == nullptr) { + return nullptr; + } + if (auto main = dir->GetFile("main")) { + return main; + } + const auto exefs = dir->GetSubdirectory("exefs"); + return exefs != nullptr ? exefs->GetFile("main") : nullptr; } return vfs->OpenFile(path, FileSys::OpenMode::Read); @@ -373,6 +387,46 @@ struct System::Impl { Settings::values.memory_layout_mode.SetGlobal(false); Settings::values.memory_layout_mode.SetValue(Settings::MemoryLayout::Memory_12Gb); } +#ifdef __ANDROID__ + // Android only: desktop users set the environment, and desktop keeps upstream's defaults. + // The KEY=VALUE lines of an env.txt in the user directory become environment variables: + // the EDEN_NX2_* switches, which Android has no other way to set. Each is read the first + // time it is used, so a change takes a restart of the app. + if (std::ifstream env{Common::FS::GetEdenPath(Common::FS::EdenPath::EdenDir) / "env.txt"}) { + for (std::string line; std::getline(env, line);) { + while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) { + line.pop_back(); + } + const size_t equals = line.find('='); + if (line.empty() || line[0] == '#' || equals == std::string::npos || equals == 0) { + continue; + } + const std::string key = line.substr(0, equals); + const std::string value = line.substr(equals + 1); +#ifdef _WIN32 + _putenv_s(key.c_str(), value.c_str()); +#else + setenv(key.c_str(), value.c_str(), 1); +#endif + LOG_INFO(Core, "env.txt: {}={}", key, value); + } + } + // A gpu_profile file in the user directory turns the GPU threads' profile on: their time + // per zone every 10 s in the log (EDEN_NX2_GPU_PROFILE, for Android). + if (Common::FS::Exists(Common::FS::GetEdenPath(Common::FS::EdenPath::EdenDir) / + "gpu_profile")) { + VideoCommon::Nx2Profile::g_enabled.store(true, std::memory_order_relaxed); + LOG_INFO(Core, "GPU thread profile on (gpu_profile file)"); + } + // Switch 2 titles build thousands of pipelines while they play: build them asynchronously + // (a hitch-free frame instead of a GPU thread stall), unless set for this game itself. + if (switch2_program && Settings::values.use_asynchronous_shaders.UsingGlobal() && + !Settings::values.use_asynchronous_shaders.GetValue()) { + LOG_INFO(Core, "Switch 2 program: building shaders asynchronously"); + Settings::values.use_asynchronous_shaders.SetGlobal(false); + Settings::values.use_asynchronous_shaders.SetValue(true); + } +#endif // The Switch 2 counter runs at 31.25 MHz and its SDK converts ticks with that constant // (nn::os::GetSystemTickFrequency, TickManager::ConvertToTimeSpan/ConvertToTick). At 19.2 MHz diff --git a/src/core/cpu_manager.cpp b/src/core/cpu_manager.cpp index 23d5ac3dfd..0a8d1f2386 100644 --- a/src/core/cpu_manager.cpp +++ b/src/core/cpu_manager.cpp @@ -18,8 +18,71 @@ #include "core/hle/kernel/physical_core.h" #include "video_core/gpu.h" +#if defined(__ANDROID__) && defined(ARCHITECTURE_arm64) +#include +#include +#include +#include +#include "common/logging.h" +#ifndef PR_PAC_SET_ENABLED_KEYS +#define PR_PAC_SET_ENABLED_KEYS 60 +#endif +#ifndef PR_PAC_APIAKEY +#define PR_PAC_APIAKEY (1UL << 0) +#define PR_PAC_APIBKEY (1UL << 1) +#define PR_PAC_APDAKEY (1UL << 2) +#define PR_PAC_APDBKEY (1UL << 3) +#endif +#endif + namespace Core { +#if defined(__ANDROID__) && defined(ARCHITECTURE_arm64) +namespace { +// Android gives every thread its own PAC keys, and NCE runs guest PAC natively: a pointer the +// guest signs on one core thread (a fiber context, a return address) fails to authenticate on +// another. NCE core threads run with the instruction and data keys disabled, so PAC/AUT pass +// pointers through unchanged as under Dynarmic. PACGA keeps the process-wide GA key. +// EDEN_NX2_NCE_PAC_KEYS=1 keeps the keys. +constexpr unsigned long NcePacKeys = + PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY; + +bool DisableNcePacKeys() { + if (!Settings::IsNceEnabled()) { + return false; + } + static const bool keep_keys = [] { + const char* const value = std::getenv("EDEN_NX2_NCE_PAC_KEYS"); + return value != nullptr && *value != '\0' && *value != '0'; + }(); + static std::once_flag logged; + if (keep_keys) { + std::call_once(logged, [] { + LOG_INFO(Core, "NCE: PAC keys kept on the CPU core threads (EDEN_NX2_NCE_PAC_KEYS)"); + }); + return false; + } + const int result = prctl(PR_PAC_SET_ENABLED_KEYS, NcePacKeys, 0, 0, 0); + const int error = errno; + if (result != 0) { + LOG_WARNING(Core, "NCE: could not disable the PAC keys on this CPU core thread (errno {})", + error); + return false; + } + std::call_once(logged, [] { + LOG_INFO(Core, "NCE: PAC instruction and data keys disabled on the CPU core threads"); + }); + return true; +} + +// Before the thread returns, so the libc frames signed before the keys were disabled +// authenticate. +void RestoreNcePacKeys() { + prctl(PR_PAC_SET_ENABLED_KEYS, NcePacKeys, NcePacKeys, 0, 0); +} +} // namespace +#endif + CpuManager::CpuManager(System& system_) : system{system_} {} CpuManager::~CpuManager() = default; @@ -192,6 +255,16 @@ void CpuManager::RunThread(std::stop_token token, std::size_t core) { system.GPU().ObtainContext(); } +#if defined(__ANDROID__) && defined(ARCHITECTURE_arm64) + // Before any guest code runs on this thread (see DisableNcePacKeys). + const bool pac_keys_disabled = DisableNcePacKeys(); + SCOPE_EXIT { + if (pac_keys_disabled) { + RestoreNcePacKeys(); + } + }; +#endif + auto& kernel = system.Kernel(); auto& scheduler = *kernel.CurrentScheduler(); auto* thread = scheduler.GetSchedulerCurrentThread(); diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index d275bdbf2d..4d476b315c 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -47,6 +47,10 @@ class DeviceMemoryManager { public: DeviceMemoryManager(const DeviceMemory& device_memory); + + /// Gives back the backing of the guest memory the device range maps (whole pages): it reads + /// as zeros until written again. + void ReleaseMemory(DAddr address, size_t size); ~DeviceMemoryManager(); static constexpr bool HAS_FLUSH_INVALIDATION = true; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 5931fd0882..65ea58edb9 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -443,7 +443,11 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o std::size_t page_index = addr >> Memory::YUZU_PAGEBITS; std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; while (remaining_size) { - const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1); + const auto phys_addr = tracked_entries[page_index].compressed_physical_ptr; + // An unmapped page says nothing about the pages after it: its run cannot be trusted, so + // only this page is handed to on_unmapped before the walk moves on. + const size_t next_pages = + phys_addr == 0 ? 1 : std::size_t(tracked_entries[page_index].continuity_tracker + 1); const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset); SCOPE_EXIT{ @@ -453,7 +457,6 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o remaining_size -= copy_amount; }; - auto phys_addr = tracked_entries[page_index].compressed_physical_ptr; if (phys_addr == 0) { on_unmapped(copy_amount, current_vaddr); continue; @@ -773,4 +776,35 @@ void DeviceMemoryManager::ArmReadTraps(DAddr addr, size_t size) { release_run(); } +template +void DeviceMemoryManager::ReleaseMemory(DAddr address, size_t size) { + // Runs of device pages whose backing is contiguous, released a run at a time. + size_t run_begin = 0; + size_t run_size = 0; + const auto flush = [&] { + if (run_size != 0) { + device_memory_ptr->buffer.ReleaseBackingRegion(run_begin, run_size); + run_size = 0; + } + }; + const DAddr first = Common::AlignUp(address, Memory::YUZU_PAGESIZE); + const DAddr last = Common::AlignDown(address + size, Memory::YUZU_PAGESIZE); + for (DAddr page = first; page < last; page += Memory::YUZU_PAGESIZE) { + const u8* const pointer = GetPointer(page); + if (pointer == nullptr) { + flush(); + continue; + } + const size_t offset = static_cast(reinterpret_cast(pointer) - physical_base); + if (run_size != 0 && run_begin + run_size == offset) { + run_size += Memory::YUZU_PAGESIZE; + } else { + flush(); + run_begin = offset; + run_size = Memory::YUZU_PAGESIZE; + } + } + flush(); +} + } // namespace Core diff --git a/src/core/file_sys/nx2_package.cpp b/src/core/file_sys/nx2_package.cpp new file mode 100644 index 0000000000..6f6ecbfdfa --- /dev/null +++ b/src/core/file_sys/nx2_package.cpp @@ -0,0 +1,295 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/alignment.h" +#include "common/common_funcs.h" +#include "common/fs/file.h" +#include "common/fs/fs.h" +#include "common/fs/fs_util.h" +#include "common/fs/path_util.h" +#include "common/literals.h" +#include "common/logging.h" +#include "common/swap.h" +#include "core/file_sys/content_archive.h" +#include "core/file_sys/fsmitm_romfsbuild.h" +#include "core/file_sys/nx2_package.h" +#include "core/file_sys/vfs/vfs.h" + +namespace FileSys { + +namespace { + +using namespace Common::Literals; + +// The PFS0 layout PartitionFilesystem reads: header, entries, string table, then the data, with +// entry offsets relative to the end of the string table. +struct PfsHeader { + u32_le magic; + u32_le num_entries; + u32_le strtab_size; + u32_le reserved; +}; +static_assert(sizeof(PfsHeader) == 0x10); + +struct PfsEntry { + u64_le offset; + u64_le size; + u32_le strtab_offset; + u32_le reserved; +}; +static_assert(sizeof(PfsEntry) == 0x18); + +constexpr size_t CopyChunkSize = 16_MiB; + +// A file of the package: one source file, or the RomFS image as its parts (offset in the image, +// file); the gaps between parts are zero. +struct PackageEntry { + std::string name; + u64 size; + std::vector> parts; +}; + +struct CopyJob { + u64 destination; + u64 size; + VirtualFile file; +}; + +bool IsSidecar(const VirtualFile& file) { + const auto extension = file->GetExtension(); + return extension == "nacp" || extension == "png" || extension == "jpg" || + extension == "jpeg" || extension == "bmp" || + (extension == "dat" && file->GetName().starts_with("icon_")); +} + +} // Anonymous namespace + +std::string PackNx2(const VirtualDir& dump_dir, const std::string& out_path, + const Nx2PackProgress& progress) { + if (dump_dir == nullptr) { + return "The dump folder could not be opened."; + } + const VirtualDir exefs = + IsDirectoryExeFS(dump_dir) ? dump_dir : dump_dir->GetSubdirectory("exefs"); + if (exefs == nullptr || !IsDirectoryExeFS(exefs)) { + return "No ExeFS found: the folder needs main and main.npdm, directly or in an exefs " + "folder."; + } + const VirtualDir title_dir = exefs == dump_dir ? exefs->GetParentDirectory() : dump_dir; + + std::vector entries; + const auto add = [&entries](std::string name, VirtualFile file) { + const bool present = std::any_of(entries.begin(), entries.end(), + [&name](const auto& entry) { return entry.name == name; }); + if (!present) { + const u64 size = file->GetSize(); + entries.push_back({std::move(name), size, {{0, std::move(file)}}}); + } + }; + + // The ExeFS (main, main.npdm, rtld, sdk, subsdk*, sidecars dropped next to them). A RomFS + // image already next to main is taken as the RomFS. + VirtualFile romfs_image; + for (const auto& file : exefs->GetFiles()) { + if (file->GetExtension() == "romfs") { + if (romfs_image == nullptr) { + romfs_image = file; + } + continue; + } + add(file->GetName(), file); + } + + // Sidecars in the game folder: a PFS0 has no parent for the loader to look in. + if (title_dir != nullptr && title_dir != exefs) { + for (const auto& file : title_dir->GetFiles()) { + if (IsSidecar(file)) { + add(file->GetName(), file); + } + } + } + + // The RomFS: that image, else the romfs/ folder next to the ExeFS laid out as one image. + if (romfs_image != nullptr) { + add(std::string{Nx2RomFSName}, romfs_image); + } else if (const auto romfs_dir = + title_dir != nullptr ? title_dir->GetSubdirectory("romfs") : nullptr) { + auto parts = RomFSBuildContext{romfs_dir}.Build(); + if (!parts.empty()) { + const u64 size = parts.back().first + parts.back().second->GetSize(); + entries.push_back({std::string{Nx2RomFSName}, size, std::move(parts)}); + } + } else { + LOG_WARNING(Loader, "Packing {} without a RomFS", out_path); + } + + // String table, padded so the data starts 0x20-aligned like Nintendo's own PFS0s. + std::vector strtab; + std::vector pfs_entries; + u64 data_size = 0; + for (const auto& entry : entries) { + pfs_entries.push_back(PfsEntry{ + .offset = data_size, + .size = entry.size, + .strtab_offset = static_cast(strtab.size()), + .reserved = 0, + }); + strtab.insert(strtab.end(), entry.name.begin(), entry.name.end()); + strtab.push_back('\0'); + data_size = Common::AlignUp(data_size + entry.size, 0x10); + } + const size_t metadata_size = + sizeof(PfsHeader) + pfs_entries.size() * sizeof(PfsEntry) + strtab.size(); + strtab.resize(strtab.size() + Common::AlignUp(metadata_size, 0x20) - metadata_size, '\0'); + const u64 data_start = Common::AlignUp(metadata_size, 0x20); + + const PfsHeader header{ + .magic = Common::MakeMagic('P', 'F', 'S', '0'), + .num_entries = static_cast(pfs_entries.size()), + .strtab_size = static_cast(strtab.size()), + .reserved = 0, + }; + + std::vector jobs; + u64 total = 0; + for (size_t i = 0; i < entries.size(); ++i) { + for (const auto& [offset, file] : entries[i].parts) { + const u64 size = file->GetSize(); + if (size != 0) { + jobs.push_back({data_start + pfs_entries[i].offset + offset, size, file}); + total += size; + } + } + } + + Common::FS::IOFile out{out_path, Common::FS::FileAccessMode::Write, + Common::FS::FileType::BinaryFile}; + if (!out.IsOpen()) { + return "The output file could not be created."; + } + const auto fail = [&out, &out_path](std::string error) { + out.Close(); + Common::FS::RemoveFile(out_path); + return error; + }; + + // Not sized up front: on Windows that writes every byte as zero first. The jobs are taken in + // offset order, so the file grows as they land; the OS zero-fills the gaps and paddings. + if (!out.WriteObject(header) || + out.WriteSpan(std::span{pfs_entries}) != pfs_entries.size() || + out.WriteSpan(std::span{strtab}) != strtab.size()) { + return fail("Writing the package failed (is the disk full?)."); + } + if (progress && !progress(0, total)) { + return fail("Cancelled."); + } + + // A RomFS is mostly small files, and copying them one by one waits on each file's open + // (antivirus scans included) with the disk idle. Several readers keep it busy; real files + // are read through their own handles, since the VFS serialises its reads behind one lock. + std::atomic next_job{0}; + std::atomic copied{0}; + std::atomic stop{false}; + std::atomic running{0}; + std::mutex out_mutex; + std::mutex error_mutex; + std::string copy_error; + const auto set_error = [&](std::string error) { + std::scoped_lock lk{error_mutex}; + if (copy_error.empty()) { + copy_error = std::move(error); + } + stop = true; + }; + + const auto copy_worker = [&] { + std::vector buffer(CopyChunkSize); + while (!stop) { + const size_t index = next_job.fetch_add(1); + if (index >= jobs.size()) { + break; + } + const CopyJob& job = jobs[index]; + + Common::FS::IOFile direct; + const auto path = job.file->GetFullPath(); + if (Common::FS::IsFile(path)) { + direct.Open(std::filesystem::path{Common::FS::ToU8String(path)}, + Common::FS::FileAccessMode::Read, Common::FS::FileType::BinaryFile); + if (direct.IsOpen() && direct.GetSize() != job.size) { + direct.Close(); + } + } + + for (u64 position = 0; position < job.size && !stop;) { + const size_t length = + static_cast((std::min)(CopyChunkSize, job.size - position)); + const auto data = std::span{buffer.data(), length}; + const size_t read = direct.IsOpen() ? direct.ReadSpan(data) + : job.file->Read(data.data(), length, position); + if (read != length) { + set_error("Reading " + path + " failed."); + break; + } + { + std::scoped_lock lk{out_mutex}; + if (!out.Seek(static_cast(job.destination + position)) || + out.WriteSpan(std::span{data}) != length) { + set_error("Writing the package failed (is the disk full?)."); + break; + } + } + position += length; + copied += length; + } + } + --running; + }; + + const size_t reader_count = + std::clamp(std::thread::hardware_concurrency() / 2, 4, 16); + std::vector readers; + running = reader_count; + for (size_t i = 0; i < reader_count; ++i) { + readers.emplace_back(copy_worker); + } + bool cancelled = false; + while (running > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + if (progress && !cancelled && !progress(copied, total)) { + cancelled = true; + stop = true; + } + } + for (auto& reader : readers) { + reader.join(); + } + + if (cancelled) { + return fail("Cancelled."); + } + if (!copy_error.empty()) { + return fail(copy_error); + } + // The padding after the last file + if (!out.SetSize(data_start + data_size) || !out.Flush()) { + return fail("Writing the package failed (is the disk full?)."); + } + if (progress) { + progress(total, total); + } + LOG_INFO(Loader, "Packed {} files ({} bytes) into {}", jobs.size(), data_start + data_size, + out_path); + return {}; +} + +} // namespace FileSys diff --git a/src/core/file_sys/nx2_package.h b/src/core/file_sys/nx2_package.h new file mode 100644 index 0000000000..984ae2db2b --- /dev/null +++ b/src/core/file_sys/nx2_package.h @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#include "common/common_types.h" +#include "core/file_sys/vfs/vfs_types.h" + +namespace FileSys { + +// .nx2: a loose Switch 2 dump (ExeFS + RomFS) packed into one file. It is an extracted-type NSP, +// a PFS0 holding the ExeFS files directly, with the RomFS as a single image and any +// control.nacp/icon sidecars next to them, so the NSP loader boots it like the loose dump. +constexpr std::string_view Nx2Extension = "nx2"; +constexpr std::string_view Nx2RomFSName = "game.romfs"; + +// Called with the bytes written so far and the total; returning false cancels. +using Nx2PackProgress = std::function; + +// Packs the dump in dump_dir (the folder holding exefs/ and romfs/, or the ExeFS folder itself) +// into out_path. Returns an error message, empty on success. A failed or cancelled pack leaves +// no output file. +std::string PackNx2(const VirtualDir& dump_dir, const std::string& out_path, + const Nx2PackProgress& progress); + +} // namespace FileSys diff --git a/src/core/file_sys/vfs/vfs_real.cpp b/src/core/file_sys/vfs/vfs_real.cpp index 7cc5315a70..2683e3c17f 100644 --- a/src/core/file_sys/vfs/vfs_real.cpp +++ b/src/core/file_sys/vfs/vfs_real.cpp @@ -68,6 +68,53 @@ bool IsWithinRoot(std::string_view root, std::string_view full_path) { return full_path.size() == root.size() || full_path[root.size()] == '/' || full_path[root.size()] == '\\'; } +bool IsContentPath(const std::string& path) { +#ifdef __ANDROID__ + return FS::Android::IsContentUri(path); +#else + return false; +#endif +} + +#ifdef __ANDROID__ +// A child of a content URI directory. Document URIs cannot be joined with '/', so children are +// found by name in the provider's listing, one path component at a time. +struct ContentChild { + FS::Android::ContentEntry entry; +}; + +std::optional FindContentChild(const std::string& dir, + std::string_view relative_path) { + std::optional found; + std::string current = dir; + size_t start = 0; + while (start <= relative_path.size()) { + const size_t end = (std::min)(relative_path.find_first_of("/\\", start), relative_path.size()); + const std::string_view component = relative_path.substr(start, end - start); + start = end + 1; + if (component.empty() || component == ".") { + continue; + } + if (found && !found->entry.is_directory) { + return std::nullopt; + } + const auto children = FS::Android::ListChildren(current); + const auto it = std::find_if(children.begin(), children.end(), + [&](const auto& child) { return child.name == component; }); + if (it == children.end()) { + return std::nullopt; + } + found = ContentChild{*it}; + current = it->uri; + } + return found; +} + +std::optional ContentEntrySize(const FS::Android::ContentEntry& entry) { + return entry.size >= 0 ? std::optional{static_cast(entry.size)} : std::nullopt; +} +#endif + constexpr FS::FileAccessMode ModeFlagsToFileAccessMode(OpenMode mode) { switch (mode) { case OpenMode::Read: @@ -395,6 +442,18 @@ std::vector RealVfsDirectory::IterateEntries( std::vector out; +#ifdef __ANDROID__ + if (FS::Android::IsContentUri(path)) { + for (const auto& entry : FS::Android::ListChildren(path)) { + if (!entry.is_directory) { + out.emplace_back( + base.OpenFileFromEntry(entry.uri, ContentEntrySize(entry), perms)); + } + } + return out; + } +#endif + const FS::DirEntryCallable callback = [this, &out](const std::filesystem::directory_entry& entry) { const auto full_path_string = FS::PathToUTF8String(entry.path()); @@ -417,6 +476,17 @@ std::vector RealVfsDirectory::IterateEntries out; +#ifdef __ANDROID__ + if (FS::Android::IsContentUri(path)) { + for (const auto& entry : FS::Android::ListChildren(path)) { + if (entry.is_directory) { + out.emplace_back(base.OpenDirectory(entry.uri, perms)); + } + } + return out; + } +#endif + const FS::DirEntryCallable callback = [this, &out](const std::filesystem::directory_entry& entry) { const auto full_path_string = FS::PathToUTF8String(entry.path()); @@ -433,9 +503,11 @@ std::vector RealVfsDirectory::IterateEntriesentry.is_directory) { + return nullptr; + } + return base.OpenFileFromEntry(child->entry.uri, ContentEntrySize(child->entry), perms); + } +#endif const auto full_path = FS::SanitizePath(path + '/' + std::string(relative_path)); if (!FS::Exists(full_path) || FS::IsDir(full_path) || !IsWithinRoot(FS::SanitizePath(path), full_path)) { @@ -452,6 +533,15 @@ VirtualFile RealVfsDirectory::GetFileRelative(std::string_view relative_path) co } VirtualDir RealVfsDirectory::GetDirectoryRelative(std::string_view relative_path) const { +#ifdef __ANDROID__ + if (FS::Android::IsContentUri(path)) { + const auto child = FindContentChild(path, relative_path); + if (!child || !child->entry.is_directory) { + return nullptr; + } + return base.OpenDirectory(child->entry.uri, perms); + } +#endif const auto full_path = FS::SanitizePath(path + '/' + std::string(relative_path)); if (!FS::Exists(full_path) || !FS::IsDir(full_path) || !IsWithinRoot(FS::SanitizePath(path), full_path)) { @@ -526,10 +616,25 @@ bool RealVfsDirectory::IsReadable() const { } std::string RealVfsDirectory::GetName() const { +#ifdef __ANDROID__ + if (FS::Android::IsContentUri(path)) { + return FS::Android::GetFilename(path); + } +#endif return path_components.empty() ? "" : std::string(path_components.back()); } VirtualDir RealVfsDirectory::GetParentDirectory() const { +#ifdef __ANDROID__ + if (FS::Android::IsContentUri(path)) { + // The provider answers with the same URI at the root of the granted tree. + const auto parent = FS::GetParentPath(path); + if (parent.empty() || parent == path) { + return nullptr; + } + return base.OpenDirectory(parent, perms); + } +#endif if (path_components.size() <= 1) { return nullptr; } @@ -575,6 +680,16 @@ std::map> RealVfsDirectory::GetEntries() std::map> out; +#ifdef __ANDROID__ + if (FS::Android::IsContentUri(path)) { + for (const auto& entry : FS::Android::ListChildren(path)) { + out.insert_or_assign(entry.name, entry.is_directory ? VfsEntryType::Directory + : VfsEntryType::File); + } + return out; + } +#endif + const FS::DirEntryCallable callback = [&out](const std::filesystem::directory_entry& entry) { const auto filename = FS::PathToUTF8String(entry.path().filename()); out.insert_or_assign(filename, diff --git a/src/core/hle/kernel/k_memory_manager.cpp b/src/core/hle/kernel/k_memory_manager.cpp index bed3dd819d..5e0a8b8378 100644 --- a/src/core/hle/kernel/k_memory_manager.cpp +++ b/src/core/hle/kernel/k_memory_manager.cpp @@ -450,6 +450,24 @@ Result KMemoryManager::AllocateForProcess(KPageGroup* out, size_t num_pages, u32 R_SUCCEED(); } +void KMemoryManager::ReleaseFreedPages(KPhysicalAddress address, size_t num_pages) { +#ifdef __ANDROID__ + // Freed guest memory stayed resident until reused: Mario Kart World's memory grew with every + // course it loaded (the last ones' regions freed but resident) until a phone killed it on the + // third. Small runs (thread stacks and the like) are left, not worth the system call. Phones + // only: desktop keeps freed memory for reuse as before. + constexpr size_t MinPages = 16; + if (num_pages < MinPages) { + return; + } + m_system.DeviceMemory().buffer.ReleaseBackingRegion( + GetInteger(address) - Core::DramMemoryMap::Base, num_pages * PageSize); +#else + static_cast(address); + static_cast(num_pages); +#endif +} + size_t KMemoryManager::Impl::Initialize(KPhysicalAddress address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p) { diff --git a/src/core/hle/kernel/k_memory_manager.h b/src/core/hle/kernel/k_memory_manager.h index 66b97fef8a..42cc144323 100644 --- a/src/core/hle/kernel/k_memory_manager.h +++ b/src/core/hle/kernel/k_memory_manager.h @@ -110,7 +110,9 @@ class KMemoryManager { { KScopedLightLock lk(m_pool_locks[static_cast(manager.GetPool())]); - manager.Close(address, cur_pages); + manager.Close(address, cur_pages, [this](KPhysicalAddress freed, size_t count) { + ReleaseFreedPages(freed, count); + }); } num_pages -= cur_pages; @@ -291,12 +293,19 @@ class KMemoryManager { } } - void Close(KPhysicalAddress address, size_t num_pages) { + // `on_free` sees each run of pages whose last reference went, before they're free. + template + void Close(KPhysicalAddress address, size_t num_pages, OnFree&& on_free) { size_t index = this->GetPageOffset(address); const size_t end = index + num_pages; size_t free_start = 0; size_t free_count = 0; + const auto free_run = [&] { + const KPhysicalAddress run = m_heap.GetAddress() + free_start * PageSize; + on_free(run, free_count); + this->Free(run, free_count); + }; while (index < end) { ASSERT(m_page_reference_counts[index] > 0); const RefCount ref_count = (--m_page_reference_counts[index]); @@ -311,7 +320,7 @@ class KMemoryManager { } } else { if (free_count > 0) { - this->Free(m_heap.GetAddress() + free_start * PageSize, free_count); + free_run(); free_count = 0; } } @@ -320,7 +329,7 @@ class KMemoryManager { } if (free_count > 0) { - this->Free(m_heap.GetAddress() + free_start * PageSize, free_count); + free_run(); } } @@ -367,6 +376,10 @@ class KMemoryManager { template using PoolArray = std::array(Pool::Count)>; + // Gives the host memory behind pages whose last reference just went back to the system, + // under the pool lock before they can be allocated again. They read as zero from then on. + void ReleaseFreedPages(KPhysicalAddress address, size_t num_pages); + Core::System& m_system; const KMemoryLayout& m_memory_layout; PoolArray m_pool_locks; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 93d51cd538..5df2ce4227 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -346,6 +346,7 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics3( nx2_caller = true; g_nx2_driver = true; Tegra::g_nx2_ampere = true; + Tegra::g_nx2_program_id = system.GetApplicationProcessProgramID(); ApplyNx2Characteristics(params.gc); params.gpu_characteristics_buf_size = Nx2CharacteristicsSize; LOG_WARNING(Service_NVDRV, "Switch 2 caller, reporting Ampere chip 0x17E"); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 0562ecb2cf..af33d99506 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -25,6 +25,7 @@ #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" namespace Service::Nvidia::Devices { namespace { @@ -181,6 +182,14 @@ NvResult nvhost_gpu::ZCullBind(IoctlZCullBind& params) { NvResult nvhost_gpu::SetErrorNotifier(IoctlSetErrorNotifier& params) { LOG_WARNING(Service_NVDRV, "(STUBBED) called, offset={:X}, size={:X}, mem={:X}", params.offset, params.size, params.mem); + // Like the host driver, clear the notifier when it is installed: NVN polls its status word and + // treats a non-zero value (whatever the buffer held before) as a channel error. + if (const auto handle = nvmap.GetHandle(params.mem); + handle && handle->address && params.size && params.size <= 0x10000 && + params.offset + params.size <= handle->size) { + const std::vector zeros(params.size); + system.ApplicationMemory().WriteBlock(handle->address + params.offset, zeros.data(), zeros.size()); + } return NvResult::Success; } @@ -440,12 +449,13 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL // frames ahead), and Mario Kart World then refills command memory it already // submitted: the end of a frame ran with the next frame's present image as its render // target, leaving an image three frames old to be shown. Keep the words as submitted. - // EDEN_NX2_PB_SNAPSHOT=0: execute from memory. + // EDEN_NX2_PB_SNAPSHOT=0 or EDEN_NX2_COMMAND_LISTS=0: execute from memory. static const bool snapshot = [] { const char* value = std::getenv("EDEN_NX2_PB_SNAPSHOT"); return value == nullptr || value[0] != '0'; }(); - if (snapshot && channel_state->memory_manager && channel_state->payload) { + if (snapshot && Tegra::Nx2CommandLists() && channel_state->memory_manager && + channel_state->payload) { // Before the copy: a suspension during it must count as one after it. entries.snapshot_suspensions = channel_state->payload->dma_pusher.suspensions.load(std::memory_order_acquire); diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h index 8dee9ccfb7..83fe8a7b31 100644 --- a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h +++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h @@ -54,17 +54,21 @@ class Nx2GpuMemoryRegistry { return registry; } - void Register(u64 address, u64 size) { - MapAt(address, address, size); + /// Returns the mapping's generation: what UnregisterIfUnchanged needs to undo this very + /// registration and not one made since over the range. + u64 Register(u64 address, u64 size) { + return MapAt(address, address, size); } - void MapAt(u64 gpu_address, u64 cpu_address, u64 size) { + u64 MapAt(u64 gpu_address, u64 cpu_address, u64 size) { std::scoped_lock lk{mutex}; TrimMappings(gpu_address, size); - mappings.push_back({gpu_address, cpu_address, size}); + const u64 generation = ++next_generation; + mappings.push_back({gpu_address, cpu_address, size, generation}); for (auto& [owner, listener] : listeners) { listener.map(gpu_address, cpu_address, size); } + return generation; } // NormalUnmapGpuMemory4K, or destroying the memory: the GPU range stops reaching it. @@ -76,6 +80,37 @@ class Nx2GpuMemoryRegistry { } } + // Undoes Register(address, size) if that mapping is still in place: unmaps it (on the GPU + // thread, after the command lists already submitted) and releases its backing unless another + // mapping of the same range shares it. A mapping made since over the range replaced it, and + // its GPU range and backing belong to that one now: then nothing is done (false). + /// `generation`: Register's result for the mapping to undo. A mapping made since over the + /// range (another Register, or 211 at the address) has a newer one and is left alone. + bool UnregisterIfUnchanged(u64 address, u64 size, u64 generation) { + bool shared{}; + { + std::scoped_lock lk{mutex}; + const auto it = std::ranges::find_if(mappings, [&](const Mapping& m) { + return m.gpu_address == address && m.cpu_address == address && m.size == size && + m.generation == generation; + }); + if (it == mappings.end()) { + return false; + } + mappings.erase(it); + shared = std::ranges::any_of(mappings, [&](const Mapping& m) { + return m.cpu_address == address && m.size == size; + }); + for (auto& [owner, listener] : listeners) { + listener.unmap(address, size); + } + } + if (!shared) { + ReleaseBacking(address, size); + } + return true; + } + // Backing memory [cpu_address, +size) is about to be freed (its GPU mappings are gone): each // address space drops its SMMU mapping of it, in GPU-thread order. void ReleaseBacking(u64 cpu_address, u64 size) { @@ -252,7 +287,9 @@ class Nx2GpuMemoryRegistry { u64 gpu_address; u64 cpu_address; u64 size; + u64 generation; ///< Which registration this is (MapAt's result) }; + u64 next_generation{}; struct Listeners { Listener map; diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index 7853d17235..0770c53200 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -5,9 +5,12 @@ // SPDX-FileCopyrightText: 2021 Skyline Team and Contributors // SPDX-License-Identifier: GPL-3.0-or-later +#include #include #include +#include "common/game_presets.h" +#include "common/settings.h" #include "core/core.h" #include "core/hle/kernel/k_event.h" #include "core/hle/service/ipc_helpers.h" @@ -134,9 +137,34 @@ Module::Module(Core::System& system) Tegra::Nx2ClearSemaphoreRegions(); Tegra::Nx2ClearSparseRegions(); Tegra::Nx2ClearGpuWrittenPages(); + Tegra::Nx2ClearGpuOnlyRegions(); Devices::g_nx2_driver = false; Tegra::g_nx2_ampere = false; Core::g_nx2_present_count = 0; + LOG_INFO(Service_NVDRV, "Switch 2 held CPU writes before command lists: {} (EDEN_NX2_HELD_WRITES)", + Tegra::Nx2HeldWritesEnabled() ? "on" : "off"); + { + // Read once per game start; EDEN_NX2_COMMAND_LISTS=0/1 overrides the setting. + bool command_lists = Settings::values.nx2_command_lists.GetValue(); +#ifdef __ANDROID__ + // On an Odin 3 the copies cost Mario Kart World ~45 ms of GPU-thread waiting a second + // and caused flicker, so it runs without them unless its own custom settings turn them + // on. Other games need them: Ocarina of Time deadlocks on the lists it rewrites. + if (system.GetApplicationProcessProgramID() == Settings::MarioKartWorldProgramId && + Settings::values.nx2_command_lists.UsingGlobal()) { + command_lists = false; + } +#endif + const char* const override_value = std::getenv("EDEN_NX2_COMMAND_LISTS"); + if (override_value != nullptr && override_value[0] != '\0') { + command_lists = override_value[0] != '0'; + } + Tegra::g_nx2_command_lists = command_lists; + LOG_INFO(Service_NVDRV, "Switch 2 command list fixes: {} ({})", command_lists ? "on" : "off", + override_value != nullptr && override_value[0] != '\0' + ? "EDEN_NX2_COMMAND_LISTS" + : "nx2_command_lists"); + } nx2_stub_builder = [this, &system](DeviceFD fd, const std::string& name) { auto device = std::make_shared(system, name); open_files.emplace(fd, device); diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index e2060c90a7..e2eecdf19e 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -356,6 +356,8 @@ class INx2GpuSession final : public ServiceFramework { ~INx2GpuSession() override { // Waits for a backing the GPU thread may be doing. Tegra::Nx2SetSparseBacking({}); + // Then the refill thread, done with the block it's on. + m_spare_thread = {}; for (auto* event : m_events) { if (event != nullptr) { m_context.CloseEvent(event); @@ -605,23 +607,30 @@ class INx2GpuSession final : public ServiceFramework { w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], w[9], w[10], w[11], w[12]); if (last > start && ((last + 1 - start) % Core::Memory::YUZU_PAGESIZE) == 0) { const u64 size = last + 1 - start; - std::scoped_lock lk{m_region_mutex}; + // Kernel and registry calls are made without m_region_mutex: the GPU thread takes it + // (BackSparseWrite), and they can wait on what waits on the GPU. FreeReleasedRegions(); if ((w[7] & SparseRegionFlag) == 0) { + std::scoped_lock lk{m_region_mutex}; m_gpu_only_ranges[start] = size; // kept out of 216, however it is backed } if ((w[7] & SparseRegionFlag) != 0 && LazySparseRegions()) { // Title 1's terrain texture arrays and layer pools (170 MB-5.8 GB, mostly empty): // backed whole, they ran the pool dry and the later ones got no memory at all. // The region gets a device range of its own, unbacked pages read zero, and - // BackSparseWrite gives a block memory before the GPU writes into it. - m_sparse_regions[start] = { - .size = size, - .blocks = std::vector(Common::DivCeil(size, SparseBlockSize)), - }; + // BackSparseWrite gives a block memory before the GPU writes into it. The device + // range first: the blocks BackSparseWrite backs are mapped in it. Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); - Tegra::Nx2SparseRegionCreated(start, size, SparseBlockSize); - Tegra::Nx2AddSparseSpan(start, size); + { + std::scoped_lock lk{m_region_mutex}; + m_sparse_regions[start] = { + .size = size, + .blocks = std::vector(Common::DivCeil(size, SparseBlockSize)), + }; + Tegra::Nx2SparseRegionCreated(start, size, SparseBlockSize); + Tegra::Nx2AddSparseSpan(start, size); + } + StartSpareRefill(); LOG_INFO(Service, "nx2:gpu 214 sparse region [{:#x}, {:#x}] backed as the GPU writes it", start, last); } else if (LazyGpuOnlyRegions()) { @@ -629,20 +638,32 @@ class INx2GpuSession final : public ServiceFramework { // title 4's render targets, 285-300 MB regions): like a sparse region it gets a // device range of its own and memory per block only when something writes guest // memory there (downloads, copies). The texture cache sees ordinary memory. - m_sparse_regions[start] = { - .gpu_only = true, - .size = size, - .blocks = std::vector(Common::DivCeil(size, SparseBlockSize)), - }; Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); - Tegra::Nx2AddSparseSpan(start, size); + { + std::scoped_lock lk{m_region_mutex}; + m_sparse_regions[start] = { + .gpu_only = true, + .size = size, + .blocks = std::vector(Common::DivCeil(size, SparseBlockSize)), + }; + Tegra::Nx2AddSparseSpan(start, size); + } + StartSpareRefill(); LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed as it is written", start, last); } else if (const auto address = BackGpuOnlyRegion(start, size)) { Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(start, *address, size); + Tegra::Nx2GpuOnlyRegionCreated(start, size); + u64 in_use{}; + u64 peak{}; + { + std::scoped_lock lk{m_region_mutex}; + in_use = m_gpu_only_bytes; + peak = m_gpu_only_peak; + } LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed at {:#x} " "({} MiB of GPU-only memory in use, peak {} MiB)", - start, last, *address, m_gpu_only_bytes >> 20, m_gpu_only_peak >> 20); + start, last, *address, in_use >> 20, peak >> 20); } } else { LOG_WARNING(Service, "nx2:gpu 214 unexpected region [{:#x}, {:#x}]", start, last); @@ -662,40 +683,62 @@ class INx2GpuSession final : public ServiceFramework { const u64 last = u64{w[5]} | (u64{w[6]} << 32); LOG_DEBUG(Service, "nx2:gpu 215 destroy sparse region [{:#x}, {:#x}] words {:#x} {:#x} {:#x} {:#x}", first, last, w[1], w[2], w[7], w[8]); - std::scoped_lock lk{m_region_mutex}; + // As in 214: kernel, registry and GPU calls without m_region_mutex. FreeReleasedRegions(); - if (m_gpu_only_ranges.erase(first) != 0 && last > first) { - DropGpuOnlyTiles(first, last - first + 1, false); - m_shared_regions.erase(first); - std::erase_if(m_shared_memory, [first](const auto& entry) { return entry.second == first; }); - } - if (last > first) { - if (const auto region = m_sparse_regions.find(first); region != m_sparse_regions.end()) { - m_tile_bytes -= Core::Memory::YUZU_PAGESIZE * - static_cast(std::count(region->second.tile_pages.begin(), - region->second.tile_pages.end(), true)); - m_sparse_regions.erase(region); - Tegra::Nx2SparseRegionDestroyed(first); + if (last <= first) { + std::scoped_lock lk{m_region_mutex}; + m_gpu_only_ranges.erase(first); + } else { + // Under the lock only what BackSparseWrite and 216 read. + bool sparse = false; + std::vector backings; + { + std::scoped_lock lk{m_region_mutex}; + if (m_gpu_only_ranges.erase(first) != 0) { + DropGpuOnlyTiles(first, last - first + 1, false); + m_shared_regions.erase(first); + std::erase_if(m_shared_memory, + [first](const auto& entry) { return entry.second == first; }); + } + if (const auto region = m_sparse_regions.find(first); + region != m_sparse_regions.end()) { + m_tile_bytes -= Core::Memory::YUZU_PAGESIZE * + static_cast(std::count(region->second.tile_pages.begin(), + region->second.tile_pages.end(), true)); + m_sparse_regions.erase(region); + Tegra::Nx2SparseRegionDestroyed(first); + sparse = true; + } else { + Tegra::Nx2GpuOnlyRegionDestroyed(first); + } + // The region's backing: all of it, or the blocks a sparse region got as it was + // written. + for (auto region = m_gpu_only_regions.lower_bound(first); + region != m_gpu_only_regions.end() && region->first <= last;) { + if (region->first + region->second.size - 1 > last) { + ++region; + continue; + } + backings.push_back(std::move(region->second)); + region = m_gpu_only_regions.erase(region); + } + } + if (sparse) { Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DestroySparse(first, last - first + 1); } else { Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1); } - // The region's backing: all of it, or the blocks a sparse region got as it was written. - std::shared_ptr> done; - for (auto region = m_gpu_only_regions.lower_bound(first); - region != m_gpu_only_regions.end() && region->first <= last;) { - if (region->first + region->second.size - 1 > last) { - ++region; - continue; + if (!backings.empty()) { + for (const auto& backing : backings) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().ReleaseBacking(backing.cpu_address, + backing.size); } - Nvidia::Devices::Nx2GpuMemoryRegistry::Get().ReleaseBacking( - region->second.cpu_address, region->second.size); - if (!done) { - done = std::make_shared>(false); - system.GPU().RunInOrder([done] { done->store(true, std::memory_order_release); }); + auto done = std::make_shared>(false); + system.GPU().RunInOrder([done] { done->store(true, std::memory_order_release); }); + std::scoped_lock lk{m_region_mutex}; + for (auto& backing : backings) { + m_released_regions.push_back({std::move(backing), done}); } - m_released_regions.push_back({std::move(region->second), done}); - region = m_gpu_only_regions.erase(region); } } SignalAfterGpu(w[8]); @@ -715,9 +758,9 @@ class INx2GpuSession final : public ServiceFramework { const auto data = ctx.BufferDescriptorA().empty() ? std::span{} : ctx.ReadBufferA(0); const size_t count = data.size() / sizeof(SparseMapRequest); std::vector kinds(count); + FreeReleasedRegions(); // takes kernel locks: before m_region_mutex { std::scoped_lock lk{m_region_mutex}; - FreeReleasedRegions(); ++m_tile_calls; for (size_t i = 0; i < count; ++i) { SparseMapRequest request; @@ -763,25 +806,46 @@ class INx2GpuSession final : public ServiceFramework { return true; } - // Returns the pages of destroyed regions the GPU thread is done with to the kernel. + // Returns the pages of destroyed regions the GPU thread is done with to the kernel. Takes + // kernel locks: called without m_region_mutex. void FreeReleasedRegions() { + std::vector regions; + { + std::scoped_lock lk{m_region_mutex}; + std::erase_if(m_released_regions, [&](ReleasedRegion& released) { + if (!released.done->load(std::memory_order_acquire)) { + return false; + } + regions.push_back(std::move(released.region)); + return true; + }); + } + if (regions.empty()) { + return; + } auto* const process = system.ApplicationProcess(); - std::erase_if(m_released_regions, [&](ReleasedRegion& released) { - if (!released.done->load(std::memory_order_acquire)) { - return false; - } - auto& region = released.region; + u64 freed = 0; + for (auto& region : regions) { if (process != nullptr && R_FAILED(process->GetPageTable().UnmapPageGroup( region.cpu_address, *region.pages, Kernel::KMemoryState::IoMemory))) { LOG_ERROR(Service, "nx2:gpu could not unmap GPU-only backing {:#x}+{:#x}", region.cpu_address, region.size); - return true; // still mapped: keep its pages + continue; // still mapped: keep its pages } + freed += region.pages->GetNumPages() * Core::Memory::YUZU_PAGESIZE; region.pages->Close(system.Kernel()); - m_gpu_only_bytes -= region.size; - return true; - }); + } + { + std::scoped_lock lk{m_region_mutex}; + m_gpu_only_bytes -= (std::min)(m_gpu_only_bytes, freed); + } + // Memory came back: a refill that found none can try again. + { + std::scoped_lock lk{m_spare_mutex}; + m_spare_freed = true; + } + m_spare_cv.notify_all(); } struct MappedBacking { @@ -848,7 +912,8 @@ class INx2GpuSession final : public ServiceFramework { return std::nullopt; } - // Records a mapped backing for the region at gpu_address. Needs m_region_mutex. + // Records a mapped backing for the region at gpu_address (backing.size bytes of it; a spare + // block for a region's short last block has more pages). Needs m_region_mutex. void RegisterGpuOnlyBacking(u64 gpu_address, MappedBacking&& backing) { // A title may create the same region again from the same start with a different size // (Mario Kart World walks 300 MB -> 600 MB -> 1 GB on the same address). Overwriting the @@ -862,22 +927,24 @@ class INx2GpuSession final : public ServiceFramework { it->second.size); auto done = std::make_shared>(false); system.GPU().RunInOrder([done] { done->store(true, std::memory_order_release); }); - m_gpu_only_bytes -= it->second.size; + // Counted out by FreeReleasedRegions, by pages, once the pages are freed. m_released_regions.push_back({std::move(it->second), done}); m_gpu_only_regions.erase(it); } + const u64 bytes = backing.pages->GetNumPages() * Core::Memory::YUZU_PAGESIZE; m_gpu_only_regions[gpu_address] = {backing.address, backing.size, std::move(backing.pages)}; - m_gpu_only_bytes += backing.size; + m_gpu_only_bytes += bytes; m_gpu_only_peak = (std::max)(m_gpu_only_peak, m_gpu_only_bytes); } - // Needs m_region_mutex. + // Allocates and maps without m_region_mutex (kernel locks), then records under it. std::optional BackGpuOnlyRegion(u64 gpu_address, u64 size) { auto backing = AllocateGpuOnlyBacking(size, true, false); if (!backing) { return std::nullopt; } const u64 address = backing->address; + std::scoped_lock lk{m_region_mutex}; RegisterGpuOnlyBacking(gpu_address, std::move(*backing)); return address; } @@ -885,7 +952,8 @@ class INx2GpuSession final : public ServiceFramework { // Runs before a GPU-side write that may land in a sparse region (Tegra::Nx2BackSparseWrite, // usually on the GPU thread): the blocks the write touches get memory, cleared, behind the // region's device pages. The GPU mapping stays as it is, so the images over the region keep - // their storage and only see the write. + // their storage and only see the write. The memory comes from the spare blocks (TakeSpare): no + // kernel locks here. void BackSparseWrite(u64 gpu_address, u64 size, const u8* data) { struct Pending { u64 region; @@ -950,7 +1018,7 @@ class INx2GpuSession final : public ServiceFramework { m_sparse_zero_bytes >> 20); } } - // Allocating and mapping happens outside the lock (see AllocateGpuOnlyBacking). + // Spare blocks are taken outside the lock (TakeSpare can wait for the refill thread). for (const auto& [start, block] : todo) { u64 block_size = SparseBlockSize; { @@ -960,7 +1028,8 @@ class INx2GpuSession final : public ServiceFramework { block_size = (std::min)(SparseBlockSize, it->second.size - block * SparseBlockSize); } } - auto backing = AllocateGpuOnlyBacking(block_size, false, true); + bool exhausted = false; + auto backing = TakeSpare(exhausted); bool stale = false; { std::scoped_lock lk{m_region_mutex}; @@ -972,7 +1041,7 @@ class INx2GpuSession final : public ServiceFramework { stale = true; it->second.blocks[block] = BlockWritten; Tegra::Nx2SparseBlockWritten(start, block); - } else if (!backing) { + } else if (!backing && exhausted) { auto& region = it->second; region.blocks[block] = BlockFailed; // out of memory: not again for every write if (!region.warned) { @@ -981,9 +1050,18 @@ class INx2GpuSession final : public ServiceFramework { "to its unbacked blocks are dropped", start); } + } else if (!backing) { + // No spare ready in time: left empty, so the next write to it tries again. + it->second.blocks[block] = BlockEmpty; + if (++m_spare_drops <= 16 || (m_spare_drops & (m_spare_drops - 1)) == 0) { + LOG_WARNING(Service, "nx2:gpu sparse block {:#x}: no memory ready for it, GPU " + "write dropped ({} so far)", + start + block * SparseBlockSize, m_spare_drops); + } } else { const u64 block_address = start + block * SparseBlockSize; const u64 address = backing->address; + backing->size = block_size; // a spare is a whole block: the last may be shorter it->second.blocks[block] = BlockBacked; RegisterGpuOnlyBacking(block_address, std::move(*backing)); Nvidia::Devices::Nx2GpuMemoryRegistry::Get().BackSparse(block_address, address, @@ -1000,13 +1078,9 @@ class INx2GpuSession final : public ServiceFramework { } m_region_cv.notify_all(); if (stale && backing) { - // Never handed to the GPU: unmap and free right away. - if (auto* const process = system.ApplicationProcess(); process != nullptr) { - process->GetPageTable().UnmapPageGroup(Common::ProcessAddress{backing->address}, - *backing->pages, - Kernel::KMemoryState::IoMemory); - } - backing->pages->Close(system.Kernel()); + // Never handed to the GPU, still clear: back to the spares (freeing it would take + // kernel locks here). + ReturnSpare(std::move(*backing)); } } if (waits_for_other) { @@ -1024,6 +1098,85 @@ class INx2GpuSession final : public ServiceFramework { } } + // Sparse blocks get memory on the GPU thread, which must never wait on the kernel: a guest + // thread holding the page table lock can need a CPU core that is waiting on the GPU. Mario + // Kart World deadlocked loading a race that way: core 2 in a fault handler invalidating GPU + // memory, the GPU behind the back thread, the back thread on the page table lock, and the + // lock's owner runnable on core 2 only. So blocks are allocated and mapped ahead, on a thread + // of their own, and BackSparseWrite takes them from here. Pool and address space only: host + // memory is used once a block is written. + static constexpr size_t SpareBlocks = 16; + static constexpr auto SpareWait = std::chrono::milliseconds{200}; + + void StartSpareRefill() { + std::scoped_lock lk{m_spare_mutex}; + if (!m_spare_thread.joinable()) { + m_spare_thread = std::jthread([this](std::stop_token stop) { RefillSpares(stop); }); + } + } + + void RefillSpares(std::stop_token stop) { + Common::SetCurrentThreadName("nx2:SparseRefill"); + bool failed = false; + while (true) { + { + std::unique_lock lk{m_spare_mutex}; + if (failed) { + // Out of memory: again once regions are freed, or in a second. + m_spare_cv.wait_for(lk, stop, std::chrono::seconds{1}, [&] { return m_spare_freed; }); + m_spare_freed = false; + } + if (!m_spare_cv.wait(lk, stop, [&] { return m_spare_blocks.size() < SpareBlocks; })) { + return; + } + } + auto backing = AllocateGpuOnlyBacking(SparseBlockSize, false, true); + { + std::scoped_lock lk{m_spare_mutex}; + failed = !backing; + m_spare_failed = failed; + if (backing) { + m_spare_blocks.push_back(std::move(*backing)); + m_spare_starved = false; + } + } + m_spare_cv.notify_all(); + } + } + + // A spare block for BackSparseWrite. None: `exhausted` when the pool ran out of memory, else + // the refill thread didn't have one ready in time (it may be stuck in the kernel behind the + // GPU itself: no more waiting then until it adds one). + std::optional TakeSpare(bool& exhausted) { + std::unique_lock lk{m_spare_mutex}; + if (m_spare_blocks.empty() && !m_spare_failed && !m_spare_starved) { + m_spare_cv.notify_all(); + if (!m_spare_cv.wait_for(lk, SpareWait, + [&] { return !m_spare_blocks.empty() || m_spare_failed; })) { + m_spare_starved = true; + } + } + if (m_spare_blocks.empty()) { + exhausted = m_spare_failed; + return std::nullopt; + } + MappedBacking backing = std::move(m_spare_blocks.back()); + m_spare_blocks.pop_back(); + lk.unlock(); + m_spare_cv.notify_all(); + return backing; + } + + // A spare TakeSpare gave out that went unused. + void ReturnSpare(MappedBacking&& backing) { + backing.size = SparseBlockSize; + { + std::scoped_lock lk{m_spare_mutex}; + m_spare_blocks.push_back(std::move(backing)); + } + m_spare_cv.notify_all(); + } + // EDEN_NX2_SPARSE_LAZY=0 backs sparse regions whole, as before. static bool LazySparseRegions() { static const bool lazy = [] { @@ -1389,6 +1542,15 @@ class INx2GpuSession final : public ServiceFramework { u64 m_gpu_only_tile_max{}; ///< Largest such tile, bounds the search u64 m_gpu_only_tile_bytes{}; ///< GPU-only region memory the game maps right now std::vector m_released_regions; + u64 m_spare_drops{}; ///< GPU writes dropped for want of a spare block, under m_region_mutex + + std::mutex m_spare_mutex; + std::condition_variable_any m_spare_cv; + std::vector m_spare_blocks; + bool m_spare_failed{}; ///< The last refill found no memory + bool m_spare_starved{}; ///< A take timed out: no waiting until the refill thread adds a block + bool m_spare_freed{}; ///< Regions were freed since the last failed refill + std::jthread m_spare_thread; ///< Stopped by the destructor, before what it uses goes struct GpuMemory { u64 address; @@ -1462,10 +1624,20 @@ struct Nx2NativeObjects { bool gpu_read{}; u32 gpu_maps{}; // outstanding 426 maps bool mapped_here{}; // 426 added its GPU mapping (no other mapping covered it) + u64 map_generation{}; // that mapping's registration (Nx2GpuMemoryRegistry::Register) + }; + // Mappings 426 made for decoder images that went while the client still had them mapped, + // dropped a while later (INx2GraphicsSystemSession::RetireDecoderMapping). + // Also a mapping 426 made, taken from its image (TakeDecoderMapping) to be dropped. + struct RetiredMapping { + u64 address; + u64 size; + u64 generation; ///< Its registration: a newer one at the address is kept when it goes }; std::mutex mutex; std::map memories; std::map images; + std::deque retired; u32 next_memory{1}; u32 next_image{1}; }; @@ -1734,10 +1906,11 @@ class INx2Vi2Session final : public ServiceFramework { // only after this present returns: while a channel stays suspended on one, wait for the // previous frame's fence instead and let the game prepare frame N+1 meanwhile (waiting // for this frame there cost 8-56 ms of every 34-105 ms frame, or never finished). - // EDEN_NX2_FRAMES_AHEAD=1 always waits for the previous frame, =0 always for this one. + // EDEN_NX2_FRAMES_AHEAD=1 always waits for the previous frame, =0 always for this one, + // =2 only ever for the previous one, even with the GPU behind. static const int frames_ahead_setting = [] { const char* value = std::getenv("EDEN_NX2_FRAMES_AHEAD"); - return value == nullptr ? -1 : value[0] == '1' ? 1 : 0; + return value == nullptr ? -1 : value[0] == '2' ? 2 : value[0] == '1' ? 1 : 0; }(); // Donkey Kong Bananza's channels wait on each other every frame, so it always ran a frame // ahead, where it rewrote memory the GPU thread hadn't read yet (misread commands, then @@ -1749,7 +1922,10 @@ class INx2Vi2Session final : public ServiceFramework { frames_ahead = 0; } const auto entered = std::chrono::steady_clock::now(); - if (frames_ahead == 1 || (frames_ahead == -1 && !WaitForFrame(~w[22], w[24], true))) { + if (frames_ahead == 2) { + WaitForFrame(m_previous_fence_id, m_previous_fence_value); + } else if (frames_ahead == 1 || + (frames_ahead == -1 && !WaitForFrame(~w[22], w[24], true))) { LOG_DEBUG(Service, "nx2:vi2 present #{} fence ({}, {:#x}), waiting for ({}, {:#x}); now {:#x}", m_presents, ~w[22], w[24], m_previous_fence_id, m_previous_fence_value, ~w[22] < 192 ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(~w[22]) : 0U); @@ -1837,6 +2013,10 @@ class INx2Vi2Session final : public ServiceFramework { // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it). std::chrono::steady_clock::time_point m_next_vblank{}; u32 m_swap_word{}; + // The game bound a native event to vsync (261) and waits on it between frames: it paces + // itself, so PaceToVblank leaves its presents alone. + bool m_game_waits_for_vsync{}; + bool m_logged_vsync_pacing{}; // timing: present word 12, (swap interval << 16) | refresh rate. Fire Emblem: Fortune's Weave // sends 0x1003c during its boot logos and 0x2003c (two vblanks per frame, 30 fps) after; paced @@ -1849,12 +2029,44 @@ class INx2Vi2Session final : public ServiceFramework { const char* value = std::getenv("EDEN_NX2_SWAP_INTERVAL"); return value ? static_cast(std::strtoul(value, nullptr, 10)) : 0U; }(); - const auto& process = Kernel::GetCurrentProcess(kernel); - const bool mkw_dynamic = - process.GetProgramId() == Settings::MarioKartWorldProgramId && - process.IsMkwDynamicFpsPatchApplied() && Settings::values.mkw_dynamic_fps.GetValue(); + // This runs on the vi2 service thread, whose current process is not the game's: look the + // application up instead. + const bool is_mkw = + system.GetApplicationProcessProgramID() == Settings::MarioKartWorldProgramId; + const auto* const app = is_mkw ? system.ApplicationProcess() : nullptr; + const bool mkw_dynamic = app != nullptr && app->IsMkwDynamicFpsPatchApplied() && + Settings::values.mkw_dynamic_fps.GetValue(); + // Mario Kart World runs at 60 fps and asks for two vblanks a frame only when a frame ran + // long, which on slower hosts is nearly every frame: honouring that holds it at 30 fps + // (12-15 ms of sleep a frame on an Odin 3, where frames take 18-25 ms). Its interval is + // honoured by default; EDEN_NX2_MKW_PACING=1 paces it at one vblank, =0 doesn't pace it. + static const int mkw_pacing = [] { + const char* value = std::getenv("EDEN_NX2_MKW_PACING"); + if (value == nullptr) { + return 2; + } + return value[0] == '0' ? 0 : (value[0] == '1' ? 1 : 2); + }(); + if (is_mkw && mkw_pacing == 0 && forced_interval == 0) { + return; + } + // A game that waits on the vsync event between frames already runs on the vblank timer. + // Pacing its presents to a grid too, of another phase, made it miss that timer's next + // vblank: Mario Kart World, which binds one, ran at 27-30 fps with 4-14 ms of sleep a + // frame while Eden's GPU threads were idle half the time. Not pacing it at all let it run + // past 60 fps on a fast PC, and its game time follows its frames. So such a game's + // presents are only capped at the swap interval on average, one present of slack + // allowed, which never sleeps a game that runs slower than that. Fortune's Weave binds no + // vsync event and keeps the grid (at every vblank it ran at double speed). + // EDEN_NX2_PACE_VSYNC_GAMES=1 paces them to the grid anyway. + static const bool pace_vsync_games = [] { + const char* value = std::getenv("EDEN_NX2_PACE_VSYNC_GAMES"); + return value != nullptr && value[0] == '1'; + }(); + const bool cap_only = m_game_waits_for_vsync && !pace_vsync_games && + forced_interval == 0 && !(is_mkw && mkw_pacing == 1); u32 swap_interval = std::clamp(timing >> 16, 1, 4); - if (mkw_dynamic) { + if (mkw_dynamic || (is_mkw && mkw_pacing == 1)) { swap_interval = 1; } if (forced_interval != 0) { @@ -1868,6 +2080,21 @@ class INx2Vi2Session final : public ServiceFramework { const auto interval = std::chrono::nanoseconds{ 100'000'000'000LL * swap_interval / refresh_rate / Settings::SpeedLimit()}; const auto now = std::chrono::steady_clock::now(); + if (cap_only) { + if (!std::exchange(m_logged_vsync_pacing, true)) { + LOG_INFO(Service, "nx2:vi2 the game waits on the vsync event: presents capped at " + "the swap interval, not paced to a grid"); + } + // m_next_vblank is the earliest time the next present is due at the capped rate. + m_next_vblank = (std::max)(m_next_vblank, now); + if (const auto early = m_next_vblank - now; early > interval) { + const auto wait = std::chrono::duration_cast(early - interval); + static_cast(Kernel::GetCurrentThread(kernel).Sleep( + kernel, kernel.HardwareTimer().GetTick() + wait.count() + 2)); + } + m_next_vblank += interval; + return; + } if (now > m_next_vblank + interval) { // Running slower than the swap interval: show at once and don't bank the lost time. m_next_vblank = now; @@ -2248,6 +2475,7 @@ class INx2Vi2Session final : public ServiceFramework { LOG_INFO(Service, "nx2:vi2 261 native event {} -> vsync", index); if (index < m_native_events.size() && m_native_events[index]) { SetVsyncSlot(index, true); + m_game_waits_for_vsync = true; } TipcReplyWords(ctx, {}); } @@ -2451,15 +2679,66 @@ class INx2VideoSession final : public ServiceFramework { static constexpr u32 NoSyncpoint = ~0U; static constexpr size_t MaxQueuedInputs = 8; + static constexpr size_t MaxQueuedAv1Inputs = 3; // fefw's MaximumPendingInputs static constexpr size_t MaxDecodedAhead = 2; // decoded frames held while no image is free static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager + static constexpr u64 SummaryFrames = 150; // AV1: a state line every 150 frames (5 s) + static constexpr std::chrono::seconds StallReport{2}; + static constexpr u32 EmptyOutputResult = 0x1c286f; // 213: nothing finished + static constexpr u32 InputBusyResult = 0x1c266f; // 109: input queue full (fefw) + + // EDEN_NX2_VDEC_TRACE=1: every video decoder call, reply and written frame at Info level. + static bool Tracing() { + static const bool trace = std::getenv("EDEN_NX2_VDEC_TRACE") != nullptr; + return trace; + } + + // With m_mutex held. + size_t InputLimit() const { + return m_av1 ? MaxQueuedAv1Inputs : MaxQueuedInputs; + } + + // With m_mutex held: where every queue stands, to tell from the log which side a movie that + // stops waits on (the client's input or releases, or the decoder). + void LogState(std::string_view what) { + using Ms = std::chrono::duration; + using S = std::chrono::duration; + const u32 reached = + m_syncpoint == NoSyncpoint + ? 0 + : static_cast( + system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(m_syncpoint)); + const double decode_ms = + m_timed_decodes ? Ms(m_decode_time).count() / static_cast(m_timed_decodes) : 0; + const double write_ms = + m_timed_writes ? Ms(m_write_time).count() / static_cast(m_timed_writes) : 0; + const double stream_s = + static_cast(m_last_fetch_pts - m_first_fetch_pts) / 1'000'000.0; + const double wall_s = + m_first_fetch_time ? S(std::chrono::steady_clock::now() - *m_first_fetch_time).count() + : 0; + LOG_INFO(Service, + "{} {} frame {}: {} inputs ({} KiB, {} busy, {} without a picture), {} fetched, " + "{} empty polls, {} releases ({} idle); waiting {} inputs, {} decoded, {} free " + "images, {} unfetched; sync {} of {} handed; decode {:.1f} ms, write {:.1f} ms; " + "pts in {} us, stream {:.2f} s fetched in {:.2f} s", + GetServiceName(), what, m_frames_out, m_submitted, m_submitted_bytes >> 10, + m_busy_submits, m_no_picture, m_fetched, m_empty_polls, m_releases, + m_idle_releases, m_inputs.size(), m_decoded.size(), m_free.size(), + m_outputs.size(), reached, m_sync_base + m_handed, decode_ms, write_ms, + m_last_pts_in, stream_s, wall_s); + m_decode_time = {}; + m_timed_decodes = 0; + m_write_time = {}; + m_timed_writes = 0; + } void Log(HLERequestContext& ctx, std::string_view what) { const auto* w = ctx.CommandBuffer(); if (Nx2FirstVideoCommand(GetServiceName(), ctx.GetCommand())) { LOG_INFO(Service, "{} TIPC {} {} (first use): words {:08x}", GetServiceName(), ctx.GetCommand(), what, fmt::join(std::span{w, 12}, " ")); - } else if (static const bool trace = std::getenv("EDEN_NX2_VDEC_TRACE") != nullptr; trace) { + } else if (Tracing()) { // EDEN_NX2_VDEC_TRACE=1: every video decoder call at Info level. LOG_INFO(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}", GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5]); @@ -2505,7 +2784,10 @@ class INx2VideoSession final : public ServiceFramework { if (!name_buf.empty()) { if (codec.empty()) { const std::string_view sv{reinterpret_cast(name_buf.data()), name_buf.size()}; - for (const auto& candidate : {"video/avc", "video/vp9", "video/vp8", "video/hevc"}) { + // Both AV1 spellings Nx2IsAv1Mime accepts, so a buffer naming AV1 off its start + // still configures the AV1 path. + for (const auto& candidate : {"video/avc", "video/vp9", "video/vp8", "video/hevc", + "video/av01", "video/av1"}) { if (sv.find(candidate) != std::string_view::npos) { codec = candidate; break; @@ -2520,8 +2802,10 @@ class INx2VideoSession final : public ServiceFramework { Log(ctx, "configure"); LOG_INFO(Service, "{} configure codec \"{}\" {}x{} rate {}", GetServiceName(), codec, w[5], w[6], w[7]); + const bool av1 = Tegra::Host1x::Nx2IsAv1Mime(codec); { std::scoped_lock lk{m_mutex}; + m_av1 = av1; m_codec = std::move(codec); m_width = w[5]; m_height = w[6]; @@ -2529,6 +2813,12 @@ class INx2VideoSession final : public ServiceFramework { if (!m_worker.joinable()) { m_worker = std::thread{[this] { DecodeLoop(); }}; } + if (av1) { + // fefw replies a u64 of zero. With the result alone the client read the words its + // request left in the buffer (the codec name's address) as that value. + TipcReplyWords(ctx, {0, 0}); + return; + } TipcReplyWords(ctx, {}); } @@ -2574,6 +2864,24 @@ class INx2VideoSession final : public ServiceFramework { u32 value; { std::scoped_lock lk{m_mutex}; + // AV1 (fefw e75ce0c202): Fortune's Weave gives back images that still wait for a + // frame, in a loop after Flush/Start. Frames advance the syncpoint by one each in + // hand-out order, so a new value would leave a hole in the sequence and put every + // later fence out of reach: answer with the value the image already has. + if (m_av1 && release_fence && IsWaiting(image)) { + if ((++m_idle_releases & 0xfff) == 1) { + LOG_WARNING(Service, "{} release of waiting image {} (x{})", + GetServiceName(), image, m_idle_releases); + } + if (Tracing() && (m_idle_releases <= 16 || (m_idle_releases & 0xff) == 0)) { + LOG_INFO(Service, "{} release of waiting image {}: sync value {} kept", + GetServiceName(), image, m_values[image]); + } + return {~m_syncpoint, 0, m_values[image], 0}; + } + if (release_fence) { + ++m_releases; + } m_free.push_back(image); if (release_fence) { m_release_fences[image] = *release_fence; @@ -2596,22 +2904,53 @@ class INx2VideoSession final : public ServiceFramework { } else { value = m_sync_base + ++m_handed; } + m_values[image] = value; + if (Tracing()) { + LOG_INFO(Service, "{} {} image {}: sync value {} ({} images wait)", + GetServiceName(), release_fence ? "release" : "register", image, value, + m_free.size()); + } } m_cv.notify_one(); return {~m_syncpoint, 0, value, 0}; } + // 311/314 reply: the NativeSyncHandle. AV1: then two u64s of zero, as fefw replies; with the + // handle alone the client read the words its request left in the buffer there. + void ReplySyncHandle(HLERequestContext& ctx, const std::array& handle) { + if (m_av1) { + TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3], 0, 0, 0, 0}); + return; + } + TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); + } + + // With m_mutex held: the image waits for a frame. + bool IsWaiting(u32 image) const { + return m_writing == image || std::ranges::find(m_free, image) != m_free.end(); + } + + // With m_mutex held: an image that holds a frame waits for one again, at the end of the sync + // sequence (its old value was reached when the frame was written). + void Requeue(u32 image) { + if (IsWaiting(image)) { + return; + } + m_free.push_back(image); + m_values[image] = m_sync_base + ++m_handed; + } + void RegisterBuffer(HLERequestContext& ctx) { Log(ctx, "register output"); const auto handle = HandOut(ctx.CommandBuffer()[2], std::nullopt); - TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); + ReplySyncHandle(ctx, handle); } void ReleaseBuffer(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); Log(ctx, "release output"); const auto handle = HandOut(w[2], Fence{~w[4], w[6]}); - TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); + ReplySyncHandle(ctx, handle); } void UnregisterBuffer(HLERequestContext& ctx) { @@ -2621,6 +2960,7 @@ class INx2VideoSession final : public ServiceFramework { std::scoped_lock lk{m_mutex}; std::erase(m_free, image); m_release_fences.erase(image); + m_values.erase(image); } TipcReplyWords(ctx, {}); } @@ -2638,15 +2978,47 @@ class INx2VideoSession final : public ServiceFramework { const auto data = ctx.ReadBufferA(0); input.data.assign(data.begin(), data.end()); } + const u64 size = input.data.size(); + u64 id{}; + bool busy{}; + size_t queued{}; { std::scoped_lock lk{m_mutex}; - m_inputs.push_back(std::move(input)); - m_stream_ended = false; - m_ready->room.store(!end && m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed); + if (m_av1 && m_inputs.size() >= MaxQueuedAv1Inputs) { + // AV1: at most three inputs wait, as in fefw, which tells the client to come back + // (0x1c266f, nothing taken); it waits for the submission event and submits the + // same frame again. + busy = true; + ++m_busy_submits; + m_ready->room.store(false, std::memory_order_relaxed); + } else { + m_inputs.push_back(std::move(input)); + m_stream_ended = false; + m_ready->room.store(!end && m_inputs.size() < InputLimit(), + std::memory_order_relaxed); + id = ++m_submitted; + m_submitted_bytes += size; + m_last_pts_in = pts; + } + queued = m_inputs.size(); + } + // Busy replies only now and then: a client may come back in a loop. + if (Tracing() && (!busy || (m_busy_submits & (m_busy_submits - 1)) == 0)) { + LOG_INFO(Service, "{} submit {} bytes, pts {} us{}: {} ({} inputs wait)", + GetServiceName(), size, pts, end ? ", end of stream" : "", + busy ? std::string{"busy"} : fmt::format("input {}", id), queued); + } + if (busy) { + IPC::ResponseBuilder rb{ctx, 4}; + rb.PushRaw(InputBusyResult); + rb.Push(0); + rb.Push(0); + return; } m_cv.notify_one(); - const u64 id = ++m_submitted; - TipcReplyWords(ctx, {static_cast(id), static_cast(id >> 32)}); + // AV1: the bytes taken, as fefw's decoder replies (Fortune's Weave plays with it). + const u64 reply = m_av1 ? size : id; + TipcReplyWords(ctx, {static_cast(reply), static_cast(reply >> 32)}); } // Flush (8) and Stop (207): drop queued input and frames not yet in an image. Images already @@ -2664,6 +3036,18 @@ class INx2VideoSession final : public ServiceFramework { if (ctx.GetCommand() == 207) { CompleteHandedFences(); } + if (m_av1) { + // fefw 2106415948/141643e8e8: Fortune's Weave flushes its character select clips + // when one ends or the character changes, then calls Start. Like + // MediaCodec.flush(), drop everything in flight: frames written but not fetched + // (the old clip's end of stream among them) made it take the new clip as + // finished. Their images wait for frames again. + ++m_flushes; + for (const auto& output : m_outputs) { + Requeue(output.image); + } + m_outputs.clear(); + } } m_cv.notify_one(); TipcReplyWords(ctx, {}); @@ -2695,21 +3079,56 @@ class INx2VideoSession final : public ServiceFramework { void GetOutputInfo(HLERequestContext& ctx) { Log(ctx, "output info"); std::optional output; + u32 width{}; + u32 height{}; + bool av1{}; { std::scoped_lock lk{m_mutex}; + av1 = m_av1; if (!m_outputs.empty()) { output = m_outputs.front(); m_outputs.pop_front(); + // AV1: words 1-2 hold the configured size, as fefw's decoder replies. + width = m_av1 ? m_width : output->width; + height = m_av1 ? m_height : output->height; + ++m_fetched; + if (!m_first_fetch_time) { + m_first_fetch_time = std::chrono::steady_clock::now(); + m_first_fetch_pts = output->pts; + } + if (!output->end) { + m_last_fetch_pts = std::max(m_last_fetch_pts, output->pts); + } + if (Tracing()) { + LOG_INFO(Service, "{} 213: image {}, pts {} us{} after {} empty polls ({} more " + "finished)", + GetServiceName(), output->image, output->pts, + output->end ? ", end of stream" : "", m_empty_streak, + m_outputs.size()); + } + m_empty_streak = 0; + } else { + ++m_empty_polls; + if (Tracing() && m_empty_streak++ == 0) { + LOG_INFO(Service, "{} 213: none ({} decoded, {} free images, {} inputs wait)", + GetServiceName(), m_decoded.size(), m_free.size(), m_inputs.size()); + } } } if (!output) { - // Nothing finished: the code the client's own Acquire uses for an empty ring. - IPC::ResponseBuilder rb{ctx, 2}; - rb.PushRaw(0x1c286f); + // Nothing finished: the code the client's own Acquire uses for an empty ring. AV1: + // with the 16 words of zero fefw replies. + IPC::ResponseBuilder rb{ctx, av1 ? 18U : 2U}; + rb.PushRaw(EmptyOutputResult); + if (av1) { + for (int i = 0; i < 16; ++i) { + rb.Push(0); + } + } return; } const u64 pts_ns = output->pts * 1000; - TipcReplyWords(ctx, {output->image, output->width, output->height, output->width, + TipcReplyWords(ctx, {output->image, width, height, output->width, output->height, 0, static_cast(pts_ns), static_cast(pts_ns >> 32), output->end ? 1U : 0U, 0, 0, 0, 0, 0, 0, 0}); @@ -2722,12 +3141,22 @@ class INx2VideoSession final : public ServiceFramework { std::unique_ptr decoder; std::vector pictures; std::vector bytes; + u64 stall_reported = ~0ULL; std::unique_lock lk{m_mutex}; while (true) { - m_cv.wait(lk, [&] { + const auto ready = [&] { return m_quit || m_flush || (!m_decoded.empty() && !m_free.empty()) || (!m_inputs.empty() && m_decoded.size() < MaxDecodedAhead); - }); + }; + if (!m_cv.wait_for(lk, StallReport, ready)) { + // A movie that stops: say once what it waits on (input, a release or a frame). + if ((m_av1 || Tracing()) && m_frames_out != 0 && !m_stream_ended && + stall_reported != m_frames_out) { + stall_reported = m_frames_out; + LogState("no frame for 2 s after"); + } + continue; + } if (m_quit) { return; } @@ -2746,6 +3175,9 @@ class INx2VideoSession final : public ServiceFramework { m_decoded.pop_front(); const u32 image = m_free.front(); m_free.pop_front(); + m_writing = image; + const u64 flushes = m_flushes; + const bool av1 = m_av1; std::optional fence; if (const auto it = m_release_fences.find(image); it != m_release_fences.end()) { fence = it->second; @@ -2755,9 +3187,22 @@ class INx2VideoSession final : public ServiceFramework { if (fence) { WaitForRelease(*fence); } - const auto [width, height] = WriteImage(decoder.get(), image, frame, bytes); + const auto write_start = std::chrono::steady_clock::now(); + const auto [width, height] = WriteImage(decoder.get(), image, frame, bytes, av1); + const auto write_time = std::chrono::steady_clock::now() - write_start; lk.lock(); + m_writing.reset(); + if (frame.picture) { + m_write_time += write_time; + ++m_timed_writes; + } // Even an image written during a flush counts: the client waits for its value. + // An AV1 flush drops the frame, though (see Flush): the image waits again. + if (m_av1 && m_flushes != flushes) { + Requeue(image); + system.Host1x().GetSyncpointManager().IncrementGuest(m_syncpoint); + continue; + } m_outputs.push_back({image, width, height, frame.pts, frame.end}); if (frame.end) { m_stream_ended = true; @@ -2769,13 +3214,19 @@ class INx2VideoSession final : public ServiceFramework { LOG_INFO(Service, "{} frame {} ({}x{}, pts {} us{}) in image {}", GetServiceName(), m_frames_out, width, height, frame.pts, frame.end ? ", end of stream" : "", image); + } else if (Tracing()) { + LOG_INFO(Service, "{} frame {} (pts {} us) in image {}, sync value {}", + GetServiceName(), m_frames_out, frame.pts, image, m_values[image]); + } + if ((av1 || Tracing()) && (m_frames_out % SummaryFrames) == 0) { + LogState("at"); } continue; } Input input = std::move(m_inputs.front()); m_inputs.pop_front(); if (!m_stream_ended) { - m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed); + m_ready->room.store(m_inputs.size() < InputLimit(), std::memory_order_relaxed); } std::string codec = m_codec; if (codec.empty() && Tegra::Host1x::IsH264Stream(input.data)) { @@ -2789,14 +3240,23 @@ class INx2VideoSession final : public ServiceFramework { decoder = std::make_unique(codec); } pictures.clear(); + const auto decode_start = std::chrono::steady_clock::now(); decoder->Decode(input.data, input.pts, pictures); if (input.end) { decoder->Drain(pictures); } + const auto decode_time = std::chrono::steady_clock::now() - decode_start; lk.lock(); if (m_flush) { continue; } + if (!input.end) { + m_decode_time += decode_time; + ++m_timed_decodes; + if (pictures.empty()) { + ++m_no_picture; + } + } for (auto& picture : pictures) { const u64 pts = picture.pts; m_decoded.push_back({std::move(picture), pts, false}); @@ -2828,7 +3288,7 @@ class INx2VideoSession final : public ServiceFramework { // Writes a decoded frame into an output image; returns the size the frame reports. std::pair WriteImage(Tegra::Host1x::Nx2VideoDecoder* decoder, u32 image_id, - const Decoded& frame, std::vector& bytes) { + const Decoded& frame, std::vector& bytes, bool av1) { u64 address{}; Nx2DecoderImageLayout layout{}; bool block_linear{}; @@ -2839,7 +3299,9 @@ class INx2VideoSession final : public ServiceFramework { return {m_width, m_height}; } layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); - block_linear = image->second.gpu_read && + // AV1 (Fortune's Weave): every RGBA image is read on the GPU, as fefw writes them, + // also the first frames, which come before the client's 428/426 for the image. + block_linear = (image->second.gpu_read || av1) && layout.format != Tegra::Host1x::Nx2PictureFormat::Nv12; const auto memory = m_objects->memories.find(image->second.memory); if (memory != m_objects->memories.end() && memory->second.address != 0) { @@ -2861,8 +3323,7 @@ class INx2VideoSession final : public ServiceFramework { decoder->Render(*frame.picture, Tegra::Host1x::Nx2PictureFormat::Rgba8, width, height, layout.pitch, 0, bytes); m_tiled.resize(layout.size); - Tegra::Texture::SwizzleTexture(m_tiled, bytes, 4, static_cast(layout.pitch / 4), - layout.height, 1, 4, 0, 1, true); + SwizzleBlockLinear(m_tiled, bytes, layout.pitch, layout.height); memory.WriteBlock(address, m_tiled.data(), m_tiled.size()); if (const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address)) { @@ -2885,6 +3346,45 @@ class INx2VideoSession final : public ServiceFramework { return {width, height}; } + // Swizzles 4-byte pixel rows `pitch` bytes apart into 16-GOB (128-row) blocks. Each row of + // blocks is contiguous in the tiled image, so bands of whole block rows are swizzled on threads + // of their own, with the same result (fefw dd31b32b74: a 1440p frame took several ms here). + static void SwizzleBlockLinear(std::span tiled, std::span linear, u64 pitch, + u32 height) { + constexpr u32 BlockHeightLog2 = 4; + constexpr u32 RowsPerBlock = 8U << BlockHeightLog2; + constexpr u32 Bands = 4; + const u32 block_rows = Common::DivCeil(height, RowsPerBlock); + const u64 block_row_bytes = pitch * RowsPerBlock; + const u32 width = static_cast(pitch / 4); + const auto swizzle = [&](u32 first_block, u32 last_block) { + if (first_block >= last_block) { + return; + } + const u32 first = first_block * RowsPerBlock; + const u32 last = std::min(last_block * RowsPerBlock, height); + Tegra::Texture::SwizzleTexture(tiled.subspan(first_block * block_row_bytes), + linear.subspan(first * pitch, (last - first) * pitch), + 4, width, last - first, 1, BlockHeightLog2, 0, 1, true); + }; + if (pitch % Tegra::Texture::GOB_SIZE_X != 0 || + tiled.size() < block_rows * block_row_bytes || + linear.size() < u64{height} * pitch) { + Tegra::Texture::SwizzleTexture(tiled, linear, 4, width, height, 1, BlockHeightLog2, 0, + 1, true); + return; + } + const u32 per_band = Common::DivCeil(block_rows, Bands); + { + std::array workers; + for (u32 i = 1; i < Bands; ++i) { + workers[i - 1] = std::jthread{swizzle, std::min(i * per_band, block_rows), + std::min((i + 1) * per_band, block_rows)}; + } + swizzle(0, std::min(per_band, block_rows)); + } + } + // EDEN_NX2_VDEC_DUMP=: save the luma of every 120th decoded frame, a quarter of its size, // as nx2-vdec-.pgm (to check what the decoder wrote, apart from how the game shows it). void DumpFrame(const std::vector& bytes, const Nx2DecoderImageLayout& layout, u32 width, @@ -2938,6 +3438,27 @@ class INx2VideoSession final : public ServiceFramework { std::map m_release_fences; std::deque m_outputs; ///< Finished, for 213 bool m_stream_ended{}; ///< End-of-stream frame handed out, nothing submitted since + std::map m_values; ///< Sync value each image last handed out completes at + std::optional m_writing; ///< Image the decode thread is writing + bool m_av1{}; ///< AV1 stream: fefw's flush and release rules apply + u64 m_flushes{}; ///< AV1 flushes so far + u64 m_idle_releases{}; + // Counters for LogState. + u64 m_submitted_bytes{}; + u64 m_busy_submits{}; + u64 m_no_picture{}; ///< Inputs that showed no frame (libgav1: a hidden frame) + u64 m_fetched{}; + u64 m_empty_polls{}; + u64 m_empty_streak{}; + u64 m_releases{}; + u64 m_last_pts_in{}; + u64 m_first_fetch_pts{}; + u64 m_last_fetch_pts{}; + std::optional m_first_fetch_time; + std::chrono::steady_clock::duration m_decode_time{}; + u64 m_timed_decodes{}; + std::chrono::steady_clock::duration m_write_time{}; + u64 m_timed_writes{}; bool m_flush{}; bool m_quit{}; std::thread m_worker; @@ -2979,6 +3500,10 @@ class INx2GraphicsSystemSession final : public ServiceFrameworkmutex}; + if (m_objects->retired.empty()) { + break; + } + } + DropRetiredMapping(*m_objects); + } + } for (auto* event : m_syncpt_events) { if (event != nullptr) { m_context.CloseEvent(event); @@ -3368,18 +3907,18 @@ class INx2GraphicsSystemSession final : public ServiceFramework>> unmaps; + std::vector> unmaps; { std::scoped_lock lk{m_objects->mutex}; for (auto& [image_id, image] : m_objects->images) { if (image.memory == id && image.mapped_here) { - unmaps.push_back(TakeDecoderMapping(image)); + unmaps.push_back(TakeDecoderMapping(image, id)); } } m_objects->memories.erase(id); } for (const auto& range : unmaps) { - DropDecoderMapping(range); + RetireDecoderMapping(range); } LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id); TipcReplyWords(ctx, {}); @@ -3539,7 +4078,12 @@ class INx2GraphicsSystemSession final : public ServiceFrameworkmutex}; + if (const auto image = m_objects->images.find(handle); image != m_objects->images.end()) { + image->second.map_generation = generation; + } } LOG_INFO(Service, "nx2:gsv {} decoder image {} at {:#x}+{:#x}{}", ctx.GetCommand(), handle, address, size, register_mapping ? ", mapped for the GPU" : ""); @@ -3571,7 +4115,7 @@ class INx2GraphicsSystemSession final : public ServiceFramework> unmap; + std::optional unmap; { std::scoped_lock lk{m_objects->mutex}; if (const auto image = m_objects->images.find(handle); @@ -3580,13 +4124,18 @@ class INx2GraphicsSystemSession final : public ServiceFrameworksecond); } } + if (unmap) { + LOG_INFO(Service, "nx2:gsv 427 decoder image {} unmapped", handle); + } LOG_DEBUG(Service, "nx2:gsv 427 image {} [{:#x}, {:#x}]", handle, Pair(w, 4), Pair(w, 6)); DropDecoderMapping(unmap); TipcReplyWords(ctx, {}); } // The range 426 mapped for an image, if it did; called with m_objects->mutex held. - std::optional> TakeDecoderMapping(Nx2NativeObjects::Image& image) { + // `freed`: the memory being destroyed (112), whose images can't keep the mapping. + std::optional TakeDecoderMapping( + Nx2NativeObjects::Image& image, std::optional freed = {}) { image.gpu_maps = 0; if (!std::exchange(image.mapped_here, false)) { return std::nullopt; @@ -3595,17 +4144,96 @@ class INx2GraphicsSystemSession final : public ServiceFrameworkmemories.end() || memory->second.address == 0) { return std::nullopt; } - return std::pair{memory->second.address + image.offset, image.size}; + const u64 address = memory->second.address + image.offset; + // An image of a pool made again at the same address maps through this mapping (its 426 + // found it in place): it keeps the mapping. + for (auto& [id, other] : m_objects->images) { + if (&other == &image || !other.decoder || other.gpu_maps == 0 || other.mapped_here || + other.size != image.size || other.memory == freed) { + continue; + } + const auto other_memory = m_objects->memories.find(other.memory); + if (other_memory != m_objects->memories.end() && + other_memory->second.address + other.offset == address) { + other.mapped_here = true; + other.map_generation = image.map_generation; + return std::nullopt; + } + } + return Nx2NativeObjects::RetiredMapping{address, image.size, image.map_generation}; } - static void DropDecoderMapping(const std::optional>& range) { - if (range) { - auto& registry = Nvidia::Devices::Nx2GpuMemoryRegistry::Get(); - registry.Unmap(range->first, range->second); - registry.ReleaseBacking(range->first, range->second); + // Drops a mapping 426 made. The GPU thread unmaps it in order, after the command lists + // already submitted, and the texture cache frees its images once the GPU is past them. A + // mapping made since over the range owns it now and is left alone. + static void DropDecoderMapping(const std::optional& range) { + if (range && !Nvidia::Devices::Nx2GpuMemoryRegistry::Get().UnregisterIfUnchanged( + range->address, range->size, range->generation)) { + LOG_INFO(Service, "nx2:gsv decoder image mapping {:#x}+{:#x} was replaced, kept", + range->address, range->size); } } + // A mapping 426 made for an image whose pool (912), memory (112) or image (430) went while the + // client still had it mapped: GPU work that reads it may still be queued or running (Fortune's + // Weave destroys its movie's pools right after its last releases, and Turnip crashed in + // VulkanWorker 0.7 s later when they went at once). fefw keeps such a pool until its mapping + // goes. Here it is dropped RetireDelay later, by then long done on the GPU, unless a pool + // made again at the address took it over meanwhile. + void RetireDecoderMapping(const std::optional& range) { + if (!range) { + return; + } + // With its registration's generation: one made since at the address (a pool made again, + // or a 211 mapping) is a newer one and stays when this one is dropped. + { + std::scoped_lock lk{m_objects->mutex}; + m_objects->retired.push_back(*range); + } + if (!m_retire_event) { + m_retire_event = Core::Timing::CreateEvent( + "nx2:gsv retire", + [objects = m_objects](s64, std::chrono::nanoseconds) + -> std::optional { + DropRetiredMapping(*objects); + return std::nullopt; + }); + } + // One event per mapping, in order: each drops the oldest. + system.CoreTiming().ScheduleEvent(RetireDelay, m_retire_event); + LOG_INFO(Service, "nx2:gsv decoder image mapping {:#x}+{:#x} goes in {} ms", range->address, + range->size, RetireDelay.count()); + } + + static void DropRetiredMapping(Nx2NativeObjects& objects) { + std::optional drop; + { + std::scoped_lock lk{objects.mutex}; + if (objects.retired.empty()) { + return; + } + const auto retired = objects.retired.front(); + objects.retired.pop_front(); + drop = retired; + // A pool made again at the address maps through it (its 426 found it in place). + for (auto& [id, image] : objects.images) { + if (!image.decoder || image.gpu_maps == 0 || image.mapped_here || + image.size != retired.size) { + continue; + } + const auto memory = objects.memories.find(image.memory); + if (memory != objects.memories.end() && + memory->second.address + image.offset == retired.address) { + image.mapped_here = true; + image.map_generation = retired.generation; + drop.reset(); + break; + } + } + } + DropDecoderMapping(drop); + } + // 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool): // result, the id of the first of `count` decoder output images; the client names image i of // the pool id + i. They share the pool's memory, one image size apart. @@ -3664,7 +4292,7 @@ class INx2GraphicsSystemSession final : public ServiceFramework> unmap; + std::optional unmap; { std::scoped_lock lk{m_objects->mutex}; if (const auto image = m_objects->images.find(id); image != m_objects->images.end()) { @@ -3672,7 +4300,41 @@ class INx2GraphicsSystemSession final : public ServiceFrameworkimages.erase(image); } } - DropDecoderMapping(unmap); + RetireDecoderMapping(unmap); + TipcReplyWords(ctx, {}); + } + + // 912(u32 first image id): destroys a decoder image pool (fefw's + // NativeVideoService::DestroyImagePool, fefw 0b1ba0db76): the image and the images that follow + // it in the same pool memory, so the decode thread no longer writes into memory the game + // reuses. A title that frees the pool's images one by one finds the later ones gone. + // A GPU mapping 426 made for them goes RetireDelay later (RetireDecoderMapping). + void DestroyDecoderImagePool(HLERequestContext& ctx) { + const u32 first = ctx.CommandBuffer()[2]; + std::vector> unmaps; + u32 count{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto pool = m_objects->images.find(first); + if (pool != m_objects->images.end() && pool->second.decoder) { + const u32 memory = pool->second.memory; + const u64 size = pool->second.size; + u64 offset = pool->second.offset; + for (auto image = pool; image != m_objects->images.end() && + image->first == first + count && image->second.decoder && + image->second.memory == memory && + image->second.offset == offset;) { + unmaps.push_back(TakeDecoderMapping(image->second)); + image = m_objects->images.erase(image); + ++count; + offset += size; + } + } + } + for (const auto& range : unmaps) { + RetireDecoderMapping(range); + } + LOG_INFO(Service, "nx2:gsv 912 decoder images {}..+{} destroyed", first, count); TipcReplyWords(ctx, {}); } @@ -3721,6 +4383,9 @@ class INx2GraphicsSystemSession final : public ServiceFramework m_retire_event; std::shared_ptr m_objects{std::make_shared()}; KernelHelpers::ServiceContext m_context; diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 491535a62d..d00f49e14d 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -286,7 +286,27 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys : AppLoader(std::move(file_)) , override_update(override_update_) { - const auto file_dir = file->GetContainingDirectory(); + ReadDumpMetadata(file->GetContainingDirectory()); +} + +AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( + FileSys::VirtualDir directory, bool override_update_, bool read_dump_metadata) + : AppLoader(directory->GetFile("main")) + , dir(std::move(directory)) + , override_update(override_update_) +{ + if (read_dump_metadata) { + ReadDumpMetadata(dir); + } else { + romfs_exefs_dir = dir; + } +} + +// Title ID, icon, control data and RomFS version of a dump whose ExeFS is file_dir. +void AppLoader_DeconstructedRomDirectory::ReadDumpMetadata(const FileSys::VirtualDir& file_dir) { + if (file_dir == nullptr) { + return; + } const auto sidecar_dirs = FindSidecarDirs(file_dir); // Title ID @@ -353,18 +373,14 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys romfs_exefs_dir = file_dir; if (const auto title_dir = file_dir->GetParentDirectory()) { romfs_version = RomFSProductVersion(FindLooseRomFSDirectory(title_dir)); + } else if (LooseRomFS() != nullptr) { + // A packed dump (.nx2) has no parent directory: its RomFS is the game.romfs image in it, + // extracted only if the version is asked for and the control data has none + // (ReadDisplayVersion): most loaders are made for the game list and never need it. + romfs_version_pending = true; } } -AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( - FileSys::VirtualDir directory, bool override_update_) - : AppLoader(directory->GetFile("main")) - , dir(std::move(directory)) - , override_update(override_update_) -{ - romfs_exefs_dir = dir; -} - // Loaders made only for metadata (the game list, title lookups) never need the RomFS, and several // of them were alive with a whole copy each. const FileSys::VirtualFile& AppLoader_DeconstructedRomDirectory::LooseRomFS() { @@ -585,6 +601,10 @@ ResultStatus AppLoader_DeconstructedRomDirectory::ReadDisplayVersion(std::string version = control->GetVersionString(); return ResultStatus::Success; } + if (romfs_version_pending) { + romfs_version_pending = false; + romfs_version = RomFSProductVersion(FileSys::ExtractRomFS(romfs)); + } if (!romfs_version.empty()) { version = romfs_version; return ResultStatus::Success; diff --git a/src/core/loader/deconstructed_rom_directory.h b/src/core/loader/deconstructed_rom_directory.h index db0120f4ad..a4cc5f1265 100644 --- a/src/core/loader/deconstructed_rom_directory.h +++ b/src/core/loader/deconstructed_rom_directory.h @@ -28,8 +28,11 @@ namespace Loader { class AppLoader_DeconstructedRomDirectory final : public AppLoader { public: explicit AppLoader_DeconstructedRomDirectory(FileSys::VirtualFile main_file, bool override_update_ = false); - // Overload to accept exefs directory. Must contain 'main' and 'main.npdm' - explicit AppLoader_DeconstructedRomDirectory(FileSys::VirtualDir directory, bool override_update_ = false); + // Overload to accept exefs directory. Must contain 'main' and 'main.npdm'. read_dump_metadata + // also takes the title, icon and version from the directory, as for a loose dump: set for a + // packed dump (.nx2), whose files have no containing directory to find them from. + explicit AppLoader_DeconstructedRomDirectory(FileSys::VirtualDir directory, bool override_update_ = false, + bool read_dump_metadata = false); /** * Identifies whether or not the given file is a deconstructed ROM directory. @@ -61,6 +64,8 @@ class AppLoader_DeconstructedRomDirectory final : public AppLoader { /// The loose RomFS next to the ExeFS, found (and built) on first use. const FileSys::VirtualFile& LooseRomFS(); + void ReadDumpMetadata(const FileSys::VirtualDir& file_dir); + FileSys::ProgramMetadata metadata; FileSys::VirtualFile romfs; FileSys::VirtualDir romfs_exefs_dir; ///< Where LooseRomFS looks @@ -71,6 +76,7 @@ class AppLoader_DeconstructedRomDirectory final : public AppLoader { std::string name; std::optional control; ///< control.nacp next to the dump, if it has one std::string romfs_version; ///< "1.8.0" from the RomFS's Product.180 tables, if any + bool romfs_version_pending{}; ///< A packed dump's RomFS version, read when asked for u64 title_id{}; bool override_update; diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 839bb4d80b..1486aa160d 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -115,6 +115,11 @@ bool HasApplicationProgramContent(const std::shared_ptr& nsp) { return false; } + // An extracted-type NSP (a packed dump, .nx2) holds its ExeFS directly, with no NCAs. + if (nsp->IsExtractedType()) { + return nsp->GetProgramStatus() == ResultStatus::Success; + } + const auto& ncas = nsp->GetNCAs(); return std::any_of(ncas.cbegin(), ncas.cend(), [](const auto& title_entry) { const auto& nca_map = title_entry.second; @@ -171,15 +176,23 @@ bool IsBootableGameContainer(FileSys::VirtualFile file, FileType type, u64 progr if (nup.GetStatus() != ResultStatus::Success) { return false; } + const auto has_usable_exefs = [&nup](std::size_t index) { + const auto exefs = nup.GetExeFSDir(index); + return exefs != nullptr && exefs->GetFile("main") != nullptr && + exefs->GetFile("main.npdm") != nullptr; + }; // A base package boots normally; a standalone update (no base) carries // a complete patched tree and boots on its own (SPEC section 4.1.2). if (const auto base = nup.FindBasePackage(); base.has_value() && nup.GetExeFSDir(*base) != nullptr) { - return true; + const auto update = nup.FindBestUpdatePackage(); + return has_usable_exefs(update && nup.GetExeFSDir(*update) != nullptr + ? *update + : *base); } if (const auto update = nup.FindBestUpdatePackage(); update.has_value() && nup.GetExeFSDir(*update) != nullptr) { - return true; + return has_usable_exefs(*update); } return false; } @@ -215,6 +228,8 @@ FileType GuessFromFilename(const std::string& name) { return FileType::NUP; else if (extension == "nsz") return FileType::NSZ; + else if (extension == "nx2") // a packed Switch 2 dump: an extracted-type NSP + return FileType::NSP; else if (extension == "kip") return FileType::KIP; return FileType::Unknown; diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp index 4b9c89d38e..b8088370ec 100644 --- a/src/core/loader/nso.cpp +++ b/src/core/loader/nso.cpp @@ -5,9 +5,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include +#include #include +#include #include +#include #include #include "common/common_funcs.h" @@ -33,6 +37,119 @@ namespace Loader { namespace { + +// Mario Kart World: reduced detail (Settings::values.mkw_lod_distance, mkw_shadow_cascades). +// Pattern-matched text and rodata patches to the module before it is mapped, each applied only +// when its pattern occurs exactly once. The settings select two shadow cascades and the LOD +// distance ratio key renamed (the game then keeps the constructor's 1.0 instead of the loaded +// 2.0). EDEN_NX2_MKW_DETAIL, when set, replaces the set: a comma list of "cascades=1|2|3" (3: no +// patch), "lod=1.0|0.75|0.5" (below 1.0 the constructor default is patched too), "elink" +// (vehicle count divisor key), "emit" (single emit key), "traffic" (mob car count and distance +// keys), e.g. "cascades=2,lod=1.0,elink,emit,traffic". +struct MkwDetailOptions { + u32 cascades{3}; + bool lod_key{}; + u32 lod_default{0}; ///< 0: keep the constructor's 1.0; else the mov w8 immediate to use + bool elink{}; + bool emit{}; + bool traffic{}; +}; + +MkwDetailOptions ParseMkwDetail(std::string_view text) { + MkwDetailOptions options{.cascades = 3, .lod_key = false}; + while (!text.empty()) { + const size_t comma = text.find(','); + const std::string_view token = text.substr(0, comma); + text = comma == std::string_view::npos ? std::string_view{} : text.substr(comma + 1); + if (token.empty()) { + continue; + } + if (token == "elink") { + options.elink = true; + } else if (token == "emit") { + options.emit = true; + } else if (token == "traffic") { + options.traffic = true; + } else if (token.starts_with("cascades=")) { + const std::string_view value = token.substr(9); + options.cascades = value == "1" ? 1U : value == "2" ? 2U : 3U; + } else if (token.starts_with("lod=")) { + const std::string_view value = token.substr(4); + options.lod_key = true; + if (value == "1.0" || value == "1") { + options.lod_default = 0; + } else if (value == "0.75") { + options.lod_default = 0x52A7E808; // mov w8, #0x3f40'0000 (0.75f) + } else if (value == "0.5") { + options.lod_default = 0x52A7E008; // mov w8, #0x3f00'0000 (0.5f) + } else { + LOG_WARNING(Loader, "EDEN_NX2_MKW_DETAIL: unknown LOD ratio '{}' (1.0, 0.75, 0.5)", + value); + options.lod_key = false; + } + } else { + LOG_WARNING(Loader, "EDEN_NX2_MKW_DETAIL: unknown item '{}'", token); + } + } + return options; +} + +/// The offset of `needle` in `hay` when it occurs exactly once; `matches` gets the count. +std::optional FindOnce(std::span hay, std::span needle, + size_t& matches) { + matches = 0; + std::optional found; + auto it = hay.begin(); + while (true) { + it = std::search(it, hay.end(), needle.begin(), needle.end()); + if (it == hay.end()) { + break; + } + if (++matches == 1) { + found = static_cast(it - hay.begin()); + } + ++it; + } + return matches == 1 ? found : std::nullopt; +} + +/// Replaces the first word of the one occurrence of `sequence` in the text segment. +void PatchMkwWords(std::span text, u64 text_offset, std::span sequence, + u32 replacement, std::string_view what) { + std::array needle{}; + const size_t bytes = sequence.size() * sizeof(u32); + ASSERT(bytes <= needle.size()); + std::memcpy(needle.data(), sequence.data(), bytes); + size_t matches = 0; + const auto at = FindOnce(text, std::span{needle.data(), bytes}, matches); + if (!at) { + LOG_WARNING(Loader, "Mario Kart World: {} patch skipped ({} matches)", what, matches); + return; + } + std::memcpy(text.data() + *at, &replacement, sizeof(replacement)); + LOG_INFO(Loader, "Mario Kart World: {} (main+{:#x})", what, text_offset + *at); +} + +/// Overwrites the one occurrence of the NUL-framed `key` in rodata with `replacement`, of the +/// same length (NUL padding included). +void PatchMkwKey(std::span rodata, u64 rodata_offset, std::string_view key, + std::string_view replacement, std::string_view what) { + ASSERT(key.size() == replacement.size()); + std::array needle{}; + ASSERT(key.size() + 2 <= needle.size()); + needle[0] = 0; + std::memcpy(needle.data() + 1, key.data(), key.size()); + needle[key.size() + 1] = 0; + size_t matches = 0; + const auto at = FindOnce(rodata, std::span{needle.data(), key.size() + 2}, matches); + if (!at) { + LOG_WARNING(Loader, "Mario Kart World: {} patch skipped ({} matches)", what, matches); + return; + } + std::memcpy(rodata.data() + *at + 1, replacement.data(), replacement.size()); + LOG_INFO(Loader, "Mario Kart World: {} (main+{:#x})", what, rodata_offset + *at + 1); +} + struct MODHeader { u32_le magic; u32_le dynamic_offset; @@ -177,15 +294,19 @@ std::optional AppLoader_NSO::LoadModule(Kernel::KProcess& process, Core:: std::copy(pi_header.begin() + sizeof(NSOHeader), pi_header.end(), patchable_section.data()); } - // Mario Kart World selects a 1- or 2-vblank frame mode in its main loop. Force its native - // 30 FPS mode, or let it choose dynamically using the whole frame time (CPU + GPU wait). - // Match the instruction sequence instead of relying on a title-version-specific address. - const bool mkw_dynamic = Settings::values.mkw_dynamic_fps.GetValue(); + // Mario Kart World selects a 1- or 2-vblank frame mode in its main loop (mode 0: dynamic, from + // the CPU frame time at [x19, #0x128]; 1: one vblank; 2: two vblanks). Force its native 30 FPS + // mode, its 60 FPS mode, or let it choose dynamically using the whole frame time (CPU + GPU + // wait), in that order of precedence. Match the instruction sequence instead of relying on a + // title-version-specific address. + const bool mkw_30 = Settings::values.mkw_30fps.GetValue(); + const bool mkw_60 = !mkw_30 && Settings::values.mkw_60fps.GetValue(); + const bool mkw_dynamic = !mkw_30 && !mkw_60 && Settings::values.mkw_dynamic_fps.GetValue(); bool mkw_dynamic_fps_patch_applied = false; - if ((mkw_dynamic || Settings::values.mkw_30fps.GetValue()) && name == "main" && + if ((mkw_30 || mkw_60 || mkw_dynamic) && name == "main" && process.GetProgramId() == Settings::MarioKartWorldProgramId) { constexpr u32 ModeLoadInsn = 0xB9421268; // ldr w8, [x19, #0x210] - const u32 mode = mkw_dynamic ? 0U : 2U; + const u32 mode = mkw_dynamic ? 0U : (mkw_60 ? 1U : 2U); const u32 mode_instruction = 0x52800008U | (mode << 5); // mov w8, #mode const auto& text = nso_header.segments[0]; u8* const code = codeset.memory.data() + module_start + text.location; @@ -233,7 +354,7 @@ std::optional AppLoader_NSO::LoadModule(Kernel::KProcess& process, Core:: mkw_dynamic_fps_patch_applied = true; } LOG_INFO(Loader, "Mario Kart World: {} FPS mode enabled (main+{:#x})", - mkw_dynamic ? "dynamic" : "30", + mkw_dynamic ? "dynamic" : (mkw_60 ? "60" : "30"), text.location + mode_site * sizeof(u32)); } else { LOG_WARNING(Loader, @@ -242,6 +363,62 @@ std::optional AppLoader_NSO::LoadModule(Kernel::KProcess& process, Core:: } } + // Mario Kart World: reduced detail (see MkwDetailOptions). The environment list, when set, + // replaces the settings (with both off too, as an opt-in for testing). + if (name == "main" && process.GetProgramId() == Settings::MarioKartWorldProgramId) { + const char* const env = std::getenv("EDEN_NX2_MKW_DETAIL"); + const bool lod_setting = Settings::values.mkw_lod_distance.GetValue(); + const bool cascades_setting = Settings::values.mkw_shadow_cascades.GetValue(); + if (lod_setting || cascades_setting || env != nullptr) { + const MkwDetailOptions options = + env != nullptr ? ParseMkwDetail(env) + : MkwDetailOptions{.cascades = cascades_setting ? 2U : 3U, + .lod_key = lod_setting}; + const auto& text = nso_header.segments[0]; + const auto& rodata = nso_header.segments[1]; + const std::span text_bytes{codeset.memory.data() + module_start + text.location, + text.size}; + const std::span rodata_bytes{ + codeset.memory.data() + module_start + rodata.location, rodata.size}; + if (options.cascades < 3) { + // DepthShadow init: ldr w8, [x21, #0x1a48] (the cascade count) -> mov w8, #N. + static constexpr std::array cascades_sequence{ + 0xB95A4AA8, 0x6B09011F, 0x1A89B108, 0xF941B6C9, + 0x7100051F, 0x1A9FC508, 0xB9022AC8}; + PatchMkwWords(text_bytes, text.location, cascades_sequence, + 0x52800008U | (options.cascades << 5), + options.cascades == 1 ? "1 shadow cascade" : "2 shadow cascades"); + } + if (options.lod_key) { + PatchMkwKey(rodata_bytes, rodata.location, "LODDistanceRatio", "LODDistanceQatio", + "LOD distance ratio key renamed (constructor default kept)"); + } + if (options.lod_default != 0) { + // The constructor: mov w8, #1.0f stored to the ratio field. + static constexpr std::array lod_sequence{ + 0x52A7F008, 0xF9001A7F, 0xF803C27F, 0xF804427F, + 0xF807C27F, 0xF808427F, 0x79011A7F, 0xB9003A68}; + PatchMkwWords(text_bytes, text.location, lod_sequence, options.lod_default, + options.lod_default == 0x52A7E808 ? "LOD distance ratio default 0.75" + : "LOD distance ratio default 0.5"); + } + if (options.elink) { + PatchMkwKey(rodata_bytes, rodata.location, "VehicleNumNoDiv", + std::string_view{"VehicleNumDiv2\0", 15}, "vehicle count divided by 2"); + } + if (options.emit) { + PatchMkwKey(rodata_bytes, rodata.location, "NoEmitSingle", + std::string_view{"NoEmitDiv2\0\0", 12}, "single emits divided by 2"); + } + if (options.traffic) { + PatchMkwKey(rodata_bytes, rodata.location, "MobCarDispNum", "MobCarQispNum", + "mob car count key renamed"); + PatchMkwKey(rodata_bytes, rodata.location, "MobCarDispDist", "MobCarQispDist", + "mob car distance key renamed"); + } + } + } + #ifdef HAS_NCE // If we are computing the process code layout and using nce backend, patch. const auto& code = codeset.CodeSegment(); diff --git a/src/core/loader/nsp.cpp b/src/core/loader/nsp.cpp index 8b2c79394d..66c77de355 100644 --- a/src/core/loader/nsp.cpp +++ b/src/core/loader/nsp.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include +#include +#include #include #include "common/common_types.h" @@ -16,6 +19,7 @@ #include "core/file_sys/registered_cache.h" #include "core/file_sys/romfs_factory.h" #include "core/file_sys/submission_package.h" +#include "core/file_sys/vfs/vfs_vector.h" #include "core/hle/kernel/k_process.h" #include "core/hle/service/filesystem/filesystem.h" #include "core/loader/deconstructed_rom_directory.h" @@ -24,6 +28,38 @@ namespace Loader { +namespace { + +std::string PackageStem(const FileSys::VirtualFile& package) { + const auto& name = package->GetName(); + return name.substr(0, name.rfind('.')); +} + +// A packed dump (.nx2) has control data only if its dump had sidecars. Cover art and a NACP can +// also sit next to the package: ".png" (.jpg, .jpeg, .bmp), ".nacp". +FileSys::VirtualFile FindPackageSidecar(const FileSys::VirtualFile& package, + std::initializer_list extensions) { + const auto dir = package != nullptr ? package->GetContainingDirectory() : nullptr; + if (dir == nullptr) { + return nullptr; + } + const auto stem = PackageStem(package); + // One listing of the directory for all the extensions (a content directory on Android is + // listed by every lookup). + const auto files = dir->GetFiles(); + for (const auto extension : extensions) { + const std::string wanted = stem + '.' + std::string{extension}; + for (const auto& file : files) { + if (file != nullptr && file->GetName() == wanted) { + return file; + } + } + } + return nullptr; +} + +} // Anonymous namespace + AppLoader_NSP::AppLoader_NSP(FileSys::VirtualFile file_, const Service::FileSystem::FileSystemController& fsc, const FileSys::ContentProvider& content_provider, u64 program_id, @@ -35,7 +71,27 @@ AppLoader_NSP::AppLoader_NSP(FileSys::VirtualFile file_, } if (nsp->IsExtractedType()) { - secondary_loader = std::make_unique(nsp->GetExeFS(), false); + // A packed dump (.nx2): title, icon and version come from its sidecars and RomFS + secondary_loader = + std::make_unique(nsp->GetExeFS(), false, true); + + // Else from sidecars next to the package, looked up now: the Read* functions are also + // called from guest service calls, which on Android must not reach the content URI + // lookups (JNI) a package outside the app's storage needs. + std::vector icon; + if (secondary_loader->ReadIcon(icon) != ResultStatus::Success) { + icon_file = FindPackageSidecar(file, {"png", "jpg", "jpeg", "bmp"}); + if (icon_file != nullptr) { + icon_file = std::make_shared(icon_file->ReadAllBytes(), + icon_file->GetName()); + } + } + FileSys::NACP control; + if (secondary_loader->ReadControlData(control) != ResultStatus::Success) { + if (const auto sidecar = FindPackageSidecar(file, {"nacp"})) { + nacp_file = std::make_unique(sidecar); + } + } } else { const auto control_nca = nsp->GetNCA(FileSys::GetBaseTitleID(nsp->GetProgramTitleID()), FileSys::ContentRecordType::Control); @@ -228,6 +284,9 @@ ResultStatus AppLoader_NSP::ReadProgramIds(std::vector& out_program_ids) { ResultStatus AppLoader_NSP::ReadIcon(std::vector& buffer) { if (icon_file == nullptr) { + if (nsp->IsExtractedType() && secondary_loader != nullptr) { + return secondary_loader->ReadIcon(buffer); + } return ResultStatus::ErrorNoControl; } @@ -237,6 +296,15 @@ ResultStatus AppLoader_NSP::ReadIcon(std::vector& buffer) { ResultStatus AppLoader_NSP::ReadTitle(std::string& title) { if (nacp_file == nullptr) { + if (nsp->IsExtractedType()) { + if (secondary_loader != nullptr && + secondary_loader->ReadTitle(title) == ResultStatus::Success && !title.empty()) { + return ResultStatus::Success; + } + // The package is named after its dump folder when packed + title = PackageStem(file); + return ResultStatus::Success; + } return ResultStatus::ErrorNoControl; } @@ -246,6 +314,9 @@ ResultStatus AppLoader_NSP::ReadTitle(std::string& title) { ResultStatus AppLoader_NSP::ReadControlData(FileSys::NACP& nacp) { if (nacp_file == nullptr) { + if (nsp->IsExtractedType() && secondary_loader != nullptr) { + return secondary_loader->ReadControlData(nacp); + } return ResultStatus::ErrorNoControl; } @@ -253,6 +324,13 @@ ResultStatus AppLoader_NSP::ReadControlData(FileSys::NACP& nacp) { return ResultStatus::Success; } +ResultStatus AppLoader_NSP::ReadDisplayVersion(std::string& version) { + if (nsp->IsExtractedType() && secondary_loader != nullptr) { + return secondary_loader->ReadDisplayVersion(version); + } + return AppLoader::ReadDisplayVersion(version); +} + ResultStatus AppLoader_NSP::ReadManualRomFS(FileSys::VirtualFile& out_file) { const auto nca = nsp->GetNCA(nsp->GetProgramTitleID(), FileSys::ContentRecordType::HtmlDocument); diff --git a/src/core/loader/nsp.h b/src/core/loader/nsp.h index 7ce436c67e..d4610bb8ad 100644 --- a/src/core/loader/nsp.h +++ b/src/core/loader/nsp.h @@ -54,6 +54,7 @@ class AppLoader_NSP final : public AppLoader { ResultStatus ReadIcon(std::vector& buffer) override; ResultStatus ReadTitle(std::string& title) override; ResultStatus ReadControlData(FileSys::NACP& nacp) override; + ResultStatus ReadDisplayVersion(std::string& version) override; ResultStatus ReadManualRomFS(FileSys::VirtualFile& out_file) override; ResultStatus ReadBanner(std::vector& buffer) override; diff --git a/src/hid_core/resources/npad/npad.cpp b/src/hid_core/resources/npad/npad.cpp index 2b04e2c670..19f4ed09f0 100644 --- a/src/hid_core/resources/npad/npad.cpp +++ b/src/hid_core/resources/npad/npad.cpp @@ -150,6 +150,18 @@ void NPad::ControllerUpdate(Kernel::KernelCore& kernel, Core::HID::ControllerTri return; } + // Only connection and battery changes are handled here. Motion and button updates arrive many + // times a second, a phone's gyro from the first frame: before the applet resource is set, + // which this callback (registered by the constructor) used to dereference. + if (type != Core::HID::ControllerTriggerType::Connected && + type != Core::HID::ControllerTriggerType::Disconnected && + type != Core::HID::ControllerTriggerType::Battery) { + return; + } + if (applet_resource_holder.applet_resource == nullptr) { + return; + } + for (std::size_t aruid_index = 0; aruid_index < AruidIndexMax; aruid_index++) { if (controller_idx >= controller_data[aruid_index].size()) { return; diff --git a/src/qt_common/config/shared_translation.cpp b/src/qt_common/config/shared_translation.cpp index c59abede5f..742aba5907 100644 --- a/src/qt_common/config/shared_translation.cpp +++ b/src/qt_common/config/shared_translation.cpp @@ -283,16 +283,36 @@ std::unique_ptr InitializeTranslations(QObject* parent) { INSERT(Settings, mkw_30fps, tr("Mario Kart World: 30 FPS mode"), tr("Forces the game to use its native 30 FPS mode at normal game speed. " "Takes effect after restarting the game.")); + INSERT(Settings, mkw_60fps, tr("Mario Kart World: 60 FPS mode"), + tr("Always renders at 60 fps. Slower hardware runs the game in slow motion. " + "Takes effect after restarting the game; the 30 FPS mode takes precedence.")); INSERT(Settings, mkw_dynamic_fps, tr("Mario Kart World: dynamic FPS (experimental)"), tr("Lets the game adjust between 30 and 60 FPS using whole-frame timing to maintain " - "normal game speed. Takes effect after restarting the game and takes precedence over " - "the 30 FPS mode.")); + "normal game speed. Takes effect after restarting the game; the 30 and 60 FPS modes " + "take precedence.")); + INSERT(Settings, mkw_lod_distance, tr("Mario Kart World: shorter LOD distance"), + tr("Objects switch to lower detail at half the distance (the game's default ratio " + "instead of 2.0). Fewer, lighter draws. Takes effect after restarting the game.")); + INSERT(Settings, mkw_shadow_cascades, tr("Mario Kart World: fewer shadow cascades"), + tr("Drops the farthest dynamic shadow cascade (160-250 m). Takes effect after " + "restarting the game.")); INSERT(Settings, barrier_feedback_loops, tr("Barrier feedback loops"), tr("Improves rendering of transparency effects in specific games.")); INSERT(Settings, enable_buffer_history, tr("Enable buffer history"), tr("Enables access to previous buffer states.\nThis option may improve rendering " "quality and performance consistency in some games.")); INSERT(Settings, fix_bloom_effects, tr("Fix bloom effects"), tr("Removes bloom in Burnout.")); + INSERT(Settings, nx2_command_lists, tr("Switch 2 command list fixes"), + tr("Upstream's fixes for Switch 2 command lists the game rewrites after submitting. Off " + "on phones: on an Odin 3 they cost ~45 ms of GPU-thread waiting per second in Mario " + "Kart World and caused flicker. Takes effect when a game starts.")); + INSERT(Settings, reduce_texture_resolution, tr("Reduce texture resolution"), + tr("Large compressed textures keep their mip levels from the second one on: a quarter " + "of the video memory, slightly blurrier up close.")); + INSERT(Settings, release_gpu_only_memory, tr("Release GPU-only memory (experimental)"), + tr("Switch 2 textures in GPU-only memory give back their guest memory once uploaded, " + "the host image holding the data. Saves RAM; may break titles that read that " + "memory directly.")); // Renderer (Extensions) INSERT(Settings, dyna_state, tr("Extended Dynamic State"), diff --git a/src/qt_common/qt_common.cpp b/src/qt_common/qt_common.cpp index 7ad7a477d2..72c80e80dd 100644 --- a/src/qt_common/qt_common.cpp +++ b/src/qt_common/qt_common.cpp @@ -59,7 +59,7 @@ const QStringList supported_file_extensions = {QStringLiteral("nro"), QStringLit QStringLiteral("nca"), QStringLiteral("xci"), QStringLiteral("xcz"), QStringLiteral("nsp"), QStringLiteral("nsz"), QStringLiteral("nup"), - QStringLiteral("kip")}; + QStringLiteral("kip"), QStringLiteral("nx2")}; Core::Frontend::WindowSystemType GetWindowSystemType() { // Determine WSI type based on Qt platform. diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index e4df9c045d..aa03648c45 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -117,6 +117,7 @@ add_library(shader_recompiler STATIC frontend/maxwell/translate/impl/impl.h frontend/maxwell/translate/impl/integer_add.cpp frontend/maxwell/translate/impl/integer_add_three_input.cpp + frontend/maxwell/translate/impl/integer_byte_permute.cpp frontend/maxwell/translate/impl/integer_compare.cpp frontend/maxwell/translate/impl/integer_compare_and_set.cpp frontend/maxwell/translate/impl/integer_floating_point_conversion.cpp diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp index d1c08c1131..f833443d5a 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp @@ -18,9 +18,7 @@ Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) { if (index_offset > 0) { index = ctx.OpIAdd(ctx.U32[1], index, ctx.Const(index_offset)); } - return ctx.uses_explicit_workgroup_layout - ? ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index) - : ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index); + return ctx.SharedWordPointer(index); } Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size) { diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_barriers.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_barriers.cpp index 0b9a3664ff..19d3365e7a 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_barriers.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_barriers.cpp @@ -18,8 +18,11 @@ void MemoryBarrier(EmitContext& ctx, spv::Scope scope) { void EmitBarrier(EmitContext& ctx) { const auto execution{spv::Scope::Workgroup}; const auto memory{spv::Scope::Workgroup}; - const auto memory_semantics{spv::MemorySemanticsMask::AcquireRelease | - spv::MemorySemanticsMask::WorkgroupMemory}; + // Emulated shared memory is a storage buffer: its writes need UniformMemory semantics. + const auto memory_semantics{ + spv::MemorySemanticsMask::AcquireRelease | spv::MemorySemanticsMask::WorkgroupMemory | + (ctx.emulates_shared_memory ? spv::MemorySemanticsMask::UniformMemory + : spv::MemorySemanticsMask::MaskNone)}; ctx.OpControlBarrier(ctx.Const(static_cast(execution)), ctx.Const(static_cast(memory)), ctx.Const(static_cast(memory_semantics))); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp index dc398aa296..dcee9529e0 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp @@ -18,7 +18,7 @@ Id Pointer(EmitContext& ctx, Id pointer_type, Id array, Id offset, u32 shift) { Id Word(EmitContext& ctx, Id offset) { const Id shift_id{ctx.Const(2U)}; const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)}; - const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)}; + const Id pointer{ctx.SharedWordPointer(index)}; return ctx.OpLoad(ctx.U32[1], pointer); } @@ -89,8 +89,8 @@ Id EmitLoadSharedU64(EmitContext& ctx, Id offset) { const Id shift_id{ctx.Const(2U)}; const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)}; const Id next_index{ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(1U))}; - const Id lhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, base_index)}; - const Id rhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, next_index)}; + const Id lhs_pointer{ctx.SharedWordPointer(base_index)}; + const Id rhs_pointer{ctx.SharedWordPointer(next_index)}; return ctx.OpCompositeConstruct(ctx.U32[2], ctx.OpLoad(ctx.U32[1], lhs_pointer), ctx.OpLoad(ctx.U32[1], rhs_pointer)); } @@ -106,7 +106,7 @@ Id EmitLoadSharedU128(EmitContext& ctx, Id offset) { std::array values{}; for (u32 i = 0; i < 4; ++i) { const Id index{i == 0 ? base_index : ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(i))}; - const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)}; + const Id pointer{ctx.SharedWordPointer(index)}; values[i] = ctx.OpLoad(ctx.U32[1], pointer); } return ctx.OpCompositeConstruct(ctx.U32[4], values); @@ -138,7 +138,7 @@ void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) { } else { const Id shift{ctx.Const(2U)}; const Id word_offset{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)}; - pointer = ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, word_offset); + pointer = ctx.SharedWordPointer(word_offset); } ctx.OpStore(pointer, value); } @@ -152,8 +152,8 @@ void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) { const Id shift{ctx.Const(2U)}; const Id word_offset{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)}; const Id next_offset{ctx.OpIAdd(ctx.U32[1], word_offset, ctx.Const(1U))}; - const Id lhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, word_offset)}; - const Id rhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, next_offset)}; + const Id lhs_pointer{ctx.SharedWordPointer(word_offset)}; + const Id rhs_pointer{ctx.SharedWordPointer(next_offset)}; ctx.OpStore(lhs_pointer, ctx.OpCompositeExtract(ctx.U32[1], value, 0U)); ctx.OpStore(rhs_pointer, ctx.OpCompositeExtract(ctx.U32[1], value, 1U)); } @@ -168,7 +168,7 @@ void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value) { const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)}; for (u32 i = 0; i < 4; ++i) { const Id index{i == 0 ? base_index : ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(i))}; - const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)}; + const Id pointer{ctx.SharedWordPointer(index)}; ctx.OpStore(pointer, ctx.OpCompositeExtract(ctx.U32[1], value, i)); } } diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index f8f966a6b5..babd2ef67e 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -400,8 +400,9 @@ Id CasLoop(EmitContext& ctx, Operation operation, Id array_pointer, Id element_p ctx.OpBranch(continue_block); ctx.AddLabel(continue_block); - const Id word_pointer{is_struct ? ctx.OpAccessChain(element_pointer, base, zero, index) - : ctx.OpAccessChain(element_pointer, base, index)}; + const Id word_pointer{is_shared ? ctx.SharedWordPointer(index) + : is_struct ? ctx.OpAccessChain(element_pointer, base, zero, index) + : ctx.OpAccessChain(element_pointer, base, index)}; if (value_type.value == ctx.F32[2].value) { const Id u32_value{ctx.OpLoad(ctx.U32[1], word_pointer)}; const Id value{ctx.OpUnpackHalf2x16(ctx.F32[2], u32_value)}; @@ -491,6 +492,19 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf DefineImageBuffers(program.info, image_binding); DefineTextures(program.info, texture_binding, bindings.texture_scaling_index); DefineImages(program.info, image_binding, bindings.image_scaling_index); + if (emulates_shared_memory && Sirit::ValidId(shared_memory_u32)) { + // SPIR-V types are unique, so the emulated buffer's are the u32 storage buffers' ones: + // decorate them only when DefineStorageBuffers did not (twice is invalid). + if (!Sirit::ValidId(storage_types.U32.array)) { + const Id array_type{TypeRuntimeArray(U32[1])}; + Decorate(array_type, spv::Decoration::ArrayStride, 4U); + const Id struct_type{TypeStruct(array_type)}; + Decorate(struct_type, spv::Decoration::Block); + MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U); + } + // After every other descriptor: the host appends the buffer in the same place. + Decorate(shared_memory_u32, spv::Decoration::Binding, storage_binding++); + } DefineAttributeMemAccess(program.info); DefineWriteStorageCasLoopFunction(program.info); DefineGlobalMemoryFunctions(program.info); @@ -500,6 +514,33 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf EmitContext::~EmitContext() = default; +Id EmitContext::SharedWordPointer(Id word_index) { + if (emulates_shared_memory) { + // This workgroup's slice: linear workgroup index x + nx * (y + ny * z). + const Id id{OpLoad(U32[3], workgroup_id)}; + const Id count{OpLoad(U32[3], num_workgroups)}; + const Id x{OpCompositeExtract(U32[1], id, 0U)}; + const Id y{OpCompositeExtract(U32[1], id, 1U)}; + const Id z{OpCompositeExtract(U32[1], id, 2U)}; + const Id nx{OpCompositeExtract(U32[1], count, 0U)}; + const Id ny{OpCompositeExtract(U32[1], count, 1U)}; + const Id linear{OpIAdd(U32[1], x, OpIMul(U32[1], nx, OpIAdd(U32[1], y, OpIMul(U32[1], ny, z))))}; + // Never past the buffer, which a write there can turn into a GPU fault (device lost): an + // indirect dispatch can have more workgroups than slices, and a program can address past + // its shared memory size. The host binds at least this many slices. + const u32 slices{EmulatedSharedMemorySlices( + shared_memory_words * 4, EmulatedSharedMemoryMaxBytes(profile.emulated_shared_memory_max_bytes))}; + const Id slice{OpUMin(U32[1], linear, Const(slices - 1))}; + const Id word{OpUMin(U32[1], word_index, Const(shared_memory_words - 1))}; + const Id base{OpIMul(U32[1], slice, Const(shared_memory_words))}; + return OpAccessChain(shared_u32, shared_memory_u32, u32_zero_value, + OpIAdd(U32[1], base, word)); + } + return uses_explicit_workgroup_layout + ? OpAccessChain(shared_u32, shared_memory_u32, u32_zero_value, word_index) + : OpAccessChain(shared_u32, shared_memory_u32, word_index); +} + Id EmitContext::Def(const IR::Value& value) { if (!value.IsImmediate()) { return value.InstRecursive()->Definition(); @@ -611,6 +652,10 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) { profile.support_explicit_workgroup_layout && (!program.info.uses_int8 || profile.support_workgroup_layout_8bit_access) && (!program.info.uses_int16 || profile.support_workgroup_layout_16bit_access); + emulates_shared_memory = program.info.emulated_shared_memory_size != 0; + if (emulates_shared_memory) { + uses_explicit_workgroup_layout = false; + } if (program.shared_memory_size == 0) { return; } @@ -650,13 +695,35 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) { std::tie(shared_memory_u32x4, shared_u32x4, std::ignore) = make(U32[4], 16); return; } - const u32 num_elements{Common::DivCeil(program.shared_memory_size, 4U)}; - const Id type{TypeArray(U32[1], Const(num_elements))}; - shared_memory_u32_type = TypePointer(spv::StorageClass::Workgroup, type); + if (emulates_shared_memory) { + // More shared memory than the device has (48 KB Switch 2 programs on a 32 KB Adreno): a + // storage buffer holds a slice per workgroup (SharedWordPointer). Its binding comes last, + // see the constructor. + shared_memory_words = Common::DivCeil(program.info.emulated_shared_memory_size, 4U); + // Decorated in the constructor: these are the u32 storage buffers' types too. + const Id struct_type{TypeStruct(TypeRuntimeArray(U32[1]))}; + shared_memory_u32_type = TypePointer(spv::StorageClass::StorageBuffer, struct_type); + shared_u32 = TypePointer(spv::StorageClass::StorageBuffer, U32[1]); + shared_memory_u32 = + AddGlobalVariable(shared_memory_u32_type, spv::StorageClass::StorageBuffer); + Decorate(shared_memory_u32, spv::Decoration::DescriptorSet, 0U); + Name(shared_memory_u32, "shared_memory_emulation"); + if (profile.supported_spirv >= 0x00010400) { + interfaces.push_back(shared_memory_u32); + } + if (!Sirit::ValidId(workgroup_id)) { + workgroup_id = DefineInput(*this, U32[3], false, spv::BuiltIn::WorkgroupId); + } + num_workgroups = DefineInput(*this, U32[3], false, spv::BuiltIn::NumWorkgroups); + } else { + const u32 num_elements{Common::DivCeil(program.shared_memory_size, 4U)}; + const Id type{TypeArray(U32[1], Const(num_elements))}; + shared_memory_u32_type = TypePointer(spv::StorageClass::Workgroup, type); - shared_u32 = TypePointer(spv::StorageClass::Workgroup, U32[1]); - shared_memory_u32 = AddGlobalVariable(shared_memory_u32_type, spv::StorageClass::Workgroup); - interfaces.push_back(shared_memory_u32); + shared_u32 = TypePointer(spv::StorageClass::Workgroup, U32[1]); + shared_memory_u32 = AddGlobalVariable(shared_memory_u32_type, spv::StorageClass::Workgroup); + interfaces.push_back(shared_memory_u32); + } const Id func_type{TypeFunction(void_id, U32[1], U32[1])}; const auto make_function{[&](u32 mask, u32 size) { @@ -679,7 +746,7 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) { OpBranch(continue_block); AddLabel(continue_block); - const Id word_pointer{OpAccessChain(shared_u32, shared_memory_u32, word_offset)}; + const Id word_pointer{SharedWordPointer(word_offset)}; const Id old_value{OpLoad(U32[1], word_pointer)}; const Id new_value{OpBitFieldInsert(U32[1], old_value, insert_value, bit_offset, count)}; const Id atomic_res{OpAtomicCompareExchange(U32[1], word_pointer, Const(1U), u32_zero_value, diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index 1f9b534aaf..04edc8888d 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -7,8 +7,10 @@ #pragma once #include +#include #include +#include "common/container/unordered_map.h" #include "common/container/unordered_set.h" #include "shader_recompiler/backend/bindings.h" @@ -174,7 +176,7 @@ class EmitContext final : public Sirit::Module { [[nodiscard]] Id BitOffset16(const IR::Value& offset); Id Const(u32 value) { - return Constant(U32[1], value); + return CachedConstant(u32_constants, U32[1], value, value); } Id Const(u32 element_1, u32 element_2) { @@ -208,7 +210,19 @@ class EmitContext final : public Sirit::Module { } Id Const(f32 value) { - return Constant(F32[1], value); + return CachedConstant(f32_constants, F32[1], std::bit_cast(value), value); + } + + /// Sirit deduplicates constants, but by hashing a freshly allocated copy of their words on + /// every call; immediates are emitted over and over, so remember the ids of scalar constants. + /// Keys are bit patterns, like Sirit's (0.0f and -0.0f stay distinct). + template + Id CachedConstant(::Common::unordered_map& cache, Id type, u32 bits, T value) { + const auto [it, is_new]{cache.try_emplace(bits)}; + if (is_new) { + it->second = Constant(type, value); + } + return it->second; } const Profile& profile; @@ -228,6 +242,9 @@ class EmitContext final : public Sirit::Module { VectorTypes F16; VectorTypes F64; + ::Common::unordered_map u32_constants; + ::Common::unordered_map f32_constants; + Id true_value{}; Id false_value{}; Id u32_zero_value{}; @@ -263,6 +280,14 @@ class EmitContext final : public Sirit::Module { std::vector images; Id workgroup_id{}; + Id num_workgroups{}; + + /// Shared memory lives in a storage buffer, a slice of shared_memory_words per workgroup. + bool emulates_shared_memory{}; + u32 shared_memory_words{}; + + /// Pointer to a 32-bit word of shared memory, native or emulated. + Id SharedWordPointer(Id word_index); Id local_invocation_id{}; Id invocation_id{}; Id patch_vertices_in{}; diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h index 66748f2bcd..f963334d5d 100644 --- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -2224,6 +2224,43 @@ inline LoweredProgram LowerToMaxwellWith(std::span code, u32 const bool depth_compare = insn.Bits(78, 1); const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + if (!bindless && !insn.Bits(60, 1) && dimension == 7 && lod == 3) { + const u32 components = std::popcount(mask); + if (depth_compare || !components || split != (components > 2) || + insn.Bits(76, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + source_a >= 246 || source_b >= 250 || dest == 255 || + (components > 1 && ((dest & 1) || dest >= 254)) || + (components == 4 && ((second_dest & 1) || second_dest >= 254))) + return fail(index, "native array-cube vector sample mode"); + if (!bind_texture_buffer(2)) + return fail(index, "native texture descriptor buffer conflict"); + // Vector coordinates already use layer,X,Y,Z order. LOD is + // supplied separately, matching the Maxwell TEX operand ABI. + emit(0xc000000000000000ULL | predicate | 250 | (u64{source_a} << 8) | + (u64{source_b} << 20) | (u64{7} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(77, 1)} << 35) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{lod} << 55)); + if (insn.Bits(79, 1)) { + // FP16 results, packed as the general path below does: (c0, c1) into the + // first destination, (c2, c3) into the second (F2F writes the half in the + // low 16 bits; ISCADD merges, op_a shifted up as the high half). + const auto pack = [&](u32 target, u32 low, u32 high) { + emit(0x5ca8000000000000ULL | predicate | 246 | (u64{1} << 8) | (u64{2} << 10) | (u64{low} << 20)); + emit(0x5ca8000000000000ULL | predicate | 247 | (u64{1} << 8) | (u64{2} << 10) | (u64{high} << 20)); + emit(0x5c18000000000000ULL | predicate | target | (u64{247} << 8) | (u64{246} << 20) | (u64{16} << 39)); + }; + pack(static_cast(dest), 250, components > 1 ? 251U : 255U); + if (components > 2) + pack(second_dest, 252, components > 3 ? 253U : 255U); + break; + } + for (u32 i = 0; i < components; ++i) + emit(0x5c98000000000000ULL | predicate | + (i < 2 ? dest + i : second_dest + i - 2) | + (u64{15} << 39) | (u64{250 + i} << 20)); + break; + } if (!bindless && dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || diff --git a/src/shader_recompiler/frontend/ir/basic_block.h b/src/shader_recompiler/frontend/ir/basic_block.h index d3a66caac9..e86f05dc1f 100644 --- a/src/shader_recompiler/frontend/ir/basic_block.h +++ b/src/shader_recompiler/frontend/ir/basic_block.h @@ -89,6 +89,17 @@ class Block { return ssa_reg_values[RegIndex(reg)]; } + /// Number of other fixed-count variables the SSA pass stores in blocks + /// (predicates, the four condition flags and the indirect branch variable). + static constexpr size_t NUM_SSA_VALUES = NUM_USER_PREDS + 5; + + void SetSsaValue(size_t index, const Value& value) noexcept { + ssa_values[index] = value; + } + const Value& SsaValue(size_t index) const noexcept { + return ssa_values[index]; + } + void SsaSeal() noexcept { is_ssa_sealed = true; } @@ -182,6 +193,8 @@ class Block { /// Intrusively store the value of a register in the block. std::array ssa_reg_values; + /// Intrusively store the value of the other fixed-count SSA variables in the block. + std::array ssa_values; /// Intrusively store if the block is sealed in the SSA pass. bool is_ssa_sealed{false}; diff --git a/src/shader_recompiler/frontend/maxwell/control_flow.cpp b/src/shader_recompiler/frontend/maxwell/control_flow.cpp index 9fb30df664..a67c02581a 100644 --- a/src/shader_recompiler/frontend/maxwell/control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/control_flow.cpp @@ -256,12 +256,18 @@ void CFG::AnalyzeLabel(FunctionId function_id, Label& label) { bool CFG::InspectVisitedBlocks(FunctionId function_id, const Label& label) { const Location pc{label.address}; Function& function{functions[function_id]}; - const auto it{ - std::ranges::find_if(function.blocks, [pc](auto& block) { return block.Contains(pc); })}; - if (it == function.blocks.end()) { + // Blocks are sorted by their first instruction and do not overlap; empty blocks (virtual + // blocks) contain nothing. So only the last non-empty block starting at or before pc can + // contain it. A linear search over all blocks for every label was quadratic. + auto it{function.blocks.upper_bound(pc, Compare{})}; + while (it != function.blocks.begin() && std::prev(it)->begin == std::prev(it)->end) { + --it; + } + if (it == function.blocks.begin() || !std::prev(it)->Contains(pc)) { // Address has not been visited return false; } + --it; Block* const visited_block{&*it}; if (visited_block->begin == pc) { throw LogicError("Dangling block"); diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp index c8c9e26248..0e07d0a239 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp @@ -14,7 +14,7 @@ namespace Shader::Maxwell { namespace { enum class DestFormat : u64 { - Invalid, + I8, I16, I32, I64, @@ -48,6 +48,8 @@ union F2I { size_t BitSize(DestFormat dest_format) { switch (dest_format) { + case DestFormat::I8: + return 8; case DestFormat::I16: return 16; case DestFormat::I32: @@ -62,6 +64,9 @@ size_t BitSize(DestFormat dest_format) { std::pair ClampBounds(DestFormat format, bool is_signed) { if (is_signed) { switch (format) { + case DestFormat::I8: + return {static_cast((std::numeric_limits::max)()), + static_cast((std::numeric_limits::min)())}; case DestFormat::I16: return {static_cast((std::numeric_limits::max)()), static_cast((std::numeric_limits::min)())}; @@ -76,6 +81,9 @@ std::pair ClampBounds(DestFormat format, bool is_signed) { } } else { switch (format) { + case DestFormat::I8: + return {static_cast((std::numeric_limits::max)()), + static_cast((std::numeric_limits::min)())}; case DestFormat::I16: return {static_cast((std::numeric_limits::max)()), static_cast((std::numeric_limits::min)())}; diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/integer_byte_permute.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/integer_byte_permute.cpp new file mode 100644 index 0000000000..66c87db909 --- /dev/null +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/integer_byte_permute.cpp @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "common/bit_field.h" +#include "common/common_types.h" +#include "shader_recompiler/frontend/maxwell/translate/impl/impl.h" + +namespace Shader::Maxwell { +namespace { +// PRMT in its default mode: each result byte is picked by a selector nibble from the eight bytes +// {c, a} (0-3 from a, 4-7 from c); bit 3 of the nibble replicates the picked byte's sign bit. +void PRMT(TranslatorVisitor& v, u64 insn, const IR::U32& selector, const IR::U32& src_c) { + union { + u64 raw; + BitField<0, 8, IR::Reg> dest_reg; + BitField<8, 8, IR::Reg> src_a_reg; + BitField<48, 3, u64> mode; + } const prmt{insn}; + if (prmt.mode != 0) { + throw NotImplementedException("PRMT mode {}", prmt.mode.Value()); + } + const IR::U32 src_a{v.X(prmt.src_a_reg)}; + IR::U32 result{v.ir.Imm32(0)}; + for (u32 byte = 0; byte < 4; ++byte) { + const IR::U32 nibble{v.ir.BitFieldExtract(selector, v.ir.Imm32(byte * 4), v.ir.Imm32(4))}; + const IR::U32 index{v.ir.BitwiseAnd(nibble, v.ir.Imm32(7))}; + const IR::U32 source{ + IR::U32{v.ir.Select(v.ir.ILessThan(index, v.ir.Imm32(4), false), src_a, src_c)}}; + const IR::U32 offset{v.ir.ShiftLeftLogical(v.ir.BitwiseAnd(index, v.ir.Imm32(3)), v.ir.Imm32(3))}; + const IR::U32 value{v.ir.BitFieldExtract(source, offset, v.ir.Imm32(8))}; + const IR::U32 sign{v.ir.BitFieldExtract(source, v.ir.IAdd(offset, v.ir.Imm32(7)), v.ir.Imm32(1), true)}; + const IR::U1 replicate{v.ir.INotEqual(v.ir.BitwiseAnd(nibble, v.ir.Imm32(8)), v.ir.Imm32(0))}; + const IR::U32 picked{IR::U32{v.ir.Select(replicate, sign, value)}}; + result = v.ir.BitFieldInsert(result, picked, v.ir.Imm32(byte * 8), v.ir.Imm32(8)); + } + v.X(prmt.dest_reg, result); +} +} // Anonymous namespace + +void TranslatorVisitor::PRMT_reg(u64 insn) { + PRMT(*this, insn, GetReg20(insn), GetReg39(insn)); +} + +void TranslatorVisitor::PRMT_rc(u64 insn) { + PRMT(*this, insn, GetReg39(insn), GetCbuf(insn)); +} + +void TranslatorVisitor::PRMT_cr(u64 insn) { + PRMT(*this, insn, GetCbuf(insn), GetReg39(insn)); +} + +void TranslatorVisitor::PRMT_imm(u64 insn) { + PRMT(*this, insn, GetImm20(insn), GetReg39(insn)); +} + +} // namespace Shader::Maxwell diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp index 369af8e03a..4165ca2f66 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp @@ -195,22 +195,6 @@ void TranslatorVisitor::PRET(u64) { ThrowNotImplemented(Opcode::PRET); } -void TranslatorVisitor::PRMT_reg(u64) { - ThrowNotImplemented(Opcode::PRMT_reg); -} - -void TranslatorVisitor::PRMT_rc(u64) { - ThrowNotImplemented(Opcode::PRMT_rc); -} - -void TranslatorVisitor::PRMT_cr(u64) { - ThrowNotImplemented(Opcode::PRMT_cr); -} - -void TranslatorVisitor::PRMT_imm(u64) { - ThrowNotImplemented(Opcode::PRMT_imm); -} - void TranslatorVisitor::R2B(u64) { ThrowNotImplemented(Opcode::R2B); } diff --git a/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp b/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp index b5cd233992..b8fabee46f 100644 --- a/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp +++ b/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp @@ -54,8 +54,29 @@ struct IndirectBranchVariable { }; using Variant = std::variant; -// TODO: majority of these require stable iterators, test with XC beforehand -using ValueMap = std::unordered_map; +// Definitions are copied out as soon as they are read, so the tables need no stable references. +// Fixed-count variables live in the blocks, like registers: every read walks back through +// blocks and looks up or writes a definition in each, and node-based maps per variable made +// those lookups and their allocations a large part of the pass. +constexpr size_t SSA_ZERO_FLAG = IR::NUM_USER_PREDS; +constexpr size_t SSA_SIGN_FLAG = IR::NUM_USER_PREDS + 1; +constexpr size_t SSA_CARRY_FLAG = IR::NUM_USER_PREDS + 2; +constexpr size_t SSA_OVERFLOW_FLAG = IR::NUM_USER_PREDS + 3; +constexpr size_t SSA_INDIRECT_BRANCH = IR::NUM_USER_PREDS + 4; +static_assert(SSA_INDIRECT_BRANCH < IR::Block::NUM_SSA_VALUES); + +struct GotoKey { + bool operator==(const GotoKey&) const noexcept = default; + + IR::Block* block; + u32 index; +}; + +struct GotoKeyHash { + size_t operator()(const GotoKey& key) const noexcept { + return std::hash{}(key.block) ^ (static_cast(key.index) << 32); + } +}; struct DefTable { const IR::Value& Def(IR::Block* block, IR::Reg variable) { @@ -66,62 +87,57 @@ struct DefTable { } const IR::Value& Def(IR::Block* block, IR::Pred variable) { - return preds[IR::PredIndex(variable)][block]; + return block->SsaValue(IR::PredIndex(variable)); } void SetDef(IR::Block* block, IR::Pred variable, const IR::Value& value) { - preds[IR::PredIndex(variable)].insert_or_assign(block, value); + block->SetSsaValue(IR::PredIndex(variable), value); } const IR::Value& Def(IR::Block* block, GotoVariable variable) { - return goto_vars[variable.index][block]; + const auto it{goto_vars.find(GotoKey{block, variable.index})}; + return it != goto_vars.end() ? it->second : empty; } void SetDef(IR::Block* block, GotoVariable variable, const IR::Value& value) { - goto_vars[variable.index].insert_or_assign(block, value); + goto_vars.insert_or_assign(GotoKey{block, variable.index}, value); } const IR::Value& Def(IR::Block* block, IndirectBranchVariable) { - return indirect_branch_var[block]; + return block->SsaValue(SSA_INDIRECT_BRANCH); } void SetDef(IR::Block* block, IndirectBranchVariable, const IR::Value& value) { - indirect_branch_var.insert_or_assign(block, value); + block->SetSsaValue(SSA_INDIRECT_BRANCH, value); } const IR::Value& Def(IR::Block* block, ZeroFlagTag) { - return zero_flag[block]; + return block->SsaValue(SSA_ZERO_FLAG); } void SetDef(IR::Block* block, ZeroFlagTag, const IR::Value& value) { - zero_flag.insert_or_assign(block, value); + block->SetSsaValue(SSA_ZERO_FLAG, value); } const IR::Value& Def(IR::Block* block, SignFlagTag) { - return sign_flag[block]; + return block->SsaValue(SSA_SIGN_FLAG); } void SetDef(IR::Block* block, SignFlagTag, const IR::Value& value) { - sign_flag.insert_or_assign(block, value); + block->SetSsaValue(SSA_SIGN_FLAG, value); } const IR::Value& Def(IR::Block* block, CarryFlagTag) { - return carry_flag[block]; + return block->SsaValue(SSA_CARRY_FLAG); } void SetDef(IR::Block* block, CarryFlagTag, const IR::Value& value) { - carry_flag.insert_or_assign(block, value); + block->SetSsaValue(SSA_CARRY_FLAG, value); } const IR::Value& Def(IR::Block* block, OverflowFlagTag) { - return overflow_flag[block]; + return block->SsaValue(SSA_OVERFLOW_FLAG); } void SetDef(IR::Block* block, OverflowFlagTag, const IR::Value& value) { - overflow_flag.insert_or_assign(block, value); + block->SetSsaValue(SSA_OVERFLOW_FLAG, value); } - std::array preds; - // TODO: Requires stable iterators - std::unordered_map goto_vars; - ValueMap indirect_branch_var; - ValueMap zero_flag; - ValueMap sign_flag; - ValueMap carry_flag; - ValueMap overflow_flag; + Common::unordered_map goto_vars; + const IR::Value empty{}; }; IR::Opcode UndefOpcode(IR::Reg) noexcept { @@ -147,16 +163,19 @@ enum class Status { PushPhiArgument, }; -template struct ReadState { ReadState(IR::Block* block_) : block{block_} {} ReadState() = default; IR::Block* block{}; IR::Value result{}; + /// Phi of a sealed block with several predecessors, null while it is only pending IR::Inst* phi{}; IR::Block* const* pred_it{}; IR::Block* const* pred_end{}; + /// Operands of a pending phi, in Pass::pending_operands + size_t operands_begin{}; + size_t num_operands{}; Status pc{Status::Start}; }; @@ -169,16 +188,21 @@ class Pass { template IR::Value ReadVariable(Type variable, IR::Block* root_block) { - // TODO: Windows commits sodoku if you use small_vector - std::vector> stack{ - ReadState(nullptr), - ReadState(root_block), - }; + // Reads walk back through every block until they find a definition, often dozens deep: + // reuse one stack instead of growing a new vector on every read. + auto& stack{read_stack}; + stack.clear(); + stack.emplace_back(nullptr); + stack.emplace_back(root_block); + const IR::Opcode undef_opcode{UndefOpcode(variable)}; const auto prepare_phi_operand{[&] { - if (stack.back().pred_it == stack.back().pred_end) { - IR::Inst* const phi{stack.back().phi}; - IR::Block* const block{stack.back().block}; - const IR::Value result{TryRemoveTrivialPhi(*phi, block, UndefOpcode(variable))}; + ReadState& state{stack.back()}; + if (state.pred_it == state.pred_end) { + IR::Block* const block{state.block}; + const IR::Value result{state.phi + ? TryRemoveTrivialPhi(*state.phi, block, undef_opcode) + : ResolvePendingPhi(state, undef_opcode)}; + pending_operands.resize(state.operands_begin); stack.pop_back(); stack.back().result = result; WriteVariable(variable, block, result); @@ -192,7 +216,12 @@ class Pass { IR::Block* const block{stack.back().block}; switch (stack.back().pc) { case Status::Start: { - if (const IR::Value& def = current_def.Def(block, variable); !def.IsEmpty()) { + if (IR::Value def{current_def.Def(block, variable)}; !def.IsEmpty()) { + if (def == pending_phi) { + // A cycle reached a pending phi: it needs to exist now + def = IR::Value{MaterializePhi(PendingFrame(block), undef_opcode)}; + WriteVariable(variable, block, def); + } stack.back().result = def; } else if (!block->IsSsaSealed()) { // Incomplete CFG @@ -208,13 +237,16 @@ class Pass { stack.emplace_back(imm_preds.front()); break; } else { - // Break potential cycles with operandless phi - IR::Inst* const phi{&*block->PrependNewInst(block->begin(), IR::Opcode::Phi)}; - phi->SetFlags(IR::TypeOf(UndefOpcode(variable))); - - WriteVariable(variable, block, IR::Value{phi}); - - stack.back().phi = phi; + // Break potential cycles with an operandless phi. Most of these phis turn out + // trivial, so it stays pending (pending_phi marks the block) and is only + // created when a cycle reads it or when it merges different values. Nothing + // is inserted at the front of this block until then, so it is created where + // it would have been created right away. + WriteVariable(variable, block, pending_phi); + + stack.back().phi = nullptr; + stack.back().operands_begin = pending_operands.size(); + stack.back().num_operands = 0; stack.back().pred_it = imm_preds.data(); stack.back().pred_end = imm_preds.data() + imm_preds.size(); prepare_phi_operand(); @@ -230,9 +262,14 @@ class Pass { break; } case Status::PushPhiArgument: { - IR::Inst* const phi{stack.back().phi}; - phi->AddPhiOperand(*stack.back().pred_it, stack.back().result); - ++stack.back().pred_it; + ReadState& state{stack.back()}; + if (state.phi) { + state.phi->AddPhiOperand(*state.pred_it, state.result); + } else { + pending_operands.emplace_back(*state.pred_it, state.result); + ++state.num_operands; + } + ++state.pred_it; } [[fallthrough]]; case Status::PreparePhiArgument: @@ -256,6 +293,51 @@ class Pass { } private: + ReadState& PendingFrame(IR::Block* block) { + for (auto it = read_stack.rbegin(); it != read_stack.rend(); ++it) { + if (it->block == block && it->pc == Status::PushPhiArgument && !it->phi) { + return *it; + } + } + throw LogicError("Pending phi without a frame"); + } + + /// Creates the phi of a pending frame with the operands it collected so far + IR::Inst* MaterializePhi(ReadState& state, IR::Opcode undef_opcode) { + IR::Block* const block{state.block}; + IR::Inst* const phi{&*block->PrependNewInst(block->begin(), IR::Opcode::Phi)}; + phi->SetFlags(IR::TypeOf(undef_opcode)); + for (size_t i = 0; i < state.num_operands; ++i) { + const auto& [pred, value]{pending_operands[state.operands_begin + i]}; + phi->AddPhiOperand(pred, value); + } + state.phi = phi; + return phi; + } + + /// TryRemoveTrivialPhi for a phi nothing has seen yet: no self-references, and nothing to + /// remove or reroute when it is trivial + IR::Value ResolvePendingPhi(ReadState& state, IR::Opcode undef_opcode) { + IR::Value same; + for (size_t i = 0; i < state.num_operands; ++i) { + const IR::Value& op{pending_operands[state.operands_begin + i].second}; + if (op.Resolve() == same.Resolve()) { + continue; + } + if (!same.IsEmpty()) { + return IR::Value{MaterializePhi(state, undef_opcode)}; + } + same = op; + } + if (same.IsEmpty()) { + // Unreachable, or the start block: same insertion point as in TryRemoveTrivialPhi + IR::Block* const block{state.block}; + const auto insert_point{std::ranges::find_if_not(block->Instructions(), IR::IsPhi)}; + same = IR::Value{&*block->PrependNewInst(insert_point, undef_opcode)}; + } + return same; + } + template IR::Value AddPhiOperands(Type variable, IR::Inst& phi, IR::Block* block) { for (IR::Block* const imm_pred : block->ImmPredecessors()) { @@ -303,6 +385,12 @@ class Pass { // TODO: Windows dies with stack exhaustion? std::unordered_map> incomplete_phis; DefTable current_def; + // ReadVariable never nests, so its walk stack can be shared + std::vector read_stack; + std::vector> pending_operands; + // Definition of blocks whose phi is pending + IR::Inst pending_phi_inst{IR::Opcode::Void, 0}; + const IR::Value pending_phi{&pending_phi_inst}; }; void VisitInst(Pass& pass, IR::Block* block, IR::Inst& inst) { @@ -412,6 +500,20 @@ void SsaRewritePass(IR::Program& program) { } } } + // Trivial phis and replaced reads stay in their blocks as Identity instructions, often more + // of them than real instructions. Those without uses cannot gain any now that the definition + // tables are gone: drop them here, as dead code elimination would later (same order), instead + // of letting every pass until then walk them. + for (IR::Block* const block : program.post_order_blocks) { + auto it{block->end()}; + while (it != block->begin()) { + --it; + if (it->GetOpcode() == IR::Opcode::Identity && !it->HasUses()) { + it->Invalidate(); + it = block->Instructions().erase(it); + } + } + } } } // namespace Shader::Optimization diff --git a/src/shader_recompiler/profile.h b/src/shader_recompiler/profile.h index 8e9c83f888..29da75a299 100644 --- a/src/shader_recompiler/profile.h +++ b/src/shader_recompiler/profile.h @@ -113,6 +113,9 @@ struct Profile { u64 min_ssbo_alignment{}; u32 max_user_clip_distances{}; + /// Bytes an emulated shared memory buffer may span on this host (EmulatedSharedMemoryMaxBytes); + /// 0: the fixed limit. + u32 emulated_shared_memory_max_bytes{}; bool SupportsSubgroupStage(Stage stage) const { return (supported_subgroup_stages & (1u << static_cast(stage))) != 0; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 4e5fb5c6e4..e38437a768 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -267,6 +268,24 @@ struct MeshInfo { boost::container::static_vector primitive_lines; }; +/// Most bytes an emulated shared memory buffer spans: shaders clamp their workgroup slice to it +/// and the host binds at least min(workgroups, slices that fit) slices. +constexpr u32 EMULATED_SHARED_MEMORY_MAX_BYTES = 256U << 20; + +/// The limit on a host whose storage buffer range is `max_storage_buffer_range` (0: unknown, +/// the fixed limit): the shader's clamp and the host's binding must agree on it. +[[nodiscard]] constexpr u32 EmulatedSharedMemoryMaxBytes(u64 max_storage_buffer_range) { + return max_storage_buffer_range == 0 + ? EMULATED_SHARED_MEMORY_MAX_BYTES + : static_cast((std::min)(u64{EMULATED_SHARED_MEMORY_MAX_BYTES}, + max_storage_buffer_range)); +} + +[[nodiscard]] constexpr u32 EmulatedSharedMemorySlices(u32 bytes_per_workgroup, + u32 max_bytes = EMULATED_SHARED_MEMORY_MAX_BYTES) { + return bytes_per_workgroup == 0 ? 1U : (std::max)(max_bytes / bytes_per_workgroup, 1U); +} + struct Info { static constexpr size_t MAX_INDIRECT_CBUFS{14}; static constexpr size_t MAX_CBUFS{18}; @@ -274,6 +293,10 @@ struct Info { bool is_task{}; ///< Switch 2 task program (Stage::Task) bool is_mesh{}; ///< Switch 2 mesh program (Stage::Mesh) + /// Shared memory per workgroup kept in a storage buffer instead (the device has less than the + /// program needs), bound after every other descriptor. Zero when shared memory is native. + u32 emulated_shared_memory_size{}; + bool uses_workgroup_id{}; bool uses_local_invocation_id{}; bool uses_invocation_id{}; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index d23d3eb9b5..a0bf824fa4 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -112,6 +112,7 @@ add_library(video_core STATIC native_query_trace.h nx2.h nx2_profile.h + semaphore_ledger.h present.h pte_kind.h query_cache/bank_base.h @@ -442,6 +443,15 @@ target_include_directories(video_core PUBLIC ${FFmpeg_INCLUDE_DIR}) target_link_libraries(video_core PRIVATE ${FFmpeg_LIBRARIES}) target_link_options(video_core PRIVATE ${FFmpeg_LDFLAGS}) +# libgav1 decodes Switch 2 AV1 movies on Android, whose FFmpeg has no libdav1d; only the nx2 video +# decoder sees its headers. +if (ANDROID) + set_source_files_properties(host1x/nx2_video_decoder.cpp PROPERTIES + INCLUDE_DIRECTORIES "$" + COMPILE_DEFINITIONS EDEN_NX2_LIBGAV1) + target_link_libraries(video_core PRIVATE "$") +endif() + add_dependencies(video_core host_shaders) target_include_directories(video_core PRIVATE ${HOST_SHADERS_INCLUDE}) @@ -500,6 +510,12 @@ if (ARCHITECTURE_x86_64 OR ARCHITECTURE_arm64) target_link_libraries(video_core PRIVATE dynarmic::dynarmic) endif() +if (ARCHITECTURE_arm64 AND ANDROID) + # The Turing macro JIT's ARM64 emitter (macro_turing_jit_arm64.h), used on Android. + target_sources(video_core PRIVATE macro_turing_jit_arm64.cpp macro_turing_jit_arm64.h) + target_link_libraries(video_core PRIVATE merry::oaknut) +endif() + if (ANDROID AND ARCHITECTURE_arm64) target_link_libraries(video_core PRIVATE adrenotools) endif() diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index d7eb21612f..ac2481c313 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -129,6 +129,16 @@ class BufferBase { write_tick = write_tick_; } + /// The GPU tick of the last command recorded to modify this buffer (shader writes, copies, + /// clears): what a direct host read of its memory waits for. + [[nodiscard]] u64 GpuWriteTick() const noexcept { + return gpu_write_tick; + } + + void SetGpuWriteTick(u64 tick) noexcept { + gpu_write_tick = tick; + } + private: VAddr cpu_addr = 0; BufferFlagBits flags{}; @@ -136,6 +146,7 @@ class BufferBase { size_t lru_id = SIZE_MAX; size_t size_bytes = 0; u64 write_tick = 0; + u64 gpu_write_tick = 0; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f357a3583c..bf9cdd735c 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -269,8 +269,48 @@ std::optional BufferCache

::GetFlushArea(DA template void BufferCache

::DownloadMemory(DAddr device_addr, u64 size) { + // EDEN_NX2_DOWNLOAD_WINDOW=1 also takes the pages around the range, below. Off: even with + // only whole, protected pages of the buffer, Mario Kart World faulted the GPU within two + // minutes (CP opcode errors), and raced 12 minutes without one with it off. Why is not found. + static const bool window = [] { + const char* value = std::getenv("EDEN_NX2_DOWNLOAD_WINDOW"); + return value != nullptr && value[0] != '0'; + }(); ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) { - DownloadBufferMemory(buffer, device_addr, size); + if (!window) { + DownloadBufferMemory(buffer, device_addr, size); + return; + } + // Each download waits for the whole GPU, and Mario Kart World's macros and CPU read + // GPU-written data a few words at a time: also take the GPU-written pages of the same + // buffer in the 64 KiB around the range, so the reads after them find it flushed. + // Only whole pages of this buffer: the tracking is per page, and clearing a page this + // buffer shares with another would drop the other's GPU-written state and copy bytes + // from outside this buffer (taking every page in the window corrupted guest memory + // and faulted the GPU within minutes). And only pages still write-protected: a CPU write + // there faults and flushes first, so the GPU's data is the newest. + constexpr u64 Window = u64{64} << 10; + constexpr u64 Page = Core::DEVICE_PAGESIZE; + const DAddr owned_begin = Common::AlignUp(buffer.CpuAddr(), Page); + const DAddr owned_end = Common::AlignDown(buffer.CpuAddr() + buffer.SizeBytes(), Page); + const DAddr begin = (std::max)(Common::AlignDown(device_addr, Window), owned_begin); + const DAddr end = (std::min)(Common::AlignUp(device_addr + size, Window), owned_end); + const DAddr asked_begin = Common::AlignDown(device_addr, Page); + const DAddr asked_end = Common::AlignUp(device_addr + size, Page); + boost::container::small_vector, 8> ranges{{device_addr, size}}; + for (DAddr page = begin; page < end; page += Page) { + if ((page >= asked_begin && page < asked_end) || + !memory_tracker.IsRegionGpuModified(page, Page) || + memory_tracker.IsRegionUntracked(page, Page)) { + continue; + } + if (ranges.size() > 1 && ranges.back().first + ranges.back().second == page) { + ranges.back().second += Page; + } else { + ranges.emplace_back(page, Page); + } + } + DownloadBufferRanges(buffer, ranges); }); } @@ -344,6 +384,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); dest_buffer.MarkUsage(copy.dst_offset, copy.size); + NoteGpuWrite(dest_buffer); runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); @@ -377,6 +418,7 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast(size), false); Buffer& dest_buffer = slot_buffers[buffer]; const u32 offset = dest_buffer.Offset(*cpu_dst_address); + NoteGpuWrite(dest_buffer); runtime.ClearBuffer(dest_buffer, offset, size, value); dest_buffer.MarkUsage(offset, size); return true; @@ -412,6 +454,9 @@ std::pair BufferCache

::ObtainCPUBuffer( case ObtainBufferOperation::MarkAsWritten: MarkWrittenBuffer(buffer_id, device_addr, size); break; + case ObtainBufferOperation::NoteGpuWrite: + NoteGpuWrite(slot_buffers[buffer_id]); + break; case ObtainBufferOperation::DiscardWrite: { ClearDownload(device_addr, size); gpu_modified_ranges.Subtract(device_addr, size); @@ -900,6 +945,7 @@ void BufferCache

::BindHostIndexBuffer() { auto upload_staging = runtime.UploadStagingBuffer(size); std::array copies{{BufferCopy{.src_offset = upload_staging.offset, .dst_offset = 0, .size = size}}}; std::memcpy(upload_staging.mapped_span.data(), draw_state.inline_index_draw_indexes.data(), size); + NoteGpuWrite(buffer); runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true); } else { buffer.ImmediateUpload(0, draw_state.inline_index_draw_indexes); @@ -1174,17 +1220,24 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 // stream allocation again instead of allocating and copying (uniform_stream_reuse.h). // EDEN_NX2_UBOREUSE=0 turns it off. if (VideoCommon::UniformStreamReuseEnabled()) { + Nx2Profile::SampledScope<3> scope{Nx2Profile::Zone::UboReuse}; streamed = VideoCommon::BindSmallUniformReusing( runtime, device_memory, stage, index, binding_index, device_addr, size); + if (streamed) { + Nx2Profile::Add(Nx2Profile::Counter::UboReuseHits, 1); + } } } if (!streamed) { + Nx2Profile::SampledScope<3> scope{Nx2Profile::Zone::UboCopy}; + Nx2Profile::Add(Nx2Profile::Counter::UboCopyBytes, size); const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); device_memory.ReadBlockUnsafe(device_addr, span.data(), size); } return; } // Classic cached path + Nx2Profile::SampledScope<3> cached_scope{Nx2Profile::Zone::UboCached}; if (SynchronizeBuffer(buffer, device_addr, size)) { ++channel_state->uniform_cache_hits[0]; } @@ -1262,6 +1315,15 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt if (binding.segment_first + binding.segment_count > pool.size()) { return false; } + // A segment whose buffer was joined into another (deleted) since it was resolved: binding + // the dead slot put its freed LRU item back in the list, and the collector later deleted + // the slot again, wiping another buffer's page entries. + for (u32 index = 0; index < binding.segment_count; ++index) { + const BufferId id = pool[binding.segment_first + index].buffer_id; + if (!id || !slot_buffers.contains(id)) [[unlikely]] { + return false; + } + } const u64 key = (static_cast(gpu_memory->GetID()) << 48) ^ binding.gpu_addr; runtime.ResetMultiRange(); for (u32 index = 0; index < binding.segment_count; ++index) { @@ -1495,18 +1557,26 @@ void BufferCache

::DoUpdateGraphicsBuffers(bool is_indexed) { graphics_segments.clear(); BufferOperations([&]() { if (is_indexed) { + Nx2Profile::Scope scope{Nx2Profile::Zone::UpdateIndex}; UpdateIndexBuffer(); } - UpdateVertexBuffers(); + { + Nx2Profile::Scope scope{Nx2Profile::Zone::UpdateVertex}; + UpdateVertexBuffers(); + } UpdateTransformFeedbackBuffers(); - for (size_t stage = 0; stage < NUM_STAGES; ++stage) { - UpdateUniformBuffers(stage); - UpdateStorageBuffers(stage); - UpdateTextureBuffers(stage); + { + Nx2Profile::Scope scope{Nx2Profile::Zone::UpdateStages}; + for (size_t stage = 0; stage < NUM_STAGES; ++stage) { + UpdateUniformBuffers(stage); + UpdateStorageBuffers(stage); + UpdateTextureBuffers(stage); + } } if (current_draw_indirect) { UpdateDrawIndirect(); } + RefreshMultiRangeSegments(graphics_segments); }); } @@ -1517,9 +1587,31 @@ void BufferCache

::DoUpdateComputeBuffers() { UpdateComputeUniformBuffers(); UpdateComputeStorageBuffers(); UpdateComputeTextureBuffers(); + RefreshMultiRangeSegments(compute_segments); }); } +template +void BufferCache

::RefreshMultiRangeSegments(std::vector& pool) { + if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { + // Resolving one segment can join, and so delete, the buffer of another (DeleteBuffer + // clears its id): repeat until every segment points at a live buffer. + for (int pass = 0; pass < 16; ++pass) { + bool resolved = false; + for (MultiRangeSegment& segment : pool) { + if (!segment.buffer_id) { + segment.buffer_id = FindBuffer(segment.device_addr, segment.size, + runtime.PrefersSparseSources()); + resolved = true; + } + } + if (!resolved) { + return; + } + } + } +} + template void BufferCache

::UpdateIndexBuffer() { // We have to check for the dirty flags and index count @@ -1535,10 +1627,14 @@ void BufferCache

::UpdateIndexBuffer() { auto inline_index_size = static_cast(draw_state.inline_index_draw_indexes.size()); u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE); if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] { + LOG_INFO(HW_GPU, "Creating inline index buffer of {:#x} bytes", buffer_size); inline_buffer_id = CreateBuffer(0, buffer_size, false); } if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] { - slot_buffers.erase(inline_buffer_id); + LOG_INFO(HW_GPU, "Growing inline index buffer to {:#x} bytes", buffer_size); + // Erasing the slot directly left its page table entries and LRU item behind, so a + // later CreateBuffer could reuse the slot and join the new buffer with itself. + DeleteBuffer(inline_buffer_id, true); inline_buffer_id = CreateBuffer(0, buffer_size, false); } channel_state->index_buffer = Binding{ @@ -1588,9 +1684,16 @@ void BufferCache

::UpdateVertexBuffer(u32 index) { const auto& limit = maxwell3d->regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); const bool native = maxwell3d->GetNativeMacroEngine().native_turing; - const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0; + u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0; if (native && array.enable != 0 && native_size > (std::numeric_limits::max)()) { - throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size"); + // A misread or stale size: bounded by what's mapped below instead. Thrown on the GPU + // thread, it aborted Mario Kart World as it booted. + static std::atomic reports{}; + if (reports.fetch_add(1, std::memory_order_relaxed) < 8) { + LOG_WARNING(HW_GPU, "Native vertex stream {} size {:#x} exceeds 32 bits, clamped", index, + native_size); + } + native_size = (std::numeric_limits::max)(); } const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1; const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); @@ -1748,11 +1851,19 @@ void BufferCache

::UpdateComputeTextureBuffers() { }); } +template +void BufferCache

::NoteGpuWrite(Buffer& buffer) { + if constexpr (!IS_OPENGL) { + buffer.SetGpuWriteTick(runtime.CurrentTick()); + } +} + template void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync) { if constexpr (!IS_OPENGL) { + Buffer& buffer = slot_buffers[buffer_id]; + NoteGpuWrite(buffer); if (needs_sync) { - Buffer& buffer = slot_buffers[buffer_id]; buffer.setWriteTick(runtime.CurrentTick()); } } @@ -2042,6 +2153,18 @@ typename BufferCache

::OverlapResult BufferCache

::ResolveOverlaps(DAddr dev if (!overlap_id) { continue; } + if (!slot_buffers.contains(overlap_id)) [[unlikely]] { + // A deleted buffer is still in the page table. Joining it would delete whichever + // buffer reuses the slot next, possibly the one being created here. + static u32 stale_logs = 0; + if (stale_logs < 16) { + ++stale_logs; + LOG_ERROR(HW_GPU, "Stale buffer page: device_addr={:#x} buffer_id={}", + device_addr, overlap_id.index); + } + page_table[device_addr >> CACHING_PAGEBITS] = BufferId{}; + continue; + } Buffer& overlap = slot_buffers[overlap_id]; if (overlap.IsPicked()) { continue; @@ -2095,6 +2218,7 @@ void BufferCache

::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, .size = overlap.SizeBytes(), }); new_buffer.MarkUsage(copies[0].dst_offset, copies[0].size); + NoteGpuWrite(new_buffer); runtime.CopyBuffer(new_buffer, overlap, copies, true); #ifdef YUZU_LEGACY if (immediately_free) @@ -2115,6 +2239,7 @@ BufferId BufferCache

::CreateBuffer(DAddr device_addr, u32 wanted_size, slot_buffers.insert(runtime, overlap.begin, size, sparse_compatible); auto& new_buffer = slot_buffers[new_buffer_id]; const size_t size_bytes = new_buffer.SizeBytes(); + NoteGpuWrite(new_buffer); runtime.ClearBuffer(new_buffer, 0, size_bytes, 0); new_buffer.MarkUsage(0, size_bytes); for (const BufferId overlap_id : overlap.ids) { @@ -2147,7 +2272,11 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { } else { buffer_bytes -= Common::AlignUp(size, 1024); total_used_memory -= Common::AlignUp(size, 1024); - lru_cache.Free(buffer.getLRUID()); + // Only a registered buffer has an LRU item (freeing SIZE_MAX crashed in Free). + if (buffer.getLRUID() != SIZE_MAX) { + lru_cache.Free(buffer.getLRUID()); + buffer.setLRUID(SIZE_MAX); + } } const DAddr device_addr_begin = buffer.CpuAddr(); const DAddr device_addr_end = device_addr_begin + size; @@ -2156,7 +2285,8 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { for (u64 page = page_begin; page != page_end; ++page) { if constexpr (insert) { page_table[page] = buffer_id; - } else { + } else if (page_table[page] == buffer_id) { + // Only its own entries: another buffer's would go stale. page_table[page] = BufferId{}; } } @@ -2164,13 +2294,35 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { template void BufferCache

::TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept { - if (buffer_id != NULL_BUFFER_ID) { + if (buffer_id != NULL_BUFFER_ID && buffer.getLRUID() != SIZE_MAX) { lru_cache.Touch(buffer.getLRUID(), frame_tick); } } template bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size) { + // Most binds come back to a range nothing was written to since it was synchronized: skip + // walking its pages while its region saw no CPU writes (they took 5% of Mario Kart World's + // GPU back thread). + // EDEN_NX2_SYNC_CACHE=0 synchronizes every range every time. Android only: desktop + // synchronizes every range every time as before. +#ifdef __ANDROID__ + static const bool sync_cache = [] { + const char* value = std::getenv("EDEN_NX2_SYNC_CACHE"); + return value == nullptr || value[0] != '0'; + }(); +#else + constexpr bool sync_cache = false; +#endif + const u64 generation = sync_cache ? memory_tracker.RangeGeneration(device_addr, size) : 0; + SyncedRange& synced = synced_ranges[((device_addr >> 8) ^ size) % synced_ranges.size()]; + if (generation != 0 && synced.generation == generation && synced.device_addr == device_addr && + synced.size == size && synced.buffer == &buffer) { + return true; + } + if (generation != 0) { + synced = SyncedRange{&buffer, device_addr, size, generation}; + } #ifdef __ANDROID__ const bool use_optimized_vertex_buffers = Settings::values.use_optimized_vertex_buffers.GetValue(); #else @@ -2292,6 +2444,7 @@ void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, copy.src_offset += upload_staging.offset; } const bool can_reorder = runtime.CanReorderUpload(buffer, copies); + NoteGpuWrite(buffer); runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, can_reorder); } } @@ -2335,6 +2488,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ u8* const src_pointer = upload_staging.mapped_span.data(); std::memcpy(src_pointer, inlined_buffer.data(), copy_size); const bool can_reorder = runtime.CanReorderUpload(buffer, copies); + NoteGpuWrite(buffer); runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, can_reorder); } else { buffer.ImmediateUpload(buffer.Offset(dest_address), inlined_buffer.first(copy_size)); @@ -2394,34 +2548,76 @@ void BufferCache

::DownloadBuffersMemory(std::span buffer_ids) template void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 size) { + const std::array ranges{std::pair{device_addr, size}}; + DownloadBufferRanges(buffer, ranges); +} + +template +void BufferCache

::DownloadBufferRanges(Buffer& buffer, + std::span> ranges) { boost::container::small_vector copies; u64 total_size_bytes = 0; u64 largest_copy = 0; - memory_tracker.ForEachDownloadRangeAndClear( - device_addr, size, [&](u64 device_addr_out, u64 range_size) { - const DAddr buffer_addr = buffer.CpuAddr(); - const auto add_download = [&](DAddr start, DAddr end) { - const u64 new_offset = start - buffer_addr; - const u64 new_size = end - start; - copies.push_back(BufferCopy{ - .src_offset = new_offset, - .dst_offset = total_size_bytes, - .size = new_size, - }); - // Align up to avoid cache conflicts - constexpr u64 align = 64ULL; - constexpr u64 mask = ~(align - 1ULL); - total_size_bytes += (new_size + align - 1) & mask; - largest_copy = (std::max)(largest_copy, new_size); - }; - - gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, add_download); - ClearDownload(device_addr_out, range_size); - gpu_modified_ranges.Subtract(device_addr_out, range_size); - }); + const DAddr buffer_addr = buffer.CpuAddr(); + const DAddr buffer_end = buffer_addr + buffer.SizeBytes(); + for (const auto& [range_addr, range_size_in] : ranges) { + // Only this buffer's part: the tracker and gpu_modified_ranges are shared by every buffer, + // so a range past the buffer gave copies outside it (offsets past its size, or below its + // start) and cleared other buffers' GPU-written state without downloading it. + const DAddr range_begin = (std::max)(range_addr, buffer_addr); + const DAddr range_end = (std::min)(range_addr + range_size_in, buffer_end); + if (range_begin >= range_end) { + continue; + } + memory_tracker.ForEachDownloadRangeAndClear( + range_begin, range_end - range_begin, [&](u64 device_addr_out, u64 range_size) { + const auto add_download = [&](DAddr start, DAddr end) { + start = (std::max)(start, buffer_addr); + end = (std::min)(end, buffer_end); + if (start >= end) { + return; + } + const u64 new_offset = start - buffer_addr; + const u64 new_size = end - start; + copies.push_back(BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }); + // Align up to avoid cache conflicts + constexpr u64 align = 64ULL; + constexpr u64 mask = ~(align - 1ULL); + total_size_bytes += (new_size + align - 1) & mask; + largest_copy = (std::max)(largest_copy, new_size); + }; + + gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, add_download); + ClearDownload(device_addr_out, range_size); + gpu_modified_ranges.Subtract(device_addr_out, range_size); + }); + } if (total_size_bytes == 0) { return; } + if constexpr (requires { runtime.CanReadBuffersDirectly(); buffer.HostView(); }) { + // An integrated GPU's buffers are in the memory the host reads: wait for the GPU to + // complete the last command that wrote this buffer (often done already) and read it, + // instead of copying it out and waiting for all the GPU's work. Mario Kart World's CPU + // reads GPU-written data all the time, each such wait 17% of the GPU back thread. + const u64 tick = buffer.GpuWriteTick(); + if (tick != 0 && runtime.CanReadBuffersDirectly()) { + const std::span host = buffer.HostView(); + if (!host.empty()) { + runtime.WaitTick(tick); + buffer.InvalidateHostView(); + for (const BufferCopy& copy : copies) { + device_memory.WriteBlockUnsafe(buffer.CpuAddr() + copy.src_offset, + host.data() + copy.src_offset, copy.size); + } + return; + } + } + } if constexpr (USE_MEMORY_MAPS) { auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); @@ -2453,6 +2649,10 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 template void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { + if (buffer_id == inline_buffer_id) { + // The garbage collector can delete the inline index buffer; recreate it on next use. + inline_buffer_id = NULL_BUFFER_ID; + } if constexpr (requires { runtime.OnBufferDeleted(slot_buffers[buffer_id]); }) { runtime.OnBufferDeleted(slot_buffers[buffer_id]); } @@ -2473,6 +2673,14 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { channel_state->index_buffer.buffer_id = BufferId{}; dirty_index = true; } + // Multi-range storage segments too: the update pass resolves them again. + for (auto* const pool : {&graphics_segments, &compute_segments}) { + for (MultiRangeSegment& segment : *pool) { + if (segment.buffer_id == buffer_id) { + segment.buffer_id = BufferId{}; + } + } + } for (u32 index = 0; index < channel_state->vertex_buffers.size(); index++) { auto& binding = channel_state->vertex_buffers[index]; @@ -2488,6 +2696,7 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { replace(channel_state->compute_uniform_buffers); replace(channel_state->compute_storage_buffers); + synced_ranges.fill({}); // Mark the whole buffer as CPU written to stop tracking CPU writes if (!do_not_mark) { Buffer& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index ec442676d2..9c865c9212 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -35,6 +35,7 @@ #include "video_core/buffer_cache/buffer_base.h" #include "video_core/buffer_cache/virtual_range_cache.h" #include "video_core/control/channel_state_cache.h" +#include "video_core/nx2.h" #include "video_core/delayed_destruction_ring.h" #include "video_core/dirty_flags.h" #include "video_core/engines/maxwell_3d.h" @@ -80,6 +81,9 @@ enum class ObtainBufferOperation : u32 { MarkAsWritten = 1, DiscardWrite = 2, MarkQuery = 3, + /// The buffer is written on the GPU by what follows: record the write tick (a direct host + /// read waits for it) without marking the range modified or trapping its reads. + NoteGpuWrite = 4, }; static constexpr BufferId NULL_BUFFER_ID{0}; @@ -491,6 +495,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& pool); + bool SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size); void UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy, @@ -505,6 +511,12 @@ class BufferCache : public VideoCommon::ChannelSetupCaches> ranges); + /// Downloads the GPU-written data of several buffers with one wait for the GPU. void DownloadBuffersMemory(std::span buffer_ids); @@ -561,6 +573,16 @@ class BufferCache : public VideoCommon::ChannelSetupCaches upload_copies; + /// Ranges SynchronizeBuffer left with nothing to upload, at their region's CPU write + /// generation (direct mapped by address and size). + struct SyncedRange { + const Buffer* buffer; + DAddr device_addr; + u32 size; + u64 generation; + }; + std::array synced_ranges{}; + MemoryTracker memory_tracker; Common::RangeSet uncommitted_gpu_modified_ranges; @@ -570,7 +592,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches= 0) { return forced == 1; } - return gpu_memory == nullptr || !gpu_memory->UsesNativeTiling(); + return gpu_memory == nullptr || !gpu_memory->UsesNativeTiling() || + Tegra::Nx2NeedsBufferDownloads(); } /// Whether a GPU write to this range is copied back at the next fence. Ranges the guest CPU diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h index 8936da023f..b13e3577b4 100644 --- a/src/video_core/buffer_cache/memory_tracker_base.h +++ b/src/video_core/buffer_cache/memory_tracker_base.h @@ -63,6 +63,14 @@ class MemoryTrackerBase { }); } + /// Returns true if a region has pages CPU writes no longer fault on (written by the CPU and + /// not uploaded since, or written through the cached path) + [[nodiscard]] bool IsRegionUntracked(VAddr query_cpu_addr, u64 query_size) noexcept { + return IteratePages(query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { + return manager->IsRegionModified(Type::Untracked, offset, size); + }); + } + /// Returns true if a region has been marked as Preflushable [[nodiscard]] bool IsRegionPreflushable(VAddr query_cpu_addr, u64 query_size) noexcept { return IteratePages(query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { @@ -77,6 +85,18 @@ class MemoryTrackerBase { }); } + /// The CPU write generation of the region holding the range (see WordManager), or 0 when + /// the range spans regions or has none yet. A range synchronized at one generation has + /// nothing new to upload while its region's generation stays the same. + [[nodiscard]] u64 RangeGeneration(VAddr query_cpu_addr, u64 query_size) const noexcept { + const u64 page = query_cpu_addr >> HIGHER_PAGE_BITS; + if (query_size == 0 || page != (query_cpu_addr + query_size - 1) >> HIGHER_PAGE_BITS || + page >= NUM_HIGH_PAGES || top_tier[page] == nullptr) { + return 0; + } + return top_tier[page]->CpuGeneration(); + } + /// Unmark region as CPU modified, notifying the device_tracker about this change void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) { IteratePages(dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h index 7ba31c4717..9fd6496b20 100644 --- a/src/video_core/buffer_cache/uniform_stream_reuse.h +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -97,24 +97,38 @@ template } Reuse& reuse = UniformStreamReuseState(); typename Reuse::Slot& slot = reuse.slots[stage][index]; - u8* const scratch = reuse.scratch.data(); - device_memory.ReadBlockUnsafe(device_addr, scratch, size); + // The guest bytes are compared in place when they are contiguous in host memory (one + // read instead of a copy and a read; nearly every bind compares equal); otherwise through + // the scratch copy. EDEN_NX2_UBO_DIRECT=0 always copies first. + static const bool direct = [] { + const char* value = std::getenv("EDEN_NX2_UBO_DIRECT"); + return value == nullptr || value[0] != '0'; + }(); + const u8* bytes = direct ? device_memory.GetSpan(device_addr, size) : nullptr; + if (bytes == nullptr) { + u8* const scratch = reuse.scratch.data(); + device_memory.ReadBlockUnsafe(device_addr, scratch, size); + bytes = scratch; + } const bool same_bytes = - slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; + slot.size == size && std::memcmp(slot.bytes.data(), bytes, size) == 0; if (same_bytes && slot.tick == runtime.CurrentTick()) { runtime.RebindStreamUniform(slot.ref, size); return true; } + if (!same_bytes) { + // One snapshot, the slot's copy, which the stream is then written from: the guest may + // write the range meanwhile, and two reads of it could leave the stream and the copy + // (what the next bind compares against) different. + slot.size = size; + std::memcpy(slot.bytes.data(), bytes, size); + } const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - std::memcpy(span.data(), scratch, size); + std::memcpy(span.data(), slot.bytes.data(), size); slot.ref = runtime.LastStreamUniformRef(); // After the allocation: the tick the stream region is tagged with is the one current now. slot.tick = runtime.CurrentTick(); - if (!same_bytes) { - slot.size = size; - std::memcpy(slot.bytes.data(), scratch, size); - } return true; } diff --git a/src/video_core/buffer_cache/word_manager.h b/src/video_core/buffer_cache/word_manager.h index 76d608bbe0..05989d87c7 100644 --- a/src/video_core/buffer_cache/word_manager.h +++ b/src/video_core/buffer_cache/word_manager.h @@ -51,6 +51,10 @@ struct WordManager { } explicit WordManager() = default; + [[nodiscard]] u64 CpuGeneration() const noexcept { + return cpu_generation; + } + static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) { constexpr size_t number_bits = sizeof(u64) * 8; const size_t limit_page_end = number_bits - (std::min)(page_end, number_bits); @@ -116,6 +120,9 @@ struct WordManager { /// @param dirty_addr Base address to mark or unmark as modified /// @param size Size in bytes to mark or unmark as modified void ChangeRegionState(Type type, bool enable, u64 dirty_addr, u64 size) noexcept { + if (enable && type == Type::CPU) { + ++cpu_generation; + } std::span state_words = Span(type); [[maybe_unused]] std::span untracked_words = Span(Type::Untracked); [[maybe_unused]] std::span cached_words = Span(Type::CachedCPU); @@ -253,6 +260,7 @@ struct WordManager { CollectChangedRanges(false, word_index, untracked_words[word_index], cached_bits, ranges); untracked_words[word_index] |= cached_bits; cpu_words[word_index] |= cached_bits; + cpu_generation += cached_bits != 0 ? 1 : 0; cached_words[word_index] = 0; } if (!ranges.empty()) { @@ -322,6 +330,8 @@ struct WordManager { std::array heap = {}; DeviceTracker* tracker = nullptr; VAddr cpu_addr = 0; + /// Counts the times pages were marked CPU modified (MemoryTrackerBase::RangeGeneration). + u64 cpu_generation = 1; }; } // namespace VideoCommon diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 19319606bb..bc70afc74c 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -110,6 +110,9 @@ bool DmaPusher::Step() { auto& current_command = command_list.command_lists[dma_pushbuffer_subindex]; const CommandListHeader& header = current_command; dma_state.dma_get = header.addr; + if (command_resume_offset == 0) { + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::GpfifoEntries, 1); + } if (signal_sync && !synced) { std::unique_lock lk(sync_mutex); @@ -178,7 +181,11 @@ bool DmaPusher::Step() { // billions of instances and lost the device loading a race). bool gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); - if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && + // Only with the command list series on (nx2_command_lists): without it no page is ever + // marked, IsGpuWrittenCommands is always false, and the wait (for every compute dispatch + // or copy queued for the back) ran for every entry, ~130,000 times a second, for nothing. + if (!gpu_written && Tegra::g_nx2_ampere.load(std::memory_order_relaxed) && + Tegra::Nx2CommandLists() && VideoCommon::GpuSplit::Get().Active() && VideoCommon::GpuSplit::Get().WaitForWriters()) { gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); @@ -244,6 +251,9 @@ bool DmaPusher::Step() { } } parked_methods.clear(); + // A rejected block cannot continue past its submission; do not let its skip eat the + // headers of the next one. + dma_state.skip_words = 0; dma_pushbuffer.pop_front(); dma_pushbuffer_subindex = 0; } else { @@ -274,6 +284,16 @@ static const bool g_record_parsed_entries = std::getenv("EDEN_NX2_PB_CHECK") != size_t DmaPusher::ProcessCommands(std::span commands, size_t word_offset) { for (size_t index = 0; index < commands.size();) { + // Discard the rest of a command block that was rejected (illegal method). The block's + // declared count still tells where the next header starts, so the stream stays aligned + // instead of running thousands of bogus methods. + if (dma_state.skip_words != 0) { + const u32 skip = + static_cast(std::min(dma_state.skip_words, commands.size() - index)); + index += skip; + dma_state.skip_words -= skip; + continue; + } // Data word of methods command if (dma_state.method_count && dma_state.non_incrementing) { auto const& command_header = commands[index]; //must ref (MUltiMethod re) @@ -287,6 +307,38 @@ size_t DmaPusher::ProcessCommands(std::span commands, size_ dma_state.is_last_call = true; index += max_write; } else if (dma_state.method_count) { + // A whole incrementing run within the 3D engine's constant buffer offset and data + // registers goes to the split's back as one entry (Maxwell3D::LoadConstantRun), not + // a method call per word. EDEN_NX2_CB_BATCH_PB=0 turns it off. + static const bool cb_runs = [] { + const char* value = std::getenv("EDEN_NX2_CB_BATCH_PB"); + return value == nullptr || value[0] != '0'; + }(); + using Engines::Maxwell3D; + constexpr u32 cb_offset = MAXWELL3D_REG_INDEX(const_buffer.offset); + constexpr u32 cb_last = MAXWELL3D_REG_INDEX(const_buffer.buffer) + 15; + if (cb_runs && !dma_state.non_incrementing && !dma_increment_once && + dma_state.method >= cb_offset && dma_state.method <= cb_last && + dma_state.method_count <= Engines::Maxwell3D::MaxConstantRun && + dma_state.method_count <= cb_last - dma_state.method + 1 && + index + dma_state.method_count <= commands.size() && g_semaphore_watch == 0 && + subchannel_type[dma_state.subchannel] == Engines::EngineTypes::Maxwell3D && + subchannels[dma_state.subchannel] != nullptr) { + auto& split = VideoCommon::GpuSplit::Get(); + auto* const maxwell3d = + static_cast(subchannels[dma_state.subchannel]); + if (split.Active() && split.FrontMacros() && + maxwell3d->GetNativeMacroEngine().native_turing && + maxwell3d->FrontLoadConstantRun(system, split, dma_state.method, + &commands[index].argument, + dma_state.method_count)) { + index += dma_state.method_count; + dma_state.method += dma_state.method_count; + dma_state.method_count = 0; + dma_state.is_last_call = true; + continue; + } + } // Fast-path batching for consecutive non-execution methods in increasing mode if (!dma_state.non_incrementing && !dma_increment_once && dma_state.method >= non_puller_methods && @@ -428,6 +480,25 @@ bool DmaPusher::WantsSegment() const { } void DmaPusher::CallMethod(u32 argument) { + // Method 0x1 is illegal in the host class: a stream that reaches it has been misparsed (for + // example, a split method continued into a different command buffer). Discard the rest of + // the block using its declared count instead of running thousands of bogus methods against + // the engines; the next header then parses from an aligned position. + if (dma_state.method == static_cast(BufferMethods::Illegal)) { + const u32 remaining = dma_state.method_count > 0 ? dma_state.method_count - 1 : 0; + static std::atomic reported{}; + if (reported.fetch_add(1) < 8) { + LOG_ERROR(HW_GPU, + "Illegal host method on subchannel {}: discarding {} words of the block", + dma_state.subchannel, remaining); + } + dma_state.skip_words = remaining; + // The caller's decrement then ends the block. An inline header (no block active, + // method_count 0) must stay at 0: at 1 its next word would be read as data. + dma_state.method_count = dma_state.method_count > 0 ? 1 : 0; + dma_state.non_incrementing = false; + return; + } if (g_semaphore_watch != 0) [[unlikely]] { WatchArgument(dma_state.method, argument); } @@ -558,7 +629,8 @@ void DmaPusher::RefreshSnapshot(CommandList& list, u64 now) { } bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { - if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || !Tegra::Nx2CommandLists() || + size == 0) { return false; } for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { @@ -578,8 +650,8 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { // second in a Mario Kart World race, during which the back had nothing to do). Pages // FlushGpuWrittenCommands flushed whole and no GPU work wrote since are read from guest // memory. - if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || count == 0 || - rasterizer == nullptr) { + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || !Tegra::Nx2CommandLists() || + count == 0 || rasterizer == nullptr) { return false; } const GPUVAddr end = address + u64{count} * sizeof(u32); diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index a245c39819..78b26a5ee2 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -236,6 +236,7 @@ class DmaPusher final { u32 subchannel; ///< Current subchannel u32 method_count; ///< Current method count u32 length_pending; ///< Large NI command length pending + u32 skip_words; ///< Words left to discard from an invalid command block GPUVAddr dma_get; ///< Currently read segment u64 dma_word_offset; ///< Current word offset from address bool non_incrementing; ///< Current command's NI flag diff --git a/src/video_core/engines/engine_interface.h b/src/video_core/engines/engine_interface.h index 83dbf13059..fb73c584b6 100644 --- a/src/video_core/engines/engine_interface.h +++ b/src/video_core/engines/engine_interface.h @@ -37,6 +37,15 @@ class EngineInterface { /// Write multiple values to the register identified by method. virtual void CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) = 0; + /// A run of `count` incrementing methods from `first_method`, each with its word, as the + /// pushbuffer would call them one by one (GpuSplit::ConstantRun). Engines with a cheaper + /// equivalent override it. + virtual void LoadConstantRun(Core::System& system, u32 first_method, const u32* words, u32 count) { + for (u32 i = 0; i < count; ++i) { + CallMethod(system, first_method + i, words[i], i + 1 == count); + } + } + void ConsumeSink(Core::System& system) { if (!method_sink.empty()) { ConsumeSinkImpl(system); diff --git a/src/video_core/engines/fermi_2d.cpp b/src/video_core/engines/fermi_2d.cpp index 996661f8dd..a9efada227 100644 --- a/src/video_core/engines/fermi_2d.cpp +++ b/src/video_core/engines/fermi_2d.cpp @@ -40,7 +40,11 @@ void Fermi2D::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { } void Fermi2D::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < Regs::NUM_REGS, "Invalid Fermi2D register, increase the size of the Regs structure"); + if (method >= Regs::NUM_REGS) { + // Switch 2 games can send methods past this register file; don't write past it. + LOG_DEBUG(HW_GPU, "Ignoring Fermi2D method {:#x} past the register file", method); + return; + } regs.reg_array[method] = method_argument; if (method == FERMI2D_REG_INDEX(pixels_from_memory.src_y0) + 1) { @@ -56,7 +60,9 @@ void Fermi2D::CallMultiMethod(Core::System& system, u32 method, const u32* base_ void Fermi2D::ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { - regs.reg_array[method] = value; + if (method < Regs::NUM_REGS) { + regs.reg_array[method] = value; + } } method_sink.clear(); } diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 23ffbad790..d2af44f706 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -31,6 +31,9 @@ #ifdef ARCHITECTURE_x86_64 #include "video_core/macro_turing_jit.h" #endif +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) +#include "video_core/macro_turing_jit_arm64.h" +#endif #include "video_core/rasterizer_interface.h" #include "video_core/textures/decoders.h" @@ -180,8 +183,9 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar CallMethod(system, address, value, true); return true; }; -#ifdef ARCHITECTURE_x86_64 - // Compiled macro code, as for 3D macros (macro_turing_jit.h); EDEN_NX2_MME_JIT=0 off. +#if defined(ARCHITECTURE_x86_64) || (defined(ARCHITECTURE_arm64) && defined(__ANDROID__)) + // Compiled macro code, as for 3D macros (macro_turing_jit.h, macro_turing_jit_arm64.h); + // EDEN_NX2_MME_JIT=0 off. static const bool use_jit = [] { const char* value = std::getenv("EDEN_NX2_MME_JIT"); return value == nullptr || value[0] != '0'; @@ -192,6 +196,8 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar } native_macro->native_jit->Prepare(native_macro->native_code, native_macro->native_code_generation); + } + if (use_jit && !watched && native_macro->native_jit->Usable()) { const auto result = native_macro->native_jit->Execute( start, native_parameters, std::span{*native_macro->native_interpreter.active_dram}, @@ -622,7 +628,20 @@ constexpr u32 DLSS_CHECK_MISMATCH_PERCENT = 3; // shaders 32 threads wide. VideoCore::GameDlssReason KeplerCompute::GameDlssBlocker() const { using VideoCore::GameDlssReason; - static const bool forced_replacement = std::getenv("EDEN_NX2_DLSS_REPLACEMENT") != nullptr; + // On Android the replacement is the default: compiling the network's emulated tensor passes + // took Turnip from 3.1 to 7.6 GB in 4 s on a 24 GB phone and crashed it (Fire Emblem: + // Fortune's Weave). EDEN_NX2_GAME_DLSS=1 tries the game's network there. + static const bool forced_replacement = [] { + if (std::getenv("EDEN_NX2_DLSS_REPLACEMENT") != nullptr) { + return true; + } +#ifdef __ANDROID__ + const char* const game = std::getenv("EDEN_NX2_GAME_DLSS"); + return game == nullptr || game[0] != '1'; +#else + return false; +#endif + }(); if (forced_replacement) { return GameDlssReason::TurnedOff; } @@ -1251,7 +1270,7 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& fs) { -#ifdef ARCHITECTURE_x86_64 +#if defined(ARCHITECTURE_x86_64) || (defined(ARCHITECTURE_arm64) && defined(__ANDROID__)) const auto positions = FrontMacroPositions(fs); const u32 start = positions[((*fs.executing_macro - 0xe00) / 2) % positions.size()]; fs.executing_macro.reset(); @@ -1264,23 +1283,28 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s macro.native_jit = std::make_shared(); } macro.native_jit->Prepare(macro.native_code, macro.native_code_generation); - const auto result = macro.native_jit->Execute( - start, fs.parameters, std::span{*macro.native_interpreter.active_dram}, - [&fs](u32 address) { - return address < fs.registers.size() ? fs.registers[address] : 0U; - }, - [&](u32 address, u32 value) { - if (address >= 0xe00) { - return false; - } - FrontApply(fs, address, value); - split.MacroWrite(this, address, value); - return true; - }); + const auto read = [&fs](u32 address) { + return address < fs.registers.size() ? fs.registers[address] : 0U; + }; + const auto emit = [&](u32 address, u32 value) { + if (address >= 0xe00) { + return false; + } + FrontApply(fs, address, value); + split.MacroWrite(this, address, value); + return true; + }; + // As Maxwell3D::FrontRunMacro: interpreted when the ARM64 code is not used. + const bool jit = macro.native_jit->Usable(); + const auto result = + jit ? macro.native_jit->Execute(start, fs.parameters, + std::span{*macro.native_interpreter.active_dram}, + read, emit) + : macro.native_interpreter.Execute(macro.native_code, start, fs.parameters, read, emit); if (!result.success) { macro.native_execution_failed = true; - LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: compute start={:#x}, consumed={}, error={}", - start, result.consumed, result.error); + LOG_ERROR(HW_GPU, "Native MME ({}, split front) failed: compute start={:#x}, consumed={}, error={}", + jit ? "JIT" : "interpreter", start, result.consumed, result.error); } #else fs.executing_macro.reset(); diff --git a/src/video_core/engines/kepler_memory.cpp b/src/video_core/engines/kepler_memory.cpp index 3af5f91271..3c5cc76f36 100644 --- a/src/video_core/engines/kepler_memory.cpp +++ b/src/video_core/engines/kepler_memory.cpp @@ -30,13 +30,19 @@ void KeplerMemory::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { void KeplerMemory::ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { - regs.reg_array[method] = value; + if (method < Regs::NUM_REGS) { + regs.reg_array[method] = value; + } } method_sink.clear(); } void KeplerMemory::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < Regs::NUM_REGS, "Invalid KeplerMemory register, increase the size of the Regs structure"); + if (method >= Regs::NUM_REGS) { + // Switch 2 games can send methods past this register file; don't write past it. + LOG_DEBUG(HW_GPU, "Ignoring KeplerMemory method {:#x} past the register file", method); + return; + } regs.reg_array[method] = method_argument; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index bff0787bc9..dca140409d 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -28,6 +28,9 @@ #ifdef ARCHITECTURE_x86_64 #include "video_core/macro_turing_jit.h" #endif +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) +#include "video_core/macro_turing_jit_arm64.h" +#endif namespace Tegra::Engines { @@ -316,12 +319,22 @@ u32 Maxwell3D::ProcessShadowRam(u32 method, u32 argument) { void Maxwell3D::ConsumeSinkImpl(Core::System& system) { const auto control = shadow_state.shadow_ram_control; // Sunk methods are state writes only (not in execution_mask): skip unchanged values, as - // CallMethod does. + // CallMethod does. A deferred instanced draw is issued before the first write that is not + // part of its loop (ProcessDirtyRegisters); without one when the sink is consumed, none can + // appear while it is (only draw methods make one), so the check is made once. const bool skip_same = SkipSameValueWrites(); + const bool deferred_draw = draw_manager.HasDeferredDraw(); const auto process = [&](u32 method, u32 value) { - if (!skip_same || regs.reg_array[method] != value) { + if (skip_same && regs.reg_array[method] == value) { + return; + } + if (deferred_draw) [[unlikely]] { ProcessDirtyRegisters(method, value); + return; } + regs.reg_array[method] = value; + for (auto const& table : dirty.tables) + dirty.flags[table[method]] = true; }; if (control == Regs::ShadowRamControl::Track || control == Regs::ShadowRamControl::TrackWithFilter) { for (auto [method, value] : method_sink) { @@ -591,9 +604,16 @@ void Maxwell3D::ProcessMacroUpload(u32 data) { void Maxwell3D::ProcessMacroBind(u32 data) { const auto positions = GetNativeMacroPositions(); const u32 index = regs.load_mme.start_address_ptr++; - if (index < positions.size()) { - positions[index] = data; + if (index >= positions.size()) { + static std::atomic rejected{}; + if (rejected.fetch_add(1) < 8) { + LOG_ERROR(HW_GPU, + "Rejecting out-of-range macro bind: index={:#x}, entries={}, data={:#x}", + index, positions.size(), data); + } + return; } + positions[index] = data; } void Maxwell3D::ProcessFirmwareCall4() { @@ -845,6 +865,15 @@ struct Maxwell3D::FrontState { /// then can it have a deferred instanced draw to issue at the end of a macro. bool draw_pending{true}; bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once + /// Per bind group: the raw config and constant buffer size/address of the last bind sent; + /// a bind equal to it changes nothing on the back (ProcessCBBind) and is not sent. + std::array, 5> last_bind{}; +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) + /// Methods whose emits the macro JIT must always call back for (TuringMacroJit::Elision): + /// executable ones, the MME DMA reads (FrontDmaRead) and the macro range. + std::array jit_must_call{}; + TuringMacroJit::Elision elision{}; +#endif }; namespace { @@ -891,6 +920,51 @@ bool FrontHandledOutsideDrawManager(u32 method) { } } +// EDEN_NX2_JIT_SKIP_SAME=0: the ARM64 macro JIT calls back for every emit; the same-value +// writes are then skipped in FrontWrite as before (A/B switch for the in-JIT elision). +bool JitSkipSameEnabled() { + static const bool skip = [] { + const char* value = std::getenv("EDEN_NX2_JIT_SKIP_SAME"); + return value == nullptr || value[0] != '0'; + }(); + return skip; +} + +// EDEN_NX2_CB_BIND_SKIP=0: send a constant buffer bind even when its stage's bind group holds +// the same configuration, buffer address and size as the last bind sent (ProcessCBBind would +// set the same binding again). +bool CbBindSkipEnabled() { + static const bool skip = [] { + const char* value = std::getenv("EDEN_NX2_CB_BIND_SKIP"); + return value == nullptr || value[0] != '0'; + }(); + return skip; +} + +constexpr u32 BindGroupStride = MAXWELL3D_REG_INDEX(bind_groups[1].raw_config) - + MAXWELL3D_REG_INDEX(bind_groups[0].raw_config); + +// EDEN_NX2_CB_BATCH=0: a front macro's constant buffer data words (CB_DATA) go to the back one +// by one (CallMethod each) instead of as one multi-method entry per run. +bool CbBatchEnabled() { + static const bool batch = [] { + const char* value = std::getenv("EDEN_NX2_CB_BATCH"); + return value == nullptr || value[0] != '0'; + }(); + return batch; +} + +// EDEN_NX2_MACRO_SINK=0: the back runs every write a front macro makes through CallMethod at +// once (GpuSplit::MacroWrite), instead of putting state writes in its method sink like the +// pushbuffer's and applying them before the next executable method. +bool MacroSinkEnabled() { + static const bool sink = [] { + const char* value = std::getenv("EDEN_NX2_MACRO_SINK"); + return value == nullptr || value[0] != '0'; + }(); + return sink; +} + // EDEN_NX2_SPLIT_SKIP=0: send state writes that keep the register's value too. bool FrontSkipSame() { static const bool skip = [] { @@ -913,6 +987,16 @@ Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSp std::memcpy(front->shadow.data(), shadow_state.reg_array.data(), sizeof(front->shadow)); front->control = shadow_state.shadow_ram_control; front->skip_same = SkipSameValueWrites() && FrontSkipSame(); + for (auto& bind : front->last_bind) { + bind.fill(~u32{0}); + } +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) + for (u32 method = 0; method < front->jit_must_call.size(); ++method) { + const bool dma_read = method == 0x157 || method == 0x158; + front->jit_must_call[method] = + method >= Regs::NUM_REGS || execution_mask[method] || dma_read ? 1 : 0; + } +#endif front->executing_macro = executing_macro; front->parameters = std::move(macro_params); macro_params.clear(); @@ -925,6 +1009,21 @@ Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSp void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, u32 argument, bool is_last_call, bool from_macro) { namespace Profile = VideoCommon::Nx2Profile; + // A macro's constant buffer data: a run of words goes as one multi-method entry, which the + // back uploads with ProcessCBMultiData as it does the pushbuffer's runs: the same bytes at + // the same offsets, without touching the data register or its shadow copy (nor does the + // front then). Not in replay mode, where a word written is the shadow RAM's. + constexpr u32 cb_data_first = MAXWELL3D_REG_INDEX(const_buffer.buffer); + if (from_macro && method >= cb_data_first && method < cb_data_first + 16 && + fs.control != Regs::ShadowRamControl::Replay && CbBatchEnabled()) { + if (Profile::Enabled()) [[unlikely]] { + Profile::Add(Profile::Counter::MethodWrites, 1); + Profile::Add(Profile::Counter::MacroWrites, 1); + } + fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += sizeof(u32); + split.QueueWord(this, cb_data_first, argument); + return; + } // As ProcessShadowRam does on the back. u32 value = argument; bool shadow_kept = true; @@ -937,6 +1036,9 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } if (Profile::Enabled()) [[unlikely]] { Profile::Add(Profile::Counter::MethodWrites, 1); + if (from_macro) { + Profile::Add(Profile::Counter::MacroWrites, 1); + } } if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] { fs.control_writes[fs.control_count++ % fs.control_writes.size()] = { @@ -980,6 +1082,26 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met case MAXWELL3D_REG_INDEX(falcon[4]): fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1; break; + case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config): { + // ProcessCBBind sets the stage's slot from this configuration and the constant + // buffer registers: the same ones as last time set the binding it already has. + const size_t stage = (method - MAXWELL3D_REG_INDEX(bind_groups[0].raw_config)) / BindGroupStride; + const std::array bind{value, fs.regs[MAXWELL3D_REG_INDEX(const_buffer.size)], + fs.regs[MAXWELL3D_REG_INDEX(const_buffer.address_high)], + fs.regs[MAXWELL3D_REG_INDEX(const_buffer.address_low)]}; + if (fs.last_bind[stage] == bind && CbBindSkipEnabled()) { + if (Profile::Enabled()) [[unlikely]] { + Profile::Add(Profile::Counter::CbBindsSkipped, 1); + } + return; + } + fs.last_bind[stage] = bind; + break; + } default: if (method >= cb_data && method < cb_data + 16) { fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += sizeof(u32); @@ -989,9 +1111,12 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met break; } } - if (from_macro) { + if (from_macro && (execution_mask[method] || !MacroSinkEnabled())) { split.MacroWrite(this, method, argument); } else { + // A state write: the back sinks it and applies it (shadow RAM, same-value skip, dirty + // flags) before the next executable method, as CallMethod would have at once. The + // macro's end consumes the sink too (FinishFrontMacro). split.Method(this, method, argument, is_last_call, nullptr); } } @@ -1090,6 +1215,15 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit case MAXWELL3D_REG_INDEX(falcon[4]): fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1; break; + case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config): + // Bound on the back word by word, not through FrontWrite: forget the last bind. + fs.last_bind[(method - MAXWELL3D_REG_INDEX(bind_groups[0].raw_config)) / BindGroupStride] + .fill(~u32{0}); + break; default: if (!FrontHandledOutsideDrawManager(method)) { fs.draw_pending = true; @@ -1103,7 +1237,7 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& fs) { -#ifdef ARCHITECTURE_x86_64 +#if defined(ARCHITECTURE_x86_64) || (defined(ARCHITECTURE_arm64) && defined(__ANDROID__)) const u32 method = fs.executing_macro; fs.executing_macro = 0; auto& engine = GetNativeMacroEngine(); @@ -1117,24 +1251,64 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split } engine.native_jit->Prepare(engine.native_code, engine.native_code_generation); const size_t parameter_count = fs.parameters.size(); - const auto result = engine.native_jit->Execute( - start, fs.parameters, std::span{native_data_ram}, - [&fs](u32 address) { return address < Regs::NUM_REGS ? fs.regs[address] : 0U; }, - [&](u32 address, u32 value) { - if (address >= MacroRegistersStart) { - return false; - } - FrontWrite(split, fs, address, value, true, true); - // MME_DMA_READ (into data RAM) and MME_DMA_READ_FIFOED (into the load FIFO). - if (address == 0x157 || address == 0x158) { - FrontDmaRead(split, fs, address, value); - } - return true; - }); + const auto read = [&fs](u32 address) { + return address < Regs::NUM_REGS ? fs.regs[address] : 0U; + }; + const auto emit = [&](u32 address, u32 value) { + if (address >= MacroRegistersStart) { + return false; + } + FrontWrite(split, fs, address, value, true, true); + // MME_DMA_READ (into data RAM) and MME_DMA_READ_FIFOED (into the load FIFO). + if (address == 0x157 || address == 0x158) { + FrontDmaRead(split, fs, address, value); + } + return true; + }; + // The ARM64 code stops being used when it differs from the interpreter on a checked + // call (or could not be compiled): the front interprets the macros then. With macros + // on the front the back never runs them, so the interpreter is the front's. + const bool jit = engine.native_jit->Usable(); +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) + // Same-value state writes are elided inside the compiled code, as FrontWrite would skip + // them (never when the front sends them all, EDEN_NX2_SPLIT_SKIP=0). + if (jit && fs.skip_same && JitSkipSameEnabled()) { + fs.elision = TuringMacroJit::Elision{ + .must_call = fs.jit_must_call.data(), + .regs = fs.regs.data(), + .shadow = fs.shadow.data(), + .control = reinterpret_cast(&fs.control), + .skipped = 0, + }; + engine.native_jit->SetElision(&fs.elision); + } else { + engine.native_jit->SetElision(nullptr); + } +#endif + const auto result = + jit ? engine.native_jit->Execute(start, fs.parameters, std::span{native_data_ram}, read, + emit) + : engine.native_interpreter.Execute(engine.native_code, start, fs.parameters, read, emit); +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) + // Not left behind: the compute engine's front macros can run through the same JIT, and + // must never elide against this engine's registers. + engine.native_jit->SetElision(nullptr); + if (const u32 skipped = fs.elision.skipped; skipped != 0) { + fs.elision.skipped = 0; + if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] { + namespace Profile = VideoCommon::Nx2Profile; + Profile::Add(Profile::Counter::MethodWrites, skipped); + Profile::Add(Profile::Counter::MacroWrites, skipped); + Profile::Add(Profile::Counter::SameValueWrites, skipped); + Profile::Add(Profile::Counter::JitSkippedWrites, skipped); + } + } +#endif if (!result.success) { engine.native_execution_failed = true; - LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: start={:#x}, consumed={}/{}, error={}", - start, result.consumed, parameter_count, result.error); + LOG_ERROR(HW_GPU, "Native MME ({}, split front) failed: start={:#x}, consumed={}/{}, error={}", + jit ? "JIT" : "interpreter", start, result.consumed, parameter_count, + result.error); } } fs.parameters.clear(); @@ -1155,7 +1329,7 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m u32 count) { // The words may be written by methods still queued for the back (indirect draw arguments // from a culling dispatch), and the caches that may hold newer ones belong to it. - split.Drain(); + split.Drain(VideoCommon::Nx2Profile::Zone::SplitDmaRead); const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155]; const u32 ram = fs.regs[0x156]; const u32 words = std::min(count, 0x1000); @@ -1234,6 +1408,62 @@ void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) { } } +bool Maxwell3D::FrontLoadConstantRun(Core::System& system, VideoCommon::GpuSplit& split, + u32 first_method, const u32* words, u32 count) { + namespace Profile = VideoCommon::Nx2Profile; + constexpr u32 cb_offset = MAXWELL3D_REG_INDEX(const_buffer.offset); + auto& fs = Front(system, split); + if (fs.control == Regs::ShadowRamControl::Replay) { + return false; // A word written is then the shadow RAM's: word by word (FrontWrite) + } + const bool tracked = fs.control == Regs::ShadowRamControl::Track || + fs.control == Regs::ShadowRamControl::TrackWithFilter; + // What FrontWrite mirrors for each word: the shadow RAM, the register, the offset the data + // words advance. The back makes the same changes in LoadConstantRun. + for (u32 i = 0; i < count; ++i) { + const u32 method = first_method + i; + const u32 word = words[i]; + if (tracked) { + fs.shadow[method] = word; + } + fs.regs[method] = word; + if (method != cb_offset) { + fs.regs[cb_offset] += sizeof(u32); + } + } + if (Profile::Enabled()) [[unlikely]] { + Profile::Add(Profile::Counter::MethodWrites, count); + } + split.ConstantRun(this, first_method, words, count); + return true; +} + +void Maxwell3D::LoadConstantRun(Core::System& system, u32 first_method, const u32* words, + u32 count) { + constexpr u32 cb_offset = MAXWELL3D_REG_INDEX(const_buffer.offset); + u32 index = 0; + if (first_method == cb_offset) { + // The offset: a state write, as CallMethod applies it (shadow RAM, same-value skip, + // dirty flags). + CallMethod(system, cb_offset, words[0], count == 1); + index = 1; + } + const u32 data_count = count - index; + if (data_count == 0) { + return; + } + // The data words: the shadow RAM, register and dirty flags per word as CallMethod makes + // them, then one upload from the offset (ProcessCBData per word advances it the same). + std::array values; + for (u32 i = 0; i < data_count; ++i) { + const u32 method = first_method + index + i; + const u32 value = ProcessShadowRam(method, words[index + i]); + ProcessDirtyRegisters(method, value); + values[i] = value; + } + ProcessCBMultiData(values.data(), data_count); +} + void Maxwell3D::FinishFrontMacro(Core::System& system) { // What ProcessMacro and CallMacroMethod do around the macro. ConsumeSink(system); diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 7646ddbbe3..f1660db464 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3202,6 +3202,15 @@ class Maxwell3D final : public EngineInterface { /// Write multiple values to the register identified by method. void CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) override; + /// A pushbuffer run from the constant buffer offset or data registers: the offset write as + /// CallMethod makes it, the data words' register and shadow RAM effects, one upload. + void LoadConstantRun(Core::System& system, u32 first_method, const u32* words, u32 count) override; + /// Split front: mirrors such a run and sends it as one entry. False (nothing done) when it + /// must go word by word (shadow RAM replay mode). + bool FrontLoadConstantRun(Core::System& system, VideoCommon::GpuSplit& split, u32 first_method, + const u32* words, u32 count); + static constexpr u32 MaxConstantRun = 32; + /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by /// the front instead of CallMethod: runs macros and macro uploads against a copy of the /// registers and sends the register writes to the back thread. diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index 0b0541799b..875ff2af9e 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -37,7 +37,9 @@ void MaxwellDMA::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { void MaxwellDMA::ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { - regs.reg_array[method] = value; + if (method < NUM_REGS) { + regs.reg_array[method] = value; + } } method_sink.clear(); } diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 0ac798567f..2bb967f4ba 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -25,6 +25,7 @@ #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/rasterizer_interface.h" +#include "video_core/semaphore_ledger.h" namespace Tegra::Engines { @@ -59,6 +60,69 @@ void DrainBack() { VideoCommon::GpuSplit::Get().Drain(); } +using Ledger = Nx2SemaphoreLedger; + +// Records a 32-bit puller release the front just queued for the back (semaphore_ledger.h). +void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload) { + if (!Ledger::Enabled()) { + return; + } + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address)) { + Ledger::Get().NoteRelease(*device, payload, VideoCommon::GpuSplit::Get().WritePosition()); + } +} + +void ForgetRelease(DmaPusher& dma_pusher, GPUVAddr address) { + if (!Ledger::Enabled()) { + return; + } + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address)) { + Ledger::Get().Forget(*device); + } +} + +// Whether an acquire is met without draining the split's back (semaphore_ledger.h): the last +// puller release queued for the word meets it and is still ahead of the back in the ring, or +// guest memory already meets it and no queued puller release moves the word away. The word is +// read without flushing the caches, which belong to the back: a value they hold but memory +// does not yet is newer, and sync words count up, so memory meeting the acquire is enough. +bool AcquireMetWithoutDrain(DmaPusher& dma_pusher, GPUVAddr address, u32 value, + Ledger::Compare compare) { + auto& split = VideoCommon::GpuSplit::Get(); + if (!Ledger::Enabled() || !split.Active()) { + return false; + } + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + if (!device) { + return false; + } + if (const Ledger::Entry* entry = Ledger::Get().Find(*device)) { + const bool entry_met = Ledger::Met(compare, entry->value, value); + const bool queued = entry->order > split.Consumed(); + if (queued) { + // The word will hold this release's value when the back passes the acquire. + return entry_met; + } + } + return Ledger::Met(compare, dma_pusher.memory_manager.Read(address), value); +} + +Ledger::Compare CompareOf(u32 operation) { + // Puller GpuSemaphoreOperation values (PendingAcquire::Matches). + switch (operation) { + case 0x1: + return Ledger::Compare::Equal; + case 0x4: + return Ledger::Compare::Gequal; + default: + return Ledger::Compare::AndNonZero; + } +} + +void CountLedgerAcquire() { + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireLedger, 1); +} + } // Anonymous namespace bool Puller::PendingAcquire::Matches(u32 current) const { @@ -78,6 +142,11 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (!pending_acquire) { return true; } + if (AcquireMetWithoutDrain(dma_pusher, pending_acquire->address, pending_acquire->value, + CompareOf(pending_acquire->operation))) { + CountLedgerAcquire(); + return true; + } // The release may still be queued for the split GPU thread's back half (and the caches // ReadSemaphoreWord asks are its state). DrainBack(); @@ -233,6 +302,10 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0); } }); + NoteRelease(dma_pusher, sequence_address, payload); + if (mirror != 0) { + NoteRelease(dma_pusher, mirror, payload); + } } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -250,6 +323,11 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + if (AcquireMetWithoutDrain(dma_pusher, acquire.address, acquire.value, + CompareOf(acquire.operation))) { + CountLedgerAcquire(); + return; + } DrainBack(); if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { return; @@ -307,12 +385,18 @@ void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + NoteRelease(dma_pusher, sequence_address, payload); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { + const auto value = regs.semaphore_acquire; + if (AcquireMetWithoutDrain(dma_pusher, regs.semaphore_address.SemaphoreAddress(), value, + Ledger::Compare::Equal)) { + CountLedgerAcquire(); + return; + } DrainBack(); u32 word = dma_pusher.memory_manager.Read(regs.semaphore_address.SemaphoreAddress()); - const auto value = regs.semaphore_acquire; while (word != value) { regs.acquire_active = true; regs.acquire_value = value; @@ -331,13 +415,35 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { (regs.reg_array[0x17] & ~3U); const u32 payload = regs.reg_array[0x19]; const u32 operation = argument & 7; + // PAYLOAD_SIZE (bit 24): a 64-bit payload (SEM_PAYLOAD_HI, 0x1a). Only the low word is + // written and compared here; the ledger stays out of such words. + const bool payload_64 = (argument & (1U << 24)) != 0; if (operation == 1) { // RELEASE OnBack([rasterizer = dma_pusher.rasterizer, address, payload] { rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + if (payload_64) { + ForgetRelease(dma_pusher, address); + } else { + NoteRelease(dma_pusher, address, payload); + } return; } + if (operation == 6) { // REDUCTION: the word's value depends on what it held + ForgetRelease(dma_pusher, address); + } + if (!payload_64 && operation <= 5 && operation != 1) { + const Ledger::Compare compare = operation == 0 ? Ledger::Compare::Equal + : operation == 2 ? Ledger::Compare::Gequal + : operation == 3 ? Ledger::Compare::CircularGequal + : operation == 4 ? Ledger::Compare::AndNonZero + : Ledger::Compare::Nor; + if (AcquireMetWithoutDrain(dma_pusher, address, payload, compare)) { + CountLedgerAcquire(); + return; + } + } // Acquires: check once after flushing pending fences; never block the GPU thread on them. DrainBack(); dma_pusher.rasterizer->ReleaseFences(); diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 6dbabcb43a..d6e82a1d95 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -35,6 +35,7 @@ #include "video_core/host1x/host1x.h" #include "video_core/host1x/syncpoint_manager.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/renderer_base.h" #include "video_core/shader_notify.h" @@ -114,7 +115,11 @@ struct GPU::Impl { /// Synchronizes CPU writes with Host GPU memory. void InvalidateGPUCache() { std::function callback_writes([this](PAddr address, size_t size) { - renderer->ReadRasterizer()->OnCacheInvalidation(address, size); + if (Tegra::Nx2HeldWrites()) { + renderer->ReadRasterizer()->OnCachedWrite(address, size); + } else { + renderer->ReadRasterizer()->OnCacheInvalidation(address, size); + } }); system.GatherGPUDirtyMemory(callback_writes); } @@ -131,6 +136,7 @@ struct GPU::Impl { std::unique_lock lck{sync_request_mutex}; const u64 fence = ++last_sync_fence; sync_requests.emplace_back(std::forward(action)); + has_sync_requests.store(true, std::memory_order_release); return fence; } @@ -152,7 +158,12 @@ struct GPU::Impl { /// Tick pending requests within the GPU. void TickWork() { + // Runs at every draw: skip the lock while nothing is queued. + if (!has_sync_requests.load(std::memory_order_acquire)) { + return; + } std::unique_lock lck{sync_request_mutex}; + has_sync_requests.store(false, std::memory_order_relaxed); while (!sync_requests.empty()) { auto request = std::move(sync_requests.front()); sync_requests.pop_front(); @@ -356,6 +367,7 @@ struct GPU::Impl { std::condition_variable sync_cv; std::list> sync_requests; + std::atomic has_sync_requests{}; std::atomic current_sync_fence{}; u64 last_sync_fence{}; std::mutex sync_request_mutex; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 07ac984222..c55c586e17 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -2,7 +2,10 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include +#include +#include #include +#include #include #ifdef _MSC_VER @@ -13,6 +16,16 @@ #define NOMINMAX #endif #include +#pragma comment(lib, "Synchronization.lib") +#elif defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#include #endif #include @@ -22,6 +35,7 @@ #include "core/core.h" #include "video_core/engines/engine_interface.h" #include "video_core/gpu_split.h" +#include "video_core/semaphore_ledger.h" #include "video_core/nx2.h" #include "video_core/nx2_profile.h" @@ -41,6 +55,56 @@ void Pause() { #endif } +// Sleeps until `word` moves on from `seen`, for at most `timeout`: a lost wake-up costs one +// timeout instead of a hang. Raw futexes: libc++ skips the wake syscall when its own waiter count +// for the address is zero, so its notify can't pair with these waits. +void WaitWord(std::atomic& word, u32 seen, std::chrono::microseconds timeout) { + static_assert(sizeof(std::atomic) == sizeof(u32)); +#if defined(_WIN32) + WaitOnAddress(&word, &seen, sizeof(seen), + static_cast(std::max(timeout.count() / 1000, 1))); +#elif defined(__linux__) + timespec ts{}; + ts.tv_sec = static_cast(timeout.count() / 1'000'000); + ts.tv_nsec = static_cast(timeout.count() % 1'000'000) * 1000; + syscall(SYS_futex, reinterpret_cast(&word), FUTEX_WAIT_PRIVATE, seen, &ts, nullptr, 0); +#else + (void)word; + (void)seen; + (void)timeout; + std::this_thread::yield(); +#endif +} + +void WakeWord(std::atomic& word) { +#if defined(_WIN32) + WakeByAddressAll(&word); +#elif defined(__linux__) + syscall(SYS_futex, reinterpret_cast(&word), FUTEX_WAKE_PRIVATE, INT_MAX, nullptr, nullptr, + 0); +#else + (void)word; +#endif +} + +// Phones sleep in the waits below on futexes: spinning or yielding, the front kept a big core +// busy and the phone hot. The back parks on a raw futex too (libc++'s std::atomic::wait can miss +// a notify). Desktop keeps the spin, yield and std::atomic waits. +#ifdef __ANDROID__ +constexpr bool SleepingWaits = true; +#else +constexpr bool SleepingWaits = false; +#endif + +// Wakes the back thread parked on `wake`. +void WakeBack(std::atomic& wake) { + if constexpr (SleepingWaits) { + WakeWord(wake); + } else { + wake.notify_all(); + } +} + [[maybe_unused]] std::atomic g_back_thread_id{}; std::atomic g_back_busy{}; @@ -77,6 +141,48 @@ void GpuSplit::CheckOwner(const char* where) { reinterpret_cast(module))); } std::fflush(stderr); +#elif defined(__linux__) + if (!g_back_busy.load(std::memory_order_acquire)) { + return; + } + const u32 thread = static_cast(gettid()); + if (thread == g_back_thread_id.load(std::memory_order_relaxed)) { + return; + } + static std::atomic reports{}; + if (reports.fetch_add(1) >= 12) { + return; + } + char name[16] = "?"; + pthread_getname_np(pthread_self(), name, sizeof(name)); + struct Frames { + std::array pcs{}; + size_t count{}; + } frames; + _Unwind_Backtrace( + [](_Unwind_Context* context, void* arg) { + auto& f = *static_cast(arg); + if (f.count == f.pcs.size()) { + return _URC_END_OF_STACK; + } + f.pcs[f.count++] = reinterpret_cast(_Unwind_GetIP(context)); + return _URC_NO_REASON; + }, + &frames); + std::string stack; + for (size_t i = 1; i < frames.count; ++i) { + Dl_info info{}; + const auto pc = reinterpret_cast(frames.pcs[i]); + if (dladdr(frames.pcs[i], &info) != 0 && info.dli_fname != nullptr) { + const char* base = std::strrchr(info.dli_fname, '/'); + stack += fmt::format(" {}+{:#x}", base ? base + 1 : info.dli_fname, + pc - reinterpret_cast(info.dli_fbase)); + } else { + stack += fmt::format(" {:#x}", pc); + } + } + LOG_ERROR(HW_GPU, "GPU split owner check: {} on thread {} ({}) while the back runs:{}", where, + thread, name, stack); #else (void)where; #endif @@ -114,8 +220,9 @@ void GpuSplit::MaybeStart(Core::System& system_) { ring = std::make_unique(RingWords); back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); }); active = true; -#ifdef ARCHITECTURE_x86_64 - // Front macros run compiled (macro_turing_jit.h) only. +#if defined(ARCHITECTURE_x86_64) || (defined(ARCHITECTURE_arm64) && defined(__ANDROID__)) + // Front macros run compiled (macro_turing_jit.h, macro_turing_jit_arm64.h), or interpreted + // where the ARM64 code is not used. const char* jit = std::getenv("EDEN_NX2_MME_JIT"); front_macros = mode == '2' && (jit == nullptr || jit[0] != '0'); #endif @@ -129,7 +236,7 @@ void GpuSplit::Stop() { } back_thread.request_stop(); wake.fetch_add(1, std::memory_order_seq_cst); - wake.notify_one(); + WakeBack(wake); back_thread.join(); active = false; // The split is static and outlives the emulation session. The next session's back thread @@ -145,9 +252,13 @@ void GpuSplit::Stop() { front_engine = nullptr; back_engine = nullptr; front_macros = false; + pending_count = 0; + // Its entries carry ring positions (orders) of this session: stale once they restart. + Tegra::Nx2SemaphoreLedger::Get().Clear(); published.store(0, std::memory_order_relaxed); consumed.store(0, std::memory_order_relaxed); back_waiting.store(false, std::memory_order_relaxed); + front_waiting.store(false, std::memory_order_relaxed); } u64* GpuSplit::ReserveSlow(size_t words) { @@ -178,12 +289,51 @@ void GpuSplit::WaitForSpace(size_t words) { if (spins < 4096) { Pause(); } else { - std::this_thread::yield(); + SleepOnBack(); } } } +void GpuSplit::RequestIdleHook() { + hook_requested.store(true, std::memory_order_seq_cst); + wake.fetch_add(1, std::memory_order_seq_cst); + WakeBack(wake); +} + +void GpuSplit::SleepOnBack() { + // Re-wake a back thread whose wake-up was lost (it froze Mario Kart World on Android with the + // front waiting here and the back parked). + if (back_waiting.load(std::memory_order_seq_cst)) { + wake.fetch_add(1, std::memory_order_seq_cst); + WakeBack(wake); + } + if constexpr (!SleepingWaits) { + std::this_thread::yield(); + return; + } + // The back bumps `drained` after storing `consumed` when it sees front_waiting: read it + // before the last look at `consumed`, so a store in between ends the wait at once. + front_waiting.store(true, std::memory_order_seq_cst); + const u32 seen = drained.load(std::memory_order_seq_cst); + if (consumed.load(std::memory_order_seq_cst) == consumed_cache) { + WaitWord(drained, seen, std::chrono::milliseconds{2}); + } + front_waiting.store(false, std::memory_order_relaxed); +} + +void GpuSplit::FlushPending() { + const u32 count = pending_count; + if (count == 0) { + return; + } + pending_count = 0; + Nx2Profile::Add(Nx2Profile::Counter::CbBatchWords, count); + Nx2Profile::Add(Nx2Profile::Counter::CbBatchEntries, 1); + MultiMethod(pending_engine, pending_method, pending_words.data(), count, count, nullptr); +} + void GpuSplit::Publish() { + FlushPending(); if (published_local == write_position) { return; } @@ -192,17 +342,23 @@ void GpuSplit::Publish() { published.store(write_position, std::memory_order_seq_cst); if (back_waiting.load(std::memory_order_seq_cst)) { wake.fetch_add(1, std::memory_order_seq_cst); - wake.notify_one(); + WakeBack(wake); } } void GpuSplit::Drain() { + Drain(Nx2Profile::Zone::SplitDrain); +} + +void GpuSplit::Drain(Nx2Profile::Zone zone) { + FlushPending(); consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache == write_position && BackQuiet()) { return; } Publish(); - Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain}; + Nx2Profile::Scope scope{zone}; + auto next_report = std::chrono::steady_clock::now() + std::chrono::seconds{5}; // Everything consumed is not enough: the back submits deferred work (Scheduler::DeferFlush) // from its idle hook afterwards, and the caller may use the scheduler once this returns. bool hook_asked = false; @@ -218,15 +374,28 @@ void GpuSplit::Drain() { hook_asked = true; } } - if (spins < 16384) { + if (spins < (SleepingWaits ? 8192U : 16384U)) { Pause(); } else { - std::this_thread::yield(); + // Sleep instead of spinning on: the front spent most of each frame here, keeping a big + // core busy and the phone hot. + SleepOnBack(); + if ((SleepingWaits || (spins & 1023) == 0) && + std::chrono::steady_clock::now() >= next_report) { + next_report += std::chrono::seconds{5}; + LOG_WARNING(HW_GPU, + "GPU split drain waiting on the back thread: {} of {} words consumed, " + "back {} ({})", + consumed_cache, write_position, + back_waiting.load() ? "parked" : "running", + g_back_busy.load() ? "inside a command" : "between batches"); + } } } } bool GpuSplit::WaitForWriters() { + FlushPending(); const u64 target = std::min(last_writer_position, write_position); consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache >= target) { @@ -234,6 +403,7 @@ bool GpuSplit::WaitForWriters() { } wait_target.store(target, std::memory_order_seq_cst); Publish(); + Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWriters}; for (u32 spins = 0;; ++spins) { consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache >= target) { @@ -243,11 +413,20 @@ bool GpuSplit::WaitForWriters() { if (spins < 16384) { Pause(); } else { - std::this_thread::yield(); + SleepOnBack(); } } } +void GpuSplit::HandBack(u64 position) { + // Seq_cst store and load against SleepOnBack's: one of the two sides sees the other. + consumed.store(position, std::memory_order_seq_cst); + if (front_waiting.load(std::memory_order_seq_cst)) { + drained.fetch_add(1, std::memory_order_seq_cst); + WakeWord(drained); + } +} + void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) { front_engine = engine; u64* const entry = Reserve(2); @@ -259,6 +438,7 @@ void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) { void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, u32 amount, u32 methods_pending, const GPUVAddr* segment) { SelectEngine(engine); + Nx2Profile::Add(Nx2Profile::Counter::MultiWords, amount); // Long runs (constant buffer and inline uploads) go in pieces, as when a method's data // continues in the next GPFIFO entry. while (amount > 0) { @@ -281,6 +461,20 @@ void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, } } +void GpuSplit::ConstantRun(Tegra::Engines::EngineInterface* engine, u32 first_method, + const u32* words, u32 count) { + SelectEngine(engine); + const size_t size = 1 + (size_t{count} + 1) / 2; + u64* const entry = Reserve(size); + entry[0] = static_cast(Kind::ConstantRun) | (u64{first_method & 0xffff} << 8) | + (u64{count} << 32); + entry[size - 1] = 0; + std::memcpy(entry + 1, words, size_t{count} * sizeof(u32)); + Commit(size); + Nx2Profile::Add(Nx2Profile::Counter::CbRunWords, count); + Nx2Profile::Add(Nx2Profile::Counter::CbRunEntries, 1); +} + void GpuSplit::DeferFunction(std::function&& func) { auto* const heap = new std::function(std::move(func)); Defer([heap] { @@ -302,13 +496,18 @@ size_t GpuSplit::Execute(const u64* entry) { const u32 argument = static_cast(head >> 32); auto* const engine = back_engine; if ((head & Direct) != 0) { - Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method}; + Nx2Profile::SampledScope<4> method_scope{Nx2Profile::Zone::Method}; + Nx2Profile::Add(Nx2Profile::Counter::BackMethodCalls, 1); + Nx2Profile::CountMethod(method); engine->ConsumeSink(*system); engine->CallMethod(*system, method, argument, true); } else if (!engine->execution_mask[method]) { + Nx2Profile::Add(Nx2Profile::Counter::BackSinkWrites, 1); engine->method_sink.emplace_back(method, argument); } else { - Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method}; + Nx2Profile::SampledScope<4> method_scope{Nx2Profile::Zone::Method}; + Nx2Profile::Add(Nx2Profile::Counter::BackMethodCalls, 1); + Nx2Profile::CountMethod(method); engine->ConsumeSink(*system); if ((head & HasSegment) != 0) { engine->current_dma_segment = entry[1]; @@ -321,8 +520,9 @@ size_t GpuSplit::Execute(const u64* entry) { const u32 method = static_cast((head >> 8) & 0xffff); const u32 amount = static_cast(head >> 32); const size_t data = (head & HasSegment) != 0 ? 3 : 2; + Nx2Profile::CountMethod(method, amount); auto* const engine = back_engine; - Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method}; + Nx2Profile::SampledScope<4> method_scope{Nx2Profile::Zone::Method}; engine->ConsumeSink(*system); if ((head & HasSegment) != 0) { engine->current_dma_segment = entry[2]; @@ -331,8 +531,19 @@ size_t GpuSplit::Execute(const u64* entry) { amount, static_cast(entry[1])); return data + (amount + 1) / 2; } + case Kind::ConstantRun: { + const u32 method = static_cast((head >> 8) & 0xffff); + const u32 count = static_cast(head >> 32); + auto* const engine = back_engine; + Nx2Profile::SampledScope<4> method_scope{Nx2Profile::Zone::Method}; + Nx2Profile::CountMethod(method, count); + engine->ConsumeSink(*system); + engine->LoadConstantRun(*system, method, reinterpret_cast(entry + 1), count); + return 1 + (size_t{count} + 1) / 2; + } case Kind::Call: { const size_t payload = static_cast((head >> 8) & 0xffffff); + Nx2Profile::Add(Nx2Profile::Counter::BackDeferredCalls, 1); reinterpret_cast(entry[1])(entry + 2); return 2 + payload; } @@ -349,6 +560,8 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { system->RegisterHostThread(); #ifdef _WIN32 g_back_thread_id.store(GetCurrentThreadId(), std::memory_order_relaxed); +#elif defined(__linux__) + g_back_thread_id.store(static_cast(gettid()), std::memory_order_relaxed); #endif u64 read_position = 0; while (!stop_token.stop_requested()) { @@ -384,7 +597,13 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { // A request made after the hook ran (RequestIdleHook) runs it before sleeping. if (available == read_position && !stop_token.stop_requested() && !hook_requested.load(std::memory_order_seq_cst)) { - wake.wait(seen, std::memory_order_seq_cst); + if constexpr (SleepingWaits) { + // A futex, not std::atomic::wait: libc++ spins with sched_yield first, + // and parks waiters in a shared table where a notify could miss this one. + WaitWord(wake, seen, std::chrono::milliseconds{2}); + } else { + wake.wait(seen, std::memory_order_seq_cst); + } } back_waiting.store(false, std::memory_order_relaxed); continue; @@ -408,12 +627,12 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { if ((read_position - last_store >= 4096 || read_position >= wait_target.load(std::memory_order_relaxed)) && read_position != available) { - consumed.store(read_position, std::memory_order_release); + HandBack(read_position); last_store = read_position; } } g_back_busy.store(false, std::memory_order_release); - consumed.store(read_position, std::memory_order_release); + HandBack(read_position); } back_running.store(false, std::memory_order_release); } diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 836c46c795..1187d5b331 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -20,6 +21,10 @@ namespace Tegra::Engines { class EngineInterface; } +namespace VideoCommon::Nx2Profile { +enum class Zone : u32; +} + namespace VideoCommon { // Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the @@ -69,11 +74,7 @@ class GpuSplit { return back_running.load(std::memory_order_acquire); } /// Any thread: has the back call the idle hook soon, between entries or at once if it sleeps. - void RequestIdleHook() { - hook_requested.store(true, std::memory_order_seq_cst); - wake.fetch_add(1, std::memory_order_seq_cst); - wake.notify_one(); - } + void RequestIdleHook(); /// Starts the back thread once the split is wanted (EDEN_NX2_SPLIT=1, a Switch 2 title, /// commands read without cache flushes). Called by the front between commands. @@ -97,6 +98,25 @@ class GpuSplit { } void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, u32 amount, u32 methods_pending, const GPUVAddr* segment); + /// A word of a run of one engine method (a macro's constant buffer data): goes with the + /// words queued before it as one multi-method entry, sent before anything else enters the + /// ring (Reserve, Publish, Drain and WaitForWriters flush it), so the order is kept. + void QueueWord(Tegra::Engines::EngineInterface* engine, u32 method, u32 word) { + if (pending_count != 0 && (engine != pending_engine || method != pending_method || + pending_count == PendingWords)) [[unlikely]] { + FlushPending(); + } + if (pending_count == 0) { + pending_engine = engine; + pending_method = method; + } + pending_words[pending_count++] = word; + } + void FlushPending(); + /// A pushbuffer run of `count` incrementing methods from `first_method` within the 3D + /// engine's constant buffer offset and data registers (EngineInterface::LoadConstantRun). + void ConstantRun(Tegra::Engines::EngineInterface* engine, u32 first_method, const u32* words, + u32 count); /// A write made by a macro: CallMethod without the method sink, as the engines' macros do. void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument) { SelectEngine(engine); @@ -126,8 +146,21 @@ class GpuSplit { /// Makes what was sent visible to the back. void Publish(); - /// Waits until the back executed everything sent so far. + /// Waits until the back executed everything sent so far. `zone`: the profile zone the wait + /// counts as (SplitDrain, or SplitDmaRead for a macro DMA read). void Drain(); + void Drain(Nx2Profile::Zone zone); + + /// Front thread: the ring position after everything sent so far (an entry sent now ends + /// below the position returned after it is sent). + [[nodiscard]] u64 WritePosition() const { + return write_position; + } + /// Front thread: the ring position the back has executed up to (as last seen; refreshed). + [[nodiscard]] u64 Consumed() { + consumed_cache = consumed.load(std::memory_order_acquire); + return consumed_cache; + } /// Compute or copy work was just sent: what can write GPU-written command lists. void NoteWriter() { @@ -144,6 +177,7 @@ class GpuSplit { Method, ///< Bits 8-23 method, 32-63 argument; flags IsLast, HasSegment (word 1) Multi, ///< Bits 8-23 method, 32-63 amount; word 1 methods pending, [segment], words Call, ///< Bits 8-31 payload size in words; word 1 thunk, then the payload + ConstantRun, ///< Bits 8-23 first method, 32-63 count; then the words (two per u64) }; static constexpr u64 IsLast = 1 << 4; static constexpr u64 HasSegment = 1 << 5; @@ -161,6 +195,9 @@ class GpuSplit { /// Room for an entry of `words` at the write position (never across the ring's end). u64* Reserve(size_t words) { + if (pending_count != 0) [[unlikely]] { + FlushPending(); + } const size_t index = write_position & RingMask; if (index + words <= RingWords && write_position + words - consumed_cache <= RingWords) [[likely]] { @@ -176,7 +213,16 @@ class GpuSplit { } } void WaitForSpace(size_t words); + /// Sleeps a while on the back thread handing ring space back. + void SleepOnBack(); + /// Back side: hands the ring up to `position` back, waking a sleeping front. + void HandBack(u64 position); void SelectEngine(Tegra::Engines::EngineInterface* engine) { + // Words queued (QueueWord) go first, before the engine is compared: their entry selects + // their own engine, which may differ from this one, and from front_engine as it is now. + if (pending_count != 0) [[unlikely]] { + FlushPending(); + } if (engine != front_engine) [[unlikely]] { SwitchEngine(engine); } @@ -199,6 +245,12 @@ class GpuSplit { /// Words between publishes: small right after a drain, while the back has nothing to do. size_t publish_every{PublishEvery}; Tegra::Engines::EngineInterface* front_engine{}; + /// QueueWord: a run not yet sent. + static constexpr u32 PendingWords = 256; + std::array pending_words{}; + u32 pending_count{}; + u32 pending_method{}; + Tegra::Engines::EngineInterface* pending_engine{}; // Back side. Tegra::Engines::EngineInterface* back_engine{}; @@ -227,6 +279,10 @@ class GpuSplit { alignas(128) std::atomic wait_target{~u64{0}}; alignas(128) std::atomic back_waiting{}; std::atomic wake{}; + /// Set while the front sleeps on the back thread; the back bumps `drained` when it hands ring + /// space back then. + alignas(64) std::atomic front_waiting{}; + std::atomic drained{}; std::jthread back_thread; }; diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 3dbf5a8d42..fed09a102d 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -18,6 +18,7 @@ #include "video_core/gpu.h" #include "video_core/gpu_thread.h" #include "video_core/gpu_split.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" #include "video_core/renderer_base.h" @@ -96,6 +97,10 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto* const rasterizer_ptr = renderer.ReadRasterizer(); if (auto* submit_list = std::get_if(&next.data)) { Profile::Scope scope{Profile::Zone::Push}; + // Held-back guest CPU writes first (see the single-thread path below). + if (Tegra::Nx2HeldWrites()) { + split.Defer([gpu = &system.GPU()] { gpu->InvalidateGPUCache(); }); + } scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative(next.data)) { split.Defer([gpu = &system.GPU()] { gpu->TickWork(); }); @@ -125,6 +130,15 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } if (auto* submit_list = std::get_if(&next.data)) { Profile::Scope scope{Profile::Zone::Push}; + // Guest CPU writes to GPU-written memory are held back until the GPU thread + // applies them, by default at the next fence. The guest made them before it + // submitted this list, which may read them: apply them first. Pokemon Pokopia + // keeps constants in pages its native storage bindings mark GPU-written, and some + // frames drew stale ones (flickering shadows, stretched geometry). Switch 2 + // programs only; EDEN_NX2_HELD_WRITES=0 turns it off. + if (Tegra::Nx2HeldWrites()) { + system.GPU().InvalidateGPUCache(); + } scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative(next.data)) { Profile::Scope scope{Profile::Zone::Tick}; diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp index 6163ef3d98..3a2c3fd2ef 100644 --- a/src/video_core/host1x/nx2_video_decoder.cpp +++ b/src/video_core/host1x/nx2_video_decoder.cpp @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include #include #include #include #include +#include + +#ifdef EDEN_NX2_LIBGAV1 +#include +#endif #include "common/logging.h" #include "common/settings.h" @@ -46,6 +52,80 @@ std::optional CodecFromMime(std::string_view mime) { return std::nullopt; } +#ifdef EDEN_NX2_LIBGAV1 +AVPixelFormat Av1PixelFormat(const libgav1::DecoderBuffer& buffer) { + if (buffer.bitdepth != 8 && buffer.bitdepth != 10) { + return AV_PIX_FMT_NONE; + } + const bool ten_bit = buffer.bitdepth == 10; + switch (buffer.image_format) { + case kLibgav1ImageFormatYuv420: + return ten_bit ? AV_PIX_FMT_YUV420P10LE : AV_PIX_FMT_YUV420P; + case kLibgav1ImageFormatYuv422: + return ten_bit ? AV_PIX_FMT_YUV422P10LE : AV_PIX_FMT_YUV422P; + case kLibgav1ImageFormatYuv444: + return ten_bit ? AV_PIX_FMT_YUV444P10LE : AV_PIX_FMT_YUV444P; + case kLibgav1ImageFormatMonochrome400: + return ten_bit ? AV_PIX_FMT_GRAY10LE : AV_PIX_FMT_GRAY8; + } + return AV_PIX_FMT_NONE; +} + +// Copies a picture libgav1 decoded into a frame of its own (libgav1 reuses its buffers), tagged +// with the stream's matrix and range for Render. +std::shared_ptr CopyAv1Picture(const libgav1::DecoderBuffer& buffer) { + const AVPixelFormat format = Av1PixelFormat(buffer); + const AVPixFmtDescriptor* descriptor = av_pix_fmt_desc_get(format); + if (descriptor == nullptr || buffer.displayed_width[0] <= 0 || + buffer.displayed_height[0] <= 0) { + return nullptr; + } + auto frame = std::make_shared(); + AVFrame* f = frame->GetFrame(); + f->format = format; + f->width = buffer.displayed_width[0]; + f->height = buffer.displayed_height[0]; + if (av_frame_get_buffer(f, 0) < 0) { + return nullptr; + } + const int sample_bytes = buffer.bitdepth > 8 ? 2 : 1; + for (int plane = 0; plane < buffer.NumPlanes(); ++plane) { + const int width = + plane == 0 ? f->width : AV_CEIL_RSHIFT(f->width, descriptor->log2_chroma_w); + const int height = + plane == 0 ? f->height : AV_CEIL_RSHIFT(f->height, descriptor->log2_chroma_h); + const size_t row_bytes = + static_cast(std::min(width, buffer.displayed_width[plane]) * sample_bytes); + for (int y = 0; y < std::min(height, buffer.displayed_height[plane]); ++y) { + std::memcpy(f->data[plane] + static_cast(y) * f->linesize[plane], + buffer.plane[plane] + static_cast(y) * buffer.stride[plane], + row_bytes); + } + } + // AV1 numbers its matrices as FFmpeg does (ITU-T H.273). + const int matrix = static_cast(buffer.matrix_coefficients); + f->colorspace = matrix <= static_cast(AVCOL_SPC_ICTCP) ? static_cast(matrix) + : AVCOL_SPC_UNSPECIFIED; + f->color_range = + buffer.color_range == kLibgav1ColorRangeFull ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + return frame; +} +#endif + +// The stream's matrix and range for a conversion to RGB; untagged HD video is BT.709. +void SetRgbColorspace(SwsContext* scaler, const AVFrame* f) { + int space = SWS_CS_ITU601; + if (f->colorspace == AVCOL_SPC_BT709 || + (f->colorspace == AVCOL_SPC_UNSPECIFIED && f->height >= 720)) { + space = SWS_CS_ITU709; + } else if (f->colorspace == AVCOL_SPC_BT2020_NCL || f->colorspace == AVCOL_SPC_BT2020_CL) { + space = SWS_CS_BT2020; + } + const int* coefficients = sws_getCoefficients(space); + sws_setColorspaceDetails(scaler, coefficients, f->color_range == AVCOL_RANGE_JPEG, coefficients, + 1, 0, 1 << 16, 1 << 16); +} + } // namespace bool IsH264Stream(std::span data) { @@ -87,12 +167,23 @@ bool IsH264Stream(std::span data) { return false; } +bool Nx2IsAv1Mime(std::string_view mime) { + return mime.find("av01") != std::string_view::npos || + mime.find("av1") != std::string_view::npos; +} + struct Nx2VideoDecoder::Impl { std::string name; std::optional decoder; std::optional hardware; std::optional context; +#ifdef EDEN_NX2_LIBGAV1 + std::unique_ptr av1; ///< Instead of FFmpeg, for AV1 + u64 av1_errors{}; +#endif SwsContext* scaler{}; + static constexpr size_t Bands = 4; + std::array band_scalers{}; ///< RenderBands bool open{}; H264PacketConverter h264_packets; /// Timestamps of submitted frames not yet output. Frames copied back from a host GPU decoder @@ -102,6 +193,9 @@ struct Nx2VideoDecoder::Impl { ~Impl() { sws_freeContext(scaler); + for (SwsContext* band_scaler : band_scalers) { + sws_freeContext(band_scaler); + } context.reset(); hardware.reset(); } @@ -125,10 +219,117 @@ struct Nx2VideoDecoder::Impl { std::move(frame)}); } } + + // Converts a frame to RGB at its own size in bands of rows, each with a context of its own, on + // threads (fefw converts its AV1 frames to RGBA on four). Bands start at rows the chroma + // subsampling divides, so no chroma row is split. Returns false if it can't. + bool RenderBands(const AVFrame* f, AVPixelFormat target, u32 height, u64 pitch, + std::span out) { + const auto source = static_cast(f->format); + const AVPixFmtDescriptor* descriptor = av_pix_fmt_desc_get(source); + const int plane_count = av_pix_fmt_count_planes(source); + if (descriptor == nullptr || (descriptor->flags & AV_PIX_FMT_FLAG_HWACCEL) != 0 || + plane_count <= 0 || plane_count > 4) { + return false; + } + const u32 row_alignment = 1U << std::max(1, descriptor->log2_chroma_h); + std::array rows{}; + for (size_t band = 0; band < Bands; ++band) { + rows[band] = static_cast(u64{height} * band / Bands) & ~(row_alignment - 1); + } + rows[Bands] = height; + for (size_t band = 0; band < Bands; ++band) { + const int band_height = static_cast(rows[band + 1] - rows[band]); + band_scalers[band] = sws_getCachedContext(band_scalers[band], f->width, band_height, + source, f->width, band_height, target, + SWS_BILINEAR, nullptr, nullptr, nullptr); + if (band_scalers[band] == nullptr) { + return false; + } + SetRgbColorspace(band_scalers[band], f); + } + const auto convert = [&](size_t index) { + std::array source_planes{}; + for (int plane = 0; plane < plane_count; ++plane) { + const bool chroma = plane == 1 || plane == 2; + const u32 first_row = + chroma ? rows[index] >> descriptor->log2_chroma_h : rows[index]; + source_planes[plane] = + f->data[plane] + static_cast(first_row) * f->linesize[plane]; + } + const std::array target_planes{out.data() + rows[index] * pitch, nullptr, + nullptr, nullptr}; + const std::array target_strides{static_cast(pitch), 0, 0, 0}; + sws_scale(band_scalers[index], source_planes.data(), f->linesize, 0, + static_cast(rows[index + 1] - rows[index]), target_planes.data(), + target_strides.data()); + }; + { + std::array workers; + for (size_t band = 1; band < Bands; ++band) { + workers[band - 1] = std::jthread{convert, band}; + } + convert(0); + } + return true; + } + +#ifdef EDEN_NX2_LIBGAV1 + // Not in frame parallel mode, libgav1 decodes one temporal unit per DequeueFrame and holds + // no picture back, so each one is decoded right away (EnqueueFrame keeps a pointer to it). + void DecodeAv1(std::span data, u64 pts, std::vector& pictures) { + auto status = + av1->EnqueueFrame(data.data(), data.size(), static_cast(pts), nullptr); + const libgav1::DecoderBuffer* buffer{}; + if (status == libgav1::kStatusOk) { + status = av1->DequeueFrame(&buffer); + } + if (status != libgav1::kStatusOk) { + if (av1_errors++ < 8 || (av1_errors % 600) == 0) { + LOG_ERROR(HW_GPU, "nx2 video: AV1 frame of {} bytes not decoded ({}), {} so far", + data.size(), libgav1::GetErrorString(status), av1_errors); + } + return; + } + if (buffer == nullptr) { + return; // nothing shown by this temporal unit + } + auto frame = CopyAv1Picture(*buffer); + if (!frame) { + if (av1_errors++ < 8) { + LOG_ERROR(HW_GPU, "nx2 video: unsupported AV1 picture: {}-bit, format {}, {}x{}", + buffer->bitdepth, static_cast(buffer->image_format), + buffer->displayed_width[0], buffer->displayed_height[0]); + } + return; + } + pictures.push_back({static_cast(buffer->displayed_width[0]), + static_cast(buffer->displayed_height[0]), + static_cast(buffer->user_private_data), std::move(frame)}); + } +#endif }; Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique()} { impl->name = std::string{mime}; +#ifdef EDEN_NX2_LIBGAV1 + // AV1 goes to libgav1 where FFmpeg has no libdav1d (Android's FFmpeg package); FFmpeg's own + // "av1" decoder only drives hardware decoders. + if (Nx2IsAv1Mime(mime) && avcodec_find_decoder_by_name("libdav1d") == nullptr) { + auto av1 = std::make_unique(); + libgav1::DecoderSettings settings; + settings.threads = + static_cast(std::clamp(std::thread::hardware_concurrency(), 1U, 4U)); + const auto status = av1->Init(&settings); + impl->open = status == libgav1::kStatusOk; + if (impl->open) { + impl->av1 = std::move(av1); + } + LOG_INFO(HW_GPU, "nx2 video: libgav1 decoder for \"{}\" {} ({} threads)", mime, + impl->open ? "open" : libgav1::GetErrorString(status), settings.threads); + return; + } +#endif const auto codec = CodecFromMime(mime); if (!codec) { LOG_ERROR(HW_GPU, "nx2 video: no decoder for codec \"{}\"", mime); @@ -162,6 +363,12 @@ void Nx2VideoDecoder::Decode(std::span data, u64 pts, std::vectoropen || data.empty()) { return; } +#ifdef EDEN_NX2_LIBGAV1 + if (impl->av1) { + impl->DecodeAv1(data, pts, pictures); + return; + } +#endif impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0); std::memcpy(impl->packet.data(), data.data(), data.size()); @@ -185,12 +392,26 @@ void Nx2VideoDecoder::Drain(std::vector& pictures) { if (!impl->open) { return; } +#ifdef EDEN_NX2_LIBGAV1 + if (impl->av1) { + Reset(); // libgav1 holds no picture back (see DecodeAv1) + return; + } +#endif avcodec_send_packet(impl->context->GetCodecContext(), nullptr); impl->Receive(pictures); Reset(); } void Nx2VideoDecoder::Reset() { +#ifdef EDEN_NX2_LIBGAV1 + if (impl->av1) { + // Releases its frames; the next frame may start a new sequence. + impl->av1->SignalEOS(); + impl->pending_pts.clear(); + return; + } +#endif if (impl->open) { avcodec_flush_buffers(impl->context->GetCodecContext()); } @@ -223,6 +444,12 @@ void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format, } return; } + // Large RGB images at the frame's own size convert on several threads (NV12 doesn't). + if (format != Nx2PictureFormat::Nv12 && static_cast(f->width) == width && + static_cast(f->height) == height && height >= 256 && + impl->RenderBands(f, target, height, pitch, out)) { + return; + } impl->scaler = sws_getCachedContext(impl->scaler, f->width, f->height, static_cast(f->format), static_cast(width), static_cast(height), target, @@ -234,17 +461,7 @@ void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format, return; } if (format != Nx2PictureFormat::Nv12) { - // The stream's matrix and range; untagged HD video is BT.709. - int space = SWS_CS_ITU601; - if (f->colorspace == AVCOL_SPC_BT709 || - (f->colorspace == AVCOL_SPC_UNSPECIFIED && f->height >= 720)) { - space = SWS_CS_ITU709; - } else if (f->colorspace == AVCOL_SPC_BT2020_NCL || f->colorspace == AVCOL_SPC_BT2020_CL) { - space = SWS_CS_BT2020; - } - const int* coefficients = sws_getCoefficients(space); - sws_setColorspaceDetails(impl->scaler, coefficients, f->color_range == AVCOL_RANGE_JPEG, - coefficients, 1, 0, 1 << 16, 1 << 16); + SetRgbColorspace(impl->scaler, f); } std::array planes{out.data(), nullptr, nullptr, nullptr}; std::array strides{static_cast(pitch), 0, 0, 0}; diff --git a/src/video_core/host1x/nx2_video_decoder.h b/src/video_core/host1x/nx2_video_decoder.h index 347b0e6525..dd5b867211 100644 --- a/src/video_core/host1x/nx2_video_decoder.h +++ b/src/video_core/host1x/nx2_video_decoder.h @@ -35,9 +35,13 @@ enum class Nx2PictureFormat : u32 { /// Checks if stream data begins with an H.264 Annex-B start code or avcC/MP4 length-prefixed NALUs. bool IsH264Stream(std::span data); +/// Whether `mime` names AV1 ("video/av1", "video/av01"). +bool Nx2IsAv1Mime(std::string_view mime); + /// The decoder behind the Switch 2 video decoder service (nn::codec::video): FFmpeg decodes the -/// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting). -/// Used from one thread at a time. +/// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting), +/// and on Android libgav1 decodes AV1, which its FFmpeg package lacks. Used from one thread at a +/// time. class Nx2VideoDecoder { public: /// mime: the codec name nn::codec sends with Configure ("video/vp9", "video/vp8", ...). diff --git a/src/video_core/host1x/syncpoint_manager.cpp b/src/video_core/host1x/syncpoint_manager.cpp index 7b7353b6ca..955c3c304e 100644 --- a/src/video_core/host1x/syncpoint_manager.cpp +++ b/src/video_core/host1x/syncpoint_manager.cpp @@ -4,12 +4,25 @@ // SPDX-FileCopyrightText: 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include "common/logging.h" #include "video_core/host1x/syncpoint_manager.h" namespace Tegra { namespace Host1x { +bool SyncpointManager::IsValid(u32 id) { + if (id < NUM_MAX_SYNCPOINTS) [[likely]] { + return true; + } + // Reported a few times: a bad id is usually a misread register, and it repeats every frame. + static std::atomic reports{}; + if (reports.fetch_add(1, std::memory_order_relaxed) < 16) { + LOG_ERROR(HW_GPU, "Syncpoint id {} out of range (max {}), ignored", id, NUM_MAX_SYNCPOINTS); + } + return false; +} + SyncpointManager::ActionHandle SyncpointManager::RegisterAction( std::atomic& syncpoint, std::list& action_storage, u32 expected_value, std::function&& action) { @@ -50,31 +63,51 @@ void SyncpointManager::DeregisterAction(std::list& action_stor } void SyncpointManager::DeregisterGuestAction(u32 syncpoint_id, const ActionHandle& handle) { + if (!IsValid(syncpoint_id)) { + return; + } DeregisterAction(guest_action_storage[syncpoint_id], handle); } void SyncpointManager::DeregisterHostAction(u32 syncpoint_id, const ActionHandle& handle) { + if (!IsValid(syncpoint_id)) { + return; + } DeregisterAction(host_action_storage[syncpoint_id], handle); } void SyncpointManager::IncrementGuest(u32 syncpoint_id) { - Increment(syncpoints_guest[syncpoint_id], wait_guest_cv, guest_action_storage[syncpoint_id]); + if (!IsValid(syncpoint_id)) { + return; + } + Increment(syncpoints_guest[syncpoint_id], wait_guest_cv, guest_waiters, + guest_action_storage[syncpoint_id]); } void SyncpointManager::IncrementHost(u32 syncpoint_id) { - Increment(syncpoints_host[syncpoint_id], wait_host_cv, host_action_storage[syncpoint_id]); + if (!IsValid(syncpoint_id)) { + return; + } + Increment(syncpoints_host[syncpoint_id], wait_host_cv, host_waiters, + host_action_storage[syncpoint_id]); } void SyncpointManager::WaitGuest(u32 syncpoint_id, u32 expected_value) { - Wait(syncpoints_guest[syncpoint_id], wait_guest_cv, expected_value); + if (!IsValid(syncpoint_id)) { + return; + } + Wait(syncpoints_guest[syncpoint_id], wait_guest_cv, guest_waiters, expected_value); } void SyncpointManager::WaitHost(u32 syncpoint_id, u32 expected_value) { - Wait(syncpoints_host[syncpoint_id], wait_host_cv, expected_value); + if (!IsValid(syncpoint_id)) { + return; + } + Wait(syncpoints_host[syncpoint_id], wait_host_cv, host_waiters, expected_value); } void SyncpointManager::Increment(std::atomic& syncpoint, std::condition_variable& wait_cv, - std::list& action_storage) { + u32& waiters, std::list& action_storage) { auto new_value{syncpoint.fetch_add(1, std::memory_order_acq_rel) + 1}; std::scoped_lock lk(guard); @@ -86,18 +119,25 @@ void SyncpointManager::Increment(std::atomic& syncpoint, std::condition_var it->action(); it = action_storage.erase(it); } - wait_cv.notify_all(); + // Only with someone waiting: a notify is a system call even without waiters (bionic tracks + // none), and syncpoints are incremented thousands of times a second (7.6% of the GPU back + // thread in Mario Kart World on Android). Waiters count themselves under `guard`. + if (waiters != 0) { + wait_cv.notify_all(); + } } void SyncpointManager::Wait(std::atomic& syncpoint, std::condition_variable& wait_cv, - u32 expected_value) { + u32& waiters, u32 expected_value) { const auto pred = [&]() { return syncpoint.load(std::memory_order_acquire) >= expected_value; }; if (pred()) { return; } std::unique_lock lk(guard); + ++waiters; wait_cv.wait(lk, pred); + --waiters; } } // namespace Host1x diff --git a/src/video_core/host1x/syncpoint_manager.h b/src/video_core/host1x/syncpoint_manager.h index d6e65af3eb..27498e2604 100644 --- a/src/video_core/host1x/syncpoint_manager.h +++ b/src/video_core/host1x/syncpoint_manager.h @@ -20,12 +20,17 @@ class SyncpointManager { public: static constexpr size_t NUM_MAX_SYNCPOINTS = 192; + /// Whether `id` names a syncpoint. Ids come from guest registers (16 bits in the 3D engine's + /// sync_info, 24 in the puller's fence action) and index fixed arrays; out of range ones are + /// reported and treated as always signalled instead of reaching past them. + static bool IsValid(u32 id); + u32 GetGuestSyncpointValue(u32 id) const { - return syncpoints_guest[id].load(std::memory_order_acquire); + return IsValid(id) ? syncpoints_guest[id].load(std::memory_order_acquire) : 0; } u32 GetHostSyncpointValue(u32 id) const { - return syncpoints_host[id].load(std::memory_order_acquire); + return IsValid(id) ? syncpoints_host[id].load(std::memory_order_acquire) : 0; } struct RegisteredAction { @@ -38,12 +43,20 @@ class SyncpointManager { template ActionHandle RegisterGuestAction(u32 syncpoint_id, u32 expected_value, Func&& action) { + if (!IsValid(syncpoint_id)) { + action(); + return {}; + } return RegisterAction(syncpoints_guest[syncpoint_id], guest_action_storage[syncpoint_id], expected_value, std::move(action)); } template ActionHandle RegisterHostAction(u32 syncpoint_id, u32 expected_value, Func&& action) { + if (!IsValid(syncpoint_id)) { + action(); + return {}; + } return RegisterAction(syncpoints_host[syncpoint_id], host_action_storage[syncpoint_id], expected_value, std::move(action)); } @@ -61,15 +74,17 @@ class SyncpointManager { void WaitHost(u32 syncpoint_id, u32 expected_value); bool IsReadyGuest(u32 syncpoint_id, u32 expected_value) const { - return syncpoints_guest[syncpoint_id].load(std::memory_order_acquire) >= expected_value; + return !IsValid(syncpoint_id) || + syncpoints_guest[syncpoint_id].load(std::memory_order_acquire) >= expected_value; } bool IsReadyHost(u32 syncpoint_id, u32 expected_value) const { - return syncpoints_host[syncpoint_id].load(std::memory_order_acquire) >= expected_value; + return !IsValid(syncpoint_id) || + syncpoints_host[syncpoint_id].load(std::memory_order_acquire) >= expected_value; } private: - void Increment(std::atomic& syncpoint, std::condition_variable& wait_cv, + void Increment(std::atomic& syncpoint, std::condition_variable& wait_cv, u32& waiters, std::list& action_storage); ActionHandle RegisterAction(std::atomic& syncpoint, @@ -78,7 +93,8 @@ class SyncpointManager { void DeregisterAction(std::list& action_storage, const ActionHandle& handle); - void Wait(std::atomic& syncpoint, std::condition_variable& wait_cv, u32 expected_value); + void Wait(std::atomic& syncpoint, std::condition_variable& wait_cv, u32& waiters, + u32 expected_value); std::array, NUM_MAX_SYNCPOINTS> syncpoints_guest{}; std::array, NUM_MAX_SYNCPOINTS> syncpoints_host{}; @@ -89,6 +105,8 @@ class SyncpointManager { std::mutex guard; std::condition_variable wait_guest_cv; std::condition_variable wait_host_cv; + u32 guest_waiters{}; ///< Threads in Wait on wait_guest_cv, under guard + u32 host_waiters{}; ///< Threads in Wait on wait_host_cv, under guard }; } // namespace Host1x diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index 94297e8fea..8f314bf121 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -19,6 +19,9 @@ #include "common/x64/xbyak.h" #include "video_core/macro_turing_jit.h" #endif +#if defined(ARCHITECTURE_arm64) && defined(__ANDROID__) +#include "video_core/macro_turing_jit_arm64.h" +#endif #include "common/assert.h" #include "common/scope_exit.h" @@ -1466,9 +1469,11 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u } return true; }; -#ifdef ARCHITECTURE_x86_64 - // Macro code runs compiled to x86-64 (macro_turing_jit.h) instead of through the memo - // and the interpreter; EDEN_NX2_MME_JIT=0 goes back to those. +#if defined(ARCHITECTURE_x86_64) || (defined(ARCHITECTURE_arm64) && defined(__ANDROID__)) + // Macro code runs compiled to x86-64 or ARM64 (macro_turing_jit.h, + // macro_turing_jit_arm64.h) instead of through the memo and the interpreter; + // EDEN_NX2_MME_JIT=0 goes back to those, as does ARM64 code that did not match the + // interpreter on its checked calls. static const bool use_jit = [] { const char* value = std::getenv("EDEN_NX2_MME_JIT"); return value == nullptr || value[0] != '0'; @@ -1478,6 +1483,8 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u native_jit = std::make_shared(); } native_jit->Prepare(native_code, native_code_generation); + } + if (use_jit && native_jit->Usable()) { fifo.assign(parameters.begin(), parameters.end()); const auto result = native_jit->Execute( method, fifo, std::span{*native_interpreter.active_dram}, @@ -1487,8 +1494,9 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u emit); if (!result.success) { native_execution_failed = true; - LOG_ERROR(HW_GPU, "Native MME (JIT) failed: start={:#x}, consumed={}/{}, error={}", - method, result.consumed, parameters.size(), result.error); + LOG_ERROR(HW_GPU, + "Native MME (JIT) failed: start={:#x}, pc={:#x}, consumed={}/{}, error={}", + method, result.pc, result.consumed, parameters.size(), result.error); } return; } diff --git a/src/video_core/macro.h b/src/video_core/macro.h index 0d574636e9..3840394a67 100644 --- a/src/video_core/macro.h +++ b/src/video_core/macro.h @@ -261,7 +261,8 @@ struct MacroEngine { // Calls replayed instead of interpreted (macro_memo.h), and what the current call wrote. u64 native_code_generation{}; BasicMacroMemo<::Common::unordered_map> native_memo; - /// The macro code compiled to x86-64 (EDEN_NX2_MME_JIT=0 turns it off), created on first use. + /// The macro code compiled to x86-64 or ARM64 (EDEN_NX2_MME_JIT=0 turns it off), created on + /// first use. std::shared_ptr native_jit; MacroMemoPairs native_writes; u64 native_calls{}; diff --git a/src/video_core/macro_turing_jit.h b/src/video_core/macro_turing_jit.h index d0f24080a3..d0b591a4f5 100644 --- a/src/video_core/macro_turing_jit.h +++ b/src/video_core/macro_turing_jit.h @@ -99,6 +99,11 @@ class TuringMacroJit { compiled_generation = generation; } + /// Always true here; the ARM64 port (macro_turing_jit_arm64.h) can give up on a program. + bool Usable() const { + return true; + } + template Result Execute(Word start, std::vector& parameters, std::span data_ram, Read read, Emit emit) { @@ -299,6 +304,8 @@ class TuringMacroJit { } void Instruction(Word pc, Word count, const Decoded& d) { + // Keep the failing PC available for diagnostics; it used to stay zero. + mov(dword[st + offsetof(State, fail_pc)], pc); // FIFO loads, in lane order. for (Word i = 0; i < 2; ++i) { const Xbyak::Reg32 load = i == 0 ? r12d : r13d; diff --git a/src/video_core/macro_turing_jit_arm64.cpp b/src/video_core/macro_turing_jit_arm64.cpp new file mode 100644 index 0000000000..59296e7dee --- /dev/null +++ b/src/video_core/macro_turing_jit_arm64.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +#include "video_core/macro_turing_jit_arm64.h" + +namespace Tegra { + +void* TuringMacroJit::AllocateCode(std::size_t bytes) { +#ifdef _WIN32 + return VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); +#else + void* const memory = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + return memory == MAP_FAILED ? nullptr : memory; +#endif +} + +bool TuringMacroJit::SealCode(void* memory, std::size_t bytes) { +#ifdef _WIN32 + DWORD old_protection{}; + if (!VirtualProtect(memory, bytes, PAGE_EXECUTE_READ, &old_protection)) { + return false; + } + return FlushInstructionCache(GetCurrentProcess(), memory, bytes) != 0; +#else + if (mprotect(memory, bytes, PROT_READ | PROT_EXEC) != 0) { + return false; + } + char* const begin = static_cast(memory); + __builtin___clear_cache(begin, begin + bytes); + return true; +#endif +} + +void TuringMacroJit::FreeCode(void* memory, std::size_t bytes) { +#ifdef _WIN32 + VirtualFree(memory, 0, MEM_RELEASE); +#else + munmap(memory, bytes); +#endif +} + +} // namespace Tegra diff --git a/src/video_core/macro_turing_jit_arm64.h b/src/video_core/macro_turing_jit_arm64.h new file mode 100644 index 0000000000..d517fb15e3 --- /dev/null +++ b/src/video_core/macro_turing_jit_arm64.h @@ -0,0 +1,1138 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "common/logging.h" +#include "video_core/macro_turing.h" + +namespace Tegra { + +// ARM64 code for the TU104/GA10x MME, the port of the x86-64 TuringMacroJit (macro_turing_jit.h) +// with the same interface and semantics: every instruction of the uploaded macro memory compiled +// once (each a block that falls through to the next one, branches and loops through a jump +// table), emits (method writes) and register reads calling back into C++. +// +// Nobody can step through this code on a phone, so it checks itself: after each compile, the +// first CheckedCalls calls of each entry point also run TuringMacroInterpreter, which replays what +// the compiled code saw (register reads, the FIFO words and data RAM words its emits added) +// instead of calling out, and every emit, read, the result and the data RAM are compared. A +// difference is logged once and the program goes back to the interpreter (Usable() is false). +class TuringMacroJit { +public: + using Word = std::uint32_t; + using Result = TuringMacroInterpreter::Result; + + /// Same-value elision (Maxwell3D::FrontRunMacro): an emit of a method whose `must_call` + /// byte is 0 (a state register without side effects) is skipped inside the compiled code + /// when `regs[method]` already holds the value and, with the shadow RAM tracking + /// (`*control` 0 or 1), `shadow[method]` does too; mode 3 (replay) never elides. Exactly + /// the writes Maxwell3D::FrontWrite would return from without sending; `skipped` counts + /// them. Set per call with SetElision; the checked calls (ExecuteChecked) never elide, so + /// the comparison with the interpreter sees every emit. + struct Elision { + const std::uint8_t* must_call; + const Word* regs; + const Word* shadow; + const Word* control; + Word skipped; + }; + static constexpr Word MustCallMethods = 0x1000; ///< Entries in Elision::must_call + + void SetElision(Elision* elision_) { + elision = elision_; + } + + struct State { + std::array regs; + Word carry; + Word method; + Word increment; + Word loop_count; + Word loop_start; + Word loop_end; + Word next_pc; + Word steps; + Word start; + Word consumed; + Word params_size; + Word fail_pc; + std::uint8_t end_next; + std::uint8_t stopped; + std::uint8_t cancel_end; + std::uint8_t pad; + Word scratch_x; + Word scratch_y; + const Word* params; + Word* dram; + void* user; + bool (*emit)(State*, Word method, Word value); + Word (*read)(State*, Word address); + const std::vector* fifo; + const Elision* elision; ///< Null: every emit calls back + }; + + enum Error : Word { + None = 0, + FifoUnderflow, + InvalidSource, + InvalidDestination, + UnsupportedExtended, + InvalidMerge, + InvalidLoop, + DataRamReadOutOfBounds, + DataRamWriteOutOfBounds, + InvalidOutput, + UnsupportedEmit, + OutsideCode, + StepLimit, + }; + + static std::string_view ErrorText(Word error) { + constexpr std::array texts{ + "", + "parameter FIFO underflow", + "invalid source register", + "invalid destination register", + "unsupported extended operation", + "invalid merge destination", + "invalid loop", + "data RAM read out of bounds", + "data RAM write out of bounds", + "invalid output selector", + "unsupported emitted method", + "instruction outside uploaded RAM", + "instruction limit exceeded", + }; + return error < texts.size() ? texts[error] : "unknown"; + } + + /// Calls of each entry point that also run the interpreter after a compile. + static constexpr std::uint8_t CheckedCalls = 64; + + /// Compiles `code` (3 words per instruction) when it differs from what was compiled. + void Prepare(std::span code, std::uint64_t generation) { + if (generation == compiled_generation) { + return; + } + compiled_generation = generation; + program.reset(); + diverged = false; + try { + program = std::make_unique(code); + } catch (const std::exception& exception) { + if (!compile_failure_logged.exchange(true)) { + LOG_ERROR(HW_GPU, "Turing macro JIT: could not compile the macro code to ARM64 ({}); " + "it runs in the interpreter", exception.what()); + } + return; + } + compiled_code.assign(code.begin(), code.end()); + checks_left.assign(program->instruction_count + 1, CheckedCalls); + if (!active_logged.exchange(true)) { + LOG_INFO(HW_GPU, "Turing macro JIT: macro code compiled to ARM64 ({} instructions, " + "{} KiB); the first {} calls of each entry are checked against the " + "interpreter", + program->instruction_count, program->code_bytes / 1024, CheckedCalls); + } + } + + /// False when the code could not be compiled or differed from the interpreter: the caller + /// then runs the macro memo and the interpreter. + bool Usable() const { + return program != nullptr && !diverged; + } + + template + Result Execute(Word start, std::vector& parameters, std::span data_ram, Read read, + Emit emit) { + if (!program) [[unlikely]] { + return {false, 0, 0, ErrorText(OutsideCode)}; + } + std::uint8_t& left = checks_left[(std::min)(start, checks_left.size() - 1)]; + if (left != 0) [[unlikely]] { + --left; + return ExecuteChecked(start, parameters, data_ram, read, emit); + } + return Run(start, parameters, data_ram, read, emit, elision); + } + + /// The code for `code` without its jump table (see Relocate), for tests. + struct Assembly { + std::vector words; + std::size_t table_offset{}; + std::size_t entry_offset{}; + /// Byte offset of each instruction's code. + std::vector targets; + }; + + static Assembly Assemble(std::span code) { + return Assembler{code}.Finish(); + } + + /// Writes the jump table of `assembly`, copied to `words`, for code placed at `base`. + static void Relocate(std::span words, const Assembly& assembly, + std::uintptr_t base) { + for (std::size_t pc = 0; pc < assembly.targets.size(); ++pc) { + const std::uint64_t address = std::uint64_t{base} + assembly.targets[pc]; + const std::size_t index = assembly.table_offset / sizeof(std::uint32_t) + pc * 2; + words[index] = static_cast(address); + words[index + 1] = static_cast(address >> 32); + } + } + +private: + using Decoded = TuringMacroInterpreter::Decoded; + + template + struct Thunks { + Read& read; + Emit& emit; + static bool CallEmit(State* state, Word method, Word value) { + const bool ok = static_cast(state->user)->emit(method, value); + // MME_DMA_READ_FIFOED appends to the FIFO from inside the emit. + state->params = state->fifo->data(); + state->params_size = static_cast(state->fifo->size()); + return ok; + } + static Word CallRead(State* state, Word address) { + return static_cast(state->user)->read(address); + } + }; + + template + Result Run(Word start, std::vector& parameters, std::span data_ram, Read& read, + Emit& emit, const Elision* elide) { + State state; + state.elision = elide; + state.regs.fill(0); + state.carry = state.method = state.increment = 0; + state.loop_count = state.loop_start = state.loop_end = 0; + state.consumed = 0; + state.fail_pc = 0; + state.end_next = state.stopped = state.cancel_end = 0; + state.start = start; + state.steps = 1000000; + state.params = parameters.data(); + state.params_size = static_cast(parameters.size()); + state.dram = data_ram.data(); + state.fifo = ¶meters; + Thunks thunks{read, emit}; + state.user = &thunks; + state.emit = &Thunks::CallEmit; + state.read = &Thunks::CallRead; + const Word error = program->entry(&state); + if (error == None) { + return {true, state.fail_pc, state.consumed, {}}; + } + return {false, state.fail_pc, state.consumed, ErrorText(error)}; + } + + struct AssemblerBuffer { + std::vector words; + }; + + // Register use: x19 State*; w20/w21 the instruction's FIFO loads; w22/w23 its ALU results; + // x24 the jump table. w0 (value), w1 (x) and w2 (y) as eax/ecx/edx in the x86-64 code; + // x16/x17 scratch. Callee-saved x19-x24, x29 and x30 are kept on a 64-byte frame, so sp stays + // 16-byte aligned at the callbacks. + class Assembler : private AssemblerBuffer, public oaknut::VectorCodeGenerator { + public: + explicit Assembler(std::span code) : oaknut::VectorCodeGenerator(words) { + Generate(code); + } + + Assembly Finish() { + Assembly assembly; + assembly.table_offset = table_offset; + assembly.entry_offset = entry_offset; + assembly.targets.reserve(labels.size()); + for (const oaknut::Label& label : labels) { + assembly.targets.push_back(static_cast(label.offset())); + } + assembly.words = std::move(words); + return assembly; + } + + private: + using Cond = oaknut::Cond; + using WReg = oaknut::WReg; + using XReg = oaknut::XReg; + using Label = oaknut::Label; + + static constexpr WReg W0{0}, W1{1}, W2{2}, W16{16}, W17{17}; + static constexpr XReg X0{0}, X1{1}, X16{16}, X19{19}, X20{20}, X21{21}, X22{22}, X23{23}, + X24{24}, X29{29}, X30{30}; + static constexpr oaknut::WzrReg WZR{}; + static constexpr oaknut::SpReg sp{}; + static constexpr XReg st{19}; + static constexpr WReg load0{20}, load1{21}, alu0{22}, alu1{23}; + static constexpr XReg table{24}; + + static constexpr std::size_t RegOffset(Word reg) { + return offsetof(State, regs) + reg * sizeof(Word); + } + + static WReg Alu(Word lane) { + return lane == 0 ? alu0 : alu1; + } + + void Generate(std::span code) { + const Word count = static_cast(code.size() / 3); + labels.resize(count); + std::vector decoded(count); + for (Word pc = 0; pc < count; ++pc) { + decoded[pc] = TuringMacroInterpreter::Decode( + {code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}); + } + Analyze(decoded); + + // The jump table comes first (Relocate fills it), within ADR range of the entry. + table_offset = static_cast(offset()); + l(table_label); + for (Word pc = 0; pc < count; ++pc) { + dx(0); + } + + entry_offset = static_cast(offset()); + STP(X29, X30, sp, oaknut::PreIndexed{}, -64); + MOV(X29, sp); + STP(X19, X20, sp, 16); + STP(X21, X22, sp, 32); + STP(X23, X24, sp, 48); + MOV(st, X0); + ADR(table, table_label); + LDR(W0, st, offsetof(State, start)); + CmpImm(W0, count); + B(Cond::HS, Far(fail_outside)); + LDR(X16, table, X0, oaknut::IndexExt::LSL, 3); + BR(X16); + + for (Word pc = 0; pc < count; ++pc) { + // B.cond and CBZ reach 1 MiB: out-of-line jumps to the shared labels go through + // stubs placed every 256 KiB. + if (!stubs.empty() && static_cast(offset()) - window_start > 256 * 1024) { + FlushStubs(true); + } + l(labels[pc]); + Instruction(pc, count, decoded[pc]); + } + // Every instruction ends with a jump. + FlushStubs(false); + + // Dispatch on w0 (the next pc). Every backward edge comes through here, so the + // instruction limit is counted here. + l(dispatch); + LDR(W16, st, offsetof(State, steps)); + SUBS(W16, W16, 1); + STR(W16, st, offsetof(State, steps)); + B(Cond::EQ, fail_steps); + CmpImm(W0, count); + B(Cond::HS, fail_outside); + LDR(X16, table, X0, oaknut::IndexExt::LSL, 3); + BR(X16); + + l(success); + MOV(W0, WZR); + l(leave); + LDP(X23, X24, sp, 48); + LDP(X21, X22, sp, 32); + LDP(X19, X20, sp, 16); + LDP(X29, X30, sp, oaknut::PostIndexed{}, 64); + RET(); + const auto fail_with = [&](Label& label, Word error) { + l(label); + MOV(W0, error); + B(leave); + }; + fail_with(fail_underflow, FifoUnderflow); + fail_with(fail_source, InvalidSource); + fail_with(fail_destination, InvalidDestination); + fail_with(fail_extended, UnsupportedExtended); + fail_with(fail_merge, InvalidMerge); + fail_with(fail_loop, InvalidLoop); + fail_with(fail_dread, DataRamReadOutOfBounds); + fail_with(fail_dwrite, DataRamWriteOutOfBounds); + fail_with(fail_output, InvalidOutput); + fail_with(fail_emit, UnsupportedEmit); + fail_with(fail_outside, OutsideCode); + fail_with(fail_steps, StepLimit); + } + + // A label near the current position that jumps to `target` (placed by FlushStubs). + Label& Far(Label& target) { + for (const auto& stub : stubs) { + if (stub->target == &target) { + return stub->label; + } + } + if (stubs.empty()) { + window_start = static_cast(offset()); + } + stubs.push_back(std::make_unique()); + stubs.back()->target = ⌖ + return stubs.back()->label; + } + + void FlushStubs(bool reachable) { + if (stubs.empty()) { + return; + } + Label over; + if (reachable) { + B(over); + } + for (const auto& stub : stubs) { + l(stub->label); + B(*stub->target); + } + if (reachable) { + l(over); + } + stubs.clear(); + } + + void CmpImm(WReg reg, Word value) { + if (value < 4096) { + CMP(reg, value); + } else { + MOV(W17, value); + CMP(reg, W17); + } + } + + // Loads MME register `reg` (as an operand of `lane`) into `dst`. + void Get(WReg dst, const Decoded& d, Word reg, Word lane) { + if (reg < 24) { + LDR(dst, st, RegOffset(reg)); + return; + } + switch (reg) { + case 24: MOV(dst, WZR); return; + case 25: MOV(dst, Word(std::int32_t(std::int16_t(d.imm[lane])))); return; + case 26: MOV(dst, Word(std::int32_t(std::int16_t(d.imm[1 - lane])))); return; + case 27: MOV(dst, (d.imm[0] << 16) | d.imm[1]); return; + case 28: MOV(dst, load0); return; + case 29: MOV(dst, load1); return; + default: B(fail_source); return; + } + } + + static bool IsFlow(Word op) { + return op >= 23 && op <= 29; + } + + // Where a loop can end (its back edge is checked only there) and which instructions can + // run right after an END (only those check for the end of the macro). + void Analyze(const std::vector& decoded) { + const Word count = static_cast(decoded.size()); + loop_end.assign(count, false); + after_end.assign(count, false); + const auto mark = [&](std::vector& set, std::int64_t pc) { + if (pc >= 0 && pc < count) set[static_cast(pc)] = true; + }; + for (Word pc = 0; pc < count; ++pc) { + for (Word lane = 0; lane < 2; ++lane) { + const Decoded& d = decoded[pc]; + if (d.op[lane] == 23 && d.imm[lane] >= 2 && pc + d.imm[lane] - 1 < count) { + loop_end[pc + d.imm[lane] - 1] = true; + } + } + } + for (Word pc = 0; pc < count; ++pc) { + const Decoded& d = decoded[pc]; + if (!d.end) continue; + mark(after_end, std::int64_t{pc} + 1); + for (Word lane = 0; lane < 2; ++lane) { + const Word imm = d.imm[lane]; + switch (d.op[lane]) { + case 23: mark(after_end, std::int64_t{pc} + imm); break; + case 24: mark(after_end, std::int64_t{pc} + (imm & 0x7fff)); break; + case 25: case 26: case 27: case 28: case 29: + mark(after_end, std::int64_t{pc} + (std::int32_t(imm << 19) >> 19)); + break; + default: break; + } + } + } + // An END at a loop's last instruction can be followed by the loop's second one. + for (Word pc = 0; pc < count; ++pc) { + for (Word lane = 0; lane < 2; ++lane) { + const Decoded& d = decoded[pc]; + if (d.op[lane] == 23 && d.imm[lane] >= 2) { + const Word end = pc + d.imm[lane] - 1; + if (end < count && decoded[end].end) mark(after_end, std::int64_t{pc} + 1); + } + } + } + } + + void Instruction(Word pc, Word count, const Decoded& d) { + // Keep the failing PC available for diagnostics, as the x86-64 code does. + MOV(W16, pc); + STR(W16, st, offsetof(State, fail_pc)); + // FIFO loads, in lane order. + for (Word i = 0; i < 2; ++i) { + const WReg load = i == 0 ? load0 : load1; + if (!d.load[i]) { + MOV(load, WZR); + continue; + } + LDR(W0, st, offsetof(State, consumed)); + LDR(W16, st, offsetof(State, params_size)); + CMP(W0, W16); + B(Cond::HS, Far(fail_underflow)); + LDR(X16, st, offsetof(State, params)); + LDR(load, X16, X0, oaknut::IndexExt::LSL, 2); + ADD(W0, W0, 1); + STR(W0, st, offsetof(State, consumed)); + } + if (!d.pred_mode) { + Body(pc, count, d, 15, true); + return; + } + Label when_false; + Get(W0, d, d.pred_reg, 0); + CBZ(W0, when_false); + Body(pc, count, d, d.mask_true, false); + l(when_false); + Body(pc, count, d, d.mask_false, true); + } + + // `last`: the code that follows is the next instruction's. + void Body(Word pc, Word count, const Decoded& d, Word mask, bool last) { + // An inactive lane's result reads as 0 when an output selects it. + for (Word lane = 0; lane < 2; ++lane) { + if (mask & (1U << lane)) continue; + bool selected = false; + for (Word out = 0; out < 2; ++out) { + if (!(mask & (4U << out))) continue; + selected |= d.out_method[out] == lane + 1 || d.out_emit[out] == lane + 1; + } + if (selected) { + MOV(Alu(lane), WZR); + } + } + const bool flow = ((mask & 1) && IsFlow(d.op[0])) || ((mask & 2) && IsFlow(d.op[1])); + if (flow) { + MOV(W16, pc + 1); + STR(W16, st, offsetof(State, next_pc)); + STRB(WZR, st, offsetof(State, cancel_end)); + } + // Lane 1's operands are read before lane 0 commits its destination (or calls out). + const bool lane0_active = (mask & 1) != 0; + const bool early_operands = + (mask & 2) && lane0_active && + (d.op[0] == 22 || (d.dst[0] < 24 && (d.dst[0] == d.src_a[1] || d.dst[0] == d.src_b[1]))); + if (early_operands) { + Get(W0, d, d.src_a[1], 1); + STR(W0, st, offsetof(State, scratch_x)); + Get(W0, d, d.src_b[1], 1); + STR(W0, st, offsetof(State, scratch_y)); + } + for (Word lane = 0; lane < 2; ++lane) { + if (!(mask & (1U << lane))) { + continue; + } + if (lane == 1 && early_operands) { + LDR(W1, st, offsetof(State, scratch_x)); + LDR(W2, st, offsetof(State, scratch_y)); + } else { + Get(W1, d, d.src_a[lane], lane); + Get(W2, d, d.src_b[lane], lane); + } + Lane(pc, d, lane); + // w0 holds the lane's value. + MOV(Alu(lane), W0); + if (d.dst[lane] < 24) { + STR(W0, st, RegOffset(d.dst[lane])); + } else if (d.dst[lane] != 24) { + B(fail_destination); + } + } + for (Word lane = 0; lane < 2; ++lane) { + if (!(mask & (4U << lane))) { + continue; + } + if (d.out_method[lane]) { + Output(W0, d, d.out_method[lane]); + AND(W1, W0, 0xfff); + STR(W1, st, offsetof(State, method)); + UBFX(W0, W0, 12, 4); + STR(W0, st, offsetof(State, increment)); + } + if (d.out_emit[lane]) { + Output(W2, d, d.out_emit[lane]); + LDR(W1, st, offsetof(State, method)); + // Same-value elision (State::elision): skip the callback for a state write + // that changes nothing. W1 (x1, zero-extended) is the method, < 0x1000. + Label call_emit, elided, after_emit; + LDR(X16, st, offsetof(State, elision)); + CBZ(X16, call_emit); + LDR(X0, X16, offsetof(Elision, must_call)); + LDRB(W17, X0, X1); + CBNZ(W17, call_emit); + LDR(X0, X16, offsetof(Elision, regs)); + LDR(W17, X0, X1, oaknut::IndexExt::LSL, 2); + CMP(W17, W2); + B(Cond::NE, call_emit); + LDR(X0, X16, offsetof(Elision, control)); + LDR(W17, X0, 0); + CMP(W17, 3); // Replay: the value written comes from the shadow RAM + B(Cond::EQ, call_emit); + CMP(W17, 2); // Passthrough: the shadow RAM is not involved + B(Cond::EQ, elided); + LDR(X0, X16, offsetof(Elision, shadow)); + LDR(W17, X0, X1, oaknut::IndexExt::LSL, 2); + CMP(W17, W2); + B(Cond::NE, call_emit); + l(elided); + LDR(W17, X16, offsetof(Elision, skipped)); + ADD(W17, W17, 1); + STR(W17, X16, offsetof(Elision, skipped)); + B(after_emit); + l(call_emit); + MOV(X0, st); + LDR(X16, st, offsetof(State, emit)); + BLR(X16); + TST(W0, 0xff); + B(Cond::EQ, Far(fail_emit)); + l(after_emit); + LDR(W0, st, offsetof(State, increment)); + LDR(W1, st, offsetof(State, method)); + ADD(W1, W1, W0); + STR(W1, st, offsetof(State, method)); + } + } + // An END in the previous instruction ends the macro after this one. + if (after_end[pc]) { + LDRB(W16, st, offsetof(State, end_next)); + CBNZ(W16, Far(success)); + } + if (flow) { + LDRB(W16, st, offsetof(State, stopped)); + CBNZ(W16, Far(success)); + } + if (d.end) { + if (flow) { + LDRB(W16, st, offsetof(State, cancel_end)); + EOR(W16, W16, 1); + STRB(W16, st, offsetof(State, end_next)); + } else { + MOV(W16, 1); + STRB(W16, st, offsetof(State, end_next)); + } + } + if (loop_end[pc]) { + Label no_loop; + LDR(W0, st, offsetof(State, loop_count)); + CBZ(W0, no_loop); + LDR(W16, st, offsetof(State, loop_end)); + CmpImm(W16, pc); + B(Cond::NE, no_loop); + SUB(W0, W0, 1); + STR(W0, st, offsetof(State, loop_count)); + LDR(W0, st, offsetof(State, loop_start)); + ADD(W0, W0, 1); + B(dispatch); + l(no_loop); + } + if (flow) { + LDR(W0, st, offsetof(State, next_pc)); + if (pc + 1 < count) { + CmpImm(W0, pc + 1); + B(Cond::NE, Far(dispatch)); + if (!last) { + B(labels[pc + 1]); + } + } else { + B(dispatch); + } + } else if (pc + 1 < count) { + if (!last) { + B(labels[pc + 1]); + } + } else { + B(fail_outside); + } + } + + void Output(WReg dst, const Decoded& d, Word selector) { + switch (selector) { + case 1: MOV(dst, alu0); return; + case 2: MOV(dst, alu1); return; + case 3: MOV(dst, load0); return; + case 4: MOV(dst, load1); return; + case 5: MOV(dst, d.imm[0]); return; + case 6: MOV(dst, d.imm[1]); return; + case 8: MOV(dst, d.imm[0] >> 12); return; + case 9: MOV(dst, d.imm[1] >> 12); return; + case 10: MOV(dst, (d.imm[0] << 16) | d.imm[1]); return; + default: B(fail_output); return; + } + } + + // The condition of comparison `op` (CMP x, y). + static Cond Condition(Word op) { + switch (op) { + case 17: case 25: return Cond::LT; + case 18: case 26: return Cond::LO; + case 19: case 27: return Cond::LE; + case 20: case 28: return Cond::LS; + default: return Cond::EQ; + } + } + + // w1 = x, w2 = y; leaves the value in w0. + void Lane(Word pc, const Decoded& d, Word lane) { + const Word imm = d.imm[lane]; + const Word op = d.op[lane]; + switch (op) { + case 0: + ADDS(W0, W1, W2); + CSET(W16, Cond::CS); + STR(W16, st, offsetof(State, carry)); + return; + case 1: + ADD(W0, W1, W2); + LDR(W16, st, offsetof(State, carry)); + ADD(W0, W0, W16); + return; + case 2: + // The carry is the borrow (x < y): ARM64's C flag is its inverse. + SUBS(W0, W1, W2); + CSET(W16, Cond::LO); + STR(W16, st, offsetof(State, carry)); + return; + case 3: + SUB(W0, W1, W2); + LDR(W16, st, offsetof(State, carry)); + SUB(W0, W0, W16); + return; + case 4: + SMULL(X0, W1, W2); + LSR(X16, X0, 32); + STR(W16, st, offsetof(State, carry)); + MOV(W0, W0); + return; + case 6: + UMULL(X0, W1, W2); + LSR(X16, X0, 32); + STR(W16, st, offsetof(State, carry)); + MOV(W0, W0); + return; + case 5: + LDR(W0, st, offsetof(State, carry)); + return; + case 7: + CmpImm(W1, 0x1000); + B(Cond::NE, Far(fail_extended)); + MOV(W0, WZR); + return; + case 8: + CLZ(W0, W1); + return; + case 9: LSL(W0, W1, W2); return; + case 10: LSR(W0, W1, W2); return; + case 11: ASR(W0, W1, W2); return; + case 12: AND(W0, W1, W2); return; + case 13: AND(W0, W1, W2); MVN(W0, W0); return; + case 14: ORR(W0, W1, W2); return; + case 15: EOR(W0, W1, W2); return; + case 16: { + const Word dest = (imm >> 10) & 63, bit_count = (imm >> 5) & 31; + const Word source = imm & 31; + if (dest >= 32) { + B(fail_merge); + return; + } + const Word bits_mask = (1U << bit_count) - 1; + MOV(W16, ~(bits_mask << dest)); + AND(W0, W1, W16); + LSR(W2, W2, source); + MOV(W16, bits_mask); + AND(W2, W2, W16); + LSL(W2, W2, dest); + ORR(W0, W0, W2); + return; + } + case 17: case 18: case 19: case 20: case 21: + CMP(W1, W2); + CSETM(W0, Condition(op)); + return; + case 22: + ADD(W16, W1, W2); + AND(W1, W16, 0xffff); + MOV(X0, st); + LDR(X16, st, offsetof(State, read)); + BLR(X16); + return; + case 23: { + if (imm < 2) { + B(fail_loop); + return; + } + Label skip, done; + LDR(W16, st, offsetof(State, loop_count)); + CBNZ(W16, Far(fail_loop)); + CBZ(W1, skip); + SUB(W1, W1, 1); + STR(W1, st, offsetof(State, loop_count)); + MOV(W16, pc); + STR(W16, st, offsetof(State, loop_start)); + MOV(W16, pc + imm - 1); + STR(W16, st, offsetof(State, loop_end)); + B(done); + l(skip); + MOV(W16, pc + imm); + STR(W16, st, offsetof(State, next_pc)); + l(done); + MOV(W0, WZR); + return; + } + case 24: + MOV(W16, pc + (imm & 0x7fff)); + STR(W16, st, offsetof(State, next_pc)); + MOV(W0, WZR); + return; + case 25: case 26: case 27: case 28: case 29: { + // Taken when the comparison equals bit 15. + Label not_taken; + CMP(W1, W2); + const Cond condition = Condition(op); + B((imm & 0x8000) ? oaknut::invert(condition) : condition, not_taken); + const auto branch_offset = std::int32_t(imm << 19) >> 19; + if (branch_offset == -4096) { + MOV(W16, 1); + STRB(W16, st, offsetof(State, stopped)); + } else { + MOV(W16, Word(std::int32_t(pc) + branch_offset)); + STR(W16, st, offsetof(State, next_pc)); + if (imm & 0x4000) { + MOV(W16, 1); + STRB(W16, st, offsetof(State, cancel_end)); + } else { + STRB(WZR, st, offsetof(State, cancel_end)); + } + } + l(not_taken); + MOV(W0, WZR); + return; + } + case 30: + CmpImm(W1, 0xc00); + B(Cond::HS, Far(fail_dread)); + LDR(X16, st, offsetof(State, dram)); + LDR(W0, X16, X1, oaknut::IndexExt::LSL, 2); + return; + case 31: + CmpImm(W1, 0xc00); + B(Cond::HS, Far(fail_dwrite)); + LDR(X16, st, offsetof(State, dram)); + STR(W2, X16, X1, oaknut::IndexExt::LSL, 2); + MOV(W0, WZR); + return; + default: + MOV(W0, WZR); + return; + } + } + + struct Stub { + Label label; + Label* target{}; + }; + + std::vector

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManager& device_memory_) : runtime{runtime_}, device_memory{device_memory_} { @@ -390,7 +431,7 @@ void TextureCache

::TickFrame() { std::vector> largest; std::map> by_format; for (const auto [image_id, image] : slot_images) { - const u64 bytes = GetScaledImageSizeBytes(*image); + const u64 bytes = ImageMemoryBytes(*image); image_bytes += bytes; ++images; if (debug_stats) { @@ -405,9 +446,9 @@ void TextureCache

::TickFrame() { image_bytes >> 20); std::partial_sort(largest.begin(), largest.begin() + std::min(12, largest.size()), largest.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); - for (size_t i = 0; i < std::min(12, largest.size()); ++i) { + for (size_t i = 0; i < std::min(8, largest.size()); ++i) { const Image& image = *largest[i].second; - LOG_DEBUG(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}", + LOG_INFO(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}", largest[i].first >> 20, image.gpu_addr, image.info.size.width, image.info.size.height, image.info.size.depth, image.info.resources.layers, image.info.resources.levels, static_cast(image.info.format), static_cast(image.info.type), @@ -417,7 +458,7 @@ void TextureCache

::TickFrame() { for (const auto& [format, total] : by_format) formats.emplace_back(total.first, format); std::sort(formats.rbegin(), formats.rend()); for (size_t i = 0; i < std::min(8, formats.size()); ++i) { - LOG_DEBUG(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20, + LOG_INFO(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20, by_format[formats[i].second].second); } if constexpr (requires { runtime.LogMemoryStatistics(); }) { @@ -1713,6 +1754,8 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); } + // Before the cold hint: memory given back is not hinted (the image is GPU modified then). + ReleaseGpuOnlyMemory(image); // The host GPU has its own copy now and the guest rarely reads this memory again: TickFrame // marks it cold, so the OS pages it out before anything else when it needs RAM (title 1 // streams several GB of textures through it). @@ -1764,6 +1807,40 @@ bool TextureCache

::UploadInChunks(Image& image) { return true; } +template +void TextureCache

::ReleaseGpuOnlyMemory(Image& image) { + // release_gpu_only_memory: a Switch 2 texture in GPU-only memory (regions nx2:gpu backs, whole + // or sparse) is never read by the guest CPU, and once uploaded its host image holds the data. + // Mario Kart World kept textures and render targets there twice, in RAM and on the GPU. + if (!Settings::values.release_gpu_only_memory.GetValue() || + !Settings::CurrentProgramIsSwitch2() || image.info.num_samples != 1 || + True(image.flags & (ImageFlagBits::Converted | ImageFlagBits::AcceleratedUpload | + ImageFlagBits::SparseLayers | ImageFlagBits::Unbacked | + ImageFlagBits::Sparse | ImageFlagBits::Remapped | + ImageFlagBits::Rescaled | ImageFlagBits::CpuModified)) || + !(Tegra::Nx2InGpuOnlyRegion(image.gpu_addr, image.guest_size_bytes) || + Tegra::Nx2InSparseRegion(image.gpu_addr, image.guest_size_bytes))) { + return; + } + if (True(image.flags & ImageFlagBits::TrimmedLevels)) { + // Only the levels the host doesn't keep (most of the bytes): nothing reads them, and the + // image uploads the levels it keeps from the guest memory left, after an eviction too. + const LevelArray offsets = CalculateMipLevelOffsets(image.info); + const u64 dropped = offsets[image.host_level_offset]; + const u64 layer_stride = + image.info.resources.layers > 1 ? image.info.layer_stride : image.guest_size_bytes; + for (s32 layer = 0; layer < image.info.resources.layers; ++layer) { + device_memory.ReleaseMemory(image.cpu_addr + layer * layer_stride, dropped); + } + return; + } + // The whole image, marked GPU modified: whatever reads that memory through the caches + // (flushes, eviction) downloads it back first. Only images the host keeps byte for byte as + // the guest does, so a download restores them. + MarkModification(image); + device_memory.ReleaseMemory(image.cpu_addr, image.guest_size_bytes); +} + template bool TextureCache

::UploadsByLayer(const ImageBase& image) const noexcept { // Title 1's terrain arrays (8192x8192x68 ASTC, 1.45 GB each) get a few layers streamed in at a @@ -1948,7 +2025,14 @@ void TextureCache

::UploadLayers(Image& image, std::span layers) { // Plain formats: each layer unswizzled on its own into its layer of the image. ImageInfo layer_info = image.info; layer_info.resources.layers = 1; - const size_t layer_size = CalculateUnswizzledSizeBytes(layer_info); + // Only the levels the host keeps (TrimmedLevels): a dropped 8192x8192 level 0 alone + // would be 64 MB of staging per layer. + const s32 first_level = static_cast(image.host_level_offset); + ImageInfo host_layer_info = layer_info; + host_layer_info.size.width = (std::max)(layer_info.size.width >> first_level, 1U); + host_layer_info.size.height = (std::max)(layer_info.size.height >> first_level, 1U); + host_layer_info.resources.levels -= first_level; + const size_t layer_size = CalculateUnswizzledSizeBytes(host_layer_info); auto staging = runtime.UploadStagingBuffer(layer_size * layers.size()); std::vector copies; for (size_t i = 0; i < layers.size(); ++i) { @@ -1958,7 +2042,8 @@ void TextureCache

::UploadLayers(Image& image, std::span layers) { &swizzle_data_buffer); auto layer_copies = UnswizzleImage(*gpu_memory, layer_addr, layer_info, swizzle_data, - staging.mapped_span.subspan(i * layer_size, layer_size)); + staging.mapped_span.subspan(i * layer_size, layer_size), + first_level); for (BufferImageCopy copy : layer_copies) { copy.buffer_offset += i * layer_size; copy.image_subresource.base_layer = static_cast(layers[i]); @@ -2031,9 +2116,11 @@ void TextureCache

::UploadImageContents(Image& image, StagingBuffer& staging) unswizzle_data_buffer.resize_destructive(image.unswizzled_size_bytes); auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data, unswizzle_data_buffer); ConvertImage(unswizzle_data_buffer, image.info, mapped_span, copies); + TrimHostLevels(image.host_level_offset, copies); image.UploadMemory(staging, copies); } else { - const auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data, mapped_span); + const auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data, + mapped_span, static_cast(image.host_level_offset)); image.UploadMemory(staging, copies); } } @@ -2224,6 +2311,17 @@ u64 TextureCache

::ImageMemoryBytes(const Image& image) const { } return 0; } + // A trimmed image (reduce_texture_resolution, or too large for the host) counts what its host + // image really takes, where the runtime knows it: a quarter of its guest size or less. + // Counted at guest size, Mario Kart World's trimmed terrain arrays kept the total over the + // critical limit for whole races, and the collector evicted (and re-uploaded) thousands of + // textures every few seconds. Other images count as before. + if constexpr (requires { image.HostMemoryBytes(); }) { + if (const std::optional host_bytes = image.HostMemoryBytes(); + host_bytes && True(image.flags & ImageFlagBits::TrimmedLevels)) { + return Common::AlignUp(*host_bytes, 1024); + } + } u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); if ((IsPixelFormatASTC(image.info.format) && True(image.flags & ImageFlagBits::AcceleratedUpload)) || @@ -2468,6 +2566,9 @@ ImageId TextureCache

::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, static_cast(info.format), info.block.width, info.block.height, info.block.depth); } const ImageId image_id = JoinImages(storage_info, gpu_addr, *cpu_addr); + if (!image_id) { + return ImageId{}; // Could not be created (see JoinImages) + } const Image& image = slot_images[image_id]; // Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr); @@ -2589,7 +2690,21 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA } } } - const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr); + ImageId new_image_id; + try { + new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr); + } catch (const std::exception& e) { + // The host cannot hold it at all: an 8192x8192x68 BC6H virtual texture is 6 GB on a + // driver with a 4 GB allocation limit and no real sparse images (Adreno). Sample it as + // the null image rather than aborting; nothing has been joined into it yet. + static u32 logged{}; + if (logged++ < 16) { + LOG_ERROR(HW_GPU, "Image {}x{}x{} ({} levels) at {:#x} could not be created ({}); using the null image", + new_info.size.width, new_info.size.height, new_info.resources.layers, + new_info.resources.levels, gpu_addr, e.what()); + } + return ImageId{}; + } Image& new_image = slot_images[new_image_id]; new_image.allocation_tick = frame_tick; @@ -2698,10 +2813,13 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA const u32 up_scale = can_rescale ? resolution.up_scale : 1; const u32 down_shift = can_rescale ? resolution.down_shift : 0; auto copies = MakeShrinkImageCopies(new_info, overlap.info, base, up_scale, down_shift); - if (overlap.info.num_samples != new_image.info.num_samples) { - runtime.CopyImageMSAA(new_image, overlap, copies); - } else { - runtime.CopyImage(new_image, overlap, copies); + TrimImageCopies(new_image.host_level_offset, overlap.host_level_offset, copies); + if (!copies.empty()) { + if (overlap.info.num_samples != new_image.info.num_samples) { + runtime.CopyImageMSAA(new_image, overlap, copies); + } else { + runtime.CopyImage(new_image, overlap, copies); + } } new_image.modification_tick = overlap.modification_tick; } @@ -2776,6 +2894,9 @@ std::optional::BlitImages> TextureCache

::GetBlitImag dst_id = InsertImage(dst_info, dst_addr, RelaxedOptions{}); } } while (has_deleted_images); + if (!src_id || !dst_id) { + return std::nullopt; // An image that could not be created (see JoinImages) + } const ImageBase& src_image = slot_images[src_id]; const ImageBase& dst_image = slot_images[dst_id]; const bool native_bgr = runtime.HasNativeBgr(); @@ -2791,6 +2912,9 @@ std::optional::BlitImages> TextureCache

::GetBlitImag src_id = FindOrInsertImage(src_info, src_addr, RelaxedOptions{}); dst_id = FindOrInsertImage(dst_info, dst_addr, RelaxedOptions{}); } while (has_deleted_images); + if (!src_id || !dst_id) { + return std::nullopt; + } } return {BlitImages{ .dst_id = dst_id, @@ -2907,6 +3031,21 @@ ImageViewId TextureCache

::FindDepthBuffer() { template ImageViewId TextureCache

::FindRenderTargetView(const ImageInfo& info, GPUVAddr gpu_addr) { + if (info.size.width == 0 || info.size.height == 0 || info.size.depth == 0) { + // No render target, as for a garbage one below: NULL_IMAGE_VIEW_ID is the null view + // (slot 0, a valid id), which BindRenderTarget would attach, and it has no format. + return ImageViewId{}; + } + // Render target registers filled by garbage commands (sizes like 2147487260x2147485323, or + // 0x0 with 36232 layers): creating the image would abort. + static constexpr u32 MaxRenderTargetSize = 32768; + static constexpr s32 MaxRenderTargetLayers = 2048; + if (info.size.width > MaxRenderTargetSize || info.size.height > MaxRenderTargetSize || + info.resources.layers > MaxRenderTargetLayers) { + LOG_ERROR(HW_GPU, "Ignoring {}x{} render target with {} layers at {:#x}", info.size.width, + info.size.height, info.resources.layers, gpu_addr); + return ImageViewId{}; // No render target: the null view has no format to size + } ImageId image_id{}; bool delete_state = has_deleted_images; do { @@ -3513,6 +3652,9 @@ void TextureCache

::SynchronizeAliases(ImageId image_id) { template void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool invalidate) { + if (image_id == NULL_IMAGE_ID) { + return; // See PrepareImageView + } Image& image = slot_images[image_id]; if (invalidate) { image.flags &= ~(ImageFlagBits::CpuModified | ImageFlagBits::GpuModified); @@ -3567,7 +3709,9 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool template void TextureCache

::PrepareImageView(ImageViewId image_view_id, bool is_modification, bool invalidate) { - if (!image_view_id) { + // The null view stands for textures that are unmapped or could not be created: it has no + // contents to upload (its image's format is invalid). + if (!image_view_id || image_view_id == NULL_IMAGE_VIEW_ID) { return; } const ImageViewBase& image_view = slot_image_views[image_view_id]; @@ -3590,6 +3734,10 @@ void TextureCache

::CopyImage(ImageId dst_id, ImageId src_id, std::vector BindSparseLayers(Image& image, std::span layers); + /// release_gpu_only_memory: gives back the guest memory of an uploaded image in Switch 2 + /// GPU-only memory, the host image holding the data from then on. + void ReleaseGpuOnlyMemory(Image& image); + /// Uploads `layers` of the image from guest memory. void UploadLayers(Image& image, std::span layers); @@ -320,6 +324,11 @@ class TextureCache : public VideoCommon::ChannelSetupCaches DmaBufferImageCopy( const Tegra::DMA::ImageCopy& copy_info, const Tegra::DMA::BufferOperand& buffer_operand, const Tegra::DMA::ImageOperand& image_operand, ImageId image_id, bool modifies_image); diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index 09e9513a5b..52f9161d78 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -854,7 +854,8 @@ boost::container::small_vector UnswizzleImage(Tegra::Memory GPUVAddr gpu_addr, const ImageInfo& info, std::span input, - std::span output) { + std::span output, + s32 first_level) { const size_t guest_size_bytes = input.size_bytes(); const u32 bpp_log2 = BytesPerBlockLog2(info.format); const Extent2D tile_size = DefaultBlockSize(info.format); @@ -890,20 +891,25 @@ boost::container::small_vector UnswizzleImage(Tegra::Memory info.tile_width_spacing); size_t guest_offset = 0; u32 host_offset = 0; - boost::container::small_vector copies(num_levels); + first_level = std::clamp(first_level, 0, num_levels - 1); + boost::container::small_vector copies(num_levels - first_level); for (s32 level = 0; level < num_levels; ++level) { + if (level < first_level) { + guest_offset += level_sizes[level]; + continue; + } const Extent3D level_size = AdjustMipSize(size, level); const u32 num_blocks_per_layer = NumBlocks(level_size, tile_size); const u32 host_bytes_per_layer = num_blocks_per_layer << bpp_log2; - copies[level] = BufferImageCopy{ + copies[level - first_level] = BufferImageCopy{ .buffer_offset = host_offset, .buffer_size = static_cast(host_bytes_per_layer) * num_layers, .buffer_row_length = Common::AlignUp(level_size.width, tile_size.width), .buffer_image_height = Common::AlignUp(level_size.height, tile_size.height), .image_subresource = { - .base_level = level, + .base_level = level - first_level, .base_layer = 0, .num_layers = info.resources.layers, }, diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index d91ee74c52..489ca3ad17 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -99,9 +99,11 @@ struct UnswizzlePlan { void ExecuteUnswizzlePlan(Tegra::MemoryManager& gpu_memory, const UnswizzlePlan& plan, std::span output); +/// first_level: unswizzle only the levels from it on, packed from the start of `output` and +/// numbered from 0 (for a host image that keeps only those levels). [[nodiscard]] boost::container::small_vector UnswizzleImage( Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info, - std::span input, std::span output); + std::span input, std::span output, s32 first_level = 0); /// For a single-layer 2D block-linear image of 1x1-texel blocks: the host sparse blocks /// (tile_width x tile_height texels) that the guest byte ranges (offset, size) touch, merged into diff --git a/src/video_core/transform_feedback.cpp b/src/video_core/transform_feedback.cpp index 53d29c08e2..3661e19332 100644 --- a/src/video_core/transform_feedback.cpp +++ b/src/video_core/transform_feedback.cpp @@ -70,7 +70,10 @@ std::pair, u32> MakeTransformF for (size_t buffer = 0; buffer < state.layouts.size(); ++buffer) { const auto& locations = state.varyings[buffer]; const auto& layout = state.layouts[buffer]; - const u32 varying_count = layout.varying_count; + // From guest registers: at most the 128 attributes `locations` holds (Mario Kart World + // read past the pipeline key with more). + const u32 varying_count = + (std::min)(layout.varying_count, static_cast(locations.size() * 4)); u32 highest = 0; for (u32 offset = 0; offset < varying_count; ++offset) { const auto get_attribute = [&locations](u32 index) -> u32 { diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 9ced844732..c07c7f369d 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -24,6 +24,7 @@ #include "common/fs/fs.h" #include "common/fs/path_util.h" #include "common/literals.h" +#include "common/memory_detect.h" #include #include "common/settings.h" #include "common/settings_enums.h" @@ -1597,9 +1598,17 @@ void Device::RemoveUnsuitableExtensions() { features.mesh_shader.multiviewMeshShader = VK_FALSE; features.mesh_shader.primitiveFragmentShadingRateMeshShader = VK_FALSE; features.mesh_shader.meshShaderQueries = VK_FALSE; + const bool has_mesh_extension = extensions.mesh_shader; + const bool task_shader = features.mesh_shader.taskShader != VK_FALSE; + const bool mesh_shader = features.mesh_shader.meshShader != VK_FALSE; extensions.mesh_shader = features.mesh_shader.taskShader && features.mesh_shader.meshShader; RemoveExtensionFeatureIfUnsuitable(extensions.mesh_shader, features.mesh_shader, VK_EXT_MESH_SHADER_EXTENSION_NAME); + LOG_INFO(Render_Vulkan, + "VK_EXT_mesh_shader: extension {}, taskShader {}, meshShader {}: Switch 2 task/mesh " + "draws {}", + has_mesh_extension ? "present" : "absent", task_shader, mesh_shader, + extensions.mesh_shader ? "on" : "skipped"); // VK_KHR_shader_quad_control extensions.shader_quad_control = features.shader_quad_control.shaderQuadControl; @@ -1715,6 +1724,23 @@ void Device::ReleaseCustomBorderColorSamplers(size_t count) const { } u64 Device::GetDeviceMemoryUsage() const { + // Without VK_EXT_memory_budget (Qualcomm's driver), what VMA's allocations use. The caches' + // own estimate counted sparse and unbacked images at their full size: Mario Kart World + // "used" 8.5 GB with 4.7 GB allocated, and evicted textures nonstop until the phone ran out of + // memory. Android only (CanReportMemoryUsage). +#ifdef __ANDROID__ + if (!extensions.memory_budget) { + u64 used{}; + if (allocator != nullptr) { + std::array heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + for (const size_t heap : valid_heap_memory) { + used += heap_budgets[heap].statistics.allocationBytes; + } + } + return used; + } +#endif VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; physical.GetMemoryProperties(&budget); @@ -1740,9 +1766,23 @@ u64 Device::GetDeviceMemorySlack() const { } u64 Device::GetDeviceMemoryBudget() const { +#ifdef ANDROID + // A phone GPU shares the system's RAM with the guest (12 GiB for a Switch 2 program): a + // fifth of it (at least 4 GiB), ~4.7 GiB on a 24 GiB phone. Without VK_EXT_memory_budget + // (Qualcomm) the integrated default of 4 GiB was under what Mario Kart World keeps in use; + // with it, a Turnip build reported ~9 GiB, and the textures it let in got the game killed + // for want of memory. Even with a quarter, loading a race (16-17 GB for the process, the + // guest's own peak) left the phone without memory. + const u64 integrated_budget = + (std::max)(Common::GetMemInfo().TotalPhysicalMemory / 5, u64{4} << 30); + if (!extensions.memory_budget) { + return is_integrated ? integrated_budget : 0; + } +#else if (!extensions.memory_budget) { return 0; } +#endif VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; physical.GetMemoryProperties(&budget); @@ -1768,7 +1808,12 @@ u64 Device::GetDeviceMemoryBudget() const { // budget): following it let the texture cache fill that much RAM before collecting, and // kept ASTC uncompressed. Its budget is at most the share CollectPhysicalMemoryInfo // gives it (4 GiB, 6 GiB with aggressive VRAM use). +#ifdef ANDROID + // On a phone: a quarter of the RAM (see above). + result = (std::min)(result, integrated_budget); +#else result = (std::min)(result, device_access_memory); +#endif } return result; } diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index f3f9209c99..0f207c5059 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -1026,8 +1026,13 @@ FN_MAX_LIMIT_LIST return device_access_memory; } + /// With VK_EXT_memory_budget from the driver, or VMA's allocations without it. bool CanReportMemoryUsage() const { +#ifdef __ANDROID__ + return extensions.memory_budget || allocator != nullptr; +#else return extensions.memory_budget; +#endif } u64 GetDeviceMemoryUsage() const; @@ -1043,6 +1048,11 @@ FN_MAX_LIMIT_LIST /// yet the driver does: with the heap at its budget it is not available to new images. u64 GetDeviceMemorySlack() const; + /// Most bytes a sparse (unbacked) resource may span. + u64 GetSparseAddressSpaceSize() const { + return properties.properties.limits.sparseAddressSpaceSize; + } + u32 GetSetsPerPool() const { return sets_per_pool; } diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index a0a39abe39..57ecaa65ad 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -206,7 +206,8 @@ MemoryAllocator::MemoryAllocator(const Device &device_) MemoryAllocator::~MemoryAllocator() = default; -vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated) const +vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated, + bool back_huge) const { const VmaAllocationCreateInfo alloc_ci = { // Best fit: images freed by the thousand leave holes in the blocks, which new ones @@ -231,7 +232,7 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat // backed: VMA would quietly place them in system memory once VRAM is full. They take the // unbacked sparse fallback below. const u64 texels = u64{ci.extent.width} * ci.extent.height * ci.extent.depth * ci.arrayLayers; - const bool huge = texels > (u64{1} << 28); + const bool huge = !back_huge && texels > (u64{1} << 28); VkResult result = huge ? VK_ERROR_OUT_OF_DEVICE_MEMORY : vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 8b9e72b3e3..7e7e0c213b 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -108,7 +108,9 @@ namespace Vulkan { /// `dedicated`: own device memory, returned to the driver on destruction (short-lived /// images would otherwise leave holes in the shared blocks). - vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false) const; + /// back_huge: back an image even past the size this keeps unbacked (unified memory). + vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false, + bool back_huge = false) const; /// A sparse resident image with nothing bound (reads return zero, writes are dropped), or /// a null image when the device can't make one. diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index d0bca88125..ee9e47ff81 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -555,8 +555,40 @@ void Buffer::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER, name); } +std::span Buffer::HostView() { + if (!host_view.empty()) { + return host_view; + } + if (allocator == nullptr || allocation == nullptr) { + return {}; + } + VkMemoryPropertyFlags flags{}; + vmaGetAllocationMemoryProperties(allocator, allocation, &flags); + if ((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) { + return {}; + } + void* data{}; + if (vmaMapMemory(allocator, allocation, &data) != VK_SUCCESS || data == nullptr) { + return {}; + } + VmaAllocationInfo info{}; + vmaGetAllocationInfo(allocator, allocation, &info); + host_view = std::span{static_cast(data), static_cast(info.size)}; + host_view_coherent = (flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; + return host_view; +} + +void Buffer::InvalidateHostView() const { + if (!host_view.empty() && !host_view_coherent) { + vmaInvalidateAllocation(allocator, allocation, 0, VK_WHOLE_SIZE); + } +} + void Buffer::Release() const noexcept { if (handle) { + if (!host_view.empty()) { + vmaUnmapMemory(allocator, allocation); + } vmaDestroyBuffer(allocator, handle, allocation); } } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index f49e9518bb..cd50153c4d 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -696,6 +696,13 @@ class Image { return handle != VK_NULL_HANDLE && allocation == VK_NULL_HANDLE; } + /// The memory the image needs, as the driver reports it. + [[nodiscard]] VkDeviceSize MemorySize() const noexcept { + VkMemoryRequirements requirements{}; + dld->vkGetImageMemoryRequirements(owner, handle, &requirements); + return requirements.size; + } + Image(const Image&) = delete; Image& operator=(const Image&) = delete; @@ -779,7 +786,8 @@ class Buffer { Buffer(Buffer&& rhs) noexcept : handle{std::exchange(rhs.handle, VkBuffer{})}, owner{rhs.owner}, allocator{rhs.allocator}, allocation{rhs.allocation}, mapped{rhs.mapped}, location{rhs.location}, - is_coherent{rhs.is_coherent}, dld{rhs.dld} {} + is_coherent{rhs.is_coherent}, dld{rhs.dld}, host_view{std::exchange(rhs.host_view, {})}, + host_view_coherent{rhs.host_view_coherent} {} Buffer& operator=(Buffer&& rhs) noexcept { Release(); @@ -791,6 +799,8 @@ class Buffer { location = rhs.location; is_coherent = rhs.is_coherent; dld = rhs.dld; + host_view = std::exchange(rhs.host_view, {}); + host_view_coherent = rhs.host_view_coherent; return *this; } @@ -834,6 +844,14 @@ class Buffer { void Invalidate() const; + /// The memory for the host to read, mapped on first use when its type is host visible (a + /// device-local buffer of an integrated GPU); empty otherwise. Unlike Mapped(), it does not + /// make IsHostVisible() true: writes still go through copies ordered with the GPU's work. + std::span HostView(); + + /// Makes what the device wrote visible to HostView() reads. + void InvalidateHostView() const; + void SetObjectNameEXT(const char* name) const; MemoryLocation Location() const noexcept { @@ -851,6 +869,8 @@ class Buffer { MemoryLocation location{}; bool is_coherent = false; const DeviceDispatch* dld = nullptr; + std::span host_view = {}; + bool host_view_coherent = false; }; class Queue { diff --git a/src/yuzu/main.ui b/src/yuzu/main.ui index 5fe25aa252..c35df3f262 100644 --- a/src/yuzu/main.ui +++ b/src/yuzu/main.ui @@ -219,6 +219,7 @@ + @@ -290,6 +291,11 @@ &Verify Installed Contents + + + &Pack Switch 2 Dump to .nx2... + + &About Eglantine diff --git a/src/yuzu/main_window.cpp b/src/yuzu/main_window.cpp index 741b248e4d..007a248bf4 100644 --- a/src/yuzu/main_window.cpp +++ b/src/yuzu/main_window.cpp @@ -8,6 +8,11 @@ #undef VMA_IMPLEMENTATION #endif +#include +#include +#include +#include + #include #include "common/fs/path_util.h" #include "common/game_presets.h" @@ -155,6 +160,7 @@ static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::Virtual #include "core/file_sys/card_image.h" #include "core/file_sys/common_funcs.h" #include "core/file_sys/nup_writer.h" +#include "core/file_sys/nx2_package.h" #include "core/file_sys/romfs.h" #include "core/file_sys/savedata_factory.h" @@ -1639,6 +1645,7 @@ void MainWindow::ConnectMenuEvents() { connect_menu(ui->action_Log_Folder, &MainWindow::OnOpenLogFolder); connect_menu(ui->action_Verify_installed_contents, &MainWindow::OnVerifyInstalledContents); + connect_menu(ui->action_Pack_Nx2, &MainWindow::OnPackNx2); connect_menu(ui->action_Firmware_From_Folder, &MainWindow::OnInstallFirmware); connect_menu(ui->action_Firmware_From_ZIP, &MainWindow::OnInstallFirmwareFromZIP); connect_menu(ui->action_Install_Keys, &MainWindow::OnInstallDecryptionKeys); @@ -4005,6 +4012,122 @@ void MainWindow::OnVerifyInstalledContents() { QtCommon::Content::VerifyInstalledContents(); } +// Packs a loose Switch 2 dump (exefs/ + romfs/) into one .nx2 file, which loads like any other +// game file on PC and Android. +void MainWindow::OnPackNx2() { + const QString dump_path = QFileDialog::getExistingDirectory( + this, tr("Select a Switch 2 Dump (the folder holding exefs and romfs)"), + QString::fromStdString(UISettings::values.roms_path)); + if (dump_path.isEmpty()) { + return; + } + + // Name the package after the dump folder, next to it. + QDir title_dir{dump_path}; + if (title_dir.dirName().compare(QStringLiteral("exefs"), Qt::CaseInsensitive) == 0) { + title_dir.cdUp(); + } + const QString name = title_dir.dirName(); + QDir parent_dir = title_dir; + parent_dir.cdUp(); + const QString out_path = QFileDialog::getSaveFileName( + this, tr("Save Packed Dump"), + parent_dir.filePath(name + QStringLiteral(".") + + QString::fromUtf8(FileSys::Nx2Extension.data(), + static_cast(FileSys::Nx2Extension.size()))), + tr("Packed Switch 2 Dump (*.nx2)")); + if (out_path.isEmpty()) { + return; + } + + const auto dump_dir = + QtCommon::vfs->OpenDirectory(dump_path.toStdString(), FileSys::OpenMode::Read); + const std::string out_path_string = out_path.toStdString(); + + QProgressDialog progress(tr("Packing %1...\nIndexing the RomFS").arg(name), tr("Cancel"), 0, + 1000, this); + progress.setWindowModality(Qt::WindowModal); + progress.setMinimumDuration(0); + progress.setAutoClose(false); + progress.setAutoReset(false); + progress.setValue(0); + + // Packed on its own thread: this one keeps the window (and the game list, which reloads when + // the package appears in a game folder) responsive and shows progress. + std::atomic done{0}; + std::atomic total{0}; + std::atomic cancel{false}; + std::atomic finished{false}; + std::string error; + std::thread packer([&] { + error = FileSys::PackNx2(dump_dir, out_path_string, [&](u64 done_bytes, u64 total_bytes) { + done = done_bytes; + total = total_bytes; + return !cancel; + }); + finished = true; + }); + + const auto format_duration = [this](qint64 seconds) { + if (seconds >= 3600) { + return tr("%1 h %2 min").arg(seconds / 3600).arg((seconds % 3600) / 60); + } + if (seconds >= 60) { + return tr("%1 min").arg((seconds + 30) / 60); + } + return tr("%1 s").arg(seconds); + }; + std::optional copy_start; + QEventLoop loop; + QTimer timer; + connect(&timer, &QTimer::timeout, &loop, [&] { + if (progress.wasCanceled()) { + cancel = true; + } + const u64 done_bytes = done; + const u64 total_bytes = total; + if (total_bytes != 0) { + if (!copy_start) { + copy_start = std::chrono::steady_clock::now(); + } + progress.setValue(static_cast(done_bytes * 1000 / total_bytes)); + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - *copy_start) + .count(); + QString speed_eta; + if (done_bytes != 0 && elapsed >= 3.0) { + const double rate = static_cast(done_bytes) / elapsed; + speed_eta = tr(" · %1 MB/s · about %2 left") + .arg(rate / 1048576.0, 0, 'f', 0) + .arg(format_duration(static_cast( + static_cast(total_bytes - done_bytes) / rate))); + } + progress.setLabelText(tr("Packing %1...\n%2 of %3 GB%4") + .arg(name) + .arg(static_cast(done_bytes) / 1073741824.0, 0, 'f', 1) + .arg(static_cast(total_bytes) / 1073741824.0, 0, 'f', 1) + .arg(speed_eta)); + } + if (finished) { + loop.quit(); + } + }); + timer.start(250); + loop.exec(); + // Stopped before the result box: a tick during its event loop would show the dialog again. + timer.stop(); + packer.join(); + progress.close(); + + if (!error.empty()) { + QMessageBox::warning(this, tr("Packing Failed"), QString::fromStdString(error)); + return; + } + QMessageBox::information( + this, tr("Packing Succeeded"), + tr("Saved %1.\nIt loads like any other game file, on PC and on Android.").arg(out_path)); +} + void MainWindow::OnInstallFirmware() { QtCommon::Content::InstallFirmware(); OnCheckFirmwareDecryption(); diff --git a/src/yuzu/main_window.h b/src/yuzu/main_window.h index 9e98b97ef9..b2a6535e13 100644 --- a/src/yuzu/main_window.h +++ b/src/yuzu/main_window.h @@ -393,6 +393,7 @@ private slots: void OnOpenModFolder(); void OnOpenLogFolder(); void OnVerifyInstalledContents(); + void OnPackNx2(); void OnInstallFirmware(); void OnInstallFirmwareFromZIP(); void OnInstallDecryptionKeys();