diff --git a/.kres.yaml b/.kres.yaml index 16f2a9a2d..05cb8297b 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -74,6 +74,7 @@ spec: - mellanox-mstflint-pkg - nvidia-open-gpu-kernel-modules-lts-pkg - nvidia-open-gpu-kernel-modules-production-pkg + - nvidia-tegra-nvgpu-pkg - px-fuse-pkg - tenstorrent-pkg - xdma-driver-pkg diff --git a/Pkgfile b/Pkgfile index a74f24ba8..c8e285a15 100644 --- a/Pkgfile +++ b/Pkgfile @@ -311,6 +311,20 @@ vars: gdrcopy_sha256: 32bc7b2c198dd97ec251de0ff4823252c95e31a4c79a5f843c82514c9af2052b gdrcopy_sha512: c717f118eff8cd5a8dc35613c3881818f8b71dc493461dd0151ce7c882f8e2c2d852e22733fab4e2bec57219e10eec874c11b4fad90dd4815ae572840ed19d28 + # OE4T (NVIDIA Tegra) kernel modules for Jetson Orin NX (Tegra234 / GA10B) + # renovate: datasource=git-refs versioning=git depName=https://github.com/OE4T/linux-nvgpu.git + oe4t_nvgpu_commit: d530a48d64f9ad3020d9f3307f53e8dde8e3fba1 + oe4t_nvgpu_sha256: adc5864edf76d986866e386803a9e628ee229e69ea34867b92b978a0b44f3d54 + oe4t_nvgpu_sha512: a7c7f0b5d3174bf41abc77c77009f46182358f93936aedbe4993e63ff7fc94e21bfd83c3fa0b41af5836866b9c200427504d8f26685d567c11722e7a7bfd3ed9 + # renovate: datasource=git-refs versioning=git depName=https://github.com/OE4T/linux-nv-oot.git + oe4t_nv_oot_commit: ea32e7f97dd04c3f75aadc22424dc63568387120 + oe4t_nv_oot_sha256: 9d2d70a121a418be307e3d1cd3c74d9ae9398e7abc756304d614e998dfd6f342 + oe4t_nv_oot_sha512: 5645163e964bfb13d7aa2ee1749188fe40a1fe9012080f548548f7dc70e4397a762c161041d8d209d2cd969cbb4aab36ea5c560ef5967946eeb3f1dd16335b9c + # renovate: datasource=git-refs versioning=git depName=https://github.com/OE4T/linux-hwpm.git + oe4t_hwpm_commit: 4d8a6998760d85f98637dbf61597bfbb88158206 + oe4t_hwpm_sha256: 96c7656bdad0bf330e7fd58981b8a4eec4717a76840cefbe84e720d88b46be55 + oe4t_hwpm_sha512: 971b91fcae284c59dbe411356109bce9b1a7884b8fac41c9683c79bf3eddef606e71ebaa9c06ad2389b2ba382c3c1125fabe0cbaeb5edac857e218077ed24ef9 + # renovate: datasource=github-releases depName=NVIDIA/gds-nvidia-fs gds_nvidia_fs_version: v2.29.4 gds_nvidia_fs_sha256: 6936aeacfb519a1d6fe66e16281799c20ffe177c2022f831d29f90895f11e339 diff --git a/nvidia-tegra-nvgpu/patches/nvgpu/0001-nvhost-syncpt-retry-and-skip-id0.patch b/nvidia-tegra-nvgpu/patches/nvgpu/0001-nvhost-syncpt-retry-and-skip-id0.patch new file mode 100644 index 000000000..b268da848 --- /dev/null +++ b/nvidia-tegra-nvgpu/patches/nvgpu/0001-nvhost-syncpt-retry-and-skip-id0.patch @@ -0,0 +1,77 @@ +nvgpu: make host1x syncpoint allocation robust on GA10B + +Two fixes for CUDA error 999 on the first cudaStreamSynchronize(): +1. Retry host1x_syncpt_alloc() for up to 5 ms; it returns NULL during + the short async GR init window after the first kernel submit. +2. Never hand out syncpt id 0. Without a "gpu" syncpt pool in the DT, + host1x allocates id 0 first, which GA10B rejects + (NVGPU_ERRATA_SYNCPT_INVALID_ID_0). + +diff --git a/drivers/gpu/nvgpu/os/linux/nvhost_host1x.c b/drivers/gpu/nvgpu/os/linux/nvhost_host1x.c +index 945fff7..bf88a52 100644 +--- a/drivers/gpu/nvgpu/os/linux/nvhost_host1x.c ++++ b/drivers/gpu/nvgpu/os/linux/nvhost_host1x.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. + * +@@ -237,17 +238,50 @@ void nvgpu_nvhost_syncpt_put_ref_ext(struct nvgpu_nvhost_dev *nvhost_dev, + u32 nvgpu_nvhost_get_syncpt_client_managed(struct nvgpu_nvhost_dev *nvhost_dev, + const char *syncpt_name) + { +- struct host1x_syncpt *sp; ++ struct host1x_syncpt *sp = NULL; + struct host1x *host1x; ++ int retry; + +- host1x = platform_get_drvdata(nvhost_dev->host1x_pdev); +- if (!host1x) +- return 0; +- +- sp = host1x_syncpt_alloc(host1x, HOST1X_SYNCPT_CLIENT_MANAGED | HOST1X_SYNCPT_GPU, +- syncpt_name); +- if (!sp) ++ /* ++ * During the ~1-2 ms async GR init window after the first kernel submit, ++ * host1x_syncpt_alloc() can return NULL. Returning 0 here makes channel ++ * sync creation fail (GA10B rejects syncpt id 0) and CUDA reports error 999 ++ * on the first cudaStreamSynchronize(). Retry briefly to cover the window. ++ */ ++ for (retry = 0; retry < 5; retry++) { ++ host1x = platform_get_drvdata(nvhost_dev->host1x_pdev); ++ if (!host1x) { ++ pr_warn_ratelimited("nvgpu: host1x not ready, syncpt retry %d/5\n", retry + 1); ++ msleep(1); ++ continue; ++ } ++ sp = host1x_syncpt_alloc(host1x, ++ HOST1X_SYNCPT_CLIENT_MANAGED | HOST1X_SYNCPT_GPU, ++ syncpt_name); ++ if (sp) ++ break; ++ pr_warn_ratelimited("nvgpu: syncpt_alloc NULL, retry %d/5\n", retry + 1); ++ msleep(1); ++ } ++ if (!sp) { ++ pr_err_ratelimited("nvgpu: get_syncpt_client_managed: failed after retries\n"); + return 0; ++ } ++ /* ++ * GA10B (NVGPU_ERRATA_SYNCPT_INVALID_ID_0) rejects syncpt id 0. Without a ++ * "gpu" syncpt pool in the DT, host1x hands out id 0 first. Hold it and ++ * allocate again so the GPU gets id >= 1; the Tegra234 syncpt shim covers ++ * all 1024 ids, so any non-zero id is GPU-signable. ++ */ ++ if (host1x_syncpt_id(sp) == 0U) { ++ struct host1x_syncpt *sp_skip = sp; ++ sp = host1x_syncpt_alloc(host1x, ++ HOST1X_SYNCPT_CLIENT_MANAGED | HOST1X_SYNCPT_GPU, ++ syncpt_name); ++ host1x_syncpt_put(sp_skip); ++ if (!sp) ++ return 0; ++ } + + return host1x_syncpt_id(sp); + } diff --git a/nvidia-tegra-nvgpu/patches/nvgpu/0002-netlist-flexible-array.patch b/nvidia-tegra-nvgpu/patches/nvgpu/0002-netlist-flexible-array.patch new file mode 100644 index 000000000..bd0b79eeb --- /dev/null +++ b/nvidia-tegra-nvgpu/patches/nvgpu/0002-netlist-flexible-array.patch @@ -0,0 +1,19 @@ +nvgpu: netlist regions as flexible array member + +regions[1] is indexed by the region count from the firmware header, so +UBSAN flags every access with index >= 1. A flexible array member has the +same layout and silences the false positive. + +diff --git a/drivers/gpu/nvgpu/common/netlist/netlist_priv.h b/drivers/gpu/nvgpu/common/netlist/netlist_priv.h +index 0ad50e2..b224ed9 100644 +--- a/drivers/gpu/nvgpu/common/netlist/netlist_priv.h ++++ b/drivers/gpu/nvgpu/common/netlist/netlist_priv.h +@@ -113,7 +113,7 @@ struct netlist_image_header { + + struct netlist_image { + struct netlist_image_header header; +- struct netlist_region regions[1]; ++ struct netlist_region regions[]; + }; + + struct netlist_gr_ucode { diff --git a/nvidia-tegra-nvgpu/patches/nvidia-oot/0001-tegra-drm-headless-no-fbdev.patch b/nvidia-tegra-nvgpu/patches/nvidia-oot/0001-tegra-drm-headless-no-fbdev.patch new file mode 100644 index 000000000..dcc7c0aa5 --- /dev/null +++ b/nvidia-tegra-nvgpu/patches/nvidia-oot/0001-tegra-drm-headless-no-fbdev.patch @@ -0,0 +1,29 @@ +tegra-drm: build headless, without fbdev emulation + +The kernel config has CONFIG_DRM_FBDEV_EMULATION=y, which the module +inherits through autoconf.h and cannot undefine via ccflags. fbdev.c +relies on drm_fb_helper internals that changed heavily between 6.0 and +6.18 and is not needed for a render node, so always take the stub path. + +diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h +index 147ce75..1224e96 100644 +--- a/drivers/gpu/drm/tegra/drm.h ++++ b/drivers/gpu/drm/tegra/drm.h +@@ -228,7 +228,7 @@ struct drm_framebuffer *tegra_fb_create(struct drm_device *drm, + const struct drm_mode_fb_cmd2 *cmd); + + #if defined(NV_DRM_DRIVER_HAS_FBDEV_PROBE) /* Linux v6.13 */ +-#ifdef CONFIG_DRM_FBDEV_EMULATION ++#if 0 /* fbdev not needed for headless CUDA */ + int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, + struct drm_fb_helper_surface_size *sizes); + #define TEGRA_FBDEV_DRIVER_OPS \ +@@ -238,7 +238,7 @@ int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, + .fbdev_probe = NULL + #endif + #else +-#ifdef CONFIG_DRM_FBDEV_EMULATION ++#if 0 /* fbdev not needed for headless CUDA */ + void tegra_fbdev_setup(struct drm_device *drm); + #else + static inline void tegra_fbdev_setup(struct drm_device *drm) diff --git a/nvidia-tegra-nvgpu/pkg.yaml b/nvidia-tegra-nvgpu/pkg.yaml new file mode 100644 index 000000000..97088dda3 --- /dev/null +++ b/nvidia-tegra-nvgpu/pkg.yaml @@ -0,0 +1,137 @@ +name: nvidia-tegra-nvgpu-pkg +variant: scratch +shell: /bin/bash +dependencies: + - stage: base + - stage: kernel-build + - image: "{{ .LLVM_IMAGE }}:{{ .TOOLS_REV }}" +steps: + - sources: + # OE4T patched nvgpu - supports kernel 6.x (fixes platform_driver.remove, hrtimer, struct fd) + - url: https://github.com/OE4T/linux-nvgpu/archive/{{ .oe4t_nvgpu_commit }}.tar.gz + destination: nvgpu.tar.gz + sha256: "{{ .oe4t_nvgpu_sha256 }}" + sha512: "{{ .oe4t_nvgpu_sha512 }}" + # OE4T patched nvidia-oot - patches-r36.5 branch (kernel 6.18 compat: __assign_str, + # f_count->f_ref, __alloc_pages_bulk 5-arg, and all earlier 6.x fixes) + - url: https://github.com/OE4T/linux-nv-oot/archive/{{ .oe4t_nv_oot_commit }}.tar.gz + destination: nvidia-oot.tar.gz + sha256: "{{ .oe4t_nv_oot_sha256 }}" + sha512: "{{ .oe4t_nv_oot_sha512 }}" + # OE4T patched hwpm - supports kernel 6.x (fixes platform_driver.remove, MODULE_IMPORT_NS) + - url: https://github.com/OE4T/linux-hwpm/archive/{{ .oe4t_hwpm_commit }}.tar.gz + destination: hwpm.tar.gz + sha256: "{{ .oe4t_hwpm_sha256 }}" + sha512: "{{ .oe4t_hwpm_sha512 }}" + env: + ARCH: arm64 + LLVM: "1" + LLVM_IAS: "1" + # conftest compiles its probes with $CC (Makefile default: cc = GCC). Probing + # Clang-built kernel headers with GCC fails wholesale, which leaves every NV_* + # macro undefined and selects the wrong code paths. The kernel build itself + # ignores CC because LLVM=1 selects clang. + CC: clang + # The OE4T top-level Makefiles add these include paths for every module; the + # module directories are built individually here, so pass them globally. + KCFLAGS: >- + -I/oot-src/out/nvidia-conftest + -I/oot-src/nvidia-oot/include + -I/oot-src/nvidia-oot/drivers/gpu/host1x/include + -I/oot-src/nvidia-oot/drivers/video/tegra/nvmap/include + # Common make arguments for every out-of-tree module directory. -Werror is + # dropped from the OE4T Makefiles because Clang warns where GCC does not. + OOT_MAKE: >- + CONFIG_TEGRA_OOT_MODULE=m ccflags-remove-y=-Werror LLVM=1 + srctree.nvidia-oot=/oot-src/nvidia-oot srctree.nvidia=/oot-src/nvidia-oot + srctree.hwpm=/oot-src/hwpm srctree.nvconftest=/oot-src/out/nvidia-conftest + KBUILD_EXTRA_SYMBOLS=/oot-src/out/Module.symvers.nvidia + prepare: + - | + for t in nvgpu nvidia-oot hwpm; do + mkdir -p /oot-src/$t + tar xzf $t.tar.gz -C /oot-src/$t --strip-components=1 + done + # patches//*.patch applies to /oot-src/ + for p in /pkg/patches/*/*.patch; do t=${p%/*}; patch -p1 -d /oot-src/${t##*/} < $p; done + build: + - | + # conftest probes the kernel API and writes conftest/*.h, which selects the + # code paths in every module (see CC above). Its Makefile hardcodes -Werror, + # which turns Clang-only warnings into failed probes: 26 of 129 macros go + # missing (measured), e.g. NV___ASSIGN_STR_HAS_NO_SRC_ARG, and one inverted + # "function present" probe reports the wrong answer. + CONFTEST=/oot-src/out/nvidia-conftest/nvidia + mkdir -p $CONFTEST + cp -a /oot-src/nvidia-oot/scripts/conftest/. $CONFTEST/ + sed -i 's/ -Werror / /' $CONFTEST/Makefile + make -j$(nproc) -f $CONFTEST/Makefile src=$CONFTEST obj=$CONFTEST \ + NV_KERNEL_SOURCES=/src NV_KERNEL_OUTPUT=/src + # a probe every 6.3+ kernel must pass: fails the build if conftest is broken + grep -rq '#define NV_IOMMU_MAP_HAS_GFP_ARG' $CONFTEST/conftest/ + - | + OOT=/oot-src/nvidia-oot + : > /oot-src/out/Module.symvers.nvidia + + # Build one module directory against the kernel and add its exported symbols + # for the directories that follow, so the order below matters. + oot() { + make -j$(nproc) -C /src M=$1 "${@:2}" $OOT_MAKE modules + cat $1/Module.symvers >> /oot-src/out/Module.symvers.nvidia + } + + oot $OOT/drivers/gpu/host1x + oot $OOT/drivers/platform/tegra/mc-utils + oot $OOT/drivers/gpu/host1x-fence + oot $OOT/drivers/gpu/host1x-nvhost + oot /oot-src/hwpm/drivers/tegra/hwpm + # headless: no fbdev (the kernel config has it enabled, so it must be forced off) + oot $OOT/drivers/gpu/drm/tegra CONFIG_DRM_FBDEV_EMULATION=n + # the OE4T defaults for 5.10+ are handle-as-id and NvSciIpc support, neither is wanted here + oot $OOT/drivers/video/tegra/nvmap NVMAP_CONFIG_HANDLE_AS_ID=n NVMAP_CONFIG_SCIIPC=n + oot $OOT/drivers/devfreq + # syncpoints through the OE4T host1x (the kernel has neither GRHOST nor HOST1X_NEXT) + oot /oot-src/nvgpu/drivers/gpu/nvgpu CONFIG_TEGRA_GK20A_NVHOST=y CONFIG_TEGRA_GK20A_NVHOST_HOST1X=y + install: + - | + OOT=/oot-src/nvidia-oot + MOD=/rootfs/usr/lib/modules/$(cat /src/include/config/kernel.release) + mkdir -p $MOD + cp /src/modules.order /src/modules.builtin /src/modules.builtin.modinfo $MOD/ + + # Modules are stripped after signing, see + # https://www.kernel.org/doc/html/v4.15/admin-guide/module-signing.html#signed-modules-and-stripping + oot_install() { + make -j$(nproc) -C /src M=$1 "${@:3}" $OOT_MAKE \ + INSTALL_MOD_PATH=/rootfs/usr INSTALL_MOD_DIR=$2 \ + INSTALL_MOD_STRIP=1 CONFIG_MODULE_SIG_ALL=y modules_install + } + + # host1x and tegra-drm are installed at the in-tree paths they shadow. depmod + # resolves kernel/drivers/gpu/drm/tegra/tegra-drm.ko against + # kernel/drivers/gpu/host1x/host1x.ko, so installing only under extra/ leaves + # modprobe pointing at the vanilla tegra-drm.ko, which then fails with + # "tegra_drm: disagrees about version of symbol host1x_job_alloc". + oot_install $OOT/drivers/gpu/host1x kernel/drivers/gpu/host1x + oot_install $OOT/drivers/gpu/drm/tegra kernel/drivers/gpu/drm/tegra CONFIG_DRM_FBDEV_EMULATION=n + oot_install $OOT/drivers/gpu/host1x-fence extra/nvidia-tegra + oot_install $OOT/drivers/gpu/host1x-nvhost extra/nvidia-tegra + oot_install /oot-src/hwpm/drivers/tegra/hwpm extra/nvidia-tegra + oot_install $OOT/drivers/video/tegra/nvmap extra/nvidia-tegra NVMAP_CONFIG_HANDLE_AS_ID=n NVMAP_CONFIG_SCIIPC=n + oot_install $OOT/drivers/platform/tegra/mc-utils extra/nvidia-tegra + oot_install $OOT/drivers/devfreq extra/nvidia-tegra + oot_install /oot-src/nvgpu/drivers/gpu/nvgpu extra/nvidia-tegra \ + CONFIG_TEGRA_GK20A_NVHOST=y CONFIG_TEGRA_GK20A_NVHOST_HOST1X=y + + # Load order: host1x provides syncpoint alloc, host1x-fence the dma_fence + # bridge, host1x-nvhost the nvhost device layer. tegra-drm needs all three and + # registers /dev/dri/renderD128, which libcuda requires to enumerate the iGPU. + # nvmap provides GPU memory, mc-utils emc_freq_to_bw, nvgpu comes last. + mkdir -p /rootfs/usr/lib/modprobe.d + printf 'softdep tegra-drm pre: host1x host1x-fence host1x-nvhost\n' \ + > /rootfs/usr/lib/modprobe.d/nvidia-tegra.conf + printf 'softdep nvgpu pre: host1x nvmap host1x-fence host1x-nvhost tegra-drm mc-utils\n' \ + >> /rootfs/usr/lib/modprobe.d/nvidia-tegra.conf +finalize: + - from: /rootfs + to: /