From db4ec7f913e783eb9f9d0b339ca718bef2bbcdd4 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 6 Aug 2026 02:09:43 +0100 Subject: [PATCH 1/3] feat(clang): assemble CUDA resources from NVIDIA manifest --- ClangAstParser/build.gradle | 3 + ClangAstParser/cuda-release.tag | 1 + .../src/pt/up/fe/specs/clang/CacheFiles.java | 10 + .../fe/specs/clang/ClangAstWebResource.java | 31 +- .../pt/up/fe/specs/clang/ClangResources.java | 38 +- .../pt/up/fe/specs/clang/CudaResources.java | 678 ++++++++++++++++++ .../fe/specs/clang/codeparser/CodeParser.java | 2 +- .../up/fe/specs/clang/ClangResourcesTest.java | 25 +- .../up/fe/specs/clang/CudaResourcesTest.java | 537 ++++++++++++++ .../fe/specs/clang/parser/CxxCudaTester.java | 5 +- .../specs/clang/parser/tests/CxxCudaTest.java | 2 +- Clava-JS/code/sideEffects.ts | 8 +- 12 files changed, 1293 insertions(+), 47 deletions(-) create mode 100644 ClangAstParser/cuda-release.tag create mode 100644 ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java diff --git a/ClangAstParser/build.gradle b/ClangAstParser/build.gradle index 81313b2d7..26f49755e 100644 --- a/ClangAstParser/build.gradle +++ b/ClangAstParser/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation 'com.google.code.gson:gson:2.12.1' implementation 'com.google.guava:guava:33.4.0-jre' + implementation 'org.apache.commons:commons-compress:1.27.1' + implementation 'org.tukaani:xz:1.9' testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' testRuntimeOnly 'org.junit.platform:junit-platform-launcher:1.10.0' @@ -52,6 +54,7 @@ sourceSets { processResources { from('clang-dumper-release.tag') + from('cuda-release.tag') } // Test coverage configuration diff --git a/ClangAstParser/cuda-release.tag b/ClangAstParser/cuda-release.tag new file mode 100644 index 000000000..1701b30e1 --- /dev/null +++ b/ClangAstParser/cuda-release.tag @@ -0,0 +1 @@ +12.3.2 diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/CacheFiles.java b/ClangAstParser/src/pt/up/fe/specs/clang/CacheFiles.java index 3a4290753..62fc87a58 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/CacheFiles.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/CacheFiles.java @@ -154,6 +154,11 @@ public void close() { static File installFile(Path cacheRoot, File destination, FileResourceProvider resource, String expectedSha256, String description) { + return installFile(cacheRoot, destination, resource, expectedSha256, -1, description); + } + + static File installFile(Path cacheRoot, File destination, FileResourceProvider resource, String expectedSha256, + long expectedSize, String description) { if (destination.isFile()) { return destination; } @@ -166,6 +171,11 @@ static File installFile(Path cacheRoot, File destination, FileResourceProvider r throw new RuntimeException("Could not download " + description); } + if (expectedSize >= 0 && stagedFile.length() != expectedSize) { + throw new RuntimeException("Downloaded " + description + " does not match expected size '" + + expectedSize + "' (actual: " + stagedFile.length() + ")"); + } + if (expectedSha256 != null && !hasExpectedSha256(stagedFile, expectedSha256)) { throw new RuntimeException("Downloaded " + description + " does not match expected SHA-256 '" + expectedSha256 + "'"); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index f4e6535df..37c62f01d 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -28,12 +28,8 @@ public final class ClangAstWebResource { private static final String RELEASE_ROOT = "https://github.com/specs-feup/clang-dumper/releases/download/"; - private static final String CUDA_RELEASE_ROOT = - "https://github.com/specs-feup/clava/releases/download/clang_ast_dumper_v12.0.7.1/"; - public static final String CUDA_LIB_FILENAME = "cudalib.zip"; - static final WebResourceProvider CUDA_LIB = - WebResourceProvider.newInstance(CUDA_RELEASE_ROOT, CUDA_LIB_FILENAME, "v11.3.0"); private static final String RELEASE_TAG_RESOURCE = "clang-dumper-release.tag"; + private static final String CUDA_RELEASE_TAG_RESOURCE = "cuda-release.tag"; public static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; private static final Gson GSON = new Gson(); @@ -47,24 +43,28 @@ public static DumperSource getDumperSource() { } private static DumperSource readDumperSource() { - var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(RELEASE_TAG_RESOURCE); + return parseDumperSource(readReleaseTag(RELEASE_TAG_RESOURCE)); + } + + private static String readReleaseTag(String resourceName) { + var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(resourceName); if (inputStream == null) { - throw new RuntimeException("Could not find resource '" + RELEASE_TAG_RESOURCE + "'"); + throw new RuntimeException("Could not find resource '" + resourceName + "'"); } String value; try (inputStream) { value = SpecsIo.read(inputStream).trim(); } catch (IOException e) { - throw new UncheckedIOException("Could not read resource '" + RELEASE_TAG_RESOURCE + "'", e); + throw new UncheckedIOException("Could not read resource '" + resourceName + "'", e); } if (value.isBlank()) { - throw new RuntimeException("Resource '" + RELEASE_TAG_RESOURCE + "' is empty"); + throw new RuntimeException("Resource '" + resourceName + "' is empty"); } - return parseDumperSource(value); + return value; } static DumperSource parseDumperSource(String value) { @@ -90,6 +90,17 @@ public static String getReleaseTag() { throw new IllegalStateException("The clang-dumper resource points to a local build"); } + public static String getCudaReleaseTag() { + var releaseTag = readReleaseTag(CUDA_RELEASE_TAG_RESOURCE); + if (releaseTag.equals(".") || releaseTag.equals("..") + || releaseTag.contains("/") || releaseTag.contains("\\")) { + throw new RuntimeException("Release resource '" + CUDA_RELEASE_TAG_RESOURCE + + "' must contain a single path component: '" + releaseTag + "'"); + } + + return releaseTag; + } + public static ClangDumperManifest getManifest(File resourceFolder) { var releaseTag = getReleaseTag(); var manifestResource = WebResourceProvider.newInstance(getReleaseBaseUrl(releaseTag), MANIFEST_FILENAME, diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 228eba6b7..4432867aa 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -43,6 +43,7 @@ public class ClangResources { private static final Map CLANG_FILES_CACHE = new ConcurrentHashMap<>(); private static final String CLANG_FOLDERNAME = "clang_ast_exe"; + private static final String CLANG_CACHE_FOLDERNAME = "clang-dumper"; private static final String RELEASES_FOLDERNAME = "releases"; private static final String INCLUDES_FOLDERNAME = "includes"; private static final Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); @@ -55,21 +56,12 @@ public ClangResources(CodeParser options) { this.options = options; } - public File getBuiltinCudaLib() { - var cudaResourceFolder = SpecsIo.mkdir(options.get(CodeParser.DUMPER_FOLDER), "cuda"); - var cudaFolder = SpecsIo.mkdir(cudaResourceFolder, "cudalib"); - var zipFile = ClangAstWebResource.CUDA_LIB.writeVersioned(cudaResourceFolder, ClangResources.class); - - if (zipFile.isNewFile() || !isCudaInstallation(cudaFolder)) { - SpecsIo.deleteFolderContents(cudaFolder); - SpecsIo.extractZip(zipFile.getFile(), cudaFolder); - } - - return cudaFolder; + public static boolean isBuiltinCudaSupported() { + return CudaResources.isSupportedPlatform(); } - private static boolean isCudaInstallation(File folder) { - return folder.isDirectory() && new File(folder, "include/cuda_runtime.h").isFile(); + public File getBuiltinCudaLib() { + return CudaResources.getBuiltinCudaLib(options.get(CodeParser.DUMPER_FOLDER).toPath()); } public ClangFiles getClangFiles(LibcMode libcMode) { @@ -119,7 +111,7 @@ private boolean isUsable(CachedClangFiles cached) { return false; } - return CacheFiles.withMaintenanceLock(options.get(CodeParser.DUMPER_FOLDER).toPath(), () -> { + return CacheFiles.withMaintenanceLock(getClangCacheRoot().toPath(), () -> { if (!cached.files().clangExecutable().isFile()) { return false; } @@ -143,7 +135,7 @@ private boolean isUsable(CachedClangFiles cached) { } private void touchUse(File resourceFolder, File includesFolder) { - CacheFiles.withMaintenanceLock(options.get(CodeParser.DUMPER_FOLDER).toPath(), () -> { + CacheFiles.withMaintenanceLock(getClangCacheRoot().toPath(), () -> { CacheFiles.touch(resourceFolder.toPath()); if (includesFolder != null) { CacheFiles.touch(includesFolder.toPath()); @@ -178,7 +170,7 @@ private File prepareResources(ClangDumperManifest manifest, File resourceFolder) var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; var asset = getCurrentAsset(manifest, executableKind); - File executable = CacheFiles.installFile(options.get(CodeParser.DUMPER_FOLDER).toPath(), + File executable = CacheFiles.installFile(getClangCacheRoot().toPath(), new File(resourceFolder, asset.filename()), ClangAstWebResource.getAssetResource(asset), asset.sha256(), "clang-dumper asset '" + asset.filename() + "'"); @@ -214,7 +206,7 @@ private void unblockWindowsFile(File executable) { } public File getClangResourceFolder() { - var cacheFolder = options.get(CodeParser.DUMPER_FOLDER); + var cacheFolder = getClangCacheRoot(); return CacheFiles.withMaintenanceLock(cacheFolder.toPath(), () -> { var releaseFolder = SpecsIo.mkdir(getReleasesFolder(), ClangAstWebResource.getReleaseTag()); CacheFiles.touch(releaseFolder.toPath()); @@ -227,11 +219,15 @@ public static File getDefaultTempFolder() { } private File getReleasesFolder() { - return SpecsIo.mkdir(options.get(CodeParser.DUMPER_FOLDER), RELEASES_FOLDERNAME); + return SpecsIo.mkdir(getClangCacheRoot(), RELEASES_FOLDERNAME); } private File getIncludesRoot() { - return new File(options.get(CodeParser.DUMPER_FOLDER), INCLUDES_FOLDERNAME); + return new File(getClangCacheRoot(), INCLUDES_FOLDERNAME); + } + + private File getClangCacheRoot() { + return new File(options.get(CodeParser.DUMPER_FOLDER), CLANG_CACHE_FOLDERNAME); } static File getSharedIncludesFolder(File cacheFolder, String sha256) { @@ -311,7 +307,7 @@ private PreparedIncludes prepareIncludes(ClangDumperManifest manifest, File clan private File prepareIncludesFolder(ClangDumperManifest manifest) { var includesAsset = getCurrentAsset(manifest, "includes"); - return resolveIncludes(options.get(CodeParser.DUMPER_FOLDER), includesAsset, + return resolveIncludes(getClangCacheRoot(), includesAsset, ClangAstWebResource.getAssetResource(includesAsset)); } @@ -453,7 +449,7 @@ void deleteStaleVersions(Instant now, File currentVersionFolder) { private void deleteStaleVersions(Instant now, File currentVersionFolder, File currentIncludesFolder) { var cutoff = now.minus(STALE_CACHE_MAX_AGE); - var cacheRoot = options.get(CodeParser.DUMPER_FOLDER).toPath(); + var cacheRoot = getClangCacheRoot().toPath(); try { CacheFiles.deleteStaleDirectories(cacheRoot, getReleasesFolder().toPath(), cutoff, currentVersionFolder.toPath()); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java new file mode 100644 index 000000000..b1a27491e --- /dev/null +++ b/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java @@ -0,0 +1,678 @@ +/** + * Copyright 2026 SPeCS. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import pt.up.fe.specs.util.SpecsIo; +import pt.up.fe.specs.util.SpecsLogs; +import pt.up.fe.specs.util.providers.FileResourceProvider; +import pt.up.fe.specs.util.providers.WebResourceProvider; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; + +/** + * Downloads the NVIDIA redistribution packages required by Clang and assembles them into the CUDA root expected by + * the bundled dumper. + * + *

CUDA resources are release-addressed. Archives from different CUDA releases therefore never share a cache + * destination, even when NVIDIA publishes identical bytes for both releases.

+ */ +final class CudaResources { + + static final String NVIDIA_REDIST_ROOT = "https://developer.download.nvidia.com/compute/cuda/redist/"; + static final String LINUX_X86_64_PLATFORM = "linux-x86_64"; + static final String LINUX_PPC64LE_PLATFORM = "linux-ppc64le"; + static final String LINUX_SBSA_PLATFORM = "linux-sbsa"; + static final String WINDOWS_X86_64_PLATFORM = "windows-x86_64"; + static final List REQUIRED_COMPONENTS = List.of("cuda_cudart", "cuda_nvcc", "libcurand", "cuda_cccl"); + static final String PLATFORM_FILENAME = ".platform"; + + private static final String CUDA_FOLDERNAME = "cuda"; + private static final String CUDA_LIB_FOLDERNAME = "cudalib"; + private static final String ARCHIVES_FOLDERNAME = "archives"; + private static final String MANIFEST_FILENAME_PREFIX = "redistrib_"; + private static final String MANIFEST_FILENAME_SUFFIX = ".json"; + private static final Set MANIFEST_FIELDS = Set.of("release_date", "release_label", "release_product"); + private static final Set COMPONENT_FIELDS = Set.of("name", "license", "license_path", "version"); + private static final Pattern SHA256_PATTERN = Pattern.compile("[0-9a-fA-F]{64}"); + private static final List REQUIRED_FILES = List.of( + "include/cuda.h", + "include/cuda_runtime.h", + "include/texture_fetch_functions.h", + "include/curand_mtgp32_kernel.h", + "include/nv/target", + "include/crt/host_config.h", + "nvvm/libdevice/libdevice.10.bc"); + + private CudaResources() { + } + + static File getBuiltinCudaLib(Path cacheRoot) { + var releaseTag = ClangAstWebResource.getCudaReleaseTag(); + var platform = requireSupportedPlatform(); + var platformFolder = getPlatformFolder(cacheRoot, releaseTag, platform); + var installationFolder = getInstallationFolder(platformFolder); + + // A published installation is immutable. A malformed one is an operator error, not an invitation to repair it + // in place, because doing so could race with a reader that already selected this release. + if (Files.exists(installationFolder.toPath(), LinkOption.NOFOLLOW_LINKS)) { + return useExistingInstallation(cacheRoot, platformFolder, platform, installationFolder); + } + + var manifest = getManifest(cacheRoot, platformFolder); + return install(cacheRoot, platformFolder, releaseTag, platform, manifest, CudaResources::getArchiveResource); + } + + static CudaPlatform requireSupportedPlatform() { + return getCurrentPlatform(); + } + + static boolean isSupportedPlatform() { + try { + getCurrentPlatform(); + return true; + } catch (RuntimeException e) { + return false; + } + } + + static CudaPlatform getCurrentPlatform() { + return new CudaPlatform(getManifestPlatform(SupportedPlatform.getCurrentPlatform(), System.getProperty("os.arch"))); + } + + static String getManifestPlatform(SupportedPlatform platform, String architecture) { + Objects.requireNonNull(platform, "platform"); + Objects.requireNonNull(architecture, "architecture"); + var normalizedArchitecture = architecture.toLowerCase(Locale.ROOT); + + if (platform.isWindows() && isX86_64(normalizedArchitecture)) { + return WINDOWS_X86_64_PLATFORM; + } + + if (platform.isLinux()) { + if (isX86_64(normalizedArchitecture)) { + return LINUX_X86_64_PLATFORM; + } + + if (normalizedArchitecture.equals("ppc64le")) { + return LINUX_PPC64LE_PLATFORM; + } + + if (normalizedArchitecture.equals("aarch64") || normalizedArchitecture.equals("arm64")) { + return LINUX_SBSA_PLATFORM; + } + } + + throw new RuntimeException("The CUDA manifest does not provide a supported package for platform '" + + platform + "' and architecture '" + architecture + "'"); + } + + private static boolean isX86_64(String architecture) { + return architecture.equals("amd64") || architecture.equals("x86_64"); + } + + static File getPlatformFolder(Path cacheRoot, String releaseTag, CudaPlatform platform) { + return cacheRoot.resolve(CUDA_FOLDERNAME).resolve(releaseTag).resolve(platform.manifestName()).toFile(); + } + + static File getInstallationFolder(File platformFolder) { + return new File(platformFolder, CUDA_LIB_FOLDERNAME); + } + + static File getArchiveFile(File platformFolder, CudaPackage cudaPackage) { + var relativePath = cudaPackage.archive().relativePath(); + var archiveName = relativePath.substring(relativePath.lastIndexOf('/') + 1); + return new File(new File(new File(platformFolder, ARCHIVES_FOLDERNAME), cudaPackage.component()), archiveName); + } + + static NvidiaCudaManifest getManifest(File resourceFolder) { + return getManifest(getCacheRoot(resourceFolder), resourceFolder); + } + + static NvidiaCudaManifest getManifest(Path cacheRoot, File resourceFolder) { + var releaseTag = ClangAstWebResource.getCudaReleaseTag(); + var manifestFilename = getManifestFilename(releaseTag); + var resource = WebResourceProvider.newInstance(NVIDIA_REDIST_ROOT, manifestFilename, releaseTag); + var manifestFile = CacheFiles.installFile(cacheRoot, new File(resourceFolder, manifestFilename), resource, null, + "NVIDIA CUDA redistribution manifest"); + var manifest = parseManifest(SpecsIo.read(manifestFile)); + validateManifest(manifest, releaseTag); + return manifest; + } + + private static Path getCacheRoot(File resourceFolder) { + var cacheRoot = resourceFolder.toPath(); + for (int i = 0; i < 3; i++) { + cacheRoot = cacheRoot.getParent(); + if (cacheRoot == null) { + throw new RuntimeException("CUDA resource folder is not below a cache root: '" + resourceFolder + "'"); + } + } + + return cacheRoot; + } + + static String getManifestFilename(String releaseTag) { + return MANIFEST_FILENAME_PREFIX + releaseTag + MANIFEST_FILENAME_SUFFIX; + } + + static WebResourceProvider getArchiveResource(CudaPackage cudaPackage) { + var archive = cudaPackage.archive(); + return WebResourceProvider.newInstance(NVIDIA_REDIST_ROOT, archive.relativePath(), + "cuda-" + cudaPackage.component() + "-" + archive.sha256()); + } + + static NvidiaCudaManifest parseManifest(String json) { + if (json == null || json.isBlank()) { + throw new RuntimeException("NVIDIA CUDA redistribution manifest is empty"); + } + + final JsonObject root; + try { + root = JsonParser.parseString(json).getAsJsonObject(); + } catch (RuntimeException e) { + throw new RuntimeException("Could not parse NVIDIA CUDA redistribution manifest", e); + } + + var releaseDate = getRequiredString(root, "release_date", "manifest"); + var releaseLabel = getRequiredString(root, "release_label", "manifest"); + var releaseProduct = getRequiredString(root, "release_product", "manifest"); + var components = new LinkedHashMap(); + + for (var entry : root.entrySet()) { + if (MANIFEST_FIELDS.contains(entry.getKey())) { + continue; + } + + if (!entry.getValue().isJsonObject()) { + throw new RuntimeException("NVIDIA CUDA manifest component '" + entry.getKey() + + "' is not an object"); + } + + components.put(entry.getKey(), parseComponent(entry.getKey(), entry.getValue().getAsJsonObject())); + } + + if (components.isEmpty()) { + throw new RuntimeException("NVIDIA CUDA redistribution manifest does not contain components"); + } + + return new NvidiaCudaManifest(releaseDate, releaseLabel, releaseProduct, components); + } + + static File install(Path cacheRoot, File resourceFolder, String releaseTag, CudaPlatform platform, + NvidiaCudaManifest manifest, + Function archiveResourceFactory) { + validateManifest(manifest, releaseTag); + Objects.requireNonNull(archiveResourceFactory, "archiveResourceFactory"); + + var installationFolder = getInstallationFolder(resourceFolder); + if (Files.exists(installationFolder.toPath(), LinkOption.NOFOLLOW_LINKS)) { + return useExistingInstallation(cacheRoot, resourceFolder, platform, installationFolder); + } + + var downloadedPackages = manifest.getRequiredPackages(platform.manifestName()).stream() + .map(cudaPackage -> downloadPackage(cacheRoot, resourceFolder, cudaPackage, archiveResourceFactory)) + .toList(); + + CacheFiles.deleteUnlockedStagingLocks(cacheRoot, resourceFolder.toPath()); + var stagingDirectory = CacheFiles.createStagingDirectory(cacheRoot, resourceFolder.toPath(), ".cudalib.tmp-"); + try { + try { + assemble(stagingDirectory.path().toFile(), platform, downloadedPackages); + } catch (IOException e) { + throw new UncheckedIOException("Could not assemble CUDA resources in '" + + stagingDirectory.path() + "'", e); + } + + if (!isCudaInstallation(stagingDirectory.path().toFile(), platform.manifestName())) { + throw new RuntimeException("Assembled CUDA resources failed structural validation in '" + + stagingDirectory.path() + "'"); + } + + var publishedFolder = CacheFiles.publish(stagingDirectory.path(), installationFolder.toPath()).toFile(); + return useExistingInstallation(cacheRoot, resourceFolder, platform, publishedFolder); + } finally { + try { + CacheFiles.delete(stagingDirectory.path()); + } finally { + stagingDirectory.close(); + } + } + } + + private static CudaResources.DownloadedPackage downloadPackage(Path cacheRoot, File resourceFolder, + CudaPackage cudaPackage, + Function factory) { + var destination = getArchiveFile(resourceFolder, cudaPackage); + var archiveParent = destination.getParentFile().toPath(); + CacheFiles.deleteUnlockedStagingLocks(cacheRoot, archiveParent); + var archive = CacheFiles.installFile(cacheRoot, destination, factory.apply(cudaPackage), + cudaPackage.archive().sha256(), cudaPackage.archive().size(), + "NVIDIA CUDA archive '" + destination.getName() + "'"); + return new DownloadedPackage(cudaPackage, archive); + } + + private static File useExistingInstallation(Path cacheRoot, File resourceFolder, CudaPlatform platform, + File installationFolder) { + var validInstallation = CacheFiles.withMaintenanceLock(cacheRoot, () -> { + if (!isCudaInstallation(installationFolder, platform.manifestName())) { + throw invalidInstallation(installationFolder, platform.manifestName()); + } + + CacheFiles.touch(resourceFolder.toPath()); + CacheFiles.touch(resourceFolder.getParentFile().toPath()); + return installationFolder; + }); + + cleanup(cacheRoot, resourceFolder); + SpecsLogs.debug(() -> "Using cached CUDA resources: " + validInstallation); + return validInstallation; + } + + private static void cleanup(Path cacheRoot, File resourceFolder) { + var cudaRoot = cacheRoot.resolve(CUDA_FOLDERNAME); + var releaseFolder = resourceFolder.toPath().getParent(); + var cutoff = Instant.now().minus(Duration.ofDays(60)); + try { + CacheFiles.deleteStaleDirectories(cacheRoot, cudaRoot, cutoff, releaseFolder); + CacheFiles.deleteUnlockedStagingLocks(cacheRoot, releaseFolder); + CacheFiles.deleteUnlockedStagingLocks(cacheRoot, resourceFolder.toPath()); + for (var component : REQUIRED_COMPONENTS) { + CacheFiles.deleteUnlockedStagingLocks(cacheRoot, + resourceFolder.toPath().resolve(ARCHIVES_FOLDERNAME).resolve(component)); + } + } catch (RuntimeException e) { + SpecsLogs.warn("Could not clean stale CUDA cache resources", e); + } + } + + static boolean isCudaInstallation(File folder, String platform) { + Path root = folder.toPath().toAbsolutePath().normalize(); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS) + || !Files.isDirectory(root.resolve("bin"), LinkOption.NOFOLLOW_LINKS)) { + return false; + } + + for (var requiredFile : REQUIRED_FILES) { + if (!Files.isRegularFile(root.resolve(requiredFile), LinkOption.NOFOLLOW_LINKS)) { + return false; + } + } + + var platformFile = root.resolve(PLATFORM_FILENAME); + if (!Files.isRegularFile(platformFile, LinkOption.NOFOLLOW_LINKS)) { + return false; + } + + try { + return platform.equals(Files.readString(platformFile).trim()); + } catch (IOException e) { + return false; + } + } + + private static RuntimeException invalidInstallation(File folder, String platform) { + return new RuntimeException("Invalid published CUDA installation '" + folder.getAbsolutePath() + + "' for platform '" + platform + "'; delete this directory manually to regenerate"); + } + + private static void validateManifest(NvidiaCudaManifest manifest, String releaseTag) { + Objects.requireNonNull(manifest, "manifest"); + if (!releaseTag.equals(manifest.releaseLabel())) { + throw new RuntimeException("NVIDIA CUDA manifest release label '" + manifest.releaseLabel() + + "' does not match the configured CUDA release tag '" + releaseTag + "'"); + } + + if (!"cuda".equals(manifest.releaseProduct())) { + throw new RuntimeException("NVIDIA redistribution manifest is not a CUDA manifest: '" + + manifest.releaseProduct() + "'"); + } + } + + private static NvidiaCudaComponent parseComponent(String componentName, JsonObject component) { + var name = getRequiredString(component, "name", "component '" + componentName + "'"); + var version = getRequiredString(component, "version", "component '" + componentName + "'"); + var archives = new LinkedHashMap(); + + for (var entry : component.entrySet()) { + if (COMPONENT_FIELDS.contains(entry.getKey())) { + continue; + } + + if (!entry.getValue().isJsonObject()) { + continue; + } + + var platform = entry.getKey(); + var platformObject = entry.getValue().getAsJsonObject(); + var relativePath = getRequiredString(platformObject, "relative_path", + "component '" + componentName + "', platform '" + platform + "'"); + var sha256 = getRequiredString(platformObject, "sha256", + "component '" + componentName + "', platform '" + platform + "'"); + validateSha256(sha256, componentName, platform); + var size = getRequiredLong(platformObject, "size", + "component '" + componentName + "', platform '" + platform + "'"); + validateRelativePath(relativePath, "component '" + componentName + "', platform '" + platform + "'"); + if (size < 0) { + throw new RuntimeException("NVIDIA CUDA archive size must not be negative for component '" + + componentName + "', platform '" + platform + "'"); + } + + archives.put(platform, new CudaArchive(relativePath, sha256, size)); + } + + if (archives.isEmpty()) { + throw new RuntimeException("NVIDIA CUDA component '" + componentName + "' has no platform archives"); + } + + return new NvidiaCudaComponent(name, version, archives); + } + + private static String getRequiredString(JsonObject object, String field, String owner) { + JsonElement value = object.get(field); + if (value == null || value.isJsonNull() || !value.isJsonPrimitive() + || !value.getAsJsonPrimitive().isString()) { + throw new RuntimeException("NVIDIA CUDA " + owner + " is missing string field '" + field + "'"); + } + + var stringValue = value.getAsString().trim(); + if (stringValue.isEmpty()) { + throw new RuntimeException("NVIDIA CUDA " + owner + " has an empty field '" + field + "'"); + } + + return stringValue; + } + + private static long getRequiredLong(JsonObject object, String field, String owner) { + JsonElement value = object.get(field); + if (value == null || value.isJsonNull() || !value.isJsonPrimitive()) { + throw new RuntimeException("NVIDIA CUDA " + owner + " is missing numeric field '" + field + "'"); + } + + try { + return value.getAsLong(); + } catch (RuntimeException e) { + throw new RuntimeException("NVIDIA CUDA " + owner + " has an invalid numeric field '" + field + "'", e); + } + } + + private static void validateSha256(String sha256, String component, String platform) { + if (!SHA256_PATTERN.matcher(sha256).matches()) { + throw new RuntimeException("NVIDIA CUDA archive for component '" + component + "', platform '" + + platform + "' has an invalid SHA-256: '" + sha256 + "'"); + } + } + + private static void validateRelativePath(String relativePath, String owner) { + if (relativePath.startsWith("/") || relativePath.startsWith("\\") || relativePath.contains("\\") + || hasWindowsDrivePrefix(relativePath)) { + throw new RuntimeException("NVIDIA CUDA " + owner + " has an unsafe relative path: '" + relativePath + "'"); + } + + for (var segment : relativePath.split("/", -1)) { + if (segment.isEmpty() || segment.equals(".") || segment.equals("..")) { + throw new RuntimeException("NVIDIA CUDA " + owner + " has an unsafe relative path: '" + relativePath + "'"); + } + } + } + + static void assemble(File stagingFolder, CudaPlatform platform, List packages) throws IOException { + Files.writeString(new File(stagingFolder, PLATFORM_FILENAME).toPath(), platform.manifestName()); + Files.createDirectories(new File(stagingFolder, "bin").toPath()); + + for (var downloadedPackage : packages) { + var component = downloadedPackage.cudaPackage().component(); + var sourceRoots = switch (component) { + case "cuda_cudart", "libcurand", "cuda_cccl" -> List.of("include"); + case "cuda_nvcc" -> List.of("include/crt", "nvvm/libdevice/libdevice.10.bc"); + default -> throw new RuntimeException("Unsupported NVIDIA CUDA component '" + component + "'"); + }; + + extractArchive(downloadedPackage.archiveFile(), stagingFolder, sourceRoots); + } + } + + private static void extractArchive(File archive, File destination, List sourceRoots) throws IOException { + if (archive.getName().endsWith(".zip")) { + try (InputStream input = Files.newInputStream(archive.toPath()); + var archiveInput = new ZipArchiveInputStream(input)) { + extractArchiveEntries(archiveInput, archive, destination, sourceRoots, + entry -> entry instanceof ZipArchiveEntry zipEntry && isRegularZipEntry(zipEntry)); + } + return; + } + + if (!archive.getName().endsWith(".tar.xz")) { + throw new RuntimeException("Unsupported NVIDIA CUDA archive format: '" + archive + "'"); + } + + try (InputStream input = Files.newInputStream(archive.toPath()); + var xzInput = new XZCompressorInputStream(input); + var tarInput = new TarArchiveInputStream(xzInput)) { + extractArchiveEntries(tarInput, archive, destination, sourceRoots, + entry -> entry instanceof TarArchiveEntry tarEntry && tarEntry.isFile()); + } + } + + private static void extractArchiveEntries(ArchiveInputStream archiveInput, File archive, File destination, + List sourceRoots, ArchiveEntryPolicy entryPolicy) throws IOException { + var foundRoots = new HashSet(); + String archiveRoot = null; + ArchiveEntry entry; + while ((entry = archiveInput.getNextEntry()) != null) { + var entryName = validateArchiveEntryName(entry.getName(), archive); + var topLevel = getTopLevelPath(entryName); + + if (archiveRoot == null) { + archiveRoot = topLevel; + } else if (!archiveRoot.equals(topLevel)) { + throw new RuntimeException("NVIDIA CUDA archive contains multiple top-level folders: '" + + archiveRoot + "' and '" + topLevel + "'"); + } + + var relativeName = entryName.length() == archiveRoot.length() + ? "" + : entryName.substring(archiveRoot.length() + 1); + var sourceRoot = findSourceRoot(relativeName, sourceRoots); + if (sourceRoot == null) { + continue; + } + + if (!entry.isDirectory() && !entryPolicy.isRegular(entry)) { + throw new RuntimeException("NVIDIA CUDA archive contains a non-regular selected entry: '" + + entryName + "'"); + } + + if (entry.isDirectory()) { + Files.createDirectories(destination.toPath().resolve(relativeName)); + continue; + } + + foundRoots.add(sourceRoot); + copyArchiveFile(archiveInput, destination.toPath().resolve(relativeName), entryName); + } + + if (!foundRoots.containsAll(sourceRoots)) { + var missingRoots = new ArrayList<>(sourceRoots); + missingRoots.removeAll(foundRoots); + throw new RuntimeException("NVIDIA CUDA archive '" + archive + "' is missing selected paths: " + missingRoots); + } + } + + private static boolean isRegularZipEntry(ZipArchiveEntry entry) { + if (entry.isUnixSymlink()) { + return false; + } + + var unixMode = entry.getUnixMode(); + return unixMode == 0 || (unixMode & 0170000) == 0100000; + } + + private static String validateArchiveEntryName(String entryName, File archive) { + if (entryName == null || entryName.isBlank() || entryName.startsWith("/") || entryName.contains("\\") + || hasWindowsDrivePrefix(entryName)) { + throw new RuntimeException("NVIDIA CUDA archive '" + archive + "' contains an unsafe path: '" + entryName + "'"); + } + + var normalizedName = entryName.endsWith("/") ? entryName.substring(0, entryName.length() - 1) : entryName; + if (normalizedName.isEmpty()) { + throw new RuntimeException("NVIDIA CUDA archive '" + archive + "' contains an empty path"); + } + + for (var segment : normalizedName.split("/", -1)) { + if (segment.isEmpty() || segment.equals(".") || segment.equals("..")) { + throw new RuntimeException("NVIDIA CUDA archive '" + archive + "' contains an unsafe path: '" + entryName + "'"); + } + } + + return normalizedName; + } + + private static boolean hasWindowsDrivePrefix(String path) { + return path.length() >= 2 && Character.isLetter(path.charAt(0)) && path.charAt(1) == ':'; + } + + private static String getTopLevelPath(String entryName) { + var separator = entryName.indexOf('/'); + return separator == -1 ? entryName : entryName.substring(0, separator); + } + + private static String findSourceRoot(String relativeName, List sourceRoots) { + for (var sourceRoot : sourceRoots) { + if (relativeName.equals(sourceRoot) || relativeName.startsWith(sourceRoot + "/")) { + return sourceRoot; + } + } + + return null; + } + + private static void copyArchiveFile(InputStream input, Path destination, String entryName) throws IOException { + Files.createDirectories(destination.getParent()); + var temporaryFile = Files.createTempFile(destination.getParent(), ".cuda-entry-", ".tmp"); + try { + try (OutputStream output = Files.newOutputStream(temporaryFile)) { + input.transferTo(output); + } + + if (Files.exists(destination, LinkOption.NOFOLLOW_LINKS)) { + if (!Files.isRegularFile(destination, LinkOption.NOFOLLOW_LINKS) + || Files.mismatch(destination, temporaryFile) != -1) { + throw new RuntimeException("NVIDIA CUDA archives contain conflicting files at '" + entryName + "'"); + } + return; + } + + try { + Files.move(temporaryFile, destination, StandardCopyOption.ATOMIC_MOVE); + } catch (java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporaryFile, destination); + } catch (java.nio.file.FileAlreadyExistsException e) { + if (!Files.isRegularFile(destination, LinkOption.NOFOLLOW_LINKS) + || Files.mismatch(destination, temporaryFile) != -1) { + throw new RuntimeException("NVIDIA CUDA archives contain conflicting files at '" + entryName + "'"); + } + } + } finally { + Files.deleteIfExists(temporaryFile); + } + } + + record NvidiaCudaManifest(String releaseDate, String releaseLabel, String releaseProduct, + Map components) { + + NvidiaCudaManifest { + components = Map.copyOf(components); + } + + List getRequiredPackages(String platform) { + return REQUIRED_COMPONENTS.stream() + .map(component -> new CudaPackage(component, getComponent(component).getArchive(platform))) + .toList(); + } + + private NvidiaCudaComponent getComponent(String component) { + var value = components.get(component); + if (value == null) { + throw new RuntimeException("NVIDIA CUDA manifest is missing required component '" + component + "'"); + } + + return value; + } + } + + record NvidiaCudaComponent(String name, String version, Map archives) { + + NvidiaCudaComponent { + archives = Map.copyOf(archives); + } + + CudaArchive getArchive(String platform) { + var archive = archives.get(platform); + if (archive == null) { + throw new RuntimeException("NVIDIA CUDA component '" + name + "' has no archive for platform '" + + platform + "'"); + } + + return archive; + } + } + + record CudaPackage(String component, CudaArchive archive) { + } + + record CudaArchive(String relativePath, String sha256, long size) { + } + + record DownloadedPackage(CudaPackage cudaPackage, File archiveFile) { + } + + record CudaPlatform(String manifestName) { + } + + @FunctionalInterface + private interface ArchiveEntryPolicy { + + boolean isRegular(ArchiveEntry entry); + } +} diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java index f7441ed77..e65c941a9 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java @@ -45,7 +45,7 @@ public abstract class CodeParser extends ADataClass { .setLabel("CUDA Path (empty: uses system installed; : uses builtin version)") .setDefaultString(""); public static final DataKey DUMPER_FOLDER = KeyFactory.folder("dumperFolder") - .setLabel("The base cache folder for the clang-dumper. Clava stores each dumper release in a versioned subfolder and downloads it if not found. If not set, a temporary folder will be used.") + .setLabel("The base cache folder for Clava's downloaded resources. Clava stores each clang-dumper and CUDA release in a versioned subfolder and downloads it if not found. If not set, a temporary folder will be used.") .setDefault(ClangResources::getDefaultTempFolder); /** diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 4ba36b52e..4b30a9067 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -313,11 +313,13 @@ public void orphanedStagingLocksAreCleaned() throws IOException { @Test public void staleReleaseAndSharedIncludesAreRemovedAfterSixtyDays() throws IOException { - var releases = Files.createDirectories(tempFolder.resolve("releases")); + var clangCacheRoot = clangCacheRoot(); + var releases = Files.createDirectories(clangCacheRoot.resolve("releases")); var current = Files.createDirectories(releases.resolve("current")); var staleRelease = Files.createDirectories(releases.resolve("stale")); var staleIncludes = Files.createDirectories( - ClangResources.getSharedIncludesFolder(tempFolder.toFile(), "c".repeat(64)).toPath().resolve("builtin")); + ClangResources.getSharedIncludesFolder(clangCacheRoot.toFile(), "c".repeat(64)).toPath() + .resolve("builtin")); Files.writeString(staleIncludes.getParent().resolve("entrypoints.txt"), "builtin\n"); var old = FileTime.from(Instant.now().minus(Duration.ofDays(61))); Files.setLastModifiedTime(staleRelease, old); @@ -335,21 +337,23 @@ public void staleReleaseAndSharedIncludesAreRemovedAfterSixtyDays() throws IOExc @Test public void usingSharedIncludesRefreshesItsLastUsedTime() throws IOException { var sha = "d".repeat(64); - var shared = Files.createDirectories(ClangResources.getSharedIncludesFolder(tempFolder.toFile(), sha).toPath()); + var clangCacheRoot = clangCacheRoot(); + var shared = Files.createDirectories( + ClangResources.getSharedIncludesFolder(clangCacheRoot.toFile(), sha).toPath()); Files.createDirectories(shared.resolve("builtin")); Files.writeString(shared.resolve("entrypoints.txt"), "builtin\n"); Files.setLastModifiedTime(shared, FileTime.from(Instant.now().minus(Duration.ofDays(61)))); var writes = new AtomicInteger(); var unusedArchive = tempFolder.resolve("unused.zip"); var asset = new ClangDumperManifestAsset("includes.zip", "includes", "linux", "x64", 18, sha); - assertEquals(shared.toFile(), ClangResources.resolveIncludes(tempFolder.toFile(), asset, + assertEquals(shared.toFile(), ClangResources.resolveIncludes(clangCacheRoot.toFile(), asset, copyingResource(unusedArchive, writes))); assertEquals(0, writes.get()); var parser = CodeParser.newInstance(); parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); new ClangResources(parser).deleteStaleVersions(Instant.now(), - Files.createDirectories(tempFolder.resolve("releases/current")).toFile()); + Files.createDirectories(clangCacheRoot.resolve("releases/current")).toFile()); assertTrue(Files.exists(shared)); } @@ -433,7 +437,8 @@ public void releaseResourcesCanBeInitializedBySeparateJvms() throws Exception { var secondExecutable = new File(Files.readString(secondDone).trim()); assertEquals(firstExecutable.getAbsoluteFile(), secondExecutable.getAbsoluteFile()); assertTrue(firstExecutable.isFile()); - assertTrue(cacheFolder.toPath().resolve("releases").resolve(ClangAstWebResource.getReleaseTag()).toFile().isDirectory()); + assertTrue(cacheFolder.toPath().resolve("clang-dumper").resolve("releases") + .resolve(ClangAstWebResource.getReleaseTag()).toFile().isDirectory()); } @Test @@ -533,7 +538,9 @@ public void builtinCudaArchiveHasCanonicalInstallationLayout() { var parser = newParser(CodeParser.getBuiltinOption()); var cudaFolder = new ClangResources(parser).getBuiltinCudaLib(); - assertEquals(tempFolder.resolve("cuda/cudalib").toFile().getAbsolutePath(), cudaFolder.getAbsolutePath()); + var cudaPlatform = CudaResources.getCurrentPlatform().manifestName(); + assertEquals(tempFolder.resolve("cuda").resolve(ClangAstWebResource.getCudaReleaseTag()) + .resolve(cudaPlatform).resolve("cudalib").toFile().getAbsolutePath(), cudaFolder.getAbsolutePath()); assertTrue(new File(cudaFolder, "include/cuda.h").isFile()); assertTrue(new File(cudaFolder, "include/cuda_runtime.h").isFile()); assertTrue(new File(cudaFolder, "nvvm/libdevice/libdevice.10.bc").isFile()); @@ -563,6 +570,10 @@ private CodeParser newParser(String cudaPath) { return parser; } + private Path clangCacheRoot() { + return tempFolder.resolve("clang-dumper"); + } + private static ClangDumperManifestAsset asset(String filename, String kind, String platform, String arch) { return new ClangDumperManifestAsset(filename, kind, platform, arch, 18, HELLO_SHA256); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java new file mode 100644 index 000000000..f3169882a --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java @@ -0,0 +1,537 @@ +/** + * Copyright 2026 SPeCS. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import pt.up.fe.specs.clang.codeparser.CodeParser; +import pt.up.fe.specs.util.providers.FileResourceProvider; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CudaResourcesTest { + + private static final String PLATFORM = CudaResources.LINUX_X86_64_PLATFORM; + private static final String RELEASE = "12.3.2"; + private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10); + + @TempDir + Path tempFolder; + + @Test + public void manifestSelectsRequiredComponentsAndValidatesMetadata() { + var manifest = CudaResources.parseManifest(manifestJson()); + + assertEquals(RELEASE, ClangAstWebResource.getCudaReleaseTag()); + assertEquals(RELEASE, manifest.releaseLabel()); + assertEquals("cuda", manifest.releaseProduct()); + assertEquals(List.of("cuda_cudart", "cuda_nvcc", "libcurand", "cuda_cccl"), + manifest.getRequiredPackages(PLATFORM).stream() + .map(CudaResources.CudaPackage::component) + .toList()); + assertEquals(CudaResources.NVIDIA_REDIST_ROOT + "cuda_cudart/linux-x86_64/cuda_cudart.tar.xz", + CudaResources.getArchiveResource(manifest.getRequiredPackages(PLATFORM).get(0)).getUrlString()); + assertEquals("redistrib_12.3.2.json", CudaResources.getManifestFilename(RELEASE)); + + assertThrows(RuntimeException.class, () -> CudaResources.parseManifest("")); + assertThrows(RuntimeException.class, + () -> CudaResources.parseManifest(manifestJson().replace(SHA256, "not-a-sha"))); + assertThrows(RuntimeException.class, + () -> CudaResources.parseManifest(manifestJson().replace( + "cuda_cudart/linux-x86_64/cuda_cudart.tar.xz", "../cuda_cudart.tar.xz"))); + assertThrows(RuntimeException.class, + () -> CudaResources.parseManifest(manifestJson().replace("\"size\": \"1\"", "\"size\": \"-1\""))); + + var wrongRelease = CudaResources.parseManifest(manifestJson() + .replace("\"release_label\": \"12.3.2\"", "\"release_label\": \"12.3.1\"")); + var releaseError = assertThrows(RuntimeException.class, () -> CudaResources.install( + tempFolder, tempFolder.resolve("wrong-release").toFile(), RELEASE, + new CudaResources.CudaPlatform(PLATFORM), wrongRelease, ignored -> { + throw new AssertionError("Archive downloads must not start for an invalid manifest"); + })); + assertTrue(releaseError.getMessage().contains("release label")); + + var wrongProduct = CudaResources.parseManifest(manifestJson() + .replace("\"release_product\": \"cuda\"", "\"release_product\": \"other\"")); + var productError = assertThrows(RuntimeException.class, () -> CudaResources.install( + tempFolder, tempFolder.resolve("wrong-product").toFile(), RELEASE, + new CudaResources.CudaPlatform(PLATFORM), wrongProduct, ignored -> { + throw new AssertionError("Archive downloads must not start for an invalid manifest"); + })); + assertTrue(productError.getMessage().contains("not a CUDA manifest")); + + var missingComponents = new LinkedHashMap<>(manifest.components()); + missingComponents.remove("cuda_cccl"); + var missingComponent = new CudaResources.NvidiaCudaManifest( + manifest.releaseDate(), manifest.releaseLabel(), manifest.releaseProduct(), missingComponents); + assertThrows(RuntimeException.class, () -> missingComponent.getRequiredPackages(PLATFORM)); + } + + @Test + public void manifestPlatformMatchesNvidiaKeysAndRejectsUnavailablePlatforms() { + assertEquals(CudaResources.LINUX_X86_64_PLATFORM, + CudaResources.getManifestPlatform(SupportedPlatform.LINUX, "amd64")); + assertEquals(CudaResources.LINUX_PPC64LE_PLATFORM, + CudaResources.getManifestPlatform(SupportedPlatform.LINUX, "ppc64le")); + assertEquals(CudaResources.LINUX_SBSA_PLATFORM, + CudaResources.getManifestPlatform(SupportedPlatform.LINUX, "aarch64")); + assertEquals(CudaResources.WINDOWS_X86_64_PLATFORM, + CudaResources.getManifestPlatform(SupportedPlatform.WINDOWS, "x86_64")); + + assertThrows(RuntimeException.class, + () -> CudaResources.getManifestPlatform(SupportedPlatform.MAC_OS, "aarch64")); + assertThrows(RuntimeException.class, + () -> CudaResources.getManifestPlatform(SupportedPlatform.WINDOWS, "aarch64")); + } + + @Test + public void installationFetchesOnlyTheSelectedPlatformArchives() throws IOException { + var archives = createArchives(); + var manifest = addUnusedPlatforms(archives.manifest()); + var platform = new CudaResources.CudaPlatform(PLATFORM); + var platformFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, platform); + var writes = new AtomicInteger(); + + var installation = CudaResources.install(tempFolder, platformFolder, RELEASE, platform, manifest, + cudaPackage -> { + assertTrue(cudaPackage.archive().relativePath().contains("/" + PLATFORM + "/")); + var source = archives.files().get(cudaPackage.component()); + return copyingResource(source, source.getFileName().toString(), writes); + }); + + assertTrue(CudaResources.isCudaInstallation(installation, PLATFORM)); + assertEquals(CudaResources.REQUIRED_COMPONENTS.size(), writes.get()); + } + + @Test + public void archiveDownloadsRequireBothExpectedSizeAndSha256() throws IOException { + var source = Files.writeString(tempFolder.resolve("cuda_cudart.tar.xz"), "archive"); + var actualSize = Files.size(source); + var actualSha = sha256(source); + var platformFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, new CudaResources.CudaPlatform(PLATFORM)); + + var wrongSize = manifestForSingleArchive(source, actualSha, actualSize + 1); + var sizeError = assertThrows(RuntimeException.class, + () -> install(wrongSize, platformFolder, source, new AtomicInteger())); + assertTrue(sizeError.getMessage().contains("expected size")); + assertFalse(CudaResources.getArchiveFile(platformFolder, + wrongSize.getRequiredPackages(PLATFORM).get(0)).isFile()); + + var wrongSha = manifestForSingleArchive(source, "0".repeat(64), actualSize); + var shaError = assertThrows(RuntimeException.class, + () -> install(wrongSha, platformFolder, source, new AtomicInteger())); + assertTrue(shaError.getMessage().contains("expected SHA-256")); + } + + @Test + public void assembleSupportsTarXzAndZipPackages() throws IOException { + var archives = createArchives(); + var stagingFolder = Files.createDirectory(tempFolder.resolve("cudalib")); + + CudaResources.assemble(stagingFolder.toFile(), new CudaResources.CudaPlatform(PLATFORM), + downloadedPackages(archives)); + + assertEquals(PLATFORM, Files.readString(stagingFolder.resolve(CudaResources.PLATFORM_FILENAME))); + assertTrue(Files.isDirectory(stagingFolder.resolve("bin"))); + assertEquals("cuda.h", Files.readString(stagingFolder.resolve("include/cuda.h"))); + assertEquals("cuda_runtime.h", Files.readString(stagingFolder.resolve("include/cuda_runtime.h"))); + assertEquals("texture", Files.readString(stagingFolder.resolve("include/texture_fetch_functions.h"))); + assertEquals("curand", Files.readString(stagingFolder.resolve("include/curand_mtgp32_kernel.h"))); + assertEquals("target", Files.readString(stagingFolder.resolve("include/nv/target"))); + assertEquals("host_config", Files.readString(stagingFolder.resolve("include/crt/host_config.h"))); + assertEquals("libdevice", Files.readString(stagingFolder.resolve("nvvm/libdevice/libdevice.10.bc"))); + assertFalse(Files.exists(stagingFolder.resolve("bin/discarded"))); + assertFalse(Files.exists(stagingFolder.resolve("bin/discarded.exe"))); + assertTrue(CudaResources.isCudaInstallation(stagingFolder.toFile(), PLATFORM)); + } + + @Test + public void extractionRejectsTraversalEntries() throws IOException { + var traversalArchive = tempFolder.resolve("traversal.tar.xz"); + writeTarXz(traversalArchive, Map.of( + "cuda_cudart/include/../escape.h", bytes("escape"))); + var traversalPackage = new CudaResources.DownloadedPackage( + new CudaResources.CudaPackage("cuda_cudart", + new CudaResources.CudaArchive("cuda_cudart/linux-x86_64/cuda_cudart.tar.xz", SHA256, 1)), + traversalArchive.toFile()); + + assertThrows(RuntimeException.class, () -> CudaResources.assemble( + Files.createDirectory(tempFolder.resolve("traversal-out")).toFile(), + new CudaResources.CudaPlatform(PLATFORM), List.of(traversalPackage))); + assertFalse(Files.exists(tempFolder.resolve("escape.h"))); + + var driveArchive = tempFolder.resolve("drive.tar.xz"); + writeTarXz(driveArchive, Map.of("C:/escape.h", bytes("escape"))); + var drivePackage = new CudaResources.DownloadedPackage( + new CudaResources.CudaPackage("cuda_cudart", + new CudaResources.CudaArchive("cuda_cudart/linux-x86_64/cuda_cudart.tar.xz", SHA256, 1)), + driveArchive.toFile()); + assertThrows(RuntimeException.class, () -> CudaResources.assemble( + Files.createDirectory(tempFolder.resolve("drive-out")).toFile(), + new CudaResources.CudaPlatform(PLATFORM), List.of(drivePackage))); + } + + @Test + public void requiredComponentsAreStoredPerReleaseWithoutDeduplication() throws IOException { + var archives = createArchives(); + var firstRelease = CudaResources.getPlatformFolder(tempFolder, RELEASE, + new CudaResources.CudaPlatform(PLATFORM)); + var secondRelease = CudaResources.getPlatformFolder(tempFolder, "13.3.1", + new CudaResources.CudaPlatform(PLATFORM)); + + var first = install(archives, firstRelease, new AtomicInteger()); + var second = install(archives, secondRelease, new AtomicInteger()); + + assertTrue(CudaResources.isCudaInstallation(first, PLATFORM)); + assertTrue(CudaResources.isCudaInstallation(second, PLATFORM)); + assertNotEquals(CudaResources.getArchiveFile(firstRelease, + archives.manifest().getRequiredPackages(PLATFORM).get(0)).toPath(), + CudaResources.getArchiveFile(secondRelease, + archives.manifest().getRequiredPackages(PLATFORM).get(0)).toPath()); + } + + @Test + public void existingValidInstallationIsReusedAndUsageIsRefreshed() throws IOException { + var platform = new CudaResources.CudaPlatform(hostPlatform()); + var platformFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, platform); + var installation = CudaResources.getInstallationFolder(platformFolder); + writeValidInstallation(installation.toPath(), PLATFORM); + var old = FileTime.from(Instant.now().minus(Duration.ofDays(61))); + Files.setLastModifiedTime(platformFolder.toPath().getParent(), old); + + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + var result = new ClangResources(parser).getBuiltinCudaLib(); + + assertEquals(installation.getAbsoluteFile(), result.getAbsoluteFile()); + assertTrue(Files.getLastModifiedTime(platformFolder.toPath().getParent()).toInstant() + .isAfter(Instant.now().minus(Duration.ofDays(1)))); + } + + @Test + public void invalidPublishedInstallationFailsWithoutRepair() throws IOException { + var platform = new CudaResources.CudaPlatform(hostPlatform()); + var platformFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, platform); + var installation = CudaResources.getInstallationFolder(platformFolder); + Files.createDirectories(installation.toPath()); + Files.writeString(installation.toPath().resolve("sentinel"), "do not repair"); + + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + var error = assertThrows(RuntimeException.class, () -> new ClangResources(parser).getBuiltinCudaLib()); + + assertTrue(error.getMessage().contains(installation.getAbsolutePath())); + assertTrue(error.getMessage().contains("delete this directory manually to regenerate")); + assertEquals("do not repair", Files.readString(installation.toPath().resolve("sentinel"))); + } + + @Test + public void concurrentPublicationLeavesOneValidInstallation() throws Exception { + var archives = createArchives(); + var platform = new CudaResources.CudaPlatform(PLATFORM); + var platformFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, platform); + var writes = new AtomicInteger(); + var executor = Executors.newFixedThreadPool(4); + var futures = new ArrayList>(); + + try { + for (int i = 0; i < 4; i++) { + futures.add(executor.submit(() -> install(archives, platformFolder, writes))); + } + + for (var future : futures) { + assertEquals(CudaResources.getInstallationFolder(platformFolder).getAbsoluteFile(), + future.get(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).getAbsoluteFile()); + } + } finally { + executor.shutdownNow(); + executor.awaitTermination(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } + + assertTrue(CudaResources.isCudaInstallation(CudaResources.getInstallationFolder(platformFolder), PLATFORM)); + try (var children = Files.list(platformFolder.toPath())) { + assertTrue(children.noneMatch(path -> path.getFileName().toString().startsWith(".cudalib.tmp-"))); + } + try (var children = Files.list(platformFolder.toPath().resolve("archives/cuda_cudart"))) { + assertTrue(children.noneMatch(path -> path.getFileName().toString().startsWith(".cuda_cudart"))); + } + assertTrue(writes.get() >= 4); + } + + @Test + public void staleCudaReleasesAreRemovedAfterSixtyDays() throws IOException { + var platform = new CudaResources.CudaPlatform(hostPlatform()); + var currentFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, platform); + var currentInstallation = CudaResources.getInstallationFolder(currentFolder); + writeValidInstallation(currentInstallation.toPath(), PLATFORM); + + var staleFolder = CudaResources.getPlatformFolder(tempFolder, "11.8.0", platform); + Files.createDirectories(staleFolder.toPath()); + Files.setLastModifiedTime(staleFolder.toPath().getParent(), + FileTime.from(Instant.now().minus(Duration.ofDays(61)))); + + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + new ClangResources(parser).getBuiltinCudaLib(); + + assertTrue(currentInstallation.isDirectory()); + assertFalse(staleFolder.getParentFile().exists()); + } + + private File install(CudaResources.NvidiaCudaManifest manifest, File platformFolder, Path source, + AtomicInteger writes) { + return CudaResources.install(tempFolder, platformFolder, RELEASE, + new CudaResources.CudaPlatform(PLATFORM), manifest, + cudaPackage -> copyingResource(source, + source.getFileName().toString(), writes)); + } + + private File install(ArchiveSet archives, File platformFolder, AtomicInteger writes) { + return CudaResources.install(tempFolder, platformFolder, RELEASE, + new CudaResources.CudaPlatform(PLATFORM), archives.manifest(), + cudaPackage -> { + var source = archives.files().get(cudaPackage.component()); + return copyingResource(source, source.getFileName().toString(), writes); + }); + } + + private ArchiveSet createArchives() throws IOException { + var files = new LinkedHashMap(); + var cudart = tempFolder.resolve("cuda_cudart.tar.xz"); + writeTarXz(cudart, Map.of( + "cuda_cudart/include/cuda.h", bytes("cuda.h"), + "cuda_cudart/include/cuda_runtime.h", bytes("cuda_runtime.h"), + "cuda_cudart/include/texture_fetch_functions.h", bytes("texture"), + "cuda_cudart/bin/discarded", bytes("discarded"))); + files.put("cuda_cudart", cudart); + + var nvcc = tempFolder.resolve("cuda_nvcc.zip"); + writeZip(nvcc, Map.of( + "cuda_nvcc/include/crt/host_config.h", bytes("host_config"), + "cuda_nvcc/nvvm/libdevice/libdevice.10.bc", bytes("libdevice"), + "cuda_nvcc/bin/discarded.exe", bytes("discarded"))); + files.put("cuda_nvcc", nvcc); + + var curand = tempFolder.resolve("libcurand.tar.xz"); + writeTarXz(curand, Map.of( + "libcurand/include/curand_mtgp32_kernel.h", bytes("curand"))); + files.put("libcurand", curand); + + var cccl = tempFolder.resolve("cuda_cccl.zip"); + writeZip(cccl, Map.of("cuda_cccl/include/nv/target", bytes("target"))); + files.put("cuda_cccl", cccl); + + var components = new LinkedHashMap(); + for (var component : CudaResources.REQUIRED_COMPONENTS) { + var archive = files.get(component); + var archivePath = component + "/linux-x86_64/" + archive.getFileName(); + var cudaArchive = new CudaResources.CudaArchive(archivePath, sha256(archive), Files.size(archive)); + components.put(component, new CudaResources.NvidiaCudaComponent(component, RELEASE, + Map.of(PLATFORM, cudaArchive))); + } + + return new ArchiveSet(new CudaResources.NvidiaCudaManifest("2024-01-02", RELEASE, "cuda", components), files); + } + + private List downloadedPackages(ArchiveSet archives) { + return archives.manifest().getRequiredPackages(PLATFORM).stream() + .map(cudaPackage -> new CudaResources.DownloadedPackage(cudaPackage, + archives.files().get(cudaPackage.component()).toFile())) + .toList(); + } + + private CudaResources.NvidiaCudaManifest manifestForSingleArchive(Path source, String sha256, long size) { + var components = new LinkedHashMap(); + for (var component : CudaResources.REQUIRED_COMPONENTS) { + var archiveName = component.equals("cuda_cudart") ? "cuda_cudart.tar.xz" : source.getFileName().toString(); + var relativePath = component + "/linux-x86_64/" + archiveName; + components.put(component, new CudaResources.NvidiaCudaComponent(component, RELEASE, + Map.of(PLATFORM, new CudaResources.CudaArchive(relativePath, sha256, size)))); + } + + return new CudaResources.NvidiaCudaManifest("2024-01-02", RELEASE, "cuda", components); + } + + private CudaResources.NvidiaCudaManifest addUnusedPlatforms(CudaResources.NvidiaCudaManifest manifest) { + var components = new LinkedHashMap(); + for (var entry : manifest.components().entrySet()) { + var selectedArchive = entry.getValue().archives().get(PLATFORM); + var archives = new LinkedHashMap<>(entry.getValue().archives()); + for (var unusedPlatform : List.of(CudaResources.LINUX_PPC64LE_PLATFORM, + CudaResources.LINUX_SBSA_PLATFORM, CudaResources.WINDOWS_X86_64_PLATFORM)) { + archives.put(unusedPlatform, new CudaResources.CudaArchive( + entry.getKey() + "/" + unusedPlatform + "/unused.tar.xz", + selectedArchive.sha256(), selectedArchive.size())); + } + components.put(entry.getKey(), new CudaResources.NvidiaCudaComponent( + entry.getValue().name(), entry.getValue().version(), archives)); + } + + return new CudaResources.NvidiaCudaManifest(manifest.releaseDate(), manifest.releaseLabel(), + manifest.releaseProduct(), components); + } + + private void writeValidInstallation(Path installation, String platform) throws IOException { + Files.createDirectories(installation.resolve("bin")); + Files.writeString(installation.resolve(CudaResources.PLATFORM_FILENAME), platform); + for (var requiredFile : List.of( + "include/cuda.h", + "include/cuda_runtime.h", + "include/texture_fetch_functions.h", + "include/curand_mtgp32_kernel.h", + "include/nv/target", + "include/crt/host_config.h", + "nvvm/libdevice/libdevice.10.bc")) { + var file = installation.resolve(requiredFile); + Files.createDirectories(file.getParent()); + Files.writeString(file, requiredFile); + } + } + + private FileResourceProvider copyingResource(Path source, String filename, AtomicInteger writes) { + return new FileResourceProvider() { + @Override + public File write(java.io.File folder) { + writes.incrementAndGet(); + try { + var destination = folder.toPath().resolve(filename); + Files.copy(source, destination); + return destination.toFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String version() { + return "test"; + } + + @Override + public String getFilename() { + return filename; + } + }; + } + + private static String hostPlatform() { + return CudaResources.getCurrentPlatform().manifestName(); + } + + private static void writeTarXz(Path archive, Map files) throws IOException { + try (OutputStream output = Files.newOutputStream(archive); + var xzOutput = new XZCompressorOutputStream(output); + var tarOutput = new TarArchiveOutputStream(xzOutput)) { + tarOutput.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + for (var file : files.entrySet()) { + var entry = new TarArchiveEntry(file.getKey()); + entry.setSize(file.getValue().length); + tarOutput.putArchiveEntry(entry); + tarOutput.write(file.getValue()); + tarOutput.closeArchiveEntry(); + } + } + } + + private static void writeZip(Path archive, Map files) throws IOException { + try (OutputStream output = Files.newOutputStream(archive); + var zipOutput = new ZipArchiveOutputStream(output)) { + for (var file : files.entrySet()) { + var entry = new ZipArchiveEntry(file.getKey()); + zipOutput.putArchiveEntry(entry); + zipOutput.write(file.getValue()); + zipOutput.closeArchiveEntry(); + } + } + } + + private static String manifestJson() { + return """ + { + "release_date": "2024-01-02", + "release_label": "12.3.2", + "release_product": "cuda", + "cuda_cudart": %s, + "cuda_nvcc": %s, + "libcurand": %s, + "cuda_cccl": %s + } + """.formatted( + component("CUDA Runtime", "cuda_cudart/linux-x86_64/cuda_cudart.tar.xz"), + component("CUDA NVCC", "cuda_nvcc/linux-x86_64/cuda_nvcc.tar.xz"), + component("cuRAND", "libcurand/linux-x86_64/libcurand.tar.xz"), + component("CCCL", "cuda_cccl/linux-x86_64/cuda_cccl.tar.xz")); + } + + private static String component(String name, String relativePath) { + return """ + { + "name": "%s", + "version": "12.3.101", + "linux-x86_64": { + "relative_path": "%s", + "sha256": "%s", + "size": "1" + } + } + """.formatted(name, relativePath, SHA256); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static String sha256(Path file) throws IOException { + try { + return java.util.HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(Files.readAllBytes(file))); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + private static final String SHA256 = "0".repeat(64); + + private record ArchiveSet(CudaResources.NvidiaCudaManifest manifest, Map files) { + } +} diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/CxxCudaTester.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/CxxCudaTester.java index 45fdfffce..fde8a56bc 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/CxxCudaTester.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/CxxCudaTester.java @@ -16,7 +16,7 @@ import java.util.Arrays; import java.util.List; -import pt.up.fe.specs.lang.SpecsPlatforms; +import pt.up.fe.specs.clang.ClangResources; public class CxxCudaTester extends AClangAstTester { @@ -28,8 +28,7 @@ public CxxCudaTester(List files) { // super("cxx/cuda", files, Arrays.asList("-std=cuda")); super("cxx/cuda", files); - // Windows currently not supported - if (SpecsPlatforms.isWindows()) { + if (!ClangResources.isBuiltinCudaSupported()) { doNotRun(); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java index 2a857db62..a0e13c028 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java @@ -19,7 +19,7 @@ import pt.up.fe.specs.clang.LibcMode; import pt.up.fe.specs.clang.parser.CxxCudaTester; -/** Verifies built-in CUDA parsing through Clava's historical cudalib archive. */ +/** Verifies built-in CUDA parsing through the pinned NVIDIA redistribution packages. */ public class CxxCudaTest { @Test public void testAtomicAdd() { diff --git a/Clava-JS/code/sideEffects.ts b/Clava-JS/code/sideEffects.ts index 2c62d3bf4..090da8743 100644 --- a/Clava-JS/code/sideEffects.ts +++ b/Clava-JS/code/sideEffects.ts @@ -18,14 +18,14 @@ const datastore = Weaver.getWeaverEngine().getData().get(); datastore.set(CxxWeaverOptions.DISABLE_CLAVA_INFO, true); datastore.set( CodeParser.DUMPER_FOLDER, - new JavaTypes.File(getClangDumperCacheDir()) + new JavaTypes.File(getClavaCacheDir()) ); /** Code to obtain temporary folder **/ -function getClangDumperCacheDir(): string { - // The version will be added by the installer to isolate different installed versions - return path.join(getCacheBaseDir(), pkg.name, "clang-dumper"); +function getClavaCacheDir(): string { + // Java adds separate namespaces for the dumper and CUDA resources. + return path.join(getCacheBaseDir(), pkg.name); } function getCacheBaseDir(): string { From 003d248b91adf64137b830d60f2e570ea33a8ee7 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 6 Aug 2026 02:27:35 +0100 Subject: [PATCH 2/3] fix(clang): protect initializing CUDA releases --- .../pt/up/fe/specs/clang/CudaResources.java | 21 +++++++ .../up/fe/specs/clang/CudaResourcesTest.java | 55 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java index b1a27491e..1e0672552 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java @@ -99,10 +99,31 @@ static File getBuiltinCudaLib(Path cacheRoot) { return useExistingInstallation(cacheRoot, platformFolder, platform, installationFolder); } + claimInUse(cacheRoot, platformFolder); var manifest = getManifest(cacheRoot, platformFolder); return install(cacheRoot, platformFolder, releaseTag, platform, manifest, CudaResources::getArchiveResource); } + static void claimInUse(Path cacheRoot, File platformFolder) { + CacheFiles.withMaintenanceLock(cacheRoot, () -> { + var platformPath = platformFolder.toPath(); + var releasePath = platformPath.getParent(); + if (releasePath == null) { + throw new RuntimeException("CUDA platform folder is not below a release folder: '" + + platformFolder + "'"); + } + + try { + Files.createDirectories(platformPath); + } catch (IOException e) { + throw new UncheckedIOException("Could not create CUDA platform folder '" + platformPath + "'", e); + } + + CacheFiles.touch(releasePath); + CacheFiles.touch(platformPath); + }); + } + static CudaPlatform requireSupportedPlatform() { return getCurrentPlatform(); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java index f3169882a..c51f893c9 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java @@ -38,6 +38,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -265,6 +266,49 @@ public void invalidPublishedInstallationFailsWithoutRepair() throws IOException assertEquals("do not repair", Files.readString(installation.toPath().resolve("sentinel"))); } + @Test + public void oldPartialReleaseIsProtectedWhileInitializationContinues() throws Exception { + var platformFolder = CudaResources.getPlatformFolder(tempFolder, RELEASE, + new CudaResources.CudaPlatform(PLATFORM)); + var releaseFolder = platformFolder.toPath().getParent(); + Files.createDirectories(platformFolder.toPath().resolve("partial")); + Files.writeString(platformFolder.toPath().resolve("partial/manifest-download"), "in progress"); + var old = FileTime.from(Instant.now().minus(Duration.ofDays(61))); + Files.setLastModifiedTime(releaseFolder, old); + Files.setLastModifiedTime(platformFolder.toPath(), old); + + var claimed = new CountDownLatch(1); + var allowInitializationToFinish = new CountDownLatch(1); + var executor = Executors.newFixedThreadPool(2); + + try { + var initialization = executor.submit(() -> { + CudaResources.claimInUse(tempFolder, platformFolder); + claimed.countDown(); + awaitLatch(allowInitializationToFinish); + }); + assertTrue(claimed.await(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + + var cleanup = executor.submit(() -> CacheFiles.deleteStaleDirectories(tempFolder, + tempFolder.resolve("cuda"), Instant.now().minus(Duration.ofDays(60)), null)); + cleanup.get(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + + assertTrue(Files.isDirectory(releaseFolder)); + assertTrue(Files.isDirectory(platformFolder.toPath())); + assertTrue(Files.getLastModifiedTime(releaseFolder).toInstant() + .isAfter(Instant.now().minus(Duration.ofDays(1)))); + assertTrue(Files.getLastModifiedTime(platformFolder.toPath()).toInstant() + .isAfter(Instant.now().minus(Duration.ofDays(1)))); + + allowInitializationToFinish.countDown(); + initialization.get(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } finally { + allowInitializationToFinish.countDown(); + executor.shutdownNow(); + executor.awaitTermination(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } + } + @Test public void concurrentPublicationLeavesOneValidInstallation() throws Exception { var archives = createArchives(); @@ -532,6 +576,17 @@ private static String sha256(Path file) throws IOException { private static final String SHA256 = "0".repeat(64); + private static void awaitLatch(CountDownLatch latch) { + try { + if (!latch.await(TEST_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)) { + throw new AssertionError("Timed out waiting for CUDA initialization test coordination"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + private record ArchiveSet(CudaResources.NvidiaCudaManifest manifest, Map files) { } } From 7c4f1f24de9cda61dd343cf98d051270f04149de Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 6 Aug 2026 16:44:10 +0100 Subject: [PATCH 3/3] fix(clang): select CUDA platform from NVIDIA manifest Keep the clang-dumper includes archive limited to built-in libc/libc++ and configure the system Clang resource directory for SYSTEM-mode CUDA parsing. Built-in CUDA supplies only the NVIDIA cudalib. Select a host-compatible CUDA platform from the intersection of all required NVIDIA manifest components, then reuse that parsed manifest for installation. Tests: CudaResourcesTest, ClangResourcesTest, and CxxCudaTest --- ClangAstParser/clang-dumper-release.tag | 2 +- .../src/pt/up/fe/specs/clang/ClangFiles.java | 2 +- .../pt/up/fe/specs/clang/ClangResources.java | 73 ++++++++- .../pt/up/fe/specs/clang/CudaResources.java | 145 ++++++++++++++---- .../clang/codeparser/ParallelCodeParser.java | 9 +- .../fe/specs/clang/dumper/ClangAstDumper.java | 10 +- .../up/fe/specs/clang/ClangResourcesTest.java | 29 ++-- .../up/fe/specs/clang/CudaResourcesTest.java | 70 ++++++--- .../specs/clang/parser/tests/CxxCudaTest.java | 6 +- 9 files changed, 269 insertions(+), 77 deletions(-) diff --git a/ClangAstParser/clang-dumper-release.tag b/ClangAstParser/clang-dumper-release.tag index 9acbd12ca..73e74e98f 100644 --- a/ClangAstParser/clang-dumper-release.tag +++ b/ClangAstParser/clang-dumper-release.tag @@ -1 +1 @@ -v18.1.8_1 +v18.1.8_2 diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java index 731cac065..1fcbf321b 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java @@ -16,6 +16,6 @@ import java.io.File; import java.util.List; -public record ClangFiles(File clangExecutable, List builtinIncludes) { +public record ClangFiles(File clangExecutable, List builtinIncludes, File systemResourceDir) { } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 4432867aa..c7fcffe77 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -67,12 +67,16 @@ public File getBuiltinCudaLib() { public ClangFiles getClangFiles(LibcMode libcMode) { var source = ClangAstWebResource.getDumperSource(); + var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); if (source instanceof LocalBuild localBuild) { - return new ClangFiles(getLocalExecutable(localBuild.folder()), List.of()); + var clangExecutable = getLocalExecutable(localBuild.folder()); + var systemResourceDir = libcMode == LibcMode.SYSTEM && useBuiltinCuda + ? findSystemClangResourceDir(null) + : null; + return new ClangFiles(clangExecutable, List.of(), systemResourceDir); } - var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); var resourceFolder = getClangResourceFolder(); var key = libcMode.name() + "_" + useBuiltinCuda + "_" + source + "_" + resourceFolder.getAbsolutePath(); @@ -89,6 +93,9 @@ public ClangFiles getClangFiles(LibcMode libcMode) { var manifest = ClangAstWebResource.getManifest(resourceFolder); File clangExecutable = prepareResources(manifest, resourceFolder); var includes = prepareIncludes(manifest, clangExecutable, libcMode); + var systemResourceDir = libcMode == LibcMode.SYSTEM && useBuiltinCuda + ? prepareSystemClangResourceDir(manifest) + : null; if (useBuiltinCuda) { getBuiltinCudaLib(); @@ -97,7 +104,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { touchUse(resourceFolder, includes.extractedFolder()); updateLastUsedAndCleanupStaleVersions(resourceFolder, includes.extractedFolder()); - var newFiles = new CachedClangFiles(new ClangFiles(clangExecutable, includes.folders()), + var newFiles = new CachedClangFiles(new ClangFiles(clangExecutable, includes.folders(), systemResourceDir), includes.extractedFolder()); var existingFiles = CLANG_FILES_CACHE.putIfAbsent(key, newFiles); var selectedFiles = existingFiles == null ? newFiles : existingFiles; @@ -116,6 +123,11 @@ private boolean isUsable(CachedClangFiles cached) { return false; } + if (cached.files().systemResourceDir() != null + && !cached.files().systemResourceDir().isDirectory()) { + return false; + } + var includesFolder = cached.includesFolder(); if (includesFolder == null) { return true; @@ -292,9 +304,8 @@ private static ProcessOutputAsString runClangAstDumper(File clangExecutable, Fil private PreparedIncludes prepareIncludes(ClangDumperManifest manifest, File clangExecutable, LibcMode libcMode) { var useBuiltinLibc = useBuiltinLibc(clangExecutable, libcMode); - var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); - if (!useBuiltinLibc && !useBuiltinCuda) { + if (!useBuiltinLibc) { return new PreparedIncludes(List.of(), null); } @@ -305,6 +316,58 @@ private PreparedIncludes prepareIncludes(ClangDumperManifest manifest, File clan return new PreparedIncludes(includeFolders.stream().map(File::getAbsolutePath).toList(), extractedFolder); } + private File prepareSystemClangResourceDir(ClangDumperManifest manifest) { + var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; + var llvmMajor = getCurrentAsset(manifest, executableKind).llvm_major(); + return findSystemClangResourceDir(llvmMajor); + } + + private File findSystemClangResourceDir(Integer llvmMajor) { + var commandNames = getSystemClangCommandNames(llvmMajor); + for (var commandName : commandNames) { + final ProcessOutputAsString output; + try { + output = SpecsSystem.runProcess(List.of(commandName, "-print-resource-dir"), true, false); + } catch (RuntimeException e) { + continue; + } + + if (output.getReturnValue() != 0 || output.getStdOut() == null) { + continue; + } + + var resourceDir = new File(output.getStdOut().trim()); + if (isSystemClangResourceDir(resourceDir, llvmMajor)) { + SpecsLogs.debug(() -> "Using system Clang resource directory '" + + resourceDir.getAbsolutePath() + "'"); + return resourceDir; + } + } + + var expectedVersion = llvmMajor == null ? "the local clang-dumper build's version" + : "LLVM " + llvmMajor; + throw new RuntimeException("Could not find a system Clang resource directory for SYSTEM mode with built-in CUDA" + + " on host '" + SupportedPlatform.getCurrentPlatform() + "' (expected " + expectedVersion + + "). Tried: " + commandNames); + } + + private static List getSystemClangCommandNames(Integer llvmMajor) { + var suffix = SupportedPlatform.getCurrentPlatform().isWindows() ? ".exe" : ""; + if (llvmMajor == null) { + return List.of("clang++" + suffix); + } + + return List.of("clang++-" + llvmMajor + suffix, "clang++" + suffix); + } + + private static boolean isSystemClangResourceDir(File resourceDir, Integer llvmMajor) { + if (!resourceDir.isDirectory() || !new File(resourceDir, "include").isDirectory()) { + return false; + } + + return llvmMajor == null || resourceDir.getName().equals(Integer.toString(llvmMajor)); + } + private File prepareIncludesFolder(ClangDumperManifest manifest) { var includesAsset = getCurrentAsset(manifest, "includes"); return resolveIncludes(getClangCacheRoot(), includesAsset, diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java index 1e0672552..4cf8718d9 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/CudaResources.java @@ -42,6 +42,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -60,10 +61,6 @@ final class CudaResources { static final String NVIDIA_REDIST_ROOT = "https://developer.download.nvidia.com/compute/cuda/redist/"; - static final String LINUX_X86_64_PLATFORM = "linux-x86_64"; - static final String LINUX_PPC64LE_PLATFORM = "linux-ppc64le"; - static final String LINUX_SBSA_PLATFORM = "linux-sbsa"; - static final String WINDOWS_X86_64_PLATFORM = "windows-x86_64"; static final List REQUIRED_COMPONENTS = List.of("cuda_cudart", "cuda_nvcc", "libcurand", "cuda_cccl"); static final String PLATFORM_FILENAME = ".platform"; @@ -89,7 +86,10 @@ private CudaResources() { static File getBuiltinCudaLib(Path cacheRoot) { var releaseTag = ClangAstWebResource.getCudaReleaseTag(); - var platform = requireSupportedPlatform(); + var releaseFolder = getReleaseFolder(cacheRoot, releaseTag); + claimReleaseInUse(cacheRoot, releaseFolder); + var manifest = getManifest(cacheRoot, releaseFolder); + var platform = requireSupportedPlatform(manifest); var platformFolder = getPlatformFolder(cacheRoot, releaseTag, platform); var installationFolder = getInstallationFolder(platformFolder); @@ -100,10 +100,21 @@ static File getBuiltinCudaLib(Path cacheRoot) { } claimInUse(cacheRoot, platformFolder); - var manifest = getManifest(cacheRoot, platformFolder); return install(cacheRoot, platformFolder, releaseTag, platform, manifest, CudaResources::getArchiveResource); } + private static void claimReleaseInUse(Path cacheRoot, File releaseFolder) { + CacheFiles.withMaintenanceLock(cacheRoot, () -> { + try { + Files.createDirectories(releaseFolder.toPath()); + } catch (IOException e) { + throw new UncheckedIOException("Could not create CUDA release folder '" + releaseFolder + "'", e); + } + + CacheFiles.touch(releaseFolder.toPath()); + }); + } + static void claimInUse(Path cacheRoot, File platformFolder) { CacheFiles.withMaintenanceLock(cacheRoot, () -> { var platformPath = platformFolder.toPath(); @@ -128,9 +139,17 @@ static CudaPlatform requireSupportedPlatform() { return getCurrentPlatform(); } + static CudaPlatform requireSupportedPlatform(NvidiaCudaManifest manifest) { + return getCurrentPlatform(manifest); + } + static boolean isSupportedPlatform() { + return isSupportedPlatform(ClangResources.getDefaultTempFolder().toPath()); + } + + static boolean isSupportedPlatform(Path cacheRoot) { try { - getCurrentPlatform(); + getCurrentPlatform(cacheRoot); return true; } catch (RuntimeException e) { return false; @@ -138,42 +157,114 @@ static boolean isSupportedPlatform() { } static CudaPlatform getCurrentPlatform() { - return new CudaPlatform(getManifestPlatform(SupportedPlatform.getCurrentPlatform(), System.getProperty("os.arch"))); + return getCurrentPlatform(ClangResources.getDefaultTempFolder().toPath()); + } + + static CudaPlatform getCurrentPlatform(Path cacheRoot) { + var releaseTag = ClangAstWebResource.getCudaReleaseTag(); + var releaseFolder = getReleaseFolder(cacheRoot, releaseTag); + claimReleaseInUse(cacheRoot, releaseFolder); + return getCurrentPlatform(getManifest(cacheRoot, releaseFolder)); } - static String getManifestPlatform(SupportedPlatform platform, String architecture) { + static CudaPlatform getCurrentPlatform(NvidiaCudaManifest manifest) { + return new CudaPlatform(getManifestPlatform(manifest, SupportedPlatform.getCurrentPlatform(), + System.getProperty("os.arch"))); + } + + static String getManifestPlatform(NvidiaCudaManifest manifest, SupportedPlatform platform, String architecture) { + Objects.requireNonNull(manifest, "manifest"); Objects.requireNonNull(platform, "platform"); Objects.requireNonNull(architecture, "architecture"); - var normalizedArchitecture = architecture.toLowerCase(Locale.ROOT); - - if (platform.isWindows() && isX86_64(normalizedArchitecture)) { - return WINDOWS_X86_64_PLATFORM; - } - if (platform.isLinux()) { - if (isX86_64(normalizedArchitecture)) { - return LINUX_X86_64_PLATFORM; + var commonPlatforms = new LinkedHashSet(); + var missingComponents = new ArrayList(); + var firstComponent = true; + for (var componentName : REQUIRED_COMPONENTS) { + var component = manifest.components().get(componentName); + if (component == null) { + missingComponents.add(componentName); + continue; } - if (normalizedArchitecture.equals("ppc64le")) { - return LINUX_PPC64LE_PLATFORM; + if (firstComponent) { + commonPlatforms.addAll(component.archives().keySet()); + firstComponent = false; + } else { + commonPlatforms.retainAll(component.archives().keySet()); } + } - if (normalizedArchitecture.equals("aarch64") || normalizedArchitecture.equals("arm64")) { - return LINUX_SBSA_PLATFORM; - } + var selectedPlatform = commonPlatforms.stream() + .filter(candidate -> isCompatiblePlatform(candidate, platform, architecture)) + .findFirst(); + if (missingComponents.isEmpty() && selectedPlatform.isPresent()) { + return selectedPlatform.get(); + } + + var reason = missingComponents.isEmpty() + ? "no platform key is present in all required components and is compatible with this host" + : "the manifest is missing required components " + missingComponents; + throw new RuntimeException("Built-in CUDA is unsupported for host '" + platform + " (" + architecture + + ")': " + reason + ". Available manifest platform keys: " + + getAvailablePlatformKeys(manifest)); + } + + private static boolean isCompatiblePlatform(String manifestPlatform, SupportedPlatform hostPlatform, + String hostArchitecture) { + var separator = manifestPlatform.indexOf('-'); + if (separator <= 0 || separator == manifestPlatform.length() - 1) { + return false; + } + + var manifestOs = normalizeOs(manifestPlatform.substring(0, separator)); + var manifestArchitecture = normalizeArchitecture(manifestPlatform.substring(separator + 1)); + return manifestOs.equals(normalizeOs(hostPlatform)) + && manifestArchitecture.equals(normalizeArchitecture(hostArchitecture)); + } + + private static String normalizeOs(SupportedPlatform platform) { + return switch (platform) { + case WINDOWS -> "windows"; + case LINUX -> "linux"; + case MAC_OS -> "macos"; + }; + } + + private static String normalizeOs(String os) { + var normalized = os.toLowerCase(Locale.ROOT).replace("_", "").replace("-", ""); + return switch (normalized) { + case "darwin", "mac" -> "macos"; + default -> normalized; + }; + } + + private static String normalizeArchitecture(String architecture) { + var normalized = architecture.toLowerCase(Locale.ROOT).replace("_", "").replace("-", ""); + return switch (normalized) { + case "amd64", "x8664", "x64" -> "x8664"; + case "aarch64", "arm64", "armv8", "armv8l", "sbsa" -> "arm64"; + default -> normalized; + }; + } + + private static Map> getAvailablePlatformKeys(NvidiaCudaManifest manifest) { + var available = new LinkedHashMap>(); + for (var componentName : REQUIRED_COMPONENTS) { + var component = manifest.components().get(componentName); + var platforms = component == null ? List.of() : component.archives().keySet().stream().sorted().toList(); + available.put(componentName, platforms); } - throw new RuntimeException("The CUDA manifest does not provide a supported package for platform '" - + platform + "' and architecture '" + architecture + "'"); + return available; } - private static boolean isX86_64(String architecture) { - return architecture.equals("amd64") || architecture.equals("x86_64"); + private static File getReleaseFolder(Path cacheRoot, String releaseTag) { + return cacheRoot.resolve(CUDA_FOLDERNAME).resolve(releaseTag).toFile(); } static File getPlatformFolder(Path cacheRoot, String releaseTag, CudaPlatform platform) { - return cacheRoot.resolve(CUDA_FOLDERNAME).resolve(releaseTag).resolve(platform.manifestName()).toFile(); + return getReleaseFolder(cacheRoot, releaseTag).toPath().resolve(platform.manifestName()).toFile(); } static File getInstallationFolder(File platformFolder) { diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java index d16c2ce82..9313859bd 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java @@ -89,6 +89,7 @@ public App parse(List inputSources, List compilerOptions, ClavaCon ConcurrentLinkedQueue clangDump = new ConcurrentLinkedQueue<>(); DataStore options = ClangAstKeys.toDataStore(compilerOptions); + options.set(ClangAstKeys.LIBC_CXX_MODE, get(ClangAstKeys.LIBC_CXX_MODE)); // Add context to config // ClavaContext context = new ClavaContext(); @@ -145,7 +146,8 @@ public App parse(List inputSources, List compilerOptions, ClavaCon Future tUnit = executor .submit(() -> parseSource(source, id, standard, options, clangDump, - counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes())); + counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes(), + clangFiles.systemResourceDir())); futureTUnits.add(tUnit); @@ -362,7 +364,7 @@ private Standard getStandard(Collection sources, DataStore options) { private ClangAstData parseSource(File sourceFile, String id, Standard standard, DataStore options, ConcurrentLinkedQueue clangDump, ParallelProgressCounter counter, File parsingFolder, - File clangExecutable, List builtinIncludes) { + File clangExecutable, List builtinIncludes, File systemResourceDir) { // ConcurrentLinkedQueue clangDump, ConcurrentLinkedQueue workingFolders) { @@ -373,7 +375,8 @@ private ClangAstData parseSource(File sourceFile, String id, Standard standard, // Only show output of console after parsing is done, when using parallel parsing boolean streamConsoleOutput = !get(PARALLEL_PARSING); - ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes, this) + ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes, + systemResourceDir, this) .setBaseFolder(parsingFolder) .setSystemIncludesThreshold(get(SYSTEM_INCLUDES_THRESHOLD)); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index ba7f29de3..8b56c1c29 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -86,6 +86,7 @@ public static List getTempFiles() { private File baseFolder; private File clangExecutable; private List builtinIncludes; + private File systemResourceDir; private int systemIncludesThreshold; private final ClangResources clangResources; @@ -99,15 +100,18 @@ public static List getTempFiles() { * @param streamConsoleOutput * @param clangExecutable * @param builtinIncludes + * @param systemResourceDir * @param parserConfig */ public ClangAstDumper(boolean streamConsoleOutput, - File clangExecutable, List builtinIncludes, CodeParser parserConfig) { + File clangExecutable, List builtinIncludes, File systemResourceDir, + CodeParser parserConfig) { this.streamConsoleOutput = streamConsoleOutput; this.clangExecutable = clangExecutable; this.builtinIncludes = builtinIncludes; + this.systemResourceDir = systemResourceDir; this.workingFolders = new ArrayList<>(); this.lastWorkingFolder = null; @@ -252,6 +256,10 @@ else if (SourceType.isHeader(sourceFile)) { arguments.add(standard.isCxx() ? "c++" : "c"); } + if (systemResourceDir != null) { + arguments.add("-resource-dir=" + systemResourceDir.getAbsolutePath()); + } + // If it was determined that built-in includes will be used, disable system includes if (ClangResources.useBuiltinLibc(clangExecutable, config.get(ClangAstKeys.LIBC_CXX_MODE))) { arguments.add("-nostdinc"); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 4b30a9067..bdd8f2379 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -55,6 +55,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -486,10 +487,10 @@ public void maintenanceLockIsSharedAcrossJvmProcesses() throws Exception { } @Test - public void sameJvmInstancesReuseReleaseFilesAndRefreshSharedIncludes() throws Exception { - var firstParser = newParser(CodeParser.getBuiltinOption()); - var secondParser = newParser(CodeParser.getBuiltinOption()); - var thirdParser = newParser(CodeParser.getBuiltinOption()); + public void sameJvmInstancesReuseReleaseFilesAndPrepareIncludesOnlyForBuiltinLibc() throws Exception { + var firstParser = newParser(""); + var secondParser = newParser(""); + var thirdParser = newParser(""); var firstResources = new ClangResources(firstParser); var secondResources = new ClangResources(secondParser); var thirdResources = new ClangResources(thirdParser); @@ -507,14 +508,16 @@ public void sameJvmInstancesReuseReleaseFilesAndRefreshSharedIncludes() throws E assertEquals(firstFiles, secondFiles); assertEquals(firstFiles.clangExecutable().getAbsoluteFile(), thirdFiles.clangExecutable().getAbsoluteFile()); assertTrue(firstFiles.clangExecutable().isFile()); + assertTrue(firstFiles.builtinIncludes().isEmpty()); + assertTrue(secondFiles.builtinIncludes().isEmpty()); + assertFalse(thirdFiles.builtinIncludes().isEmpty()); - assumeTrue(!firstFiles.builtinIncludes().isEmpty()); - var shared = new File(firstFiles.builtinIncludes().get(0)).toPath(); + var shared = new File(thirdFiles.builtinIncludes().get(0)).toPath(); while (!Files.isRegularFile(shared.resolve("entrypoints.txt"))) { shared = shared.getParent(); } Files.setLastModifiedTime(shared, FileTime.from(Instant.now().minus(Duration.ofDays(61)))); - firstResources.getClangFiles(LibcMode.SYSTEM); + thirdResources.getClangFiles(LibcMode.BUILTIN_AND_LIBC); assertTrue(Files.getLastModifiedTime(shared).toInstant().isAfter(Instant.now().minus(Duration.ofDays(1)))); } finally { executor.shutdownNow(); @@ -523,14 +526,14 @@ public void sameJvmInstancesReuseReleaseFilesAndRefreshSharedIncludes() throws E } @Test - public void builtinCudaIncludesAreAvailableWithSystemLibc() { + public void builtinCudaUsesSystemClangResourceWithSystemLibc() { var parser = newParser(CodeParser.getBuiltinOption()); var clangFiles = new ClangResources(parser).getClangFiles(LibcMode.SYSTEM); - var hasCudaWrapper = clangFiles.builtinIncludes().stream() - .map(folder -> new File(folder, "__clang_cuda_runtime_wrapper.h")) - .anyMatch(File::isFile); - assertTrue(hasCudaWrapper, "Built-in CUDA must provide Clang's CUDA runtime wrapper independently of libc"); + assertTrue(clangFiles.builtinIncludes().isEmpty()); + assertNotNull(clangFiles.systemResourceDir()); + assertTrue(clangFiles.systemResourceDir().isDirectory()); + assertFalse(Files.exists(clangCacheRoot().resolve("includes"))); } @Test @@ -538,7 +541,7 @@ public void builtinCudaArchiveHasCanonicalInstallationLayout() { var parser = newParser(CodeParser.getBuiltinOption()); var cudaFolder = new ClangResources(parser).getBuiltinCudaLib(); - var cudaPlatform = CudaResources.getCurrentPlatform().manifestName(); + var cudaPlatform = cudaFolder.getParentFile().getName(); assertEquals(tempFolder.resolve("cuda").resolve(ClangAstWebResource.getCudaReleaseTag()) .resolve(cudaPlatform).resolve("cudalib").toFile().getAbsolutePath(), cudaFolder.getAbsolutePath()); assertTrue(new File(cudaFolder, "include/cuda.h").isFile()); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java index c51f893c9..3a2576bbe 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/CudaResourcesTest.java @@ -52,7 +52,7 @@ public class CudaResourcesTest { - private static final String PLATFORM = CudaResources.LINUX_X86_64_PLATFORM; + private static final String PLATFORM = "linux-x86_64"; private static final String RELEASE = "12.3.2"; private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10); @@ -109,20 +109,39 @@ public void manifestSelectsRequiredComponentsAndValidatesMetadata() { } @Test - public void manifestPlatformMatchesNvidiaKeysAndRejectsUnavailablePlatforms() { - assertEquals(CudaResources.LINUX_X86_64_PLATFORM, - CudaResources.getManifestPlatform(SupportedPlatform.LINUX, "amd64")); - assertEquals(CudaResources.LINUX_PPC64LE_PLATFORM, - CudaResources.getManifestPlatform(SupportedPlatform.LINUX, "ppc64le")); - assertEquals(CudaResources.LINUX_SBSA_PLATFORM, - CudaResources.getManifestPlatform(SupportedPlatform.LINUX, "aarch64")); - assertEquals(CudaResources.WINDOWS_X86_64_PLATFORM, - CudaResources.getManifestPlatform(SupportedPlatform.WINDOWS, "x86_64")); + public void manifestAcceptsHostWhenAllRequiredComponentsExposeACompatiblePlatform() { + var manifest = CudaResources.parseManifest(manifestJson()); - assertThrows(RuntimeException.class, - () -> CudaResources.getManifestPlatform(SupportedPlatform.MAC_OS, "aarch64")); - assertThrows(RuntimeException.class, - () -> CudaResources.getManifestPlatform(SupportedPlatform.WINDOWS, "aarch64")); + assertEquals(PLATFORM, + CudaResources.getManifestPlatform(manifest, SupportedPlatform.LINUX, "amd64")); + } + + @Test + public void manifestRejectsHostWhenARequiredComponentLacksThePlatform() { + var manifest = CudaResources.parseManifest(manifestJson()); + var missingPlatformComponent = manifest.components().get("cuda_cccl"); + var archives = new LinkedHashMap<>(missingPlatformComponent.archives()); + archives.remove(PLATFORM); + var components = new LinkedHashMap<>(manifest.components()); + components.put("cuda_cccl", new CudaResources.NvidiaCudaComponent( + missingPlatformComponent.name(), missingPlatformComponent.version(), archives)); + var incompleteManifest = new CudaResources.NvidiaCudaManifest(manifest.releaseDate(), manifest.releaseLabel(), + manifest.releaseProduct(), components); + + var error = assertThrows(RuntimeException.class, + () -> CudaResources.getManifestPlatform(incompleteManifest, SupportedPlatform.LINUX, "amd64")); + assertTrue(error.getMessage().contains("linux (amd64)")); + assertTrue(error.getMessage().contains(PLATFORM)); + assertTrue(error.getMessage().contains("cuda_cccl")); + } + + @Test + public void additionalCompatibleManifestPlatformNeedsNoJavaSupportWhitelist() { + var additionalPlatform = "linux-riscv64"; + var manifest = CudaResources.parseManifest(manifestJson(additionalPlatform)); + + assertEquals(additionalPlatform, + CudaResources.getManifestPlatform(manifest, SupportedPlatform.LINUX, "riscv64")); } @Test @@ -441,8 +460,7 @@ private CudaResources.NvidiaCudaManifest addUnusedPlatforms(CudaResources.Nvidia for (var entry : manifest.components().entrySet()) { var selectedArchive = entry.getValue().archives().get(PLATFORM); var archives = new LinkedHashMap<>(entry.getValue().archives()); - for (var unusedPlatform : List.of(CudaResources.LINUX_PPC64LE_PLATFORM, - CudaResources.LINUX_SBSA_PLATFORM, CudaResources.WINDOWS_X86_64_PLATFORM)) { + for (var unusedPlatform : List.of("linux-riscv64", "windows-x86_64")) { archives.put(unusedPlatform, new CudaResources.CudaArchive( entry.getKey() + "/" + unusedPlatform + "/unused.tar.xz", selectedArchive.sha256(), selectedArchive.size())); @@ -499,7 +517,7 @@ public String getFilename() { } private static String hostPlatform() { - return CudaResources.getCurrentPlatform().manifestName(); + return PLATFORM; } private static void writeTarXz(Path archive, Map files) throws IOException { @@ -530,6 +548,10 @@ private static void writeZip(Path archive, Map files) throws IOE } private static String manifestJson() { + return manifestJson(PLATFORM); + } + + private static String manifestJson(String platform) { return """ { "release_date": "2024-01-02", @@ -541,24 +563,24 @@ private static String manifestJson() { "cuda_cccl": %s } """.formatted( - component("CUDA Runtime", "cuda_cudart/linux-x86_64/cuda_cudart.tar.xz"), - component("CUDA NVCC", "cuda_nvcc/linux-x86_64/cuda_nvcc.tar.xz"), - component("cuRAND", "libcurand/linux-x86_64/libcurand.tar.xz"), - component("CCCL", "cuda_cccl/linux-x86_64/cuda_cccl.tar.xz")); + component("CUDA Runtime", platform, "cuda_cudart/" + platform + "/cuda_cudart.tar.xz"), + component("CUDA NVCC", platform, "cuda_nvcc/" + platform + "/cuda_nvcc.tar.xz"), + component("cuRAND", platform, "libcurand/" + platform + "/libcurand.tar.xz"), + component("CCCL", platform, "cuda_cccl/" + platform + "/cuda_cccl.tar.xz")); } - private static String component(String name, String relativePath) { + private static String component(String name, String platform, String relativePath) { return """ { "name": "%s", "version": "12.3.101", - "linux-x86_64": { + "%s": { "relative_path": "%s", "sha256": "%s", "size": "1" } } - """.formatted(name, relativePath, SHA256); + """.formatted(name, platform, relativePath, SHA256); } private static byte[] bytes(String value) { diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java index a0e13c028..7d45f6770 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java @@ -22,8 +22,10 @@ /** Verifies built-in CUDA parsing through the pinned NVIDIA redistribution packages. */ public class CxxCudaTest { @Test - public void testAtomicAdd() { - new CxxCudaTester("atomicAdd.cu").test(); + public void testAtomicAddWithBuiltinLibc() { + new CxxCudaTester("atomicAdd.cu") + .set(ClangAstKeys.LIBC_CXX_MODE, LibcMode.BUILTIN_AND_LIBC) + .test(); } @Test