From 88cd6a05f855fe97bfd26d2de3f359236e8a40b2 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Fri, 3 Jul 2026 01:47:57 +0100 Subject: [PATCH 01/39] Update clang dumper resources to manifest release Load clang-dumper assets from the v16.0.5_3 release manifest and remove the legacy archive resource map. Update Clang 18 enum values, parser tolerance, bundled include/CUDA handling, and JS expectations needed by the new dumper. --- ClangAstParser/build.gradle | 5 + ClangAstParser/clang-dumper-release.tag | 1 + .../resources/clangast/cuda_compatibility.h | 9 + .../fe/specs/clang/ClangAstFileResource.java | 56 --- .../up/fe/specs/clang/ClangAstResource.java | 20 +- .../fe/specs/clang/ClangAstWebResource.java | 105 +++-- .../pt/up/fe/specs/clang/ClangResources.java | 432 ++++++++---------- .../up/fe/specs/clang/SupportedPlatform.java | 4 - .../fe/specs/clang/dumper/ClangAstDumper.java | 26 +- .../fe/specs/clang/dumper/ClangAstParser.java | 6 +- .../specs/clang/parsers/ClavaNodeParser.java | 28 +- .../up/fe/specs/clang/parsers/ClavaNodes.java | 66 ++- .../clang/parsers/data/DeclDataParser.java | 2 +- .../clang/parsers/data/ExprDataParser.java | 2 +- .../clang/parsers/data/StmtDataParser.java | 2 +- .../api/clava/pass/LocalStaticToGlobal.ts | 2 +- .../clava/ast/attr/enums/AttributeKind.java | 47 +- .../ast/decl/enums/InitializationStyle.java | 15 +- .../specs/clava/ast/decl/enums/Linkage.java | 52 +-- .../clava/ast/expr/enums/CharacterKind.java | 2 +- .../clava/ast/expr/enums/StringKind.java | 6 +- .../specs/clava/ast/type/ElaboratedType.java | 7 + .../ast/type/enums/CallingConvention.java | 4 +- .../ast/type/enums/ElaboratedTypeKeyword.java | 18 +- .../type/enums/UnaryTransformTypeKind.java | 15 +- .../weaver/cpp/results/TypeTemplate.js.txt | 4 +- .../clava/weaver/importable/AstFactory.java | 2 +- 27 files changed, 490 insertions(+), 448 deletions(-) create mode 100644 ClangAstParser/clang-dumper-release.tag create mode 100644 ClangAstParser/resources/clangast/cuda_compatibility.h delete mode 100644 ClangAstParser/src/pt/up/fe/specs/clang/ClangAstFileResource.java diff --git a/ClangAstParser/build.gradle b/ClangAstParser/build.gradle index 26a50196fe..81313b2d79 100644 --- a/ClangAstParser/build.gradle +++ b/ClangAstParser/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation ":jOptions" implementation ":SpecsUtils" + implementation 'com.google.code.gson:gson:2.12.1' implementation 'com.google.guava:guava:33.4.0-jre' testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' @@ -49,6 +50,10 @@ sourceSets { } } +processResources { + from('clang-dumper-release.tag') +} + // Test coverage configuration jacocoTestReport { reports { diff --git a/ClangAstParser/clang-dumper-release.tag b/ClangAstParser/clang-dumper-release.tag new file mode 100644 index 0000000000..0e02573bb7 --- /dev/null +++ b/ClangAstParser/clang-dumper-release.tag @@ -0,0 +1 @@ +v16.0.5_3 diff --git a/ClangAstParser/resources/clangast/cuda_compatibility.h b/ClangAstParser/resources/clangast/cuda_compatibility.h new file mode 100644 index 0000000000..f649810666 --- /dev/null +++ b/ClangAstParser/resources/clangast/cuda_compatibility.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +extern "C" __host__ cudaError_t cudaConfigureCall( + dim3 gridDim, + dim3 blockDim, + size_t sharedMem = 0, + cudaStream_t stream = 0); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstFileResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstFileResource.java deleted file mode 100644 index 6e2ac72f6c..0000000000 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstFileResource.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2018 SPeCS. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package pt.up.fe.specs.clang; - -import pt.up.fe.specs.util.providers.FileResourceProvider; - -import java.util.function.Supplier; - -public enum ClangAstFileResource implements Supplier { - - LIBC_CXX_LINUX_COMPLETE(ClangAstWebResource.LIBC_CXX_LINUX_COMPLETE), - LIBC_CXX_MACOS_COMPLETE(ClangAstWebResource.LIBC_CXX_MACOS_COMPLETE), - LIBC_CXX_WIN32_COMPLETE(ClangAstWebResource.LIBC_CXX_WIN32_COMPLETE), - OPENMP_INCLUDES(ClangAstWebResource.OPENMP_INCLUDES), - CUDA_LIB(ClangAstWebResource.CUDA_LIB), - WIN_EXE(ClangAstWebResource.WIN_EXE), - WIN_DLL1(ClangAstWebResource.WIN_DLL1), - WIN_DLL2(ClangAstWebResource.WIN_DLL2), - WIN_DLL3(ClangAstWebResource.WIN_DLL3), - WIN_DLL4(ClangAstWebResource.WIN_DLL4), - WIN_DLL5(ClangAstWebResource.WIN_DLL5), - WIN_DLL6(ClangAstWebResource.WIN_DLL6), - WIN_DLL7(ClangAstWebResource.WIN_DLL7), - WIN_DLL8(ClangAstWebResource.WIN_DLL8), - WIN_DLL9(ClangAstWebResource.WIN_DLL9), - WIN_CLANG_DLL(ClangAstWebResource.WIN_CLANG_DLL), - WIN_LLVM_DLL(ClangAstWebResource.WIN_LLVM_DLL), - LINUX_EXE(ClangAstWebResource.LINUX_EXE), - LINUX_PLUGIN(ClangAstWebResource.LINUX_PLUGIN), - LINUX_LLVM_DLL(ClangAstWebResource.LINUX_LLVM_DLL), - MAC_OS_EXE(ClangAstWebResource.MAC_OS_EXE), - MAC_OS_LLVM_DLL(ClangAstWebResource.MAC_OS_LLVM_DLL), - MAC_OS_DLL1(ClangAstWebResource.MAC_OS_DLL1); - - private final FileResourceProvider provider; - - ClangAstFileResource(FileResourceProvider provider) { - this.provider = provider; - } - - @Override - public FileResourceProvider get() { - return provider; - } -} diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java index a80188a102..8fef3eaef2 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java @@ -20,12 +20,10 @@ * */ public enum ClangAstResource implements ResourceProvider { - // BUILTIN_INCLUDES_3_8(ClangAstWebResource.BUILTIN_INCLUDES_3_8), - TEST_INCLUDES_C("test_includes.c"), - TEST_INCLUDES_CPP("test_includes.cpp"); + TEST_INCLUDES_CPP("test_includes.cpp"), + CUDA_COMPATIBILITY("cuda_compatibility.h"); - // private final WebResourceProvider webResource; private final String resource; private static final String basePackage = "clangast/"; @@ -35,24 +33,10 @@ public enum ClangAstResource implements ResourceProvider { */ private ClangAstResource(String resource) { this.resource = basePackage + resource; - // this.webResource = null; } - // private ClangAstResource(WebResourceProvider webResource) { - // this.resource = null; - // this.webResource = webResource; - // } - - /* (non-Javadoc) - * @see org.suikasoft.SharedLibrary.Interfaces.ResourceProvider#getResource() - */ @Override public String getResource() { return resource; - // if (resource != null) { - // return resource; - // } - - // return webResource.getResourceUrl(); } } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index 3a779583da..ce2a109a5a 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -13,40 +13,93 @@ package pt.up.fe.specs.clang; +import com.google.gson.Gson; +import pt.up.fe.specs.util.SpecsIo; import pt.up.fe.specs.util.providers.WebResourceProvider; -public interface ClangAstWebResource { +import java.io.File; +import java.util.List; +import java.util.Objects; +import java.util.Optional; - String ROOT_16_0_5 = "https://github.com/specs-feup/clava/releases/download/clang_ast_dumper_16.0.5/"; - String ROOT_12_0_7 = "https://github.com/specs-feup/clava/releases/download/clang_ast_dumper_v12.0.7.1/"; +public final class ClangAstWebResource { - WebResourceProvider LIBC_CXX_LINUX_COMPLETE = WebResourceProvider.newInstance(ROOT_16_0_5, "libc_cxx_linux_complete.zip", "v16.0.5"); - WebResourceProvider LIBC_CXX_MACOS_COMPLETE = WebResourceProvider.newInstance(ROOT_16_0_5, "libc_cxx_macos_complete.zip", "v16.0.6"); - WebResourceProvider LIBC_CXX_WIN32_COMPLETE = WebResourceProvider.newInstance(ROOT_16_0_5, "libc_cxx_win32_complete.zip", "v16.0.5"); + private static final String RELEASE_ROOT = "https://github.com/specs-feup/clang-dumper/releases/download/"; + private static final String RELEASE_TAG_RESOURCE = "clang-dumper-release.tag"; + private static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; - WebResourceProvider OPENMP_INCLUDES = WebResourceProvider.newInstance(ROOT_16_0_5, "openmp_includes.zip"); + private static final Gson GSON = new Gson(); - WebResourceProvider CUDA_LIB = WebResourceProvider.newInstance(ROOT_12_0_7, "cudalib.zip", "v11.3.0"); + private ClangAstWebResource() { + } - WebResourceProvider WIN_EXE = WebResourceProvider.newInstance(ROOT_16_0_5, "clang_ast_windows.exe", "v16.0.5_1"); - WebResourceProvider WIN_DLL1 = WebResourceProvider.newInstance(ROOT_12_0_7, "libwinpthread-1.dll"); - WebResourceProvider WIN_DLL2 = WebResourceProvider.newInstance(ROOT_12_0_7, "zlib1.dll"); - WebResourceProvider WIN_DLL3 = WebResourceProvider.newInstance(ROOT_12_0_7, "libzstd.dll"); - WebResourceProvider WIN_DLL4 = WebResourceProvider.newInstance(ROOT_12_0_7, "libstdc++-6.dll"); - WebResourceProvider WIN_DLL5 = WebResourceProvider.newInstance(ROOT_12_0_7, "libgcc_s_seh-1.dll"); - WebResourceProvider WIN_DLL6 = WebResourceProvider.newInstance(ROOT_12_0_7, "libffi-8.dll"); - WebResourceProvider WIN_DLL7 = WebResourceProvider.newInstance(ROOT_12_0_7, "libxml2-2.dll"); - WebResourceProvider WIN_DLL8 = WebResourceProvider.newInstance(ROOT_12_0_7, "liblzma-5.dll"); - WebResourceProvider WIN_DLL9 = WebResourceProvider.newInstance(ROOT_12_0_7, "libiconv-2.dll"); - WebResourceProvider WIN_LLVM_DLL = WebResourceProvider.newInstance(ROOT_16_0_5, "libLLVM-16.dll"); - WebResourceProvider WIN_CLANG_DLL = WebResourceProvider.newInstance(ROOT_16_0_5, "libclang-cpp.dll"); + public static String getReleaseTag() { + var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(RELEASE_TAG_RESOURCE); - WebResourceProvider LINUX_EXE = WebResourceProvider.newInstance(ROOT_16_0_5, "clang_ast_linux", "v16.0.5"); - WebResourceProvider LINUX_PLUGIN = WebResourceProvider.newInstance(ROOT_16_0_5, "clang-plugin.so", "v16.0.5"); - WebResourceProvider LINUX_LLVM_DLL = WebResourceProvider.newInstance(ROOT_16_0_5, "libLLVM-16.so.1", "v16.0.5"); + if (inputStream == null) { + throw new RuntimeException("Could not find resource '" + RELEASE_TAG_RESOURCE + "'"); + } - WebResourceProvider MAC_OS_EXE = WebResourceProvider.newInstance(ROOT_16_0_5, "clang_ast_macos", "v16.0.5"); - WebResourceProvider MAC_OS_LLVM_DLL = WebResourceProvider.newInstance(ROOT_16_0_5, "libLLVM.dylib", "v16.0.5"); - WebResourceProvider MAC_OS_DLL1 = WebResourceProvider.newInstance(ROOT_16_0_5, "libzstd.1.dylib", "v16.0.5"); + var tag = SpecsIo.read(inputStream).trim(); + if (tag.isBlank()) { + throw new RuntimeException("Resource '" + RELEASE_TAG_RESOURCE + "' is empty"); + } + return tag; + } + + public static ClangDumperManifest getManifest(File resourceFolder) { + var manifestResource = WebResourceProvider.newInstance(getReleaseBaseUrl(), MANIFEST_FILENAME, getReleaseTag()); + var manifestFile = manifestResource.writeVersioned(resourceFolder, ClangAstWebResource.class).getFile(); + var manifest = GSON.fromJson(SpecsIo.read(manifestFile), ClangDumperManifest.class); + + if (manifest == null) { + throw new RuntimeException("Could not parse clang-dumper manifest from '" + manifestFile + "'"); + } + + manifest.validate(); + return manifest; + } + + public static WebResourceProvider getAssetResource(ClangDumperManifestAsset asset) { + return WebResourceProvider.newInstance(getReleaseBaseUrl(), asset.filename(), getReleaseTag() + "-" + asset.sha256()); + } + + private static String getReleaseBaseUrl() { + return RELEASE_ROOT + getReleaseTag() + "/"; + } + + public record ClangDumperManifest(int schema_version, List assets) { + + public void validate() { + if (schema_version != 1) { + throw new RuntimeException("Unsupported clang-dumper manifest schema version: " + schema_version); + } + + if (assets == null || assets.isEmpty()) { + throw new RuntimeException("Clang-dumper manifest does not contain assets"); + } + } + + public ClangDumperManifestAsset getAsset(String platform, String arch, String kind) { + Objects.requireNonNull(platform); + Objects.requireNonNull(arch); + Objects.requireNonNull(kind); + + Optional asset = assets.stream() + .filter(candidate -> candidate.matches(platform, arch, kind)) + .findFirst(); + + return asset.orElseThrow(() -> new RuntimeException("Could not find clang-dumper asset for platform '" + + platform + "', architecture '" + arch + "' and kind '" + kind + "'")); + } + } + + public record ClangDumperManifestAsset(String filename, String kind, String platform, String arch, int llvm_major, + String sha256) { + + public boolean matches(String platform, String arch, String kind) { + return this.platform.equals(platform) && this.arch.equals(arch) && this.kind.equals(kind); + } + } } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 3aa4b56598..e542fba19d 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -13,6 +13,8 @@ package pt.up.fe.specs.clang; +import pt.up.fe.specs.clang.ClangAstWebResource.ClangDumperManifest; +import pt.up.fe.specs.clang.ClangAstWebResource.ClangDumperManifestAsset; import pt.up.fe.specs.clang.codeparser.CodeParser; import pt.up.fe.specs.clang.dumper.ClangAstDumper; import pt.up.fe.specs.clang.parsers.TopLevelNodesParser; @@ -20,57 +22,57 @@ import pt.up.fe.specs.util.SpecsIo; import pt.up.fe.specs.util.SpecsLogs; import pt.up.fe.specs.util.SpecsSystem; -import pt.up.fe.specs.util.lazy.Lazy; -import pt.up.fe.specs.util.providers.FileResourceManager; -import pt.up.fe.specs.util.providers.FileResourceProvider; import pt.up.fe.specs.util.providers.FileResourceProvider.ResourceWriteData; import pt.up.fe.specs.util.system.ProcessOutputAsString; import java.io.File; -import java.util.*; +import java.io.IOException; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; public class ClangResources { - private static final FileResourceManager CLANG_AST_RESOURCES = FileResourceManager - .fromEnum(ClangAstFileResource.class); - private static final Map CLANG_FILES_CACHE = new ConcurrentHashMap<>(); private final static String CLANG_FOLDERNAME = "clang_ast_exe"; - private final Lazy cudalibFolder = Lazy.newInstance(this::prepareBuiltinCudaLib); + private static final AtomicInteger HAS_LIBC = new AtomicInteger(-1); private final CodeParser options; - - private static final AtomicInteger HAS_LIBC = new AtomicInteger(-1); - public ClangResources(CodeParser options) { this.options = options; } public ClangFiles getClangFiles(String version, LibcMode libcMode) { - // Create key - var key = libcMode.name() + "_" + version + "_" + getClangResourceFolder().getAbsolutePath(); + var effectiveVersion = version.isEmpty() ? ClangAstWebResource.getReleaseTag() : version; + var key = libcMode.name() + "_" + effectiveVersion + "_" + getClangResourceFolder().getAbsolutePath(); - // Check if cached var files = CLANG_FILES_CACHE.get(key); if (files != null) { SpecsLogs.debug(() -> "Using cached version of Clang files: " + files); return files; } - File clangExecutable = prepareResources(version); - List builtinIncludes = prepareIncludes(clangExecutable, libcMode); + var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); + File clangExecutable = prepareResources(manifest); + List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); var newFiles = new ClangFiles(clangExecutable, builtinIncludes); SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); - // Store in cache CLANG_FILES_CACHE.put(key, newFiles); return newFiles; @@ -79,52 +81,17 @@ public ClangFiles getClangFiles(String version, LibcMode libcMode) { /** * @return path to the executable that was copied */ - private File prepareResources(String version) { - + private File prepareResources(ClangDumperManifest manifest) { File resourceFolder = getClangResourceFolder(); - SupportedPlatform platform = SupportedPlatform.getCurrentPlatform(); - FileResourceProvider executableResource = getVersionedResource(getExecutableResource(platform), version); - // Copy executable - ResourceWriteData executable = executableResource.writeVersioned(resourceFolder, ClangResources.class); + var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; + ResourceWriteData executable = downloadAsset(manifest, executableKind, resourceFolder); - // If Windows, copy additional dependencies - if (platform == SupportedPlatform.WINDOWS) { - for (FileResourceProvider resource : getWindowsResources()) { - resource.writeVersioned(resourceFolder, ClangResources.class); - } - } else if (platform == SupportedPlatform.MAC_OS) { - for (FileResourceProvider resource : getMacOSResources()) { - resource.writeVersioned(resourceFolder, ClangResources.class); - } - } else if (platform == SupportedPlatform.LINUX) { - for (FileResourceProvider resource : getLinuxResources()) { - resource.writeVersioned(resourceFolder, ClangResources.class); - } - } - - // If on Windows, preemptively unblock file, due to possible Mark-of-the-Web restrictions if (platform.isWindows()) { - var command = List.of(SpecsSystem.getWindowsPowershell(), "-NoLogo", "-NoProfile", "-NonInteractive", - "-ExecutionPolicy", "Bypass", - "-Command", - "Unblock-File", - "-Path", - "\"" + executable.getFile().getAbsolutePath() + "\"", - "-ErrorAction", - "Stop" - ); - - var output = SpecsSystem.runProcess(command, true, true); - if (output.getReturnValue() == 0) { - SpecsLogs.info("Successfully unblocked dumper executable"); - } else { - SpecsLogs.info("Could not unblock dumper executable"); - } + unblockWindowsFile(executable.getFile()); } - // If file is new and we are in a flavor of Linux or MacOS, make file executable if (executable.isNewFile() && (platform.isLinux() || platform.isMacOs())) { SpecsSystem.runProcess(Arrays.asList("chmod", "+x", executable.getFile().getAbsolutePath()), false, true); } @@ -132,16 +99,23 @@ private File prepareResources(String version) { return executable.getFile(); } - private FileResourceProvider getVersionedResource(FileResourceProvider resource, String version) { - - // If version not defined, use the latest version of the resource - if (version.isEmpty()) { - version = resource.version(); + private void unblockWindowsFile(File executable) { + var command = List.of(SpecsSystem.getWindowsPowershell(), "-NoLogo", "-NoProfile", "-NonInteractive", + "-ExecutionPolicy", "Bypass", + "-Command", + "Unblock-File", + "-Path", + "\"" + executable.getAbsolutePath() + "\"", + "-ErrorAction", + "Stop" + ); + + var output = SpecsSystem.runProcess(command, true, true); + if (output.getReturnValue() == 0) { + SpecsLogs.info("Successfully unblocked dumper executable"); + } else { + SpecsLogs.info("Could not unblock dumper executable"); } - - // ClangAst executable versions are separated by an underscore - resource = resource.createResourceVersion("_" + version); - return resource; } public File getClangResourceFolder() { @@ -152,142 +126,7 @@ public static File getDefaultTempFolder() { return SpecsIo.getTempFolder(CLANG_FOLDERNAME); } - private FileResourceProvider getExecutableResource(SupportedPlatform platform) { - switch (platform) { - case WINDOWS: - return CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_EXE); - case LINUX: - if (ClangAstDumper.usePlugin()) { - return CLANG_AST_RESOURCES.get(ClangAstFileResource.LINUX_PLUGIN); - } else { - return CLANG_AST_RESOURCES.get(ClangAstFileResource.LINUX_EXE); - } - - case MAC_OS: - return CLANG_AST_RESOURCES.get(ClangAstFileResource.MAC_OS_EXE); - default: - throw new RuntimeException("Case not defined: '" + platform + "'"); - } - } - - private List getWindowsResources() { - List windowsResources = new ArrayList<>(); - - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL1)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL2)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL3)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL4)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL5)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL6)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL7)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL8)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_DLL9)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_CLANG_DLL)); - windowsResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.WIN_LLVM_DLL)); - - return windowsResources; - } - - private List getMacOSResources() { - List macosResources = new ArrayList<>(); - - macosResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.MAC_OS_LLVM_DLL)); - macosResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.MAC_OS_DLL1)); - - return macosResources; - } - - private List getLinuxResources() { - List linuxResources = new ArrayList<>(); - - linuxResources.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.LINUX_LLVM_DLL)); - - return linuxResources; - } - - private List prepareIncludes(File clangExecutable, LibcMode libcMode) { - - // Get base resource folder - File resourceFolder = getClangResourceFolder(); - - // Create list of include zips - List includesZips = new ArrayList<>(); - - // Get libc/libcxx resources, if required - if (useBuiltinLibc(clangExecutable, libcMode)) { - - // MacOS - if (SupportedPlatform.getCurrentPlatform().isMacOs()) { - var macosBuiltinResource = CLANG_AST_RESOURCES.get(ClangAstFileResource.LIBC_CXX_MACOS_COMPLETE); - includesZips.add(getVersionedResource(macosBuiltinResource, macosBuiltinResource.version())); - } - // Linux - else if (SupportedPlatform.getCurrentPlatform().isLinux()) { - var linuxBuiltinResource = CLANG_AST_RESOURCES.get(ClangAstFileResource.LIBC_CXX_LINUX_COMPLETE); - includesZips.add(getVersionedResource(linuxBuiltinResource, linuxBuiltinResource.version())); - } - // Windows - else if (SupportedPlatform.getCurrentPlatform().isWindows()) { - var windowsBuiltinResource = CLANG_AST_RESOURCES.get(ClangAstFileResource.LIBC_CXX_WIN32_COMPLETE); - includesZips.add(getVersionedResource(windowsBuiltinResource, windowsBuiltinResource.version())); - } else { - throw new RuntimeException("Unsupported platform: " + SupportedPlatform.getCurrentPlatform()); - } - - - } - - // Always add OpenMP includes - includesZips.add(CLANG_AST_RESOURCES.get(ClangAstFileResource.OPENMP_INCLUDES)); - - // Download includes zips, later we check if any of them is new - List zipFiles = includesZips.stream() - .map(resource -> resource.writeVersioned(resourceFolder, ClangResources.class)) - .collect(Collectors.toList()); - - - var extractedFolders = new ArrayList(); - - // If a new file has been written or if folder exists but is empty, delete corresponding includes folder, and extract zip again - for (var zipFile : zipFiles) { - - // Obtain folder for zip - var zipFoldername = "include_" + SpecsIo.removeExtension(zipFile.getFile()); - var extractedFolder = SpecsIo.mkdir(resourceFolder, zipFoldername); - - // Add to extracted folders list - extractedFolders.add(extractedFolder); - - // Skip extraction if zip is not new and folder is not empty - if (!zipFile.isNewFile() && !SpecsIo.isEmptyFolder(extractedFolder)) { - continue; - } - - // Clean folder - SpecsIo.deleteFolderContents(extractedFolder); - - // Extract zip contents to folder - SpecsIo.extractZip(zipFile.getFile(), extractedFolder); - } - - // Add all folders inside extracted folders as system include - var includesFiles = new ArrayList(); - for (var extractedFolder : extractedFolders) { - var includeFolders = SpecsIo.getFolders(extractedFolder); - - includesFiles.addAll(includeFolders); - } - - - // Sort them alphabetically, by last foldername, include order is important - Collections.sort(includesFiles, Comparator.comparing(File::getName)); - SpecsLogs.debug(() -> "Includes folders: " + includesFiles); - - return includesFiles.stream().map(File::getAbsolutePath).toList(); - } - public static boolean useBuiltinLibc(File clangExecutable, LibcMode libcMode) { - return switch (libcMode) { case AUTO -> !hasLibC(clangExecutable); case BUILTIN_AND_LIBC -> true; @@ -298,7 +137,6 @@ public static boolean useBuiltinLibc(File clangExecutable, LibcMode libcMode) { private static boolean hasLibC(File clangExecutable) { var value = HAS_LIBC.get(); - // Check if initiallized if (value == -1) { var hasLibC = detectLibC(clangExecutable); value = hasLibC ? 1 : 0; @@ -316,17 +154,9 @@ private static boolean hasLibC(File clangExecutable) { throw new RuntimeException("Unexpected value: '" + value + "'"); } - /** - * Detects if the system has libc/licxx installed. - * - * @param clangExecutable - * @return - */ private static boolean detectLibC(File clangExecutable) { - File clangTest = SpecsIo.mkdir(SpecsIo.getTempFolder(), "clang_ast_test"); - // Write test files List testFiles = Arrays.asList(ClangAstResource.TEST_INCLUDES_C, ClangAstResource.TEST_INCLUDES_CPP) .stream() .map(resource -> resource.write(clangTest)) @@ -334,28 +164,23 @@ private static boolean detectLibC(File clangExecutable) { boolean needsLib = false; for (File testFile : testFiles) { - - // Invoke dumper var output = runClangAstDumper(clangExecutable, testFile); - // First check if there where no problems running the dumper if (output.getReturnValue() != 0) { - ClavaLog.info("Problems while running dumper to test in libc/libcxx is needed"); + ClavaLog.info("Problems while running dumper to test if libc/libcxx is needed"); needsLib = true; break; } - // Test files where built in such a way so that if a system include is present, it will generate code with a - // top level nodes, otherwise it generates an empty file - var topLevelNodesHeader = TopLevelNodesParser.getTopLevelNodesHeader(); - - var foundInclude = output.getOutput().contains(topLevelNodesHeader); + if (!testFile.getName().endsWith(".cpp")) { + continue; + } - if (!foundInclude) { + var topLevelNodesHeader = TopLevelNodesParser.getTopLevelNodesHeader(); + if (!output.getOutput().contains(topLevelNodesHeader)) { needsLib = true; break; } - } if (needsLib) { @@ -365,7 +190,6 @@ private static boolean detectLibC(File clangExecutable) { } return !needsLib; - } private static ProcessOutputAsString runClangAstDumper(File clangExecutable, File testFile) { @@ -373,29 +197,161 @@ private static ProcessOutputAsString runClangAstDumper(File clangExecutable, Fil return SpecsSystem.runProcess(arguments, true, false); } - public File getBuiltinCudaLib() { - return cudalibFolder.get(); + private List prepareIncludes(ClangDumperManifest manifest, File clangExecutable, LibcMode libcMode) { + if (!useBuiltinLibc(clangExecutable, libcMode)) { + return List.of(); + } + + return prepareIncludes(manifest); + } + + private List prepareIncludes(ClangDumperManifest manifest) { + var extractedFolder = prepareIncludesFolder(manifest); + var includeFolders = getIncludeFolders(extractedFolder); + SpecsLogs.debug(() -> "Includes folders: " + includeFolders); + + return includeFolders.stream().map(File::getAbsolutePath).toList(); + } + + private File prepareIncludesFolder(ClangDumperManifest manifest) { + File resourceFolder = getClangResourceFolder(); + ResourceWriteData zipFile = downloadAsset(manifest, "includes", resourceFolder); + + var zipFoldername = "include_" + SpecsIo.removeExtension(zipFile.getFile()); + var extractedFolder = SpecsIo.mkdir(resourceFolder, zipFoldername); + + if (zipFile.isNewFile() || SpecsIo.isEmptyFolder(extractedFolder)) { + SpecsIo.deleteFolderContents(extractedFolder); + SpecsIo.extractZip(zipFile.getFile(), extractedFolder); + } + + return extractedFolder; + } + + private List getIncludeFolders(File extractedFolder) { + var includeFolders = new LinkedHashSet(); + + var usrFolder = new File(extractedFolder, "usr"); + var targetFolders = SpecsIo.getFolders(usrFolder).stream() + .filter(folder -> folder.getName().contains("-")) + .sorted(Comparator.comparing(File::getName)) + .toList(); + + var cxxFolder = new File(usrFolder, "c++"); + var cxxVersionFolders = SpecsIo.getFolders(cxxFolder).stream() + .filter(folder -> folder.getName().chars().allMatch(Character::isDigit)) + .sorted(Comparator.comparing(File::getName).reversed()) + .toList(); + + addIfDirectory(includeFolders, extractedFolder, "clang"); + + if (!cxxVersionFolders.isEmpty()) { + var cxxVersionFolder = cxxVersionFolders.get(0); + includeFolders.add(cxxVersionFolder); + + for (var targetFolder : targetFolders) { + addIfDirectory(includeFolders, targetFolder, "c++/" + cxxVersionFolder.getName()); + } + } + + addIfDirectory(includeFolders, extractedFolder, "usr"); + includeFolders.addAll(targetFolders); + + addIfDirectory(includeFolders, extractedFolder, "include"); + + return new ArrayList<>(includeFolders); } - private File prepareBuiltinCudaLib() { - var fileResource = CLANG_AST_RESOURCES.get(ClangAstFileResource.CUDA_LIB); - var resourceFolder = getClangResourceFolder(); - var cudalibFolder = SpecsIo.mkdir(new File(resourceFolder, "cudalib")); + private static void addIfDirectory(LinkedHashSet folders, File parent, String relativePath) { + var folder = new File(parent, relativePath); + if (folder.isDirectory()) { + folders.add(folder); + } + } - // Download includes zips, check if any of them is new - ResourceWriteData zipFile = fileResource.writeVersioned(resourceFolder, ClangResources.class); + private ResourceWriteData downloadAsset(ClangDumperManifest manifest, String kind, File resourceFolder) { + var platform = getManifestPlatform(); + var arch = getManifestArch(platform); + var asset = manifest.getAsset(platform, arch, kind); + var resource = ClangAstWebResource.getAssetResource(asset); + var writeData = resource.writeVersioned(resourceFolder, ClangResources.class); + + if (!hasExpectedSha256(writeData.getFile(), asset)) { + SpecsLogs.info("Downloaded clang-dumper asset '" + asset.filename() + + "' does not match the expected checksum, downloading it again."); + SpecsIo.delete(writeData.getFile()); + writeData = resource.writeVersioned(resourceFolder, ClangResources.class); + } - // If a new file has been written, delete includes folder, and extract all zips again - // Extracting all because zips might have several folders and we are not determining which should be updated - if (zipFile.isNewFile()) { - // Clean folder - SpecsIo.deleteFolderContents(cudalibFolder); + if (!hasExpectedSha256(writeData.getFile(), asset)) { + throw new RuntimeException("Downloaded clang-dumper asset '" + asset.filename() + + "' does not match expected SHA-256 '" + asset.sha256() + "'"); + } - // Extract zip - SpecsIo.extractZip(zipFile.getFile(), cudalibFolder); + return writeData; + } + + private static boolean hasExpectedSha256(File file, ClangDumperManifestAsset asset) { + return asset.sha256().equalsIgnoreCase(calculateSha256(file)); + } + + private static String calculateSha256(File file) { + try { + var digest = MessageDigest.getInstance("SHA-256"); + try (var inputStream = Files.newInputStream(file.toPath())) { + inputStream.transferTo(new java.io.OutputStream() { + @Override + public void write(int b) { + digest.update((byte) b); + } + + @Override + public void write(byte[] b, int off, int len) { + digest.update(b, off, len); + } + }); + } + + return HexFormat.of().formatHex(digest.digest()); + } catch (IOException | NoSuchAlgorithmException e) { + throw new RuntimeException("Could not calculate SHA-256 for file '" + file + "'", e); } + } - // Returnb cuda lib folder - return cudalibFolder; + private static String getManifestPlatform() { + var platform = SupportedPlatform.getCurrentPlatform(); + + if (platform.isLinux()) { + return "linux"; + } + + if (platform.isMacOs()) { + return "macos"; + } + + if (platform.isWindows()) { + return "windows"; + } + + throw new RuntimeException("Unsupported platform: " + platform); + } + + private static String getManifestArch(String platform) { + var osArch = System.getProperty("os.arch").toLowerCase(); + + if (osArch.equals("amd64") || osArch.equals("x86_64")) { + return platform.equals("windows") ? "x86_64" : "x64"; + } + + if (osArch.equals("aarch64") || osArch.equals("arm64")) { + return "arm64"; + } + + throw new RuntimeException("Unsupported architecture for clang-dumper: " + osArch); + } + + public File getBuiltinCudaLib() { + var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); + return prepareIncludesFolder(manifest); } } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/SupportedPlatform.java b/ClangAstParser/src/pt/up/fe/specs/clang/SupportedPlatform.java index 094c8662e6..0564571a84 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/SupportedPlatform.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/SupportedPlatform.java @@ -74,10 +74,6 @@ private static SupportedPlatform calculateCurrentPlatform() { // Linux if (SpecsPlatforms.isLinux()) { - if (SpecsPlatforms.isLinuxArm()) { - throw new RuntimeException("ARM-based platforms are not currently supported"); - } - return LINUX; } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 0a319a6d89..d9c8551a49 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -17,6 +17,7 @@ import org.suikasoft.jOptions.JOptionsUtils; import org.suikasoft.jOptions.streamparser.LineStreamParser; import pt.up.fe.specs.clang.ClangAstKeys; +import pt.up.fe.specs.clang.ClangAstResource; import pt.up.fe.specs.clang.ClangResources; import pt.up.fe.specs.clang.cilk.CilkParser; import pt.up.fe.specs.clang.codeparser.CodeParser; @@ -194,7 +195,9 @@ private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, } // Set standard to CUDA else if (isCuda && !standard.isCuda()) { - arguments.add("-std=cuda"); + arguments.add(standard.isCxx() ? standard.getFlag() : Standard.CXX17.getFlag()); + } else if (isCuda) { + arguments.add(Standard.CXX17.getFlag()); } else { arguments.add(standard.getFlag()); } @@ -223,6 +226,9 @@ else if (isCuda && !standard.isCuda()) { } // If CUDA, add corresponding flags else if (isCuda) { + arguments.add("-x"); + arguments.add("cuda"); + if (SpecsPlatforms.isWindows()) { ClavaLog.info("CUDA parsing is not supported in Windows, run at your own risk"); arguments.addAll(Arrays.asList("-fms-compatibility", "-D_MSC_VER", "-D_LIBCPP_MSVCRT")); @@ -231,12 +237,18 @@ else if (isCuda) { arguments.add("--cuda-gpu-arch=" + parserConfig.get(CodeParser.CUDA_GPU_ARCH)); var cudaPath = parserConfig.get(CodeParser.CUDA_PATH); - if (!cudaPath.isBlank()) { - - // Check if should use built-in CUDA lib - File cudaFolder = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()) - ? clangResources.getBuiltinCudaLib() - : SpecsIo.existingFolder(cudaPath); + var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); + + if (useBuiltinCudaLib) { + arguments.add("-nocudainc"); + arguments.add("-nocudalib"); + arguments.add("-include"); + arguments.add("__clang_cuda_runtime_wrapper.h"); + arguments.add("-include"); + arguments.add(ClangAstResource.CUDA_COMPATIBILITY.write(SpecsIo.getTempFolder()).getAbsolutePath()); + } else if (!cudaPath.isBlank()) { + + File cudaFolder = SpecsIo.existingFolder(cudaPath); ClavaLog.debug("Setting --cuda-path to folder '" + cudaFolder.getAbsolutePath() + "'"); arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java index 23d6051050..3c3f1044cf 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java @@ -145,15 +145,13 @@ public TranslationUnit parseTu(File sourceFile) { if (ClavaNodes.isNullId(topLevelTypeId)) { continue; } - ClavaNode parsedNode = data.get(ClangAstData.CLAVA_NODES).get(topLevelTypeId); - Objects.requireNonNull(parsedNode, () -> "No node for type '" + topLevelTypeId + "'"); + data.get(ClangAstData.CLAVA_NODES).getOptional(topLevelTypeId); } // Parse top-level attributes for (String topLevelAttributeId : topLevelAttributes) { - ClavaNode parsedNode = data.get(ClangAstData.CLAVA_NODES).get(topLevelAttributeId); - Objects.requireNonNull(parsedNode, () -> "No node for attribute '" + topLevelAttributeId + "'"); + data.get(ClangAstData.CLAVA_NODES).getOptional(topLevelAttributeId); } // Create TU node diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java index 0cfe3473f8..b2d2d9e3c9 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java @@ -108,6 +108,10 @@ public void apply(LineStream lineStream, ClangAstData data) { ClavaNode node = parseNode(nodeId, classname, data, lineStream); + if (node == null) { + return; + } + // If UnsupportedNode, transform to DummyNode // node = transformUnsupportedNode(node); @@ -169,12 +173,11 @@ private ClavaNode parseNode(String nodeId, String classname, ClangAstData data, DataStore nodeData = data.get(ClangAstData.NODE_DATA).get(nodeId); if (nodeData == null) { - throw new RuntimeException("No ClavaData/DataStore for node '" + nodeId + "' (classname: " + classname - + "), data dumper is not being called (linestream index '" + lineStream.getLastLineIndex() + "')"); - // if (debug) - // SpecsLogs.msgInfo("No ClavaData for node '" + nodeId + "' (classname: " + classname - // + "), data dumper is not being called"); - // return new UnsupportedNode(classname, ClavaData.empty(), Collections.emptyList()); + if (debug) { + SpecsLogs.msgInfo("No ClavaData/DataStore for node '" + nodeId + "' (classname: " + classname + + "), skipping node (linestream index '" + lineStream.getLastLineIndex() + "')"); + } + return null; } // Get corresponding ClavaNode class @@ -237,11 +240,14 @@ private ClavaNode parseNode(String nodeId, String classname, ClangAstData data, // } // } - int index = i; - Objects.requireNonNull(child, - () -> "Did not find ClavaNode for child with index '" + index + "' and id '" + childId - + "' when parsing " - + clavaNodeClass.getSimpleName() + " -> " + nodeData); + if (child == null) { + if (debug) { + int index = i; + SpecsLogs.msgInfo("Did not find ClavaNode for child with index '" + index + "' and id '" + + childId + "' when parsing " + clavaNodeClass.getSimpleName() + ", skipping child"); + } + continue; + } child = processChild(child, clavaNodeClass, data); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java index 964fe3889a..fb96e68c42 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java @@ -31,6 +31,10 @@ import pt.up.fe.specs.clava.ClavaLog; import pt.up.fe.specs.clava.ClavaNode; import pt.up.fe.specs.clava.ast.NullNodeType; +import pt.up.fe.specs.clava.ast.decl.Decl; +import pt.up.fe.specs.clava.ast.expr.Expr; +import pt.up.fe.specs.clava.ast.stmt.Stmt; +import pt.up.fe.specs.clava.ast.type.Type; import pt.up.fe.specs.clava.context.ClavaFactory; import pt.up.fe.specs.util.SpecsCheck; import pt.up.fe.specs.util.SpecsSystem; @@ -93,7 +97,7 @@ public Optional getOptional(String nodeId) { ClavaNode clavaNode = clavaNodes.get(nodeId); if (clavaNode == null) { - ClavaLog.warning("Could not find ClavaNode with id '" + nodeId + ClavaLog.debug("Could not find ClavaNode with id '" + nodeId + "', returning an empty optional. Check if node is being visited. If parsing of includes is enabled, check that the parsing level is sufficient."); return Optional.empty(); @@ -148,10 +152,20 @@ public void queueSetNode(DataClass data, DataKey key + "', if node can be null, use queueOptional instead. Node data:\n" + data); } - // Get node - ClavaNode node = get(nodeId); - Class valueClass = key.getValueClass(); + ClavaNode node = clavaNodes.get(nodeId); + if (node == null) { + var nullNode = nullNodeFor(valueClass); + if (nullNode != null) { + data.set(key, valueClass.cast(nullNode)); + return; + } + + ClavaLog.debug("Could not find ClavaNode with id '" + nodeId + + "', skipping key '" + key.getName() + + "'. Check if node is being visited. If parsing of includes is enabled, check that the parsing level is sufficient."); + return; + } ClavaNode adaptedNode = adaptNode(node, valueClass); @@ -194,6 +208,26 @@ private ClavaNode adaptNode(ClavaNode node, Class value return node; } + private ClavaNode nullNodeFor(Class valueClass) { + if (valueClass.equals(Type.class)) { + return factory.nullType(); + } + + if (valueClass.equals(Decl.class)) { + return factory.nullDecl(); + } + + if (valueClass.equals(Stmt.class)) { + return factory.nullStmt(); + } + + if (valueClass.equals(Expr.class)) { + return factory.nullExpr(); + } + + return null; + } + public void queueSetOptionalNode(DataClass data, DataKey> key, String nodeId) { @@ -227,7 +261,29 @@ public void queueSetNodeList(DataClass data, DataKey

  • { @SuppressWarnings("unchecked") // If the nodes exist, they should be of the requested type - List nodes = nodeIds.stream().map(id -> (T) get(id)).collect(Collectors.toList()); + List nodes = nodeIds.stream() + .map(this::getOptional) + .flatMap(Optional::stream) + .map(node -> (T) node) + .collect(Collectors.toList()); + + data.set(key, nodes); + }; + + queuedActions.add(nodeToAdd); + } + + public void queueSetOptionalNodeList(DataClass data, DataKey> key, + List nodeIds) { + + Runnable nodeToAdd = () -> { + + @SuppressWarnings("unchecked") // If present, the nodes should be of the requested type + List nodes = nodeIds.stream() + .map(this::getOptional) + .flatMap(Optional::stream) + .map(node -> (T) node) + .collect(Collectors.toList()); data.set(key, nodes); }; diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java index b108271017..f7f11dc97a 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java @@ -87,7 +87,7 @@ public static DataStore parseDeclData(LineStream lines, ClangAstData dataStore) data.add(Decl.IS_INVALID_DECL, LineStreamParsers.oneOrZero(lines)); data.add(Decl.IS_MODULE_PRIVATE, LineStreamParsers.oneOrZero(lines)); - dataStore.getClavaNodes().queueSetNodeList(data, Decl.ATTRIBUTES, LineStreamParsers.stringList(lines)); + dataStore.getClavaNodes().queueSetOptionalNodeList(data, Decl.ATTRIBUTES, LineStreamParsers.stringList(lines)); // dataStore.getClavaNodes().queueSetOptionalNode(data, Decl.DECL_CONTEXT, lines.nextLine()); // dataStore.getClavaNodes().queueSetNodeList(data, Decl.DECL_CONTEXT, LineStreamParsers.stringList(lines)); // data.add(Decl.DECL_CONTEXT_IDS, LineStreamParsers.stringList(lines)); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/ExprDataParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/ExprDataParser.java index 3687d9e622..8582135167 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/ExprDataParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/ExprDataParser.java @@ -119,7 +119,7 @@ public static DataStore parseCharacterLiteralData(LineStream lines, ClangAstData DataStore data = parseLiteralData(lines, dataStore); data.add(CharacterLiteral.VALUE, LineStreamParsers.longInt(lines)); - data.add(CharacterLiteral.KIND, LineStreamParsers.enumFromInt(CharacterKind.getEnumHelper(), lines)); + data.add(CharacterLiteral.KIND, LineStreamParsers.enumFromName(CharacterKind.getEnumHelper(), lines)); return data; } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java index 43c2b2efb0..a52881b784 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java @@ -64,7 +64,7 @@ public static DataStore parseGotoStmtData(LineStream lines, ClangAstData dataSto public static DataStore parseAttributedStmtData(LineStream lines, ClangAstData dataStore) { DataStore data = parseStmtData(lines, dataStore); - dataStore.getClavaNodes().queueSetNodeList(data, AttributedStmt.STMT_ATTRIBUTES, + dataStore.getClavaNodes().queueSetOptionalNodeList(data, AttributedStmt.STMT_ATTRIBUTES, LineStreamParsers.stringList(lines)); return data; diff --git a/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts b/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts index 205e431aed..123ec0d47b 100644 --- a/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts +++ b/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts @@ -42,7 +42,7 @@ export default class LocalStaticToGlobal extends SimplePass { } // With static storage - if ($jp.storageClass !== StorageClass.STATIC) { + if ($jp.storageClass.toLowerCase() !== StorageClass.STATIC) { return false; } diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/attr/enums/AttributeKind.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/attr/enums/AttributeKind.java index 8158847f4a..5b0e3d7e71 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/attr/enums/AttributeKind.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/attr/enums/AttributeKind.java @@ -24,12 +24,19 @@ import pt.up.fe.specs.util.utilities.CachedItems; public enum AttributeKind implements StringProvider { - AddressSpace, AnnotateType, + ArmIn, + ArmInOut, ArmMveStrictPolymorphism, + ArmOut, + ArmPreserves, + ArmStreaming, + ArmStreamingCompatible, BTFTypeTag, CmseNSCall, + HLSLGroupSharedAddressSpace, + HLSLParamModifier, NoDeref, ObjCGC, ObjCInertUnsafeUnretained, @@ -49,15 +56,17 @@ public enum AttributeKind implements StringProvider { TypeNullable, TypeNullableResult, UPtr, + WebAssemblyFuncref, + CodeAlign, FallThrough, Likely, MustTail, OpenCLUnrollHint, - Suppress, Unlikely, AlwaysInline, NoInline, NoMerge, + Suppress, AArch64SVEPcs, AArch64VectorPcs, AMDGPUKernelCall, @@ -67,6 +76,7 @@ public enum AttributeKind implements StringProvider { FastCall, IntelOclBicc, LifetimeBound, + M68kRTD, MSABI, NSReturnsRetained, ObjCOwnership, @@ -94,6 +104,8 @@ public enum AttributeKind implements StringProvider { PassObjectSize, ReleaseHandle, UseHandle, + HLSLSV_DispatchThreadID, + HLSLSV_GroupIndex, AMDGPUFlatWorkGroupSize, AMDGPUNumSGPR, AMDGPUNumVGPR, @@ -116,6 +128,8 @@ public enum AttributeKind implements StringProvider { ArcWeakrefUnavailable, ArgumentWithTypeTag, ArmBuiltinAlias, + ArmLocallyStreaming, + ArmNew, Artificial, AsmLabel, AssertCapability, @@ -124,7 +138,9 @@ public enum AttributeKind implements StringProvider { AssumeAligned, Assumption, Availability, + AvailableOnlyInDefaultEvalMethod, BPFPreserveAccessIndex, + BPFPreserveStaticOffset, BTFDeclTag, Blocks, Builtin, @@ -153,6 +169,7 @@ public enum AttributeKind implements StringProvider { CapturedRecord, Cleanup, CmseNSEntry, + CodeModel, CodeSeg, Cold, Common, @@ -163,6 +180,12 @@ public enum AttributeKind implements StringProvider { ConsumableAutoCast, ConsumableSetOnRead, Convergent, + CoroDisableLifetimeBound, + CoroLifetimeBound, + CoroOnlyDestroyWhenComplete, + CoroReturnType, + CoroWrapper, + CountedBy, DLLExport, DLLExportStaticLocal, DLLImport, @@ -193,7 +216,8 @@ public enum AttributeKind implements StringProvider { GuardedVar, HIPManaged, HLSLNumThreads, - HLSLSV_GroupIndex, + HLSLResource, + HLSLResourceBinding, HLSLShader, Hot, IBAction, @@ -209,6 +233,7 @@ public enum AttributeKind implements StringProvider { M68kInterrupt, MIGServerRoutine, MSAllocator, + MSConstexpr, MSInheritance, MSNoVTable, MSP430Interrupt, @@ -216,6 +241,7 @@ public enum AttributeKind implements StringProvider { MSVtorDisp, MaxFieldAlignment, MayAlias, + MaybeUndef, MicroMips, MinSize, MinVectorWidth, @@ -227,6 +253,7 @@ public enum AttributeKind implements StringProvider { NSErrorDomain, NSReturnsAutoreleased, NSReturnsNotRetained, + NVPTXKernel, Naked, NoAlias, NoCommon, @@ -246,6 +273,7 @@ public enum AttributeKind implements StringProvider { NoThreadSafetyAnalysis, NoThrow, NoUniqueAddress, + NoUwtable, NotTailCalled, OMPAllocateDecl, OMPCaptureNoInit, @@ -288,11 +316,13 @@ public enum AttributeKind implements StringProvider { PragmaClangRodataSection, PragmaClangTextSection, PreferredName, + PreferredType, PtGuardedBy, PtGuardedVar, Pure, RISCVInterrupt, RandomizeLayout, + ReadOnlyPlacement, Reinitializes, ReleaseCapability, ReqdWorkGroupSize, @@ -313,6 +343,7 @@ public enum AttributeKind implements StringProvider { SpeculativeLoadHardening, StandaloneDebug, StrictFP, + StrictGuardStackCheck, SwiftAsync, SwiftAsyncError, SwiftAsyncName, @@ -320,12 +351,15 @@ public enum AttributeKind implements StringProvider { SwiftBridge, SwiftBridgedTypedef, SwiftError, + SwiftImportAsNonGeneric, + SwiftImportPropertyAsAccessors, SwiftName, SwiftNewType, SwiftPrivate, TLSModel, Target, TargetClones, + TargetVersion, TestTypestate, TransparentUnion, TrivialABI, @@ -334,6 +368,7 @@ public enum AttributeKind implements StringProvider { TypeVisibility, Unavailable, Uninitialized, + UnsafeBufferUsage, Unused, Used, UsingIfExists, @@ -382,6 +417,8 @@ public enum AttributeKind implements StringProvider { Overloadable, RenderScriptKernel, SwiftObjCMembers, + SwiftVersionedAddition, + SwiftVersionedRemoval, Thread, FirstAttr, LastAttr, @@ -398,7 +435,9 @@ public enum AttributeKind implements StringProvider { FirstInheritableParamAttr, LastInheritableParamAttr, FirstParameterABIAttr, - LastParameterABIAttr; + LastParameterABIAttr, + FirstHLSLAnnotationAttr, + LastHLSLAnnotationAttr; private static final Lazy> ENUM_HELPER = EnumHelperWithValue .newLazyHelperWithValue(AttributeKind.class); diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java index 21e3022c48..cae31033f3 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java @@ -24,13 +24,13 @@ import pt.up.fe.specs.util.providers.StringProvider; public enum InitializationStyle implements StringProvider { - NO_INIT, - CINIT, // C-style initialization with assignment - CALL_INIT("callinit"), // Call-style initialization (C++98) - LIST_INIT("listinit"); // Direct list-initialization (C++11) + CINIT, + CALL_INIT, + LIST_INIT, + ParenListInit; private static Lazy> ENUM_HELPER = EnumHelperWithValue - .newLazyHelperWithValue(InitializationStyle.class, NO_INIT); + .newLazyHelperWithValue(InitializationStyle.class); public static EnumHelperWithValue getHelper() { return ENUM_HELPER.get(); @@ -56,9 +56,8 @@ public String getCode(VarDecl node) { switch (this) { case CINIT: return cinitCode(node); - case NO_INIT: - return ""; case CALL_INIT: + case ParenListInit: return callInitCode(node); case LIST_INIT: return listInitCode(node); @@ -111,4 +110,4 @@ private String listInitCode(VarDecl node) { // .orElseThrow(() -> new RuntimeException()); } -} \ No newline at end of file +} diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java index 6ae4a3394d..1aa999c0fd 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java @@ -22,49 +22,11 @@ * */ public enum Linkage { - - /** - * The entity is unique and can only be referred to from within its scope. - */ - NoLinkage, - - /** - * the entity can be referred to from within the translation unit, but not other translation units. - */ - InternalLinkage, - - /** - * External linkage within a unique namespace. - * - *

    - * From the language perspective, these entities have external linkage. However, since they reside in an anonymous - * namespace, their names are unique to this translation unit, which is equivalent to having internal linkage from - * the code-generation point of view. - */ - UniqueExternalLinkage, - - /** - * No linkage according to the standard, but is visible from other translation units because of types defined in a - * inline function. - */ - VisibleNoLinkage, - - /** - * Internal linkage according to the Modules TS, but can be referred to from other translation units indirectly - * through inline functions and templates in the module interface. - * - */ - ModuleInternalLinkage, - - /** - * Module linkage, which indicates that the entity can be referred to from other translation units within the same - * module, and indirectly from arbitrary other translation units through inline functions and templates in the - * module interface. - */ - ModuleLinkage, - - /** - * The entity can be referred to from other translation units. - */ - ExternalLinkage; + Invalid, + None, + Internal, + UniqueExternal, + VisibleNone, + Module, + External; } diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java index 3e4ab11745..ee772ac786 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java @@ -18,7 +18,7 @@ import pt.up.fe.specs.util.providers.StringProvider; public enum CharacterKind implements StringProvider { - ASCII, + ORDINARY, WIDE, UTF8, UTF16, diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/StringKind.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/StringKind.java index afdcfbb29b..8f141a1a51 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/StringKind.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/StringKind.java @@ -18,12 +18,12 @@ import pt.up.fe.specs.util.exceptions.NotImplementedException; public enum StringKind { - ORDINARY, WIDE, UTF8(true), UTF16(true), - UTF32(true); + UTF32(true), + UNEVALUATED; private final boolean isUTF; @@ -47,6 +47,8 @@ public String getPrefix() { return "u"; case UTF32: return "U"; + case UNEVALUATED: + return ""; default: throw new NotImplementedException(this); } diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java index 0772dcacef..8f29b6be9e 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java @@ -50,6 +50,10 @@ public ElaboratedType(DataStore data, Collection children) @Override public Type desugar() { + if (!hasValue(NAMED_TYPE)) { + return this; + } + return getNamedType(); } @@ -63,6 +67,9 @@ public void setNamedType(Type namedType) { @Override public String getCode(ClavaNode sourceNode, String name) { + if (!hasValue(NAMED_TYPE) || getNamedType() instanceof NullType) { + return super.getCode(sourceNode, name); + } String code = getKeyword().getCode(); if (!code.isEmpty()) { diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/CallingConvention.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/CallingConvention.java index b5f58e3af6..47d8f46fbc 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/CallingConvention.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/CallingConvention.java @@ -18,7 +18,6 @@ import pt.up.fe.specs.util.providers.StringProvider; public enum CallingConvention implements StringProvider { - C, X86StdCall, X86FastCall, @@ -39,7 +38,8 @@ public enum CallingConvention implements StringProvider { PreserveAll, AArch64VectorCall, AArch64SVEPCS, - AMDGPUKernelCall; + AMDGPUKernelCall, + M68kRTD; private static final Lazy> HELPER = EnumHelperWithValue .newLazyHelperWithValue(CallingConvention.class); diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/ElaboratedTypeKeyword.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/ElaboratedTypeKeyword.java index b9a3be4baa..3827d3650c 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/ElaboratedTypeKeyword.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/ElaboratedTypeKeyword.java @@ -19,13 +19,13 @@ import pt.up.fe.specs.util.providers.StringProvider; public enum ElaboratedTypeKeyword implements StringProvider { - STRUCT, - INTERFACE, - UNION, - CLASS, - ENUM, - TYPENAME, - NONE; + Struct, + Interface, + Union, + Class, + Enum, + Typename, + None; private static final Lazy> HELPER = EnumHelperWithValue .newLazyHelperWithValue(ElaboratedTypeKeyword.class); @@ -35,7 +35,7 @@ public static EnumHelperWithValue getHelper() { } public String getCode() { - if (this == NONE) { + if (this == None) { return ""; } @@ -46,4 +46,4 @@ public String getCode() { public String getString() { return SpecsStrings.toCamelCase(name()); } -} \ No newline at end of file +} diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/UnaryTransformTypeKind.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/UnaryTransformTypeKind.java index 920be2ab40..7710528a14 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/UnaryTransformTypeKind.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/enums/UnaryTransformTypeKind.java @@ -14,8 +14,21 @@ package pt.up.fe.specs.clava.ast.type.enums; public enum UnaryTransformTypeKind { - + AddLvalueReference, + AddPointer, + AddRvalueReference, Decay, + MakeSigned, + MakeUnsigned, + RemoveAllExtents, + RemoveConst, + RemoveCV, + RemoveCVRef, + RemoveExtent, + RemovePointer, + RemoveReference, + RemoveRestrict, + RemoveVolatile, EnumUnderlyingType; } diff --git a/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt b/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt index 3030db32df..fb6efbf514 100644 --- a/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt +++ b/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt @@ -9,5 +9,5 @@ After setting int with array: int Original template args: int,float After second arg to double: int,double After setting with array [double, int]: double,int -After setting typedef_to_change: std::vector::const_iterator -After setting changed_typedef_type: std::vector::const_iterator \ No newline at end of file +After setting typedef_to_change: std::vector::const_iterator +After setting changed_typedef_type: std::vector::const_iterator diff --git a/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/importable/AstFactory.java b/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/importable/AstFactory.java index a0eac8c1e8..df96d1bb6d 100644 --- a/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/importable/AstFactory.java +++ b/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/importable/AstFactory.java @@ -680,7 +680,7 @@ public static ATypedefDecl typedefDecl(CxxWeaver weaver, AType underlyingT public static AElaboratedType structType(CxxWeaver weaver, AStruct struct) { var namedType = (Type) struct.getTypeImpl().getNodeImpl(); - var elaboratedType = weaver.getFactory().elaboratedType(ElaboratedTypeKeyword.STRUCT, namedType); + var elaboratedType = weaver.getFactory().elaboratedType(ElaboratedTypeKeyword.Struct, namedType); return CxxJoinpoints.create(elaboratedType, weaver, AElaboratedType.class); } From 33c7c2fe4b6848105ff28f83c4060609fb124b50 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Fri, 3 Jul 2026 02:52:17 +0100 Subject: [PATCH 02/39] Read clang dumper include entrypoints Use entrypoints.txt from extracted clang-dumper include archives as the ordered system include roots. Throw when entrypoints.txt is missing instead of inferring roots from archive folder names. --- .../pt/up/fe/specs/clang/ClangResources.java | 47 ++++--------------- 1 file changed, 8 insertions(+), 39 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index e542fba19d..28662bd010 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -30,11 +30,8 @@ import java.nio.file.Files; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HexFormat; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -229,44 +226,16 @@ private File prepareIncludesFolder(ClangDumperManifest manifest) { } private List getIncludeFolders(File extractedFolder) { - var includeFolders = new LinkedHashSet(); - - var usrFolder = new File(extractedFolder, "usr"); - var targetFolders = SpecsIo.getFolders(usrFolder).stream() - .filter(folder -> folder.getName().contains("-")) - .sorted(Comparator.comparing(File::getName)) - .toList(); - - var cxxFolder = new File(usrFolder, "c++"); - var cxxVersionFolders = SpecsIo.getFolders(cxxFolder).stream() - .filter(folder -> folder.getName().chars().allMatch(Character::isDigit)) - .sorted(Comparator.comparing(File::getName).reversed()) - .toList(); - - addIfDirectory(includeFolders, extractedFolder, "clang"); - - if (!cxxVersionFolders.isEmpty()) { - var cxxVersionFolder = cxxVersionFolders.get(0); - includeFolders.add(cxxVersionFolder); - - for (var targetFolder : targetFolders) { - addIfDirectory(includeFolders, targetFolder, "c++/" + cxxVersionFolder.getName()); - } + var entrypointsFile = new File(extractedFolder, "entrypoints.txt"); + if (!entrypointsFile.isFile()) { + throw new RuntimeException("Could not find include archive entrypoints file '" + entrypointsFile + "'"); } - addIfDirectory(includeFolders, extractedFolder, "usr"); - includeFolders.addAll(targetFolders); - - addIfDirectory(includeFolders, extractedFolder, "include"); - - return new ArrayList<>(includeFolders); - } - - private static void addIfDirectory(LinkedHashSet folders, File parent, String relativePath) { - var folder = new File(parent, relativePath); - if (folder.isDirectory()) { - folders.add(folder); - } + return SpecsIo.read(entrypointsFile).lines() + .map(String::trim) + .filter(line -> !line.isEmpty()) + .map(line -> new File(extractedFolder, line)) + .toList(); } private ResourceWriteData downloadAsset(ClangDumperManifest manifest, String kind, File resourceFolder) { From 9f62bb8b67a7c883d046dd07f664cd6fc7076958 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Mon, 6 Jul 2026 18:47:31 +0100 Subject: [PATCH 03/39] Cache clang dumper by release version --- .../pt/up/fe/specs/clang/ClangAstKeys.java | 10 - .../fe/specs/clang/ClangAstWebResource.java | 2 +- .../pt/up/fe/specs/clang/ClangResources.java | 211 +++++++++++++++++- .../fe/specs/clang/codeparser/CodeParser.java | 2 +- .../clang/codeparser/ParallelCodeParser.java | 4 +- Clava-JS/code/sideEffects.ts | 8 +- 6 files changed, 207 insertions(+), 30 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java index fd9907689b..ec17d8a8d6 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java @@ -27,8 +27,6 @@ public interface ClangAstKeys { - DataKey CLANGAST_VERSION = KeyFactory.string("clangast_version", ""); - /** * What libc/libcxx mode should be used. */ @@ -55,7 +53,6 @@ public static String getFlagIgnoreIncludes() { static DataStore toDataStore(List flags) { DataStore config = DataStore.newInstance(ClavaOptions.STORE_DEFINITION, false); final String stdPrefix = "-std="; - final String clangAstDumperPrefix = "-clang-dumper="; final String cilkFlag = "-fcilkplus"; // Search options @@ -78,13 +75,6 @@ static DataStore toDataStore(List flags) { continue; } - // If ClangAstDumper version, parse option - if (flag.startsWith(clangAstDumperPrefix)) { - String version = flag.substring(clangAstDumperPrefix.length()); - config.set(ClangAstKeys.CLANGAST_VERSION, version); - continue; - } - // If Cilk flag, add option if (flag.equals(cilkFlag)) { config.set(ClangAstKeys.USES_CILK); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index ce2a109a5a..5299cbaf1a 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -26,7 +26,7 @@ public final class ClangAstWebResource { private static final String RELEASE_ROOT = "https://github.com/specs-feup/clang-dumper/releases/download/"; private static final String RELEASE_TAG_RESOURCE = "clang-dumper-release.tag"; - private static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; + public static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; private static final Gson GSON = new Gson(); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 28662bd010..6ddc090cff 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -27,22 +27,34 @@ import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.HexFormat; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.Stream; public class ClangResources { private static final Map CLANG_FILES_CACHE = new ConcurrentHashMap<>(); private final static String CLANG_FOLDERNAME = "clang_ast_exe"; + private final static String INCLUDES_FOLDERNAME = "includes"; + private final static String INCLUDES_HASHES_FILENAME = "includes.sha256"; + private final static String LAST_USED_FILENAME = "last-used.txt"; + private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); private static final AtomicInteger HAS_LIBC = new AtomicInteger(-1); @@ -52,13 +64,13 @@ public ClangResources(CodeParser options) { this.options = options; } - public ClangFiles getClangFiles(String version, LibcMode libcMode) { + public ClangFiles getClangFiles(LibcMode libcMode) { - var effectiveVersion = version.isEmpty() ? ClangAstWebResource.getReleaseTag() : version; - var key = libcMode.name() + "_" + effectiveVersion + "_" + getClangResourceFolder().getAbsolutePath(); + var key = libcMode.name() + "_" + getClangResourceFolder().getAbsolutePath(); var files = CLANG_FILES_CACHE.get(key); if (files != null) { + writeLastUsed(Instant.now()); SpecsLogs.debug(() -> "Using cached version of Clang files: " + files); return files; } @@ -67,6 +79,9 @@ public ClangFiles getClangFiles(String version, LibcMode libcMode) { File clangExecutable = prepareResources(manifest); List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); + validateTopLevelCacheFiles(clangExecutable); + updateLastUsedAndCleanupStaleVersions(); + var newFiles = new ClangFiles(clangExecutable, builtinIncludes); SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); @@ -116,7 +131,7 @@ private void unblockWindowsFile(File executable) { } public File getClangResourceFolder() { - return options.get(CodeParser.DUMPER_FOLDER); + return SpecsIo.mkdir(options.get(CodeParser.DUMPER_FOLDER), ClangAstWebResource.getReleaseTag()); } public static File getDefaultTempFolder() { @@ -212,14 +227,23 @@ private List prepareIncludes(ClangDumperManifest manifest) { private File prepareIncludesFolder(ClangDumperManifest manifest) { File resourceFolder = getClangResourceFolder(); - ResourceWriteData zipFile = downloadAsset(manifest, "includes", resourceFolder); + var includesAsset = getCurrentAsset(manifest, "includes"); + var extractedFolder = new File(resourceFolder, INCLUDES_FOLDERNAME); + var includesHashesFile = new File(resourceFolder, INCLUDES_HASHES_FILENAME); - var zipFoldername = "include_" + SpecsIo.removeExtension(zipFile.getFile()); - var extractedFolder = SpecsIo.mkdir(resourceFolder, zipFoldername); + if (isIncludesCacheValid(extractedFolder, includesHashesFile, includesAsset)) { + return extractedFolder; + } + + ResourceWriteData zipFile = downloadAsset(manifest, "includes", resourceFolder); - if (zipFile.isNewFile() || SpecsIo.isEmptyFolder(extractedFolder)) { + try { + SpecsIo.mkdir(extractedFolder); SpecsIo.deleteFolderContents(extractedFolder); SpecsIo.extractZip(zipFile.getFile(), extractedFolder); + writeIncludesHashes(extractedFolder, includesHashesFile, includesAsset); + } finally { + SpecsIo.delete(zipFile.getFile()); } return extractedFolder; @@ -239,9 +263,7 @@ private List getIncludeFolders(File extractedFolder) { } private ResourceWriteData downloadAsset(ClangDumperManifest manifest, String kind, File resourceFolder) { - var platform = getManifestPlatform(); - var arch = getManifestArch(platform); - var asset = manifest.getAsset(platform, arch, kind); + var asset = getCurrentAsset(manifest, kind); var resource = ClangAstWebResource.getAssetResource(asset); var writeData = resource.writeVersioned(resourceFolder, ClangResources.class); @@ -260,6 +282,173 @@ private ResourceWriteData downloadAsset(ClangDumperManifest manifest, String kin return writeData; } + private ClangDumperManifestAsset getCurrentAsset(ClangDumperManifest manifest, String kind) { + var platform = getManifestPlatform(); + var arch = getManifestArch(platform); + return manifest.getAsset(platform, arch, kind); + } + + private boolean isIncludesCacheValid(File includesFolder, File includesHashesFile, + ClangDumperManifestAsset includesAsset) { + + if (!includesFolder.isDirectory() || !includesHashesFile.isFile()) { + return false; + } + + var expectedHashes = readIncludesHashes(includesHashesFile, includesAsset); + if (expectedHashes == null) { + return false; + } + + var expectedFiles = expectedHashes.keySet(); + var actualFiles = listRegularFiles(includesFolder); + + if (!actualFiles.equals(expectedFiles)) { + SpecsLogs.info("Cached clang-dumper includes contain missing or extra files, extracting them again."); + return false; + } + + for (var entry : expectedHashes.entrySet()) { + var file = new File(includesFolder, entry.getKey()); + if (!entry.getValue().equalsIgnoreCase(calculateSha256(file))) { + SpecsLogs.info("Cached clang-dumper include file '" + file + + "' does not match the expected checksum, extracting includes again."); + return false; + } + } + + return true; + } + + private static Map readIncludesHashes(File includesHashesFile, + ClangDumperManifestAsset includesAsset) { + + var lines = SpecsIo.read(includesHashesFile).lines().toList(); + if (lines.size() < 2 || !lines.get(0).equals("# clang-dumper-includes " + includesAsset.sha256())) { + return null; + } + + Map hashes = new HashMap<>(); + for (int i = 1; i < lines.size(); i++) { + var line = lines.get(i); + if (line.isBlank()) { + continue; + } + + if (line.length() <= 65 || line.charAt(64) != ' ') { + return null; + } + + var hash = line.substring(0, 64); + var relativePath = line.substring(65); + hashes.put(relativePath, hash); + } + + return hashes; + } + + private static void writeIncludesHashes(File includesFolder, File includesHashesFile, + ClangDumperManifestAsset includesAsset) { + + var actualFiles = listRegularFiles(includesFolder).stream() + .sorted() + .toList(); + + StringBuilder hashes = new StringBuilder(); + hashes.append("# clang-dumper-includes ").append(includesAsset.sha256()).append(System.lineSeparator()); + + for (var relativePath : actualFiles) { + var file = new File(includesFolder, relativePath); + hashes.append(calculateSha256(file)) + .append(' ') + .append(relativePath) + .append(System.lineSeparator()); + } + + SpecsIo.write(includesHashesFile, hashes.toString()); + } + + private static Set listRegularFiles(File folder) { + try (Stream paths = Files.walk(folder.toPath())) { + return paths + .filter(Files::isRegularFile) + .map(folder.toPath()::relativize) + .map(Path::toString) + .map(path -> path.replace(File.separatorChar, '/')) + .collect(Collectors.toCollection(HashSet::new)); + } catch (IOException e) { + throw new UncheckedIOException("Could not list files in folder '" + folder + "'", e); + } + } + + private void validateTopLevelCacheFiles(File clangExecutable) { + File resourceFolder = getClangResourceFolder(); + Set expectedNames = new HashSet<>(); + expectedNames.add(ClangAstWebResource.MANIFEST_FILENAME); + expectedNames.add(clangExecutable.getName()); + expectedNames.add(INCLUDES_FOLDERNAME); + expectedNames.add(INCLUDES_HASHES_FILENAME); + expectedNames.add(LAST_USED_FILENAME); + + var files = resourceFolder.listFiles(); + if (files == null) { + return; + } + + for (var file : files) { + if (expectedNames.contains(file.getName())) { + continue; + } + + SpecsLogs.info("Deleting unexpected file from clang-dumper cache: " + file); + SpecsIo.delete(file); + } + } + + private void updateLastUsedAndCleanupStaleVersions() { + var now = Instant.now(); + File resourceFolder = getClangResourceFolder(); + writeLastUsed(now); + + var staleCleanup = new Thread(() -> deleteStaleVersions(now, resourceFolder), + "clang-dumper-stale-cache-cleanup"); + staleCleanup.setDaemon(true); + staleCleanup.start(); + } + + private void writeLastUsed(Instant timestamp) { + SpecsIo.write(new File(getClangResourceFolder(), LAST_USED_FILENAME), timestamp.toString()); + } + + private void deleteStaleVersions(Instant now, File currentVersionFolder) { + File cacheBaseFolder = options.get(CodeParser.DUMPER_FOLDER); + var versions = cacheBaseFolder.listFiles(File::isDirectory); + if (versions == null) { + return; + } + + for (var versionFolder : versions) { + if (versionFolder.equals(currentVersionFolder)) { + continue; + } + + var lastUsedFile = new File(versionFolder, LAST_USED_FILENAME); + if (!lastUsedFile.isFile()) { + continue; + } + + try { + var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); + if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { + SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); + SpecsIo.deleteFolder(versionFolder); + } + } catch (RuntimeException e) { + SpecsLogs.warn("Could not inspect clang-dumper cache timestamp '" + lastUsedFile + "'", e); + } + } + } + private static boolean hasExpectedSha256(File file, ClangDumperManifestAsset asset) { return asset.sha256().equalsIgnoreCase(calculateSha256(file)); } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java index 5c154cd24e..f7441ed77d 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java @@ -45,7 +45,7 @@ public abstract class CodeParser extends ADataClass { .setLabel("CUDA Path (empty: uses system installed; : uses builtin version)") .setDefaultString(""); public static final DataKey DUMPER_FOLDER = KeyFactory.folder("dumperFolder") - .setLabel("The work folder for the clang-dumper. Clava will look for it in this folder, and if not found, will download it. If not set, a temporary folder will be used.") + .setLabel("The base cache folder for the clang-dumper. Clava stores each dumper release in a versioned subfolder and downloads it if not found. If not set, a temporary folder will be used.") .setDefault(ClangResources::getDefaultTempFolder); /** diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java index 399d916211..cf71e63bc4 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java @@ -104,8 +104,6 @@ public App parse(List inputSources, List compilerOptions, ClavaCon // Standard standard = getStandard(allUserSources.values(), options); // config.getTry(ClavaOptions.STANDARD).ifPresent(standard -> arguments.add(standard.getFlag())); - // Get version for the executable - String version = options.get(ClangAstKeys.CLANGAST_VERSION); // System.out.println("PARALLEL OPTIONS: " + options); // Prepare resources before execution // ClangResources clangResources = new ClangResources(get(SHOW_CLANG_DUMP)); @@ -117,7 +115,7 @@ public App parse(List inputSources, List compilerOptions, ClavaCon ClavaLog.debug(() -> "In Linux, ClangAstDumper is a plugin. LIBC_CXX_MODE is reset to SYSTEM."); } - var clangFiles = clangResources.getClangFiles(version, get(ClangAstKeys.LIBC_CXX_MODE)); + var clangFiles = clangResources.getClangFiles(get(ClangAstKeys.LIBC_CXX_MODE)); // File clangExecutable = clangResources.prepareResources(version); // List builtinIncludes = clangResources.prepareIncludes(clangExecutable, // get(ClangAstKeys.USE_PLATFORM_INCLUDES)); diff --git a/Clava-JS/code/sideEffects.ts b/Clava-JS/code/sideEffects.ts index 59d7bf76ff..2c62d3bf47 100644 --- a/Clava-JS/code/sideEffects.ts +++ b/Clava-JS/code/sideEffects.ts @@ -18,14 +18,14 @@ const datastore = Weaver.getWeaverEngine().getData().get(); datastore.set(CxxWeaverOptions.DISABLE_CLAVA_INFO, true); datastore.set( CodeParser.DUMPER_FOLDER, - new JavaTypes.File(getVersionedCacheDir()) + new JavaTypes.File(getClangDumperCacheDir()) ); /** Code to obtain temporary folder **/ -function getVersionedCacheDir(): string { - // Use name+version to isolate different installed versions - return path.join(getCacheBaseDir(), pkg.name, pkg.version); +function getClangDumperCacheDir(): string { + // The version will be added by the installer to isolate different installed versions + return path.join(getCacheBaseDir(), pkg.name, "clang-dumper"); } function getCacheBaseDir(): string { From 37d0b27fb8f7f2e71f9b24fb1189771f455d1424 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 7 Jul 2026 03:08:38 +0100 Subject: [PATCH 04/39] Throttle clang dumper include validation --- .../pt/up/fe/specs/clang/ClangResources.java | 86 ++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 6ddc090cff..19003acaf8 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -49,12 +49,15 @@ public class ClangResources { private static final Map CLANG_FILES_CACHE = new ConcurrentHashMap<>(); + private static final Set INCLUDES_DEEP_VALIDATION_CACHE = ConcurrentHashMap.newKeySet(); private final static String CLANG_FOLDERNAME = "clang_ast_exe"; private final static String INCLUDES_FOLDERNAME = "includes"; private final static String INCLUDES_HASHES_FILENAME = "includes.sha256"; + private final static String INCLUDES_VALIDATED_FILENAME = "includes-validated.txt"; private final static String LAST_USED_FILENAME = "last-used.txt"; private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); + private final static Duration INCLUDES_DEEP_VALIDATION_INTERVAL = Duration.ofDays(1); private static final AtomicInteger HAS_LIBC = new AtomicInteger(-1); @@ -230,18 +233,21 @@ private File prepareIncludesFolder(ClangDumperManifest manifest) { var includesAsset = getCurrentAsset(manifest, "includes"); var extractedFolder = new File(resourceFolder, INCLUDES_FOLDERNAME); var includesHashesFile = new File(resourceFolder, INCLUDES_HASHES_FILENAME); + var includesValidatedFile = new File(resourceFolder, INCLUDES_VALIDATED_FILENAME); if (isIncludesCacheValid(extractedFolder, includesHashesFile, includesAsset)) { + validateIncludesCacheInBackground(extractedFolder, includesHashesFile, includesValidatedFile, includesAsset); return extractedFolder; } - ResourceWriteData zipFile = downloadAsset(manifest, "includes", resourceFolder); + ResourceWriteData zipFile = downloadAsset(includesAsset, resourceFolder); try { SpecsIo.mkdir(extractedFolder); SpecsIo.deleteFolderContents(extractedFolder); SpecsIo.extractZip(zipFile.getFile(), extractedFolder); writeIncludesHashes(extractedFolder, includesHashesFile, includesAsset); + writeTimestamp(includesValidatedFile, Instant.now()); } finally { SpecsIo.delete(zipFile.getFile()); } @@ -264,6 +270,10 @@ private List getIncludeFolders(File extractedFolder) { private ResourceWriteData downloadAsset(ClangDumperManifest manifest, String kind, File resourceFolder) { var asset = getCurrentAsset(manifest, kind); + return downloadAsset(asset, resourceFolder); + } + + private ResourceWriteData downloadAsset(ClangDumperManifestAsset asset, File resourceFolder) { var resource = ClangAstWebResource.getAssetResource(asset); var writeData = resource.writeVersioned(resourceFolder, ClangResources.class); @@ -295,6 +305,63 @@ private boolean isIncludesCacheValid(File includesFolder, File includesHashesFil return false; } + if (!hasExpectedIncludesAssetHash(includesHashesFile, includesAsset)) { + return false; + } + + var entrypointsFile = new File(includesFolder, "entrypoints.txt"); + if (!entrypointsFile.isFile()) { + SpecsLogs.info("Cached clang-dumper includes are missing entrypoints, extracting them again."); + return false; + } + + return true; + } + + private void validateIncludesCacheInBackground(File includesFolder, File includesHashesFile, + File includesValidatedFile, + ClangDumperManifestAsset includesAsset) { + + if (!shouldRunIncludesDeepValidation(includesValidatedFile)) { + return; + } + + var validationKey = includesFolder.getAbsolutePath() + "_" + includesAsset.sha256(); + if (!INCLUDES_DEEP_VALIDATION_CACHE.add(validationKey)) { + return; + } + + var deepValidation = new Thread(() -> { + if (isIncludesCacheDeepValid(includesFolder, includesHashesFile, includesAsset)) { + writeTimestamp(includesValidatedFile, Instant.now()); + return; + } + + SpecsLogs.info("Invalidating clang-dumper includes cache metadata: " + includesHashesFile); + SpecsIo.delete(includesHashesFile); + SpecsIo.delete(includesValidatedFile); + }, "clang-dumper-includes-cache-validation"); + + deepValidation.setDaemon(true); + deepValidation.start(); + } + + private static boolean shouldRunIncludesDeepValidation(File includesValidatedFile) { + if (!includesValidatedFile.isFile()) { + return true; + } + + try { + var lastValidated = Instant.parse(SpecsIo.read(includesValidatedFile).trim()); + return lastValidated.isBefore(Instant.now().minus(INCLUDES_DEEP_VALIDATION_INTERVAL)); + } catch (RuntimeException e) { + return true; + } + } + + private boolean isIncludesCacheDeepValid(File includesFolder, File includesHashesFile, + ClangDumperManifestAsset includesAsset) { + var expectedHashes = readIncludesHashes(includesHashesFile, includesAsset); if (expectedHashes == null) { return false; @@ -320,6 +387,16 @@ private boolean isIncludesCacheValid(File includesFolder, File includesHashesFil return true; } + private static boolean hasExpectedIncludesAssetHash(File includesHashesFile, + ClangDumperManifestAsset includesAsset) { + + try (var reader = Files.newBufferedReader(includesHashesFile.toPath())) { + return ("# clang-dumper-includes " + includesAsset.sha256()).equals(reader.readLine()); + } catch (IOException e) { + return false; + } + } + private static Map readIncludesHashes(File includesHashesFile, ClangDumperManifestAsset includesAsset) { @@ -388,6 +465,7 @@ private void validateTopLevelCacheFiles(File clangExecutable) { expectedNames.add(clangExecutable.getName()); expectedNames.add(INCLUDES_FOLDERNAME); expectedNames.add(INCLUDES_HASHES_FILENAME); + expectedNames.add(INCLUDES_VALIDATED_FILENAME); expectedNames.add(LAST_USED_FILENAME); var files = resourceFolder.listFiles(); @@ -417,7 +495,11 @@ private void updateLastUsedAndCleanupStaleVersions() { } private void writeLastUsed(Instant timestamp) { - SpecsIo.write(new File(getClangResourceFolder(), LAST_USED_FILENAME), timestamp.toString()); + writeTimestamp(new File(getClangResourceFolder(), LAST_USED_FILENAME), timestamp); + } + + private static void writeTimestamp(File file, Instant timestamp) { + SpecsIo.write(file, timestamp.toString()); } private void deleteStaleVersions(Instant now, File currentVersionFolder) { From 9bb46c096e57cd4600bca6eb91ba82125eb8ce7d Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Wed, 15 Jul 2026 18:01:17 +0100 Subject: [PATCH 05/39] Restore strict clang dumper integration --- .../pt/up/fe/specs/clang/ClangResources.java | 102 ++++++++++-------- .../fe/specs/clang/dumper/ClangAstParser.java | 7 +- .../specs/clang/parsers/ClavaNodeParser.java | 23 ++-- .../up/fe/specs/clang/parsers/ClavaNodes.java | 46 +------- .../specs/clang/parsers/LanguageParser.java | 1 + .../up/fe/specs/clang/ClangResourcesTest.java | 38 +++++++ .../api/clava/pass/LocalStaticToGlobal.ts | 2 +- .../specs/clava/ast/expr/StringLiteral.java | 2 +- .../specs/clava/ast/extra/data/Language.java | 5 + .../specs/clava/ast/type/ElaboratedType.java | 8 -- .../weaver/cpp/results/TypeTemplate.js.txt | 4 +- 11 files changed, 118 insertions(+), 120 deletions(-) create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 19003acaf8..6a03e64a6a 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -28,8 +28,10 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Duration; @@ -49,13 +51,13 @@ public class ClangResources { private static final Map CLANG_FILES_CACHE = new ConcurrentHashMap<>(); - private static final Set INCLUDES_DEEP_VALIDATION_CACHE = ConcurrentHashMap.newKeySet(); - + private static final Map CLANG_FILES_LOCKS = new ConcurrentHashMap<>(); private final static String CLANG_FOLDERNAME = "clang_ast_exe"; private final static String INCLUDES_FOLDERNAME = "includes"; private final static String INCLUDES_HASHES_FILENAME = "includes.sha256"; private final static String INCLUDES_VALIDATED_FILENAME = "includes-validated.txt"; private final static String LAST_USED_FILENAME = "last-used.txt"; + private final static String CACHE_LOCK_FILENAME = ".cache.lock"; private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); private final static Duration INCLUDES_DEEP_VALIDATION_INTERVAL = Duration.ofDays(1); @@ -69,28 +71,45 @@ public ClangResources(CodeParser options) { public ClangFiles getClangFiles(LibcMode libcMode) { - var key = libcMode.name() + "_" + getClangResourceFolder().getAbsolutePath(); + var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); + var key = libcMode.name() + "_" + useBuiltinCuda + "_" + getClangResourceFolder().getAbsolutePath(); - var files = CLANG_FILES_CACHE.get(key); - if (files != null) { + var cachedFiles = CLANG_FILES_CACHE.get(key); + if (cachedFiles != null) { writeLastUsed(Instant.now()); - SpecsLogs.debug(() -> "Using cached version of Clang files: " + files); - return files; + SpecsLogs.debug(() -> "Using cached version of Clang files: " + cachedFiles); + return cachedFiles; } - var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); - File clangExecutable = prepareResources(manifest); - List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); + var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(key, ignored -> new Object()); + synchronized (jvmLock) { + var files = CLANG_FILES_CACHE.get(key); + if (files != null) { + writeLastUsed(Instant.now()); + return files; + } + + var lockFile = new File(getClangResourceFolder(), CACHE_LOCK_FILENAME); + try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, + StandardOpenOption.WRITE); + var ignored = lockChannel.lock()) { - validateTopLevelCacheFiles(clangExecutable); - updateLastUsedAndCleanupStaleVersions(); + var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); + File clangExecutable = prepareResources(manifest); + List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); - var newFiles = new ClangFiles(clangExecutable, builtinIncludes); - SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); + validateTopLevelCacheFiles(clangExecutable); + updateLastUsedAndCleanupStaleVersions(); - CLANG_FILES_CACHE.put(key, newFiles); + var newFiles = new ClangFiles(clangExecutable, builtinIncludes); + SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); - return newFiles; + CLANG_FILES_CACHE.put(key, newFiles); + return newFiles; + } catch (IOException e) { + throw new UncheckedIOException("Could not lock clang-dumper cache '" + lockFile + "'", e); + } + } } /** @@ -213,7 +232,10 @@ private static ProcessOutputAsString runClangAstDumper(File clangExecutable, Fil } private List prepareIncludes(ClangDumperManifest manifest, File clangExecutable, LibcMode libcMode) { - if (!useBuiltinLibc(clangExecutable, libcMode)) { + var useBuiltinLibc = useBuiltinLibc(clangExecutable, libcMode); + var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); + + if (!useBuiltinLibc && !useBuiltinCuda) { return List.of(); } @@ -235,8 +257,7 @@ private File prepareIncludesFolder(ClangDumperManifest manifest) { var includesHashesFile = new File(resourceFolder, INCLUDES_HASHES_FILENAME); var includesValidatedFile = new File(resourceFolder, INCLUDES_VALIDATED_FILENAME); - if (isIncludesCacheValid(extractedFolder, includesHashesFile, includesAsset)) { - validateIncludesCacheInBackground(extractedFolder, includesHashesFile, includesValidatedFile, includesAsset); + if (isIncludesCacheValid(extractedFolder, includesHashesFile, includesValidatedFile, includesAsset)) { return extractedFolder; } @@ -298,7 +319,7 @@ private ClangDumperManifestAsset getCurrentAsset(ClangDumperManifest manifest, S return manifest.getAsset(platform, arch, kind); } - private boolean isIncludesCacheValid(File includesFolder, File includesHashesFile, + private boolean isIncludesCacheValid(File includesFolder, File includesHashesFile, File includesValidatedFile, ClangDumperManifestAsset includesAsset) { if (!includesFolder.isDirectory() || !includesHashesFile.isFile()) { @@ -315,35 +336,27 @@ private boolean isIncludesCacheValid(File includesFolder, File includesHashesFil return false; } - return true; - } - - private void validateIncludesCacheInBackground(File includesFolder, File includesHashesFile, - File includesValidatedFile, - ClangDumperManifestAsset includesAsset) { - if (!shouldRunIncludesDeepValidation(includesValidatedFile)) { - return; + return true; } - var validationKey = includesFolder.getAbsolutePath() + "_" + includesAsset.sha256(); - if (!INCLUDES_DEEP_VALIDATION_CACHE.add(validationKey)) { - return; + var validationStart = Instant.now(); + SpecsLogs.info("Validating clang-dumper includes cache: " + includesFolder); + if (isIncludesCacheDeepValid(includesFolder, includesHashesFile, includesAsset)) { + writeTimestamp(includesValidatedFile, Instant.now()); + SpecsLogs.info("Validated clang-dumper includes cache in " + + Duration.between(validationStart, Instant.now()).toMillis() + " ms"); + return true; } - var deepValidation = new Thread(() -> { - if (isIncludesCacheDeepValid(includesFolder, includesHashesFile, includesAsset)) { - writeTimestamp(includesValidatedFile, Instant.now()); - return; - } - - SpecsLogs.info("Invalidating clang-dumper includes cache metadata: " + includesHashesFile); + SpecsLogs.info("Invalidating clang-dumper includes cache metadata: " + includesHashesFile); + if (includesHashesFile.isFile()) { SpecsIo.delete(includesHashesFile); + } + if (includesValidatedFile.isFile()) { SpecsIo.delete(includesValidatedFile); - }, "clang-dumper-includes-cache-validation"); - - deepValidation.setDaemon(true); - deepValidation.start(); + } + return false; } private static boolean shouldRunIncludesDeepValidation(File includesValidatedFile) { @@ -467,6 +480,7 @@ private void validateTopLevelCacheFiles(File clangExecutable) { expectedNames.add(INCLUDES_HASHES_FILENAME); expectedNames.add(INCLUDES_VALIDATED_FILENAME); expectedNames.add(LAST_USED_FILENAME); + expectedNames.add(CACHE_LOCK_FILENAME); var files = resourceFolder.listFiles(); if (files == null) { @@ -590,8 +604,4 @@ private static String getManifestArch(String platform) { throw new RuntimeException("Unsupported architecture for clang-dumper: " + osArch); } - public File getBuiltinCudaLib() { - var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); - return prepareIncludesFolder(manifest); - } } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java index 3c3f1044cf..ccfb24385e 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstParser.java @@ -145,13 +145,14 @@ public TranslationUnit parseTu(File sourceFile) { if (ClavaNodes.isNullId(topLevelTypeId)) { continue; } - data.get(ClangAstData.CLAVA_NODES).getOptional(topLevelTypeId); - + ClavaNode parsedNode = data.get(ClangAstData.CLAVA_NODES).get(topLevelTypeId); + Objects.requireNonNull(parsedNode, () -> "No node for type '" + topLevelTypeId + "'"); } // Parse top-level attributes for (String topLevelAttributeId : topLevelAttributes) { - data.get(ClangAstData.CLAVA_NODES).getOptional(topLevelAttributeId); + ClavaNode parsedNode = data.get(ClangAstData.CLAVA_NODES).get(topLevelAttributeId); + Objects.requireNonNull(parsedNode, () -> "No node for attribute '" + topLevelAttributeId + "'"); } // Create TU node diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java index b2d2d9e3c9..19d3bfbcfb 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodeParser.java @@ -108,10 +108,6 @@ public void apply(LineStream lineStream, ClangAstData data) { ClavaNode node = parseNode(nodeId, classname, data, lineStream); - if (node == null) { - return; - } - // If UnsupportedNode, transform to DummyNode // node = transformUnsupportedNode(node); @@ -173,11 +169,8 @@ private ClavaNode parseNode(String nodeId, String classname, ClangAstData data, DataStore nodeData = data.get(ClangAstData.NODE_DATA).get(nodeId); if (nodeData == null) { - if (debug) { - SpecsLogs.msgInfo("No ClavaData/DataStore for node '" + nodeId + "' (classname: " + classname - + "), skipping node (linestream index '" + lineStream.getLastLineIndex() + "')"); - } - return null; + throw new RuntimeException("No ClavaData/DataStore for node '" + nodeId + "' (classname: " + classname + + "), data dumper is not being called (linestream index '" + lineStream.getLastLineIndex() + "')"); } // Get corresponding ClavaNode class @@ -240,14 +233,10 @@ private ClavaNode parseNode(String nodeId, String classname, ClangAstData data, // } // } - if (child == null) { - if (debug) { - int index = i; - SpecsLogs.msgInfo("Did not find ClavaNode for child with index '" + index + "' and id '" - + childId + "' when parsing " + clavaNodeClass.getSimpleName() + ", skipping child"); - } - continue; - } + int index = i; + Objects.requireNonNull(child, + () -> "Did not find ClavaNode for child with index '" + index + "' and id '" + childId + + "' when parsing " + clavaNodeClass.getSimpleName() + " -> " + nodeData); child = processChild(child, clavaNodeClass, data); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java index fb96e68c42..a0cc364dd7 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java @@ -31,10 +31,6 @@ import pt.up.fe.specs.clava.ClavaLog; import pt.up.fe.specs.clava.ClavaNode; import pt.up.fe.specs.clava.ast.NullNodeType; -import pt.up.fe.specs.clava.ast.decl.Decl; -import pt.up.fe.specs.clava.ast.expr.Expr; -import pt.up.fe.specs.clava.ast.stmt.Stmt; -import pt.up.fe.specs.clava.ast.type.Type; import pt.up.fe.specs.clava.context.ClavaFactory; import pt.up.fe.specs.util.SpecsCheck; import pt.up.fe.specs.util.SpecsSystem; @@ -152,20 +148,10 @@ public void queueSetNode(DataClass data, DataKey key + "', if node can be null, use queueOptional instead. Node data:\n" + data); } + ClavaNode node = Objects.requireNonNull(clavaNodes.get(nodeId), + () -> "Could not resolve required node '" + nodeId + "' for key '" + key.getName() + + "'. Node data:\n" + data); Class valueClass = key.getValueClass(); - ClavaNode node = clavaNodes.get(nodeId); - if (node == null) { - var nullNode = nullNodeFor(valueClass); - if (nullNode != null) { - data.set(key, valueClass.cast(nullNode)); - return; - } - - ClavaLog.debug("Could not find ClavaNode with id '" + nodeId - + "', skipping key '" + key.getName() - + "'. Check if node is being visited. If parsing of includes is enabled, check that the parsing level is sufficient."); - return; - } ClavaNode adaptedNode = adaptNode(node, valueClass); @@ -208,26 +194,6 @@ private ClavaNode adaptNode(ClavaNode node, Class value return node; } - private ClavaNode nullNodeFor(Class valueClass) { - if (valueClass.equals(Type.class)) { - return factory.nullType(); - } - - if (valueClass.equals(Decl.class)) { - return factory.nullDecl(); - } - - if (valueClass.equals(Stmt.class)) { - return factory.nullStmt(); - } - - if (valueClass.equals(Expr.class)) { - return factory.nullExpr(); - } - - return null; - } - public void queueSetOptionalNode(DataClass data, DataKey> key, String nodeId) { @@ -261,11 +227,7 @@ public void queueSetNodeList(DataClass data, DataKey

  • { @SuppressWarnings("unchecked") // If the nodes exist, they should be of the requested type - List nodes = nodeIds.stream() - .map(this::getOptional) - .flatMap(Optional::stream) - .map(node -> (T) node) - .collect(Collectors.toList()); + List nodes = nodeIds.stream().map(id -> (T) get(id)).collect(Collectors.toList()); data.set(key, nodes); }; diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/LanguageParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/LanguageParser.java index e366767bce..3524f18bad 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/LanguageParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/LanguageParser.java @@ -55,6 +55,7 @@ public void apply(LineStream lineStream, ClangAstData data) { .set(Language.C_PLUS_PLUS_17, LineStreamParsers.oneOrZero(lineStream)) .set(Language.C_PLUS_PLUS_20, LineStreamParsers.oneOrZero(lineStream)) .set(Language.C_PLUS_PLUS_23, LineStreamParsers.oneOrZero(lineStream)) + .set(Language.C_PLUS_PLUS_26, LineStreamParsers.oneOrZero(lineStream)) .set(Language.HAS_DIGRAPHS, LineStreamParsers.oneOrZero(lineStream)) .set(Language.IS_GNU, LineStreamParsers.oneOrZero(lineStream)) .set(Language.HEX_FLOATS, LineStreamParsers.oneOrZero(lineStream)) diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java new file mode 100644 index 0000000000..e46824d1f5 --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -0,0 +1,38 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; + +import org.junit.jupiter.api.Test; + +import pt.up.fe.specs.clang.codeparser.CodeParser; + +public class ClangResourcesTest { + + @Test + public void builtinCudaIncludesAreAvailableWithSystemLibc() { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); + + var clangFiles = new ClangResources(parser).getClangFiles(LibcMode.SYSTEM); + var hasCudaWrapper = clangFiles.builtinIncludes().stream() + .map(folder -> new File(folder, "__clang_cuda_runtime_wrapper.h")) + .anyMatch(File::isFile); + + assertTrue(hasCudaWrapper, "Built-in CUDA must provide Clang's CUDA runtime wrapper independently of libc"); + } +} diff --git a/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts b/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts index 123ec0d47b..205e431aed 100644 --- a/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts +++ b/Clava-JS/api/clava/pass/LocalStaticToGlobal.ts @@ -42,7 +42,7 @@ export default class LocalStaticToGlobal extends SimplePass { } // With static storage - if ($jp.storageClass.toLowerCase() !== StorageClass.STATIC) { + if ($jp.storageClass !== StorageClass.STATIC) { return false; } diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java index fc9b3f5569..9cf85bec42 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java @@ -91,7 +91,7 @@ private String getStringFromBytes() { } // If ASCII, convert each byte directly - if (kind == StringKind.ORDINARY) { + if (kind == StringKind.ORDINARY || kind == StringKind.UNEVALUATED) { var literal = new StringBuilder(); for (var aByte : get(STRING_BYTES)) { diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/extra/data/Language.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/extra/data/Language.java index b0a37b3991..d930ebb620 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/extra/data/Language.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/extra/data/Language.java @@ -70,6 +70,11 @@ public class Language extends ADataClass { */ public static final DataKey C_PLUS_PLUS_23 = KeyFactory.bool("c++23"); + /** + * True if is a C++26 variant (or later). + */ + public static final DataKey C_PLUS_PLUS_26 = KeyFactory.bool("c++26"); + /** * True if supports digraphs. */ diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java index 8f29b6be9e..cb08094295 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/ElaboratedType.java @@ -50,10 +50,6 @@ public ElaboratedType(DataStore data, Collection children) @Override public Type desugar() { - if (!hasValue(NAMED_TYPE)) { - return this; - } - return getNamedType(); } @@ -67,10 +63,6 @@ public void setNamedType(Type namedType) { @Override public String getCode(ClavaNode sourceNode, String name) { - if (!hasValue(NAMED_TYPE) || getNamedType() instanceof NullType) { - return super.getCode(sourceNode, name); - } - String code = getKeyword().getCode(); if (!code.isEmpty()) { code += " "; diff --git a/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt b/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt index fb6efbf514..18167a3213 100644 --- a/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt +++ b/ClavaWeaver/resources/clava/test/weaver/cpp/results/TypeTemplate.js.txt @@ -9,5 +9,5 @@ After setting int with array: int Original template args: int,float After second arg to double: int,double After setting with array [double, int]: double,int -After setting typedef_to_change: std::vector::const_iterator -After setting changed_typedef_type: std::vector::const_iterator +After setting typedef_to_change: std::vector::const_iterator +After setting changed_typedef_type: std::vector::const_iterator From b367fa9304a6f920f48f0536b1d9c1b66815a1df Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 16 Jul 2026 00:07:49 +0100 Subject: [PATCH 06/39] Support testing local clang-dumper builds Allow parser tests to select a local dumper without disturbing release cache assets. Restore strict resolution of referenced nodes so incomplete dumper output fails at its source. --- .../pt/up/fe/specs/clang/ClangResources.java | 27 +++++++++++++---- .../fe/specs/clang/codeparser/CodeParser.java | 3 ++ .../up/fe/specs/clang/parsers/ClavaNodes.java | 30 +++++++------------ .../clang/parsers/data/DeclDataParser.java | 2 +- .../clang/parsers/data/StmtDataParser.java | 2 +- .../specs/clang/parser/AClangAstTester.java | 4 +++ 6 files changed, 42 insertions(+), 26 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 6a03e64a6a..2b8a38f674 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -72,7 +72,9 @@ public ClangResources(CodeParser options) { public ClangFiles getClangFiles(LibcMode libcMode) { var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); - var key = libcMode.name() + "_" + useBuiltinCuda + "_" + getClangResourceFolder().getAbsolutePath(); + var dumperExecutable = options.get(CodeParser.DUMPER_EXECUTABLE).trim(); + var key = libcMode.name() + "_" + useBuiltinCuda + "_" + dumperExecutable + "_" + + getClangResourceFolder().getAbsolutePath(); var cachedFiles = CLANG_FILES_CACHE.get(key); if (cachedFiles != null) { @@ -95,10 +97,10 @@ public ClangFiles getClangFiles(LibcMode libcMode) { var ignored = lockChannel.lock()) { var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); - File clangExecutable = prepareResources(manifest); + File clangExecutable = getClangExecutable(manifest, dumperExecutable); List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); - validateTopLevelCacheFiles(clangExecutable); + validateTopLevelCacheFiles(manifest); updateLastUsedAndCleanupStaleVersions(); var newFiles = new ClangFiles(clangExecutable, builtinIncludes); @@ -115,6 +117,20 @@ public ClangFiles getClangFiles(LibcMode libcMode) { /** * @return path to the executable that was copied */ + private File getClangExecutable(ClangDumperManifest manifest, String dumperExecutable) { + if (dumperExecutable.isEmpty()) { + return prepareResources(manifest); + } + + var executable = new File(dumperExecutable).getAbsoluteFile(); + if (!executable.isFile()) { + throw new RuntimeException("Could not find custom clang-dumper executable '" + executable + "'"); + } + + SpecsLogs.info("Using custom clang-dumper executable: " + executable); + return executable; + } + private File prepareResources(ClangDumperManifest manifest) { File resourceFolder = getClangResourceFolder(); SupportedPlatform platform = SupportedPlatform.getCurrentPlatform(); @@ -471,11 +487,12 @@ private static Set listRegularFiles(File folder) { } } - private void validateTopLevelCacheFiles(File clangExecutable) { + private void validateTopLevelCacheFiles(ClangDumperManifest manifest) { File resourceFolder = getClangResourceFolder(); Set expectedNames = new HashSet<>(); expectedNames.add(ClangAstWebResource.MANIFEST_FILENAME); - expectedNames.add(clangExecutable.getName()); + var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; + expectedNames.add(getCurrentAsset(manifest, executableKind).filename()); expectedNames.add(INCLUDES_FOLDERNAME); expectedNames.add(INCLUDES_HASHES_FILENAME); expectedNames.add(INCLUDES_VALIDATED_FILENAME); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java index f7441ed77d..0d0bba4ea6 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java @@ -47,6 +47,9 @@ public abstract class CodeParser extends ADataClass { public static final DataKey DUMPER_FOLDER = KeyFactory.folder("dumperFolder") .setLabel("The base cache folder for the clang-dumper. Clava stores each dumper release in a versioned subfolder and downloads it if not found. If not set, a temporary folder will be used.") .setDefault(ClangResources::getDefaultTempFolder); + public static final DataKey DUMPER_EXECUTABLE = KeyFactory.string("dumperExecutable") + .setLabel("Optional local clang-dumper executable. Intended for development and testing; release includes are still prepared normally.") + .setDefaultString(""); /** * Execution information, such as execution time and memory used. diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java index a0cc364dd7..fc58715dd7 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java @@ -199,8 +199,11 @@ public void queueSetOptionalNode(DataClass data, DataKe Runnable nodeToAdd = () -> { + @SuppressWarnings("unchecked") Optional value = isNullId(nodeId) ? Optional.empty() - : key.getValueClass().cast(getOptional(nodeId)); + : Optional.of((T) Objects.requireNonNull(clavaNodes.get(nodeId), + () -> "Could not resolve optional node '" + nodeId + "' for key '" + key.getName() + + "'. Node data:\n" + data)); data.set(key, value); }; @@ -213,7 +216,10 @@ public void queueSetNullableNode(DataClass data, DataKe Runnable nodeToAdd = () -> { - ClavaNode value = isNullId(nodeId) ? getNullNodeType(nodeId).newNullNode(factory) : get(nodeId); + ClavaNode value = isNullId(nodeId) ? getNullNodeType(nodeId).newNullNode(factory) + : Objects.requireNonNull(clavaNodes.get(nodeId), + () -> "Could not resolve nullable node '" + nodeId + "' for key '" + key.getName() + + "'. Node data:\n" + data); data.set(key, key.getValueClass().cast(value)); }; @@ -227,24 +233,10 @@ public void queueSetNodeList(DataClass data, DataKey

  • { @SuppressWarnings("unchecked") // If the nodes exist, they should be of the requested type - List nodes = nodeIds.stream().map(id -> (T) get(id)).collect(Collectors.toList()); - - data.set(key, nodes); - }; - - queuedActions.add(nodeToAdd); - } - - public void queueSetOptionalNodeList(DataClass data, DataKey> key, - List nodeIds) { - - Runnable nodeToAdd = () -> { - - @SuppressWarnings("unchecked") // If present, the nodes should be of the requested type List nodes = nodeIds.stream() - .map(this::getOptional) - .flatMap(Optional::stream) - .map(node -> (T) node) + .map(id -> (T) Objects.requireNonNull(clavaNodes.get(id), + () -> "Could not resolve node '" + id + "' in list for key '" + key.getName() + + "'. Node data:\n" + data)) .collect(Collectors.toList()); data.set(key, nodes); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java index f7f11dc97a..b108271017 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/DeclDataParser.java @@ -87,7 +87,7 @@ public static DataStore parseDeclData(LineStream lines, ClangAstData dataStore) data.add(Decl.IS_INVALID_DECL, LineStreamParsers.oneOrZero(lines)); data.add(Decl.IS_MODULE_PRIVATE, LineStreamParsers.oneOrZero(lines)); - dataStore.getClavaNodes().queueSetOptionalNodeList(data, Decl.ATTRIBUTES, LineStreamParsers.stringList(lines)); + dataStore.getClavaNodes().queueSetNodeList(data, Decl.ATTRIBUTES, LineStreamParsers.stringList(lines)); // dataStore.getClavaNodes().queueSetOptionalNode(data, Decl.DECL_CONTEXT, lines.nextLine()); // dataStore.getClavaNodes().queueSetNodeList(data, Decl.DECL_CONTEXT, LineStreamParsers.stringList(lines)); // data.add(Decl.DECL_CONTEXT_IDS, LineStreamParsers.stringList(lines)); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java index a52881b784..43c2b2efb0 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/StmtDataParser.java @@ -64,7 +64,7 @@ public static DataStore parseGotoStmtData(LineStream lines, ClangAstData dataSto public static DataStore parseAttributedStmtData(LineStream lines, ClangAstData dataStore) { DataStore data = parseStmtData(lines, dataStore); - dataStore.getClavaNodes().queueSetOptionalNodeList(data, AttributedStmt.STMT_ATTRIBUTES, + dataStore.getClavaNodes().queueSetNodeList(data, AttributedStmt.STMT_ATTRIBUTES, LineStreamParsers.stringList(lines)); return data; diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java index eeb1c44ce1..8cc622dca5 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java @@ -92,6 +92,10 @@ public AClangAstTester(Collection resources, List comp this.outputFoldername = OUTPUT_FOLDERNAME_PREFIX + System.nanoTime() + "-" + Thread.currentThread().getId(); codeParser = CodeParser.newInstance(); + var testDumperExecutable = System.getenv("CLAVA_TEST_DUMPER_EXECUTABLE"); + if (testDumperExecutable != null && !testDumperExecutable.isBlank()) { + codeParser.set(CodeParser.DUMPER_EXECUTABLE, testDumperExecutable); + } // Set strict mode // ClangAstParser.strictMode(true); } From 5411fb4bb907dc4530d04d2f49e8ec5a8cfe8775 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 16 Jul 2026 02:50:56 +0100 Subject: [PATCH 07/39] Use canonical dumper threshold option Share the system-header threshold spelling between Clava's tool and plugin invocation paths. --- .../src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index d9c8551a49..84d85f24ae 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -53,6 +53,7 @@ public class ClangAstDumper { private final static boolean USE_PLUGIN = false; + private final static String SYSTEM_HEADER_THRESHOLD_OPTION = "-system-header-threshold="; public static boolean usePlugin() { return USE_PLUGIN; @@ -169,7 +170,7 @@ private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, arguments.add("-Xclang"); arguments.add("-plugin-arg-DumpAst"); arguments.add("-Xclang"); - arguments.add("-system-threshold=" + systemIncludesThreshold); + arguments.add(SYSTEM_HEADER_THRESHOLD_OPTION + systemIncludesThreshold); } else { arguments.add(clangExecutable.getAbsolutePath()); @@ -177,7 +178,7 @@ private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, arguments.add("-id=" + id); - arguments.add("-system-header-threshold=" + systemIncludesThreshold); + arguments.add(SYSTEM_HEADER_THRESHOLD_OPTION + systemIncludesThreshold); arguments.add("--"); } From a1f5256d3d02dc6ce9d803fd52474cf9e2ae5434 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Mon, 27 Jul 2026 15:27:21 +0100 Subject: [PATCH 08/39] Update character literal kind for LLVM 18 --- .../src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java index ee772ac786..3e4ab11745 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/enums/CharacterKind.java @@ -18,7 +18,7 @@ import pt.up.fe.specs.util.providers.StringProvider; public enum CharacterKind implements StringProvider { - ORDINARY, + ASCII, WIDE, UTF8, UTF16, From 04f367b977a74376cc18027daa580d4c71c28718 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Mon, 27 Jul 2026 15:54:30 +0100 Subject: [PATCH 09/39] Support local clang-dumper builds from tag file Treat absolute tag-file values as build directories, use their local tool or plugin without release downloads or checksum validation, and force system-provided headers. Remove the superseded executable override and cover source parsing and local tool selection. --- .../fe/specs/clang/ClangAstWebResource.java | 41 +++++++++++++-- .../src/pt/up/fe/specs/clang/ClangFiles.java | 2 +- .../pt/up/fe/specs/clang/ClangResources.java | 51 +++++++++++++------ .../fe/specs/clang/codeparser/CodeParser.java | 3 -- .../clang/codeparser/ParallelCodeParser.java | 7 +-- .../fe/specs/clang/dumper/ClangAstDumper.java | 10 ++-- .../up/fe/specs/clang/ClangResourcesTest.java | 46 +++++++++++++++++ .../specs/clang/parser/AClangAstTester.java | 4 -- 8 files changed, 130 insertions(+), 34 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index 5299cbaf1a..c48f982528 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -18,6 +18,7 @@ import pt.up.fe.specs.util.providers.WebResourceProvider; import java.io.File; +import java.nio.file.Path; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -33,19 +34,42 @@ public final class ClangAstWebResource { private ClangAstWebResource() { } - public static String getReleaseTag() { + public static DumperSource getDumperSource() { var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(RELEASE_TAG_RESOURCE); if (inputStream == null) { throw new RuntimeException("Could not find resource '" + RELEASE_TAG_RESOURCE + "'"); } - var tag = SpecsIo.read(inputStream).trim(); - if (tag.isBlank()) { + var value = SpecsIo.read(inputStream).trim(); + if (value.isBlank()) { throw new RuntimeException("Resource '" + RELEASE_TAG_RESOURCE + "' is empty"); } - return tag; + return parseDumperSource(value); + } + + static DumperSource parseDumperSource(String value) { + var path = Path.of(value); + if (path.isAbsolute()) { + return new LocalBuild(path.toFile()); + } + + if (value.contains("/") || value.contains("\\") || value.equals(".") || value.equals("..")) { + throw new RuntimeException("Relative paths are not supported in resource '" + RELEASE_TAG_RESOURCE + + "': '" + value + "'"); + } + + return new Release(value); + } + + public static String getReleaseTag() { + var source = getDumperSource(); + if (source instanceof Release release) { + return release.tag(); + } + + throw new IllegalStateException("The clang-dumper resource points to a local build"); } public static ClangDumperManifest getManifest(File resourceFolder) { @@ -69,6 +93,15 @@ private static String getReleaseBaseUrl() { return RELEASE_ROOT + getReleaseTag() + "/"; } + public sealed interface DumperSource permits Release, LocalBuild { + } + + public record Release(String tag) implements DumperSource { + } + + public record LocalBuild(File folder) implements DumperSource { + } + public record ClangDumperManifest(int schema_version, List assets) { public void validate() { diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java index 731cac065f..b11d0f347e 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java @@ -16,6 +16,6 @@ import java.io.File; import java.util.List; -public record ClangFiles(File clangExecutable, List builtinIncludes) { +public record ClangFiles(File clangExecutable, List builtinIncludes, boolean useSystemHeaders) { } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 2b8a38f674..7cd3e84e17 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -15,6 +15,8 @@ import pt.up.fe.specs.clang.ClangAstWebResource.ClangDumperManifest; import pt.up.fe.specs.clang.ClangAstWebResource.ClangDumperManifestAsset; +import pt.up.fe.specs.clang.ClangAstWebResource.LocalBuild; +import pt.up.fe.specs.clang.ClangAstWebResource.Release; import pt.up.fe.specs.clang.codeparser.CodeParser; import pt.up.fe.specs.clang.dumper.ClangAstDumper; import pt.up.fe.specs.clang.parsers.TopLevelNodesParser; @@ -71,14 +73,18 @@ public ClangResources(CodeParser options) { public ClangFiles getClangFiles(LibcMode libcMode) { + var source = ClangAstWebResource.getDumperSource(); var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); - var dumperExecutable = options.get(CodeParser.DUMPER_EXECUTABLE).trim(); - var key = libcMode.name() + "_" + useBuiltinCuda + "_" + dumperExecutable + "_" - + getClangResourceFolder().getAbsolutePath(); + var sourceKey = source instanceof Release + ? source + "_" + getClangResourceFolder().getAbsolutePath() + : source.toString(); + var key = libcMode.name() + "_" + useBuiltinCuda + "_" + sourceKey; var cachedFiles = CLANG_FILES_CACHE.get(key); if (cachedFiles != null) { - writeLastUsed(Instant.now()); + if (source instanceof Release) { + writeLastUsed(Instant.now()); + } SpecsLogs.debug(() -> "Using cached version of Clang files: " + cachedFiles); return cachedFiles; } @@ -87,23 +93,31 @@ public ClangFiles getClangFiles(LibcMode libcMode) { synchronized (jvmLock) { var files = CLANG_FILES_CACHE.get(key); if (files != null) { - writeLastUsed(Instant.now()); + if (source instanceof Release) { + writeLastUsed(Instant.now()); + } return files; } + if (source instanceof LocalBuild localBuild) { + var newFiles = new ClangFiles(getLocalExecutable(localBuild.folder()), List.of(), true); + CLANG_FILES_CACHE.put(key, newFiles); + return newFiles; + } + var lockFile = new File(getClangResourceFolder(), CACHE_LOCK_FILENAME); try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); var ignored = lockChannel.lock()) { var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); - File clangExecutable = getClangExecutable(manifest, dumperExecutable); + File clangExecutable = prepareResources(manifest); List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); validateTopLevelCacheFiles(manifest); updateLastUsedAndCleanupStaleVersions(); - var newFiles = new ClangFiles(clangExecutable, builtinIncludes); + var newFiles = new ClangFiles(clangExecutable, builtinIncludes, false); SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); CLANG_FILES_CACHE.put(key, newFiles); @@ -114,20 +128,25 @@ public ClangFiles getClangFiles(LibcMode libcMode) { } } - /** - * @return path to the executable that was copied - */ - private File getClangExecutable(ClangDumperManifest manifest, String dumperExecutable) { - if (dumperExecutable.isEmpty()) { - return prepareResources(manifest); + static File getLocalExecutable(File buildFolder) { + if (!buildFolder.isDirectory()) { + throw new RuntimeException("Local clang-dumper build directory does not exist: '" + buildFolder + "'"); + } + + String filename; + if (ClangAstDumper.usePlugin()) { + filename = System.mapLibraryName("plugin"); + } else { + filename = SupportedPlatform.getCurrentPlatform().isWindows() ? "tool.exe" : "tool"; } - var executable = new File(dumperExecutable).getAbsoluteFile(); + var executable = new File(buildFolder, filename); if (!executable.isFile()) { - throw new RuntimeException("Could not find custom clang-dumper executable '" + executable + "'"); + throw new RuntimeException("Could not find local clang-dumper " + + (ClangAstDumper.usePlugin() ? "plugin" : "tool") + " '" + executable + "'"); } - SpecsLogs.info("Using custom clang-dumper executable: " + executable); + SpecsLogs.info("Using local clang-dumper build: " + executable); return executable; } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java index 0d0bba4ea6..f7441ed77d 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/CodeParser.java @@ -47,9 +47,6 @@ public abstract class CodeParser extends ADataClass { public static final DataKey DUMPER_FOLDER = KeyFactory.folder("dumperFolder") .setLabel("The base cache folder for the clang-dumper. Clava stores each dumper release in a versioned subfolder and downloads it if not found. If not set, a temporary folder will be used.") .setDefault(ClangResources::getDefaultTempFolder); - public static final DataKey DUMPER_EXECUTABLE = KeyFactory.string("dumperExecutable") - .setLabel("Optional local clang-dumper executable. Intended for development and testing; release includes are still prepared normally.") - .setDefaultString(""); /** * Execution information, such as execution time and memory used. diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java index cf71e63bc4..6f0d7fe238 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java @@ -145,7 +145,8 @@ public App parse(List inputSources, List compilerOptions, ClavaCon Future tUnit = executor .submit(() -> parseSource(source, id, standard, options, clangDump, - counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes())); + counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes(), + clangFiles.useSystemHeaders())); futureTUnits.add(tUnit); @@ -362,7 +363,7 @@ private Standard getStandard(Collection sources, DataStore options) { private ClangAstData parseSource(File sourceFile, String id, Standard standard, DataStore options, ConcurrentLinkedQueue clangDump, ParallelProgressCounter counter, File parsingFolder, - File clangExecutable, List builtinIncludes) { + File clangExecutable, List builtinIncludes, boolean useSystemHeaders) { // ConcurrentLinkedQueue clangDump, ConcurrentLinkedQueue workingFolders) { @@ -374,7 +375,7 @@ private ClangAstData parseSource(File sourceFile, String id, Standard standard, boolean streamConsoleOutput = !get(PARALLEL_PARSING); ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes, - this) + useSystemHeaders, this) .setBaseFolder(parsingFolder) .setSystemIncludesThreshold(get(SYSTEM_INCLUDES_THRESHOLD)); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 84d85f24ae..26c20087ae 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -87,6 +87,7 @@ public static List getTempFiles() { private File baseFolder; private File clangExecutable; private List builtinIncludes; + private final boolean useSystemHeaders; private int systemIncludesThreshold; private ClangResources clangResources; @@ -103,12 +104,14 @@ public static List getTempFiles() { * @param parserConfig */ public ClangAstDumper(boolean streamConsoleOutput, - File clangExecutable, List builtinIncludes, CodeParser parserConfig) { + File clangExecutable, List builtinIncludes, boolean useSystemHeaders, + CodeParser parserConfig) { this.streamConsoleOutput = streamConsoleOutput; this.clangExecutable = clangExecutable; this.builtinIncludes = builtinIncludes; + this.useSystemHeaders = useSystemHeaders; this.workingFolders = new ArrayList<>(); this.lastWorkingFolder = null; @@ -240,7 +243,7 @@ else if (isCuda) { var cudaPath = parserConfig.get(CodeParser.CUDA_PATH); var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); - if (useBuiltinCudaLib) { + if (useBuiltinCudaLib && !useSystemHeaders) { arguments.add("-nocudainc"); arguments.add("-nocudalib"); arguments.add("-include"); @@ -268,7 +271,8 @@ else if (SourceType.isHeader(sourceFile)) { } // If it was determined that built-in includes will be used, disable system includes - if (ClangResources.useBuiltinLibc(clangExecutable, config.get(ClangAstKeys.LIBC_CXX_MODE))) { + if (!useSystemHeaders + && ClangResources.useBuiltinLibc(clangExecutable, config.get(ClangAstKeys.LIBC_CXX_MODE))) { arguments.add("-nostdinc"); arguments.add("-nostdinc++"); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index e46824d1f5..4008e33a2d 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -13,16 +13,62 @@ package pt.up.fe.specs.clang; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import pt.up.fe.specs.clang.ClangAstWebResource.LocalBuild; +import pt.up.fe.specs.clang.ClangAstWebResource.Release; import pt.up.fe.specs.clang.codeparser.CodeParser; public class ClangResourcesTest { + @TempDir + Path tempFolder; + + @Test + public void releaseTagIsParsedAsRelease() { + var release = assertInstanceOf(Release.class, ClangAstWebResource.parseDumperSource("v16.0.5_3")); + + assertEquals("v16.0.5_3", release.tag()); + } + + @Test + public void absolutePathIsParsedAsLocalBuild() { + var localBuild = assertInstanceOf(LocalBuild.class, + ClangAstWebResource.parseDumperSource(tempFolder.toString())); + + assertEquals(tempFolder.toFile(), localBuild.folder()); + } + + @Test + public void relativePathIsRejected() { + assertThrows(RuntimeException.class, + () -> ClangAstWebResource.parseDumperSource("../clang-dumper/build")); + } + + @Test + public void localBuildSelectsExpectedTool() throws IOException { + var toolName = SupportedPlatform.getCurrentPlatform().isWindows() ? "tool.exe" : "tool"; + var tool = tempFolder.resolve(toolName).toFile(); + assertTrue(tool.createNewFile()); + + assertEquals(tool, ClangResources.getLocalExecutable(tempFolder.toFile())); + } + + @Test + public void localBuildRequiresExpectedTool() { + assertThrows(RuntimeException.class, () -> ClangResources.getLocalExecutable(tempFolder.toFile())); + } + @Test public void builtinCudaIncludesAreAvailableWithSystemLibc() { var parser = CodeParser.newInstance(); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java index 8cc622dca5..eeb1c44ce1 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/AClangAstTester.java @@ -92,10 +92,6 @@ public AClangAstTester(Collection resources, List comp this.outputFoldername = OUTPUT_FOLDERNAME_PREFIX + System.nanoTime() + "-" + Thread.currentThread().getId(); codeParser = CodeParser.newInstance(); - var testDumperExecutable = System.getenv("CLAVA_TEST_DUMPER_EXECUTABLE"); - if (testDumperExecutable != null && !testDumperExecutable.isBlank()) { - codeParser.set(CodeParser.DUMPER_EXECUTABLE, testDumperExecutable); - } // Set strict mode // ClangAstParser.strictMode(true); } From a4f50ce2ac8df63c1805866bb2df73c55556fe8d Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Mon, 27 Jul 2026 16:14:38 +0100 Subject: [PATCH 10/39] Allow unresolved unary transform types --- .../up/fe/specs/clang/parsers/data/TypeDataParser.java | 2 +- .../up/fe/specs/clava/ast/type/UnaryTransformType.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/TypeDataParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/TypeDataParser.java index 2dbe71624f..48d743ff2b 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/TypeDataParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/data/TypeDataParser.java @@ -366,7 +366,7 @@ public static DataStore parseUnaryTransformTypeData(LineStream lines, ClangAstDa DataStore data = parseTypeData(lines, parserData); data.add(UnaryTransformType.KIND, LineStreamParsers.enumFromName(UnaryTransformTypeKind.class, lines)); - parserData.getClavaNodes().queueSetNode(data, UnaryTransformType.UNDERLYING_TYPE, lines.nextLine()); + parserData.getClavaNodes().queueSetOptionalNode(data, UnaryTransformType.UNDERLYING_TYPE, lines.nextLine()); parserData.getClavaNodes().queueSetNode(data, UnaryTransformType.BASE_TYPE, lines.nextLine()); return data; diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/UnaryTransformType.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/UnaryTransformType.java index 24c1773693..827d4bf60d 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/type/UnaryTransformType.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/type/UnaryTransformType.java @@ -14,6 +14,7 @@ package pt.up.fe.specs.clava.ast.type; import java.util.Collection; +import java.util.Optional; import org.suikasoft.jOptions.Datakey.DataKey; import org.suikasoft.jOptions.Datakey.KeyFactory; @@ -35,7 +36,11 @@ public class UnaryTransformType extends Type { public final static DataKey KIND = KeyFactory .enumeration("kind", UnaryTransformTypeKind.class); - public final static DataKey UNDERLYING_TYPE = KeyFactory.object("underlyingType", Type.class); + /** + * The transformed type, when Clang has resolved it. Dependent unary + * transforms can have no underlying type while still having a base type. + */ + public final static DataKey> UNDERLYING_TYPE = KeyFactory.optional("underlyingType"); public final static DataKey BASE_TYPE = KeyFactory.object("baseType", Type.class); @@ -49,7 +54,7 @@ public Type getBaseType() { return get(BASE_TYPE); } - public Type getUnderlyingType() { + public Optional getUnderlyingType() { return get(UNDERLYING_TYPE); } From 021800ae23d05e9680e2afefbb36fc99fffb8116 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Mon, 27 Jul 2026 18:06:11 +0100 Subject: [PATCH 11/39] Test source locations in ClangAstParser --- .../test-resources/cxx/source_locations.cpp | 17 ++++ .../parser/tests/SourceLocationsTest.java | 88 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 ClangAstParser/test-resources/cxx/source_locations.cpp create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/SourceLocationsTest.java diff --git a/ClangAstParser/test-resources/cxx/source_locations.cpp b/ClangAstParser/test-resources/cxx/source_locations.cpp new file mode 100644 index 0000000000..a80c933e55 --- /dev/null +++ b/ClangAstParser/test-resources/cxx/source_locations.cpp @@ -0,0 +1,17 @@ +#define VALUE 7 +#define CAT_IMPL(left, right) left##right +#define CAT(left, right) CAT_IMPL(left, right) +#define DECL(name) int name = VALUE; + +DECL(CAT(macro_, value)) +int ordinary = 0; +int foobar = 1; +int pasted_reference = CAT(foo, bar); + +namespace std { +using uint8_t = unsigned char; +template class vector; +} // namespace std + +template > +class Holder {}; diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/SourceLocationsTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/SourceLocationsTest.java new file mode 100644 index 0000000000..c9e24c6c84 --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/SourceLocationsTest.java @@ -0,0 +1,88 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang.parser.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Predicate; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import pt.up.fe.specs.clang.codeparser.CodeParser; +import pt.up.fe.specs.clava.ClavaNode; +import pt.up.fe.specs.clava.SourceRange; +import pt.up.fe.specs.clava.ast.decl.TemplateTypeParmDecl; +import pt.up.fe.specs.clava.ast.decl.VarDecl; +import pt.up.fe.specs.clava.ast.extra.App; +import pt.up.fe.specs.util.SpecsIo; +import pt.up.fe.specs.util.SpecsSystem; + +public class SourceLocationsTest { + + private static final String RESOURCE = "cxx/source_locations.cpp"; + + @TempDir + Path tempFolder; + + @Test + public void sourceLocationsUseRealFileCoordinates() { + SpecsSystem.programStandardInit(); + + File sourceFile = SpecsIo.resourceCopy(RESOURCE, tempFolder.toFile(), false, true); + App app = CodeParser.newInstance().parse(List.of(sourceFile), List.of("-std=c++11")); + + VarDecl ordinary = find(app.getDescendants(VarDecl.class), varDecl -> varDecl.getDeclName().equals("ordinary")); + assertRange(ordinary, false, 7, 1, 7, 16); + + VarDecl macro = find(app.getDescendants(VarDecl.class), varDecl -> varDecl.getDeclName().equals("macro_value")); + assertRange(macro, true, 6, 1, 6, 24); + assertRange(macro.getInit().orElseThrow(), true, 6, 1, 6, 24); + + VarDecl pastedReference = find(app.getDescendants(VarDecl.class), + varDecl -> varDecl.getDeclName().equals("pasted_reference")); + assertRange(pastedReference.getInit().orElseThrow(), true, 9, 24, 9, 36); + + TemplateTypeParmDecl templateParameter = find(app.getDescendantsAndFields(TemplateTypeParmDecl.class), + parameter -> parameter.getDeclName().equals("BinaryType")); + assertRange(templateParameter, false, 16, 11, 16, 54); + } + + private static T find(List nodes, Predicate predicate) { + return nodes.stream() + .filter(predicate) + .findFirst() + .orElseThrow(); + } + + private static void assertRange(ClavaNode node, boolean isMacro, int startLine, int startColumn, int endLine, + int endColumn) { + + SourceRange location = node.getLocation(); + + assertEquals("source_locations.cpp", location.getFilename()); + assertEquals("source_locations.cpp", Path.of(location.getStartFilepath()).getFileName().toString()); + assertEquals("source_locations.cpp", Path.of(location.getEndFilepath()).getFileName().toString()); + assertEquals(startLine, location.getStartLine()); + assertEquals(startColumn, location.getStartCol()); + assertEquals(endLine, location.getEndLine()); + assertEquals(endColumn, location.getEndCol()); + assertEquals(isMacro, node.get(ClavaNode.IS_MACRO)); + assertFalse(location.toString().contains("")); + } +} From a8b23d762514dbe12df87c03f9267d3d8f2c2f57 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 28 Jul 2026 13:28:26 +0100 Subject: [PATCH 12/39] Prevent races during stale dumper cache cleanup --- .../pt/up/fe/specs/clang/ClangResources.java | 41 +++++++++++++++---- .../up/fe/specs/clang/ClangResourcesTest.java | 29 +++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 7cd3e84e17..b0e5880e6a 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -31,6 +31,8 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -552,7 +554,7 @@ private static void writeTimestamp(File file, Instant timestamp) { SpecsIo.write(file, timestamp.toString()); } - private void deleteStaleVersions(Instant now, File currentVersionFolder) { + void deleteStaleVersions(Instant now, File currentVersionFolder) { File cacheBaseFolder = options.get(CodeParser.DUMPER_FOLDER); var versions = cacheBaseFolder.listFiles(File::isDirectory); if (versions == null) { @@ -569,18 +571,41 @@ private void deleteStaleVersions(Instant now, File currentVersionFolder) { continue; } - try { - var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); - if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { - SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); - SpecsIo.deleteFolder(versionFolder); + var lockFile = new File(versionFolder, CACHE_LOCK_FILENAME); + try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, + StandardOpenOption.WRITE)) { + + var lock = tryLock(lockChannel); + if (lock == null) { + SpecsLogs.debug(() -> "Skipping locked clang-dumper cache folder: " + versionFolder); + continue; + } + + try (lock) { + if (!lastUsedFile.isFile()) { + continue; + } + + var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); + if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { + SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); + SpecsIo.deleteFolder(versionFolder); + } } - } catch (RuntimeException e) { - SpecsLogs.warn("Could not inspect clang-dumper cache timestamp '" + lastUsedFile + "'", e); + } catch (IOException | RuntimeException e) { + SpecsLogs.warn("Could not inspect clang-dumper cache folder '" + versionFolder + "'", e); } } } + private static FileLock tryLock(FileChannel lockChannel) throws IOException { + try { + return lockChannel.tryLock(); + } catch (OverlappingFileLockException e) { + return null; + } + } + private static boolean hasExpectedSha256(File file, ClangDumperManifestAsset asset) { return asset.sha256().equalsIgnoreCase(calculateSha256(file)); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 4008e33a2d..597207278e 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -14,13 +14,19 @@ package pt.up.fe.specs.clang; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.time.Instant; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -69,6 +75,29 @@ public void localBuildRequiresExpectedTool() { assertThrows(RuntimeException.class, () -> ClangResources.getLocalExecutable(tempFolder.toFile())); } + @Test + public void staleCacheCleanupSkipsLockedVersions() throws IOException { + var currentVersion = Files.createDirectory(tempFolder.resolve("current")).toFile(); + var staleVersion = Files.createDirectory(tempFolder.resolve("stale")).toFile(); + Files.writeString(staleVersion.toPath().resolve("last-used.txt"), + Instant.now().minus(Duration.ofDays(61)).toString()); + + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + var resources = new ClangResources(parser); + + var lockFile = staleVersion.toPath().resolve(".cache.lock"); + try (var lockChannel = FileChannel.open(lockFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var ignored = lockChannel.lock()) { + + resources.deleteStaleVersions(Instant.now(), currentVersion); + assertTrue(staleVersion.isDirectory()); + } + + resources.deleteStaleVersions(Instant.now(), currentVersion); + assertFalse(staleVersion.exists()); + } + @Test public void builtinCudaIncludesAreAvailableWithSystemLibc() { var parser = CodeParser.newInstance(); From 7dd123c1cb43e5a9fa3cad72957ffec74d7df0b3 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 28 Jul 2026 15:20:32 +0100 Subject: [PATCH 13/39] Refactor bundled include handling --- .../src/pt/up/fe/specs/clang/ClangFiles.java | 2 +- .../src/pt/up/fe/specs/clang/ClangResources.java | 4 ++-- .../specs/clang/codeparser/ParallelCodeParser.java | 8 +++----- .../pt/up/fe/specs/clang/dumper/ClangAstDumper.java | 13 +++++-------- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java index b11d0f347e..731cac065f 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java @@ -16,6 +16,6 @@ import java.io.File; import java.util.List; -public record ClangFiles(File clangExecutable, List builtinIncludes, boolean useSystemHeaders) { +public record ClangFiles(File clangExecutable, List builtinIncludes) { } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index b0e5880e6a..33d2781606 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -102,7 +102,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { } if (source instanceof LocalBuild localBuild) { - var newFiles = new ClangFiles(getLocalExecutable(localBuild.folder()), List.of(), true); + var newFiles = new ClangFiles(getLocalExecutable(localBuild.folder()), List.of()); CLANG_FILES_CACHE.put(key, newFiles); return newFiles; } @@ -119,7 +119,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { validateTopLevelCacheFiles(manifest); updateLastUsedAndCleanupStaleVersions(); - var newFiles = new ClangFiles(clangExecutable, builtinIncludes, false); + var newFiles = new ClangFiles(clangExecutable, builtinIncludes); SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); CLANG_FILES_CACHE.put(key, newFiles); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java index 6f0d7fe238..d16c2ce827 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java @@ -145,8 +145,7 @@ public App parse(List inputSources, List compilerOptions, ClavaCon Future tUnit = executor .submit(() -> parseSource(source, id, standard, options, clangDump, - counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes(), - clangFiles.useSystemHeaders())); + counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes())); futureTUnits.add(tUnit); @@ -363,7 +362,7 @@ private Standard getStandard(Collection sources, DataStore options) { private ClangAstData parseSource(File sourceFile, String id, Standard standard, DataStore options, ConcurrentLinkedQueue clangDump, ParallelProgressCounter counter, File parsingFolder, - File clangExecutable, List builtinIncludes, boolean useSystemHeaders) { + File clangExecutable, List builtinIncludes) { // ConcurrentLinkedQueue clangDump, ConcurrentLinkedQueue workingFolders) { @@ -374,8 +373,7 @@ private ClangAstData parseSource(File sourceFile, String id, Standard standard, // Only show output of console after parsing is done, when using parallel parsing boolean streamConsoleOutput = !get(PARALLEL_PARSING); - ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes, - useSystemHeaders, this) + ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes, this) .setBaseFolder(parsingFolder) .setSystemIncludesThreshold(get(SYSTEM_INCLUDES_THRESHOLD)); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 26c20087ae..c5597eb3fe 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -87,7 +87,6 @@ public static List getTempFiles() { private File baseFolder; private File clangExecutable; private List builtinIncludes; - private final boolean useSystemHeaders; private int systemIncludesThreshold; private ClangResources clangResources; @@ -104,14 +103,12 @@ public static List getTempFiles() { * @param parserConfig */ public ClangAstDumper(boolean streamConsoleOutput, - File clangExecutable, List builtinIncludes, boolean useSystemHeaders, - CodeParser parserConfig) { + File clangExecutable, List builtinIncludes, CodeParser parserConfig) { this.streamConsoleOutput = streamConsoleOutput; this.clangExecutable = clangExecutable; this.builtinIncludes = builtinIncludes; - this.useSystemHeaders = useSystemHeaders; this.workingFolders = new ArrayList<>(); this.lastWorkingFolder = null; @@ -243,14 +240,14 @@ else if (isCuda) { var cudaPath = parserConfig.get(CodeParser.CUDA_PATH); var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); - if (useBuiltinCudaLib && !useSystemHeaders) { + if (useBuiltinCudaLib && !builtinIncludes.isEmpty()) { arguments.add("-nocudainc"); arguments.add("-nocudalib"); arguments.add("-include"); arguments.add("__clang_cuda_runtime_wrapper.h"); arguments.add("-include"); arguments.add(ClangAstResource.CUDA_COMPATIBILITY.write(SpecsIo.getTempFolder()).getAbsolutePath()); - } else if (!cudaPath.isBlank()) { + } else if (!useBuiltinCudaLib && !cudaPath.isBlank()) { File cudaFolder = SpecsIo.existingFolder(cudaPath); @@ -270,8 +267,8 @@ else if (SourceType.isHeader(sourceFile)) { arguments.add(standard.isCxx() ? "c++" : "c"); } - // If it was determined that built-in includes will be used, disable system includes - if (!useSystemHeaders + // Bundled include directories replace system headers when built-in libc/cxx is required. + if (!builtinIncludes.isEmpty() && ClangResources.useBuiltinLibc(clangExecutable, config.get(ClangAstKeys.LIBC_CXX_MODE))) { arguments.add("-nostdinc"); arguments.add("-nostdinc++"); From 09098bdedec4e07d73d012edb61fa5bb02cc3549 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 28 Jul 2026 18:52:32 +0100 Subject: [PATCH 14/39] Regenerate Clava-JS enum interfaces --- Clava-JS/api/Joinpoints.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Clava-JS/api/Joinpoints.ts b/Clava-JS/api/Joinpoints.ts index 659971877f..98997a143d 100644 --- a/Clava-JS/api/Joinpoints.ts +++ b/Clava-JS/api/Joinpoints.ts @@ -2811,12 +2811,12 @@ export class TypedefType extends Type { * This and the "type" declaration below. */ export const StorageClass = { - NONE: "none", - AUTO: "auto", - EXTERN: "extern", - PRIVATE_EXTERN: "private_extern", - REGISTER: "register", - STATIC: "static", + NONE: "NONE", + AUTO: "AUTO", + EXTERN: "EXTERN", + PRIVATE_EXTERN: "PRIVATE_EXTERN", + REGISTER: "REGISTER", + STATIC: "STATIC", } as const; export type StorageClass = typeof StorageClass[keyof typeof StorageClass]; @@ -2826,12 +2826,12 @@ export type StorageClass = typeof StorageClass[keyof typeof StorageClass]; * This and the "type" declaration below. */ export const Relation = { - LE: "le", - LT: "lt", - GE: "ge", - GT: "gt", - EQ: "eq", - NE: "ne", + LE: "LE", + LT: "LT", + GE: "GE", + GT: "GT", + EQ: "EQ", + NE: "NE", } as const; export type Relation = typeof Relation[keyof typeof Relation]; From 94396270729e1f87df26b60b3e03a6366b8d275a Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 17:13:01 +0100 Subject: [PATCH 15/39] Update the dumper tag --- ClangAstParser/clang-dumper-release.tag | 2 +- .../ast/decl/enums/InitializationStyle.java | 6 ++-- .../specs/clava/ast/decl/enums/Linkage.java | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/ClangAstParser/clang-dumper-release.tag b/ClangAstParser/clang-dumper-release.tag index 0e02573bb7..9acbd12ca8 100644 --- a/ClangAstParser/clang-dumper-release.tag +++ b/ClangAstParser/clang-dumper-release.tag @@ -1 +1 @@ -v16.0.5_3 +v18.1.8_1 diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java index cae31033f3..90f2f724a7 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java @@ -24,9 +24,9 @@ import pt.up.fe.specs.util.providers.StringProvider; public enum InitializationStyle implements StringProvider { - CINIT, - CALL_INIT, - LIST_INIT, + CINIT, // C-style initialization with assignment + CALL_INIT, // Call-style initialization (C++98) + LIST_INIT, // Direct list-initialization (C++11) ParenListInit; private static Lazy> ENUM_HELPER = EnumHelperWithValue diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java index 1aa999c0fd..f8c27a9319 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/Linkage.java @@ -23,10 +23,42 @@ */ public enum Linkage { Invalid, + + /** + * The entity is unique and can only be referred to from within its scope. + */ None, + + /** + * the entity can be referred to from within the translation unit, but not other translation units. + */ Internal, + + /** + * External linkage within a unique namespace. + * + *

    + * From the language perspective, these entities have external linkage. However, since they reside in an anonymous + * namespace, their names are unique to this translation unit, which is equivalent to having internal linkage from + * the code-generation point of view. + */ UniqueExternal, + + /** + * No linkage according to the standard, but is visible from other translation units because of types defined in a + * inline function. + */ VisibleNone, + + /** + * Module linkage, which indicates that the entity can be referred to from other translation units within the same + * module, and indirectly from arbitrary other translation units through inline functions and templates in the + * module interface. + */ Module, + + /** + * The entity can be referred to from other translation units. + */ External; } From 91c8dd48434e77e85c29c6d921711957369070d2 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 22:30:34 +0100 Subject: [PATCH 16/39] Fix C++20 parenthesized list initialization Add support for Clang's CXXParenListInitExpr node and preserve the original parenthesized argument list during code generation. Add an integration test covering aggregate initialization under C++20. --- .../cxx/paren_list_initialization.cpp | 8 +++ .../parser/tests/InitializationStyleTest.java | 50 +++++++++++++++++++ .../ast/decl/enums/InitializationStyle.java | 13 ++++- .../clava/ast/expr/CXXParenListInitExpr.java | 30 +++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 ClangAstParser/test-resources/cxx/paren_list_initialization.cpp create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java create mode 100644 ClavaAst/src/pt/up/fe/specs/clava/ast/expr/CXXParenListInitExpr.java diff --git a/ClangAstParser/test-resources/cxx/paren_list_initialization.cpp b/ClangAstParser/test-resources/cxx/paren_list_initialization.cpp new file mode 100644 index 0000000000..8b84bb9cfb --- /dev/null +++ b/ClangAstParser/test-resources/cxx/paren_list_initialization.cpp @@ -0,0 +1,8 @@ +struct Point { + int x; + int y; +}; + +void test() { + Point point(1, 2); +} diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java new file mode 100644 index 0000000000..2860f16b17 --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java @@ -0,0 +1,50 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang.parser.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import pt.up.fe.specs.clang.codeparser.CodeParser; +import pt.up.fe.specs.clava.ast.decl.VarDecl; +import pt.up.fe.specs.clava.ast.extra.App; +import pt.up.fe.specs.util.SpecsIo; +import pt.up.fe.specs.util.SpecsSystem; + +public class InitializationStyleTest { + + @TempDir + Path tempFolder; + + @Test + public void parenthesizedListInitializationKeepsAllArguments() { + SpecsSystem.programStandardInit(); + + File sourceFile = SpecsIo.resourceCopy("cxx/paren_list_initialization.cpp", tempFolder.toFile(), false, true); + App app = CodeParser.newInstance().parse(List.of(sourceFile), List.of("-std=c++2a")); + + VarDecl point = app.getDescendants(VarDecl.class).stream() + .filter(varDecl -> varDecl.getDeclName().equals("point")) + .findFirst() + .orElseThrow(); + + assertEquals("Point point(1, 2)", point.getCode()); + } +} diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java index 90f2f724a7..30cf723e20 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java @@ -57,8 +57,9 @@ public String getCode(VarDecl node) { case CINIT: return cinitCode(node); case CALL_INIT: - case ParenListInit: return callInitCode(node); + case ParenListInit: + return parenListInitCode(node); case LIST_INIT: return listInitCode(node); default: @@ -97,6 +98,16 @@ private String callInitCode(VarDecl node) { return "(" + init.getCode() + ")"; } + private String parenListInitCode(VarDecl node) { + Preconditions.checkArgument(node.getNumChildren() == 1, "Expected one child"); + ClavaNode init = node.getChild(0); + + Preconditions.checkArgument(init instanceof Expr, + "Expected an Expr, got '" + init.getClass().getSimpleName() + "'"); + + return init.getCode(); + } + private String listInitCode(VarDecl node) { // Must be present Expr initList = node.getInit().get(); diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/CXXParenListInitExpr.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/CXXParenListInitExpr.java new file mode 100644 index 0000000000..92f93c195e --- /dev/null +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/CXXParenListInitExpr.java @@ -0,0 +1,30 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clava.ast.expr; + +import java.util.Collection; + +import org.suikasoft.jOptions.Interfaces.DataStore; + +import pt.up.fe.specs.clava.ClavaNode; + +/** + * A C++20 parenthesized list-initialization expression. + */ +public class CXXParenListInitExpr extends ParenListExpr { + + public CXXParenListInitExpr(DataStore data, Collection children) { + super(data, children); + } +} From 93290bc5897acceb8a5940b18f0d5139f8275779 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 22:31:27 +0100 Subject: [PATCH 17/39] Synchronize clang cache initialization locks Key the in-process lock by the shared release cache lock file so concurrent libc and CUDA configurations cannot acquire different JVM locks for the same cache folder. --- .../src/pt/up/fe/specs/clang/ClangResources.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 33d2781606..deb873f1c3 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -91,7 +91,13 @@ public ClangFiles getClangFiles(LibcMode libcMode) { return cachedFiles; } - var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(key, ignored -> new Object()); + // The JVM lock must protect the same cache file as the inter-process lock. The cache key also contains the + // libc and CUDA configuration, so using it here would let two configurations acquire different JVM locks for + // the same release folder and trigger OverlappingFileLockException. + var jvmLockKey = source instanceof Release + ? new File(getClangResourceFolder(), CACHE_LOCK_FILENAME).getAbsolutePath() + : source.toString(); + var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(jvmLockKey, ignored -> new Object()); synchronized (jvmLock) { var files = CLANG_FILES_CACHE.get(key); if (files != null) { From 04969802be9273bf10101ceee32407806061d505 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 22:33:13 +0100 Subject: [PATCH 18/39] Make stale cache cleanup portable Store per-version cache locks beside the version directory so cleanup can delete the directory while holding the lock on Windows, Linux, and macOS. Update the cache cleanup test to use the shared lock-path helper. --- .../pt/up/fe/specs/clang/ClangResources.java | 18 ++++++++++++++---- .../up/fe/specs/clang/ClangResourcesTest.java | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index deb873f1c3..9c21d4b540 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -95,7 +95,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { // libc and CUDA configuration, so using it here would let two configurations acquire different JVM locks for // the same release folder and trigger OverlappingFileLockException. var jvmLockKey = source instanceof Release - ? new File(getClangResourceFolder(), CACHE_LOCK_FILENAME).getAbsolutePath() + ? getCacheLockFile(getClangResourceFolder()).getAbsolutePath() : source.toString(); var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(jvmLockKey, ignored -> new Object()); synchronized (jvmLock) { @@ -113,7 +113,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { return newFiles; } - var lockFile = new File(getClangResourceFolder(), CACHE_LOCK_FILENAME); + var lockFile = getCacheLockFile(getClangResourceFolder()); try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); var ignored = lockChannel.lock()) { @@ -524,7 +524,6 @@ private void validateTopLevelCacheFiles(ClangDumperManifest manifest) { expectedNames.add(INCLUDES_HASHES_FILENAME); expectedNames.add(INCLUDES_VALIDATED_FILENAME); expectedNames.add(LAST_USED_FILENAME); - expectedNames.add(CACHE_LOCK_FILENAME); var files = resourceFolder.listFiles(); if (files == null) { @@ -577,7 +576,7 @@ void deleteStaleVersions(Instant now, File currentVersionFolder) { continue; } - var lockFile = new File(versionFolder, CACHE_LOCK_FILENAME); + var lockFile = getCacheLockFile(versionFolder); try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { @@ -612,6 +611,17 @@ private static FileLock tryLock(FileChannel lockChannel) throws IOException { } } + /** + * Returns the lock file for a cache version. + * + *

    The lock file must be outside the version folder because stale cleanup deletes that folder while holding the + * lock. Keeping the lock inside the folder prevents deletion on Windows.

    + */ + static File getCacheLockFile(File versionFolder) { + var absoluteVersionFolder = versionFolder.getAbsoluteFile(); + return new File(absoluteVersionFolder.getParentFile(), absoluteVersionFolder.getName() + CACHE_LOCK_FILENAME); + } + private static boolean hasExpectedSha256(File file, ClangDumperManifestAsset asset) { return asset.sha256().equalsIgnoreCase(calculateSha256(file)); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 597207278e..59459a2965 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -86,8 +86,8 @@ public void staleCacheCleanupSkipsLockedVersions() throws IOException { parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); var resources = new ClangResources(parser); - var lockFile = staleVersion.toPath().resolve(".cache.lock"); - try (var lockChannel = FileChannel.open(lockFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var lockFile = ClangResources.getCacheLockFile(staleVersion); + try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); var ignored = lockChannel.lock()) { resources.deleteStaleVersions(Instant.now(), currentVersion); From cc2e57cc31a2fc3215d867f8e284ac3ebb462884 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 22:43:21 +0100 Subject: [PATCH 19/39] Support CUDA discovery for local clang-dumper builds Discover a system CUDA toolkit from portable environment, PATH, and platform-specific locations when bundled headers are unavailable. Continue with a single warning when no toolkit is found instead of rejecting local builds. --- .../pt/up/fe/specs/clang/ClangResources.java | 72 +++++++++++++++++++ .../fe/specs/clang/dumper/ClangAstDumper.java | 13 ++++ .../up/fe/specs/clang/ClangResourcesTest.java | 23 ++++++ 3 files changed, 108 insertions(+) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 9c21d4b540..adbbf93f47 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -40,17 +40,21 @@ import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.HexFormat; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.regex.Pattern; public class ClangResources { @@ -73,6 +77,74 @@ public ClangResources(CodeParser options) { this.options = options; } + public Optional getSystemCudaInstallation() { + return findCudaInstallation( + Arrays.asList(System.getenv("CUDA_HOME"), System.getenv("CUDA_PATH"), System.getenv("CUDA_ROOT")), + System.getenv("PATH"), getConventionalCudaRoots()); + } + + static Optional findCudaInstallation(List configuredRoots, String pathValue, + List conventionalRoots) { + var candidates = new ArrayList(); + + configuredRoots.stream() + .filter(root -> root != null && !root.isBlank()) + .map(File::new) + .forEach(candidates::add); + + if (pathValue != null) { + for (var pathEntry : pathValue.split(Pattern.quote(File.pathSeparator))) { + if (pathEntry.isBlank()) { + continue; + } + + var binFolder = new File(pathEntry); + for (var executableName : List.of("nvcc", "nvcc.exe")) { + var executable = new File(binFolder, executableName); + if (executable.isFile() && binFolder.getParentFile() != null) { + candidates.add(binFolder.getParentFile()); + } + } + } + } + + candidates.addAll(conventionalRoots); + + return candidates.stream() + .map(File::getAbsoluteFile) + .filter(ClangResources::isCudaInstallation) + .findFirst(); + } + + private static boolean isCudaInstallation(File folder) { + return folder.isDirectory() && new File(folder, "include/cuda_runtime.h").isFile(); + } + + private static List getConventionalCudaRoots() { + if (!SupportedPlatform.getCurrentPlatform().isWindows()) { + return List.of( + new File("/usr/local/cuda"), + new File("/opt/cuda"), + new File("/opt/homebrew/opt/cuda")); + } + + var roots = new ArrayList(); + for (var programFiles : Arrays.asList(System.getenv("ProgramFiles"), System.getenv("ProgramFiles(x86)"))) { + if (programFiles == null || programFiles.isBlank()) { + continue; + } + + var cudaRoot = new File(programFiles, "NVIDIA GPU Computing Toolkit/CUDA"); + var versions = cudaRoot.listFiles(File::isDirectory); + if (versions != null) { + Arrays.sort(versions, Comparator.comparing(File::getName).reversed()); + roots.addAll(Arrays.asList(versions)); + } + } + + return roots; + } + public ClangFiles getClangFiles(LibcMode libcMode) { var source = ClangAstWebResource.getDumperSource(); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index c5597eb3fe..f1a9fbbadc 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -44,6 +44,7 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; /** * Calls the ClangAstDumper executable and returns the dumped information. Clava AST can be built based on this output. @@ -61,6 +62,7 @@ public static boolean usePlugin() { private final static String CLANG_DUMP_FILENAME = "clangDump.txt"; private final static String STDERR_DUMP_FILENAME = "stderr.txt"; + private static final AtomicBoolean CUDA_UNAVAILABLE_WARNING_SHOWN = new AtomicBoolean(); private static final List CLANG_AST_DUMPER_TEMP_FILES = Arrays.asList("includes.txt", CLANG_DUMP_FILENAME, // "clavaDump.txt", "nodetypes.txt", "types.txt", "is_temporary.txt", "template_args.txt", @@ -247,6 +249,17 @@ else if (isCuda) { arguments.add("__clang_cuda_runtime_wrapper.h"); arguments.add("-include"); arguments.add(ClangAstResource.CUDA_COMPATIBILITY.write(SpecsIo.getTempFolder()).getAbsolutePath()); + } else if (useBuiltinCudaLib) { + clangResources.getSystemCudaInstallation().ifPresentOrElse(cudaFolder -> { + ClavaLog.debug("Setting --cuda-path to discovered CUDA folder '" + + cudaFolder.getAbsolutePath() + "'"); + arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); + }, () -> { + if (CUDA_UNAVAILABLE_WARNING_SHOWN.compareAndSet(false, true)) { + ClavaLog.warning("CUDA parsing requested, but no system CUDA toolkit was found. " + + "Continuing without CUDA headers; CUDA parsing may fail."); + } + }); } else if (!useBuiltinCudaLib && !cudaPath.isBlank()) { File cudaFolder = SpecsIo.existingFolder(cudaPath); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 59459a2965..7f27ac5af9 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -27,6 +27,7 @@ import java.nio.file.StandardOpenOption; import java.time.Duration; import java.time.Instant; +import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -75,6 +76,28 @@ public void localBuildRequiresExpectedTool() { assertThrows(RuntimeException.class, () -> ClangResources.getLocalExecutable(tempFolder.toFile())); } + @Test + public void cudaInstallationIsFoundFromConfiguredRoot() throws IOException { + var cudaFolder = Files.createDirectories(tempFolder.resolve("cuda")); + Files.createDirectories(cudaFolder.resolve("include")); + Files.createFile(cudaFolder.resolve("include/cuda_runtime.h")); + + assertEquals(cudaFolder.toFile().getAbsoluteFile(), + ClangResources.findCudaInstallation(List.of(cudaFolder.toString()), "", List.of()).orElseThrow()); + } + + @Test + public void cudaInstallationIsFoundFromNvccPath() throws IOException { + var cudaFolder = Files.createDirectories(tempFolder.resolve("cuda")); + var binFolder = Files.createDirectories(cudaFolder.resolve("bin")); + Files.createDirectories(cudaFolder.resolve("include")); + Files.createFile(cudaFolder.resolve("include/cuda_runtime.h")); + Files.createFile(binFolder.resolve("nvcc")); + + assertEquals(cudaFolder.toFile().getAbsoluteFile(), + ClangResources.findCudaInstallation(List.of(), binFolder.toString(), List.of()).orElseThrow()); + } + @Test public void staleCacheCleanupSkipsLockedVersions() throws IOException { var currentVersion = Files.createDirectory(tempFolder.resolve("current")).toFile(); From 2b06418346b33c5e80998a590e2ce0c47ae5be35 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 22:57:25 +0100 Subject: [PATCH 20/39] Remove persistent clang cache lock artifacts Coordinate cache initialization and stale cleanup with a temporary directory claim that is removed on release. Keep the claim outside version folders for Windows deletion and recover claims left by terminated processes. --- .../pt/up/fe/specs/clang/ClangResources.java | 188 ++++++++++++++---- .../up/fe/specs/clang/ClangResourcesTest.java | 37 +++- 2 files changed, 184 insertions(+), 41 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index adbbf93f47..7cbe0e673f 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -30,11 +30,11 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.nio.channels.OverlappingFileLockException; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -50,6 +50,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -65,7 +66,10 @@ public class ClangResources { private final static String INCLUDES_HASHES_FILENAME = "includes.sha256"; private final static String INCLUDES_VALIDATED_FILENAME = "includes-validated.txt"; private final static String LAST_USED_FILENAME = "last-used.txt"; - private final static String CACHE_LOCK_FILENAME = ".cache.lock"; + private final static String CACHE_LOCK_FOLDERNAME = ".cache.lock"; + private final static String CACHE_LOCK_OWNER_FILENAME = "owner"; + private final static Duration CACHE_LOCK_RETRY_INTERVAL = Duration.ofMillis(100); + private final static Duration CACHE_LOCK_STALE_MAX_AGE = Duration.ofMinutes(5); private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); private final static Duration INCLUDES_DEEP_VALIDATION_INTERVAL = Duration.ofDays(1); @@ -163,11 +167,11 @@ public ClangFiles getClangFiles(LibcMode libcMode) { return cachedFiles; } - // The JVM lock must protect the same cache file as the inter-process lock. The cache key also contains the + // The JVM lock must protect the same cache folder as the inter-process lock. The cache key also contains the // libc and CUDA configuration, so using it here would let two configurations acquire different JVM locks for - // the same release folder and trigger OverlappingFileLockException. + // the same release folder and race while initializing it. var jvmLockKey = source instanceof Release - ? getCacheLockFile(getClangResourceFolder()).getAbsolutePath() + ? getCacheLockFolder(getClangResourceFolder()).getAbsolutePath() : source.toString(); var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(jvmLockKey, ignored -> new Object()); synchronized (jvmLock) { @@ -185,10 +189,8 @@ public ClangFiles getClangFiles(LibcMode libcMode) { return newFiles; } - var lockFile = getCacheLockFile(getClangResourceFolder()); - try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, - StandardOpenOption.WRITE); - var ignored = lockChannel.lock()) { + var lockFolder = getCacheLockFolder(getClangResourceFolder()); + try (var ignored = acquireCacheLock(getClangResourceFolder())) { var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); File clangExecutable = prepareResources(manifest); @@ -203,7 +205,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { CLANG_FILES_CACHE.put(key, newFiles); return newFiles; } catch (IOException e) { - throw new UncheckedIOException("Could not lock clang-dumper cache '" + lockFile + "'", e); + throw new UncheckedIOException("Could not lock clang-dumper cache '" + lockFolder + "'", e); } } } @@ -648,26 +650,20 @@ void deleteStaleVersions(Instant now, File currentVersionFolder) { continue; } - var lockFile = getCacheLockFile(versionFolder); - try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, - StandardOpenOption.WRITE)) { - - var lock = tryLock(lockChannel); + try (var lock = tryAcquireCacheLock(versionFolder)) { if (lock == null) { SpecsLogs.debug(() -> "Skipping locked clang-dumper cache folder: " + versionFolder); continue; } - try (lock) { - if (!lastUsedFile.isFile()) { - continue; - } + if (!lastUsedFile.isFile()) { + continue; + } - var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); - if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { - SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); - SpecsIo.deleteFolder(versionFolder); - } + var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); + if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { + SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); + SpecsIo.deleteFolder(versionFolder); } } catch (IOException | RuntimeException e) { SpecsLogs.warn("Could not inspect clang-dumper cache folder '" + versionFolder + "'", e); @@ -675,23 +671,143 @@ void deleteStaleVersions(Instant now, File currentVersionFolder) { } } - private static FileLock tryLock(FileChannel lockChannel) throws IOException { - try { - return lockChannel.tryLock(); - } catch (OverlappingFileLockException e) { - return null; + static CacheLock acquireCacheLock(File versionFolder) throws IOException { + return acquireCacheLock(versionFolder, true); + } + + private static CacheLock tryAcquireCacheLock(File versionFolder) throws IOException { + return acquireCacheLock(versionFolder, false); + } + + private static CacheLock acquireCacheLock(File versionFolder, boolean wait) throws IOException { + var lockFolder = getCacheLockFolder(versionFolder); + Files.createDirectories(lockFolder.getParentFile().toPath()); + + while (true) { + try { + Files.createDirectory(lockFolder.toPath()); + } catch (FileAlreadyExistsException e) { + if (!isCacheLockStale(lockFolder)) { + if (!wait) { + return null; + } + + waitForCacheLock(); + continue; + } + + deleteCacheLock(lockFolder); + continue; + } + + var ownerFile = new File(lockFolder, CACHE_LOCK_OWNER_FILENAME); + try { + Files.writeString(ownerFile.toPath(), getProcessIdentity(), StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE); + } catch (FileAlreadyExistsException e) { + // A stale-lock recovery raced with another process that claimed this directory. + continue; + } + + return new CacheLock(lockFolder); } } /** - * Returns the lock file for a cache version. + * Returns the temporary lock folder for a cache version. * - *

    The lock file must be outside the version folder because stale cleanup deletes that folder while holding the - * lock. Keeping the lock inside the folder prevents deletion on Windows.

    + *

    The lock folder is outside the version folder because stale cleanup deletes that folder while holding the + * lock. The folder is removed when the lock is released, so normal operation leaves no lock artifact behind.

    */ - static File getCacheLockFile(File versionFolder) { + static File getCacheLockFolder(File versionFolder) { var absoluteVersionFolder = versionFolder.getAbsoluteFile(); - return new File(absoluteVersionFolder.getParentFile(), absoluteVersionFolder.getName() + CACHE_LOCK_FILENAME); + return new File(absoluteVersionFolder.getParentFile(), absoluteVersionFolder.getName() + CACHE_LOCK_FOLDERNAME); + } + + private static String getProcessIdentity() { + var process = ProcessHandle.current(); + var startTime = process.info().startInstant().map(Instant::toString).orElse(""); + return process.pid() + System.lineSeparator() + startTime; + } + + private static boolean isCacheLockStale(File lockFolder) throws IOException { + if (!lockFolder.isDirectory()) { + throw new IOException("Cache lock path is not a directory: '" + lockFolder + "'"); + } + + var ownerFile = new File(lockFolder, CACHE_LOCK_OWNER_FILENAME); + if (!ownerFile.isFile()) { + return isCacheLockOld(lockFolder); + } + + var lines = Files.readAllLines(ownerFile.toPath()); + if (lines.isEmpty()) { + return isCacheLockOld(lockFolder); + } + + try { + var pid = Long.parseLong(lines.get(0).trim()); + var process = ProcessHandle.of(pid); + if (process.isEmpty() || !process.get().isAlive()) { + return true; + } + + if (lines.size() > 1 && !lines.get(1).isBlank()) { + var processStart = process.get().info().startInstant(); + if (processStart.isPresent() && !processStart.get().toString().equals(lines.get(1).trim())) { + return true; + } + } + + return false; + } catch (NumberFormatException e) { + return isCacheLockOld(lockFolder); + } + } + + private static boolean isCacheLockOld(File lockFolder) throws IOException { + return Files.getLastModifiedTime(lockFolder.toPath()).toInstant() + .isBefore(Instant.now().minus(CACHE_LOCK_STALE_MAX_AGE)); + } + + private static void waitForCacheLock() throws IOException { + try { + Thread.sleep(CACHE_LOCK_RETRY_INTERVAL.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting for clang-dumper cache lock", e); + } + } + + private static void deleteCacheLock(File lockFolder) throws IOException { + Files.deleteIfExists(new File(lockFolder, CACHE_LOCK_OWNER_FILENAME).toPath()); + Files.deleteIfExists(lockFolder.toPath()); + } + + static final class CacheLock implements AutoCloseable { + + private final File lockFolder; + + private CacheLock(File lockFolder) { + this.lockFolder = lockFolder; + } + + @Override + public void close() { + try { + var releasedFolder = new File(lockFolder.getParentFile(), + lockFolder.getName() + ".released-" + UUID.randomUUID()); + try { + Files.move(lockFolder.toPath(), releasedFolder.toPath(), StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(lockFolder.toPath(), releasedFolder.toPath()); + } + + deleteCacheLock(releasedFolder); + } catch (IOException e) { + SpecsLogs.warn("Could not remove temporary clang-dumper cache lock '" + lockFolder + "'", e); + } + } } private static boolean hasExpectedSha256(File file, ClangDumperManifestAsset asset) { diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 7f27ac5af9..d3946cfb63 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -21,13 +21,14 @@ import java.io.File; import java.io.IOException; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -109,18 +110,44 @@ public void staleCacheCleanupSkipsLockedVersions() throws IOException { parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); var resources = new ClangResources(parser); - var lockFile = ClangResources.getCacheLockFile(staleVersion); - try (var lockChannel = FileChannel.open(lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); - var ignored = lockChannel.lock()) { + try (var ignored = ClangResources.acquireCacheLock(staleVersion)) { resources.deleteStaleVersions(Instant.now(), currentVersion); assertTrue(staleVersion.isDirectory()); } + assertFalse(ClangResources.getCacheLockFolder(staleVersion).exists()); + resources.deleteStaleVersions(Instant.now(), currentVersion); assertFalse(staleVersion.exists()); } + @Test + public void cacheLockSerializesConcurrentAcquisition() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var executor = Executors.newSingleThreadExecutor(); + ClangResources.CacheLock firstLock = ClangResources.acquireCacheLock(versionFolder); + + try { + var secondLock = executor.submit(() -> ClangResources.acquireCacheLock(versionFolder)); + assertFalse(secondLock.isDone()); + + firstLock.close(); + firstLock = null; + try (var ignored = secondLock.get(10, TimeUnit.SECONDS)) { + assertTrue(versionFolder.isDirectory()); + } + } finally { + if (firstLock != null) { + firstLock.close(); + } + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } + + assertFalse(ClangResources.getCacheLockFolder(versionFolder).exists()); + } + @Test public void builtinCudaIncludesAreAvailableWithSystemLibc() { var parser = CodeParser.newInstance(); From d0e99f963052eac2c403c03bd1dae18b99fad9bd Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 22:58:09 +0100 Subject: [PATCH 21/39] Publish CUDA compatibility header atomically Write the shared CUDA compatibility resource to a unique temporary file and install it with an atomic move, preventing concurrent Clava instances from reading a partial header. --- .../fe/specs/clang/dumper/ClangAstDumper.java | 69 ++++++++++++++++++- .../clang/dumper/ClangAstDumperTest.java | 61 ++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index f1a9fbbadc..441ef8d579 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -39,7 +39,14 @@ import pt.up.fe.specs.util.utilities.LineStream; import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -63,6 +70,7 @@ public static boolean usePlugin() { private final static String CLANG_DUMP_FILENAME = "clangDump.txt"; private final static String STDERR_DUMP_FILENAME = "stderr.txt"; private static final AtomicBoolean CUDA_UNAVAILABLE_WARNING_SHOWN = new AtomicBoolean(); + private static final Object CUDA_COMPATIBILITY_LOCK = new Object(); private static final List CLANG_AST_DUMPER_TEMP_FILES = Arrays.asList("includes.txt", CLANG_DUMP_FILENAME, // "clavaDump.txt", "nodetypes.txt", "types.txt", "is_temporary.txt", "template_args.txt", @@ -74,6 +82,65 @@ public static List getTempFiles() { return CLANG_AST_DUMPER_TEMP_FILES; } + static File getCudaCompatibilityFile(File folder) { + var destination = new File(folder, ClangAstResource.CUDA_COMPATIBILITY.getFilename()); + if (destination.isFile()) { + return destination; + } + + synchronized (CUDA_COMPATIBILITY_LOCK) { + if (destination.isFile()) { + return destination; + } + + return writeCudaCompatibilityFile(destination); + } + } + + private static File writeCudaCompatibilityFile(File destination) { + var destinationPath = destination.toPath(); + Path temporaryPath = null; + + try { + Files.createDirectories(destinationPath.getParent()); + temporaryPath = Files.createTempFile(destinationPath.getParent(), destination.getName(), ".tmp"); + + try (var inputStream = ClangAstResource.CUDA_COMPATIBILITY.toStream()) { + if (inputStream == null) { + throw new IOException("Could not load CUDA compatibility resource"); + } + + Files.copy(inputStream, temporaryPath, StandardCopyOption.REPLACE_EXISTING); + } + + try { + Files.move(temporaryPath, destinationPath, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + try { + Files.move(temporaryPath, destinationPath); + } catch (FileAlreadyExistsException ignored) { + // Another JVM installed the complete file first. + } + } catch (FileAlreadyExistsException ignored) { + // Another JVM installed the complete file first. + } + + return destination; + } catch (IOException e) { + throw new UncheckedIOException("Could not write CUDA compatibility resource '" + + destination + "'", e); + } finally { + if (temporaryPath != null) { + try { + Files.deleteIfExists(temporaryPath); + } catch (IOException e) { + SpecsLogs.warn("Could not delete temporary CUDA compatibility resource '" + + temporaryPath + "'", e); + } + } + } + } + /** * TODO: Not implemented yet *

    @@ -248,7 +315,7 @@ else if (isCuda) { arguments.add("-include"); arguments.add("__clang_cuda_runtime_wrapper.h"); arguments.add("-include"); - arguments.add(ClangAstResource.CUDA_COMPATIBILITY.write(SpecsIo.getTempFolder()).getAbsolutePath()); + arguments.add(getCudaCompatibilityFile(SpecsIo.getTempFolder()).getAbsolutePath()); } else if (useBuiltinCudaLib) { clangResources.getSystemCudaInstallation().ifPresentOrElse(cudaFolder -> { ClavaLog.debug("Setting --cuda-path to discovered CUDA folder '" diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java new file mode 100644 index 0000000000..7e0dbe140f --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java @@ -0,0 +1,61 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang.dumper; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import pt.up.fe.specs.clang.ClangAstResource; + +public class ClangAstDumperTest { + + @TempDir + Path tempFolder; + + @Test + public void cudaCompatibilityFileIsCompleteWhenRequestedConcurrently() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + var futures = new ArrayList>(); + for (int index = 0; index < 32; index++) { + futures.add(executor.submit(() -> ClangAstDumper.getCudaCompatibilityFile(tempFolder.toFile()))); + } + + Set files = new HashSet<>(); + for (var future : futures) { + files.add(future.get(10, TimeUnit.SECONDS)); + } + + var expectedFile = tempFolder.resolve(ClangAstResource.CUDA_COMPATIBILITY.getFilename()).toFile(); + assertEquals(Set.of(expectedFile), files); + assertEquals(ClangAstResource.CUDA_COMPATIBILITY.read(), Files.readString(expectedFile.toPath())); + } finally { + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } + } +} From ab0682b7811e703078f93435884d7483fc4d631e Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Thu, 30 Jul 2026 23:19:39 +0100 Subject: [PATCH 22/39] Preserve JavaScript initialization style names Restore callinit and listinit as explicit enum values because the generated TypeScript API exposes raw strings without an enum conversion. --- .../specs/clang/parser/tests/InitializationStyleTest.java | 7 +++++++ .../fe/specs/clava/ast/decl/enums/InitializationStyle.java | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java index 2860f16b17..4a6207ae8c 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java @@ -24,6 +24,7 @@ import pt.up.fe.specs.clang.codeparser.CodeParser; import pt.up.fe.specs.clava.ast.decl.VarDecl; +import pt.up.fe.specs.clava.ast.decl.enums.InitializationStyle; import pt.up.fe.specs.clava.ast.extra.App; import pt.up.fe.specs.util.SpecsIo; import pt.up.fe.specs.util.SpecsSystem; @@ -47,4 +48,10 @@ public void parenthesizedListInitializationKeepsAllArguments() { assertEquals("Point point(1, 2)", point.getCode()); } + + @Test + public void javascriptInitializationStyleNamesRemainCompatible() { + assertEquals("callinit", InitializationStyle.CALL_INIT.getString()); + assertEquals("listinit", InitializationStyle.LIST_INIT.getString()); + } } diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java index 30cf723e20..45a8b01f6e 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/decl/enums/InitializationStyle.java @@ -25,8 +25,10 @@ public enum InitializationStyle implements StringProvider { CINIT, // C-style initialization with assignment - CALL_INIT, // Call-style initialization (C++98) - LIST_INIT, // Direct list-initialization (C++11) + // Keep these explicit names for compatibility: build-interfaces exposes strings to TS, with no TS enum to convert + // them to. + CALL_INIT("callinit"), // Call-style initialization (C++98) + LIST_INIT("listinit"), // Direct list-initialization (C++11) ParenListInit; private static Lazy> ENUM_HELPER = EnumHelperWithValue From 6b3a74408ddfd2aed3aacaf59bdab3d62a38ccbf Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Fri, 31 Jul 2026 14:54:02 +0100 Subject: [PATCH 23/39] test: exercise clang dumper cache races Add same-JVM, cross-JVM, multi-version, crash-recovery, stale-cleanup, and real release initialization coverage for the clang-dumper cache. Keep red diagnostic tests for reclaimed-lock release and cached last-used updates bypassing the cache lock. --- .../up/fe/specs/clang/ClangResourcesTest.java | 481 +++++++++++++++++- 1 file changed, 480 insertions(+), 1 deletion(-) diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index d3946cfb63..69136414e8 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -16,19 +16,23 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.concurrent.ExecutorService; +import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -39,6 +43,8 @@ public class ClangResourcesTest { + private static final Duration PROCESS_TIMEOUT = Duration.ofSeconds(10); + @TempDir Path tempFolder; @@ -148,6 +154,479 @@ public void cacheLockSerializesConcurrentAcquisition() throws Exception { assertFalse(ClangResources.getCacheLockFolder(versionFolder).exists()); } + @Test + public void cacheLockSerializesAcquisitionAcrossJvms() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var firstAcquired = tempFolder.resolve("first.acquired"); + var firstRelease = tempFolder.resolve("first.release"); + var secondAcquired = tempFolder.resolve("second.acquired"); + var secondRelease = tempFolder.resolve("second.release"); + var firstLog = tempFolder.resolve("first.log"); + var secondLog = tempFolder.resolve("second.log"); + + Process first = startCacheLockProcess(versionFolder, firstAcquired, firstRelease, firstLog); + Process second = null; + try { + assertTrue(waitForFile(firstAcquired, PROCESS_TIMEOUT)); + + second = startCacheLockProcess(versionFolder, secondAcquired, secondRelease, secondLog); + assertFalse(waitForFile(secondAcquired, Duration.ofMillis(750)), + "A second JVM acquired a cache lock that was still held"); + + Files.createFile(firstRelease); + assertTrue(waitForFile(secondAcquired, PROCESS_TIMEOUT)); + + Files.createFile(secondRelease); + waitForProcess(first, firstLog); + waitForProcess(second, secondLog); + } finally { + releaseProcess(firstRelease); + releaseProcess(secondRelease); + stopProcess(first); + stopProcess(second); + } + + assertFalse(ClangResources.getCacheLockFolder(versionFolder).exists()); + } + + @Test + public void cacheLockAllowsDifferentVersionsToProceedAcrossJvms() throws Exception { + var firstVersion = Files.createDirectory(tempFolder.resolve("version-1")).toFile(); + var secondVersion = Files.createDirectory(tempFolder.resolve("version-2")).toFile(); + var firstAcquired = tempFolder.resolve("first.acquired"); + var firstRelease = tempFolder.resolve("first.release"); + var secondAcquired = tempFolder.resolve("second.acquired"); + var secondRelease = tempFolder.resolve("second.release"); + var firstLog = tempFolder.resolve("first.log"); + var secondLog = tempFolder.resolve("second.log"); + + Process first = startCacheLockProcess(firstVersion, firstAcquired, firstRelease, firstLog); + Process second = null; + try { + assertTrue(waitForFile(firstAcquired, PROCESS_TIMEOUT)); + + second = startCacheLockProcess(secondVersion, secondAcquired, secondRelease, secondLog); + assertTrue(waitForFile(secondAcquired, PROCESS_TIMEOUT), + "A different dumper version was blocked by an unrelated cache lock"); + + Files.createFile(firstRelease); + Files.createFile(secondRelease); + waitForProcess(first, firstLog); + waitForProcess(second, secondLog); + } finally { + releaseProcess(firstRelease); + releaseProcess(secondRelease); + stopProcess(first); + stopProcess(second); + } + + assertFalse(ClangResources.getCacheLockFolder(firstVersion).exists()); + assertFalse(ClangResources.getCacheLockFolder(secondVersion).exists()); + } + + @Test + public void cacheLockRecoversAfterOwningJvmIsTerminated() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var firstAcquired = tempFolder.resolve("first.acquired"); + var firstRelease = tempFolder.resolve("first.release"); + var secondAcquired = tempFolder.resolve("second.acquired"); + var secondRelease = tempFolder.resolve("second.release"); + var firstLog = tempFolder.resolve("first.log"); + var secondLog = tempFolder.resolve("second.log"); + + Process first = startCacheLockProcess(versionFolder, firstAcquired, firstRelease, firstLog); + Process second = null; + try { + assertTrue(waitForFile(firstAcquired, PROCESS_TIMEOUT)); + first.destroyForcibly(); + assertTrue(first.waitFor(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + + second = startCacheLockProcess(versionFolder, secondAcquired, secondRelease, secondLog); + assertTrue(waitForFile(secondAcquired, PROCESS_TIMEOUT), + "A cache lock left by a terminated JVM was not recovered"); + + Files.createFile(secondRelease); + waitForProcess(second, secondLog); + } finally { + releaseProcess(firstRelease); + releaseProcess(secondRelease); + stopProcess(first); + stopProcess(second); + } + + assertFalse(ClangResources.getCacheLockFolder(versionFolder).exists()); + } + + @Test + public void cacheLockDoesNotTreatLiveOwnerAsStaleWhenDirectoryIsOld() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var lockFolder = ClangResources.getCacheLockFolder(versionFolder).toPath(); + var secondAcquired = tempFolder.resolve("second.acquired"); + var secondRelease = tempFolder.resolve("second.release"); + var secondLog = tempFolder.resolve("second.log"); + + ClangResources.CacheLock first = ClangResources.acquireCacheLock(versionFolder); + Process second = null; + try { + Files.setLastModifiedTime(lockFolder, FileTime.from(Instant.now().minus(Duration.ofHours(1)))); + + second = startCacheLockProcess(versionFolder, secondAcquired, secondRelease, secondLog); + assertFalse(waitForFile(secondAcquired, Duration.ofMillis(750)), + "A live cache-lock owner was incorrectly treated as stale"); + + first.close(); + first = null; + assertTrue(waitForFile(secondAcquired, PROCESS_TIMEOUT)); + Files.createFile(secondRelease); + waitForProcess(second, secondLog); + } finally { + if (first != null) { + first.close(); + } + releaseProcess(secondRelease); + stopProcess(second); + } + + assertFalse(lockFolder.toFile().exists()); + } + + @Test + public void cacheLockUsesProcessStartTimeWhenPidIsStillAlive() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var lockFolder = ClangResources.getCacheLockFolder(versionFolder).toPath(); + Files.createDirectory(lockFolder); + assumeTrue(ProcessHandle.current().info().startInstant().isPresent(), + "The current platform does not expose process start times"); + Files.writeString(lockFolder.resolve("owner"), + ProcessHandle.current().pid() + System.lineSeparator() + Instant.EPOCH + System.lineSeparator()); + + var acquired = tempFolder.resolve("acquired"); + var release = tempFolder.resolve("release"); + var log = tempFolder.resolve("child.log"); + Process child = startCacheLockProcess(versionFolder, acquired, release, log); + try { + assertTrue(waitForFile(acquired, PROCESS_TIMEOUT), + "A lock with a reused PID and a different process start time was not recovered"); + Files.createFile(release); + waitForProcess(child, log); + } finally { + releaseProcess(release); + stopProcess(child); + } + + assertFalse(lockFolder.toFile().exists()); + } + + @Test + public void cacheLockRecoversAnOldOwnerlessLock() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var lockFolder = ClangResources.getCacheLockFolder(versionFolder).toPath(); + Files.createDirectory(lockFolder); + Files.setLastModifiedTime(lockFolder, FileTime.from(Instant.now().minus(Duration.ofHours(1)))); + + var acquired = tempFolder.resolve("acquired"); + var release = tempFolder.resolve("release"); + var log = tempFolder.resolve("child.log"); + Process child = startCacheLockProcess(versionFolder, acquired, release, log); + try { + assertTrue(waitForFile(acquired, PROCESS_TIMEOUT), "An old ownerless cache lock was not recovered"); + Files.createFile(release); + waitForProcess(child, log); + } finally { + releaseProcess(release); + stopProcess(child); + } + + assertFalse(lockFolder.toFile().exists()); + } + + @Test + public void cacheLockDoesNotStealARecentOwnerlessLock() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var lockFolder = ClangResources.getCacheLockFolder(versionFolder).toPath(); + Files.createDirectory(lockFolder); + + var acquired = tempFolder.resolve("acquired"); + var release = tempFolder.resolve("release"); + var log = tempFolder.resolve("child.log"); + Process child = startCacheLockProcess(versionFolder, acquired, release, log); + try { + assertFalse(waitForFile(acquired, Duration.ofMillis(750)), + "A lock without owner metadata was stolen before its claim was old enough"); + + Files.delete(lockFolder); + assertTrue(waitForFile(acquired, PROCESS_TIMEOUT)); + Files.createFile(release); + waitForProcess(child, log); + } finally { + releaseProcess(release); + stopProcess(child); + } + + assertFalse(lockFolder.toFile().exists()); + } + + @Test + public void staleCacheCleanupSkipsLockedVersionAcrossJvms() throws Exception { + var currentVersion = Files.createDirectory(tempFolder.resolve("current")).toFile(); + var staleVersion = Files.createDirectory(tempFolder.resolve("stale")).toFile(); + Files.writeString(staleVersion.toPath().resolve("last-used.txt"), + Instant.now().minus(Duration.ofDays(61)).toString()); + + var acquired = tempFolder.resolve("acquired"); + var release = tempFolder.resolve("release"); + var log = tempFolder.resolve("child.log"); + Process child = startCacheLockProcess(staleVersion, acquired, release, log); + try { + assertTrue(waitForFile(acquired, PROCESS_TIMEOUT)); + + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + new ClangResources(parser).deleteStaleVersions(Instant.now(), currentVersion); + assertTrue(staleVersion.isDirectory(), "Stale cleanup deleted a version locked by another JVM"); + + Files.createFile(release); + waitForProcess(child, log); + } finally { + releaseProcess(release); + stopProcess(child); + } + + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + new ClangResources(parser).deleteStaleVersions(Instant.now(), currentVersion); + assertFalse(staleVersion.exists()); + } + + @Test + public void releaseResourcesCanBeInitializedBySeparateJvms() throws Exception { + var cacheFolder = Files.createDirectory(tempFolder.resolve("cache")).toFile(); + var firstDone = tempFolder.resolve("first.done"); + var secondDone = tempFolder.resolve("second.done"); + var firstLog = tempFolder.resolve("first.log"); + var secondLog = tempFolder.resolve("second.log"); + + Process first = startResourceProcess(cacheFolder, firstDone, firstLog); + Process second = startResourceProcess(cacheFolder, secondDone, secondLog); + try { + waitForProcess(first, firstLog); + waitForProcess(second, secondLog); + } finally { + stopProcess(first); + stopProcess(second); + } + + var firstExecutable = new File(Files.readString(firstDone).trim()); + var secondExecutable = new File(Files.readString(secondDone).trim()); + assertEquals(firstExecutable.getAbsoluteFile(), secondExecutable.getAbsoluteFile()); + assertTrue(firstExecutable.isFile()); + assertFalse(cacheFolder.toPath().resolve(ClangAstWebResource.getReleaseTag() + ".cache.lock").toFile().exists()); + } + + @Test + public void cacheLockReleaseDoesNotRemoveAReclaimedLock() throws Exception { + var versionFolder = Files.createDirectory(tempFolder.resolve("version")).toFile(); + var lockFolder = ClangResources.getCacheLockFolder(versionFolder).toPath(); + ClangResources.CacheLock first = ClangResources.acquireCacheLock(versionFolder); + + Files.writeString(lockFolder.resolve("owner"), Long.MAX_VALUE + System.lineSeparator()); + ClangResources.CacheLock second = ClangResources.acquireCacheLock(versionFolder); + + try { + first.close(); + assertTrue(Files.isDirectory(lockFolder), + "A stale lock owner released and deleted a replacement owner's lock"); + } finally { + second.close(); + } + } + + @Test + public void cachedUseDoesNotBypassTheCacheLockBeforeUpdatingLastUsed() throws Exception { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + var resources = new ClangResources(parser); + var versionFolder = resources.getClangResourceFolder(); + var fakeExecutable = Files.createFile(tempFolder.resolve("fake-tool")).toFile(); + var cache = getClangFilesCache(); + var cacheKey = getClangFilesCacheKey(resources, LibcMode.SYSTEM); + var cachedFiles = new ClangFiles(fakeExecutable, List.of()); + cache.put(cacheKey, cachedFiles); + + ClangResources.CacheLock lock = ClangResources.acquireCacheLock(versionFolder); + var executor = Executors.newSingleThreadExecutor(); + try { + var future = executor.submit(() -> resources.getClangFiles(LibcMode.SYSTEM)); + assertThrows(TimeoutException.class, () -> future.get(750, TimeUnit.MILLISECONDS), + "A cached use updated last-used without coordinating with the cache lock"); + + lock.close(); + lock = null; + assertSame(cachedFiles, future.get(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + } finally { + if (lock != null) { + lock.close(); + } + executor.shutdownNow(); + executor.awaitTermination(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + cache.remove(cacheKey); + } + } + + @Test + public void sameJvmInstancesShareReleaseCacheInitialization() throws Exception { + var firstParser = CodeParser.newInstance(); + firstParser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + firstParser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); + + var secondParser = CodeParser.newInstance(); + secondParser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + secondParser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); + + var thirdParser = CodeParser.newInstance(); + thirdParser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + thirdParser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); + + var executor = Executors.newFixedThreadPool(3); + try { + var first = executor.submit(() -> new ClangResources(firstParser).getClangFiles(LibcMode.SYSTEM)); + var second = executor.submit(() -> new ClangResources(secondParser).getClangFiles(LibcMode.SYSTEM)); + var third = executor.submit( + () -> new ClangResources(thirdParser).getClangFiles(LibcMode.BUILTIN_AND_LIBC)); + + var firstFiles = first.get(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + var secondFiles = second.get(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + var thirdFiles = third.get(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + + assertSame(firstFiles, secondFiles, "Same-JVM instances did not share the ClangFiles cache entry"); + assertEquals(firstFiles.clangExecutable().getAbsoluteFile(), thirdFiles.clangExecutable().getAbsoluteFile()); + assertTrue(firstFiles.clangExecutable().isFile()); + } finally { + executor.shutdownNow(); + executor.awaitTermination(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } + } + + private Process startCacheLockProcess(File versionFolder, Path acquired, Path release, Path log) + throws IOException { + + var javaExecutable = Path.of(System.getProperty("java.home"), "bin", + SupportedPlatform.getCurrentPlatform().isWindows() ? "java.exe" : "java"); + + return new ProcessBuilder( + javaExecutable.toString(), + "-cp", + System.getProperty("java.class.path"), + CacheLockProcess.class.getName(), + versionFolder.getAbsolutePath(), + acquired.toAbsolutePath().toString(), + release.toAbsolutePath().toString()) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + } + + private Process startResourceProcess(File cacheFolder, Path done, Path log) throws IOException { + var javaExecutable = Path.of(System.getProperty("java.home"), "bin", + SupportedPlatform.getCurrentPlatform().isWindows() ? "java.exe" : "java"); + + return new ProcessBuilder( + javaExecutable.toString(), + "-cp", + System.getProperty("java.class.path"), + CacheLockProcess.class.getName(), + "resources", + cacheFolder.getAbsolutePath(), + done.toAbsolutePath().toString()) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + } + + @SuppressWarnings("unchecked") + private Map getClangFilesCache() throws ReflectiveOperationException { + var cacheField = ClangResources.class.getDeclaredField("CLANG_FILES_CACHE"); + cacheField.setAccessible(true); + return (Map) cacheField.get(null); + } + + private String getClangFilesCacheKey(ClangResources resources, LibcMode libcMode) { + var source = ClangAstWebResource.getDumperSource(); + var sourceKey = source instanceof Release + ? source + "_" + resources.getClangResourceFolder().getAbsolutePath() + : source.toString(); + return libcMode.name() + "_false_" + sourceKey; + } + + private boolean waitForFile(Path file, Duration timeout) throws InterruptedException { + var deadline = System.nanoTime() + timeout.toNanos(); + do { + if (Files.isRegularFile(file)) { + return true; + } + + Thread.sleep(10); + } while (System.nanoTime() < deadline); + + return Files.isRegularFile(file); + } + + private void waitForProcess(Process process, Path log) throws Exception { + assertTrue(process.waitFor(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS), + () -> "Child JVM did not finish. Output: " + readLog(log)); + assertEquals(0, process.exitValue(), () -> "Child JVM failed. Output: " + readLog(log)); + } + + private String readLog(Path log) { + try { + return Files.readString(log); + } catch (IOException e) { + return ""; + } + } + + private void releaseProcess(Path release) throws IOException { + if (release != null && !Files.exists(release)) { + Files.createFile(release); + } + } + + private void stopProcess(Process process) throws InterruptedException { + if (process == null || !process.isAlive()) { + return; + } + + process.destroyForcibly(); + process.waitFor(PROCESS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } + + public static final class CacheLockProcess { + + private CacheLockProcess() { + } + + public static void main(String[] args) throws Exception { + if (args[0].equals("resources")) { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, new File(args[1])); + var clangFiles = new ClangResources(parser).getClangFiles(LibcMode.SYSTEM); + Files.writeString(Path.of(args[2]), clangFiles.clangExecutable().getAbsolutePath()); + return; + } + + var versionFolder = new File(args[0]); + var acquired = Path.of(args[1]); + var release = Path.of(args[2]); + + try (var ignored = ClangResources.acquireCacheLock(versionFolder)) { + Files.writeString(acquired, Long.toString(ProcessHandle.current().pid())); + while (!Files.exists(release)) { + Thread.sleep(10); + } + } + } + } + @Test public void builtinCudaIncludesAreAvailableWithSystemLibc() { var parser = CodeParser.newInstance(); From 09861edd1cfa1e338c68e7bbe82330270f5a6077 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Fri, 31 Jul 2026 15:04:50 +0100 Subject: [PATCH 24/39] test: cover stale cleanup of cached dumper paths Add a cross-JVM diagnostic proving stale cleanup can remove an executable still referenced by the in-process ClangFiles cache. --- .../up/fe/specs/clang/ClangResourcesTest.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 69136414e8..d4bdcface3 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -473,6 +473,35 @@ public void cachedUseDoesNotBypassTheCacheLockBeforeUpdatingLastUsed() throws Ex } } + @Test + public void staleCleanupDoesNotDeleteAnExecutableStillCachedInThisJvm() throws Exception { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + var resources = new ClangResources(parser); + var versionFolder = resources.getClangResourceFolder(); + var fakeExecutable = Files.createFile(versionFolder.toPath().resolve("fake-tool")).toFile(); + Files.writeString(versionFolder.toPath().resolve("last-used.txt"), + Instant.now().minus(Duration.ofDays(61)).toString()); + + var cache = getClangFilesCache(); + var cacheKey = getClangFilesCacheKey(resources, LibcMode.SYSTEM); + cache.put(cacheKey, new ClangFiles(fakeExecutable, List.of())); + + var currentVersion = Files.createDirectory(tempFolder.resolve("current")); + var done = tempFolder.resolve("cleanup.done"); + var log = tempFolder.resolve("cleanup.log"); + Process cleanup = startCleanupProcess(tempFolder.toFile(), currentVersion.toFile(), done, log); + try { + waitForProcess(cleanup, log); + } finally { + stopProcess(cleanup); + cache.remove(cacheKey); + } + + assertTrue(fakeExecutable.isFile(), + "A separate JVM's stale cleanup deleted an executable still held by this JVM's cache"); + } + @Test public void sameJvmInstancesShareReleaseCacheInitialization() throws Exception { var firstParser = CodeParser.newInstance(); @@ -543,6 +572,26 @@ private Process startResourceProcess(File cacheFolder, Path done, Path log) thro .start(); } + private Process startCleanupProcess(File cacheFolder, File currentVersion, Path done, Path log) + throws IOException { + + var javaExecutable = Path.of(System.getProperty("java.home"), "bin", + SupportedPlatform.getCurrentPlatform().isWindows() ? "java.exe" : "java"); + + return new ProcessBuilder( + javaExecutable.toString(), + "-cp", + System.getProperty("java.class.path"), + CacheLockProcess.class.getName(), + "cleanup", + cacheFolder.getAbsolutePath(), + currentVersion.getAbsolutePath(), + done.toAbsolutePath().toString()) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + } + @SuppressWarnings("unchecked") private Map getClangFilesCache() throws ReflectiveOperationException { var cacheField = ClangResources.class.getDeclaredField("CLANG_FILES_CACHE"); @@ -614,6 +663,14 @@ public static void main(String[] args) throws Exception { return; } + if (args[0].equals("cleanup")) { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, new File(args[1])); + new ClangResources(parser).deleteStaleVersions(Instant.now(), new File(args[2])); + Files.writeString(Path.of(args[3]), "done"); + return; + } + var versionFolder = new File(args[0]); var acquired = Path.of(args[1]); var release = Path.of(args[2]); From 8cbbcfaf302b77c165cdd051c38acf045d9c016e Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Fri, 31 Jul 2026 16:24:56 +0100 Subject: [PATCH 25/39] fix: coordinate cached clang resource heartbeat updates --- .../pt/up/fe/specs/clang/ClangResources.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 7cbe0e673f..dff0041aea 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -158,15 +158,6 @@ public ClangFiles getClangFiles(LibcMode libcMode) { : source.toString(); var key = libcMode.name() + "_" + useBuiltinCuda + "_" + sourceKey; - var cachedFiles = CLANG_FILES_CACHE.get(key); - if (cachedFiles != null) { - if (source instanceof Release) { - writeLastUsed(Instant.now()); - } - SpecsLogs.debug(() -> "Using cached version of Clang files: " + cachedFiles); - return cachedFiles; - } - // The JVM lock must protect the same cache folder as the inter-process lock. The cache key also contains the // libc and CUDA configuration, so using it here would let two configurations acquire different JVM locks for // the same release folder and race while initializing it. @@ -178,8 +169,16 @@ public ClangFiles getClangFiles(LibcMode libcMode) { var files = CLANG_FILES_CACHE.get(key); if (files != null) { if (source instanceof Release) { - writeLastUsed(Instant.now()); + var resourceFolder = getClangResourceFolder(); + try (var ignored = acquireCacheLock(resourceFolder)) { + writeLastUsed(Instant.now()); + } catch (IOException e) { + var lockFolder = getCacheLockFolder(resourceFolder); + throw new UncheckedIOException("Could not lock clang-dumper cache '" + lockFolder + "'", e); + } } + + SpecsLogs.debug(() -> "Using cached version of Clang files: " + files); return files; } From fb14c490fa86b1c26e27148fa13402228188b763 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Sat, 1 Aug 2026 00:51:20 +0100 Subject: [PATCH 26/39] Avoid hashing cached clang-dumper includes --- .../pt/up/fe/specs/clang/ClangResources.java | 158 +----------------- .../up/fe/specs/clang/ClangResourcesTest.java | 19 +++ 2 files changed, 26 insertions(+), 151 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index dff0041aea..3919ae1a25 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -43,7 +43,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.HexFormat; import java.util.List; @@ -54,7 +53,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.stream.Stream; import java.util.regex.Pattern; public class ClangResources { @@ -63,15 +61,12 @@ public class ClangResources { private static final Map CLANG_FILES_LOCKS = new ConcurrentHashMap<>(); private final static String CLANG_FOLDERNAME = "clang_ast_exe"; private final static String INCLUDES_FOLDERNAME = "includes"; - private final static String INCLUDES_HASHES_FILENAME = "includes.sha256"; - private final static String INCLUDES_VALIDATED_FILENAME = "includes-validated.txt"; private final static String LAST_USED_FILENAME = "last-used.txt"; private final static String CACHE_LOCK_FOLDERNAME = ".cache.lock"; private final static String CACHE_LOCK_OWNER_FILENAME = "owner"; private final static Duration CACHE_LOCK_RETRY_INTERVAL = Duration.ofMillis(100); private final static Duration CACHE_LOCK_STALE_MAX_AGE = Duration.ofMinutes(5); private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); - private final static Duration INCLUDES_DEEP_VALIDATION_INTERVAL = Duration.ofDays(1); private static final AtomicInteger HAS_LIBC = new AtomicInteger(-1); @@ -370,10 +365,8 @@ private File prepareIncludesFolder(ClangDumperManifest manifest) { File resourceFolder = getClangResourceFolder(); var includesAsset = getCurrentAsset(manifest, "includes"); var extractedFolder = new File(resourceFolder, INCLUDES_FOLDERNAME); - var includesHashesFile = new File(resourceFolder, INCLUDES_HASHES_FILENAME); - var includesValidatedFile = new File(resourceFolder, INCLUDES_VALIDATED_FILENAME); - if (isIncludesCacheValid(extractedFolder, includesHashesFile, includesValidatedFile, includesAsset)) { + if (isIncludesCacheValid(extractedFolder)) { return extractedFolder; } @@ -383,8 +376,6 @@ private File prepareIncludesFolder(ClangDumperManifest manifest) { SpecsIo.mkdir(extractedFolder); SpecsIo.deleteFolderContents(extractedFolder); SpecsIo.extractZip(zipFile.getFile(), extractedFolder); - writeIncludesHashes(extractedFolder, includesHashesFile, includesAsset); - writeTimestamp(includesValidatedFile, Instant.now()); } finally { SpecsIo.delete(zipFile.getFile()); } @@ -414,6 +405,10 @@ private ResourceWriteData downloadAsset(ClangDumperManifestAsset asset, File res var resource = ClangAstWebResource.getAssetResource(asset); var writeData = resource.writeVersioned(resourceFolder, ClangResources.class); + if (!writeData.isNewFile()) { + return writeData; + } + if (!hasExpectedSha256(writeData.getFile(), asset)) { SpecsLogs.info("Downloaded clang-dumper asset '" + asset.filename() + "' does not match the expected checksum, downloading it again."); @@ -435,14 +430,9 @@ private ClangDumperManifestAsset getCurrentAsset(ClangDumperManifest manifest, S return manifest.getAsset(platform, arch, kind); } - private boolean isIncludesCacheValid(File includesFolder, File includesHashesFile, File includesValidatedFile, - ClangDumperManifestAsset includesAsset) { - - if (!includesFolder.isDirectory() || !includesHashesFile.isFile()) { - return false; - } + static boolean isIncludesCacheValid(File includesFolder) { - if (!hasExpectedIncludesAssetHash(includesHashesFile, includesAsset)) { + if (!includesFolder.isDirectory()) { return false; } @@ -452,141 +442,9 @@ private boolean isIncludesCacheValid(File includesFolder, File includesHashesFil return false; } - if (!shouldRunIncludesDeepValidation(includesValidatedFile)) { - return true; - } - - var validationStart = Instant.now(); - SpecsLogs.info("Validating clang-dumper includes cache: " + includesFolder); - if (isIncludesCacheDeepValid(includesFolder, includesHashesFile, includesAsset)) { - writeTimestamp(includesValidatedFile, Instant.now()); - SpecsLogs.info("Validated clang-dumper includes cache in " - + Duration.between(validationStart, Instant.now()).toMillis() + " ms"); - return true; - } - - SpecsLogs.info("Invalidating clang-dumper includes cache metadata: " + includesHashesFile); - if (includesHashesFile.isFile()) { - SpecsIo.delete(includesHashesFile); - } - if (includesValidatedFile.isFile()) { - SpecsIo.delete(includesValidatedFile); - } - return false; - } - - private static boolean shouldRunIncludesDeepValidation(File includesValidatedFile) { - if (!includesValidatedFile.isFile()) { - return true; - } - - try { - var lastValidated = Instant.parse(SpecsIo.read(includesValidatedFile).trim()); - return lastValidated.isBefore(Instant.now().minus(INCLUDES_DEEP_VALIDATION_INTERVAL)); - } catch (RuntimeException e) { - return true; - } - } - - private boolean isIncludesCacheDeepValid(File includesFolder, File includesHashesFile, - ClangDumperManifestAsset includesAsset) { - - var expectedHashes = readIncludesHashes(includesHashesFile, includesAsset); - if (expectedHashes == null) { - return false; - } - - var expectedFiles = expectedHashes.keySet(); - var actualFiles = listRegularFiles(includesFolder); - - if (!actualFiles.equals(expectedFiles)) { - SpecsLogs.info("Cached clang-dumper includes contain missing or extra files, extracting them again."); - return false; - } - - for (var entry : expectedHashes.entrySet()) { - var file = new File(includesFolder, entry.getKey()); - if (!entry.getValue().equalsIgnoreCase(calculateSha256(file))) { - SpecsLogs.info("Cached clang-dumper include file '" + file - + "' does not match the expected checksum, extracting includes again."); - return false; - } - } - return true; } - private static boolean hasExpectedIncludesAssetHash(File includesHashesFile, - ClangDumperManifestAsset includesAsset) { - - try (var reader = Files.newBufferedReader(includesHashesFile.toPath())) { - return ("# clang-dumper-includes " + includesAsset.sha256()).equals(reader.readLine()); - } catch (IOException e) { - return false; - } - } - - private static Map readIncludesHashes(File includesHashesFile, - ClangDumperManifestAsset includesAsset) { - - var lines = SpecsIo.read(includesHashesFile).lines().toList(); - if (lines.size() < 2 || !lines.get(0).equals("# clang-dumper-includes " + includesAsset.sha256())) { - return null; - } - - Map hashes = new HashMap<>(); - for (int i = 1; i < lines.size(); i++) { - var line = lines.get(i); - if (line.isBlank()) { - continue; - } - - if (line.length() <= 65 || line.charAt(64) != ' ') { - return null; - } - - var hash = line.substring(0, 64); - var relativePath = line.substring(65); - hashes.put(relativePath, hash); - } - - return hashes; - } - - private static void writeIncludesHashes(File includesFolder, File includesHashesFile, - ClangDumperManifestAsset includesAsset) { - - var actualFiles = listRegularFiles(includesFolder).stream() - .sorted() - .toList(); - - StringBuilder hashes = new StringBuilder(); - hashes.append("# clang-dumper-includes ").append(includesAsset.sha256()).append(System.lineSeparator()); - - for (var relativePath : actualFiles) { - var file = new File(includesFolder, relativePath); - hashes.append(calculateSha256(file)) - .append(' ') - .append(relativePath) - .append(System.lineSeparator()); - } - - SpecsIo.write(includesHashesFile, hashes.toString()); - } - - private static Set listRegularFiles(File folder) { - try (Stream paths = Files.walk(folder.toPath())) { - return paths - .filter(Files::isRegularFile) - .map(folder.toPath()::relativize) - .map(Path::toString) - .map(path -> path.replace(File.separatorChar, '/')) - .collect(Collectors.toCollection(HashSet::new)); - } catch (IOException e) { - throw new UncheckedIOException("Could not list files in folder '" + folder + "'", e); - } - } - private void validateTopLevelCacheFiles(ClangDumperManifest manifest) { File resourceFolder = getClangResourceFolder(); Set expectedNames = new HashSet<>(); @@ -594,8 +452,6 @@ private void validateTopLevelCacheFiles(ClangDumperManifest manifest) { var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; expectedNames.add(getCurrentAsset(manifest, executableKind).filename()); expectedNames.add(INCLUDES_FOLDERNAME); - expectedNames.add(INCLUDES_HASHES_FILENAME); - expectedNames.add(INCLUDES_VALIDATED_FILENAME); expectedNames.add(LAST_USED_FILENAME); var files = resourceFolder.listFiles(); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index d4bdcface3..ffa8d53ffd 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -83,6 +83,25 @@ public void localBuildRequiresExpectedTool() { assertThrows(RuntimeException.class, () -> ClangResources.getLocalExecutable(tempFolder.toFile())); } + @Test + public void includesCacheValidationOnlyChecksRequiredFiles() throws IOException { + var includesFolder = tempFolder.resolve("includes"); + assertFalse(ClangResources.isIncludesCacheValid(includesFolder.toFile())); + + Files.createDirectory(includesFolder); + assertFalse(ClangResources.isIncludesCacheValid(includesFolder.toFile())); + + Files.createDirectories(includesFolder.resolve("builtin")); + Files.writeString(includesFolder.resolve("entrypoints.txt"), "builtin\n"); + Files.writeString(includesFolder.resolve("builtin/header.h"), "original"); + Files.writeString(includesFolder.resolve("unexpected.txt"), "extra"); + + assertTrue(ClangResources.isIncludesCacheValid(includesFolder.toFile())); + + Files.writeString(includesFolder.resolve("builtin/header.h"), "modified"); + assertTrue(ClangResources.isIncludesCacheValid(includesFolder.toFile())); + } + @Test public void cudaInstallationIsFoundFromConfiguredRoot() throws IOException { var cudaFolder = Files.createDirectories(tempFolder.resolve("cuda")); From b320fd412176de6b0a2e0270ca4cb7f5c4477efd Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Sat, 1 Aug 2026 01:29:54 +0100 Subject: [PATCH 27/39] fix: simplify clang cache coordination --- .../pt/up/fe/specs/clang/ClangResources.java | 250 +++++++++++------- .../up/fe/specs/clang/ClangResourcesTest.java | 59 +---- 2 files changed, 150 insertions(+), 159 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 3919ae1a25..75d7535331 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -16,7 +16,6 @@ import pt.up.fe.specs.clang.ClangAstWebResource.ClangDumperManifest; import pt.up.fe.specs.clang.ClangAstWebResource.ClangDumperManifestAsset; import pt.up.fe.specs.clang.ClangAstWebResource.LocalBuild; -import pt.up.fe.specs.clang.ClangAstWebResource.Release; import pt.up.fe.specs.clang.codeparser.CodeParser; import pt.up.fe.specs.clang.dumper.ClangAstDumper; import pt.up.fe.specs.clang.parsers.TopLevelNodesParser; @@ -30,11 +29,11 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -63,7 +62,7 @@ public class ClangResources { private final static String INCLUDES_FOLDERNAME = "includes"; private final static String LAST_USED_FILENAME = "last-used.txt"; private final static String CACHE_LOCK_FOLDERNAME = ".cache.lock"; - private final static String CACHE_LOCK_OWNER_FILENAME = "owner"; + private final static String CACHE_LOCK_OWNER_PREFIX = "owner-"; private final static Duration CACHE_LOCK_RETRY_INTERVAL = Duration.ofMillis(100); private final static Duration CACHE_LOCK_STALE_MAX_AGE = Duration.ofMinutes(5); private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); @@ -148,50 +147,37 @@ public ClangFiles getClangFiles(LibcMode libcMode) { var source = ClangAstWebResource.getDumperSource(); var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); - var sourceKey = source instanceof Release - ? source + "_" + getClangResourceFolder().getAbsolutePath() - : source.toString(); - var key = libcMode.name() + "_" + useBuiltinCuda + "_" + sourceKey; - - // The JVM lock must protect the same cache folder as the inter-process lock. The cache key also contains the - // libc and CUDA configuration, so using it here would let two configurations acquire different JVM locks for - // the same release folder and race while initializing it. - var jvmLockKey = source instanceof Release - ? getCacheLockFolder(getClangResourceFolder()).getAbsolutePath() - : source.toString(); - var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(jvmLockKey, ignored -> new Object()); - synchronized (jvmLock) { - var files = CLANG_FILES_CACHE.get(key); - if (files != null) { - if (source instanceof Release) { - var resourceFolder = getClangResourceFolder(); - try (var ignored = acquireCacheLock(resourceFolder)) { - writeLastUsed(Instant.now()); - } catch (IOException e) { - var lockFolder = getCacheLockFolder(resourceFolder); - throw new UncheckedIOException("Could not lock clang-dumper cache '" + lockFolder + "'", e); - } - } - SpecsLogs.debug(() -> "Using cached version of Clang files: " + files); - return files; - } + if (source instanceof LocalBuild localBuild) { + var key = libcMode.name() + "_" + useBuiltinCuda + "_" + source; + return CLANG_FILES_CACHE.computeIfAbsent(key, + ignored -> new ClangFiles(getLocalExecutable(localBuild.folder()), List.of())); + } - if (source instanceof LocalBuild localBuild) { - var newFiles = new ClangFiles(getLocalExecutable(localBuild.folder()), List.of()); - CLANG_FILES_CACHE.put(key, newFiles); - return newFiles; - } + var resourceFolder = getClangResourceFolder(); + var key = libcMode.name() + "_" + useBuiltinCuda + "_" + source + "_" + + resourceFolder.getAbsolutePath(); + var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(resourceFolder.getAbsolutePath(), ignored -> new Object()); + synchronized (jvmLock) { + var lockFolder = getCacheLockFolder(resourceFolder); + try (var ignored = acquireCacheLock(resourceFolder)) { + var files = CLANG_FILES_CACHE.get(key); + if (files != null) { + if (files.clangExecutable().isFile()) { + writeLastUsed(resourceFolder, Instant.now()); + SpecsLogs.debug(() -> "Using cached version of Clang files: " + files); + return files; + } - var lockFolder = getCacheLockFolder(getClangResourceFolder()); - try (var ignored = acquireCacheLock(getClangResourceFolder())) { + CLANG_FILES_CACHE.remove(key, files); + } - var manifest = ClangAstWebResource.getManifest(getClangResourceFolder()); - File clangExecutable = prepareResources(manifest); - List builtinIncludes = prepareIncludes(manifest, clangExecutable, libcMode); + var manifest = ClangAstWebResource.getManifest(resourceFolder); + File clangExecutable = prepareResources(manifest, resourceFolder); + List builtinIncludes = prepareIncludes(manifest, resourceFolder, clangExecutable, libcMode); - validateTopLevelCacheFiles(manifest); - updateLastUsedAndCleanupStaleVersions(); + validateTopLevelCacheFiles(manifest, resourceFolder); + updateLastUsedAndCleanupStaleVersions(resourceFolder); var newFiles = new ClangFiles(clangExecutable, builtinIncludes); SpecsLogs.debug(() -> "Using downloaded version of Clang files: " + newFiles); @@ -226,8 +212,7 @@ static File getLocalExecutable(File buildFolder) { return executable; } - private File prepareResources(ClangDumperManifest manifest) { - File resourceFolder = getClangResourceFolder(); + private File prepareResources(ClangDumperManifest manifest, File resourceFolder) { SupportedPlatform platform = SupportedPlatform.getCurrentPlatform(); var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; @@ -342,7 +327,8 @@ private static ProcessOutputAsString runClangAstDumper(File clangExecutable, Fil return SpecsSystem.runProcess(arguments, true, false); } - private List prepareIncludes(ClangDumperManifest manifest, File clangExecutable, LibcMode libcMode) { + private List prepareIncludes(ClangDumperManifest manifest, File resourceFolder, File clangExecutable, + LibcMode libcMode) { var useBuiltinLibc = useBuiltinLibc(clangExecutable, libcMode); var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); @@ -350,19 +336,18 @@ private List prepareIncludes(ClangDumperManifest manifest, File clangExe return List.of(); } - return prepareIncludes(manifest); + return prepareIncludes(manifest, resourceFolder); } - private List prepareIncludes(ClangDumperManifest manifest) { - var extractedFolder = prepareIncludesFolder(manifest); + private List prepareIncludes(ClangDumperManifest manifest, File resourceFolder) { + var extractedFolder = prepareIncludesFolder(manifest, resourceFolder); var includeFolders = getIncludeFolders(extractedFolder); SpecsLogs.debug(() -> "Includes folders: " + includeFolders); return includeFolders.stream().map(File::getAbsolutePath).toList(); } - private File prepareIncludesFolder(ClangDumperManifest manifest) { - File resourceFolder = getClangResourceFolder(); + private File prepareIncludesFolder(ClangDumperManifest manifest, File resourceFolder) { var includesAsset = getCurrentAsset(manifest, "includes"); var extractedFolder = new File(resourceFolder, INCLUDES_FOLDERNAME); @@ -445,8 +430,7 @@ static boolean isIncludesCacheValid(File includesFolder) { return true; } - private void validateTopLevelCacheFiles(ClangDumperManifest manifest) { - File resourceFolder = getClangResourceFolder(); + private void validateTopLevelCacheFiles(ClangDumperManifest manifest, File resourceFolder) { Set expectedNames = new HashSet<>(); expectedNames.add(ClangAstWebResource.MANIFEST_FILENAME); var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; @@ -469,10 +453,9 @@ private void validateTopLevelCacheFiles(ClangDumperManifest manifest) { } } - private void updateLastUsedAndCleanupStaleVersions() { + private void updateLastUsedAndCleanupStaleVersions(File resourceFolder) { var now = Instant.now(); - File resourceFolder = getClangResourceFolder(); - writeLastUsed(now); + writeLastUsed(resourceFolder, now); var staleCleanup = new Thread(() -> deleteStaleVersions(now, resourceFolder), "clang-dumper-stale-cache-cleanup"); @@ -480,8 +463,8 @@ private void updateLastUsedAndCleanupStaleVersions() { staleCleanup.start(); } - private void writeLastUsed(Instant timestamp) { - writeTimestamp(new File(getClangResourceFolder(), LAST_USED_FILENAME), timestamp); + private static void writeLastUsed(File resourceFolder, Instant timestamp) { + writeTimestamp(new File(resourceFolder, LAST_USED_FILENAME), timestamp); } private static void writeTimestamp(File file, Instant timestamp) { @@ -496,29 +479,34 @@ void deleteStaleVersions(Instant now, File currentVersionFolder) { } for (var versionFolder : versions) { - if (versionFolder.equals(currentVersionFolder)) { + if (versionFolder.getAbsoluteFile().equals(currentVersionFolder.getAbsoluteFile())) { continue; } - var lastUsedFile = new File(versionFolder, LAST_USED_FILENAME); - if (!lastUsedFile.isFile()) { - continue; - } - - try (var lock = tryAcquireCacheLock(versionFolder)) { - if (lock == null) { - SpecsLogs.debug(() -> "Skipping locked clang-dumper cache folder: " + versionFolder); - continue; - } - - if (!lastUsedFile.isFile()) { - continue; - } + var jvmLock = CLANG_FILES_LOCKS.computeIfAbsent(versionFolder.getAbsolutePath(), ignored -> new Object()); + try { + synchronized (jvmLock) { + var lastUsedFile = new File(versionFolder, LAST_USED_FILENAME); + if (!lastUsedFile.isFile()) { + continue; + } - var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); - if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { - SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); - SpecsIo.deleteFolder(versionFolder); + try (var lock = tryAcquireCacheLock(versionFolder)) { + if (lock == null) { + SpecsLogs.debug(() -> "Skipping locked clang-dumper cache folder: " + versionFolder); + continue; + } + + if (!lastUsedFile.isFile()) { + continue; + } + + var lastUsed = Instant.parse(SpecsIo.read(lastUsedFile).trim()); + if (lastUsed.isBefore(now.minus(STALE_CACHE_MAX_AGE))) { + SpecsLogs.info("Deleting stale clang-dumper cache folder: " + versionFolder); + SpecsIo.deleteFolder(versionFolder); + } + } } } catch (IOException | RuntimeException e) { SpecsLogs.warn("Could not inspect clang-dumper cache folder '" + versionFolder + "'", e); @@ -551,20 +539,23 @@ private static CacheLock acquireCacheLock(File versionFolder, boolean wait) thro continue; } - deleteCacheLock(lockFolder); + recoverStaleCacheLock(lockFolder); continue; } - var ownerFile = new File(lockFolder, CACHE_LOCK_OWNER_FILENAME); + var ownerFile = new File(lockFolder, CACHE_LOCK_OWNER_PREFIX + UUID.randomUUID()); try { Files.writeString(ownerFile.toPath(), getProcessIdentity(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); - } catch (FileAlreadyExistsException e) { - // A stale-lock recovery raced with another process that claimed this directory. + } catch (NoSuchFileException e) { + // Stale-lock recovery removed the directory while this process was claiming it. continue; + } catch (IOException e) { + deleteEmptyCacheLock(lockFolder); + throw e; } - return new CacheLock(lockFolder); + return new CacheLock(lockFolder, ownerFile); } } @@ -586,16 +577,44 @@ private static String getProcessIdentity() { } private static boolean isCacheLockStale(File lockFolder) throws IOException { + if (!lockFolder.exists()) { + return true; + } + if (!lockFolder.isDirectory()) { throw new IOException("Cache lock path is not a directory: '" + lockFolder + "'"); } - var ownerFile = new File(lockFolder, CACHE_LOCK_OWNER_FILENAME); - if (!ownerFile.isFile()) { + var ownerFiles = lockFolder.listFiles(File::isFile); + if (ownerFiles == null) { + if (!lockFolder.exists()) { + return true; + } + + throw new IOException("Could not list cache lock folder: '" + lockFolder + "'"); + } + + if (ownerFiles.length == 0) { return isCacheLockOld(lockFolder); } - var lines = Files.readAllLines(ownerFile.toPath()); + for (var ownerFile : ownerFiles) { + if (!isCacheLockOwnerStale(lockFolder, ownerFile)) { + return false; + } + } + + return true; + } + + private static boolean isCacheLockOwnerStale(File lockFolder, File ownerFile) throws IOException { + List lines; + try { + lines = Files.readAllLines(ownerFile.toPath()); + } catch (NoSuchFileException e) { + return true; + } + if (lines.isEmpty()) { return isCacheLockOld(lockFolder); } @@ -621,8 +640,12 @@ private static boolean isCacheLockStale(File lockFolder) throws IOException { } private static boolean isCacheLockOld(File lockFolder) throws IOException { - return Files.getLastModifiedTime(lockFolder.toPath()).toInstant() - .isBefore(Instant.now().minus(CACHE_LOCK_STALE_MAX_AGE)); + try { + return Files.getLastModifiedTime(lockFolder.toPath()).toInstant() + .isBefore(Instant.now().minus(CACHE_LOCK_STALE_MAX_AGE)); + } catch (NoSuchFileException e) { + return true; + } } private static void waitForCacheLock() throws IOException { @@ -634,31 +657,56 @@ private static void waitForCacheLock() throws IOException { } } - private static void deleteCacheLock(File lockFolder) throws IOException { - Files.deleteIfExists(new File(lockFolder, CACHE_LOCK_OWNER_FILENAME).toPath()); - Files.deleteIfExists(lockFolder.toPath()); + private static void recoverStaleCacheLock(File lockFolder) throws IOException { + if (!lockFolder.isDirectory()) { + return; + } + + var ownerFiles = lockFolder.listFiles(File::isFile); + if (ownerFiles == null) { + return; + } + + for (var ownerFile : ownerFiles) { + if (isCacheLockOwnerStale(lockFolder, ownerFile)) { + Files.deleteIfExists(ownerFile.toPath()); + } + } + + deleteEmptyCacheLock(lockFolder); + } + + private static void deleteEmptyCacheLock(File lockFolder) throws IOException { + try { + Files.deleteIfExists(lockFolder.toPath()); + } catch (DirectoryNotEmptyException e) { + // A replacement owner claimed the lock while stale recovery was in progress. + } } + /** + * A temporary claim on one cache version. Each claim has its own owner marker, so releasing an old claim cannot + * remove a newer claim created after stale-lock recovery. + */ static final class CacheLock implements AutoCloseable { private final File lockFolder; + private final File ownerFile; - private CacheLock(File lockFolder) { + private CacheLock(File lockFolder, File ownerFile) { this.lockFolder = lockFolder; + this.ownerFile = ownerFile; + } + + File ownerFile() { + return ownerFile; } @Override public void close() { try { - var releasedFolder = new File(lockFolder.getParentFile(), - lockFolder.getName() + ".released-" + UUID.randomUUID()); - try { - Files.move(lockFolder.toPath(), releasedFolder.toPath(), StandardCopyOption.ATOMIC_MOVE); - } catch (AtomicMoveNotSupportedException e) { - Files.move(lockFolder.toPath(), releasedFolder.toPath()); - } - - deleteCacheLock(releasedFolder); + Files.deleteIfExists(ownerFile.toPath()); + deleteEmptyCacheLock(lockFolder); } catch (IOException e) { SpecsLogs.warn("Could not remove temporary clang-dumper cache lock '" + lockFolder + "'", e); } diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index ffa8d53ffd..03d8f9d579 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -448,7 +448,7 @@ public void cacheLockReleaseDoesNotRemoveAReclaimedLock() throws Exception { var lockFolder = ClangResources.getCacheLockFolder(versionFolder).toPath(); ClangResources.CacheLock first = ClangResources.acquireCacheLock(versionFolder); - Files.writeString(lockFolder.resolve("owner"), Long.MAX_VALUE + System.lineSeparator()); + Files.writeString(first.ownerFile().toPath(), Long.MAX_VALUE + System.lineSeparator()); ClangResources.CacheLock second = ClangResources.acquireCacheLock(versionFolder); try { @@ -492,35 +492,6 @@ public void cachedUseDoesNotBypassTheCacheLockBeforeUpdatingLastUsed() throws Ex } } - @Test - public void staleCleanupDoesNotDeleteAnExecutableStillCachedInThisJvm() throws Exception { - var parser = CodeParser.newInstance(); - parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); - var resources = new ClangResources(parser); - var versionFolder = resources.getClangResourceFolder(); - var fakeExecutable = Files.createFile(versionFolder.toPath().resolve("fake-tool")).toFile(); - Files.writeString(versionFolder.toPath().resolve("last-used.txt"), - Instant.now().minus(Duration.ofDays(61)).toString()); - - var cache = getClangFilesCache(); - var cacheKey = getClangFilesCacheKey(resources, LibcMode.SYSTEM); - cache.put(cacheKey, new ClangFiles(fakeExecutable, List.of())); - - var currentVersion = Files.createDirectory(tempFolder.resolve("current")); - var done = tempFolder.resolve("cleanup.done"); - var log = tempFolder.resolve("cleanup.log"); - Process cleanup = startCleanupProcess(tempFolder.toFile(), currentVersion.toFile(), done, log); - try { - waitForProcess(cleanup, log); - } finally { - stopProcess(cleanup); - cache.remove(cacheKey); - } - - assertTrue(fakeExecutable.isFile(), - "A separate JVM's stale cleanup deleted an executable still held by this JVM's cache"); - } - @Test public void sameJvmInstancesShareReleaseCacheInitialization() throws Exception { var firstParser = CodeParser.newInstance(); @@ -591,26 +562,6 @@ private Process startResourceProcess(File cacheFolder, Path done, Path log) thro .start(); } - private Process startCleanupProcess(File cacheFolder, File currentVersion, Path done, Path log) - throws IOException { - - var javaExecutable = Path.of(System.getProperty("java.home"), "bin", - SupportedPlatform.getCurrentPlatform().isWindows() ? "java.exe" : "java"); - - return new ProcessBuilder( - javaExecutable.toString(), - "-cp", - System.getProperty("java.class.path"), - CacheLockProcess.class.getName(), - "cleanup", - cacheFolder.getAbsolutePath(), - currentVersion.getAbsolutePath(), - done.toAbsolutePath().toString()) - .redirectErrorStream(true) - .redirectOutput(log.toFile()) - .start(); - } - @SuppressWarnings("unchecked") private Map getClangFilesCache() throws ReflectiveOperationException { var cacheField = ClangResources.class.getDeclaredField("CLANG_FILES_CACHE"); @@ -682,14 +633,6 @@ public static void main(String[] args) throws Exception { return; } - if (args[0].equals("cleanup")) { - var parser = CodeParser.newInstance(); - parser.set(CodeParser.DUMPER_FOLDER, new File(args[1])); - new ClangResources(parser).deleteStaleVersions(Instant.now(), new File(args[2])); - Files.writeString(Path.of(args[3]), "done"); - return; - } - var versionFolder = new File(args[0]); var acquired = Path.of(args[1]); var release = Path.of(args[2]); From 4eeb7948d3a62e6cf7a8d131aa5f73d40a0e5c01 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 14:31:53 +0100 Subject: [PATCH 28/39] fix: use system libc for local clang-dumper builds --- ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java | 6 +++++- ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java | 4 ++-- .../src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java | 5 ++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java index ec17d8a8d6..ad90e67c91 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java @@ -16,6 +16,7 @@ import org.suikasoft.jOptions.Datakey.DataKey; import org.suikasoft.jOptions.Datakey.KeyFactory; import org.suikasoft.jOptions.Interfaces.DataStore; +import pt.up.fe.specs.clang.ClangAstWebResource.LocalBuild; import pt.up.fe.specs.clava.ClavaLog; import pt.up.fe.specs.clava.ClavaOptions; import pt.up.fe.specs.clava.language.Standard; @@ -32,7 +33,10 @@ public interface ClangAstKeys { */ DataKey LIBC_CXX_MODE = KeyFactory.enumeration("libcCxxMode", LibcMode.class) .setLabel("Libc/Libc++ mode. builtin (default): uses built-in libc/libc++; system: uses includes available in the system; auto: detects if the built-in includes are needed") - .setDefault(() -> LibcMode.BUILTIN_AND_LIBC); + .setDefault(() -> LibcMode.BUILTIN_AND_LIBC) + .setCustomGetter((mode, ignored) -> ClangAstWebResource.getDumperSource() instanceof LocalBuild + ? LibcMode.SYSTEM + : mode); DataKey USES_CILK = KeyFactory.bool("usesCilk"); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 75d7535331..e57aab8002 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -146,14 +146,14 @@ private static List getConventionalCudaRoots() { public ClangFiles getClangFiles(LibcMode libcMode) { var source = ClangAstWebResource.getDumperSource(); - var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); if (source instanceof LocalBuild localBuild) { - var key = libcMode.name() + "_" + useBuiltinCuda + "_" + source; + var key = source.toString(); return CLANG_FILES_CACHE.computeIfAbsent(key, ignored -> new ClangFiles(getLocalExecutable(localBuild.folder()), List.of())); } + var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); var resourceFolder = getClangResourceFolder(); var key = libcMode.name() + "_" + useBuiltinCuda + "_" + source + "_" + resourceFolder.getAbsolutePath(); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 441ef8d579..21b5ed4156 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -347,9 +347,8 @@ else if (SourceType.isHeader(sourceFile)) { arguments.add(standard.isCxx() ? "c++" : "c"); } - // Bundled include directories replace system headers when built-in libc/cxx is required. - if (!builtinIncludes.isEmpty() - && ClangResources.useBuiltinLibc(clangExecutable, config.get(ClangAstKeys.LIBC_CXX_MODE))) { + // If it was determined that built-in includes will be used, disable system includes + if (ClangResources.useBuiltinLibc(clangExecutable, config.get(ClangAstKeys.LIBC_CXX_MODE))) { arguments.add("-nostdinc"); arguments.add("-nostdinc++"); } From d0a596ebda5cfefa40bcb8fefc798c0bdacf13ed Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 14:57:32 +0100 Subject: [PATCH 29/39] refactor: simplify clang dumper resource setup --- .../pt/up/fe/specs/clang/ClangAstKeys.java | 9 +- .../fe/specs/clang/ClangAstWebResource.java | 27 ++++- .../pt/up/fe/specs/clang/ClangResources.java | 111 +++++++----------- .../fe/specs/clang/dumper/ClangAstDumper.java | 8 +- .../up/fe/specs/clang/ClangResourcesTest.java | 18 +++ 5 files changed, 90 insertions(+), 83 deletions(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java index ad90e67c91..1fef8afdf7 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstKeys.java @@ -33,10 +33,7 @@ public interface ClangAstKeys { */ DataKey LIBC_CXX_MODE = KeyFactory.enumeration("libcCxxMode", LibcMode.class) .setLabel("Libc/Libc++ mode. builtin (default): uses built-in libc/libc++; system: uses includes available in the system; auto: detects if the built-in includes are needed") - .setDefault(() -> LibcMode.BUILTIN_AND_LIBC) - .setCustomGetter((mode, ignored) -> ClangAstWebResource.getDumperSource() instanceof LocalBuild - ? LibcMode.SYSTEM - : mode); + .setDefault(() -> LibcMode.BUILTIN_AND_LIBC); DataKey USES_CILK = KeyFactory.bool("usesCilk"); @@ -102,6 +99,10 @@ static DataStore toDataStore(List flags) { config.add(ClavaOptions.FLAGS_LIST, parsedFlags); + if (ClangAstWebResource.getDumperSource() instanceof LocalBuild) { + config.set(ClangAstKeys.LIBC_CXX_MODE, LibcMode.SYSTEM); + } + return config; } diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index c48f982528..f1fc8775dc 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -18,6 +18,8 @@ import pt.up.fe.specs.util.providers.WebResourceProvider; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.List; import java.util.Objects; @@ -30,18 +32,29 @@ public final class ClangAstWebResource { public static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; private static final Gson GSON = new Gson(); + private static final DumperSource DUMPER_SOURCE = readDumperSource(); private ClangAstWebResource() { } public static DumperSource getDumperSource() { + return DUMPER_SOURCE; + } + + private static DumperSource readDumperSource() { var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(RELEASE_TAG_RESOURCE); if (inputStream == null) { throw new RuntimeException("Could not find resource '" + RELEASE_TAG_RESOURCE + "'"); } - var value = SpecsIo.read(inputStream).trim(); + String value; + try (inputStream) { + value = SpecsIo.read(inputStream).trim(); + } catch (IOException e) { + throw new UncheckedIOException("Could not read resource '" + RELEASE_TAG_RESOURCE + "'", e); + } + if (value.isBlank()) { throw new RuntimeException("Resource '" + RELEASE_TAG_RESOURCE + "' is empty"); } @@ -73,7 +86,9 @@ public static String getReleaseTag() { } public static ClangDumperManifest getManifest(File resourceFolder) { - var manifestResource = WebResourceProvider.newInstance(getReleaseBaseUrl(), MANIFEST_FILENAME, getReleaseTag()); + var releaseTag = getReleaseTag(); + var manifestResource = WebResourceProvider.newInstance(getReleaseBaseUrl(releaseTag), MANIFEST_FILENAME, + releaseTag); var manifestFile = manifestResource.writeVersioned(resourceFolder, ClangAstWebResource.class).getFile(); var manifest = GSON.fromJson(SpecsIo.read(manifestFile), ClangDumperManifest.class); @@ -86,11 +101,13 @@ public static ClangDumperManifest getManifest(File resourceFolder) { } public static WebResourceProvider getAssetResource(ClangDumperManifestAsset asset) { - return WebResourceProvider.newInstance(getReleaseBaseUrl(), asset.filename(), getReleaseTag() + "-" + asset.sha256()); + var releaseTag = getReleaseTag(); + return WebResourceProvider.newInstance(getReleaseBaseUrl(releaseTag), asset.filename(), + releaseTag + "-" + asset.sha256()); } - private static String getReleaseBaseUrl() { - return RELEASE_ROOT + getReleaseTag() + "/"; + private static String getReleaseBaseUrl(String releaseTag) { + return RELEASE_ROOT + releaseTag + "/"; } public sealed interface DumperSource permits Release, LocalBuild { diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index e57aab8002..52e504eee8 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.FileAlreadyExistsException; @@ -35,6 +36,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Duration; @@ -42,7 +44,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import java.util.HashSet; import java.util.HexFormat; import java.util.List; import java.util.Map; @@ -50,8 +51,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import java.util.regex.Pattern; public class ClangResources { @@ -67,7 +66,7 @@ public class ClangResources { private final static Duration CACHE_LOCK_STALE_MAX_AGE = Duration.ofMinutes(5); private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); - private static final AtomicInteger HAS_LIBC = new AtomicInteger(-1); + private static final Map HAS_LIBC = new ConcurrentHashMap<>(); private final CodeParser options; @@ -148,9 +147,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) { var source = ClangAstWebResource.getDumperSource(); if (source instanceof LocalBuild localBuild) { - var key = source.toString(); - return CLANG_FILES_CACHE.computeIfAbsent(key, - ignored -> new ClangFiles(getLocalExecutable(localBuild.folder()), List.of())); + return new ClangFiles(getLocalExecutable(localBuild.folder()), List.of()); } var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption()); @@ -265,65 +262,49 @@ public static boolean useBuiltinLibc(File clangExecutable, LibcMode libcMode) { } private static boolean hasLibC(File clangExecutable) { - var value = HAS_LIBC.get(); - - if (value == -1) { - var hasLibC = detectLibC(clangExecutable); - value = hasLibC ? 1 : 0; - HAS_LIBC.set(value); - } - - if (value == 0) { - return false; - } - - if (value == 1) { - return true; - } - - throw new RuntimeException("Unexpected value: '" + value + "'"); + var executableKey = SpecsIo.getCanonicalPath(clangExecutable); + return HAS_LIBC.computeIfAbsent(executableKey, ignored -> detectLibC(clangExecutable)); } private static boolean detectLibC(File clangExecutable) { - File clangTest = SpecsIo.mkdir(SpecsIo.getTempFolder(), "clang_ast_test"); + File clangTest = SpecsIo.getTempFolder("clang_ast_test_" + UUID.randomUUID()); - List testFiles = Arrays.asList(ClangAstResource.TEST_INCLUDES_C, ClangAstResource.TEST_INCLUDES_CPP) - .stream() - .map(resource -> resource.write(clangTest)) - .collect(Collectors.toList()); - - boolean needsLib = false; - for (File testFile : testFiles) { - var output = runClangAstDumper(clangExecutable, testFile); - - if (output.getReturnValue() != 0) { - ClavaLog.info("Problems while running dumper to test if libc/libcxx is needed"); - needsLib = true; - break; - } + try { + var testFiles = List.of( + ClangAstResource.TEST_INCLUDES_C.write(clangTest), + ClangAstResource.TEST_INCLUDES_CPP.write(clangTest)); + + boolean needsLib = false; + for (var testFile : testFiles) { + var output = runClangAstDumper(clangExecutable, testFile); + + if (output.getReturnValue() != 0) { + ClavaLog.info("Problems while running dumper to test if libc/libcxx is needed"); + needsLib = true; + break; + } - if (!testFile.getName().endsWith(".cpp")) { - continue; + if (testFile.getName().endsWith(".cpp") + && !output.getOutput().contains(TopLevelNodesParser.getTopLevelNodesHeader())) { + needsLib = true; + break; + } } - var topLevelNodesHeader = TopLevelNodesParser.getTopLevelNodesHeader(); - if (!output.getOutput().contains(topLevelNodesHeader)) { - needsLib = true; - break; + if (needsLib) { + ClavaLog.debug("Could not find system libc/libcxx"); + } else { + ClavaLog.debug("Detected system's libc and libcxx"); } - } - if (needsLib) { - ClavaLog.debug("Could not find system libc/libcxx"); - } else { - ClavaLog.debug("Detected system's libc and libcxx"); + return !needsLib; + } finally { + SpecsIo.deleteFolder(clangTest); } - - return !needsLib; } private static ProcessOutputAsString runClangAstDumper(File clangExecutable, File testFile) { - List arguments = Arrays.asList(clangExecutable.getAbsolutePath(), testFile.getAbsolutePath(), "--"); + List arguments = List.of(clangExecutable.getAbsolutePath(), testFile.getAbsolutePath(), "--"); return SpecsSystem.runProcess(arguments, true, false); } @@ -431,12 +412,12 @@ static boolean isIncludesCacheValid(File includesFolder) { } private void validateTopLevelCacheFiles(ClangDumperManifest manifest, File resourceFolder) { - Set expectedNames = new HashSet<>(); - expectedNames.add(ClangAstWebResource.MANIFEST_FILENAME); var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool"; - expectedNames.add(getCurrentAsset(manifest, executableKind).filename()); - expectedNames.add(INCLUDES_FOLDERNAME); - expectedNames.add(LAST_USED_FILENAME); + Set expectedNames = Set.of( + ClangAstWebResource.MANIFEST_FILENAME, + getCurrentAsset(manifest, executableKind).filename(), + INCLUDES_FOLDERNAME, + LAST_USED_FILENAME); var files = resourceFolder.listFiles(); if (files == null) { @@ -720,18 +701,8 @@ private static boolean hasExpectedSha256(File file, ClangDumperManifestAsset ass private static String calculateSha256(File file) { try { var digest = MessageDigest.getInstance("SHA-256"); - try (var inputStream = Files.newInputStream(file.toPath())) { - inputStream.transferTo(new java.io.OutputStream() { - @Override - public void write(int b) { - digest.update((byte) b); - } - - @Override - public void write(byte[] b, int off, int len) { - digest.update(b, off, len); - } - }); + try (var inputStream = new DigestInputStream(Files.newInputStream(file.toPath()), digest)) { + inputStream.transferTo(OutputStream.nullOutputStream()); } return HexFormat.of().formatHex(digest.digest()); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 21b5ed4156..fae30774ec 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -72,7 +72,7 @@ public static boolean usePlugin() { private static final AtomicBoolean CUDA_UNAVAILABLE_WARNING_SHOWN = new AtomicBoolean(); private static final Object CUDA_COMPATIBILITY_LOCK = new Object(); - private static final List CLANG_AST_DUMPER_TEMP_FILES = Arrays.asList("includes.txt", CLANG_DUMP_FILENAME, + private static final List CLANG_AST_DUMPER_TEMP_FILES = List.of("includes.txt", CLANG_DUMP_FILENAME, // "clavaDump.txt", "nodetypes.txt", "types.txt", "is_temporary.txt", "template_args.txt", "clavaDump.txt", "nodetypes.txt", "types.txt", "is_temporary.txt", "omp.txt", "invalid_source.txt", "enum_integer_type.txt", "consumer_order.txt", @@ -157,7 +157,7 @@ private static File writeCudaCompatibilityFile(File destination) { private File clangExecutable; private List builtinIncludes; private int systemIncludesThreshold; - private ClangResources clangResources; + private final ClangResources clangResources; private final CodeParser parserConfig; @@ -392,7 +392,7 @@ else if (SourceType.isHeader(sourceFile)) { workingFolders.add(lastWorkingFolder); output = SpecsSystem.runProcess(arguments, lastWorkingFolder, - inputStream -> this.processOutput(sourceFile, inputStream), + this::processOutput, inputStream -> this.processStdErr(inputStream, config.get(ClavaNode.CONTEXT))); if (output.isError()) { @@ -429,7 +429,7 @@ else if (SourceType.isHeader(sourceFile)) { return parsedData; } - private String processOutput(File sourceFile, InputStream inputStream) { + private String processOutput(InputStream inputStream) { StringBuilder output = new StringBuilder(); try (LineStream lines = LineStream.newInstance(inputStream, null)) { diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 03d8f9d579..8dd92e73cf 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -40,6 +40,7 @@ import pt.up.fe.specs.clang.ClangAstWebResource.LocalBuild; import pt.up.fe.specs.clang.ClangAstWebResource.Release; import pt.up.fe.specs.clang.codeparser.CodeParser; +import pt.up.fe.specs.clang.parsers.TopLevelNodesParser; public class ClangResourcesTest { @@ -658,4 +659,21 @@ public void builtinCudaIncludesAreAvailableWithSystemLibc() { assertTrue(hasCudaWrapper, "Built-in CUDA must provide Clang's CUDA runtime wrapper independently of libc"); } + + @Test + public void libcDetectionIsScopedToTheExecutable() throws IOException { + assumeTrue(!SupportedPlatform.getCurrentPlatform().isWindows(), "Shell fixtures require a Unix executable"); + + var systemLibcDumper = tempFolder.resolve("system-libc-dumper"); + Files.writeString(systemLibcDumper, + "#!/bin/sh\nprintf '%s\\n' '" + TopLevelNodesParser.getTopLevelNodesHeader() + "'\n"); + assertTrue(systemLibcDumper.toFile().setExecutable(true)); + + var builtinLibcDumper = tempFolder.resolve("builtin-libc-dumper"); + Files.writeString(builtinLibcDumper, "#!/bin/sh\nexit 1\n"); + assertTrue(builtinLibcDumper.toFile().setExecutable(true)); + + assertFalse(ClangResources.useBuiltinLibc(systemLibcDumper.toFile(), LibcMode.AUTO)); + assertTrue(ClangResources.useBuiltinLibc(builtinLibcDumper.toFile(), LibcMode.AUTO)); + } } From 5a09c94135ce09af20e7e4343595fd47406df5b5 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 15:39:08 +0100 Subject: [PATCH 30/39] fix: preserve unevaluated string literal spelling Add a C++26 round-trip regression fixture covering ordinary text, simple escapes, and UCNs. Preserve Clang's source spelling for unevaluated strings because their byte payload cannot be safely re-emitted with numeric escapes, and recognize the c++2c standard used by the test. --- .../test-resources/cxx/unevaluated_strings.cpp | 3 +++ .../test-resources/cxx/unevaluated_strings.cpp.txt | 3 +++ .../pt/up/fe/specs/clang/parser/tests/CxxTest.java | 7 +++++++ .../pt/up/fe/specs/clava/ast/expr/StringLiteral.java | 10 ++++++++-- .../src/pt/up/fe/specs/clava/language/Standard.java | 12 ++++++++---- 5 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 ClangAstParser/test-resources/cxx/unevaluated_strings.cpp create mode 100644 ClangAstParser/test-resources/cxx/unevaluated_strings.cpp.txt diff --git a/ClangAstParser/test-resources/cxx/unevaluated_strings.cpp b/ClangAstParser/test-resources/cxx/unevaluated_strings.cpp new file mode 100644 index 0000000000..b808d729be --- /dev/null +++ b/ClangAstParser/test-resources/cxx/unevaluated_strings.cpp @@ -0,0 +1,3 @@ +static_assert(true, "plain message"); +static_assert(true, "line\nbreak"); +static_assert(true, "\u00e9"); diff --git a/ClangAstParser/test-resources/cxx/unevaluated_strings.cpp.txt b/ClangAstParser/test-resources/cxx/unevaluated_strings.cpp.txt new file mode 100644 index 0000000000..b808d729be --- /dev/null +++ b/ClangAstParser/test-resources/cxx/unevaluated_strings.cpp.txt @@ -0,0 +1,3 @@ +static_assert(true, "plain message"); +static_assert(true, "line\nbreak"); +static_assert(true, "\u00e9"); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxTest.java index b8cdfe5726..7de8c90424 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxTest.java @@ -287,6 +287,13 @@ public void testStrings() { new CxxTester("strings.cpp").test(); } + @Test + public void testUnevaluatedStrings() { + new CxxTester("unevaluated_strings.cpp") + .addFlags("-std=c++26") + .test(); + } + // -Xclang-ast-dump-nostdinc-nocudalib-nocudainc--cuda-gpu-arch=sm_30 // "--cuda-device-only" diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java index 9cf85bec42..10e4078fe6 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java @@ -55,6 +55,12 @@ public StringLiteral(DataStore data, Collection children) { @Override public String getLiteral() { + // C++26 unevaluated strings only allow simple escapes and UCNs. The evaluated byte payload does not retain + // enough information to reconstruct those spellings, so keep Clang's validated source spelling. + if (get(STRING_KIND) == StringKind.UNEVALUATED) { + return super.getLiteral(); + } + // Update: Unfortunately it is not possible to blindly use the source code literal // For instance, if directives and macros appear in the middle of the literal, they will also appear in the // generated source code @@ -90,8 +96,8 @@ private String getStringFromBytes() { return SpecsStrings.escapeJson(new String(bytes, kind.getCharset())); } - // If ASCII, convert each byte directly - if (kind == StringKind.ORDINARY || kind == StringKind.UNEVALUATED) { + // Ordinary strings use one-byte characters; convert each byte directly. + if (kind == StringKind.ORDINARY) { var literal = new StringBuilder(); for (var aByte : get(STRING_BYTES)) { diff --git a/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java b/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java index f05370f1bb..6240c2d93a 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java @@ -42,12 +42,16 @@ public enum Standard implements StringProvider { CXX11("c++11", true), CXX14("c++14", true), CXX17("c++17", true), - CXX2A("c++2a", true), + CXX20("c++20", true), + CXX23("c++23", true), + CXX26("c++26", true), GNUXX98("gnu++98", true), GNUXX11("gnu++11", true), GNUXX14("gnu++14", true), GNUXX17("gnu++17", true), - GNUXX2A("gnu++2a", true), + GNUXX20("gnu++20", true), + GNUXX23("gnu++23", true), + GNUXX26("gnu++26", true), OPENCL10("cl1.0"), OPENCL12("cl1.2"), OPENCL20("cl2.0"), @@ -63,8 +67,8 @@ public enum Standard implements StringProvider { private static final Lazy> ENUM_HELPER = EnumHelperWithValue .newLazyHelperWithValue(Standard.class); - private static final Set GNU_STANDARDS = SpecsCollections.asSet(GNU90, GNU99, GNU11, GNUXX98, GNUXX11, - GNUXX14); + private static final Set GNU_STANDARDS = SpecsCollections.asSet(GNU90, GNU99, GNU11, GNU17, GNUXX98, GNUXX11, + GNUXX14, GNUXX17, GNUXX20, GNUXX23, GNUXX26); public static EnumHelperWithValue getEnumHelper() { return ENUM_HELPER.get(); From bbd48d15c5f4223a92ebf20646f8cd3969ee3021 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 16:04:00 +0100 Subject: [PATCH 31/39] fix: preserve unevaluated string source spelling Keep Clangs validated source spelling for UNEVALUATED literals so adjacent tokens and source line breaks survive code generation. Update the dependent-scope golden accordingly. --- .../test-resources/cxx/dependent_scope_decl_ref_expr.cpp.txt | 3 ++- ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ClangAstParser/test-resources/cxx/dependent_scope_decl_ref_expr.cpp.txt b/ClangAstParser/test-resources/cxx/dependent_scope_decl_ref_expr.cpp.txt index cce55922e1..5b84d42dc7 100644 --- a/ClangAstParser/test-resources/cxx/dependent_scope_decl_ref_expr.cpp.txt +++ b/ClangAstParser/test-resources/cxx/dependent_scope_decl_ref_expr.cpp.txt @@ -7,5 +7,6 @@ void compute_boundaries(FloatType value) { // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) // If v is normalized: // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) - static_assert(std::numeric_limits::is_iec559, "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + static_assert(std::numeric_limits::is_iec559, "internal error: dtoa_short requires an IEEE-754 " + "floating-point implementation"); } diff --git a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java index 10e4078fe6..8937042aee 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/ast/expr/StringLiteral.java @@ -55,8 +55,8 @@ public StringLiteral(DataStore data, Collection children) { @Override public String getLiteral() { - // C++26 unevaluated strings only allow simple escapes and UCNs. The evaluated byte payload does not retain - // enough information to reconstruct those spellings, so keep Clang's validated source spelling. + // Unevaluated strings can contain source-level details, such as adjacent tokens separated by a line break, that + // are lost when using the evaluated byte payload. Keep Clang's validated source spelling. if (get(STRING_KIND) == StringKind.UNEVALUATED) { return super.getLiteral(); } From e4f3e211fac3b6d37a91a3c72045077a10464068 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 16:40:34 +0100 Subject: [PATCH 32/39] fix: update C++ standard versions in Standard enum --- .../specs/clang/parser/tests/InitializationStyleTest.java | 2 +- ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java index 4a6207ae8c..91e1f2d286 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java @@ -39,7 +39,7 @@ public void parenthesizedListInitializationKeepsAllArguments() { SpecsSystem.programStandardInit(); File sourceFile = SpecsIo.resourceCopy("cxx/paren_list_initialization.cpp", tempFolder.toFile(), false, true); - App app = CodeParser.newInstance().parse(List.of(sourceFile), List.of("-std=c++2a")); + App app = CodeParser.newInstance().parse(List.of(sourceFile), List.of("-std=c++20")); VarDecl point = app.getDescendants(VarDecl.class).stream() .filter(varDecl -> varDecl.getDeclName().equals("point")) diff --git a/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java b/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java index 6240c2d93a..b35aca2dda 100644 --- a/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java +++ b/ClavaAst/src/pt/up/fe/specs/clava/language/Standard.java @@ -33,10 +33,14 @@ public enum Standard implements StringProvider { C99, C11, C17, + C18, + C23, GNU90, GNU99, GNU11, GNU17, + GNU18, + GNU23, CXX98("c++98", true), CXX03("c++03", true), CXX11("c++11", true), @@ -67,8 +71,8 @@ public enum Standard implements StringProvider { private static final Lazy> ENUM_HELPER = EnumHelperWithValue .newLazyHelperWithValue(Standard.class); - private static final Set GNU_STANDARDS = SpecsCollections.asSet(GNU90, GNU99, GNU11, GNU17, GNUXX98, GNUXX11, - GNUXX14, GNUXX17, GNUXX20, GNUXX23, GNUXX26); + private static final Set GNU_STANDARDS = SpecsCollections.asSet(GNU90, GNU99, GNU11, GNU17, GNU18, GNU23, + GNUXX98, GNUXX11, GNUXX14, GNUXX17, GNUXX20, GNUXX23, GNUXX26); public static EnumHelperWithValue getEnumHelper() { return ENUM_HELPER.get(); From 2e40739bf4cadb246ed0fce34ac4975394ab9c79 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 17:34:54 +0100 Subject: [PATCH 33/39] test: cover dependent unary transform types --- .../parser/tests/UnaryTransformTypeTest.java | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/UnaryTransformTypeTest.java diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/UnaryTransformTypeTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/UnaryTransformTypeTest.java new file mode 100644 index 0000000000..27d54bd00e --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/UnaryTransformTypeTest.java @@ -0,0 +1,85 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang.parser.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import pt.up.fe.specs.clang.codeparser.CodeParser; +import pt.up.fe.specs.clava.ast.extra.App; +import pt.up.fe.specs.clava.ast.type.Type; +import pt.up.fe.specs.clava.ast.type.UnaryTransformType; +import pt.up.fe.specs.clava.ast.type.enums.UnaryTransformTypeKind; +import pt.up.fe.specs.util.SpecsSystem; + +public class UnaryTransformTypeTest { + + private static final String SOURCE = """ + template + struct Dependent { + using type = __underlying_type(T); + }; + + enum Resolved { Value }; + using ResolvedType = __underlying_type(Resolved); + """; + + @TempDir + Path tempFolder; + + @Test + public void dependentTransformsMayHaveNoUnderlyingType() throws IOException { + SpecsSystem.programStandardInit(); + + File sourceFile = tempFolder.resolve("unary_transform_type.cpp").toFile(); + Files.writeString(sourceFile.toPath(), SOURCE); + + App app = CodeParser.newInstance().parse(List.of(sourceFile), List.of("-std=c++11")); + + List transforms = app.getDescendantsAndFields(UnaryTransformType.class).stream() + .filter(transform -> transform.get(UnaryTransformType.KIND) == UnaryTransformTypeKind.EnumUnderlyingType) + .toList(); + + assertEquals(2, transforms.size()); + + UnaryTransformType dependentTransform = transforms.stream() + .filter(transform -> transform.getUnderlyingType().isEmpty()) + .findFirst() + .orElseThrow(); + + assertNotNull(dependentTransform.getBaseType()); + assertEquals(1, dependentTransform.getNodeFields().size()); + assertTrue(dependentTransform.getNodeFields().contains(dependentTransform.getBaseType())); + + UnaryTransformType resolvedTransform = transforms.stream() + .filter(transform -> transform.getUnderlyingType().isPresent()) + .findFirst() + .orElseThrow(); + + Type resolvedUnderlyingType = resolvedTransform.getUnderlyingType().orElseThrow(); + assertNotNull(resolvedTransform.getBaseType()); + assertTrue(resolvedTransform.getNodeFields().contains(resolvedTransform.getBaseType())); + assertTrue(resolvedTransform.getNodeFields().contains(resolvedUnderlyingType)); + } +} From 9726acf6edc876f77b9910b0a665d532c73e556e Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 17:40:47 +0100 Subject: [PATCH 34/39] test: cover strict optional node resolution --- .../up/fe/specs/clang/parsers/ClavaNodes.java | 2 +- .../specs/clang/parsers/ClavaNodesTest.java | 71 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/parsers/ClavaNodesTest.java diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java index fc58715dd7..30438176cc 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/parsers/ClavaNodes.java @@ -93,7 +93,7 @@ public Optional getOptional(String nodeId) { ClavaNode clavaNode = clavaNodes.get(nodeId); if (clavaNode == null) { - ClavaLog.debug("Could not find ClavaNode with id '" + nodeId + ClavaLog.warning("Could not find ClavaNode with id '" + nodeId + "', returning an empty optional. Check if node is being visited. If parsing of includes is enabled, check that the parsing level is sufficient."); return Optional.empty(); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parsers/ClavaNodesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parsers/ClavaNodesTest.java new file mode 100644 index 0000000000..f4a42736af --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parsers/ClavaNodesTest.java @@ -0,0 +1,71 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang.parsers; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.suikasoft.jOptions.Interfaces.DataStore; + +import pt.up.fe.specs.clava.ast.expr.Expr; +import pt.up.fe.specs.clava.context.ClavaContext; + +public class ClavaNodesTest { + + @Test + public void optionalNodeResolvesPresentNode() { + var context = new ClavaContext(); + var factory = context.getFactory(); + var clavaNodes = new ClavaNodes(factory); + DataStore data = factory.newDataStore(Expr.class); + var type = factory.nullType(); + + clavaNodes.getNodes().put("type-id", type); + clavaNodes.queueSetOptionalNode(data, Expr.TYPE, "type-id"); + clavaNodes.getQueuedActions().forEach(Runnable::run); + + assertSame(type, data.get(Expr.TYPE).orElseThrow()); + } + + @Test + public void optionalNodeUsesEmptyForExplicitNullId() { + var context = new ClavaContext(); + var factory = context.getFactory(); + var clavaNodes = new ClavaNodes(factory); + DataStore data = factory.newDataStore(Expr.class); + + clavaNodes.queueSetOptionalNode(data, Expr.TYPE, "nullptr_type"); + clavaNodes.getQueuedActions().forEach(Runnable::run); + + assertTrue(data.get(Expr.TYPE).isEmpty()); + } + + @Test + public void optionalNodeRejectsUnresolvedNodeId() { + var context = new ClavaContext(); + var factory = context.getFactory(); + var clavaNodes = new ClavaNodes(factory); + DataStore data = factory.newDataStore(Expr.class); + + clavaNodes.queueSetOptionalNode(data, Expr.TYPE, "missing-id"); + + var exception = assertThrows(NullPointerException.class, + () -> clavaNodes.getQueuedActions().forEach(Runnable::run)); + + assertTrue(exception.getMessage().contains("Could not resolve optional node 'missing-id'")); + assertTrue(exception.getMessage().contains("key 'type'")); + } +} From c71eb7c1c5ecf1ee389bf36afdce54503aa8d46a Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 18:28:50 +0100 Subject: [PATCH 35/39] test: assert parenthesized initialization style --- .../up/fe/specs/clang/parser/tests/InitializationStyleTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java index 91e1f2d286..5e1f99532b 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/InitializationStyleTest.java @@ -46,6 +46,7 @@ public void parenthesizedListInitializationKeepsAllArguments() { .findFirst() .orElseThrow(); + assertEquals(InitializationStyle.ParenListInit, point.get(VarDecl.INIT_STYLE)); assertEquals("Point point(1, 2)", point.getCode()); } From 9f0f3ad1bba1ae15b7bdcb8bca6184236ba21bdf Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 20:13:34 +0100 Subject: [PATCH 36/39] fix: restore built-in CUDA archive parsing Use Clava's historical cudalib.zip with standard --cuda-path handling and remove the compatibility-header workaround. --- .../resources/clangast/cuda_compatibility.h | 9 --- .../up/fe/specs/clang/ClangAstResource.java | 3 +- .../fe/specs/clang/ClangAstWebResource.java | 4 + .../pt/up/fe/specs/clang/ClangResources.java | 23 ++++++ .../fe/specs/clang/dumper/ClangAstDumper.java | 79 ++----------------- .../up/fe/specs/clang/ClangResourcesTest.java | 13 +++ .../clang/dumper/ClangAstDumperTest.java | 61 -------------- .../specs/clang/parser/tests/CxxCudaTest.java | 8 +- 8 files changed, 47 insertions(+), 153 deletions(-) delete mode 100644 ClangAstParser/resources/clangast/cuda_compatibility.h delete mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java diff --git a/ClangAstParser/resources/clangast/cuda_compatibility.h b/ClangAstParser/resources/clangast/cuda_compatibility.h deleted file mode 100644 index f649810666..0000000000 --- a/ClangAstParser/resources/clangast/cuda_compatibility.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include - -extern "C" __host__ cudaError_t cudaConfigureCall( - dim3 gridDim, - dim3 blockDim, - size_t sharedMem = 0, - cudaStream_t stream = 0); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java index 8fef3eaef2..c0c53d5ca1 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstResource.java @@ -21,8 +21,7 @@ */ public enum ClangAstResource implements ResourceProvider { TEST_INCLUDES_C("test_includes.c"), - TEST_INCLUDES_CPP("test_includes.cpp"), - CUDA_COMPATIBILITY("cuda_compatibility.h"); + TEST_INCLUDES_CPP("test_includes.cpp"); private final String resource; diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index f1fc8775dc..6ec9d6c9b4 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -28,6 +28,10 @@ public final class ClangAstWebResource { private static final String RELEASE_ROOT = "https://github.com/specs-feup/clang-dumper/releases/download/"; + private static final String LEGACY_CUDA_RELEASE_ROOT = + "https://github.com/specs-feup/clava/releases/download/clang_ast_dumper_v12.0.7.1/"; + static final WebResourceProvider CUDA_LIB = + WebResourceProvider.newInstance(LEGACY_CUDA_RELEASE_ROOT, "cudalib.zip", "v11.3.0"); private static final String RELEASE_TAG_RESOURCE = "clang-dumper-release.tag"; public static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index 52e504eee8..ef8d0b9de7 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -23,6 +23,7 @@ import pt.up.fe.specs.util.SpecsIo; import pt.up.fe.specs.util.SpecsLogs; import pt.up.fe.specs.util.SpecsSystem; +import pt.up.fe.specs.util.lazy.Lazy; import pt.up.fe.specs.util.providers.FileResourceProvider.ResourceWriteData; import pt.up.fe.specs.util.system.ProcessOutputAsString; @@ -65,15 +66,35 @@ public class ClangResources { private final static Duration CACHE_LOCK_RETRY_INTERVAL = Duration.ofMillis(100); private final static Duration CACHE_LOCK_STALE_MAX_AGE = Duration.ofMinutes(5); private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); + private static final String CUDA_ARCHIVE_FILENAME = "cudalib.zip"; + private static final String CUDA_FOLDERNAME = "cudalib"; private static final Map HAS_LIBC = new ConcurrentHashMap<>(); private final CodeParser options; + private final Lazy builtinCudaLib = Lazy.newInstance(this::prepareBuiltinCudaLib); public ClangResources(CodeParser options) { this.options = options; } + public File getBuiltinCudaLib() { + return builtinCudaLib.get(); + } + + private File prepareBuiltinCudaLib() { + var resourceFolder = getClangResourceFolder(); + var cudaFolder = SpecsIo.mkdir(new File(resourceFolder, CUDA_FOLDERNAME)); + var zipFile = ClangAstWebResource.CUDA_LIB.writeVersioned(resourceFolder, ClangResources.class); + + if (zipFile.isNewFile() || !isCudaInstallation(cudaFolder)) { + SpecsIo.deleteFolderContents(cudaFolder); + SpecsIo.extractZip(zipFile.getFile(), cudaFolder); + } + + return cudaFolder; + } + public Optional getSystemCudaInstallation() { return findCudaInstallation( Arrays.asList(System.getenv("CUDA_HOME"), System.getenv("CUDA_PATH"), System.getenv("CUDA_ROOT")), @@ -417,6 +438,8 @@ private void validateTopLevelCacheFiles(ClangDumperManifest manifest, File resou ClangAstWebResource.MANIFEST_FILENAME, getCurrentAsset(manifest, executableKind).filename(), INCLUDES_FOLDERNAME, + CUDA_ARCHIVE_FILENAME, + CUDA_FOLDERNAME, LAST_USED_FILENAME); var files = resourceFolder.listFiles(); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index fae30774ec..ea31b5da86 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -17,7 +17,6 @@ import org.suikasoft.jOptions.JOptionsUtils; import org.suikasoft.jOptions.streamparser.LineStreamParser; import pt.up.fe.specs.clang.ClangAstKeys; -import pt.up.fe.specs.clang.ClangAstResource; import pt.up.fe.specs.clang.ClangResources; import pt.up.fe.specs.clang.cilk.CilkParser; import pt.up.fe.specs.clang.codeparser.CodeParser; @@ -39,14 +38,7 @@ import pt.up.fe.specs.util.utilities.LineStream; import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.file.AtomicMoveNotSupportedException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -70,7 +62,6 @@ public static boolean usePlugin() { private final static String CLANG_DUMP_FILENAME = "clangDump.txt"; private final static String STDERR_DUMP_FILENAME = "stderr.txt"; private static final AtomicBoolean CUDA_UNAVAILABLE_WARNING_SHOWN = new AtomicBoolean(); - private static final Object CUDA_COMPATIBILITY_LOCK = new Object(); private static final List CLANG_AST_DUMPER_TEMP_FILES = List.of("includes.txt", CLANG_DUMP_FILENAME, // "clavaDump.txt", "nodetypes.txt", "types.txt", "is_temporary.txt", "template_args.txt", @@ -82,65 +73,6 @@ public static List getTempFiles() { return CLANG_AST_DUMPER_TEMP_FILES; } - static File getCudaCompatibilityFile(File folder) { - var destination = new File(folder, ClangAstResource.CUDA_COMPATIBILITY.getFilename()); - if (destination.isFile()) { - return destination; - } - - synchronized (CUDA_COMPATIBILITY_LOCK) { - if (destination.isFile()) { - return destination; - } - - return writeCudaCompatibilityFile(destination); - } - } - - private static File writeCudaCompatibilityFile(File destination) { - var destinationPath = destination.toPath(); - Path temporaryPath = null; - - try { - Files.createDirectories(destinationPath.getParent()); - temporaryPath = Files.createTempFile(destinationPath.getParent(), destination.getName(), ".tmp"); - - try (var inputStream = ClangAstResource.CUDA_COMPATIBILITY.toStream()) { - if (inputStream == null) { - throw new IOException("Could not load CUDA compatibility resource"); - } - - Files.copy(inputStream, temporaryPath, StandardCopyOption.REPLACE_EXISTING); - } - - try { - Files.move(temporaryPath, destinationPath, StandardCopyOption.ATOMIC_MOVE); - } catch (AtomicMoveNotSupportedException e) { - try { - Files.move(temporaryPath, destinationPath); - } catch (FileAlreadyExistsException ignored) { - // Another JVM installed the complete file first. - } - } catch (FileAlreadyExistsException ignored) { - // Another JVM installed the complete file first. - } - - return destination; - } catch (IOException e) { - throw new UncheckedIOException("Could not write CUDA compatibility resource '" - + destination + "'", e); - } finally { - if (temporaryPath != null) { - try { - Files.deleteIfExists(temporaryPath); - } catch (IOException e) { - SpecsLogs.warn("Could not delete temporary CUDA compatibility resource '" - + temporaryPath + "'", e); - } - } - } - } - /** * TODO: Not implemented yet *

    @@ -310,12 +242,11 @@ else if (isCuda) { var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); if (useBuiltinCudaLib && !builtinIncludes.isEmpty()) { - arguments.add("-nocudainc"); - arguments.add("-nocudalib"); - arguments.add("-include"); - arguments.add("__clang_cuda_runtime_wrapper.h"); - arguments.add("-include"); - arguments.add(getCudaCompatibilityFile(SpecsIo.getTempFolder()).getAbsolutePath()); + File cudaFolder = clangResources.getBuiltinCudaLib(); + + ClavaLog.debug("Setting --cuda-path to built-in CUDA folder '" + + cudaFolder.getAbsolutePath() + "'"); + arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); } else if (useBuiltinCudaLib) { clangResources.getSystemCudaInstallation().ifPresentOrElse(cudaFolder -> { ClavaLog.debug("Setting --cuda-path to discovered CUDA folder '" diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 8dd92e73cf..5fae23877f 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -660,6 +660,19 @@ public void builtinCudaIncludesAreAvailableWithSystemLibc() { assertTrue(hasCudaWrapper, "Built-in CUDA must provide Clang's CUDA runtime wrapper independently of libc"); } + @Test + public void builtinCudaArchiveHasCanonicalInstallationLayout() { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + parser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); + + var cudaFolder = new ClangResources(parser).getBuiltinCudaLib(); + + assertTrue(new File(cudaFolder, "include/cuda.h").isFile()); + assertTrue(new File(cudaFolder, "include/cuda_runtime.h").isFile()); + assertTrue(new File(cudaFolder, "nvvm/libdevice/libdevice.10.bc").isFile()); + } + @Test public void libcDetectionIsScopedToTheExecutable() throws IOException { assumeTrue(!SupportedPlatform.getCurrentPlatform().isWindows(), "Shell fixtures require a Unix executable"); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java deleted file mode 100644 index 7e0dbe140f..0000000000 --- a/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2026 SPeCS. - *

    - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package pt.up.fe.specs.clang.dumper; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import pt.up.fe.specs.clang.ClangAstResource; - -public class ClangAstDumperTest { - - @TempDir - Path tempFolder; - - @Test - public void cudaCompatibilityFileIsCompleteWhenRequestedConcurrently() throws Exception { - ExecutorService executor = Executors.newFixedThreadPool(8); - try { - var futures = new ArrayList>(); - for (int index = 0; index < 32; index++) { - futures.add(executor.submit(() -> ClangAstDumper.getCudaCompatibilityFile(tempFolder.toFile()))); - } - - Set files = new HashSet<>(); - for (var future : futures) { - files.add(future.get(10, TimeUnit.SECONDS)); - } - - var expectedFile = tempFolder.resolve(ClangAstResource.CUDA_COMPATIBILITY.getFilename()).toFile(); - assertEquals(Set.of(expectedFile), files); - assertEquals(ClangAstResource.CUDA_COMPATIBILITY.read(), Files.readString(expectedFile.toPath())); - } finally { - executor.shutdownNow(); - executor.awaitTermination(10, TimeUnit.SECONDS); - } - } -} diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java index 1a6b37d3f7..4960a32dda 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java @@ -17,13 +17,7 @@ import pt.up.fe.specs.clang.parser.CxxCudaTester; -/** - * Disabled tests, they are failing in the CI server. Even when passing the --cuda-path built-in library, the parser - * fails to find the CUDA library. - * - * @author JBispo - * - */ +/** Verifies built-in CUDA parsing through Clava's historical cudalib archive. */ public class CxxCudaTest { @Test public void testAtomicAdd() { From cf3ffa0e730cb04fdcb2349b5f713eda2553a331 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 20:20:25 +0100 Subject: [PATCH 37/39] fix: always use built-in CUDA archive Select Clava's historical CUDA archive whenever the built-in CUDA option is requested, including configurations with empty bundled libc includes. --- .../fe/specs/clang/dumper/ClangAstDumper.java | 46 +++++++---------- .../clang/dumper/ClangAstDumperTest.java | 51 +++++++++++++++++++ 2 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index ea31b5da86..7554b526f6 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -43,7 +43,6 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; /** * Calls the ClangAstDumper executable and returns the dumped information. Clava AST can be built based on this output. @@ -61,7 +60,6 @@ public static boolean usePlugin() { private final static String CLANG_DUMP_FILENAME = "clangDump.txt"; private final static String STDERR_DUMP_FILENAME = "stderr.txt"; - private static final AtomicBoolean CUDA_UNAVAILABLE_WARNING_SHOWN = new AtomicBoolean(); private static final List CLANG_AST_DUMPER_TEMP_FILES = List.of("includes.txt", CLANG_DUMP_FILENAME, // "clavaDump.txt", "nodetypes.txt", "types.txt", "is_temporary.txt", "template_args.txt", @@ -239,32 +237,7 @@ else if (isCuda) { arguments.add("--cuda-gpu-arch=" + parserConfig.get(CodeParser.CUDA_GPU_ARCH)); var cudaPath = parserConfig.get(CodeParser.CUDA_PATH); - var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); - - if (useBuiltinCudaLib && !builtinIncludes.isEmpty()) { - File cudaFolder = clangResources.getBuiltinCudaLib(); - - ClavaLog.debug("Setting --cuda-path to built-in CUDA folder '" - + cudaFolder.getAbsolutePath() + "'"); - arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); - } else if (useBuiltinCudaLib) { - clangResources.getSystemCudaInstallation().ifPresentOrElse(cudaFolder -> { - ClavaLog.debug("Setting --cuda-path to discovered CUDA folder '" - + cudaFolder.getAbsolutePath() + "'"); - arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); - }, () -> { - if (CUDA_UNAVAILABLE_WARNING_SHOWN.compareAndSet(false, true)) { - ClavaLog.warning("CUDA parsing requested, but no system CUDA toolkit was found. " - + "Continuing without CUDA headers; CUDA parsing may fail."); - } - }); - } else if (!useBuiltinCudaLib && !cudaPath.isBlank()) { - - File cudaFolder = SpecsIo.existingFolder(cudaPath); - - ClavaLog.debug("Setting --cuda-path to folder '" + cudaFolder.getAbsolutePath() + "'"); - arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); - } + addCudaPathArgument(arguments, cudaPath); // Since we only need parsing, enable host-only // Can help with errors such as "__float128 is not supported on this target" @@ -360,6 +333,23 @@ else if (SourceType.isHeader(sourceFile)) { return parsedData; } + void addCudaPathArgument(List arguments, String cudaPath) { + var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); + + if (useBuiltinCudaLib) { + File cudaFolder = clangResources.getBuiltinCudaLib(); + + ClavaLog.debug("Setting --cuda-path to built-in CUDA folder '" + + cudaFolder.getAbsolutePath() + "'"); + arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); + } else if (!cudaPath.isBlank()) { + File cudaFolder = SpecsIo.existingFolder(cudaPath); + + ClavaLog.debug("Setting --cuda-path to folder '" + cudaFolder.getAbsolutePath() + "'"); + arguments.add("--cuda-path=" + cudaFolder.getAbsolutePath()); + } + } + private String processOutput(InputStream inputStream) { StringBuilder output = new StringBuilder(); try (LineStream lines = LineStream.newInstance(inputStream, null)) { diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java new file mode 100644 index 0000000000..967a73003b --- /dev/null +++ b/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java @@ -0,0 +1,51 @@ +/** + * Copyright 2026 SPeCS. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package pt.up.fe.specs.clang.dumper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import pt.up.fe.specs.clang.ClangResources; +import pt.up.fe.specs.clang.codeparser.CodeParser; + +public class ClangAstDumperTest { + + @TempDir + Path tempFolder; + + @Test + public void builtinCudaUsesArchiveWithEmptyBuiltinIncludes() { + var parser = CodeParser.newInstance(); + parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); + parser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); + + var resourceFolder = new ClangResources(parser).getClangResourceFolder(); + var expectedCudaFolder = new File(resourceFolder, "cudalib"); + var dumper = new ClangAstDumper(false, tempFolder.resolve("clang").toFile(), List.of(), parser); + var arguments = new ArrayList(); + + dumper.addCudaPathArgument(arguments, parser.get(CodeParser.CUDA_PATH)); + + assertEquals(List.of("--cuda-path=" + expectedCudaFolder.getAbsolutePath()), arguments); + assertTrue(new File(expectedCudaFolder, "include/cuda_runtime.h").isFile()); + } +} From 03a4e88ee5cd051b4246b9b6c76441499924c2f4 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 20:42:05 +0100 Subject: [PATCH 38/39] fix: simplify legacy CUDA resource handling --- .../fe/specs/clang/ClangAstWebResource.java | 5 +- .../pt/up/fe/specs/clang/ClangResources.java | 87 ++----------------- .../fe/specs/clang/dumper/ClangAstDumper.java | 20 ++--- .../up/fe/specs/clang/ClangResourcesTest.java | 24 +---- .../clang/dumper/ClangAstDumperTest.java | 51 ----------- .../specs/clang/parser/tests/CxxCudaTest.java | 10 +++ 6 files changed, 32 insertions(+), 165 deletions(-) delete mode 100644 ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java index 6ec9d6c9b4..9689587a91 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java @@ -28,10 +28,11 @@ public final class ClangAstWebResource { private static final String RELEASE_ROOT = "https://github.com/specs-feup/clang-dumper/releases/download/"; - private static final String LEGACY_CUDA_RELEASE_ROOT = + private static final String CUDA_RELEASE_ROOT = "https://github.com/specs-feup/clava/releases/download/clang_ast_dumper_v12.0.7.1/"; + public static final String CUDA_LIB_FILENAME = "cudalib.zip"; static final WebResourceProvider CUDA_LIB = - WebResourceProvider.newInstance(LEGACY_CUDA_RELEASE_ROOT, "cudalib.zip", "v11.3.0"); + WebResourceProvider.newInstance(CUDA_RELEASE_ROOT, CUDA_LIB_FILENAME, "v11.3.0"); private static final String RELEASE_TAG_RESOURCE = "clang-dumper-release.tag"; public static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json"; diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java index ef8d0b9de7..9d54bf5ebb 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java @@ -23,7 +23,6 @@ import pt.up.fe.specs.util.SpecsIo; import pt.up.fe.specs.util.SpecsLogs; import pt.up.fe.specs.util.SpecsSystem; -import pt.up.fe.specs.util.lazy.Lazy; import pt.up.fe.specs.util.providers.FileResourceProvider.ResourceWriteData; import pt.up.fe.specs.util.system.ProcessOutputAsString; @@ -44,15 +43,12 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HexFormat; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; public class ClangResources { @@ -66,26 +62,19 @@ public class ClangResources { private final static Duration CACHE_LOCK_RETRY_INTERVAL = Duration.ofMillis(100); private final static Duration CACHE_LOCK_STALE_MAX_AGE = Duration.ofMinutes(5); private final static Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60); - private static final String CUDA_ARCHIVE_FILENAME = "cudalib.zip"; - private static final String CUDA_FOLDERNAME = "cudalib"; private static final Map HAS_LIBC = new ConcurrentHashMap<>(); private final CodeParser options; - private final Lazy builtinCudaLib = Lazy.newInstance(this::prepareBuiltinCudaLib); public ClangResources(CodeParser options) { this.options = options; } public File getBuiltinCudaLib() { - return builtinCudaLib.get(); - } - - private File prepareBuiltinCudaLib() { - var resourceFolder = getClangResourceFolder(); - var cudaFolder = SpecsIo.mkdir(new File(resourceFolder, CUDA_FOLDERNAME)); - var zipFile = ClangAstWebResource.CUDA_LIB.writeVersioned(resourceFolder, ClangResources.class); + var cudaResourceFolder = SpecsIo.mkdir(options.get(CodeParser.DUMPER_FOLDER), "cuda"); + var cudaFolder = SpecsIo.mkdir(cudaResourceFolder, "cudalib"); + var zipFile = ClangAstWebResource.CUDA_LIB.writeVersioned(cudaResourceFolder, ClangResources.class); if (zipFile.isNewFile() || !isCudaInstallation(cudaFolder)) { SpecsIo.deleteFolderContents(cudaFolder); @@ -95,74 +84,10 @@ private File prepareBuiltinCudaLib() { return cudaFolder; } - public Optional getSystemCudaInstallation() { - return findCudaInstallation( - Arrays.asList(System.getenv("CUDA_HOME"), System.getenv("CUDA_PATH"), System.getenv("CUDA_ROOT")), - System.getenv("PATH"), getConventionalCudaRoots()); - } - - static Optional findCudaInstallation(List configuredRoots, String pathValue, - List conventionalRoots) { - var candidates = new ArrayList(); - - configuredRoots.stream() - .filter(root -> root != null && !root.isBlank()) - .map(File::new) - .forEach(candidates::add); - - if (pathValue != null) { - for (var pathEntry : pathValue.split(Pattern.quote(File.pathSeparator))) { - if (pathEntry.isBlank()) { - continue; - } - - var binFolder = new File(pathEntry); - for (var executableName : List.of("nvcc", "nvcc.exe")) { - var executable = new File(binFolder, executableName); - if (executable.isFile() && binFolder.getParentFile() != null) { - candidates.add(binFolder.getParentFile()); - } - } - } - } - - candidates.addAll(conventionalRoots); - - return candidates.stream() - .map(File::getAbsoluteFile) - .filter(ClangResources::isCudaInstallation) - .findFirst(); - } - private static boolean isCudaInstallation(File folder) { return folder.isDirectory() && new File(folder, "include/cuda_runtime.h").isFile(); } - private static List getConventionalCudaRoots() { - if (!SupportedPlatform.getCurrentPlatform().isWindows()) { - return List.of( - new File("/usr/local/cuda"), - new File("/opt/cuda"), - new File("/opt/homebrew/opt/cuda")); - } - - var roots = new ArrayList(); - for (var programFiles : Arrays.asList(System.getenv("ProgramFiles"), System.getenv("ProgramFiles(x86)"))) { - if (programFiles == null || programFiles.isBlank()) { - continue; - } - - var cudaRoot = new File(programFiles, "NVIDIA GPU Computing Toolkit/CUDA"); - var versions = cudaRoot.listFiles(File::isDirectory); - if (versions != null) { - Arrays.sort(versions, Comparator.comparing(File::getName).reversed()); - roots.addAll(Arrays.asList(versions)); - } - } - - return roots; - } - public ClangFiles getClangFiles(LibcMode libcMode) { var source = ClangAstWebResource.getDumperSource(); @@ -194,6 +119,10 @@ public ClangFiles getClangFiles(LibcMode libcMode) { File clangExecutable = prepareResources(manifest, resourceFolder); List builtinIncludes = prepareIncludes(manifest, resourceFolder, clangExecutable, libcMode); + if (useBuiltinCuda) { + getBuiltinCudaLib(); + } + validateTopLevelCacheFiles(manifest, resourceFolder); updateLastUsedAndCleanupStaleVersions(resourceFolder); @@ -438,8 +367,6 @@ private void validateTopLevelCacheFiles(ClangDumperManifest manifest, File resou ClangAstWebResource.MANIFEST_FILENAME, getCurrentAsset(manifest, executableKind).filename(), INCLUDES_FOLDERNAME, - CUDA_ARCHIVE_FILENAME, - CUDA_FOLDERNAME, LAST_USED_FILENAME); var files = resourceFolder.listFiles(); diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 7554b526f6..829a4a31b0 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -194,10 +194,9 @@ private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, arguments.add("-std=cl2.0"); } // Set standard to CUDA - else if (isCuda && !standard.isCuda()) { + else if (isCuda) { + // Clang does not accept '-std=cuda'; CUDA sources still use a C++ standard. arguments.add(standard.isCxx() ? standard.getFlag() : Standard.CXX17.getFlag()); - } else if (isCuda) { - arguments.add(Standard.CXX17.getFlag()); } else { arguments.add(standard.getFlag()); } @@ -226,12 +225,13 @@ else if (isCuda && !standard.isCuda()) { } // If CUDA, add corresponding flags else if (isCuda) { - arguments.add("-x"); - arguments.add("cuda"); - - if (SpecsPlatforms.isWindows()) { - ClavaLog.info("CUDA parsing is not supported in Windows, run at your own risk"); - arguments.addAll(Arrays.asList("-fms-compatibility", "-D_MSC_VER", "-D_LIBCPP_MSVCRT")); + if (!SpecsPlatforms.isLinux()) { + ClavaLog.info("We only officially support CUDA parsing in Linux, run at your own risk"); + arguments.add("-fms-compatibility"); + if (SpecsPlatforms.isWindows()) { + arguments.add("-D_MSC_VER"); + arguments.add("-D_LIBCPP_MSVCRT"); + } } arguments.add("--cuda-gpu-arch=" + parserConfig.get(CodeParser.CUDA_GPU_ARCH)); @@ -333,7 +333,7 @@ else if (SourceType.isHeader(sourceFile)) { return parsedData; } - void addCudaPathArgument(List arguments, String cudaPath) { + private void addCudaPathArgument(List arguments, String cudaPath) { var useBuiltinCudaLib = cudaPath.toUpperCase().equals(CodeParser.getBuiltinOption()); if (useBuiltinCudaLib) { diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java index 5fae23877f..7da6ca019c 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/ClangResourcesTest.java @@ -103,28 +103,6 @@ public void includesCacheValidationOnlyChecksRequiredFiles() throws IOException assertTrue(ClangResources.isIncludesCacheValid(includesFolder.toFile())); } - @Test - public void cudaInstallationIsFoundFromConfiguredRoot() throws IOException { - var cudaFolder = Files.createDirectories(tempFolder.resolve("cuda")); - Files.createDirectories(cudaFolder.resolve("include")); - Files.createFile(cudaFolder.resolve("include/cuda_runtime.h")); - - assertEquals(cudaFolder.toFile().getAbsoluteFile(), - ClangResources.findCudaInstallation(List.of(cudaFolder.toString()), "", List.of()).orElseThrow()); - } - - @Test - public void cudaInstallationIsFoundFromNvccPath() throws IOException { - var cudaFolder = Files.createDirectories(tempFolder.resolve("cuda")); - var binFolder = Files.createDirectories(cudaFolder.resolve("bin")); - Files.createDirectories(cudaFolder.resolve("include")); - Files.createFile(cudaFolder.resolve("include/cuda_runtime.h")); - Files.createFile(binFolder.resolve("nvcc")); - - assertEquals(cudaFolder.toFile().getAbsoluteFile(), - ClangResources.findCudaInstallation(List.of(), binFolder.toString(), List.of()).orElseThrow()); - } - @Test public void staleCacheCleanupSkipsLockedVersions() throws IOException { var currentVersion = Files.createDirectory(tempFolder.resolve("current")).toFile(); @@ -668,6 +646,8 @@ public void builtinCudaArchiveHasCanonicalInstallationLayout() { var cudaFolder = new ClangResources(parser).getBuiltinCudaLib(); + assertEquals(tempFolder.resolve("cuda/cudalib").toFile().getAbsolutePath(), + cudaFolder.getAbsolutePath()); assertTrue(new File(cudaFolder, "include/cuda.h").isFile()); assertTrue(new File(cudaFolder, "include/cuda_runtime.h").isFile()); assertTrue(new File(cudaFolder, "nvvm/libdevice/libdevice.10.bc").isFile()); diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java deleted file mode 100644 index 967a73003b..0000000000 --- a/ClangAstParser/test/pt/up/fe/specs/clang/dumper/ClangAstDumperTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2026 SPeCS. - *

    - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package pt.up.fe.specs.clang.dumper; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import pt.up.fe.specs.clang.ClangResources; -import pt.up.fe.specs.clang.codeparser.CodeParser; - -public class ClangAstDumperTest { - - @TempDir - Path tempFolder; - - @Test - public void builtinCudaUsesArchiveWithEmptyBuiltinIncludes() { - var parser = CodeParser.newInstance(); - parser.set(CodeParser.DUMPER_FOLDER, tempFolder.toFile()); - parser.set(CodeParser.CUDA_PATH, CodeParser.getBuiltinOption()); - - var resourceFolder = new ClangResources(parser).getClangResourceFolder(); - var expectedCudaFolder = new File(resourceFolder, "cudalib"); - var dumper = new ClangAstDumper(false, tempFolder.resolve("clang").toFile(), List.of(), parser); - var arguments = new ArrayList(); - - dumper.addCudaPathArgument(arguments, parser.get(CodeParser.CUDA_PATH)); - - assertEquals(List.of("--cuda-path=" + expectedCudaFolder.getAbsolutePath()), arguments); - assertTrue(new File(expectedCudaFolder, "include/cuda_runtime.h").isFile()); - } -} diff --git a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java index 4960a32dda..2a857db62a 100644 --- a/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java +++ b/ClangAstParser/test/pt/up/fe/specs/clang/parser/tests/CxxCudaTest.java @@ -15,6 +15,8 @@ import org.junit.jupiter.api.Test; +import pt.up.fe.specs.clang.ClangAstKeys; +import pt.up.fe.specs.clang.LibcMode; import pt.up.fe.specs.clang.parser.CxxCudaTester; /** Verifies built-in CUDA parsing through Clava's historical cudalib archive. */ @@ -24,6 +26,14 @@ public void testAtomicAdd() { new CxxCudaTester("atomicAdd.cu").test(); } + @Test + public void testAtomicAddWithSystemLibc() { + new CxxCudaTester("atomicAdd.cu") + .set(ClangAstKeys.LIBC_CXX_MODE, LibcMode.SYSTEM) + .onePass() + .test(); + } + @Test public void testConvolutionCache() { new CxxCudaTester("convolution_cache.cu").test(); From ba928d3b60cbaf48f4b7329a4afed27a07a365b7 Mon Sep 17 00:00:00 2001 From: "L. Sousa" Date: Tue, 4 Aug 2026 20:50:20 +0100 Subject: [PATCH 39/39] docs: clarify CUDA standard workaround --- .../src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index 829a4a31b0..ba7f29de31 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -195,7 +195,8 @@ private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, } // Set standard to CUDA else if (isCuda) { - // Clang does not accept '-std=cuda'; CUDA sources still use a C++ standard. + // The LLVM 18 driver bundled with clang-dumper rejects '-std=cuda'. The .cu extension already + // selects CUDA mode, so use a C++ standard for host-side parsing. arguments.add(standard.isCxx() ? standard.getFlag() : Standard.CXX17.getFlag()); } else { arguments.add(standard.getFlag());