From 5957bca9bfae47047ecbaa3370a43b9f63ebfb40 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Wed, 6 May 2026 18:30:30 +0000 Subject: [PATCH] Add cache --- .run/spice build.run.xml | 2 +- .run/spice run.run.xml | 2 +- src/SourceFile.cpp | 60 ++- src/SourceFile.h | 3 + src/global/CacheManager.cpp | 253 ++++++++++- src/global/CacheManager.h | 16 +- src/global/GlobalResourceManager.h | 2 +- src/linker/ExternalLinkerInterface.cpp | 11 +- src/linker/ExternalLinkerInterface.h | 2 + src/main.cpp | 2 +- src/symboltablebuilder/SymbolTableBuilder.cpp | 11 +- test/CMakeLists.txt | 1 + test/unittest/UnitBlockAllocator.cpp | 1 - test/unittest/UnitCommonUtil.cpp | 3 +- test/unittest/UnitCompileCache.cpp | 413 ++++++++++++++++++ test/unittest/UnitDriver.cpp | 3 +- test/unittest/UnitFileUtil.cpp | 3 +- test/unittest/UnitSystemUtil.cpp | 3 +- 18 files changed, 753 insertions(+), 38 deletions(-) create mode 100644 test/unittest/UnitCompileCache.cpp diff --git a/.run/spice build.run.xml b/.run/spice build.run.xml index c4a43caf4..9ab7a44bd 100644 --- a/.run/spice build.run.xml +++ b/.run/spice build.run.xml @@ -1,5 +1,5 @@ - + diff --git a/.run/spice run.run.xml b/.run/spice run.run.xml index 29e904ff9..c5d5e9a67 100644 --- a/.run/spice run.run.xml +++ b/.run/spice run.run.xml @@ -1,5 +1,5 @@ - + diff --git a/src/SourceFile.cpp b/src/SourceFile.cpp index a062bdb2e..1bb685b02 100644 --- a/src/SourceFile.cpp +++ b/src/SourceFile.cpp @@ -2,6 +2,9 @@ #include "SourceFile.h" +#include +#include + #include #include #include @@ -87,14 +90,10 @@ void SourceFile::runLexer() { antlrCtx.lexer->addErrorListener(antlrCtx.lexerErrorHandler.get()); antlrCtx.tokenStream = std::make_unique(antlrCtx.lexer.get()); - // Calculate cache key - std::stringstream cacheKeyString; - cacheKeyString << std::hex << std::hash{}(antlrCtx.tokenStream->getText()); - cacheKey = cacheKeyString.str(); - - // Try to load from cache - if (!cliOptions.ignoreCache) - restoredFromCache = resourceManager.cacheManager.lookupSourceFile(this); + // Pre-compute a local cache key so the field is populated for cycle-aware fallbacks. + // The final key (which folds in transitive dependency cache keys) is computed at the end + // of runImportCollector, once every dependency's cache key has been finalized. + cacheKey = resourceManager.cacheManager.computeCacheKey(antlrCtx.tokenStream->getText()); previousStage = LEXER; timer.stop(); @@ -215,12 +214,35 @@ void SourceFile::runImportCollector() { // NOLINT(misc-no-recursion) importCollector.visit(ast); previousStage = IMPORT_COLLECTOR; - timer.stop(); // Run first part of pipeline for the imported source file for (SourceFile *sourceFile : dependencies | std::views::values) sourceFile->runFrontEnd(); + // Now that every transitive dependency has its final cache key, fold them into our own + // cache key. This way any change to a dependency invalidates the cache entry of every + // dependent (and transitively of the dependents' dependents), avoiding stale object files. + std::vector transitiveDepCacheKeys; + std::unordered_set visited; + std::queue worklist; + for (const SourceFile *dep : dependencies | std::views::values) + worklist.push(dep); + while (!worklist.empty()) { + const SourceFile *dep = worklist.front(); + worklist.pop(); + if (!visited.insert(dep->cacheKey).second) + continue; + transitiveDepCacheKeys.push_back(dep->cacheKey); + for (const SourceFile *transitive : dep->dependencies | std::views::values) + worklist.push(transitive); + } + cacheKey = resourceManager.cacheManager.computeCacheKey(antlrCtx.tokenStream->getText(), transitiveDepCacheKeys); + + // Try to load from the cache. Deferred from runLexer so that dep cache keys can participate. + if (!cliOptions.ignoreCache) + restoredFromCache = resourceManager.cacheManager.lookupSourceFile(this); + + timer.stop(); printStatusMessage("Import Collector", IO_AST, IO_AST, compilerOutput.times.importCollector); } @@ -517,8 +539,18 @@ void SourceFile::runObjectEmitter() { } void SourceFile::concludeCompilation() { - // Skip if restored from the cache or this stage has already been done - if (restoredFromCache || previousStage >= FINISHED) + // Handle cache-restored files: register all cached objects with linker + if (restoredFromCache) { + for (const auto &objectFilePath : cachedObjectFilePaths) + resourceManager.linker.addFileToLinkage(objectFilePath); + for (const auto &flag : sourceLinkerFlags) + resourceManager.linker.addLinkerFlag(flag); + for (const auto &path : sourceAdditionalSourcePaths) + resourceManager.linker.addAdditionalSourcePath(path); + return; + } + + if (previousStage >= FINISHED) return; // Cache the source file @@ -704,6 +736,9 @@ void SourceFile::checkForSoftErrors() const { } void SourceFile::collectAndPrintWarnings() { // NOLINT(misc-no-recursion) + // Skip if restored from cache (no scope tree available) + if (restoredFromCache) + return; // Print warnings for all dependencies for (SourceFile *sourceFile : dependencies | std::views::values) if (!sourceFile->isStdFile) @@ -755,7 +790,8 @@ void SourceFile::mergeNameRegistries(const SourceFile &importedSourceFile, const std::string newName = importName; newName += SCOPE_ACCESS_TOKEN; newName += originalName; - exportedNameRegistry.emplace(newName, NameRegistryEntry{newName, entry.typeId, entry.targetEntry, entry.targetScope, importEntry}); + exportedNameRegistry.emplace(newName, + NameRegistryEntry{newName, entry.typeId, entry.targetEntry, entry.targetScope, importEntry}); // Add the shortened name, considering the name collision const bool keepOnCollision = importedSourceFile.alwaysKeepSymbolsOnNameCollision; addNameRegistryEntry(originalName, entry.typeId, entry.targetEntry, entry.targetScope, keepOnCollision, importEntry); diff --git a/src/SourceFile.h b/src/SourceFile.h index b6960f0a1..6eec716a5 100644 --- a/src/SourceFile.h +++ b/src/SourceFile.h @@ -184,6 +184,9 @@ class SourceFile { CompilerOutput compilerOutput; SourceFile *parent; std::string cacheKey; + std::vector cachedObjectFilePaths; + std::vector sourceLinkerFlags; + std::vector sourceAdditionalSourcePaths; EntryNode *ast = nullptr; std::unique_ptr globalScope; llvm::LLVMContext context; diff --git a/src/global/CacheManager.cpp b/src/global/CacheManager.cpp index 4604dc1f4..3331c32cc 100644 --- a/src/global/CacheManager.cpp +++ b/src/global/CacheManager.cpp @@ -2,34 +2,267 @@ #include "CacheManager.h" +#include +#include +#include +#include +#include +#include + #include #include +#include #include +#include "../../lib/json/json.hpp" + namespace spice::compiler { CacheManager::CacheManager(const CliOptions &cliOptions) : cliOptions(cliOptions), cacheDir(cliOptions.cacheDir) {} -bool CacheManager::lookupSourceFile(const SourceFile *sourceFile) const { - const std::filesystem::path symbolTableFilePath = cacheDir / (sourceFile->cacheKey + ".bson"); +std::string CacheManager::computeCacheKey(const std::string &sourceCode, const std::vector &depCacheKeys) const { + std::stringstream components; + components << std::hex << std::hash{}(sourceCode); + components << static_cast(cliOptions.buildMode); + components << static_cast(cliOptions.optLevel); + components << static_cast(cliOptions.instrumentation.sanitizer); + components << cliOptions.instrumentation.generateDebugInfo; + components << cliOptions.targetTriple.str(); + components << cliOptions.useLTO; + // The output container influences codegen (PIC/PIE levels, DSO-local attributes for symbols, + // etc.), so reusing an object emitted for a different container would produce wrong output. + components << static_cast(cliOptions.outputContainer); + // Fold transitive dependency cache keys (sorted for determinism) into the key so that any + // change in a dependency invalidates every dependent's cache entry too. + std::vector sortedDepKeys = depCacheKeys; + std::ranges::sort(sortedDepKeys); + for (const std::string &depKey : sortedDepKeys) + components << depKey; + return std::to_string(std::hash{}(components.str())); +} + +bool CacheManager::lookupSourceFile(SourceFile *sourceFile) const { const char *objectFileExtension = SystemUtil::getOutputFileExtension(cliOptions, OutputContainer::OBJECT_FILE); - const std::filesystem::path objectFilePath = cacheDir / (sourceFile->cacheKey + objectFileExtension); + const std::filesystem::path metadataFilePath = cacheDir / (sourceFile->cacheKey + ".json"); + const std::filesystem::path objectFilePath = cacheDir / (sourceFile->cacheKey + "." + objectFileExtension); // Check if cache entry is available - if (!exists(symbolTableFilePath) || !exists(objectFilePath)) + if (!exists(metadataFilePath) || !exists(objectFilePath)) + return false; + + // Read metadata + std::ifstream metadataFile(metadataFilePath); + if (!metadataFile) + return false; + + nlohmann::json metadata; + try { + metadata = nlohmann::json::parse(metadataFile); + } catch (nlohmann::detail::parse_error &) { return false; + } - // Load symbol table + // Verify all transitive dependency object files exist and collect their paths. We keep + // these even though Spice imports register themselves via their own concludeCompilation, + // because runtime modules (string-rt, memory-rt, ...) are pulled in implicitly during + // symbol-table building - a stage that gets skipped for cache-restored files. Without + // this list those runtime objects would be missing from the link. The linker dedupes the + // overlap with deps that did register themselves. + if (metadata.contains("dependencies")) { + for (const auto &depKey : metadata["dependencies"]) { + const std::string key = depKey.get(); + const std::filesystem::path depObjectFilePath = cacheDir / (key + "." + objectFileExtension); + if (!exists(depObjectFilePath)) + return false; + sourceFile->cachedObjectFilePaths.push_back(depObjectFilePath); + } + } - // Set object file path + // Add this file's own object file last + sourceFile->cachedObjectFilePaths.push_back(objectFilePath); + + // Restore linker flags and additional source paths + if (metadata.contains("linkerFlags")) + for (const auto &flag : metadata["linkerFlags"]) + sourceFile->sourceLinkerFlags.push_back(flag.get()); + if (metadata.contains("additionalSourcePaths")) + for (const auto &path : metadata["additionalSourcePaths"]) + sourceFile->sourceAdditionalSourcePaths.emplace_back(path.get()); return true; } -void CacheManager::cacheSourceFile(const SourceFile * /*sourceFile*/) { - // Cache symbol table +void CacheManager::cacheSourceFile(const SourceFile *sourceFile) const { + // Don't cache if LTO is enabled and this isn't the main file (no object file produced) + if (cliOptions.useLTO && !sourceFile->isMainFile) + return; + + // Determine the source object file path (mirrors runObjectEmitter's path logic) + std::filesystem::path sourceObjFilePath = cliOptions.outputDir / sourceFile->filePath.filename(); + sourceObjFilePath.replace_extension("o"); + + // Determine cache paths + const char *objectFileExtension = SystemUtil::getOutputFileExtension(cliOptions, OutputContainer::OBJECT_FILE); + const std::filesystem::path cachedObjectFilePath = cacheDir / (sourceFile->cacheKey + "." + objectFileExtension); + const std::filesystem::path metadataFilePath = cacheDir / (sourceFile->cacheKey + ".json"); + + // Verify source object file exists + std::error_code error; + if (!std::filesystem::exists(sourceObjFilePath, error) || error) + return; + + // Ensure cache directory exists + std::filesystem::create_directories(cacheDir, error); + if (error) + return; + + // Copy object file to cache + std::filesystem::copy_file(sourceObjFilePath, cachedObjectFilePath, std::filesystem::copy_options::overwrite_existing, error); + if (error) + return; + + // Collect all transitive dependency cache keys, linker flags, and additional source paths. + // We need the transitive list so that cache-restored files can replay the full linker input + // even for implicit deps (e.g. runtime modules requested during symbol-table building, which + // a cache hit skips). The linker dedupes the overlap with deps that register themselves. + std::vector depCacheKeys; + std::vector allLinkerFlags = sourceFile->sourceLinkerFlags; + std::vector allAdditionalSourcePaths; + for (const auto &p : sourceFile->sourceAdditionalSourcePaths) + allAdditionalSourcePaths.push_back(p.string()); + std::unordered_set visited; + std::queue worklist; + for (const SourceFile *dep : sourceFile->dependencies | std::views::values) + worklist.push(dep); + while (!worklist.empty()) { + const SourceFile *dep = worklist.front(); + worklist.pop(); + if (!visited.insert(dep->cacheKey).second) + continue; + depCacheKeys.push_back(dep->cacheKey); + allLinkerFlags.insert(allLinkerFlags.end(), dep->sourceLinkerFlags.begin(), dep->sourceLinkerFlags.end()); + for (const auto &p : dep->sourceAdditionalSourcePaths) + allAdditionalSourcePaths.push_back(p.string()); + for (const SourceFile *transitiveDep : dep->dependencies | std::views::values) + worklist.push(transitiveDep); + } + + // Write metadata file + nlohmann::json metadata; + metadata["sourceFile"] = sourceFile->filePath.string(); + metadata["fileName"] = sourceFile->fileName; + metadata["cacheKey"] = sourceFile->cacheKey; + metadata["dependencies"] = depCacheKeys; + metadata["linkerFlags"] = allLinkerFlags; + metadata["additionalSourcePaths"] = allAdditionalSourcePaths; + std::ofstream metadataStream(metadataFilePath); + if (metadataStream) + metadataStream << metadata.dump(); +} + +// Hash the content of a single linker input that's not produced by the Spice cache itself +// (e.g. C/C++ files referenced via @core.linker.additionalSource). Returns a sentinel that +// folds the path in if the file can't be opened, so a vanished file still produces a stable +// (but different) cache key. +std::string hashLinkedFile(const std::filesystem::path &path) { + std::ifstream stream(path, std::ios::binary); + if (!stream) + return "missing:" + path.string(); + std::stringstream content; + content << stream.rdbuf(); + return std::to_string(std::hash{}(content.str())); +} + +std::string computeExecutableCacheKey(const std::vector &objectFileCacheKeys, + const std::vector &linkerFlags, + const std::vector &additionalSourcePaths, + const CliOptions &cliOptions) { + std::stringstream components; + for (const std::string &key : objectFileCacheKeys) + components << key; + for (const std::string &flag : linkerFlags) + components << flag; + // Sort additional source paths so traversal order doesn't perturb the key, then fold in + // path + content hash. Without this, edits to a referenced C/C++ source would leave every + // Spice object cache key unchanged, and we'd serve a stale executable. + std::vector sortedAdditionalSources = additionalSourcePaths; + std::ranges::sort(sortedAdditionalSources); + for (const std::filesystem::path &additionalSource : sortedAdditionalSources) + components << additionalSource.string() << '\0' << hashLinkedFile(additionalSource); + components << static_cast(cliOptions.outputContainer); + components << cliOptions.staticLinking; + return std::to_string(std::hash{}(components.str())); +} + +bool CacheManager::lookupExecutable(const std::vector &objectFileCacheKeys, + const std::vector &linkerFlags, + const std::vector &additionalSourcePaths, + std::filesystem::path &cachedExecutablePath) const { + const std::string execCacheKey = computeExecutableCacheKey(objectFileCacheKeys, linkerFlags, additionalSourcePaths, cliOptions); + + // Determine expected extension + const char *extension = SystemUtil::getOutputFileExtension(cliOptions, cliOptions.outputContainer); + const std::string fileName = execCacheKey + (strlen(extension) > 0 ? "." + std::string(extension) : ""); + const std::filesystem::path execPath = cacheDir / fileName; + + if (!exists(execPath)) + return false; + + cachedExecutablePath = execPath; + return true; +} + +void CacheManager::cacheExecutable(const std::vector &objFileCacheKeys, const std::vector &linkerFlags, + const std::vector &additionalSourcePaths, + const std::filesystem::path &executablePath) const { + const std::string execCacheKey = computeExecutableCacheKey(objFileCacheKeys, linkerFlags, additionalSourcePaths, cliOptions); + + // Determine cached file name + const char *extension = SystemUtil::getOutputFileExtension(cliOptions, cliOptions.outputContainer); + const std::string fileName = execCacheKey + (strlen(extension) > 0 ? "." + std::string(extension) : ""); + const std::filesystem::path cachedExecPath = cacheDir / fileName; + + // Verify executable exists + std::error_code error; + if (!std::filesystem::exists(executablePath, error) || error) + return; + + // Ensure cache directory exists + std::filesystem::create_directories(cacheDir, error); + if (error) + return; + + // Copy executable to cache + std::filesystem::copy_file(executablePath, cachedExecPath, std::filesystem::copy_options::overwrite_existing, error); +} + +void CacheManager::linkOrRestoreExecutable(GlobalResourceManager &resourceManager) const { + const ExternalLinkerInterface &linker = resourceManager.linker; + + // Collect object file cache keys and any external linker inputs (e.g. C/C++ files added + // via @core.linker.additionalSource) that participate in the executable cache key. + std::vector objectFileCacheKeys; + std::vector additionalSourcePaths; + for (const auto &sourceFile : resourceManager.sourceFiles | std::views::values) { + objectFileCacheKeys.push_back(sourceFile->cacheKey); + for (const std::filesystem::path &additionalSource : sourceFile->sourceAdditionalSourcePaths) + additionalSourcePaths.push_back(additionalSource); + } - // Cache object file + // Check if we have a cached executable + std::filesystem::path cachedExecutablePath; + if (!cliOptions.ignoreCache && + lookupExecutable(objectFileCacheKeys, linker.getLinkerFlags(), additionalSourcePaths, cachedExecutablePath)) { + // Restore cached executable + std::error_code ec; + std::filesystem::create_directories(linker.outputPath.parent_path(), ec); + std::filesystem::copy_file(cachedExecutablePath, linker.outputPath, std::filesystem::copy_options::overwrite_existing, ec); + } else { + // Link and cache the result + linker.run(); + if (!cliOptions.ignoreCache) + cacheExecutable(objectFileCacheKeys, linker.getLinkerFlags(), additionalSourcePaths, linker.outputPath); + } } -} // namespace spice::compiler \ No newline at end of file +} // namespace spice::compiler diff --git a/src/global/CacheManager.h b/src/global/CacheManager.h index 020007c2a..497f836c6 100644 --- a/src/global/CacheManager.h +++ b/src/global/CacheManager.h @@ -3,10 +3,14 @@ #pragma once #include +#include +#include namespace spice::compiler { // Forward declarations +class ExternalLinkerInterface; +class GlobalResourceManager; class SourceFile; struct CliOptions; @@ -20,8 +24,16 @@ class CacheManager { CacheManager &operator=(const CacheManager &) = delete; // Public methods - bool lookupSourceFile(const SourceFile *sourceFile) const; - void cacheSourceFile(const SourceFile *sourceFile); + std::string computeCacheKey(const std::string &sourceCode, const std::vector &depCacheKeys = {}) const; + bool lookupSourceFile(SourceFile *sourceFile) const; + void cacheSourceFile(const SourceFile *sourceFile) const; + bool lookupExecutable(const std::vector &objectFileCacheKeys, const std::vector &linkerFlags, + const std::vector &additionalSourcePaths, + std::filesystem::path &cachedExecutablePath) const; + void cacheExecutable(const std::vector &objFileCacheKeys, const std::vector &linkerFlags, + const std::vector &additionalSourcePaths, + const std::filesystem::path &executablePath) const; + void linkOrRestoreExecutable(GlobalResourceManager &resourceManager) const; private: // Private members diff --git a/src/global/GlobalResourceManager.h b/src/global/GlobalResourceManager.h index 6fb5b0ddb..75a2a5e4a 100644 --- a/src/global/GlobalResourceManager.h +++ b/src/global/GlobalResourceManager.h @@ -53,7 +53,7 @@ class GlobalResourceManager { std::unordered_map> sourceFiles; // The GlobalResourceManager owns all source files const CliOptions &cliOptions; ExternalLinkerInterface linker; - CacheManager cacheManager; + CacheManager cacheManager; RuntimeModuleManager runtimeModuleManager; Timer totalTimer; ErrorManager errorManager; diff --git a/src/linker/ExternalLinkerInterface.cpp b/src/linker/ExternalLinkerInterface.cpp index a95bb2476..39f09d788 100644 --- a/src/linker/ExternalLinkerInterface.cpp +++ b/src/linker/ExternalLinkerInterface.cpp @@ -2,6 +2,7 @@ #include "ExternalLinkerInterface.h" +#include #include #include @@ -197,14 +198,20 @@ void ExternalLinkerInterface::archive() const { * * @param path Path to the object file */ -void ExternalLinkerInterface::addFileToLinkage(const std::filesystem::path &path) { linkedFiles.push_back(path); } +void ExternalLinkerInterface::addFileToLinkage(const std::filesystem::path &path) { + if (std::ranges::find(linkedFiles, path) == linkedFiles.end()) + linkedFiles.push_back(path); +} /** * Add another linker flag for the call to the linker executable * * @param flag Linker flag */ -void ExternalLinkerInterface::addLinkerFlag(const std::string &flag) { linkerFlags.push_back(flag); } +void ExternalLinkerInterface::addLinkerFlag(const std::string &flag) { + if (std::ranges::find(linkerFlags, flag) == linkerFlags.end()) + linkerFlags.push_back(flag); +} /** * Add another source file to compile and link in (C or C++) diff --git a/src/linker/ExternalLinkerInterface.h b/src/linker/ExternalLinkerInterface.h index 04ec2fc18..fa32bfe11 100644 --- a/src/linker/ExternalLinkerInterface.h +++ b/src/linker/ExternalLinkerInterface.h @@ -28,6 +28,8 @@ class ExternalLinkerInterface { void addLinkerFlag(const std::string &flag); void addAdditionalSourcePath(std::filesystem::path additionalSource); void requestLibMathLinkage(); + [[nodiscard]] const std::vector &getLinkerFlags() const { return linkerFlags; } + [[nodiscard]] const std::vector &getLinkedFiles() const { return linkedFiles; } // Public members std::filesystem::path outputPath; diff --git a/src/main.cpp b/src/main.cpp index 727974ce6..d710cb830 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -37,7 +37,7 @@ bool compileProject(const CliOptions &cliOptions) { // Link the target executable (link object files to executable/library) if (cliOptions.outputContainer != OutputContainer::OBJECT_FILE) { resourceManager.linker.prepare(); - resourceManager.linker.run(); + resourceManager.cacheManager.linkOrRestoreExecutable(); resourceManager.linker.cleanup(); } diff --git a/src/symboltablebuilder/SymbolTableBuilder.cpp b/src/symboltablebuilder/SymbolTableBuilder.cpp index b7e7a37b1..2a1d50d48 100644 --- a/src/symboltablebuilder/SymbolTableBuilder.cpp +++ b/src/symboltablebuilder/SymbolTableBuilder.cpp @@ -652,13 +652,18 @@ std::any SymbolTableBuilder::visitModAttr(ModAttrNode *node) { values = attrs->getAttrValuesByName(ATTR_CORE_WINDOWS_LINKER_FLAG); linkerFlagValues.insert(linkerFlagValues.end(), values.begin(), values.end()); } - for (const CompileTimeValue *value : linkerFlagValues) - resourceManager.linker.addLinkerFlag(resourceManager.compileTimeStringValues.at(value->stringValueOffset)); + for (const CompileTimeValue *value : linkerFlagValues) { + const std::string &flag = resourceManager.compileTimeStringValues.at(value->stringValueOffset); + resourceManager.linker.addLinkerFlag(flag); + sourceFile->sourceLinkerFlags.push_back(flag); + } // core.linker.additionalSource for (const CompileTimeValue *value : attrs->getAttrValuesByName(ATTR_CORE_LINKER_ADDITIONAL_SOURCE)) { const std::string &stringValue = resourceManager.compileTimeStringValues.at(value->stringValueOffset); - resourceManager.linker.addAdditionalSourcePath(sourceFile->filePath.parent_path() / stringValue); + const std::filesystem::path additionalSourcePath = sourceFile->filePath.parent_path() / stringValue; + resourceManager.linker.addAdditionalSourcePath(additionalSourcePath); + sourceFile->sourceAdditionalSourcePaths.push_back(additionalSourcePath); } return nullptr; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 49e8b8e4d..83ace978f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,6 +13,7 @@ set(SOURCES # Unit tests unittest/UnitBlockAllocator.cpp unittest/UnitCommonUtil.cpp + unittest/UnitCompileCache.cpp unittest/UnitFileUtil.cpp unittest/UnitSystemUtil.cpp unittest/UnitDriver.cpp diff --git a/test/unittest/UnitBlockAllocator.cpp b/test/unittest/UnitBlockAllocator.cpp index 01ebecda6..a39e489d3 100644 --- a/test/unittest/UnitBlockAllocator.cpp +++ b/test/unittest/UnitBlockAllocator.cpp @@ -1,5 +1,4 @@ // Copyright (c) 2021-2026 ChilliBits. All rights reserved. -// LCOV_EXCL_START #include #include diff --git a/test/unittest/UnitCommonUtil.cpp b/test/unittest/UnitCommonUtil.cpp index 88799b3ce..fc5349a7b 100644 --- a/test/unittest/UnitCommonUtil.cpp +++ b/test/unittest/UnitCommonUtil.cpp @@ -1,10 +1,11 @@ // Copyright (c) 2021-2026 ChilliBits. All rights reserved. -// LCOV_EXCL_START #include #include +// LCOV_EXCL_START + namespace spice::testing { using namespace spice::compiler; diff --git a/test/unittest/UnitCompileCache.cpp b/test/unittest/UnitCompileCache.cpp new file mode 100644 index 000000000..dfea6a4ea --- /dev/null +++ b/test/unittest/UnitCompileCache.cpp @@ -0,0 +1,413 @@ +// Copyright (c) 2021-2026 ChilliBits. All rights reserved. + +#include +#include + +#include + +#include +#include +#include +#include + +#include + +// LCOV_EXCL_START + +namespace spice::testing { + +using namespace spice::compiler; + +namespace { + +std::filesystem::path makeUniqueCacheDir() { + std::random_device rd; + std::mt19937_64 rng(rd()); + const std::string suffix = "spice-cache-test-" + std::to_string(rng()); + std::filesystem::path dir = std::filesystem::temp_directory_path() / suffix; + std::filesystem::create_directories(dir); + return dir; +} + +void writeDummyFile(const std::filesystem::path &path, const std::string &content) { + std::ofstream stream(path); + stream << content; +} + +class CompileCacheTest : public ::testing::Test { +protected: + void SetUp() override { + cacheDir = makeUniqueCacheDir(); + outputDir = makeUniqueCacheDir(); + cliOptions.cacheDir = cacheDir; + cliOptions.outputDir = outputDir; + } + + void TearDown() override { + std::error_code ec; + std::filesystem::remove_all(cacheDir, ec); + std::filesystem::remove_all(outputDir, ec); + } + + CliOptions cliOptions; + std::filesystem::path cacheDir; + std::filesystem::path outputDir; +}; + +} // namespace + +TEST_F(CompileCacheTest, ComputeCacheKeyIsDeterministic) { + const CacheManager manager(cliOptions); + const std::string source = "f main() { return 0; }"; + ASSERT_EQ(manager.computeCacheKey(source), manager.computeCacheKey(source)); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForDifferentContent) { + const CacheManager manager(cliOptions); + const std::string a = "f main() { return 0; }"; + const std::string b = "f main() { return 1; }"; + ASSERT_NE(manager.computeCacheKey(a), manager.computeCacheKey(b)); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForBuildMode) { + const std::string source = "f main() { return 0; }"; + + cliOptions.buildMode = BuildMode::DEBUG; + const CacheManager managerDebug(cliOptions); + const std::string keyDebug = managerDebug.computeCacheKey(source); + + cliOptions.buildMode = BuildMode::RELEASE; + const CacheManager managerRelease(cliOptions); + const std::string keyRelease = managerRelease.computeCacheKey(source); + + ASSERT_NE(keyDebug, keyRelease); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForOptLevel) { + const std::string source = "f main() { return 0; }"; + + cliOptions.optLevel = OptLevel::O0; + const CacheManager managerO0(cliOptions); + const std::string keyO0 = managerO0.computeCacheKey(source); + + cliOptions.optLevel = OptLevel::O3; + const CacheManager managerO3(cliOptions); + const std::string keyO3 = managerO3.computeCacheKey(source); + + ASSERT_NE(keyO0, keyO3); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForSanitizer) { + const std::string source = "f main() { return 0; }"; + + cliOptions.instrumentation.sanitizer = Sanitizer::NONE; + const CacheManager managerNone(cliOptions); + const std::string keyNone = managerNone.computeCacheKey(source); + + cliOptions.instrumentation.sanitizer = Sanitizer::ADDRESS; + const CacheManager managerAsan(cliOptions); + const std::string keyAsan = managerAsan.computeCacheKey(source); + + ASSERT_NE(keyNone, keyAsan); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForDebugInfo) { + const std::string source = "f main() { return 0; }"; + + cliOptions.instrumentation.generateDebugInfo = false; + const CacheManager cmNoDebug(cliOptions); + const std::string keyNoDebug = cmNoDebug.computeCacheKey(source); + + cliOptions.instrumentation.generateDebugInfo = true; + const CacheManager cmDebug(cliOptions); + const std::string keyDebug = cmDebug.computeCacheKey(source); + + ASSERT_NE(keyNoDebug, keyDebug); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForLTO) { + const std::string source = "f main() { return 0; }"; + + cliOptions.useLTO = false; + const CacheManager managerNoLto(cliOptions); + const std::string keyNoLto = managerNoLto.computeCacheKey(source); + + cliOptions.useLTO = true; + const CacheManager managerLto(cliOptions); + const std::string keyLto = managerLto.computeCacheKey(source); + + ASSERT_NE(keyNoLto, keyLto); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyDiffersForDepCacheKeys) { + const CacheManager manager(cliOptions); + const std::string source = "f main() { return 0; }"; + + const std::string keyNoDeps = manager.computeCacheKey(source); + const std::string keyWithDep = manager.computeCacheKey(source, {"dep-v1"}); + const std::string keyWithDepChanged = manager.computeCacheKey(source, {"dep-v2"}); + + ASSERT_NE(keyNoDeps, keyWithDep); + ASSERT_NE(keyWithDep, keyWithDepChanged); +} + +TEST_F(CompileCacheTest, ComputeCacheKeyIsOrderIndependentInDeps) { + const CacheManager manager(cliOptions); + const std::string source = "f main() { return 0; }"; + // Dep cache keys must be sorted internally, so call sites do not need to provide a particular + // order. This matters because dep traversal order is not stable across builds. + ASSERT_EQ(manager.computeCacheKey(source, {"a", "b", "c"}), manager.computeCacheKey(source, {"c", "a", "b"})); +} + +TEST_F(CompileCacheTest, LookupExecutableMissReturnsFalse) { + const CacheManager manager(cliOptions); + std::filesystem::path resolved; + ASSERT_FALSE(manager.lookupExecutable({"key-a", "key-b"}, {"-lm"}, {}, resolved)); + ASSERT_TRUE(resolved.empty()); +} + +TEST_F(CompileCacheTest, CacheExecutableRoundTrip) { + const CacheManager manager(cliOptions); + + const std::filesystem::path executablePath = outputDir / "my-program"; + writeDummyFile(executablePath, "executable-bytes"); + + const std::vector objectKeys = {"obj-1", "obj-2"}; + const std::vector linkerFlags = {"-lm", "-lpthread"}; + + manager.cacheExecutable(objectKeys, linkerFlags, {}, executablePath); + + std::filesystem::path resolved; + ASSERT_TRUE(manager.lookupExecutable(objectKeys, linkerFlags, {}, resolved)); + ASSERT_TRUE(std::filesystem::exists(resolved)); + ASSERT_EQ(cacheDir, resolved.parent_path()); +} + +TEST_F(CompileCacheTest, CacheExecutableNonExistingSourceIsNoop) { + const CacheManager manager(cliOptions); + + const std::filesystem::path missingExecutable = outputDir / "does-not-exist"; + const std::vector objectKeys = {"obj-1"}; + constexpr std::vector linkerFlags; + + manager.cacheExecutable(objectKeys, linkerFlags, {}, missingExecutable); + + std::filesystem::path resolved; + ASSERT_FALSE(manager.lookupExecutable(objectKeys, linkerFlags, {}, resolved)); +} + +TEST_F(CompileCacheTest, LookupExecutableMissesWhenObjectKeysDiffer) { + const CacheManager manager(cliOptions); + + const std::filesystem::path executablePath = outputDir / "program"; + writeDummyFile(executablePath, "executable-bytes"); + const std::vector linkerFlags = {"-lm"}; + + manager.cacheExecutable({"obj-1", "obj-2"}, linkerFlags, {}, executablePath); + + std::filesystem::path resolved; + ASSERT_FALSE(manager.lookupExecutable({"obj-1", "obj-3"}, linkerFlags, {}, resolved)); +} + +TEST_F(CompileCacheTest, LookupExecutableMissesWhenLinkerFlagsDiffer) { + const CacheManager manager(cliOptions); + + const std::filesystem::path executablePath = outputDir / "program"; + writeDummyFile(executablePath, "executable-bytes"); + const std::vector objectKeys = {"obj-1"}; + + manager.cacheExecutable(objectKeys, {"-lm"}, {}, executablePath); + + std::filesystem::path resolved; + ASSERT_FALSE(manager.lookupExecutable(objectKeys, {"-lpthread"}, {}, resolved)); +} + +TEST_F(CompileCacheTest, LookupExecutableMissesWhenStaticLinkingDiffers) { + cliOptions.staticLinking = false; + const CacheManager manager(cliOptions); + + const std::filesystem::path executablePath = outputDir / "program"; + writeDummyFile(executablePath, "executable-bytes"); + const std::vector objectKeys = {"obj-1"}; + const std::vector linkerFlags = {"-lm"}; + + manager.cacheExecutable(objectKeys, linkerFlags, {}, executablePath); + + // Same cacheDir, but a CacheManager configured with staticLinking=true should miss + cliOptions.staticLinking = true; + const CacheManager cmStatic(cliOptions); + std::filesystem::path resolved; + ASSERT_FALSE(cmStatic.lookupExecutable(objectKeys, linkerFlags, {}, resolved)); +} + +TEST_F(CompileCacheTest, LookupExecutableMissesWhenOutputContainerDiffers) { + cliOptions.outputContainer = OutputContainer::EXECUTABLE; + const CacheManager managerExec(cliOptions); + + const std::filesystem::path executablePath = outputDir / "program"; + writeDummyFile(executablePath, "executable-bytes"); + const std::vector objectKeys = {"obj-1"}; + const std::vector linkerFlags = {"-lm"}; + + managerExec.cacheExecutable(objectKeys, linkerFlags, {}, executablePath); + + cliOptions.outputContainer = OutputContainer::SHARED_LIBRARY; + const CacheManager managerShared(cliOptions); + std::filesystem::path resolved; + ASSERT_FALSE(managerShared.lookupExecutable(objectKeys, linkerFlags, {}, resolved)); +} + +TEST_F(CompileCacheTest, LookupExecutableMissesWhenAdditionalSourceContentChanges) { + const CacheManager manager(cliOptions); + + // C/C++ files referenced via @core.linker.additionalSource must contribute to the executable + // cache key, otherwise editing them would silently keep serving the previously linked binary. + const std::filesystem::path additionalSource = outputDir / "extra.c"; + writeDummyFile(additionalSource, "int compute() { return 1; }\n"); + + const std::filesystem::path executablePath = outputDir / "program"; + writeDummyFile(executablePath, "executable-bytes"); + + const std::vector objectKeys = {"obj-1"}; + const std::vector linkerFlags = {"-lm"}; + const std::vector additionalSources = {additionalSource}; + + manager.cacheExecutable(objectKeys, linkerFlags, additionalSources, executablePath); + + std::filesystem::path resolved; + ASSERT_TRUE(manager.lookupExecutable(objectKeys, linkerFlags, additionalSources, resolved)); + + // Editing the additional source must invalidate the cached executable + writeDummyFile(additionalSource, "int compute() { return 2; }\n"); + ASSERT_FALSE(manager.lookupExecutable(objectKeys, linkerFlags, additionalSources, resolved)); +} + +// Walks through a 3-file dependency chain (main -> utils -> math) using the real cache-key +// folding behavior, caches every file, then mutates math's source and verifies that the change +// cascades into utils and main: their cache keys move with the dep, so all three miss and must +// be recompiled. Also verifies that each restored entry holds only its own object file (no +// transitive accumulation, which would cause duplicate linker registrations downstream). +TEST_F(CompileCacheTest, MultiFileDependencyChangeForcesRecompilation) { + // Configure native target so that SourceFile construction can look up an LLVM target + cliOptions.targetTriple = llvm::Triple(llvm::Triple::normalize(llvm::sys::getProcessTriple())); + cliOptions.isNativeTarget = true; + + // Place stub source files on disk (their contents are irrelevant: cache keys are derived from + // the in-memory "source" strings below and assigned to SourceFile::cacheKey directly). + const std::filesystem::path mathPath = outputDir / "math.spice"; + const std::filesystem::path utilsPath = outputDir / "utils.spice"; + const std::filesystem::path mainPath = outputDir / "main.spice"; + writeDummyFile(mathPath, ""); + writeDummyFile(utilsPath, ""); + writeDummyFile(mainPath, ""); + + // GlobalResourceManager initializes LLVM targets and owns a CacheManager wired to cliOptions + GlobalResourceManager resourceManager(cliOptions); + CacheManager &manager = resourceManager.cacheManager; + + // Sources that drive cache keys for the initial state + const std::string mathSrcV1 = "f add(int a, int b) { return a + b; }"; + const std::string utilsSrc = "import \"math\"; f helper(int x) { return add(x, 1); }"; + const std::string mainSrc = "import \"utils\"; f main() { return helper(41); }"; + + SourceFile *math = resourceManager.createSourceFile(nullptr, "math", mathPath, false); + SourceFile *utils = resourceManager.createSourceFile(nullptr, "utils", utilsPath, false); + SourceFile *main = resourceManager.createSourceFile(nullptr, "main", mainPath, false); + math->isMainFile = false; + utils->isMainFile = false; + main->isMainFile = true; + + // Wire up the dependency chain: main -> utils -> math + utils->dependencies["math"] = math; + main->dependencies["utils"] = utils; + + // Assign cache keys as runImportCollector would: each file's key folds in its transitive + // dep cache keys, so changes to any dep propagate up the chain. + const auto recomputeKeys = [&] { + math->cacheKey = manager.computeCacheKey(mathSrcV1); + utils->cacheKey = manager.computeCacheKey(utilsSrc, {math->cacheKey}); + main->cacheKey = manager.computeCacheKey(mainSrc, {utils->cacheKey, math->cacheKey}); + }; + recomputeKeys(); + + // Pretend the back end emitted object files into outputDir (cacheSourceFile picks them up there) + writeDummyFile(outputDir / "math.o", "math-v1-obj"); + writeDummyFile(outputDir / "utils.o", "utils-obj"); + writeDummyFile(outputDir / "main.o", "main-obj"); + + manager.cacheSourceFile(math); + manager.cacheSourceFile(utils); + manager.cacheSourceFile(main); + + // Reset lookup-output fields between lookups, since they accumulate on each call + auto lookup = [&](SourceFile *sourceFile) { + sourceFile->cachedObjectFilePaths.clear(); + sourceFile->sourceLinkerFlags.clear(); + sourceFile->sourceAdditionalSourcePaths.clear(); + return manager.lookupSourceFile(sourceFile); + }; + + // Phase 1: nothing changed since caching - every file hits and pulls its transitive dep + // objects (kept so runtime modules pulled in implicitly at symbol-table-building time still + // make it into the link for cache-restored files; the linker dedupes the overlap). + ASSERT_TRUE(lookup(math)); + ASSERT_EQ(1u, math->cachedObjectFilePaths.size()); // math.o + ASSERT_TRUE(lookup(utils)); + ASSERT_EQ(2u, utils->cachedObjectFilePaths.size()); // math.o + utils.o + ASSERT_TRUE(lookup(main)); + ASSERT_EQ(3u, main->cachedObjectFilePaths.size()); // utils.o, math.o, main.o + + // Phase 2: math's source changes -> its cache key changes -> utils and main keys also change + // (because the fold pulls in the new math key transitively). All three miss and recompile. + const std::string oldMathKey = math->cacheKey; + const std::string oldUtilsKey = utils->cacheKey; + const std::string oldMainKey = main->cacheKey; + + const std::string mathSrcV2 = "f add(int a, int b) { return a + b + 0; }"; + math->cacheKey = manager.computeCacheKey(mathSrcV2); + utils->cacheKey = manager.computeCacheKey(utilsSrc, {math->cacheKey}); + main->cacheKey = manager.computeCacheKey(mainSrc, {utils->cacheKey, math->cacheKey}); + ASSERT_NE(oldMathKey, math->cacheKey); + ASSERT_NE(oldUtilsKey, utils->cacheKey); + ASSERT_NE(oldMainKey, main->cacheKey); + + ASSERT_FALSE(lookup(math)); + ASSERT_FALSE(lookup(utils)); + ASSERT_FALSE(lookup(main)); +} + +// Provokes the bug that was fixed: before transitive dep cache keys were folded into a +// file's own cache key, a dependent whose source text was unchanged would keep cache-hitting +// against a stale object file even when one of its dependencies had been edited - causing +// the linker to consume out-of-date code. With the fold in place, mutating any dependency's +// cache key must propagate into every dependent's cache key, so the dependent's lookup misses, +// and it gets recompiled against the new dependency. +TEST_F(CompileCacheTest, DependencyChangeInvalidatesDependentCacheKey) { + const CacheManager manager(cliOptions); + + // Initial cache keys for a dependent file (utils) and its single dependency (math) + const std::string mathSrc = "f add(int a, int b) { return a + b; }"; + const std::string utilsSrc = "import \"math\"; f helper(int x) { return add(x, 1); }"; + + const std::string mathKeyV1 = manager.computeCacheKey(mathSrc); + const std::string utilsKeyV1 = manager.computeCacheKey(utilsSrc, {mathKeyV1}); + + // The dependent's own source text didn't change - only the dependency did. + const std::string mathSrcChanged = "f add(int a, int b) { return a + b + 0; }"; + const std::string mathKeyV2 = manager.computeCacheKey(mathSrcChanged); + ASSERT_NE(mathKeyV1, mathKeyV2); + + // Without the fold, recomputing utils cache key here would yield the same value as before + // (since utilsSrc is unchanged) and the stale cached utils.o would be served. The fold makes + // the dependent's cache key a function of the dependency's cache key, so utils now misses too. + const std::string utilsKeyV2 = manager.computeCacheKey(utilsSrc, {mathKeyV2}); + ASSERT_NE(utilsKeyV1, utilsKeyV2); + + // Sanity check: without any dep change the dependent's cache key stays stable, so we don't + // pay unnecessary rebuilds when nothing actually moved. + ASSERT_EQ(utilsKeyV1, manager.computeCacheKey(utilsSrc, {mathKeyV1})); +} + +} // namespace spice::testing + +// LCOV_EXCL_STOP \ No newline at end of file diff --git a/test/unittest/UnitDriver.cpp b/test/unittest/UnitDriver.cpp index cebf6cf72..e5eb23f0e 100644 --- a/test/unittest/UnitDriver.cpp +++ b/test/unittest/UnitDriver.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2021-2026 ChilliBits. All rights reserved. -// LCOV_EXCL_START #include #include #include +// LCOV_EXCL_START + namespace spice::testing { using namespace spice::compiler; diff --git a/test/unittest/UnitFileUtil.cpp b/test/unittest/UnitFileUtil.cpp index c75b28eca..6430ec0a3 100644 --- a/test/unittest/UnitFileUtil.cpp +++ b/test/unittest/UnitFileUtil.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2021-2026 ChilliBits. All rights reserved. -// LCOV_EXCL_START #include #include #include +// LCOV_EXCL_START + namespace spice::testing { using namespace spice::compiler; diff --git a/test/unittest/UnitSystemUtil.cpp b/test/unittest/UnitSystemUtil.cpp index 38f135cab..d7e9c82b1 100644 --- a/test/unittest/UnitSystemUtil.cpp +++ b/test/unittest/UnitSystemUtil.cpp @@ -1,10 +1,11 @@ // Copyright (c) 2021-2026 ChilliBits. All rights reserved. -// LCOV_EXCL_START #include #include +// LCOV_EXCL_START + namespace spice::testing { using namespace spice::compiler;