From b4fb096944e0cb0ee95607835851075ce984c19e Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Mon, 17 Nov 2025 10:45:27 -0600 Subject: [PATCH 01/23] Added support for TOML based cofiguration files This commits adds support for TOML configuration files, thus making it possible to further customize the behaviour of the current executing CAPIO-CL running instance. --- CMakeLists.txt | 11 ++- capiocl.hpp | 7 +- include/configuration.h | 21 ++++ include/engine.h | 9 ++ include/monitor.h | 10 +- include/parser.h | 12 +++ include/printer.h | 1 + src/Engine.cpp | 12 ++- src/Parser.cpp | 2 + src/configuration.cpp | 54 +++++++++++ src/monitors/Multicast.cpp | 14 ++- src/parsers/v1_1.cpp | 188 ++++++++++++++++++++++++++++++++++++ tests/jsons/V1-1_test0.json | 32 ++++++ 13 files changed, 352 insertions(+), 21 deletions(-) create mode 100644 include/configuration.h create mode 100644 src/configuration.cpp create mode 100644 src/parsers/v1_1.cpp create mode 100644 tests/jsons/V1-1_test0.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ebcf6e..0f590e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,12 @@ add_compile_options(-Wall -Wextra -Wpedantic) # External projects ##################################### +FetchContent_Declare( + tomlplusplus + GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git + GIT_TAG v3.4.0 +) + FetchContent_Declare( jsoncons GIT_REPOSITORY https://github.com/danielaparker/jsoncons.git @@ -55,7 +61,7 @@ set(JSONCONS_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(JSONCONS_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(JSONCONS_BUILD_FUZZERS OFF CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(jsoncons) +FetchContent_MakeAvailable(jsoncons tomlplusplus) if (BUILD_PYTHON_BINDINGS) FetchContent_Declare( @@ -117,10 +123,11 @@ target_include_directories(libcapio_cl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src ${jsoncons_SOURCE_DIR}/include ${CAPIOCL_JSON_SCHEMAS_DIRECTORY} + ${TOMLPLUSPLUS_SOURCE_DIR}/include ) -# jsoncons is header-only, no linking required target_link_libraries(libcapio_cl PUBLIC) +target_link_libraries(libcapio_cl PRIVATE tomlplusplus::tomlplusplus) ##################################### # Install rules diff --git a/capiocl.hpp b/capiocl.hpp index c780746..8c1cb31 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -76,7 +76,8 @@ inline std::string sanitize(const std::string &input) { /// @brief Available versions of CAPIO-CL language struct CAPIO_CL_VERSION final { /// @brief Release 1.0 of CAPIO-CL - static constexpr char V1[] = "1.0"; + static constexpr char V1[] = "1.0"; + static constexpr char V1_1[] = "1.1"; }; namespace serializer { @@ -92,6 +93,10 @@ class MonitorException; namespace engine { class Engine; } + +namespace configuration { +class CapioClConfiguration; +} } // namespace capiocl #endif // CAPIO_CL_CAPIOCL_HPP \ No newline at end of file diff --git a/include/configuration.h b/include/configuration.h new file mode 100644 index 0000000..af4e2ca --- /dev/null +++ b/include/configuration.h @@ -0,0 +1,21 @@ +#ifndef CAPIO_CL_CONFIGURATION_H +#define CAPIO_CL_CONFIGURATION_H +#include +#include + +#include "capiocl.hpp" + +/// @brief Load configuration and store it from a CAPIO-CL TOML configuration file +class capiocl::configuration::CapioClConfiguration { + std::unordered_map config; + + public: + explicit CapioClConfiguration() = default; + ~CapioClConfiguration() = default; + + void load(const std::filesystem::path &path); + void getParameter(const std::string &key, int *value, int def_value) const; + void getParameter(const std::string &key, std::string *value, std::string def_value) const; +}; + +#endif \ No newline at end of file diff --git a/include/engine.h b/include/engine.h index 87fbfe8..f441a14 100644 --- a/include/engine.h +++ b/include/engine.h @@ -27,6 +27,9 @@ class Engine final { friend class serializer::Serializer; bool store_all_in_memory = false; + ///@brief Configuration imported from CAPIO-CL config TOML file + configuration::CapioClConfiguration* configuration; + /// @brief Monitor instance to check runtime information of CAPIO-CL files monitor::Monitor monitor; @@ -384,6 +387,12 @@ class Engine final { * @return true if both this instance and other are equivalent. false otherwise. */ bool operator==(const Engine &other) const; + + /** + * Load a CAPIO-CL TOML configuration file + * @param path + */ + void loadConfiguration(const std::string & path); }; } // namespace capiocl::engine diff --git a/include/monitor.h b/include/monitor.h index e0b77bf..e52dc2b 100644 --- a/include/monitor.h +++ b/include/monitor.h @@ -1,6 +1,8 @@ #ifndef CAPIO_CL_MONITOR_H #define CAPIO_CL_MONITOR_H +#include "configuration.h" + #include #include #include @@ -204,13 +206,9 @@ class MulticastMonitor final : public MonitorInterface { /** * @brief Construct a multicast-based monitor. * - * @param commit_ip_addr Multicast commit group address to use. - * @param commit_ip_port Multicast commit port to use. - * @param home_node_ip_addr Multicast home node group address to use. - * @param home_node_ip_port Multicast home node port to use. + *@param config const reference to CAPIO-CL configuration */ - MulticastMonitor(const std::string &commit_ip_addr, int commit_ip_port, - const std::string &home_node_ip_addr, int home_node_ip_port); + MulticastMonitor(const capiocl::configuration::CapioClConfiguration &config); /** * @brief Destructor; stops listener thread and cleans resources. diff --git a/include/parser.h b/include/parser.h index 8327d54..16b43a6 100644 --- a/include/parser.h +++ b/include/parser.h @@ -42,6 +42,18 @@ class Parser final { static engine::Engine *parse_v1(const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, bool store_only_in_memory); + + /** + * Parser for the V1.1 Specification of the CAPIO-CL language + * @param source Path of CAPIO-CL configuration file + * @param resolve_prefix Prefix to prepend to path if found to be relative + * @param store_only_in_memory Flag to set to returned instance of Engine if required to + * store all files in memory + * @return Parsed Engine. + */ + static engine::Engine *parse_v1_1(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, + bool store_only_in_memory); }; /** diff --git a/include/printer.h b/include/printer.h index 22a9bed..04ea2ec 100644 --- a/include/printer.h +++ b/include/printer.h @@ -2,6 +2,7 @@ #define CAPIO_CL_PRINTER_H #include #include +#include #ifndef HOST_NAME_MAX #define HOST_NAME_MAX 1024 diff --git a/src/Engine.cpp b/src/Engine.cpp index 6055c40..14f1689 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -3,6 +3,7 @@ #include #include "capiocl.hpp" +#include "include/configuration.h" #include "include/engine.h" #include "include/printer.h" @@ -136,10 +137,7 @@ capiocl::engine::Engine::Engine() { this->workflow_name = CAPIO_CL_DEFAULT_WF_NAME; } - // TODO: make selection of monitor available to user - monitor.registerMonitorBackend( - new monitor::MulticastMonitor("224.224.224.1", 12345, "224.224.224.2", 12345)); - monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); + configuration = new configuration::CapioClConfiguration(); } void capiocl::engine::Engine::_newFile(const std::filesystem::path &path) const { @@ -752,4 +750,10 @@ bool capiocl::engine::Engine::operator==(const capiocl::engine::Engine &other) c } } return true; +} +void capiocl::engine::Engine::loadConfiguration(const std::string &path) { + configuration->load(path); + + monitor.registerMonitorBackend(new monitor::MulticastMonitor(*configuration)); + monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); } \ No newline at end of file diff --git a/src/Parser.cpp b/src/Parser.cpp index 2b80076..87a1121 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -72,6 +72,8 @@ capiocl::engine::Engine *capiocl::parser::Parser::parse(const std::filesystem::p if (capio_cl_release == CAPIO_CL_VERSION::V1) { return available_parsers::parse_v1(source, resolve_prefix, store_only_in_memory); + } else if (capio_cl_release == CAPIO_CL_VERSION::V1_1) { + return available_parsers::parse_v1_1(source, resolve_prefix, store_only_in_memory); } else { throw ParserException("Invalid CAPIO-CL specification version!"); } diff --git a/src/configuration.cpp b/src/configuration.cpp new file mode 100644 index 0000000..b97cf21 --- /dev/null +++ b/src/configuration.cpp @@ -0,0 +1,54 @@ +#include "toml++/toml.hpp" +#include +#include + +#include "include/configuration.h" +#include "include/printer.h" + +void flatten_table(const toml::table &tbl, std::unordered_map &map, + const std::string &prefix = "") { + for (const auto &[key, value] : tbl) { + std::string full_key = + prefix.empty() ? std::string{key.str()} : prefix + "." + std::string{key.str()}; + + if (value.is_table()) { + flatten_table(*value.as_table(), map, full_key); + } else { + map[full_key] = value.as_string()->get(); + } + } +} + +void capiocl::configuration::CapioClConfiguration::load(const std::filesystem::path &path) { + if (path.empty()) { + return; + } + + toml::table tbl; + try { + tbl = toml::parse_file(path.string()); + } catch (const toml::parse_error &err) { + printer::print(printer::CLI_LEVEL_ERROR, err.what()); + } + flatten_table(tbl, config); +} + +void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, int *value, + const int def_value) const { + + if (config.find(key) != config.end()) { + *value = std::stoi(config.at(key)); + } else { + *value = def_value; + } +} + +void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, + std::string *value, + std::string def_value) const { + if (config.find(key) != config.end()) { + *value = config.at(key); + } else { + *value = std::move(def_value); + } +} \ No newline at end of file diff --git a/src/monitors/Multicast.cpp b/src/monitors/Multicast.cpp index acb0a21..1c8a700 100644 --- a/src/monitors/Multicast.cpp +++ b/src/monitors/Multicast.cpp @@ -183,14 +183,12 @@ void capiocl::monitor::MulticastMonitor::_send_message(const std::string &ip_add close(out_s); } -capiocl::monitor::MulticastMonitor::MulticastMonitor(const std::string &commit_ip_addr, - const int commit_ip_port, - const std::string &home_node_ip_addr, - int home_node_ip_port) { - MULTICAST_COMMIT_ADDR = commit_ip_addr; - MULTICAST_COMMIT_PORT = commit_ip_port; - MULTICAST_HOME_NODE_ADDR = home_node_ip_addr; - MULTICAST_HOME_NODE_PORT = home_node_ip_port; +capiocl::monitor::MulticastMonitor::MulticastMonitor( + const configuration::CapioClConfiguration &config) { + config.getParameter("monitor.commit.ip", &MULTICAST_COMMIT_ADDR, "224.224.224.1"); + config.getParameter("monitor.commit.port", &MULTICAST_COMMIT_PORT, 12345); + config.getParameter("monitor.homenode.ip", &MULTICAST_HOME_NODE_ADDR, "224.224.224.2"); + config.getParameter("monitor.homenode.port", &MULTICAST_HOME_NODE_PORT, 12345); commit_thread = std::thread(&commit_listener, std::ref(_committed_files), std::ref(committed_lock), diff --git a/src/parsers/v1_1.cpp b/src/parsers/v1_1.cpp new file mode 100644 index 0000000..e1bc305 --- /dev/null +++ b/src/parsers/v1_1.cpp @@ -0,0 +1,188 @@ +#include + +#include "capiocl.hpp" +#include "include/engine.h" +#include "include/parser.h" +#include "include/printer.h" + +capiocl::engine::Engine * +capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, + bool store_only_in_memory) { + std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; + auto engine = new engine::Engine(); + + // ---- Load JSON ---- + std::ifstream file(source); + + jsoncons::json doc = jsoncons::json::parse(file); + validate_json(doc); + + // ---- workflow name ---- + workflow_name = doc["name"].as(); + engine->setWorkflowName(workflow_name); + printer::print(printer::CLI_LEVEL_JSON, "Parsing configuration for workflow: " + workflow_name); + + // ---- CAPIOCL TOML CONFIGURATION ---- + auto toml_config_path = doc["configuration"].as(); + engine.loadConfiguration(toml_config_path); + + // ---- IO_Graph ---- + for (const auto &app : doc["IO_Graph"].array_range()) { + std::string app_name = app["name"].as(); + printer::print(printer::CLI_LEVEL_JSON, "Parsing config for app " + app_name); + + // ---- input_stream ---- + printer::print(printer::CLI_LEVEL_JSON, "Parsing input_stream for app " + app_name); + for (const auto &itm : app["input_stream"].array_range()) { + auto file_path = resolve(itm.as(), resolve_prefix); + engine->newFile(file_path); + engine->addConsumer(file_path, app_name); + } + + // ---- output_stream ---- + printer::print(printer::CLI_LEVEL_JSON, "Parsing output_stream for app " + app_name); + for (const auto &itm : app["output_stream"].array_range()) { + auto file_path = resolve(itm.as(), resolve_prefix); + engine->newFile(file_path); + engine->addProducer(file_path, app_name); + } + + // ---- streaming ---- + if (app.contains("streaming")) { + printer::print(printer::CLI_LEVEL_JSON, "Parsing streaming for app " + app_name); + for (const auto &stream_item : app["streaming"].array_range()) { + bool is_file = true; + std::vector streaming_names; + std::vector file_deps; + std::string commit_rule; + std::string fire_rule; + long int n_close = 0; + int64_t n_files = 0; + + if (stream_item.contains("name")) { + for (const auto &nm : stream_item["name"].array_range()) { + auto nm_resolved = resolve(nm.as(), resolve_prefix); + streaming_names.push_back(nm_resolved); + } + } else { + // At this point we have dirname, as either name or dirname is required + // This is checked by the JSON schema validation phase + is_file = false; + for (const auto &nm : stream_item["dirname"].array_range()) { + auto nm_resolved = resolve(nm.as(), resolve_prefix); + streaming_names.push_back(nm_resolved); + } + } + + // Commit rule (optional) + if (stream_item.contains("committed")) { + auto committed = stream_item["committed"].as(); + auto pos = committed.find(':'); + if (pos != std::string::npos) { + // If we reach here, we are certain that the commit rule is on_close + // as the json schema enforces the rule that the :n is allowed only for + // the on_close commit rule. + + size_t num_len; + std::string count_str = committed.substr(pos + 1); + n_close = std::stoi(count_str, &num_len); + + // clean up committed + committed = committed.substr(0, pos); + } + + commit_rule = committed; + + if (commit_rule == commitRules::ON_FILE) { + for (const auto &dep : stream_item["file_deps"].array_range()) { + auto dep_resolved = resolve(dep.as(), resolve_prefix); + file_deps.push_back(dep_resolved); + } + } + } else { + commit_rule = commitRules::ON_TERMINATION; + } + + // Firing rule (optional) + if (stream_item.contains("mode")) { + fire_rule = stream_item["mode"].as(); + } else { + fire_rule = fireRules::NO_UPDATE; + } + + // n_files (optional) + if (stream_item.contains("n_files") && !is_file) { + n_files = stream_item["n_files"].as(); + } + + for (auto &path : streaming_names) { + if (n_files != 0) { + engine->setDirectoryFileCount(path, n_files); + } + if (is_file) { + engine->setFile(path); + } else { + engine->setDirectory(path); + } + + engine->setCommitRule(path, commit_rule); + engine->setFireRule(path, fire_rule); + engine->setCommitedCloseNumber(path, n_close); + engine->setFileDeps(path, file_deps); + } + } + } + } + + // ---- permanent ---- + if (doc.contains("permanent")) { + for (const auto &item : doc["permanent"].array_range()) { + std::filesystem::path path = resolve(item.as(), resolve_prefix); + engine->newFile(path); + engine->setPermanent(path, true); + } + } + + // ---- exclude ---- + if (doc.contains("exclude")) { + for (const auto &item : doc["exclude"].array_range()) { + std::filesystem::path path = resolve(item.as(), resolve_prefix); + engine->newFile(path); + engine->setExclude(path, true); + } + } + + // ---- storage ---- + if (doc.contains("storage")) { + const auto &storage = doc["storage"]; + + if (storage.contains("memory")) { + for (const auto &f : storage["memory"].array_range()) { + std::string file_str = f.as(); + engine->setStoreFileInMemory(file_str); + } + } else { + printer::print(printer::CLI_LEVEL_INFO, "No MEM storage section found"); + } + + if (storage.contains("fs")) { + for (const auto &f : storage["fs"].array_range()) { + std::string file_str = f.as(); + engine->setStoreFileInFileSystem(file_str); + } + } else { + printer::print(printer::CLI_LEVEL_INFO, "No FS storage section found"); + } + } else { + printer::print(printer::CLI_LEVEL_INFO, "No storage section found"); + } + + // ---- Store only in memory ---- + if (store_only_in_memory) { + printer::print(printer::CLI_LEVEL_INFO, "Storing all files in memory"); + engine->setAllStoreInMemory(); + } + + return engine; +} \ No newline at end of file diff --git a/tests/jsons/V1-1_test0.json b/tests/jsons/V1-1_test0.json new file mode 100644 index 0000000..292ddcf --- /dev/null +++ b/tests/jsons/V1-1_test0.json @@ -0,0 +1,32 @@ +{ + "version": 1.1, + "configuration" : "/tmp/capio_cl.toml", + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "/tmp/file", + "file3" + ], + "output_stream": [ + "file1", + "file2" + ], + "streaming": [ + { + "name": [ + "/tmp/file" + ], + "committed": "on_termination", + "mode": "no_update" + }, + { + "name": [ + "/tmp/file1" + ] + } + ] + } + ] +} \ No newline at end of file From 82130cddab5e7c581ac7a799621112e21dcfa8c7 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Mon, 17 Nov 2025 11:31:07 -0600 Subject: [PATCH 02/23] Added default configuration --- include/configuration.h | 4 ++++ include/engine.h | 5 +++++ src/Engine.cpp | 10 ++++++++++ src/configuration.cpp | 4 ++++ src/parsers/v1.cpp | 2 ++ src/parsers/v1_1.cpp | 14 +++++++++----- 6 files changed, 34 insertions(+), 5 deletions(-) diff --git a/include/configuration.h b/include/configuration.h index af4e2ca..10e2e6c 100644 --- a/include/configuration.h +++ b/include/configuration.h @@ -7,8 +7,12 @@ /// @brief Load configuration and store it from a CAPIO-CL TOML configuration file class capiocl::configuration::CapioClConfiguration { + friend class engine::Engine; std::unordered_map config; +protected: + void set(const std::string& key, std::string value); + public: explicit CapioClConfiguration() = default; ~CapioClConfiguration() = default; diff --git a/include/engine.h b/include/engine.h index f441a14..e46ce34 100644 --- a/include/engine.h +++ b/include/engine.h @@ -393,6 +393,11 @@ class Engine final { * @param path */ void loadConfiguration(const std::string & path); + + /** + * Use default CAPIO-CL TOML configuration. + */ + void useDefaultConfiguration(); }; } // namespace capiocl::engine diff --git a/src/Engine.cpp b/src/Engine.cpp index 14f1689..5a85dc3 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -754,6 +754,16 @@ bool capiocl::engine::Engine::operator==(const capiocl::engine::Engine &other) c void capiocl::engine::Engine::loadConfiguration(const std::string &path) { configuration->load(path); + monitor.registerMonitorBackend(new monitor::MulticastMonitor(*configuration)); + monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); +} +void capiocl::engine::Engine::useDefaultConfiguration() { + + configuration->set("monitor.commit.port", "224.224.224.1"); + configuration->set("monitor.commit.port", "12345"); + configuration->set("monitor.homenode.ip", "224.224.224.2"); + configuration->set("monitor.homenode.port", "12345"); + monitor.registerMonitorBackend(new monitor::MulticastMonitor(*configuration)); monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); } \ No newline at end of file diff --git a/src/configuration.cpp b/src/configuration.cpp index b97cf21..06d7544 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -19,6 +19,10 @@ void flatten_table(const toml::table &tbl, std::unordered_mapuseDefaultConfiguration(); + // ---- Load JSON ---- std::ifstream file(source); diff --git a/src/parsers/v1_1.cpp b/src/parsers/v1_1.cpp index e1bc305..3ebec78 100644 --- a/src/parsers/v1_1.cpp +++ b/src/parsers/v1_1.cpp @@ -7,8 +7,8 @@ capiocl::engine::Engine * capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::path &source, - const std::filesystem::path &resolve_prefix, - bool store_only_in_memory) { + const std::filesystem::path &resolve_prefix, + bool store_only_in_memory) { std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; auto engine = new engine::Engine(); @@ -23,9 +23,13 @@ capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::pa engine->setWorkflowName(workflow_name); printer::print(printer::CLI_LEVEL_JSON, "Parsing configuration for workflow: " + workflow_name); - // ---- CAPIOCL TOML CONFIGURATION ---- - auto toml_config_path = doc["configuration"].as(); - engine.loadConfiguration(toml_config_path); + // ---- CAPIO-CL TOML CONFIGURATION ---- + if (doc.contains("configuration")) { + auto toml_config_path = doc["configuration"].as(); + engine->loadConfiguration(toml_config_path); + } else { + engine->useDefaultConfiguration(); + } // ---- IO_Graph ---- for (const auto &app : doc["IO_Graph"].array_range()) { From 699794ae78cf086d1603a6e87b9131bdc4008734 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Mon, 17 Nov 2025 14:30:53 -0600 Subject: [PATCH 03/23] Added default configuration --- include/engine.h | 8 ++++---- src/Engine.cpp | 5 ++++- src/parsers/v1.cpp | 2 +- src/parsers/v1_1.cpp | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/engine.h b/include/engine.h index e46ce34..e43d031 100644 --- a/include/engine.h +++ b/include/engine.h @@ -28,7 +28,7 @@ class Engine final { bool store_all_in_memory = false; ///@brief Configuration imported from CAPIO-CL config TOML file - configuration::CapioClConfiguration* configuration; + configuration::CapioClConfiguration *configuration; /// @brief Monitor instance to check runtime information of CAPIO-CL files monitor::Monitor monitor; @@ -100,8 +100,8 @@ class Engine final { void compute_directory_entry_count(const std::filesystem::path &path) const; public: - /// @brief Class constructor - explicit Engine(); + /// @brief Class constructorw + explicit Engine(bool use_default_settings = true); /// @brief Print the current CAPIO-CL configuration. void print() const; @@ -392,7 +392,7 @@ class Engine final { * Load a CAPIO-CL TOML configuration file * @param path */ - void loadConfiguration(const std::string & path); + void loadConfiguration(const std::string &path); /** * Use default CAPIO-CL TOML configuration. diff --git a/src/Engine.cpp b/src/Engine.cpp index 5a85dc3..13b8290 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -126,7 +126,7 @@ void capiocl::engine::Engine::print() const { printer::print(printer::CLI_LEVEL_JSON, ""); } -capiocl::engine::Engine::Engine() { +capiocl::engine::Engine::Engine(const bool use_default_settings) { node_name = std::string(1024, '\0'); gethostname(node_name.data(), node_name.size()); node_name.resize(std::strlen(node_name.c_str())); @@ -138,6 +138,9 @@ capiocl::engine::Engine::Engine() { } configuration = new configuration::CapioClConfiguration(); + if (use_default_settings) { + this->useDefaultConfiguration(); + } } void capiocl::engine::Engine::_newFile(const std::filesystem::path &path) const { diff --git a/src/parsers/v1.cpp b/src/parsers/v1.cpp index ac90ac6..a217206 100644 --- a/src/parsers/v1.cpp +++ b/src/parsers/v1.cpp @@ -10,7 +10,7 @@ capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; - auto engine = new engine::Engine(); + auto engine = new engine::Engine(true); engine->useDefaultConfiguration(); diff --git a/src/parsers/v1_1.cpp b/src/parsers/v1_1.cpp index 3ebec78..d5adb5e 100644 --- a/src/parsers/v1_1.cpp +++ b/src/parsers/v1_1.cpp @@ -10,7 +10,7 @@ capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::pa const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; - auto engine = new engine::Engine(); + auto engine = new engine::Engine(false); // ---- Load JSON ---- std::ifstream file(source); From d6c22b657e17068df8615ca0f970e0b9de0c9f08 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Mon, 17 Nov 2025 16:38:22 -0600 Subject: [PATCH 04/23] Improved default configuration --- include/configuration.h | 12 ++++++------ include/monitor.h | 7 +++++-- src/Engine.cpp | 7 ++----- src/configuration.cpp | 19 +++++++++++-------- src/monitors/Multicast.cpp | 13 +++++++------ 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/include/configuration.h b/include/configuration.h index 10e2e6c..8e1ca3b 100644 --- a/include/configuration.h +++ b/include/configuration.h @@ -10,16 +10,16 @@ class capiocl::configuration::CapioClConfiguration { friend class engine::Engine; std::unordered_map config; -protected: - void set(const std::string& key, std::string value); + protected: + void set(const std::string &key, std::string value); public: - explicit CapioClConfiguration() = default; - ~CapioClConfiguration() = default; + explicit CapioClConfiguration(); + ~CapioClConfiguration() = default; void load(const std::filesystem::path &path); - void getParameter(const std::string &key, int *value, int def_value) const; - void getParameter(const std::string &key, std::string *value, std::string def_value) const; + void getParameter(const std::string &key, int *value) const; + void getParameter(const std::string &key, std::string *value) const; }; #endif \ No newline at end of file diff --git a/include/monitor.h b/include/monitor.h index e52dc2b..d2cae8c 100644 --- a/include/monitor.h +++ b/include/monitor.h @@ -150,9 +150,12 @@ class MulticastMonitor final : public MonitorInterface { /** * @brief Multicast port number. */ - int MULTICAST_COMMIT_PORT; + int MULTICAST_COMMIT_PORT{}; - int MULTICAST_HOME_NODE_PORT; + int MULTICAST_HOME_NODE_PORT{}; + + ///@brief Delay in milliseconds before checking again for a status change + int MULTICAST_DELAY_MILLIS{}; /** * @brief Supported network command types for commit messages. diff --git a/src/Engine.cpp b/src/Engine.cpp index 13b8290..b1729be 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -762,11 +762,8 @@ void capiocl::engine::Engine::loadConfiguration(const std::string &path) { } void capiocl::engine::Engine::useDefaultConfiguration() { - configuration->set("monitor.commit.port", "224.224.224.1"); - configuration->set("monitor.commit.port", "12345"); - configuration->set("monitor.homenode.ip", "224.224.224.2"); - configuration->set("monitor.homenode.port", "12345"); + const auto def_config = configuration::CapioClConfiguration(); - monitor.registerMonitorBackend(new monitor::MulticastMonitor(*configuration)); + monitor.registerMonitorBackend(new monitor::MulticastMonitor(def_config)); monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); } \ No newline at end of file diff --git a/src/configuration.cpp b/src/configuration.cpp index 06d7544..054c8fd 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -19,6 +19,14 @@ void flatten_table(const toml::table &tbl, std::unordered_mapset("monitor.mcast.commit.ip", "224.224.224.1"); + this->set("monitor.mcast.commit.port", "12345"); + this->set("monitor.mcast.homenode.ip", "224.224.224.2"); + this->set("monitor.mcast.homenode.port", "12345"); + this->set("monitor.mcast.delay_ms", "300"); +} + void capiocl::configuration::CapioClConfiguration::set(const std::string &key, std::string value) { config[key] = std::move(value); } @@ -37,22 +45,17 @@ void capiocl::configuration::CapioClConfiguration::load(const std::filesystem::p flatten_table(tbl, config); } -void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, int *value, - const int def_value) const { +void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, + int *value) const { if (config.find(key) != config.end()) { *value = std::stoi(config.at(key)); - } else { - *value = def_value; } } void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, - std::string *value, - std::string def_value) const { + std::string *value) const { if (config.find(key) != config.end()) { *value = config.at(key); - } else { - *value = std::move(def_value); } } \ No newline at end of file diff --git a/src/monitors/Multicast.cpp b/src/monitors/Multicast.cpp index 1c8a700..e633fa0 100644 --- a/src/monitors/Multicast.cpp +++ b/src/monitors/Multicast.cpp @@ -185,10 +185,11 @@ void capiocl::monitor::MulticastMonitor::_send_message(const std::string &ip_add capiocl::monitor::MulticastMonitor::MulticastMonitor( const configuration::CapioClConfiguration &config) { - config.getParameter("monitor.commit.ip", &MULTICAST_COMMIT_ADDR, "224.224.224.1"); - config.getParameter("monitor.commit.port", &MULTICAST_COMMIT_PORT, 12345); - config.getParameter("monitor.homenode.ip", &MULTICAST_HOME_NODE_ADDR, "224.224.224.2"); - config.getParameter("monitor.homenode.port", &MULTICAST_HOME_NODE_PORT, 12345); + config.getParameter("monitor.mcast.commit.ip", &MULTICAST_COMMIT_ADDR); + config.getParameter("monitor.mcast.commit.port", &MULTICAST_COMMIT_PORT); + config.getParameter("monitor.mcast.homenode.ip", &MULTICAST_HOME_NODE_ADDR); + config.getParameter("monitor.mcast.homenode.port", &MULTICAST_HOME_NODE_PORT); + config.getParameter("monitor.mcast.delay_ms", &MULTICAST_DELAY_MILLIS); commit_thread = std::thread(&commit_listener, std::ref(_committed_files), std::ref(committed_lock), @@ -219,7 +220,7 @@ bool capiocl::monitor::MulticastMonitor::isCommitted(const std::filesystem::path } _send_message(MULTICAST_COMMIT_ADDR, MULTICAST_COMMIT_PORT, path, GET); - std::this_thread::sleep_for(std::chrono::milliseconds(300)); + std::this_thread::sleep_for(std::chrono::milliseconds(MULTICAST_DELAY_MILLIS)); { const std::lock_guard lg(committed_lock); @@ -256,7 +257,7 @@ capiocl::monitor::MulticastMonitor::getHomeNode(const std::filesystem::path &pat } _send_message(MULTICAST_HOME_NODE_ADDR, MULTICAST_HOME_NODE_PORT, path.string(), GET); - std::this_thread::sleep_for(std::chrono::milliseconds(300)); + std::this_thread::sleep_for(std::chrono::milliseconds(MULTICAST_DELAY_MILLIS)); const std::lock_guard lg(home_node_lock); if (const auto itm = _home_nodes.find(path); itm != _home_nodes.end()) { From 2023e8c7b1ce914570b8138f9eece91be08e1d01 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Tue, 18 Nov 2025 21:43:01 -0600 Subject: [PATCH 05/23] defaults --- capiocl.hpp | 1 + include/configuration.h | 14 ++++++++++++++ src/configuration.cpp | 14 +++++++++----- src/defaults.cpp | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 src/defaults.cpp diff --git a/capiocl.hpp b/capiocl.hpp index 8c1cb31..d785345 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -96,6 +96,7 @@ class Engine; namespace configuration { class CapioClConfiguration; +struct defaults; } } // namespace capiocl diff --git a/include/configuration.h b/include/configuration.h index 8e1ca3b..ee2bf50 100644 --- a/include/configuration.h +++ b/include/configuration.h @@ -5,6 +5,19 @@ #include "capiocl.hpp" +typedef struct { + std::string k; + std::string v; +} ConfigurationEntry; + +struct capiocl::configuration::defaults { + static ConfigurationEntry DEFAULT_MONITOR_MCAST_IP; + static ConfigurationEntry DEFAULT_MONITOR_MCAST_PORT; + static ConfigurationEntry DEFAULT_MONITOR_MCAST_DELAY; + static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_IP; + static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_PORT; +}; + /// @brief Load configuration and store it from a CAPIO-CL TOML configuration file class capiocl::configuration::CapioClConfiguration { friend class engine::Engine; @@ -12,6 +25,7 @@ class capiocl::configuration::CapioClConfiguration { protected: void set(const std::string &key, std::string value); + void set(ConfigurationEntry entry); public: explicit CapioClConfiguration(); diff --git a/src/configuration.cpp b/src/configuration.cpp index 054c8fd..b4188d4 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -20,17 +20,21 @@ void flatten_table(const toml::table &tbl, std::unordered_mapset("monitor.mcast.commit.ip", "224.224.224.1"); - this->set("monitor.mcast.commit.port", "12345"); - this->set("monitor.mcast.homenode.ip", "224.224.224.2"); - this->set("monitor.mcast.homenode.port", "12345"); - this->set("monitor.mcast.delay_ms", "300"); + this->set(defaults::DEFAULT_MONITOR_MCAST_IP); + this->set(defaults::DEFAULT_MONITOR_MCAST_PORT); + this->set(defaults::DEFAULT_MONITOR_HOMENODE_IP); + this->set(defaults::DEFAULT_MONITOR_HOMENODE_PORT); + this->set(defaults::DEFAULT_MONITOR_MCAST_DELAY); } void capiocl::configuration::CapioClConfiguration::set(const std::string &key, std::string value) { config[key] = std::move(value); } +void capiocl::configuration::CapioClConfiguration::set(const ConfigurationEntry entry) { + this->set(entry.k, entry.v); +} + void capiocl::configuration::CapioClConfiguration::load(const std::filesystem::path &path) { if (path.empty()) { return; diff --git a/src/defaults.cpp b/src/defaults.cpp new file mode 100644 index 0000000..3e833ce --- /dev/null +++ b/src/defaults.cpp @@ -0,0 +1,16 @@ +#include "include/configuration.h" + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_IP{ + "monitor.mcast.commit.ip", "224.224.224.1"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_PORT{ + "monitor.mcast.commit.port", "12345"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_DELAY{ + "monitor.mcast.delay_ms", "300"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_HOMENODE_IP{ + "monitor.mcast.homenode.ip", "224.224.224.2"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_HOMENODE_PORT{ + "monitor.mcast.homenode.port", "12345"}; From 7548a6d1d0a293659d93639b3e32e58c56a2ab22 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 07:47:52 -0600 Subject: [PATCH 06/23] Added tests --- CMakeLists.txt | 15 ++-- tests/cpp/test_exceptions.hpp | 74 +++++++++---------- tests/cpp/test_serialize_deserialize.hpp | 26 ++++++- tests/jsons/{V1_test0.json => V1/test0.json} | 0 tests/jsons/{V1_test1.json => V1/test1.json} | 0 .../jsons/{V1_test10.json => V1/test10.json} | 0 .../jsons/{V1_test11.json => V1/test11.json} | 0 .../jsons/{V1_test12.json => V1/test12.json} | 0 .../jsons/{V1_test13.json => V1/test13.json} | 0 .../jsons/{V1_test14.json => V1/test14.json} | 0 .../jsons/{V1_test15.json => V1/test15.json} | 0 .../jsons/{V1_test16.json => V1/test16.json} | 0 .../jsons/{V1_test17.json => V1/test17.json} | 0 .../jsons/{V1_test18.json => V1/test18.json} | 0 .../jsons/{V1_test19.json => V1/test19.json} | 0 tests/jsons/{V1_test2.json => V1/test2.json} | 0 .../jsons/{V1_test20.json => V1/test20.json} | 0 .../jsons/{V1_test21.json => V1/test21.json} | 0 .../jsons/{V1_test22.json => V1/test22.json} | 0 .../jsons/{V1_test23.json => V1/test23.json} | 0 .../jsons/{V1_test24.json => V1/test24.json} | 0 .../jsons/{V1_test25.json => V1/test25.json} | 0 tests/jsons/{V1_test3.json => V1/test3.json} | 0 tests/jsons/{V1_test4.json => V1/test4.json} | 0 tests/jsons/{V1_test5.json => V1/test5.json} | 0 tests/jsons/{V1_test6.json => V1/test6.json} | 0 tests/jsons/{V1_test7.json => V1/test7.json} | 0 tests/jsons/{V1_test8.json => V1/test8.json} | 0 tests/jsons/{V1_test9.json => V1/test9.json} | 0 tests/jsons/{ => V1_1}/V1-1_test0.json | 0 tests/jsons/V1_1/test0.json | 32 ++++++++ tests/jsons/V1_1/test1.json | 3 + tests/jsons/V1_1/test10.json | 13 ++++ tests/jsons/V1_1/test11.json | 18 +++++ tests/jsons/V1_1/test12.json | 20 +++++ tests/jsons/V1_1/test13.json | 23 ++++++ tests/jsons/V1_1/test14.json | 23 ++++++ tests/jsons/V1_1/test15.json | 23 ++++++ tests/jsons/V1_1/test16.json | 23 ++++++ tests/jsons/V1_1/test17.json | 26 +++++++ tests/jsons/V1_1/test18.json | 24 ++++++ tests/jsons/V1_1/test19.json | 25 +++++++ tests/jsons/V1_1/test2.json | 4 + tests/jsons/V1_1/test20.json | 27 +++++++ tests/jsons/V1_1/test21.json | 28 +++++++ tests/jsons/V1_1/test22.json | 25 +++++++ tests/jsons/V1_1/test23.json | 24 ++++++ tests/jsons/V1_1/test24.json | 15 ++++ tests/jsons/V1_1/test25.json | 15 ++++ tests/jsons/V1_1/test3.json | 4 + tests/jsons/V1_1/test4.json | 5 ++ tests/jsons/V1_1/test5.json | 7 ++ tests/jsons/V1_1/test6.json | 11 +++ tests/jsons/V1_1/test7.json | 9 +++ tests/jsons/V1_1/test8.json | 10 +++ tests/jsons/V1_1/test9.json | 12 +++ 56 files changed, 510 insertions(+), 54 deletions(-) rename tests/jsons/{V1_test0.json => V1/test0.json} (100%) rename tests/jsons/{V1_test1.json => V1/test1.json} (100%) rename tests/jsons/{V1_test10.json => V1/test10.json} (100%) rename tests/jsons/{V1_test11.json => V1/test11.json} (100%) rename tests/jsons/{V1_test12.json => V1/test12.json} (100%) rename tests/jsons/{V1_test13.json => V1/test13.json} (100%) rename tests/jsons/{V1_test14.json => V1/test14.json} (100%) rename tests/jsons/{V1_test15.json => V1/test15.json} (100%) rename tests/jsons/{V1_test16.json => V1/test16.json} (100%) rename tests/jsons/{V1_test17.json => V1/test17.json} (100%) rename tests/jsons/{V1_test18.json => V1/test18.json} (100%) rename tests/jsons/{V1_test19.json => V1/test19.json} (100%) rename tests/jsons/{V1_test2.json => V1/test2.json} (100%) rename tests/jsons/{V1_test20.json => V1/test20.json} (100%) rename tests/jsons/{V1_test21.json => V1/test21.json} (100%) rename tests/jsons/{V1_test22.json => V1/test22.json} (100%) rename tests/jsons/{V1_test23.json => V1/test23.json} (100%) rename tests/jsons/{V1_test24.json => V1/test24.json} (100%) rename tests/jsons/{V1_test25.json => V1/test25.json} (100%) rename tests/jsons/{V1_test3.json => V1/test3.json} (100%) rename tests/jsons/{V1_test4.json => V1/test4.json} (100%) rename tests/jsons/{V1_test5.json => V1/test5.json} (100%) rename tests/jsons/{V1_test6.json => V1/test6.json} (100%) rename tests/jsons/{V1_test7.json => V1/test7.json} (100%) rename tests/jsons/{V1_test8.json => V1/test8.json} (100%) rename tests/jsons/{V1_test9.json => V1/test9.json} (100%) rename tests/jsons/{ => V1_1}/V1-1_test0.json (100%) create mode 100644 tests/jsons/V1_1/test0.json create mode 100644 tests/jsons/V1_1/test1.json create mode 100644 tests/jsons/V1_1/test10.json create mode 100644 tests/jsons/V1_1/test11.json create mode 100644 tests/jsons/V1_1/test12.json create mode 100644 tests/jsons/V1_1/test13.json create mode 100644 tests/jsons/V1_1/test14.json create mode 100644 tests/jsons/V1_1/test15.json create mode 100644 tests/jsons/V1_1/test16.json create mode 100644 tests/jsons/V1_1/test17.json create mode 100644 tests/jsons/V1_1/test18.json create mode 100644 tests/jsons/V1_1/test19.json create mode 100644 tests/jsons/V1_1/test2.json create mode 100644 tests/jsons/V1_1/test20.json create mode 100644 tests/jsons/V1_1/test21.json create mode 100644 tests/jsons/V1_1/test22.json create mode 100644 tests/jsons/V1_1/test23.json create mode 100644 tests/jsons/V1_1/test24.json create mode 100644 tests/jsons/V1_1/test25.json create mode 100644 tests/jsons/V1_1/test3.json create mode 100644 tests/jsons/V1_1/test4.json create mode 100644 tests/jsons/V1_1/test5.json create mode 100644 tests/jsons/V1_1/test6.json create mode 100644 tests/jsons/V1_1/test7.json create mode 100644 tests/jsons/V1_1/test8.json create mode 100644 tests/jsons/V1_1/test9.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f590e7..5ed4d80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,7 @@ add_compile_options(-Wall -Wextra -Wpedantic) FetchContent_Declare( tomlplusplus GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git - GIT_TAG v3.4.0 + GIT_TAG v3.4.0 ) FetchContent_Declare( @@ -196,8 +196,10 @@ if (CAPIO_CL_BUILD_TESTS) add_custom_command( TARGET CAPIO_CL_tests PRE_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory "/tmp/capio_cl_jsons" - COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_JSON_DIR} "/tmp/capio_cl_jsons" - COMMENT "Copying JSON test files to build directory" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${TEST_JSON_DIR}" + "/tmp/capio_cl_jsons" + COMMENT "Copying JSON test files with full directory structure" ) ##################################### @@ -207,13 +209,8 @@ if (CAPIO_CL_BUILD_TESTS) RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) - install(DIRECTORY ${TEST_JSON_DIR} - DESTINATION ${CMAKE_INSTALL_BINDIR}/jsons - FILES_MATCHING PATTERN "*.json" - ) - ##################################### - # Targget to execute tests + # Target to execute tests ##################################### add_custom_target(run_tests DEPENDS CAPIO_CL_tests diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index cf08145..d2bb3ac 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -43,56 +43,48 @@ TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { } TEST(EXCEPTION_SUITE_NAME, testParserException) { - std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons"; + std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons/"; + std::vector VERSIONS = {"V1", "V1_1"}; capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, "Loading jsons from " + JSON_DIR.string()); - bool exception_catched = false; std::vector test_filenames = { "", "ANonExistingFile", - JSON_DIR / "V1_test1.json", - JSON_DIR / "V1_test2.json", - JSON_DIR / "V1_test3.json", - JSON_DIR / "V1_test4.json", - JSON_DIR / "V1_test5.json", - JSON_DIR / "V1_test6.json", - JSON_DIR / "V1_test7.json", - JSON_DIR / "V1_test8.json", - JSON_DIR / "V1_test9.json", - JSON_DIR / "V1_test10.json", - JSON_DIR / "V1_test11.json", - JSON_DIR / "V1_test12.json", - JSON_DIR / "V1_test13.json", - JSON_DIR / "V1_test14.json", - JSON_DIR / "V1_test15.json", - JSON_DIR / "V1_test16.json", - JSON_DIR / "V1_test17.json", - JSON_DIR / "V1_test18.json", - JSON_DIR / "V1_test19.json", - JSON_DIR / "V1_test20.json", - JSON_DIR / "V1_test21.json", - JSON_DIR / "V1_test22.json", - JSON_DIR / "V1_test23.json", - JSON_DIR / "V1_test25.json", + "test1.json", + "test2.json", + "test3.json", + "test4.json", + "test5.json", + "test6.json", + "test7.json", + "test8.json", + "test9.json", + "test10.json", + "test11.json", + "test12.json", + "test13.json", + "test14.json", + "test15.json", + "test16.json", + "test17.json", + "test18.json", + "test19.json", + "test20.json", + "test21.json", + "test22.json", + "test23.json", + "test25.json", }; - - for (const auto &test : test_filenames) { - exception_catched = false; - try { + for (const auto &version : VERSIONS) { + for (const auto &test : test_filenames) { + const auto test_file_path = test.empty() ? test : JSON_DIR / version / test; capiocl::printer::print(capiocl::printer::CLI_LEVEL_WARNING, - "Testing on file " + test.string()); - capiocl::parser::Parser::parse(test); - } catch (std::exception &e) { - exception_catched = true; - auto demangled = demangled_name(e); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, - "Caught exception of type =" + demangled); - EXPECT_TRUE(demangled == "capiocl::parser::ParserException"); - EXPECT_GT(std::string(e.what()).size(), 0); + "Testing on file " + test_file_path.string()); + + EXPECT_THROW(capiocl::parser::Parser::parse(test_file_path), + capiocl::parser::ParserException); } - EXPECT_TRUE(exception_catched); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, "Test failed successfully\n\n"); } } diff --git a/tests/cpp/test_serialize_deserialize.hpp b/tests/cpp/test_serialize_deserialize.hpp index 5f49a2c..6b93912 100644 --- a/tests/cpp/test_serialize_deserialize.hpp +++ b/tests/cpp/test_serialize_deserialize.hpp @@ -144,8 +144,8 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeCommitOnCloseCountNoCommitRu std::filesystem::remove(path); } -TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsolute) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test0.json"); +TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1/test0.json"); auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); EXPECT_TRUE(engine->getWorkflowName() == "test"); EXPECT_TRUE(engine->contains("/tmp/file")); @@ -154,8 +154,26 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsolute) { EXPECT_TRUE(engine->contains("/tmp/file3")); } -TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSection) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test24.json"); +TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSectionV1) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1/test24.json"); + auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(engine->getWorkflowName() == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); +} + +TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1_1) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_1/test0.json"); + auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(engine->getWorkflowName() == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + EXPECT_TRUE(engine->contains("/tmp/file2")); + EXPECT_TRUE(engine->contains("/tmp/file3")); +} + +TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSectionV1_1) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_1/test24.json"); auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); EXPECT_TRUE(engine->getWorkflowName() == "test"); EXPECT_TRUE(engine->contains("/tmp/file")); diff --git a/tests/jsons/V1_test0.json b/tests/jsons/V1/test0.json similarity index 100% rename from tests/jsons/V1_test0.json rename to tests/jsons/V1/test0.json diff --git a/tests/jsons/V1_test1.json b/tests/jsons/V1/test1.json similarity index 100% rename from tests/jsons/V1_test1.json rename to tests/jsons/V1/test1.json diff --git a/tests/jsons/V1_test10.json b/tests/jsons/V1/test10.json similarity index 100% rename from tests/jsons/V1_test10.json rename to tests/jsons/V1/test10.json diff --git a/tests/jsons/V1_test11.json b/tests/jsons/V1/test11.json similarity index 100% rename from tests/jsons/V1_test11.json rename to tests/jsons/V1/test11.json diff --git a/tests/jsons/V1_test12.json b/tests/jsons/V1/test12.json similarity index 100% rename from tests/jsons/V1_test12.json rename to tests/jsons/V1/test12.json diff --git a/tests/jsons/V1_test13.json b/tests/jsons/V1/test13.json similarity index 100% rename from tests/jsons/V1_test13.json rename to tests/jsons/V1/test13.json diff --git a/tests/jsons/V1_test14.json b/tests/jsons/V1/test14.json similarity index 100% rename from tests/jsons/V1_test14.json rename to tests/jsons/V1/test14.json diff --git a/tests/jsons/V1_test15.json b/tests/jsons/V1/test15.json similarity index 100% rename from tests/jsons/V1_test15.json rename to tests/jsons/V1/test15.json diff --git a/tests/jsons/V1_test16.json b/tests/jsons/V1/test16.json similarity index 100% rename from tests/jsons/V1_test16.json rename to tests/jsons/V1/test16.json diff --git a/tests/jsons/V1_test17.json b/tests/jsons/V1/test17.json similarity index 100% rename from tests/jsons/V1_test17.json rename to tests/jsons/V1/test17.json diff --git a/tests/jsons/V1_test18.json b/tests/jsons/V1/test18.json similarity index 100% rename from tests/jsons/V1_test18.json rename to tests/jsons/V1/test18.json diff --git a/tests/jsons/V1_test19.json b/tests/jsons/V1/test19.json similarity index 100% rename from tests/jsons/V1_test19.json rename to tests/jsons/V1/test19.json diff --git a/tests/jsons/V1_test2.json b/tests/jsons/V1/test2.json similarity index 100% rename from tests/jsons/V1_test2.json rename to tests/jsons/V1/test2.json diff --git a/tests/jsons/V1_test20.json b/tests/jsons/V1/test20.json similarity index 100% rename from tests/jsons/V1_test20.json rename to tests/jsons/V1/test20.json diff --git a/tests/jsons/V1_test21.json b/tests/jsons/V1/test21.json similarity index 100% rename from tests/jsons/V1_test21.json rename to tests/jsons/V1/test21.json diff --git a/tests/jsons/V1_test22.json b/tests/jsons/V1/test22.json similarity index 100% rename from tests/jsons/V1_test22.json rename to tests/jsons/V1/test22.json diff --git a/tests/jsons/V1_test23.json b/tests/jsons/V1/test23.json similarity index 100% rename from tests/jsons/V1_test23.json rename to tests/jsons/V1/test23.json diff --git a/tests/jsons/V1_test24.json b/tests/jsons/V1/test24.json similarity index 100% rename from tests/jsons/V1_test24.json rename to tests/jsons/V1/test24.json diff --git a/tests/jsons/V1_test25.json b/tests/jsons/V1/test25.json similarity index 100% rename from tests/jsons/V1_test25.json rename to tests/jsons/V1/test25.json diff --git a/tests/jsons/V1_test3.json b/tests/jsons/V1/test3.json similarity index 100% rename from tests/jsons/V1_test3.json rename to tests/jsons/V1/test3.json diff --git a/tests/jsons/V1_test4.json b/tests/jsons/V1/test4.json similarity index 100% rename from tests/jsons/V1_test4.json rename to tests/jsons/V1/test4.json diff --git a/tests/jsons/V1_test5.json b/tests/jsons/V1/test5.json similarity index 100% rename from tests/jsons/V1_test5.json rename to tests/jsons/V1/test5.json diff --git a/tests/jsons/V1_test6.json b/tests/jsons/V1/test6.json similarity index 100% rename from tests/jsons/V1_test6.json rename to tests/jsons/V1/test6.json diff --git a/tests/jsons/V1_test7.json b/tests/jsons/V1/test7.json similarity index 100% rename from tests/jsons/V1_test7.json rename to tests/jsons/V1/test7.json diff --git a/tests/jsons/V1_test8.json b/tests/jsons/V1/test8.json similarity index 100% rename from tests/jsons/V1_test8.json rename to tests/jsons/V1/test8.json diff --git a/tests/jsons/V1_test9.json b/tests/jsons/V1/test9.json similarity index 100% rename from tests/jsons/V1_test9.json rename to tests/jsons/V1/test9.json diff --git a/tests/jsons/V1-1_test0.json b/tests/jsons/V1_1/V1-1_test0.json similarity index 100% rename from tests/jsons/V1-1_test0.json rename to tests/jsons/V1_1/V1-1_test0.json diff --git a/tests/jsons/V1_1/test0.json b/tests/jsons/V1_1/test0.json new file mode 100644 index 0000000..efaca20 --- /dev/null +++ b/tests/jsons/V1_1/test0.json @@ -0,0 +1,32 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "/tmp/file", + "file3" + ], + "output_stream": [ + "file1", + "file2" + ], + "streaming": [ + { + "name": [ + "/tmp/file" + ], + "committed": "on_termination", + "mode": "no_update" + }, + { + "name": [ + "/tmp/file1" + ] + } + ] + } + ], + "storage": {} +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test1.json b/tests/jsons/V1_1/test1.json new file mode 100644 index 0000000..0394d23 --- /dev/null +++ b/tests/jsons/V1_1/test1.json @@ -0,0 +1,3 @@ +{ + "version": 1.1 +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test10.json b/tests/jsons/V1_1/test10.json new file mode 100644 index 0000000..248bfe6 --- /dev/null +++ b/tests/jsons/V1_1/test10.json @@ -0,0 +1,13 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test11.json b/tests/jsons/V1_1/test11.json new file mode 100644 index 0000000..d7bc9b8 --- /dev/null +++ b/tests/jsons/V1_1/test11.json @@ -0,0 +1,18 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + {} + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test12.json b/tests/jsons/V1_1/test12.json new file mode 100644 index 0000000..95cfe49 --- /dev/null +++ b/tests/jsons/V1_1/test12.json @@ -0,0 +1,20 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test13.json b/tests/jsons/V1_1/test13.json new file mode 100644 index 0000000..018310e --- /dev/null +++ b/tests/jsons/V1_1/test13.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test14.json b/tests/jsons/V1_1/test14.json new file mode 100644 index 0000000..3b07731 --- /dev/null +++ b/tests/jsons/V1_1/test14.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "a:b" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test15.json b/tests/jsons/V1_1/test15.json new file mode 100644 index 0000000..1555951 --- /dev/null +++ b/tests/jsons/V1_1/test15.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test16.json b/tests/jsons/V1_1/test16.json new file mode 100644 index 0000000..f123886 --- /dev/null +++ b/tests/jsons/V1_1/test16.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_file" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test17.json b/tests/jsons/V1_1/test17.json new file mode 100644 index 0000000..b7af2f2 --- /dev/null +++ b/tests/jsons/V1_1/test17.json @@ -0,0 +1,26 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": [ + "failMe" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test18.json b/tests/jsons/V1_1/test18.json new file mode 100644 index 0000000..da4a0b2 --- /dev/null +++ b/tests/jsons/V1_1/test18.json @@ -0,0 +1,24 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test19.json b/tests/jsons/V1_1/test19.json new file mode 100644 index 0000000..bd8ad57 --- /dev/null +++ b/tests/jsons/V1_1/test19.json @@ -0,0 +1,25 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": "failMe" +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test2.json b/tests/jsons/V1_1/test2.json new file mode 100644 index 0000000..7a14ed0 --- /dev/null +++ b/tests/jsons/V1_1/test2.json @@ -0,0 +1,4 @@ +{ + "version": 1.1, + "name": [] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test20.json b/tests/jsons/V1_1/test20.json new file mode 100644 index 0000000..d8c8b57 --- /dev/null +++ b/tests/jsons/V1_1/test20.json @@ -0,0 +1,27 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test21.json b/tests/jsons/V1_1/test21.json new file mode 100644 index 0000000..665cbba --- /dev/null +++ b/tests/jsons/V1_1/test21.json @@ -0,0 +1,28 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": [], + "fs": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test22.json b/tests/jsons/V1_1/test22.json new file mode 100644 index 0000000..348d200 --- /dev/null +++ b/tests/jsons/V1_1/test22.json @@ -0,0 +1,25 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update", + "n_files": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test23.json b/tests/jsons/V1_1/test23.json new file mode 100644 index 0000000..e361dfc --- /dev/null +++ b/tests/jsons/V1_1/test23.json @@ -0,0 +1,24 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "n_files:failMe", + "mode": "no_update" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test24.json b/tests/jsons/V1_1/test24.json new file mode 100644 index 0000000..50f3917 --- /dev/null +++ b/tests/jsons/V1_1/test24.json @@ -0,0 +1,15 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test25.json b/tests/jsons/V1_1/test25.json new file mode 100644 index 0000000..d1f98f0 --- /dev/null +++ b/tests/jsons/V1_1/test25.json @@ -0,0 +1,15 @@ +{ + "version": "1234.5678", + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test3.json b/tests/jsons/V1_1/test3.json new file mode 100644 index 0000000..60fa1e8 --- /dev/null +++ b/tests/jsons/V1_1/test3.json @@ -0,0 +1,4 @@ +{ + "version": 1.1, + "name": "test" +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test4.json b/tests/jsons/V1_1/test4.json new file mode 100644 index 0000000..4dd0576 --- /dev/null +++ b/tests/jsons/V1_1/test4.json @@ -0,0 +1,5 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": "fail" +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test5.json b/tests/jsons/V1_1/test5.json new file mode 100644 index 0000000..1ccbca6 --- /dev/null +++ b/tests/jsons/V1_1/test5.json @@ -0,0 +1,7 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + {} + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test6.json b/tests/jsons/V1_1/test6.json new file mode 100644 index 0000000..15af65c --- /dev/null +++ b/tests/jsons/V1_1/test6.json @@ -0,0 +1,11 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": [ + "failMe" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test7.json b/tests/jsons/V1_1/test7.json new file mode 100644 index 0000000..59232eb --- /dev/null +++ b/tests/jsons/V1_1/test7.json @@ -0,0 +1,9 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test8.json b/tests/jsons/V1_1/test8.json new file mode 100644 index 0000000..516fd55 --- /dev/null +++ b/tests/jsons/V1_1/test8.json @@ -0,0 +1,10 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1_1/test9.json b/tests/jsons/V1_1/test9.json new file mode 100644 index 0000000..f85b1f3 --- /dev/null +++ b/tests/jsons/V1_1/test9.json @@ -0,0 +1,12 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ] + } + ] +} \ No newline at end of file From 1bc76d1eb553f171376a63abc1f7ae5bfb6c9143 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 08:05:46 -0600 Subject: [PATCH 07/23] Added V1_1 schema --- include/parser.h | 3 +- schema/v1_1.json | 232 ++++++++++++++++++++++++++++++++++ src/Parser.cpp | 16 +-- src/parsers/v1.cpp | 3 +- src/parsers/v1_1.cpp | 3 +- tests/cpp/test_exceptions.hpp | 42 ++---- 6 files changed, 260 insertions(+), 39 deletions(-) create mode 100644 schema/v1_1.json diff --git a/include/parser.h b/include/parser.h index 16b43a6..4c79256 100644 --- a/include/parser.h +++ b/include/parser.h @@ -77,8 +77,9 @@ class Parser final { /** * Validate a CAPIO-CL configuration file according to the JSON schema of the language * @param doc The loaded CAPIO-CL configuration file + * @param str_schema Raw JSON schema to use */ - static void validate_json(const jsoncons::json &doc); + static void validate_json(const jsoncons::json &doc, const char* str_schema); /** * @brief Perform the parsing of the capio_server configuration file diff --git a/schema/v1_1.json b/schema/v1_1.json new file mode 100644 index 0000000..2daa8f3 --- /dev/null +++ b/schema/v1_1.json @@ -0,0 +1,232 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CAPIO-CL Configuration Schema", + "type": "object", + + "definitions": { + "streaming_name_rule": { + "type": "object", + "properties": { + "name": { + "type": "array", + "description": "Filenames to which the rule applies.", + "items": { "type": "string" } + }, + "committed": { + "type": "string", + "description": "Commit rule associated with the file.", + "pattern": "^(on_close(:[0-9]+)?|on_file|on_termination)$" + }, + "mode": { + "type": "string", + "description": "Firing rule associated with the files.", + "enum": ["update", "no_update"] + }, + "file_deps": { + "type": "array", + "description": "List of dependent files required when committed = on_file.", + "items": { "type": "string" } + }, + "n_files": { + "type": "integer", + "description": "Number of files expected when commit rule == n_files." + } + }, + "required": ["name"], + "additionalProperties": false, + + "if": { + "properties": { + "committed": { "pattern": "^on_file" } + }, + "required": ["committed"] + }, + "then": { + "required": ["file_deps"] + } + }, + + "streaming_dirname_rule": { + "type": "object", + "properties": { + "dirname": { + "type": "array", + "description": "Directory names to which the rule applies.", + "items": { "type": "string" } + }, + "committed": { + "type": "string", + "description": "Commit rule associated with the directory.", + "pattern": "^(on_close(:[0-9]+)?|on_file|on_n_files|on_termination)$" + }, + "mode": { + "type": "string", + "description": "Firing rule associated with the directory contents.", + "enum": ["update", "no_update"] + }, + "file_deps": { + "type": "array", + "description": "List of dependent files required when committed = on_file.", + "items": { "type": "string" } + }, + "n_files": { + "type": "integer", + "description": "Number of files expected when commit rule == n_files." + } + }, + "required": ["dirname"], + "additionalProperties": false, + + "allOf": [ + { + "if": { + "properties": { + "committed": { "pattern": "^on_file" } + }, + "required": ["committed"] + }, + "then": { + "required": ["file_deps"] + } + }, + { + "if": { + "properties": { + "committed": { "pattern": "^on_n_files" } + }, + "required": ["committed"] + }, + "then": { + "required": ["n_files"] + } + } + ] + } + }, + "properties": { + + "version": { + "type": "number", + "description": "Version of the CAPIO-CL configuration file", + "default": 1.0 + }, + + "configuration": { + "type": "string", + "description": "TOML configuration file for runtime CAPIO-CL parameters" + }, + + "name": { + "type": "string", + "description": "Identifies the application workflow." + }, + + "aliases": { + "type": "array", + "description": "Groups files or directories under a convenient name.", + "items": { + "type": "object", + "properties": { + "group_name": { "type": "string" }, + "files": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["group_name", "files"], + "additionalProperties": false + } + }, + + "IO_Graph": { + "type": "array", + "description": "Describes file data dependencies among application modules.", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + + "input_stream": { + "type": "array", + "items": { "type": "string" } + }, + + "output_stream": { + "type": "array", + "items": { "type": "string" } + }, + + "streaming": { + "type": "array", + "items": { + "oneOf": [ + { "$ref": "#/definitions/streaming_name_rule" }, + { "$ref": "#/definitions/streaming_dirname_rule" } + ] + } + } + }, + "required": ["name", "input_stream", "output_stream"], + "additionalProperties": false + } + }, + + "permanent": { + "type": "array", + "items": { "type": "string" } + }, + + "exclude": { + "type": "array", + "items": { "type": "string" } + }, + + "home_node_policies": { + "type": "object", + "properties": { + "create": { + "type": "array", + "items": { "type": "string" } + }, + "hashing": { + "type": "array", + "items": { "type": "string" } + }, + "manual": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "array", + "items": { "type": "string" } + }, + "app_node": { "type": "string" } + }, + "required": ["name", "app_node"], + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + + "storage": { + "type": "object", + "properties": { + "memory": { + "type": "array", + "items": { "type": "string" } + }, + "fs": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + } + }, + + "required": ["name", "IO_Graph", "version"], + "additionalProperties": false +} diff --git a/src/Parser.cpp b/src/Parser.cpp index 87a1121..306c325 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -1,7 +1,6 @@ #include #include -#include "capio_cl_json_schemas.hpp" #include "capiocl.hpp" #include "include/engine.h" #include "include/parser.h" @@ -11,6 +10,11 @@ capiocl::parser::ParserException::ParserException(const std::string &msg) : mess printer::print(printer::CLI_LEVEL_ERROR, msg); } +jsoncons::jsonschema::json_schema +capiocl::parser::Parser::loadSchema(const char *data) { + return jsoncons::jsonschema::make_json_schema(jsoncons::json::parse(data)); +} + std::filesystem::path capiocl::parser::Parser::resolve(std::filesystem::path path, const std::filesystem::path &prefix) { if (prefix.empty()) { @@ -28,13 +32,9 @@ std::filesystem::path capiocl::parser::Parser::resolve(std::filesystem::path pat return resolved; } -jsoncons::jsonschema::json_schema -capiocl::parser::Parser::loadSchema(const char *data) { - return jsoncons::jsonschema::make_json_schema(jsoncons::json::parse(data)); -} - -void capiocl::parser::Parser::validate_json(const jsoncons::json &doc) { - jsoncons::jsonschema::json_schema schema = loadSchema(schema_v1); +void capiocl::parser::Parser::validate_json(const jsoncons::json &doc, + const char* str_schema) { + jsoncons::jsonschema::json_schema schema = loadSchema(str_schema); try { // throws jsoncons::jsonschema::validation_error on failure [[maybe_unused]] auto status = schema.validate(doc); diff --git a/src/parsers/v1.cpp b/src/parsers/v1.cpp index a217206..24533fb 100644 --- a/src/parsers/v1.cpp +++ b/src/parsers/v1.cpp @@ -4,6 +4,7 @@ #include "include/engine.h" #include "include/parser.h" #include "include/printer.h" +#include "capio_cl_json_schemas.hpp" capiocl::engine::Engine * capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path &source, @@ -18,7 +19,7 @@ capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path std::ifstream file(source); jsoncons::json doc = jsoncons::json::parse(file); - validate_json(doc); + validate_json(doc, schema_v1); // ---- workflow name ---- workflow_name = doc["name"].as(); diff --git a/src/parsers/v1_1.cpp b/src/parsers/v1_1.cpp index d5adb5e..146f13e 100644 --- a/src/parsers/v1_1.cpp +++ b/src/parsers/v1_1.cpp @@ -4,6 +4,7 @@ #include "include/engine.h" #include "include/parser.h" #include "include/printer.h" +#include "capio_cl_json_schemas.hpp" capiocl::engine::Engine * capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::path &source, @@ -16,7 +17,7 @@ capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::pa std::ifstream file(source); jsoncons::json doc = jsoncons::json::parse(file); - validate_json(doc); + validate_json(doc, schema_v1_1); // ---- workflow name ---- workflow_name = doc["name"].as(); diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index d2bb3ac..87b7424 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -5,41 +5,27 @@ #include "include/serializer.h" TEST(EXCEPTION_SUITE_NAME, testFailedDump) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - bool exception_catched = false; + const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", + "/tmp/capio_cl_jsons/V1_1/test24.json"}; - try { - capiocl::serializer::Serializer::dump(*engine, "/"); - } catch (std::exception &e) { - exception_catched = true; - auto demangled = demangled_name(e); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, - "Caught exception of type =" + demangled); - EXPECT_TRUE(demangled == "capiocl::serializer::SerializerException"); - EXPECT_GT(std::string(e.what()).size(), 0); - } + for (const auto &source : json_path) { + auto engine = capiocl::parser::Parser::parse(source, "/tmp"); - EXPECT_TRUE(exception_catched); + EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "/"), + capiocl::serializer::SerializerException); + } } TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - bool exception_catched = false; + const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", + "/tmp/capio_cl_jsons/V1_1/test24.json"}; - try { - capiocl::serializer::Serializer::dump(*engine, "test.json", "1234.5678"); - } catch (std::exception &e) { - exception_catched = true; - auto demangled = demangled_name(e); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, - "Caught exception of type =" + demangled); - EXPECT_TRUE(demangled == "capiocl::serializer::SerializerException"); - EXPECT_GT(std::string(e.what()).size(), 0); - } + for (const auto &source : json_path) { + auto engine = capiocl::parser::Parser::parse(source, "/tmp"); - EXPECT_TRUE(exception_catched); + EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "test.json", "1234.5678"), + capiocl::serializer::SerializerException); + } } TEST(EXCEPTION_SUITE_NAME, testParserException) { From 7d811eff3f74e6f0a0f959d6660eafa9d0823a25 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 09:01:50 -0600 Subject: [PATCH 08/23] Tests --- tests/cpp/test_exceptions.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index 87b7424..685d1d2 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -4,6 +4,23 @@ #define EXCEPTION_SUITE_NAME TestThrowExceptions #include "include/serializer.h" +TEST(EXCEPTION_SUITE_NAME, testWhatMEthods) { + try { + capiocl::parser::Parser::parse(""); + } catch (const capiocl::parser::ParserException &e) { + EXPECT_TRUE(demangled_name(e) == "capiocl::parser::ParserException"); + EXPECT_GT(strlen(e.what()), 0); + } + + try { + const auto engine = capiocl::engine::Engine(); + capiocl::serializer::Serializer::dump(engine, ""); + } catch (const capiocl::serializer::SerializerException &e) { + EXPECT_TRUE(demangled_name(e) == "capiocl::serializer::SerializerException"); + EXPECT_GT(strlen(e.what()), 0); + } +} + TEST(EXCEPTION_SUITE_NAME, testFailedDump) { const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", "/tmp/capio_cl_jsons/V1_1/test24.json"}; From 7c11f9c9359177c696905be0fbd374084e4afd66 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 09:25:16 -0600 Subject: [PATCH 09/23] Added serializer for v1.1 --- include/serializer.h | 11 + src/Serializer.cpp | 3 + src/serializers/v1_1.cpp | 129 ++++++++++++ tests/cpp/main.cpp | 5 +- tests/cpp/test_serialize_deserialize.hpp | 256 ++++++++++++----------- 5 files changed, 280 insertions(+), 124 deletions(-) create mode 100644 src/serializers/v1_1.cpp diff --git a/include/serializer.h b/include/serializer.h index fad0c53..2ab65ba 100644 --- a/include/serializer.h +++ b/include/serializer.h @@ -41,6 +41,17 @@ class Serializer final { */ static void serialize_v1(const engine::Engine &engine, const std::filesystem::path &filename); + + /** + * @brief Dump the current configuration loaded into an instance of Engine to a CAPIO-CL + * VERSION 1.1 configuration file. + * + * @param engine instance of Engine to dump + * @param filename path of output file + * @throws SerializerException + */ + static void serialize_v1_1(const engine::Engine &engine, + const std::filesystem::path &filename); }; public: diff --git a/src/Serializer.cpp b/src/Serializer.cpp index fe65d7c..21de09b 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -13,6 +13,9 @@ void capiocl::serializer::Serializer::dump(const engine::Engine &engine, if (version == CAPIO_CL_VERSION::V1) { printer::print(printer::CLI_LEVEL_INFO, "Serializing engine with V1 specification"); available_serializers::serialize_v1(engine, filename); + } else if (version == CAPIO_CL_VERSION::V1_1) { + printer::print(printer::CLI_LEVEL_INFO, "Serializing engine with V1.1 specification"); + available_serializers::serialize_v1_1(engine, filename); } else { const auto message = "No serializer available for CAPIO-CL version: " + version; throw SerializerException(message); diff --git a/src/serializers/v1_1.cpp b/src/serializers/v1_1.cpp new file mode 100644 index 0000000..837d49f --- /dev/null +++ b/src/serializers/v1_1.cpp @@ -0,0 +1,129 @@ +#include +#include + +#include "capiocl.hpp" +#include "include/engine.h" +#include "include/printer.h" +#include "include/serializer.h" + +void capiocl::serializer::Serializer::available_serializers::serialize_v1_1( + const engine::Engine &engine, const std::filesystem::path &filename) { + jsoncons::json doc; + doc["version"] = 1.1; + doc["name"] = engine.getWorkflowName(); + + const auto files = engine._capio_cl_entries; + + std::unordered_map> app_inputs; + std::unordered_map> app_outputs; + + std::vector permanent; + std::vector exclude; + std::vector memory_storage; + std::vector fs_storage; + + jsoncons::json storage = jsoncons::json::object(); + jsoncons::json io_graph = jsoncons::json::array(); + + for (const auto &[path, entry] : files) { + if (entry.permanent) { + permanent.push_back(path); + } + if (entry.excluded) { + exclude.push_back(path); + } + (entry.store_in_memory ? memory_storage : fs_storage).push_back(path); + + for (const auto &p : entry.producers) { + app_outputs[p].push_back(path); + } + for (const auto &c : entry.consumers) { + app_inputs[c].push_back(path); + } + } + + for (const auto &[app_name, outputs] : app_outputs) { + jsoncons::json app = jsoncons::json::object(); + jsoncons::json streaming = jsoncons::json::array(); + + for (const auto &path : outputs) { + const auto &entry = files.at(path); + + jsoncons::json streaming_item = jsoncons::json::object(); + std::string committed = entry.commit_rule; + const char *name_kind = entry.is_file ? "name" : "dirname"; + streaming_item[name_kind] = jsoncons::json::array({path}); // LCOV_EXCL_LINE + + if (entry.commit_on_close_count > 0) { + if (entry.commit_rule == commitRules::ON_CLOSE) { + const auto close_count = std::to_string(entry.commit_on_close_count); + streaming_item["committed"] = entry.commit_rule + ":" + close_count; + } else { + const auto msg = "Commit rule is not ON_CLOSE but close count > 0"; + printer::print(printer::CLI_LEVEL_WARNING, msg); + printer::print(printer::CLI_LEVEL_WARNING, "Setting commit rule = ON_CLOSE"); + streaming_item["committed"] = std::string(commitRules::ON_CLOSE) + ":" + + std::to_string(entry.commit_on_close_count); + } + } else { + streaming_item["committed"] = entry.commit_rule; + } + + if (!entry.is_file) { + streaming_item["n_files"] = entry.directory_children_count; + } + + // Convert std::vector -> std::vector + std::vector file_deps_str; + file_deps_str.reserve(entry.file_dependencies.size()); + for (const auto &p : entry.file_dependencies) { + file_deps_str.push_back(p.string()); + } + streaming_item["file_deps"] = file_deps_str; + + streaming_item["mode"] = entry.fire_rule; + + streaming.push_back(streaming_item); + } + + app["name"] = app_name; + app["input_stream"] = app_inputs[app_name]; + app["output_stream"] = outputs; + app["streaming"] = streaming; + + io_graph.push_back(app); + } + + // Ensure apps that only have inputs appear as well + for (const auto &[app_name, inputs] : app_inputs) { + bool contained = false; + for (const auto &entry : io_graph.array_range()) { + if (entry["name"].as() == app_name) { + contained = true; + break; + } + } + if (!contained) { + jsoncons::json app = jsoncons::json::object(); + app["name"] = app_name; + app["input_stream"] = inputs; + app["output_stream"] = jsoncons::json::array(); + io_graph.push_back(app); + } + } + + doc["IO_Graph"] = io_graph; + doc["permanent"] = permanent; + doc["exclude"] = exclude; + storage["memory"] = memory_storage; + storage["fs"] = fs_storage; + doc["storage"] = storage; + + std::ofstream out(filename); + if (!out.is_open()) { + throw SerializerException("Failed to open output file: " + filename.string()); + } + out << jsoncons::pretty_print(doc) << std::endl; + + printer::print(printer::CLI_LEVEL_INFO, "Configuration serialized to " + filename.string()); +} \ No newline at end of file diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index e8ed6b5..fd0b795 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -2,6 +2,10 @@ #include #include #include +#include "capiocl.hpp" + +const std::vector CAPIO_CL_AVAIL_VERSIONS = {capiocl::CAPIO_CL_VERSION::V1, + capiocl::CAPIO_CL_VERSION::V1_1}; template std::string demangled_name(const T &obj) { int status; @@ -11,7 +15,6 @@ template std::string demangled_name(const T &obj) { return status == 0 ? demangled.get() : mangled; } -#include "capiocl.hpp" #include "include/engine.h" #include "include/monitor.h" #include "include/parser.h" diff --git a/tests/cpp/test_serialize_deserialize.hpp b/tests/cpp/test_serialize_deserialize.hpp index 6b93912..60f222f 100644 --- a/tests/cpp/test_serialize_deserialize.hpp +++ b/tests/cpp/test_serialize_deserialize.hpp @@ -4,144 +4,154 @@ #define SERIALIZE_DESERIALIZE_SUITE_NAME TestSerializeAndDeserialize TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", - file_3_name = "my_command_history.txt", file_4_name = "/tmp"; - std::string producer_name = "_first", consumer_name = "_last", intermediate_name = "_middle"; - - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); - engine.addProducer(file_1_name, producer_name); - engine.addConsumer(file_1_name, intermediate_name); - engine.addProducer(file_2_name, intermediate_name); - engine.addConsumer(file_2_name, consumer_name); - engine.addConsumer(file_1_name, consumer_name); - - engine.setStoreFileInMemory(file_1_name); - engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); - engine.setCommitedCloseNumber(file_1_name, 3); - engine.setFireRule(file_1_name, capiocl::fireRules::UPDATE); - engine.setPermanent(file_1_name, true); - - engine.setStoreFileInFileSystem(file_2_name); - engine.setCommitRule(file_2_name, capiocl::commitRules::ON_TERMINATION); - engine.setFireRule(file_1_name, capiocl::fireRules::NO_UPDATE); - - engine.addProducer(file_3_name, producer_name); - engine.addProducer(file_3_name, consumer_name); - engine.addProducer(file_3_name, intermediate_name); - engine.setExclude(file_3_name, true); - - engine.setCommitRule(file_4_name, capiocl::commitRules::ON_N_FILES); - engine.setFireRule(file_4_name, capiocl::fireRules::NO_UPDATE); - engine.setDirectoryFileCount(file_4_name, 10); - engine.addProducer(file_4_name, intermediate_name); - - engine.print(); - - capiocl::serializer::Serializer::dump(engine, path); - - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); - - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - capiocl::printer::print("", ""); - EXPECT_TRUE(engine == *new_engine); - - auto new_engine1 = capiocl::parser::Parser::parse(path, resolve, true); - EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); - - std::filesystem::remove(path); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "my_command_history.txt", file_4_name = "/tmp"; + std::string producer_name = "_first", consumer_name = "_last", + intermediate_name = "_middle"; + + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, intermediate_name); + engine.addProducer(file_2_name, intermediate_name); + engine.addConsumer(file_2_name, consumer_name); + engine.addConsumer(file_1_name, consumer_name); + + engine.setStoreFileInMemory(file_1_name); + engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); + engine.setCommitedCloseNumber(file_1_name, 3); + engine.setFireRule(file_1_name, capiocl::fireRules::UPDATE); + engine.setPermanent(file_1_name, true); + + engine.setStoreFileInFileSystem(file_2_name); + engine.setCommitRule(file_2_name, capiocl::commitRules::ON_TERMINATION); + engine.setFireRule(file_1_name, capiocl::fireRules::NO_UPDATE); + + engine.addProducer(file_3_name, producer_name); + engine.addProducer(file_3_name, consumer_name); + engine.addProducer(file_3_name, intermediate_name); + engine.setExclude(file_3_name, true); + + engine.setCommitRule(file_4_name, capiocl::commitRules::ON_N_FILES); + engine.setFireRule(file_4_name, capiocl::fireRules::NO_UPDATE); + engine.setDirectoryFileCount(file_4_name, 10); + engine.addProducer(file_4_name, intermediate_name); + + engine.print(); + + capiocl::serializer::Serializer::dump(engine, path, _cl_version); + + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); + + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + capiocl::printer::print("", ""); + EXPECT_TRUE(engine == *new_engine); + + auto new_engine1 = capiocl::parser::Parser::parse(path, resolve, true); + EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1NcloseNfiles) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt"; - std::string producer_name = "_first", consumer_name = "_last"; + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt"; + std::string producer_name = "_first", consumer_name = "_last"; - engine.setDirectory(file_1_name); - engine.setDirectoryFileCount(file_1_name, 10); - engine.addProducer(file_1_name, producer_name); - engine.addConsumer(file_1_name, consumer_name); + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); - capiocl::serializer::Serializer::dump(engine, path); + engine.setDirectory(file_1_name); + engine.setDirectoryFileCount(file_1_name, 10); + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, consumer_name); - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); + capiocl::serializer::Serializer::dump(engine, path, _cl_version); - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - capiocl::printer::print("", ""); - EXPECT_TRUE(engine == *new_engine); + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); - std::filesystem::remove(path); + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + capiocl::printer::print("", ""); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1FileDeps) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", - file_3_name = "file3.txt"; - std::string producer_name = "_first", consumer_name = "_last"; - - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); - - engine.newFile(file_1_name); - engine.newFile(file_2_name); - engine.addProducer(file_1_name, producer_name); - engine.addProducer(file_2_name, producer_name); - - engine.newFile(file_3_name); - engine.addConsumer(file_3_name, consumer_name); - engine.addProducer(file_3_name, producer_name); - engine.setCommitRule(file_3_name, capiocl::commitRules::ON_FILE); - engine.setFileDeps(file_3_name, {file_1_name, file_2_name}); - - engine.print(); - capiocl::serializer::Serializer::dump(engine, path); - - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); - - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - capiocl::printer::print("", ""); - EXPECT_TRUE(engine == *new_engine); - - std::filesystem::remove(path); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "file3.txt"; + std::string producer_name = "_first", consumer_name = "_last"; + + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); + + engine.newFile(file_1_name); + engine.newFile(file_2_name); + engine.addProducer(file_1_name, producer_name); + engine.addProducer(file_2_name, producer_name); + + engine.newFile(file_3_name); + engine.addConsumer(file_3_name, consumer_name); + engine.addProducer(file_3_name, producer_name); + engine.setCommitRule(file_3_name, capiocl::commitRules::ON_FILE); + engine.setFileDeps(file_3_name, {file_1_name, file_2_name}); + + engine.print(); + capiocl::serializer::Serializer::dump(engine, path, _cl_version); + + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); + + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + capiocl::printer::print("", ""); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeCommitOnCloseCountNoCommitRule) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt"; - std::string producer_name = "_first", consumer_name = "_last"; - - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); - - engine.newFile(file_1_name); - engine.addProducer(file_1_name, producer_name); - engine.setCommitRule(file_1_name, capiocl::commitRules::ON_TERMINATION); - engine.setCommitedCloseNumber(file_1_name, 10); - - engine.print(); - capiocl::serializer::Serializer::dump(engine, path); - - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); - - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - EXPECT_FALSE(engine == *new_engine); - capiocl::printer::print("", ""); - engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); - EXPECT_TRUE(engine == *new_engine); - - std::filesystem::remove(path); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt"; + std::string producer_name = "_first", consumer_name = "_last"; + + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); + + engine.newFile(file_1_name); + engine.addProducer(file_1_name, producer_name); + engine.setCommitRule(file_1_name, capiocl::commitRules::ON_TERMINATION); + engine.setCommitedCloseNumber(file_1_name, 10); + + engine.print(); + capiocl::serializer::Serializer::dump(engine, path, _cl_version); + + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); + + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + EXPECT_FALSE(engine == *new_engine); + capiocl::printer::print("", ""); + engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1) { From e87eb0c1d682353610e20142b0b02ec7725bb4fb Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 15:41:21 -0600 Subject: [PATCH 10/23] Refactor --- CMakeLists.txt | 21 ++++++++++++++++----- schema/{v1_1.json => v1.1.json} | 0 src/parsers/{v1_1.cpp => v1.1.cpp} | 0 src/serializers/{v1_1.cpp => v1.1.cpp} | 0 tests/cpp/test_exceptions.hpp | 11 +++++------ tests/cpp/test_serialize_deserialize.hpp | 4 ++-- tests/jsons/{V1_1 => V1.1}/V1-1_test0.json | 0 tests/jsons/{V1_1 => V1.1}/test0.json | 0 tests/jsons/{V1_1 => V1.1}/test1.json | 0 tests/jsons/{V1_1 => V1.1}/test10.json | 0 tests/jsons/{V1_1 => V1.1}/test11.json | 0 tests/jsons/{V1_1 => V1.1}/test12.json | 0 tests/jsons/{V1_1 => V1.1}/test13.json | 0 tests/jsons/{V1_1 => V1.1}/test14.json | 0 tests/jsons/{V1_1 => V1.1}/test15.json | 0 tests/jsons/{V1_1 => V1.1}/test16.json | 0 tests/jsons/{V1_1 => V1.1}/test17.json | 0 tests/jsons/{V1_1 => V1.1}/test18.json | 0 tests/jsons/{V1_1 => V1.1}/test19.json | 0 tests/jsons/{V1_1 => V1.1}/test2.json | 0 tests/jsons/{V1_1 => V1.1}/test20.json | 0 tests/jsons/{V1_1 => V1.1}/test21.json | 0 tests/jsons/{V1_1 => V1.1}/test22.json | 0 tests/jsons/{V1_1 => V1.1}/test23.json | 0 tests/jsons/{V1_1 => V1.1}/test24.json | 0 tests/jsons/{V1_1 => V1.1}/test25.json | 0 tests/jsons/{V1_1 => V1.1}/test3.json | 0 tests/jsons/{V1_1 => V1.1}/test4.json | 0 tests/jsons/{V1_1 => V1.1}/test5.json | 0 tests/jsons/{V1_1 => V1.1}/test6.json | 0 tests/jsons/{V1_1 => V1.1}/test7.json | 0 tests/jsons/{V1_1 => V1.1}/test8.json | 0 tests/jsons/{V1_1 => V1.1}/test9.json | 0 33 files changed, 23 insertions(+), 13 deletions(-) rename schema/{v1_1.json => v1.1.json} (100%) rename src/parsers/{v1_1.cpp => v1.1.cpp} (100%) rename src/serializers/{v1_1.cpp => v1.1.cpp} (100%) rename tests/jsons/{V1_1 => V1.1}/V1-1_test0.json (100%) rename tests/jsons/{V1_1 => V1.1}/test0.json (100%) rename tests/jsons/{V1_1 => V1.1}/test1.json (100%) rename tests/jsons/{V1_1 => V1.1}/test10.json (100%) rename tests/jsons/{V1_1 => V1.1}/test11.json (100%) rename tests/jsons/{V1_1 => V1.1}/test12.json (100%) rename tests/jsons/{V1_1 => V1.1}/test13.json (100%) rename tests/jsons/{V1_1 => V1.1}/test14.json (100%) rename tests/jsons/{V1_1 => V1.1}/test15.json (100%) rename tests/jsons/{V1_1 => V1.1}/test16.json (100%) rename tests/jsons/{V1_1 => V1.1}/test17.json (100%) rename tests/jsons/{V1_1 => V1.1}/test18.json (100%) rename tests/jsons/{V1_1 => V1.1}/test19.json (100%) rename tests/jsons/{V1_1 => V1.1}/test2.json (100%) rename tests/jsons/{V1_1 => V1.1}/test20.json (100%) rename tests/jsons/{V1_1 => V1.1}/test21.json (100%) rename tests/jsons/{V1_1 => V1.1}/test22.json (100%) rename tests/jsons/{V1_1 => V1.1}/test23.json (100%) rename tests/jsons/{V1_1 => V1.1}/test24.json (100%) rename tests/jsons/{V1_1 => V1.1}/test25.json (100%) rename tests/jsons/{V1_1 => V1.1}/test3.json (100%) rename tests/jsons/{V1_1 => V1.1}/test4.json (100%) rename tests/jsons/{V1_1 => V1.1}/test5.json (100%) rename tests/jsons/{V1_1 => V1.1}/test6.json (100%) rename tests/jsons/{V1_1 => V1.1}/test7.json (100%) rename tests/jsons/{V1_1 => V1.1}/test8.json (100%) rename tests/jsons/{V1_1 => V1.1}/test9.json (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ed4d80..189eeef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,22 +88,33 @@ file(WRITE ${OUTPUT_HEADER} "// Bundled CAPIO-CL encoded JSON schemas\n") file(APPEND ${OUTPUT_HEADER} "#pragma once\n\n") foreach (SCHEMA_FILE ${SCHEMA_FILES}) - get_filename_component(SCHEMA_NAME ${SCHEMA_FILE} NAME_WE) get_filename_component(SCHEMA_BASENAME ${SCHEMA_FILE} NAME) message(STATUS "Bundling CAPIO-CL schema: ${SCHEMA_BASENAME}") execute_process( - COMMAND bash -c "echo 'constexpr char schema_${SCHEMA_NAME}[] = R\"(' && cat ${SCHEMA_BASENAME} && echo ')\";'" + COMMAND bash -c "echo ${SCHEMA_BASENAME} | sed 's/\\.json$//'" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/schema + OUTPUT_VARIABLE SCHEMA_NAME_RAW + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + string(REPLACE "." "_" SCHEMA_NAME "${SCHEMA_NAME_RAW}") + + execute_process( + COMMAND bash -c + "echo \"// CAPIO-CL version: ${SCHEMA_NAME_RAW}\" && \ + echo \"constexpr char schema_${SCHEMA_NAME}[] = R\\\"(\" && \ + cat \"${SCHEMA_BASENAME}\" && \ + echo \")\\\";\"" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/schema OUTPUT_VARIABLE HEXDATA RESULT_VARIABLE RES ) + if (NOT RES EQUAL 0) - message(FATAL_ERROR "Failed bundling for ${SCHEMA_FILE}: Error code is ${RES}") + message(FATAL_ERROR "Failed to bundle ${SCHEMA_BASENAME}: exit ${RES}") endif () - - file(APPEND ${OUTPUT_HEADER} "// CAPIO-CL version: ${SCHEMA_NAME}\n") file(APPEND ${OUTPUT_HEADER} "${HEXDATA}\n\n") endforeach () diff --git a/schema/v1_1.json b/schema/v1.1.json similarity index 100% rename from schema/v1_1.json rename to schema/v1.1.json diff --git a/src/parsers/v1_1.cpp b/src/parsers/v1.1.cpp similarity index 100% rename from src/parsers/v1_1.cpp rename to src/parsers/v1.1.cpp diff --git a/src/serializers/v1_1.cpp b/src/serializers/v1.1.cpp similarity index 100% rename from src/serializers/v1_1.cpp rename to src/serializers/v1.1.cpp diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index 685d1d2..ee5a92a 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -23,7 +23,7 @@ TEST(EXCEPTION_SUITE_NAME, testWhatMEthods) { TEST(EXCEPTION_SUITE_NAME, testFailedDump) { const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", - "/tmp/capio_cl_jsons/V1_1/test24.json"}; + "/tmp/capio_cl_jsons/V1.1/test24.json"}; for (const auto &source : json_path) { auto engine = capiocl::parser::Parser::parse(source, "/tmp"); @@ -35,7 +35,7 @@ TEST(EXCEPTION_SUITE_NAME, testFailedDump) { TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", - "/tmp/capio_cl_jsons/V1_1/test24.json"}; + "/tmp/capio_cl_jsons/V1.1/test24.json"}; for (const auto &source : json_path) { auto engine = capiocl::parser::Parser::parse(source, "/tmp"); @@ -46,8 +46,7 @@ TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { } TEST(EXCEPTION_SUITE_NAME, testParserException) { - std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons/"; - std::vector VERSIONS = {"V1", "V1_1"}; + std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons/"; capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, "Loading jsons from " + JSON_DIR.string()); @@ -79,9 +78,9 @@ TEST(EXCEPTION_SUITE_NAME, testParserException) { "test23.json", "test25.json", }; - for (const auto &version : VERSIONS) { + for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { for (const auto &test : test_filenames) { - const auto test_file_path = test.empty() ? test : JSON_DIR / version / test; + const auto test_file_path = test.empty() ? test : JSON_DIR / ("V" + version) / test; capiocl::printer::print(capiocl::printer::CLI_LEVEL_WARNING, "Testing on file " + test_file_path.string()); diff --git a/tests/cpp/test_serialize_deserialize.hpp b/tests/cpp/test_serialize_deserialize.hpp index 60f222f..d2b20be 100644 --- a/tests/cpp/test_serialize_deserialize.hpp +++ b/tests/cpp/test_serialize_deserialize.hpp @@ -173,7 +173,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSectionV1) { } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1_1) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_1/test0.json"); + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1.1/test0.json"); auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); EXPECT_TRUE(engine->getWorkflowName() == "test"); EXPECT_TRUE(engine->contains("/tmp/file")); @@ -183,7 +183,7 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1_1) { } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSectionV1_1) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_1/test24.json"); + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1.1/test24.json"); auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); EXPECT_TRUE(engine->getWorkflowName() == "test"); EXPECT_TRUE(engine->contains("/tmp/file")); diff --git a/tests/jsons/V1_1/V1-1_test0.json b/tests/jsons/V1.1/V1-1_test0.json similarity index 100% rename from tests/jsons/V1_1/V1-1_test0.json rename to tests/jsons/V1.1/V1-1_test0.json diff --git a/tests/jsons/V1_1/test0.json b/tests/jsons/V1.1/test0.json similarity index 100% rename from tests/jsons/V1_1/test0.json rename to tests/jsons/V1.1/test0.json diff --git a/tests/jsons/V1_1/test1.json b/tests/jsons/V1.1/test1.json similarity index 100% rename from tests/jsons/V1_1/test1.json rename to tests/jsons/V1.1/test1.json diff --git a/tests/jsons/V1_1/test10.json b/tests/jsons/V1.1/test10.json similarity index 100% rename from tests/jsons/V1_1/test10.json rename to tests/jsons/V1.1/test10.json diff --git a/tests/jsons/V1_1/test11.json b/tests/jsons/V1.1/test11.json similarity index 100% rename from tests/jsons/V1_1/test11.json rename to tests/jsons/V1.1/test11.json diff --git a/tests/jsons/V1_1/test12.json b/tests/jsons/V1.1/test12.json similarity index 100% rename from tests/jsons/V1_1/test12.json rename to tests/jsons/V1.1/test12.json diff --git a/tests/jsons/V1_1/test13.json b/tests/jsons/V1.1/test13.json similarity index 100% rename from tests/jsons/V1_1/test13.json rename to tests/jsons/V1.1/test13.json diff --git a/tests/jsons/V1_1/test14.json b/tests/jsons/V1.1/test14.json similarity index 100% rename from tests/jsons/V1_1/test14.json rename to tests/jsons/V1.1/test14.json diff --git a/tests/jsons/V1_1/test15.json b/tests/jsons/V1.1/test15.json similarity index 100% rename from tests/jsons/V1_1/test15.json rename to tests/jsons/V1.1/test15.json diff --git a/tests/jsons/V1_1/test16.json b/tests/jsons/V1.1/test16.json similarity index 100% rename from tests/jsons/V1_1/test16.json rename to tests/jsons/V1.1/test16.json diff --git a/tests/jsons/V1_1/test17.json b/tests/jsons/V1.1/test17.json similarity index 100% rename from tests/jsons/V1_1/test17.json rename to tests/jsons/V1.1/test17.json diff --git a/tests/jsons/V1_1/test18.json b/tests/jsons/V1.1/test18.json similarity index 100% rename from tests/jsons/V1_1/test18.json rename to tests/jsons/V1.1/test18.json diff --git a/tests/jsons/V1_1/test19.json b/tests/jsons/V1.1/test19.json similarity index 100% rename from tests/jsons/V1_1/test19.json rename to tests/jsons/V1.1/test19.json diff --git a/tests/jsons/V1_1/test2.json b/tests/jsons/V1.1/test2.json similarity index 100% rename from tests/jsons/V1_1/test2.json rename to tests/jsons/V1.1/test2.json diff --git a/tests/jsons/V1_1/test20.json b/tests/jsons/V1.1/test20.json similarity index 100% rename from tests/jsons/V1_1/test20.json rename to tests/jsons/V1.1/test20.json diff --git a/tests/jsons/V1_1/test21.json b/tests/jsons/V1.1/test21.json similarity index 100% rename from tests/jsons/V1_1/test21.json rename to tests/jsons/V1.1/test21.json diff --git a/tests/jsons/V1_1/test22.json b/tests/jsons/V1.1/test22.json similarity index 100% rename from tests/jsons/V1_1/test22.json rename to tests/jsons/V1.1/test22.json diff --git a/tests/jsons/V1_1/test23.json b/tests/jsons/V1.1/test23.json similarity index 100% rename from tests/jsons/V1_1/test23.json rename to tests/jsons/V1.1/test23.json diff --git a/tests/jsons/V1_1/test24.json b/tests/jsons/V1.1/test24.json similarity index 100% rename from tests/jsons/V1_1/test24.json rename to tests/jsons/V1.1/test24.json diff --git a/tests/jsons/V1_1/test25.json b/tests/jsons/V1.1/test25.json similarity index 100% rename from tests/jsons/V1_1/test25.json rename to tests/jsons/V1.1/test25.json diff --git a/tests/jsons/V1_1/test3.json b/tests/jsons/V1.1/test3.json similarity index 100% rename from tests/jsons/V1_1/test3.json rename to tests/jsons/V1.1/test3.json diff --git a/tests/jsons/V1_1/test4.json b/tests/jsons/V1.1/test4.json similarity index 100% rename from tests/jsons/V1_1/test4.json rename to tests/jsons/V1.1/test4.json diff --git a/tests/jsons/V1_1/test5.json b/tests/jsons/V1.1/test5.json similarity index 100% rename from tests/jsons/V1_1/test5.json rename to tests/jsons/V1.1/test5.json diff --git a/tests/jsons/V1_1/test6.json b/tests/jsons/V1.1/test6.json similarity index 100% rename from tests/jsons/V1_1/test6.json rename to tests/jsons/V1.1/test6.json diff --git a/tests/jsons/V1_1/test7.json b/tests/jsons/V1.1/test7.json similarity index 100% rename from tests/jsons/V1_1/test7.json rename to tests/jsons/V1.1/test7.json diff --git a/tests/jsons/V1_1/test8.json b/tests/jsons/V1.1/test8.json similarity index 100% rename from tests/jsons/V1_1/test8.json rename to tests/jsons/V1.1/test8.json diff --git a/tests/jsons/V1_1/test9.json b/tests/jsons/V1.1/test9.json similarity index 100% rename from tests/jsons/V1_1/test9.json rename to tests/jsons/V1.1/test9.json From b162025aa8ee960bd147727cc4dbcada961253a0 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 15:53:20 -0600 Subject: [PATCH 11/23] Refactor --- tests/cpp/test_exceptions.hpp | 16 +++----- tests/cpp/test_serialize_deserialize.hpp | 52 +++++++++--------------- tests/jsons/{V1 => V1.0}/test0.json | 0 tests/jsons/{V1 => V1.0}/test1.json | 0 tests/jsons/{V1 => V1.0}/test10.json | 0 tests/jsons/{V1 => V1.0}/test11.json | 0 tests/jsons/{V1 => V1.0}/test12.json | 0 tests/jsons/{V1 => V1.0}/test13.json | 0 tests/jsons/{V1 => V1.0}/test14.json | 0 tests/jsons/{V1 => V1.0}/test15.json | 0 tests/jsons/{V1 => V1.0}/test16.json | 0 tests/jsons/{V1 => V1.0}/test17.json | 0 tests/jsons/{V1 => V1.0}/test18.json | 0 tests/jsons/{V1 => V1.0}/test19.json | 0 tests/jsons/{V1 => V1.0}/test2.json | 0 tests/jsons/{V1 => V1.0}/test20.json | 0 tests/jsons/{V1 => V1.0}/test21.json | 0 tests/jsons/{V1 => V1.0}/test22.json | 0 tests/jsons/{V1 => V1.0}/test23.json | 0 tests/jsons/{V1 => V1.0}/test24.json | 0 tests/jsons/{V1 => V1.0}/test25.json | 0 tests/jsons/{V1 => V1.0}/test3.json | 0 tests/jsons/{V1 => V1.0}/test4.json | 0 tests/jsons/{V1 => V1.0}/test5.json | 0 tests/jsons/{V1 => V1.0}/test6.json | 0 tests/jsons/{V1 => V1.0}/test7.json | 0 tests/jsons/{V1 => V1.0}/test8.json | 0 tests/jsons/{V1 => V1.0}/test9.json | 0 28 files changed, 26 insertions(+), 42 deletions(-) rename tests/jsons/{V1 => V1.0}/test0.json (100%) rename tests/jsons/{V1 => V1.0}/test1.json (100%) rename tests/jsons/{V1 => V1.0}/test10.json (100%) rename tests/jsons/{V1 => V1.0}/test11.json (100%) rename tests/jsons/{V1 => V1.0}/test12.json (100%) rename tests/jsons/{V1 => V1.0}/test13.json (100%) rename tests/jsons/{V1 => V1.0}/test14.json (100%) rename tests/jsons/{V1 => V1.0}/test15.json (100%) rename tests/jsons/{V1 => V1.0}/test16.json (100%) rename tests/jsons/{V1 => V1.0}/test17.json (100%) rename tests/jsons/{V1 => V1.0}/test18.json (100%) rename tests/jsons/{V1 => V1.0}/test19.json (100%) rename tests/jsons/{V1 => V1.0}/test2.json (100%) rename tests/jsons/{V1 => V1.0}/test20.json (100%) rename tests/jsons/{V1 => V1.0}/test21.json (100%) rename tests/jsons/{V1 => V1.0}/test22.json (100%) rename tests/jsons/{V1 => V1.0}/test23.json (100%) rename tests/jsons/{V1 => V1.0}/test24.json (100%) rename tests/jsons/{V1 => V1.0}/test25.json (100%) rename tests/jsons/{V1 => V1.0}/test3.json (100%) rename tests/jsons/{V1 => V1.0}/test4.json (100%) rename tests/jsons/{V1 => V1.0}/test5.json (100%) rename tests/jsons/{V1 => V1.0}/test6.json (100%) rename tests/jsons/{V1 => V1.0}/test7.json (100%) rename tests/jsons/{V1 => V1.0}/test8.json (100%) rename tests/jsons/{V1 => V1.0}/test9.json (100%) diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index ee5a92a..0021757 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -22,11 +22,9 @@ TEST(EXCEPTION_SUITE_NAME, testWhatMEthods) { } TEST(EXCEPTION_SUITE_NAME, testFailedDump) { - const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", - "/tmp/capio_cl_jsons/V1.1/test24.json"}; - - for (const auto &source : json_path) { - auto engine = capiocl::parser::Parser::parse(source, "/tmp"); + for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path source = "/tmp/capio_cl_jsons/V" + version + "/test24.json"; + auto engine = capiocl::parser::Parser::parse(source, "/tmp"); EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "/"), capiocl::serializer::SerializerException); @@ -34,11 +32,9 @@ TEST(EXCEPTION_SUITE_NAME, testFailedDump) { } TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { - const std::vector json_path = {"/tmp/capio_cl_jsons/V1/test24.json", - "/tmp/capio_cl_jsons/V1.1/test24.json"}; - - for (const auto &source : json_path) { - auto engine = capiocl::parser::Parser::parse(source, "/tmp"); + for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path source = "/tmp/capio_cl_jsons/V" + version + "/test24.json"; + auto engine = capiocl::parser::Parser::parse(source, "/tmp"); EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "test.json", "1234.5678"), capiocl::serializer::SerializerException); diff --git a/tests/cpp/test_serialize_deserialize.hpp b/tests/cpp/test_serialize_deserialize.hpp index d2b20be..8c0cd1a 100644 --- a/tests/cpp/test_serialize_deserialize.hpp +++ b/tests/cpp/test_serialize_deserialize.hpp @@ -154,40 +154,28 @@ TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeCommitOnCloseCountNoCommitRu } } -TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1/test0.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - EXPECT_TRUE(engine->getWorkflowName() == "test"); - EXPECT_TRUE(engine->contains("/tmp/file")); - EXPECT_TRUE(engine->contains("/tmp/file1")); - EXPECT_TRUE(engine->contains("/tmp/file2")); - EXPECT_TRUE(engine->contains("/tmp/file3")); -} - -TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSectionV1) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1/test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - EXPECT_TRUE(engine->getWorkflowName() == "test"); - EXPECT_TRUE(engine->contains("/tmp/file")); - EXPECT_TRUE(engine->contains("/tmp/file1")); +TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsolute) { + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V" + _cl_version + + "/test0.json"); + auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(engine->getWorkflowName() == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + EXPECT_TRUE(engine->contains("/tmp/file2")); + EXPECT_TRUE(engine->contains("/tmp/file3")); + } } -TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsoluteV1_1) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1.1/test0.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - EXPECT_TRUE(engine->getWorkflowName() == "test"); - EXPECT_TRUE(engine->contains("/tmp/file")); - EXPECT_TRUE(engine->contains("/tmp/file1")); - EXPECT_TRUE(engine->contains("/tmp/file2")); - EXPECT_TRUE(engine->contains("/tmp/file3")); -} +TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSection) { -TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSectionV1_1) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1.1/test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - EXPECT_TRUE(engine->getWorkflowName() == "test"); - EXPECT_TRUE(engine->contains("/tmp/file")); - EXPECT_TRUE(engine->contains("/tmp/file1")); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V" + _cl_version + + "/test24.json"); + auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(engine->getWorkflowName() == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + } } - #endif // CAPIO_CL_TEST_SERIALIZE_DESERIALIZE_HPP \ No newline at end of file diff --git a/tests/jsons/V1/test0.json b/tests/jsons/V1.0/test0.json similarity index 100% rename from tests/jsons/V1/test0.json rename to tests/jsons/V1.0/test0.json diff --git a/tests/jsons/V1/test1.json b/tests/jsons/V1.0/test1.json similarity index 100% rename from tests/jsons/V1/test1.json rename to tests/jsons/V1.0/test1.json diff --git a/tests/jsons/V1/test10.json b/tests/jsons/V1.0/test10.json similarity index 100% rename from tests/jsons/V1/test10.json rename to tests/jsons/V1.0/test10.json diff --git a/tests/jsons/V1/test11.json b/tests/jsons/V1.0/test11.json similarity index 100% rename from tests/jsons/V1/test11.json rename to tests/jsons/V1.0/test11.json diff --git a/tests/jsons/V1/test12.json b/tests/jsons/V1.0/test12.json similarity index 100% rename from tests/jsons/V1/test12.json rename to tests/jsons/V1.0/test12.json diff --git a/tests/jsons/V1/test13.json b/tests/jsons/V1.0/test13.json similarity index 100% rename from tests/jsons/V1/test13.json rename to tests/jsons/V1.0/test13.json diff --git a/tests/jsons/V1/test14.json b/tests/jsons/V1.0/test14.json similarity index 100% rename from tests/jsons/V1/test14.json rename to tests/jsons/V1.0/test14.json diff --git a/tests/jsons/V1/test15.json b/tests/jsons/V1.0/test15.json similarity index 100% rename from tests/jsons/V1/test15.json rename to tests/jsons/V1.0/test15.json diff --git a/tests/jsons/V1/test16.json b/tests/jsons/V1.0/test16.json similarity index 100% rename from tests/jsons/V1/test16.json rename to tests/jsons/V1.0/test16.json diff --git a/tests/jsons/V1/test17.json b/tests/jsons/V1.0/test17.json similarity index 100% rename from tests/jsons/V1/test17.json rename to tests/jsons/V1.0/test17.json diff --git a/tests/jsons/V1/test18.json b/tests/jsons/V1.0/test18.json similarity index 100% rename from tests/jsons/V1/test18.json rename to tests/jsons/V1.0/test18.json diff --git a/tests/jsons/V1/test19.json b/tests/jsons/V1.0/test19.json similarity index 100% rename from tests/jsons/V1/test19.json rename to tests/jsons/V1.0/test19.json diff --git a/tests/jsons/V1/test2.json b/tests/jsons/V1.0/test2.json similarity index 100% rename from tests/jsons/V1/test2.json rename to tests/jsons/V1.0/test2.json diff --git a/tests/jsons/V1/test20.json b/tests/jsons/V1.0/test20.json similarity index 100% rename from tests/jsons/V1/test20.json rename to tests/jsons/V1.0/test20.json diff --git a/tests/jsons/V1/test21.json b/tests/jsons/V1.0/test21.json similarity index 100% rename from tests/jsons/V1/test21.json rename to tests/jsons/V1.0/test21.json diff --git a/tests/jsons/V1/test22.json b/tests/jsons/V1.0/test22.json similarity index 100% rename from tests/jsons/V1/test22.json rename to tests/jsons/V1.0/test22.json diff --git a/tests/jsons/V1/test23.json b/tests/jsons/V1.0/test23.json similarity index 100% rename from tests/jsons/V1/test23.json rename to tests/jsons/V1.0/test23.json diff --git a/tests/jsons/V1/test24.json b/tests/jsons/V1.0/test24.json similarity index 100% rename from tests/jsons/V1/test24.json rename to tests/jsons/V1.0/test24.json diff --git a/tests/jsons/V1/test25.json b/tests/jsons/V1.0/test25.json similarity index 100% rename from tests/jsons/V1/test25.json rename to tests/jsons/V1.0/test25.json diff --git a/tests/jsons/V1/test3.json b/tests/jsons/V1.0/test3.json similarity index 100% rename from tests/jsons/V1/test3.json rename to tests/jsons/V1.0/test3.json diff --git a/tests/jsons/V1/test4.json b/tests/jsons/V1.0/test4.json similarity index 100% rename from tests/jsons/V1/test4.json rename to tests/jsons/V1.0/test4.json diff --git a/tests/jsons/V1/test5.json b/tests/jsons/V1.0/test5.json similarity index 100% rename from tests/jsons/V1/test5.json rename to tests/jsons/V1.0/test5.json diff --git a/tests/jsons/V1/test6.json b/tests/jsons/V1.0/test6.json similarity index 100% rename from tests/jsons/V1/test6.json rename to tests/jsons/V1.0/test6.json diff --git a/tests/jsons/V1/test7.json b/tests/jsons/V1.0/test7.json similarity index 100% rename from tests/jsons/V1/test7.json rename to tests/jsons/V1.0/test7.json diff --git a/tests/jsons/V1/test8.json b/tests/jsons/V1.0/test8.json similarity index 100% rename from tests/jsons/V1/test8.json rename to tests/jsons/V1.0/test8.json diff --git a/tests/jsons/V1/test9.json b/tests/jsons/V1.0/test9.json similarity index 100% rename from tests/jsons/V1/test9.json rename to tests/jsons/V1.0/test9.json From 79382d6f5ff6479e4b714d592b222d2f68063c0b Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 16:10:00 -0600 Subject: [PATCH 12/23] Configuration tests --- CMakeLists.txt | 14 ++++++++++++++ src/configuration.cpp | 7 ++++++- tests/cpp/main.cpp | 3 ++- tests/cpp/test_configuration.hpp | 12 ++++++++++++ tests/tomls/sample1.toml | 10 ++++++++++ 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/cpp/test_configuration.hpp create mode 100644 tests/tomls/sample1.toml diff --git a/CMakeLists.txt b/CMakeLists.txt index 189eeef..b252831 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -213,6 +213,20 @@ if (CAPIO_CL_BUILD_TESTS) COMMENT "Copying JSON test files with full directory structure" ) + ##################################### + # Copy TOMLS test files + ##################################### + set(TEST_TOMLS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/tomls") + + add_custom_command( + TARGET CAPIO_CL_tests PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "/tmp/capio_cl_tomls" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${TEST_TOMLS_DIR}" + "/tmp/capio_cl_tomls" + COMMENT "Copying TOMLS test files with full directory structure" + ) + ##################################### # Install rules for tests ##################################### diff --git a/src/configuration.cpp b/src/configuration.cpp index b4188d4..65caa90 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -14,7 +14,12 @@ void flatten_table(const toml::table &tbl, std::unordered_mapget(); + const auto itm = value.as_string(); + if (itm == nullptr) { + map[full_key] = std::to_string(value.as_integer()->get()); + }else { + map[full_key] = itm->get(); + } } } } diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index fd0b795..91a5cbf 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -24,4 +24,5 @@ template std::string demangled_name(const T &obj) { #include "test_engine.hpp" #include "test_exceptions.hpp" #include "test_monitor.hpp" -#include "test_serialize_deserialize.hpp" \ No newline at end of file +#include "test_serialize_deserialize.hpp" +#include "test_configuration.hpp" \ No newline at end of file diff --git a/tests/cpp/test_configuration.hpp b/tests/cpp/test_configuration.hpp new file mode 100644 index 0000000..3f547ea --- /dev/null +++ b/tests/cpp/test_configuration.hpp @@ -0,0 +1,12 @@ +#ifndef CAPIO_CL_TEST_CONFIGURATION_HPP +#define CAPIO_CL_TEST_CONFIGURATION_HPP + +#define CONFIGURATION_SUITE_NAME TestTOMLConfiguration + +TEST(CONFIGURATION_SUITE_NAME, TestLoadConfiguration) { + capiocl::engine::Engine engine; + engine.loadConfiguration("/tmp/capio_cl_tomls/sample1.toml"); + EXPECT_TRUE(true); +} + +#endif // CAPIO_CL_TEST_CONFIGURATION_HPP diff --git a/tests/tomls/sample1.toml b/tests/tomls/sample1.toml new file mode 100644 index 0000000..b8e486f --- /dev/null +++ b/tests/tomls/sample1.toml @@ -0,0 +1,10 @@ +[monitor.mcast] +delay_ms = 300 + +[monitor.mcast.commit] +ip = "224.224.224.3" +port = 11223 + +[monitor.mcast.homenode] +ip = "224.224.224.4" +port = 22334 From 1550d5cab1fd0ddeb653fcbd1db964506f118204 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 16:30:54 -0600 Subject: [PATCH 13/23] Exception for configuration --- capiocl.hpp | 1 + include/configuration.h | 22 +++++++++++++++++++++- src/configuration.cpp | 17 +++++++++++++---- tests/cpp/test_configuration.hpp | 7 +++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index d785345..2cb2fe6 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -96,6 +96,7 @@ class Engine; namespace configuration { class CapioClConfiguration; +class CapioClConfigurationException; struct defaults; } } // namespace capiocl diff --git a/include/configuration.h b/include/configuration.h index ee2bf50..13bb693 100644 --- a/include/configuration.h +++ b/include/configuration.h @@ -25,7 +25,7 @@ class capiocl::configuration::CapioClConfiguration { protected: void set(const std::string &key, std::string value); - void set(ConfigurationEntry entry); + void set(const ConfigurationEntry &entry); public: explicit CapioClConfiguration(); @@ -36,4 +36,24 @@ class capiocl::configuration::CapioClConfiguration { void getParameter(const std::string &key, std::string *value) const; }; +/** + * @brief Custom exception thrown when handling a CAPIO-CL TOML configuration file + */ +class capiocl::configuration::CapioClConfigurationException final : public std::exception { + std::string message; + + public: + /** + * @brief Construct a new CAPIO-CL Exception + * @param msg Error Message that raised this exception + */ + explicit CapioClConfigurationException(const std::string &msg); + + /** + * Get the description of the error causing the exception + * @return + */ + [[nodiscard]] const char *what() const noexcept override { return message.c_str(); } +}; + #endif \ No newline at end of file diff --git a/src/configuration.cpp b/src/configuration.cpp index 65caa90..c298801 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -17,7 +17,7 @@ void flatten_table(const toml::table &tbl, std::unordered_mapget()); - }else { + } else { map[full_key] = itm->get(); } } @@ -36,20 +36,20 @@ void capiocl::configuration::CapioClConfiguration::set(const std::string &key, s config[key] = std::move(value); } -void capiocl::configuration::CapioClConfiguration::set(const ConfigurationEntry entry) { +void capiocl::configuration::CapioClConfiguration::set(const ConfigurationEntry &entry) { this->set(entry.k, entry.v); } void capiocl::configuration::CapioClConfiguration::load(const std::filesystem::path &path) { if (path.empty()) { - return; + throw CapioClConfigurationException("Empty pathname!"); } toml::table tbl; try { tbl = toml::parse_file(path.string()); } catch (const toml::parse_error &err) { - printer::print(printer::CLI_LEVEL_ERROR, err.what()); + throw CapioClConfigurationException(err.what()); } flatten_table(tbl, config); } @@ -59,6 +59,8 @@ void capiocl::configuration::CapioClConfiguration::getParameter(const std::strin if (config.find(key) != config.end()) { *value = std::stoi(config.at(key)); + } else { + throw CapioClConfigurationException("Key" + key + " not found!"); } } @@ -66,5 +68,12 @@ void capiocl::configuration::CapioClConfiguration::getParameter(const std::strin std::string *value) const { if (config.find(key) != config.end()) { *value = config.at(key); + } else { + throw CapioClConfigurationException("Key" + key + " not found!"); } +} +capiocl::configuration::CapioClConfigurationException::CapioClConfigurationException( + const std::string &msg) + : message(msg) { + printer::print(printer::CLI_LEVEL_ERROR, msg); } \ No newline at end of file diff --git a/tests/cpp/test_configuration.hpp b/tests/cpp/test_configuration.hpp index 3f547ea..641d529 100644 --- a/tests/cpp/test_configuration.hpp +++ b/tests/cpp/test_configuration.hpp @@ -2,6 +2,7 @@ #define CAPIO_CL_TEST_CONFIGURATION_HPP #define CONFIGURATION_SUITE_NAME TestTOMLConfiguration +#include "include/configuration.h" TEST(CONFIGURATION_SUITE_NAME, TestLoadConfiguration) { capiocl::engine::Engine engine; @@ -9,4 +10,10 @@ TEST(CONFIGURATION_SUITE_NAME, TestLoadConfiguration) { EXPECT_TRUE(true); } +TEST(CONFIGURATION_SUITE_NAME, TestLoadEmptyPath) { + capiocl::engine::Engine engine; + EXPECT_THROW(engine.loadConfiguration(""), + capiocl::configuration::CapioClConfigurationException); +} + #endif // CAPIO_CL_TEST_CONFIGURATION_HPP From c55d7a13c835c54f02e17953b2fb8d382f9ad6a2 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Wed, 19 Nov 2025 16:42:39 -0600 Subject: [PATCH 14/23] Tests --- src/configuration.cpp | 19 +++++++++++-------- tests/cpp/main.cpp | 6 +++--- tests/cpp/test_configuration.hpp | 22 ++++++++++++++++++++++ tests/tomls/sample0.toml | 1 + 4 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 tests/tomls/sample0.toml diff --git a/src/configuration.cpp b/src/configuration.cpp index c298801..ca960e5 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -8,17 +8,20 @@ void flatten_table(const toml::table &tbl, std::unordered_map &map, const std::string &prefix = "") { for (const auto &[key, value] : tbl) { - std::string full_key = - prefix.empty() ? std::string{key.str()} : prefix + "." + std::string{key.str()}; + std::string full_key; + if (prefix.empty()) { + full_key = std::string{key.str()}; + } else { + full_key = prefix + "." + std::string{key.str()}; + } if (value.is_table()) { flatten_table(*value.as_table(), map, full_key); } else { - const auto itm = value.as_string(); - if (itm == nullptr) { - map[full_key] = std::to_string(value.as_integer()->get()); + if (value.is_string()) { + map[full_key] = value.as_string()->get(); } else { - map[full_key] = itm->get(); + map[full_key] = std::to_string(value.as_integer()->get()); } } } @@ -60,7 +63,7 @@ void capiocl::configuration::CapioClConfiguration::getParameter(const std::strin if (config.find(key) != config.end()) { *value = std::stoi(config.at(key)); } else { - throw CapioClConfigurationException("Key" + key + " not found!"); + throw CapioClConfigurationException("Key " + key + " not found!"); } } @@ -69,7 +72,7 @@ void capiocl::configuration::CapioClConfiguration::getParameter(const std::strin if (config.find(key) != config.end()) { *value = config.at(key); } else { - throw CapioClConfigurationException("Key" + key + " not found!"); + throw CapioClConfigurationException("Key " + key + " not found!"); } } capiocl::configuration::CapioClConfigurationException::CapioClConfigurationException( diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index 91a5cbf..248ec97 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -1,8 +1,8 @@ +#include "capiocl.hpp" #include #include #include #include -#include "capiocl.hpp" const std::vector CAPIO_CL_AVAIL_VERSIONS = {capiocl::CAPIO_CL_VERSION::V1, capiocl::CAPIO_CL_VERSION::V1_1}; @@ -21,8 +21,8 @@ template std::string demangled_name(const T &obj) { #include "include/printer.h" #include "include/serializer.h" +#include "test_configuration.hpp" #include "test_engine.hpp" #include "test_exceptions.hpp" #include "test_monitor.hpp" -#include "test_serialize_deserialize.hpp" -#include "test_configuration.hpp" \ No newline at end of file +#include "test_serialize_deserialize.hpp" \ No newline at end of file diff --git a/tests/cpp/test_configuration.hpp b/tests/cpp/test_configuration.hpp index 641d529..e7948bb 100644 --- a/tests/cpp/test_configuration.hpp +++ b/tests/cpp/test_configuration.hpp @@ -16,4 +16,26 @@ TEST(CONFIGURATION_SUITE_NAME, TestLoadEmptyPath) { capiocl::configuration::CapioClConfigurationException); } +TEST(CONFIGURATION_SUITE_NAME, TestExceptions) { + capiocl::configuration::CapioClConfiguration config; + + EXPECT_THROW(config.getParameter("not.a.valid.key", static_cast(nullptr)), + capiocl::configuration::CapioClConfigurationException); + + EXPECT_THROW(config.getParameter("not.a.valid.key", static_cast(nullptr)), + capiocl::configuration::CapioClConfigurationException); + + try { + config.getParameter("not.a.valid.key", static_cast(nullptr)); + } catch (const capiocl::configuration::CapioClConfigurationException &err) { + EXPECT_GT(strlen(err.what()), 0); + } +} + +TEST(CONFIGURATION_SUITE_NAME, TestFailureParsingTOML) { + capiocl::configuration::CapioClConfiguration config; + EXPECT_THROW(config.load("/tmp/capio_cl_tomls/sample0.toml"), + capiocl::configuration::CapioClConfigurationException); +} + #endif // CAPIO_CL_TEST_CONFIGURATION_HPP diff --git a/tests/tomls/sample0.toml b/tests/tomls/sample0.toml new file mode 100644 index 0000000..7b80d4c --- /dev/null +++ b/tests/tomls/sample0.toml @@ -0,0 +1 @@ +[a.wrong.formatted.table \ No newline at end of file From b2e3bebee4a3a825eca3de89e0fe38e942a2327a Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 20 Nov 2025 07:09:13 -0600 Subject: [PATCH 15/23] Test configuration --- src/parsers/v1.1.cpp | 3 ++- tests/jsons/V1.1/test24.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/parsers/v1.1.cpp b/src/parsers/v1.1.cpp index 146f13e..6c21a4a 100644 --- a/src/parsers/v1.1.cpp +++ b/src/parsers/v1.1.cpp @@ -1,10 +1,10 @@ #include +#include "capio_cl_json_schemas.hpp" #include "capiocl.hpp" #include "include/engine.h" #include "include/parser.h" #include "include/printer.h" -#include "capio_cl_json_schemas.hpp" capiocl::engine::Engine * capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::path &source, @@ -27,6 +27,7 @@ capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::pa // ---- CAPIO-CL TOML CONFIGURATION ---- if (doc.contains("configuration")) { auto toml_config_path = doc["configuration"].as(); + printer::print(printer::CLI_LEVEL_JSON, "Using configuration file : " + toml_config_path); engine->loadConfiguration(toml_config_path); } else { engine->useDefaultConfiguration(); diff --git a/tests/jsons/V1.1/test24.json b/tests/jsons/V1.1/test24.json index 50f3917..5dc8be5 100644 --- a/tests/jsons/V1.1/test24.json +++ b/tests/jsons/V1.1/test24.json @@ -11,5 +11,6 @@ "file1" ] } - ] + ], + "configuration" : "/tmp/capio_cl_tomls/sample1.toml" } \ No newline at end of file From 74a5e7defae95286d8ea1c590142b339e003e296 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 20 Nov 2025 07:52:02 -0600 Subject: [PATCH 16/23] documentation --- capiocl.hpp | 3 ++- include/configuration.h | 35 +++++++++++++++++++++++++++++++++++ include/parser.h | 2 +- include/printer.h | 2 +- src/parsers/v1.cpp | 2 +- 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index 2cb2fe6..ccc663e 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -77,6 +77,7 @@ inline std::string sanitize(const std::string &input) { struct CAPIO_CL_VERSION final { /// @brief Release 1.0 of CAPIO-CL static constexpr char V1[] = "1.0"; + /// @brief Release 1.1 of CAPIO-CL static constexpr char V1_1[] = "1.1"; }; @@ -98,7 +99,7 @@ namespace configuration { class CapioClConfiguration; class CapioClConfigurationException; struct defaults; -} +} // namespace configuration } // namespace capiocl #endif // CAPIO_CL_CAPIOCL_HPP \ No newline at end of file diff --git a/include/configuration.h b/include/configuration.h index 13bb693..2c00af4 100644 --- a/include/configuration.h +++ b/include/configuration.h @@ -5,16 +5,25 @@ #include "capiocl.hpp" +/// @brief Struct containing key value pairs for capio-cl runtime configuration typedef struct { + /// @brief Key of option std::string k; + /// @brief Value of option std::string v; } ConfigurationEntry; +/// @brief Defaults values keys for runtime options of CAPIO-CL struct capiocl::configuration::defaults { + /// @brief Multicast monitor commit IP static ConfigurationEntry DEFAULT_MONITOR_MCAST_IP; + /// @brief Multicast monitor commit PORT static ConfigurationEntry DEFAULT_MONITOR_MCAST_PORT; + /// @brief Multicast monitor delay before following operation static ConfigurationEntry DEFAULT_MONITOR_MCAST_DELAY; + /// @brief Multicast monitor homenode IP static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_IP; + /// @brief Multicast monitor homenode PORT static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_PORT; }; @@ -24,15 +33,41 @@ class capiocl::configuration::CapioClConfiguration { std::unordered_map config; protected: + /** + * Set a key value pair explicitly + * @param key Option key name + * @param value Value + */ void set(const std::string &key, std::string value); + + /** + * Set a capio-cl configuration option trough a ConfigurationEntry object + * @param entry + */ void set(const ConfigurationEntry &entry); public: explicit CapioClConfiguration(); ~CapioClConfiguration() = default; + /** + * Load a configuiration from a TOML file + * @param path + */ void load(const std::filesystem::path &path); + + /** + * Get a string value + * @param key key of option to get + * @param value reference in which value will be stored + */ void getParameter(const std::string &key, int *value) const; + + /** + * Get a integer value + * @param key key of option to get + * @param value reference in which value will be stored + */ void getParameter(const std::string &key, std::string *value) const; }; diff --git a/include/parser.h b/include/parser.h index 4c79256..6ebafee 100644 --- a/include/parser.h +++ b/include/parser.h @@ -79,7 +79,7 @@ class Parser final { * @param doc The loaded CAPIO-CL configuration file * @param str_schema Raw JSON schema to use */ - static void validate_json(const jsoncons::json &doc, const char* str_schema); + static void validate_json(const jsoncons::json &doc, const char *str_schema); /** * @brief Perform the parsing of the capio_server configuration file diff --git a/include/printer.h b/include/printer.h index 04ea2ec..98d0714 100644 --- a/include/printer.h +++ b/include/printer.h @@ -1,8 +1,8 @@ #ifndef CAPIO_CL_PRINTER_H #define CAPIO_CL_PRINTER_H #include -#include #include +#include #ifndef HOST_NAME_MAX #define HOST_NAME_MAX 1024 diff --git a/src/parsers/v1.cpp b/src/parsers/v1.cpp index 24533fb..4a892ca 100644 --- a/src/parsers/v1.cpp +++ b/src/parsers/v1.cpp @@ -1,10 +1,10 @@ #include +#include "capio_cl_json_schemas.hpp" #include "capiocl.hpp" #include "include/engine.h" #include "include/parser.h" #include "include/printer.h" -#include "capio_cl_json_schemas.hpp" capiocl::engine::Engine * capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path &source, From 35e50f1cdfab9886cd84a6e2002e619ca884f025 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 20 Nov 2025 07:59:35 -0600 Subject: [PATCH 17/23] format --- capiocl.hpp | 9 --------- include/configuration.h | 2 +- include/monitor.h | 5 ++--- include/printer.h | 1 - include/serializer.h | 2 +- src/Parser.cpp | 3 +-- src/configuration.cpp | 2 +- src/monitors/FileSystem.cpp | 2 +- src/serializers/v1.1.cpp | 1 - src/serializers/v1.cpp | 1 - 10 files changed, 7 insertions(+), 21 deletions(-) diff --git a/capiocl.hpp b/capiocl.hpp index ccc663e..aeac085 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -1,17 +1,8 @@ #ifndef CAPIO_CL_CAPIOCL_HPP #define CAPIO_CL_CAPIOCL_HPP -#include -#include -#include -#include #include -#include #include -#include -#include -#include -#include /// @brief Namespace containing all the CAPIO-CL related code namespace capiocl { diff --git a/include/configuration.h b/include/configuration.h index 2c00af4..3a00cc7 100644 --- a/include/configuration.h +++ b/include/configuration.h @@ -41,7 +41,7 @@ class capiocl::configuration::CapioClConfiguration { void set(const std::string &key, std::string value); /** - * Set a capio-cl configuration option trough a ConfigurationEntry object + * Set a capio-cl configuration option through a ConfigurationEntry object * @param entry */ void set(const ConfigurationEntry &entry); diff --git a/include/monitor.h b/include/monitor.h index d2cae8c..2ab6b3d 100644 --- a/include/monitor.h +++ b/include/monitor.h @@ -1,9 +1,6 @@ #ifndef CAPIO_CL_MONITOR_H #define CAPIO_CL_MONITOR_H -#include "configuration.h" - -#include #include #include #include @@ -12,6 +9,8 @@ #include #include +#include "configuration.h" + #ifndef PATH_MAX #define PATH_MAX 4096 #endif diff --git a/include/printer.h b/include/printer.h index 98d0714..1069bf6 100644 --- a/include/printer.h +++ b/include/printer.h @@ -1,6 +1,5 @@ #ifndef CAPIO_CL_PRINTER_H #define CAPIO_CL_PRINTER_H -#include #include #include diff --git a/include/serializer.h b/include/serializer.h index 2ab65ba..1c20ef7 100644 --- a/include/serializer.h +++ b/include/serializer.h @@ -1,7 +1,7 @@ #ifndef CAPIO_CL_SERIALIZER_H #define CAPIO_CL_SERIALIZER_H -#include +#include "capiocl.hpp" /// @brief Namespace containing the CAPIO-CL Serializer component namespace capiocl::serializer { diff --git a/src/Parser.cpp b/src/Parser.cpp index 306c325..61951ea 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -32,8 +32,7 @@ std::filesystem::path capiocl::parser::Parser::resolve(std::filesystem::path pat return resolved; } -void capiocl::parser::Parser::validate_json(const jsoncons::json &doc, - const char* str_schema) { +void capiocl::parser::Parser::validate_json(const jsoncons::json &doc, const char *str_schema) { jsoncons::jsonschema::json_schema schema = loadSchema(str_schema); try { // throws jsoncons::jsonschema::validation_error on failure diff --git a/src/configuration.cpp b/src/configuration.cpp index ca960e5..4859977 100644 --- a/src/configuration.cpp +++ b/src/configuration.cpp @@ -1,9 +1,9 @@ -#include "toml++/toml.hpp" #include #include #include "include/configuration.h" #include "include/printer.h" +#include "toml++/toml.hpp" void flatten_table(const toml::table &tbl, std::unordered_map &map, const std::string &prefix = "") { diff --git a/src/monitors/FileSystem.cpp b/src/monitors/FileSystem.cpp index 4924e86..4358bf8 100644 --- a/src/monitors/FileSystem.cpp +++ b/src/monitors/FileSystem.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include "capiocl.hpp" #include "include/monitor.h" diff --git a/src/serializers/v1.1.cpp b/src/serializers/v1.1.cpp index 837d49f..eb75761 100644 --- a/src/serializers/v1.1.cpp +++ b/src/serializers/v1.1.cpp @@ -1,5 +1,4 @@ #include -#include #include "capiocl.hpp" #include "include/engine.h" diff --git a/src/serializers/v1.cpp b/src/serializers/v1.cpp index e730ab8..738f57d 100644 --- a/src/serializers/v1.cpp +++ b/src/serializers/v1.cpp @@ -1,5 +1,4 @@ #include -#include #include "capiocl.hpp" #include "include/engine.h" From a4e63c4b2dc4f3662562dff0b6a2a2a5ff89fb68 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 20 Nov 2025 09:01:42 -0600 Subject: [PATCH 18/23] cicd fixes --- .github/workflows/python-bindings.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index d2771b4..24b697d 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -54,7 +54,9 @@ jobs: - name: "Build and Install Python package" run: | mkdir -p /tmp/capio_cl_jsons - cp tests/jsons/*.json /tmp/capio_cl_jsons + mkdir -p /tmp/capio_cl_tomls + cp -r tests/jsons/ /tmp/capio_cl_jsons + cp -r tests/tomls/ /tmp/capio_cl_tomls . ./venv/bin/activate pip install . From edb0020c4f12f79df063316cf22acf7452553863 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Thu, 20 Nov 2025 09:08:47 -0600 Subject: [PATCH 19/23] tests --- tests/python/test_parser_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_parser_serializer.py b/tests/python/test_parser_serializer.py index 11793ea..6493392 100644 --- a/tests/python/test_parser_serializer.py +++ b/tests/python/test_parser_serializer.py @@ -68,7 +68,7 @@ def test_serialize_parse_py_capio_cl_v1(tmp_path): def test_parser_resolve_absolute(): - json_path = "/tmp/capio_cl_jsons/V1_test0.json" + json_path = "/tmp/capio_cl_jsons/V1.0/test0.json" engine = py_capio_cl.Parser.parse(str(json_path), "/tmp") assert engine.getWorkflowName() == "test" From 190a4859f17e67ff6cef823bd3d98a4c63ad43d7 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 21 Nov 2025 08:48:18 -0600 Subject: [PATCH 20/23] Fixed CI/CD workflow --- .github/workflows/python-bindings.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index 24b697d..67c9a6c 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -55,8 +55,8 @@ jobs: run: | mkdir -p /tmp/capio_cl_jsons mkdir -p /tmp/capio_cl_tomls - cp -r tests/jsons/ /tmp/capio_cl_jsons - cp -r tests/tomls/ /tmp/capio_cl_tomls + cp -r tests/jsons/* /tmp/capio_cl_jsons + cp -r tests/tomls/* /tmp/capio_cl_tomls . ./venv/bin/activate pip install . @@ -130,7 +130,9 @@ jobs: python3 -m pip install -r build-requirements.txt mkdir -p /tmp/capio_cl_jsons - cp tests/jsons/*.json /tmp/capio_cl_jsons + mkdir -p /tmp/capio_cl_tomls + cp -r tests/jsons/* /tmp/capio_cl_jsons + cp -r tests/tomls/* /tmp/capio_cl_tomls python3 -m build \ -Ccmake.define.ENABLE_COVERAGE=OFF \ @@ -139,6 +141,5 @@ jobs: pip install dist/*.whl python3 -m pip install -r test-requirements.txt - - echo "✅ Running unit tests" + pytest -v tests/python/test_* | tee pytest-riscv.log From 28c696b9b397e031db40a3388336f091f674198b Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 21 Nov 2025 09:13:44 -0600 Subject: [PATCH 21/23] Memory leak fixes --- include/engine.h | 2 +- src/Engine.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/engine.h b/include/engine.h index e43d031..dd35518 100644 --- a/include/engine.h +++ b/include/engine.h @@ -28,7 +28,7 @@ class Engine final { bool store_all_in_memory = false; ///@brief Configuration imported from CAPIO-CL config TOML file - configuration::CapioClConfiguration *configuration; + configuration::CapioClConfiguration configuration; /// @brief Monitor instance to check runtime information of CAPIO-CL files monitor::Monitor monitor; diff --git a/src/Engine.cpp b/src/Engine.cpp index b1729be..55e9389 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -137,7 +137,6 @@ capiocl::engine::Engine::Engine(const bool use_default_settings) { this->workflow_name = CAPIO_CL_DEFAULT_WF_NAME; } - configuration = new configuration::CapioClConfiguration(); if (use_default_settings) { this->useDefaultConfiguration(); } @@ -755,9 +754,9 @@ bool capiocl::engine::Engine::operator==(const capiocl::engine::Engine &other) c return true; } void capiocl::engine::Engine::loadConfiguration(const std::string &path) { - configuration->load(path); + configuration.load(path); - monitor.registerMonitorBackend(new monitor::MulticastMonitor(*configuration)); + monitor.registerMonitorBackend(new monitor::MulticastMonitor(configuration)); monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); } void capiocl::engine::Engine::useDefaultConfiguration() { From a25361fa2bc87635e8afcda4e5baea5f7b555460 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 21 Nov 2025 10:42:10 -0600 Subject: [PATCH 22/23] Cleanup --- include/engine.h | 2 +- src/Engine.cpp | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/include/engine.h b/include/engine.h index dd35518..a850095 100644 --- a/include/engine.h +++ b/include/engine.h @@ -100,7 +100,7 @@ class Engine final { void compute_directory_entry_count(const std::filesystem::path &path) const; public: - /// @brief Class constructorw + /// @brief Class constructor explicit Engine(bool use_default_settings = true); /// @brief Print the current CAPIO-CL configuration. diff --git a/src/Engine.cpp b/src/Engine.cpp index 55e9389..5509ae2 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -201,12 +201,9 @@ void capiocl::engine::Engine::compute_directory_entry_count( } bool capiocl::engine::Engine::contains(const std::filesystem::path &file) const { - for (auto &[fst, snd] : _capio_cl_entries) { - if (fnmatch(fst.c_str(), file.c_str(), FNM_PATHNAME) == 0) { - return true; - } - } - return false; + return std::any_of(_capio_cl_entries.begin(), _capio_cl_entries.end(), [&](auto const &entry) { + return fnmatch(entry.first.c_str(), file.c_str(), FNM_PATHNAME) == 0; + }); } size_t capiocl::engine::Engine::size() const { return this->_capio_cl_entries.size(); } From 6c2cce87cfbe41a44c2ac993f19eaa90ea3dfe3f Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 21 Nov 2025 11:36:43 -0600 Subject: [PATCH 23/23] Added doxygen documentation --- doxygen/Doxyfile | 16 +- doxygen/configuration.md | 69 +++++++++ doxygen/syntax_v1.md | 308 ++++++++++++--------------------------- doxygen/syntax_v1_1.md | 72 +++++++++ 4 files changed, 244 insertions(+), 221 deletions(-) create mode 100644 doxygen/configuration.md create mode 100644 doxygen/syntax_v1_1.md diff --git a/doxygen/Doxyfile b/doxygen/Doxyfile index 4b1dd75..7187cff 100644 --- a/doxygen/Doxyfile +++ b/doxygen/Doxyfile @@ -943,12 +943,14 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = .README.md \ - ../src \ - ../include \ - ../capiocl.hpp \ - semantics.md \ - syntax_v1.md +INPUT = .README.md \ + ../src \ + ../include \ + ../capiocl.hpp \ + semantics.md \ + syntax_v1.md \ + syntax_v1_1.md \ + configuration.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -2425,7 +2427,7 @@ INCLUDE_PATH = # used. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -INCLUDE_FILE_PATTERNS = +INCLUDE_FILE_PATTERNS = *.md # The PREDEFINED tag can be used to specify one or more macro names that are # defined before the preprocessor is started (similar to the -D option of e.g. diff --git a/doxygen/configuration.md b/doxygen/configuration.md new file mode 100644 index 0000000..96d49a2 --- /dev/null +++ b/doxygen/configuration.md @@ -0,0 +1,69 @@ +# Runtime Configuration for **CAPIO-CL** + +CAPIO-CL supports **runtime configuration** through a user-provided **TOML** configuration file. +This allows you to customize various aspects of the library without recompiling it. + +A configuration file can be loaded into a running CAPIO-CL instance by calling: + + capiocl::Engine engine; + engine.load("path/to/config.toml"); + +If no configuration file is provided, CAPIO-CL falls back to its built-in defaults. + +--- + +## TOML Configuration Structure + +The configuration file uses a structured namespace under the top-level table `monitor.mcast`. + +Available configuration parameters: + +| Key | Type | Default | Description | +|----------------------------------|----------|----------------|----------------------------------------------------------------------------------------------| +| `monitor.mcast.commit.ip` | string | `224.224.224.1` | Multicast IP address used for commit messages | +| `monitor.mcast.commit.port` | integer | `12345` | UDP port for commit messages | +| `monitor.mcast.delay_ms` | integer | `300` | Artificial delay (in milliseconds) inserted before sending multicast messages. Useful for debugging or simulating slower networks. | +| `monitor.mcast.homenode.ip` | string | `224.224.224.2` | IP address of the home node for monitoring operations | +| `monitor.mcast.homenode.port` | integer | `12345` | Port associated with the home node monitoring endpoint | + +--- + +## Example configuration file + +Below is a complete example of a `config.toml` file: + + # Example CAPIO-CL TOML configuration + + [monitor.mcast] + + # Multicast settings for commit messages + commit.ip = "224.224.224.1" + commit.port = 12345 + + # Delay (in milliseconds) + delay_ms = 300 + + # Home node information + homenode.ip = "224.224.224.2" + homenode.port = 12345 + +--- + +## How CAPIO-CL Uses These Settings + +### `commit.ip` and `commit.port` +These fields define where CAPIO-CL sends **commit broadcast messages**. +Commit messages are used for consistency coordination across distributed nodes. + +### `delay_ms` +A small configurable delay may help with: + +- debugging message-ordering issues, +- testing high-latency environments, +- reproducing specific timing-related behavior. + +A value of `0` means no delay. + +### `homenode.ip` and `homenode.port` +These define the **central monitoring endpoint** (the “home node”). +CAPIO-CL uses this endpoint to coordinate monitoring metadata and cluster-wide communication. diff --git a/doxygen/syntax_v1.md b/doxygen/syntax_v1.md index 5d002f5..b831480 100644 --- a/doxygen/syntax_v1.md +++ b/doxygen/syntax_v1.md @@ -1,43 +1,44 @@ -# Syntax of the CAPIO-CL V1 language +# CAPIO-CL V1 Language Specification -In this section we will illustrate how the semantics of the CAPIO-CL configuration language can be expressed in a json -configuration file. +This document describes the syntax and semantics of the **CAPIO-CL V1 configuration language**, expressed through a JSON-based configuration file. +A valid CAPIO-CL file defines workflow structure, file dependencies, streaming semantics, storage behavior, and home-node mapping policies for distributed execution. -## JSON syntax +## 1. JSON Syntax Overview -A valid CAPIO-CL configuration file comprises five different sections. Sections marked with `*` are mandatory. +A CAPIO-CL V1 configuration file is a single JSON object composed of the following sections. +Sections marked with `*` are mandatory. -- `Workflow name(*)`: identifies the application workflow composed by multiple application modules; -- `Alias`: groups under a convenient name a set of files or directories; -- `IO_Graph(*)`: describes file data dependencies among application modules; -- `Permanent`: defines which files must be kept on the permanent storage at the end of the workflow execution; -- `Exclude`: identifies the files and directories not handled by the CAPIO-CL implementation; -- `Home Node Policies`: defines different file mapping policies to establish which CAPIO-CL servers store which files; +Section | Required | Purpose +------- | -------- | ------- +name* | Yes | Identifies the workflow +aliases | No | Provides named groups of files/directories +IO_Graph* | Yes | Describes data dependencies among modules +permanent | No | Files to keep at the end of execution +exclude | No | Files CAPIO-CL should ignore +home_node_policy | No | Rules assigning files to CAPIO-CL servers -### Workflow name +## 2. Workflow Name -This section is identified by the keyword `name`. The name is used as an identifier for the current application -workflow. This is useful as it is possible to distinguish different application workflows running on the same machine. +The workflow name is provided under the key `name`. +This unique identifier allows distinguishing multiple workflows running on the same system. -#### Example +### Example ```json { "name": "my_workflow" } -``` - -### Alias +``` -The aliases section, identified by the keyword aliases is useful to reduce the verbosity of the enumeration of files an -application can consume or produce. It is a vector of objects composed of the following items: +## 3. Aliases -- `group_name`: the alias identifier -- `files`: an array of strings representing file names. +Aliases reduce verbosity by allowing groups of files or directories to be referenced by a single identifier. +The `aliases` section is an array of objects, where each object contains: -#### Example +- `group_name`: alias identifier +- `files`: an array of filename strings -The following snippet is a valid alias section for CAPIO-CL configuration language +### Example ```json { @@ -45,80 +46,53 @@ The following snippet is a valid alias section for CAPIO-CL configuration langua "aliases": [ { "group_name": "group-even", - "files": [ - "file0.dat", - "file2.dat", - "file4.dat" - ] + "files": ["file0.dat", "file2.dat", "file4.dat"] }, { "group_name": "group-odd", - "files": [ - "file1.dat", - "file3.dat", - "file5.dat" - ] + "files": ["file1.dat", "file3.dat", "file5.dat"] } ] } -``` - -### IO Graph - -This section defines the dependencies between input and output streams between application modules comprising the -workflow. It is identified by the keyword `IO_Graph`, and it requires an array of objects. Those objects specify the -input and output streams for each application component. Each object defines the following items: - -- `name`: The name of the application. This keyword is mandatory -- `input_stream`: This keyword identifies the input files and directories the application module is expected to read. - Its value is a vector of strings. This keyword is optional -- `output_stream`: This keyword identifies a vector of files and directory names, i.e., the files and directories the - application module is expected to produce. Its value is a vector of strings. This keyword is optional. -- `streaming`: This optional keyword identifies the files and directories with their streaming semantics, i.e., the - “commit and firing rules” (please see [Semantics](semantics.md) for a more detailed description). Its value is an - array of objects. +``` -#### Streaming section +## 4. IO_Graph -Each object inside the streaming section may define the following attributes. -For **file** entries (either files or globs): +The `IO_Graph` describes how application modules exchange data. +It is an array of objects, each representing one module. -- `name`: The filenames to which the rule applies. The value of this keyword is an array of filenames. -- `committed`: This keyword defines the “commit rule” associated with the files identified with the keywords name. Its - value can be either: - - `on_close:N`: to express the CoC semantic, where N is an integer greater than 1, or can be omitted if N=1; - - `on_termination`: to express the CoT semantic (default); - - `on_file:filename` to express the CoF semantic. `filename` is the file on which the commit rule will commit - against. -- `mode`: his keyword defines the “firing rule” associated with the files and directories identified with the keys name - and dirname, respectively. Its value can be either: - - `update`: for the FoC semantic (default behavior); - - `no_update`: for the FnU semantic; +Key | Required | Description +--- | -------- | ----------- +name | Yes | Module name +input_stream | No | Files/directories the module reads +output_stream | No | Files/directories the module writes +streaming | No | Commit and firing rules for produced data -For **directories** entries (either files or globs), that is that the rule will apply to files and subdirectories -inside the specified directory entry: +### 4.1 Streaming Rules -- `dirname`: The directory names to which the rule applies. The value of this keyword is an array of directory names; +Each entry in `streaming` may describe rules for **files** or **directories**. -- `committed`: This keyword defines the “commit rule” associated with the directories identified with the keywords - dirname. Its value can be either: - - `on_n_files`: to express the Commit on N Files commit rule. If this commit rule is chosen, then the keyword - `n_files` - must be set to `N` (integer greater than 1). The directory will be considered committed as soon as `N` files are - committed inside it.; - - `on_termination`: to express the CoT semantic (default); - - `on_file`: to express the CoF semantic. In this case, the keyword `files_deps`, whose value is an array of - filenames or directory names, defines the set of dependencies. +#### For files (`name`): -- `mode`: his keyword defines the “firing rule” associated with the files and directories identified with the keys name - and dirname, respectively. Its value can be either: +- `name`: array of filenames +- `committed`: commit rule + - `"on_close"` or `"on_close:N"` + - `"on_termination"` + - `"on_file:filename"` +- `mode`: firing rule + - `"update"` (default) + - `"no_update"` - - `update`: for the FoC semantic (default behavior); - - `no_update`: for the FnU semantic; +#### For directories (`dirname`): -#### Example +- `dirname`: array of directory names +- `committed`: directory commit rule + - `"on_n_files"` (requires field `"n_files": N`) + - `"on_termination"` + - `"on_file"` (requires `"files_deps": [...]`) +- `mode`: same semantics as file rules -The following snippet is a valid example of the `IO_Graph` section: +### Example ```json { @@ -126,133 +100,84 @@ The following snippet is a valid example of the `IO_Graph` section: "IO_Graph": [ { "name": "writer", - "output_stream": [ - "file0.dat", - "file1.dat", - "file2.dat", - "dir" - ], + "output_stream": ["file0.dat", "file1.dat", "file2.dat", "dir"], "streaming": [ { - "name": [ - "file0.dat" - ], + "name": ["file0.dat"], "committed": "on_termination", "mode": "update" }, { - "name": [ - "file1.dat" - ], + "name": ["file1.dat"], "committed": "on_close", "mode": "update" }, { - "name": [ - "file2.dat" - ], + "name": ["file2.dat"], "committed": "on_close:10", "mode": "no_update" }, { - "dirname": [ - "dir" - ], - "committed": "n_files:1000", + "dirname": ["dir"], + "committed": "on_n_files", + "n_files": 1000, "mode": "no_update" } ] }, { "name": "reader", - "input_stream": [ - "file0.dat", - "file1.dat", - "file2.dat", - "dir" - ] + "input_stream": ["file0.dat", "file1.dat", "file2.dat", "dir"] } ] } ``` -### Exclude +## 5. Exclude Section -This section, identified by the keyword `exclude`, is used to specify those files that will not be handled by CAPIO-CL -even if they will be created inside the CAPIO-CL_DIR directory. Its value is an array of file names (whose values can -also be aliases). +The `exclude` section identifies files that CAPIO-CL must ignore even if they appear inside `CAPIO-CL_DIR`. +Values may include filenames, wildcard patterns, or alias names. -#### Example - -The following snippet is a valid section for the `exclude` section: +### Example ```json { "name": "my_workflow", - "exclude": [ - "file1.dat", - "group0", - "*.tmp", - "*~" - ] + "exclude": ["file1.dat", "group0", "*.tmp", "*~"] } ``` -### Permanent +## 6. Permanent Section -This language section, identified by the keyword `permanent`, is used to specify those files that will be stored on the -filesystem at the end of the workflow execution. Its value is an array of file names (whose values can be also aliases). +The `permanent` section specifies which files should persist on the filesystem after workflow completion. -#### Example - -The following snippet is a valid example of how the `permanent` section should be specified: +### Example ```json { "name": "my_workflow", - "permanent": [ - "output.dat", - "group0" - ] + "permanent": ["output.dat", "group0"] } ``` -At the end of the execution of “my_workflow”, the file output.dat and all files belonging to group0 will be stored in -the file system. +At the end of the workflow, CAPIO-CL will store `output.dat` and all files belonging to alias `group0`. -### Home Node Policy +## 7. Home Node Policy -This section allows the CAPIO-CL user to selectively choose the CAPIO-CL server node where all files (or a subset of -them) and their associated metadata should be stored. The user can define different policies for different files. In the -current version of the CAPIO-CL language, the home node policy options are: +The `home_node_policy` section defines where files and metadata are stored across CAPIO-CL servers. +The following optional strategies may be defined: -- `create` (default option) +- `create` (default) - `manual` - `hashing` -These three keywords are optional, i.e., the user can avoid setting them explicitly in the CAPIO-CL configuration file. -In this latter case, the default policy adopted by the CAPIO-CL middleware for the file-to-node mapping is `create`. - - -> **There cannot be filename overlap among the policies specified (i.e., the intersection of the set defined by the -different home-node policies must be empty).** - - -For the `hashing` and the `create` policies, the value is an array of files. +Important constraint: no file may appear in more than one policy group. -For the `manual` configuration, the syntax is more verbose and requires defining, for each file or set of files, the -logical identifier of the application process whose associated CAPIO-CL server (i.e., the server process running on the -same node where the application process is running) will store the file. With this information, each CAPIO-CL server may -statically know the file-to-node mapping and thus retrieve the node where this process is executing at runtime. +### 7.1 Wildcard Ambiguities -#### Wildcards ambiguities +Wildcards introduce risk of conflicting rules. -The CAPIO-CL language syntax leverages wildcards to provide the user with flexibility and reduce the burden of -enumerating all files and directories. - -Using wildcards in the language syntax can introduce unexpected behavior due to unintended matches or undefined behavior -due to multiple matches associated with different semantics rules. To clarify the point, let us consider the following -example: +Example: ```json { @@ -260,83 +185,38 @@ example: "IO_Graph": [ { "name": "writer", - "output_stream": [ - "file1.txt", - "file2.txt", - "file1.dat", - "file2.dat" - ], + "output_stream": ["file1.txt", "file2.txt", "file1.dat", "file2.dat"], "streaming": [ - { - "name": [ - "file*" - ], - "committed": "on_close" - }, - { - "name": [ - "*.dat" - ], - "committed": "on_termination" - } + { "name": ["file*"], "committed": "on_close" }, + { "name": ["*.dat"], "committed": "on_termination" } ] } ] } ``` -In the example, there is an overlapping match for the files `file1.dat` and `file2.dat`. For those files, it is -ambiguous if the commit semantics should be `on_close` or `on_termination`. +Files `file1.dat` and `file2.dat` match both rules, which leads to undefined behavior in the current version of CAPIO-CL. -Even if, in most cases, the ambiguity can be eliminated considering the most specific match for the rules that must be -used (for example, `*.dat` is more specific than `file*` if we consider the context, i.e., `output_stream` list of files -specified by the user), in the current version of CAPIO-CL, all ambiguities are not solved, and an undefined behavior -exception is raised by the CAPIO-CL implementation. +### 7.2 Avoiding Ambiguity with Aliases -It is worth mentioning that, other than carefully using wildcards, proper use of the `aliases` section can help the user -write a non-ambiguous and clean configuration file. An example is shown below: +Proper use of aliases can help write a non-ambiguous and clean configuration file. ```json { "name": "my_workflow", "aliases": [ - { - "group_name": "group-dat", - "files": [ - "file1.dat", - "file2.dat" - ] - }, - { - "group_name": "group-txt", - "files": [ - "file1.txt", - "file2.txt" - ] - } + { "group_name": "group-dat", "files": ["file1.dat", "file2.dat"] }, + { "group_name": "group-txt", "files": ["file1.txt", "file2.txt"] } ], "IO_Graph": [ { "name": "writer", - "output_stream": [ - "group-dat", - "group-txt" - ], + "output_stream": ["group-dat", "group-txt"], "streaming": [ - { - "name": [ - "group-txt" - ], - "committed": "on_close" - }, - { - "name": [ - "group-dat" - ], - "committed": "on_termination" - } + { "name": ["group-txt"], "committed": "on_close" }, + { "name": ["group-dat"], "committed": "on_termination" } ] } ] } -``` \ No newline at end of file +``` diff --git a/doxygen/syntax_v1_1.md b/doxygen/syntax_v1_1.md new file mode 100644 index 0000000..3dd520b --- /dev/null +++ b/doxygen/syntax_v1_1.md @@ -0,0 +1,72 @@ +# CAPIO-CL V1.1 Language Specification + +This document describes the syntax and semantics of the CAPIO-CL V1.1 +language.\ +Version 1.1 extends Version 1.0 by introducing two additional fields: +`version` and `configuration`. + +------------------------------------------------------------------------ + +## 1. Version Field + +The `version` field introduces an explicit versioning mechanism, +allowing CAPIO-CL files to clearly indicate which syntax standard they +follow.\ +This ensures compatibility and avoids ambiguity between V1.0 and V1.1 +documents. + +**Type:** float\ +**Purpose:** Identifies the CAPIO-CL language version\ +**Behavior:**\ +- If omitted, tools may assume legacy V1.0 behavior.\ +- If present (e.g., `1.1`), tools can activate V1.1 features. + +### Example (JSON) + +``` +{ + "version": 1.1, + ... +} +``` + +------------------------------------------------------------------------ + +## 2. Configuration Field + +CAPIO-CL V1.1 introduces a new optional field, `configuration`, which +allows users to define runtime options for the `capiocl::Engine` class. + +The configuration is stored in a separate **TOML** file.\ +For details on valid TOML configuration syntax and supported engine +parameters, refer to **configuration.md**. + +**Type:** string (file path)\ +**Purpose:** Supplies runtime engine settings\ +**Behavior:**\ +- File must be a valid TOML document.\ +- Engine loads configuration before execution. + +### Example (JSON + TOML) + +**capio.cl file:** + +``` +{ + "version": 1.1, + "configuration": "engine-config.toml", + ... +} +``` + +**engine-config.toml:** + +``` + [monitor.mcast] + commit.ip = "224.224.224.1" + commit.port = 12345 + delay_ms = 300 + homenode.ip = "224.224.224.2" + homenode.port = 12345 +``` +