diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index d2771b4..67c9a6c 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -54,7 +54,9 @@ jobs: - name: "Build and Install Python package" run: | mkdir -p /tmp/capio_cl_jsons - cp tests/jsons/*.json /tmp/capio_cl_jsons + mkdir -p /tmp/capio_cl_tomls + cp -r tests/jsons/* /tmp/capio_cl_jsons + cp -r tests/tomls/* /tmp/capio_cl_tomls . ./venv/bin/activate pip install . @@ -128,7 +130,9 @@ jobs: python3 -m pip install -r build-requirements.txt mkdir -p /tmp/capio_cl_jsons - cp tests/jsons/*.json /tmp/capio_cl_jsons + mkdir -p /tmp/capio_cl_tomls + cp -r tests/jsons/* /tmp/capio_cl_jsons + cp -r tests/tomls/* /tmp/capio_cl_tomls python3 -m build \ -Ccmake.define.ENABLE_COVERAGE=OFF \ @@ -137,6 +141,5 @@ jobs: pip install dist/*.whl python3 -m pip install -r test-requirements.txt - - echo "✅ Running unit tests" + pytest -v tests/python/test_* | tee pytest-riscv.log diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ebcf6e..b252831 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,12 @@ add_compile_options(-Wall -Wextra -Wpedantic) # External projects ##################################### +FetchContent_Declare( + tomlplusplus + GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git + GIT_TAG v3.4.0 +) + FetchContent_Declare( jsoncons GIT_REPOSITORY https://github.com/danielaparker/jsoncons.git @@ -55,7 +61,7 @@ set(JSONCONS_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(JSONCONS_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(JSONCONS_BUILD_FUZZERS OFF CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(jsoncons) +FetchContent_MakeAvailable(jsoncons tomlplusplus) if (BUILD_PYTHON_BINDINGS) FetchContent_Declare( @@ -82,22 +88,33 @@ file(WRITE ${OUTPUT_HEADER} "// Bundled CAPIO-CL encoded JSON schemas\n") file(APPEND ${OUTPUT_HEADER} "#pragma once\n\n") foreach (SCHEMA_FILE ${SCHEMA_FILES}) - get_filename_component(SCHEMA_NAME ${SCHEMA_FILE} NAME_WE) get_filename_component(SCHEMA_BASENAME ${SCHEMA_FILE} NAME) message(STATUS "Bundling CAPIO-CL schema: ${SCHEMA_BASENAME}") execute_process( - COMMAND bash -c "echo 'constexpr char schema_${SCHEMA_NAME}[] = R\"(' && cat ${SCHEMA_BASENAME} && echo ')\";'" + COMMAND bash -c "echo ${SCHEMA_BASENAME} | sed 's/\\.json$//'" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/schema + OUTPUT_VARIABLE SCHEMA_NAME_RAW + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + string(REPLACE "." "_" SCHEMA_NAME "${SCHEMA_NAME_RAW}") + + execute_process( + COMMAND bash -c + "echo \"// CAPIO-CL version: ${SCHEMA_NAME_RAW}\" && \ + echo \"constexpr char schema_${SCHEMA_NAME}[] = R\\\"(\" && \ + cat \"${SCHEMA_BASENAME}\" && \ + echo \")\\\";\"" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/schema OUTPUT_VARIABLE HEXDATA RESULT_VARIABLE RES ) + if (NOT RES EQUAL 0) - message(FATAL_ERROR "Failed bundling for ${SCHEMA_FILE}: Error code is ${RES}") + message(FATAL_ERROR "Failed to bundle ${SCHEMA_BASENAME}: exit ${RES}") endif () - - file(APPEND ${OUTPUT_HEADER} "// CAPIO-CL version: ${SCHEMA_NAME}\n") file(APPEND ${OUTPUT_HEADER} "${HEXDATA}\n\n") endforeach () @@ -117,10 +134,11 @@ target_include_directories(libcapio_cl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src ${jsoncons_SOURCE_DIR}/include ${CAPIOCL_JSON_SCHEMAS_DIRECTORY} + ${TOMLPLUSPLUS_SOURCE_DIR}/include ) -# jsoncons is header-only, no linking required target_link_libraries(libcapio_cl PUBLIC) +target_link_libraries(libcapio_cl PRIVATE tomlplusplus::tomlplusplus) ##################################### # Install rules @@ -189,8 +207,24 @@ if (CAPIO_CL_BUILD_TESTS) add_custom_command( TARGET CAPIO_CL_tests PRE_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory "/tmp/capio_cl_jsons" - COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_JSON_DIR} "/tmp/capio_cl_jsons" - COMMENT "Copying JSON test files to build directory" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${TEST_JSON_DIR}" + "/tmp/capio_cl_jsons" + COMMENT "Copying JSON test files with full directory structure" + ) + + ##################################### + # Copy TOMLS test files + ##################################### + set(TEST_TOMLS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/tomls") + + add_custom_command( + TARGET CAPIO_CL_tests PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "/tmp/capio_cl_tomls" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${TEST_TOMLS_DIR}" + "/tmp/capio_cl_tomls" + COMMENT "Copying TOMLS test files with full directory structure" ) ##################################### @@ -200,13 +234,8 @@ if (CAPIO_CL_BUILD_TESTS) RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) - install(DIRECTORY ${TEST_JSON_DIR} - DESTINATION ${CMAKE_INSTALL_BINDIR}/jsons - FILES_MATCHING PATTERN "*.json" - ) - ##################################### - # Targget to execute tests + # Target to execute tests ##################################### add_custom_target(run_tests DEPENDS CAPIO_CL_tests diff --git a/capiocl.hpp b/capiocl.hpp index c780746..aeac085 100644 --- a/capiocl.hpp +++ b/capiocl.hpp @@ -1,17 +1,8 @@ #ifndef CAPIO_CL_CAPIOCL_HPP #define CAPIO_CL_CAPIOCL_HPP -#include -#include -#include -#include #include -#include #include -#include -#include -#include -#include /// @brief Namespace containing all the CAPIO-CL related code namespace capiocl { @@ -76,7 +67,9 @@ inline std::string sanitize(const std::string &input) { /// @brief Available versions of CAPIO-CL language struct CAPIO_CL_VERSION final { /// @brief Release 1.0 of CAPIO-CL - static constexpr char V1[] = "1.0"; + static constexpr char V1[] = "1.0"; + /// @brief Release 1.1 of CAPIO-CL + static constexpr char V1_1[] = "1.1"; }; namespace serializer { @@ -92,6 +85,12 @@ class MonitorException; namespace engine { class Engine; } + +namespace configuration { +class CapioClConfiguration; +class CapioClConfigurationException; +struct defaults; +} // namespace configuration } // namespace capiocl #endif // CAPIO_CL_CAPIOCL_HPP \ No newline at end of file diff --git a/doxygen/Doxyfile b/doxygen/Doxyfile index 4b1dd75..7187cff 100644 --- a/doxygen/Doxyfile +++ b/doxygen/Doxyfile @@ -943,12 +943,14 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = .README.md \ - ../src \ - ../include \ - ../capiocl.hpp \ - semantics.md \ - syntax_v1.md +INPUT = .README.md \ + ../src \ + ../include \ + ../capiocl.hpp \ + semantics.md \ + syntax_v1.md \ + syntax_v1_1.md \ + configuration.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -2425,7 +2427,7 @@ INCLUDE_PATH = # used. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -INCLUDE_FILE_PATTERNS = +INCLUDE_FILE_PATTERNS = *.md # The PREDEFINED tag can be used to specify one or more macro names that are # defined before the preprocessor is started (similar to the -D option of e.g. diff --git a/doxygen/configuration.md b/doxygen/configuration.md new file mode 100644 index 0000000..96d49a2 --- /dev/null +++ b/doxygen/configuration.md @@ -0,0 +1,69 @@ +# Runtime Configuration for **CAPIO-CL** + +CAPIO-CL supports **runtime configuration** through a user-provided **TOML** configuration file. +This allows you to customize various aspects of the library without recompiling it. + +A configuration file can be loaded into a running CAPIO-CL instance by calling: + + capiocl::Engine engine; + engine.load("path/to/config.toml"); + +If no configuration file is provided, CAPIO-CL falls back to its built-in defaults. + +--- + +## TOML Configuration Structure + +The configuration file uses a structured namespace under the top-level table `monitor.mcast`. + +Available configuration parameters: + +| Key | Type | Default | Description | +|----------------------------------|----------|----------------|----------------------------------------------------------------------------------------------| +| `monitor.mcast.commit.ip` | string | `224.224.224.1` | Multicast IP address used for commit messages | +| `monitor.mcast.commit.port` | integer | `12345` | UDP port for commit messages | +| `monitor.mcast.delay_ms` | integer | `300` | Artificial delay (in milliseconds) inserted before sending multicast messages. Useful for debugging or simulating slower networks. | +| `monitor.mcast.homenode.ip` | string | `224.224.224.2` | IP address of the home node for monitoring operations | +| `monitor.mcast.homenode.port` | integer | `12345` | Port associated with the home node monitoring endpoint | + +--- + +## Example configuration file + +Below is a complete example of a `config.toml` file: + + # Example CAPIO-CL TOML configuration + + [monitor.mcast] + + # Multicast settings for commit messages + commit.ip = "224.224.224.1" + commit.port = 12345 + + # Delay (in milliseconds) + delay_ms = 300 + + # Home node information + homenode.ip = "224.224.224.2" + homenode.port = 12345 + +--- + +## How CAPIO-CL Uses These Settings + +### `commit.ip` and `commit.port` +These fields define where CAPIO-CL sends **commit broadcast messages**. +Commit messages are used for consistency coordination across distributed nodes. + +### `delay_ms` +A small configurable delay may help with: + +- debugging message-ordering issues, +- testing high-latency environments, +- reproducing specific timing-related behavior. + +A value of `0` means no delay. + +### `homenode.ip` and `homenode.port` +These define the **central monitoring endpoint** (the “home node”). +CAPIO-CL uses this endpoint to coordinate monitoring metadata and cluster-wide communication. diff --git a/doxygen/syntax_v1.md b/doxygen/syntax_v1.md index 5d002f5..b831480 100644 --- a/doxygen/syntax_v1.md +++ b/doxygen/syntax_v1.md @@ -1,43 +1,44 @@ -# Syntax of the CAPIO-CL V1 language +# CAPIO-CL V1 Language Specification -In this section we will illustrate how the semantics of the CAPIO-CL configuration language can be expressed in a json -configuration file. +This document describes the syntax and semantics of the **CAPIO-CL V1 configuration language**, expressed through a JSON-based configuration file. +A valid CAPIO-CL file defines workflow structure, file dependencies, streaming semantics, storage behavior, and home-node mapping policies for distributed execution. -## JSON syntax +## 1. JSON Syntax Overview -A valid CAPIO-CL configuration file comprises five different sections. Sections marked with `*` are mandatory. +A CAPIO-CL V1 configuration file is a single JSON object composed of the following sections. +Sections marked with `*` are mandatory. -- `Workflow name(*)`: identifies the application workflow composed by multiple application modules; -- `Alias`: groups under a convenient name a set of files or directories; -- `IO_Graph(*)`: describes file data dependencies among application modules; -- `Permanent`: defines which files must be kept on the permanent storage at the end of the workflow execution; -- `Exclude`: identifies the files and directories not handled by the CAPIO-CL implementation; -- `Home Node Policies`: defines different file mapping policies to establish which CAPIO-CL servers store which files; +Section | Required | Purpose +------- | -------- | ------- +name* | Yes | Identifies the workflow +aliases | No | Provides named groups of files/directories +IO_Graph* | Yes | Describes data dependencies among modules +permanent | No | Files to keep at the end of execution +exclude | No | Files CAPIO-CL should ignore +home_node_policy | No | Rules assigning files to CAPIO-CL servers -### Workflow name +## 2. Workflow Name -This section is identified by the keyword `name`. The name is used as an identifier for the current application -workflow. This is useful as it is possible to distinguish different application workflows running on the same machine. +The workflow name is provided under the key `name`. +This unique identifier allows distinguishing multiple workflows running on the same system. -#### Example +### Example ```json { "name": "my_workflow" } -``` - -### Alias +``` -The aliases section, identified by the keyword aliases is useful to reduce the verbosity of the enumeration of files an -application can consume or produce. It is a vector of objects composed of the following items: +## 3. Aliases -- `group_name`: the alias identifier -- `files`: an array of strings representing file names. +Aliases reduce verbosity by allowing groups of files or directories to be referenced by a single identifier. +The `aliases` section is an array of objects, where each object contains: -#### Example +- `group_name`: alias identifier +- `files`: an array of filename strings -The following snippet is a valid alias section for CAPIO-CL configuration language +### Example ```json { @@ -45,80 +46,53 @@ The following snippet is a valid alias section for CAPIO-CL configuration langua "aliases": [ { "group_name": "group-even", - "files": [ - "file0.dat", - "file2.dat", - "file4.dat" - ] + "files": ["file0.dat", "file2.dat", "file4.dat"] }, { "group_name": "group-odd", - "files": [ - "file1.dat", - "file3.dat", - "file5.dat" - ] + "files": ["file1.dat", "file3.dat", "file5.dat"] } ] } -``` - -### IO Graph - -This section defines the dependencies between input and output streams between application modules comprising the -workflow. It is identified by the keyword `IO_Graph`, and it requires an array of objects. Those objects specify the -input and output streams for each application component. Each object defines the following items: - -- `name`: The name of the application. This keyword is mandatory -- `input_stream`: This keyword identifies the input files and directories the application module is expected to read. - Its value is a vector of strings. This keyword is optional -- `output_stream`: This keyword identifies a vector of files and directory names, i.e., the files and directories the - application module is expected to produce. Its value is a vector of strings. This keyword is optional. -- `streaming`: This optional keyword identifies the files and directories with their streaming semantics, i.e., the - “commit and firing rules” (please see [Semantics](semantics.md) for a more detailed description). Its value is an - array of objects. +``` -#### Streaming section +## 4. IO_Graph -Each object inside the streaming section may define the following attributes. -For **file** entries (either files or globs): +The `IO_Graph` describes how application modules exchange data. +It is an array of objects, each representing one module. -- `name`: The filenames to which the rule applies. The value of this keyword is an array of filenames. -- `committed`: This keyword defines the “commit rule” associated with the files identified with the keywords name. Its - value can be either: - - `on_close:N`: to express the CoC semantic, where N is an integer greater than 1, or can be omitted if N=1; - - `on_termination`: to express the CoT semantic (default); - - `on_file:filename` to express the CoF semantic. `filename` is the file on which the commit rule will commit - against. -- `mode`: his keyword defines the “firing rule” associated with the files and directories identified with the keys name - and dirname, respectively. Its value can be either: - - `update`: for the FoC semantic (default behavior); - - `no_update`: for the FnU semantic; +Key | Required | Description +--- | -------- | ----------- +name | Yes | Module name +input_stream | No | Files/directories the module reads +output_stream | No | Files/directories the module writes +streaming | No | Commit and firing rules for produced data -For **directories** entries (either files or globs), that is that the rule will apply to files and subdirectories -inside the specified directory entry: +### 4.1 Streaming Rules -- `dirname`: The directory names to which the rule applies. The value of this keyword is an array of directory names; +Each entry in `streaming` may describe rules for **files** or **directories**. -- `committed`: This keyword defines the “commit rule” associated with the directories identified with the keywords - dirname. Its value can be either: - - `on_n_files`: to express the Commit on N Files commit rule. If this commit rule is chosen, then the keyword - `n_files` - must be set to `N` (integer greater than 1). The directory will be considered committed as soon as `N` files are - committed inside it.; - - `on_termination`: to express the CoT semantic (default); - - `on_file`: to express the CoF semantic. In this case, the keyword `files_deps`, whose value is an array of - filenames or directory names, defines the set of dependencies. +#### For files (`name`): -- `mode`: his keyword defines the “firing rule” associated with the files and directories identified with the keys name - and dirname, respectively. Its value can be either: +- `name`: array of filenames +- `committed`: commit rule + - `"on_close"` or `"on_close:N"` + - `"on_termination"` + - `"on_file:filename"` +- `mode`: firing rule + - `"update"` (default) + - `"no_update"` - - `update`: for the FoC semantic (default behavior); - - `no_update`: for the FnU semantic; +#### For directories (`dirname`): -#### Example +- `dirname`: array of directory names +- `committed`: directory commit rule + - `"on_n_files"` (requires field `"n_files": N`) + - `"on_termination"` + - `"on_file"` (requires `"files_deps": [...]`) +- `mode`: same semantics as file rules -The following snippet is a valid example of the `IO_Graph` section: +### Example ```json { @@ -126,133 +100,84 @@ The following snippet is a valid example of the `IO_Graph` section: "IO_Graph": [ { "name": "writer", - "output_stream": [ - "file0.dat", - "file1.dat", - "file2.dat", - "dir" - ], + "output_stream": ["file0.dat", "file1.dat", "file2.dat", "dir"], "streaming": [ { - "name": [ - "file0.dat" - ], + "name": ["file0.dat"], "committed": "on_termination", "mode": "update" }, { - "name": [ - "file1.dat" - ], + "name": ["file1.dat"], "committed": "on_close", "mode": "update" }, { - "name": [ - "file2.dat" - ], + "name": ["file2.dat"], "committed": "on_close:10", "mode": "no_update" }, { - "dirname": [ - "dir" - ], - "committed": "n_files:1000", + "dirname": ["dir"], + "committed": "on_n_files", + "n_files": 1000, "mode": "no_update" } ] }, { "name": "reader", - "input_stream": [ - "file0.dat", - "file1.dat", - "file2.dat", - "dir" - ] + "input_stream": ["file0.dat", "file1.dat", "file2.dat", "dir"] } ] } ``` -### Exclude +## 5. Exclude Section -This section, identified by the keyword `exclude`, is used to specify those files that will not be handled by CAPIO-CL -even if they will be created inside the CAPIO-CL_DIR directory. Its value is an array of file names (whose values can -also be aliases). +The `exclude` section identifies files that CAPIO-CL must ignore even if they appear inside `CAPIO-CL_DIR`. +Values may include filenames, wildcard patterns, or alias names. -#### Example - -The following snippet is a valid section for the `exclude` section: +### Example ```json { "name": "my_workflow", - "exclude": [ - "file1.dat", - "group0", - "*.tmp", - "*~" - ] + "exclude": ["file1.dat", "group0", "*.tmp", "*~"] } ``` -### Permanent +## 6. Permanent Section -This language section, identified by the keyword `permanent`, is used to specify those files that will be stored on the -filesystem at the end of the workflow execution. Its value is an array of file names (whose values can be also aliases). +The `permanent` section specifies which files should persist on the filesystem after workflow completion. -#### Example - -The following snippet is a valid example of how the `permanent` section should be specified: +### Example ```json { "name": "my_workflow", - "permanent": [ - "output.dat", - "group0" - ] + "permanent": ["output.dat", "group0"] } ``` -At the end of the execution of “my_workflow”, the file output.dat and all files belonging to group0 will be stored in -the file system. +At the end of the workflow, CAPIO-CL will store `output.dat` and all files belonging to alias `group0`. -### Home Node Policy +## 7. Home Node Policy -This section allows the CAPIO-CL user to selectively choose the CAPIO-CL server node where all files (or a subset of -them) and their associated metadata should be stored. The user can define different policies for different files. In the -current version of the CAPIO-CL language, the home node policy options are: +The `home_node_policy` section defines where files and metadata are stored across CAPIO-CL servers. +The following optional strategies may be defined: -- `create` (default option) +- `create` (default) - `manual` - `hashing` -These three keywords are optional, i.e., the user can avoid setting them explicitly in the CAPIO-CL configuration file. -In this latter case, the default policy adopted by the CAPIO-CL middleware for the file-to-node mapping is `create`. - - -> **There cannot be filename overlap among the policies specified (i.e., the intersection of the set defined by the -different home-node policies must be empty).** - - -For the `hashing` and the `create` policies, the value is an array of files. +Important constraint: no file may appear in more than one policy group. -For the `manual` configuration, the syntax is more verbose and requires defining, for each file or set of files, the -logical identifier of the application process whose associated CAPIO-CL server (i.e., the server process running on the -same node where the application process is running) will store the file. With this information, each CAPIO-CL server may -statically know the file-to-node mapping and thus retrieve the node where this process is executing at runtime. +### 7.1 Wildcard Ambiguities -#### Wildcards ambiguities +Wildcards introduce risk of conflicting rules. -The CAPIO-CL language syntax leverages wildcards to provide the user with flexibility and reduce the burden of -enumerating all files and directories. - -Using wildcards in the language syntax can introduce unexpected behavior due to unintended matches or undefined behavior -due to multiple matches associated with different semantics rules. To clarify the point, let us consider the following -example: +Example: ```json { @@ -260,83 +185,38 @@ example: "IO_Graph": [ { "name": "writer", - "output_stream": [ - "file1.txt", - "file2.txt", - "file1.dat", - "file2.dat" - ], + "output_stream": ["file1.txt", "file2.txt", "file1.dat", "file2.dat"], "streaming": [ - { - "name": [ - "file*" - ], - "committed": "on_close" - }, - { - "name": [ - "*.dat" - ], - "committed": "on_termination" - } + { "name": ["file*"], "committed": "on_close" }, + { "name": ["*.dat"], "committed": "on_termination" } ] } ] } ``` -In the example, there is an overlapping match for the files `file1.dat` and `file2.dat`. For those files, it is -ambiguous if the commit semantics should be `on_close` or `on_termination`. +Files `file1.dat` and `file2.dat` match both rules, which leads to undefined behavior in the current version of CAPIO-CL. -Even if, in most cases, the ambiguity can be eliminated considering the most specific match for the rules that must be -used (for example, `*.dat` is more specific than `file*` if we consider the context, i.e., `output_stream` list of files -specified by the user), in the current version of CAPIO-CL, all ambiguities are not solved, and an undefined behavior -exception is raised by the CAPIO-CL implementation. +### 7.2 Avoiding Ambiguity with Aliases -It is worth mentioning that, other than carefully using wildcards, proper use of the `aliases` section can help the user -write a non-ambiguous and clean configuration file. An example is shown below: +Proper use of aliases can help write a non-ambiguous and clean configuration file. ```json { "name": "my_workflow", "aliases": [ - { - "group_name": "group-dat", - "files": [ - "file1.dat", - "file2.dat" - ] - }, - { - "group_name": "group-txt", - "files": [ - "file1.txt", - "file2.txt" - ] - } + { "group_name": "group-dat", "files": ["file1.dat", "file2.dat"] }, + { "group_name": "group-txt", "files": ["file1.txt", "file2.txt"] } ], "IO_Graph": [ { "name": "writer", - "output_stream": [ - "group-dat", - "group-txt" - ], + "output_stream": ["group-dat", "group-txt"], "streaming": [ - { - "name": [ - "group-txt" - ], - "committed": "on_close" - }, - { - "name": [ - "group-dat" - ], - "committed": "on_termination" - } + { "name": ["group-txt"], "committed": "on_close" }, + { "name": ["group-dat"], "committed": "on_termination" } ] } ] } -``` \ No newline at end of file +``` diff --git a/doxygen/syntax_v1_1.md b/doxygen/syntax_v1_1.md new file mode 100644 index 0000000..3dd520b --- /dev/null +++ b/doxygen/syntax_v1_1.md @@ -0,0 +1,72 @@ +# CAPIO-CL V1.1 Language Specification + +This document describes the syntax and semantics of the CAPIO-CL V1.1 +language.\ +Version 1.1 extends Version 1.0 by introducing two additional fields: +`version` and `configuration`. + +------------------------------------------------------------------------ + +## 1. Version Field + +The `version` field introduces an explicit versioning mechanism, +allowing CAPIO-CL files to clearly indicate which syntax standard they +follow.\ +This ensures compatibility and avoids ambiguity between V1.0 and V1.1 +documents. + +**Type:** float\ +**Purpose:** Identifies the CAPIO-CL language version\ +**Behavior:**\ +- If omitted, tools may assume legacy V1.0 behavior.\ +- If present (e.g., `1.1`), tools can activate V1.1 features. + +### Example (JSON) + +``` +{ + "version": 1.1, + ... +} +``` + +------------------------------------------------------------------------ + +## 2. Configuration Field + +CAPIO-CL V1.1 introduces a new optional field, `configuration`, which +allows users to define runtime options for the `capiocl::Engine` class. + +The configuration is stored in a separate **TOML** file.\ +For details on valid TOML configuration syntax and supported engine +parameters, refer to **configuration.md**. + +**Type:** string (file path)\ +**Purpose:** Supplies runtime engine settings\ +**Behavior:**\ +- File must be a valid TOML document.\ +- Engine loads configuration before execution. + +### Example (JSON + TOML) + +**capio.cl file:** + +``` +{ + "version": 1.1, + "configuration": "engine-config.toml", + ... +} +``` + +**engine-config.toml:** + +``` + [monitor.mcast] + commit.ip = "224.224.224.1" + commit.port = 12345 + delay_ms = 300 + homenode.ip = "224.224.224.2" + homenode.port = 12345 +``` + diff --git a/include/configuration.h b/include/configuration.h new file mode 100644 index 0000000..3a00cc7 --- /dev/null +++ b/include/configuration.h @@ -0,0 +1,94 @@ +#ifndef CAPIO_CL_CONFIGURATION_H +#define CAPIO_CL_CONFIGURATION_H +#include +#include + +#include "capiocl.hpp" + +/// @brief Struct containing key value pairs for capio-cl runtime configuration +typedef struct { + /// @brief Key of option + std::string k; + /// @brief Value of option + std::string v; +} ConfigurationEntry; + +/// @brief Defaults values keys for runtime options of CAPIO-CL +struct capiocl::configuration::defaults { + /// @brief Multicast monitor commit IP + static ConfigurationEntry DEFAULT_MONITOR_MCAST_IP; + /// @brief Multicast monitor commit PORT + static ConfigurationEntry DEFAULT_MONITOR_MCAST_PORT; + /// @brief Multicast monitor delay before following operation + static ConfigurationEntry DEFAULT_MONITOR_MCAST_DELAY; + /// @brief Multicast monitor homenode IP + static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_IP; + /// @brief Multicast monitor homenode PORT + static ConfigurationEntry DEFAULT_MONITOR_HOMENODE_PORT; +}; + +/// @brief Load configuration and store it from a CAPIO-CL TOML configuration file +class capiocl::configuration::CapioClConfiguration { + friend class engine::Engine; + std::unordered_map config; + + protected: + /** + * Set a key value pair explicitly + * @param key Option key name + * @param value Value + */ + void set(const std::string &key, std::string value); + + /** + * Set a capio-cl configuration option through a ConfigurationEntry object + * @param entry + */ + void set(const ConfigurationEntry &entry); + + public: + explicit CapioClConfiguration(); + ~CapioClConfiguration() = default; + + /** + * Load a configuiration from a TOML file + * @param path + */ + void load(const std::filesystem::path &path); + + /** + * Get a string value + * @param key key of option to get + * @param value reference in which value will be stored + */ + void getParameter(const std::string &key, int *value) const; + + /** + * Get a integer value + * @param key key of option to get + * @param value reference in which value will be stored + */ + void getParameter(const std::string &key, std::string *value) const; +}; + +/** + * @brief Custom exception thrown when handling a CAPIO-CL TOML configuration file + */ +class capiocl::configuration::CapioClConfigurationException final : public std::exception { + std::string message; + + public: + /** + * @brief Construct a new CAPIO-CL Exception + * @param msg Error Message that raised this exception + */ + explicit CapioClConfigurationException(const std::string &msg); + + /** + * Get the description of the error causing the exception + * @return + */ + [[nodiscard]] const char *what() const noexcept override { return message.c_str(); } +}; + +#endif \ No newline at end of file diff --git a/include/engine.h b/include/engine.h index 87fbfe8..a850095 100644 --- a/include/engine.h +++ b/include/engine.h @@ -27,6 +27,9 @@ class Engine final { friend class serializer::Serializer; bool store_all_in_memory = false; + ///@brief Configuration imported from CAPIO-CL config TOML file + configuration::CapioClConfiguration configuration; + /// @brief Monitor instance to check runtime information of CAPIO-CL files monitor::Monitor monitor; @@ -98,7 +101,7 @@ class Engine final { public: /// @brief Class constructor - explicit Engine(); + explicit Engine(bool use_default_settings = true); /// @brief Print the current CAPIO-CL configuration. void print() const; @@ -384,6 +387,17 @@ class Engine final { * @return true if both this instance and other are equivalent. false otherwise. */ bool operator==(const Engine &other) const; + + /** + * Load a CAPIO-CL TOML configuration file + * @param path + */ + void loadConfiguration(const std::string &path); + + /** + * Use default CAPIO-CL TOML configuration. + */ + void useDefaultConfiguration(); }; } // namespace capiocl::engine diff --git a/include/monitor.h b/include/monitor.h index e0b77bf..2ab6b3d 100644 --- a/include/monitor.h +++ b/include/monitor.h @@ -1,7 +1,6 @@ #ifndef CAPIO_CL_MONITOR_H #define CAPIO_CL_MONITOR_H -#include #include #include #include @@ -10,6 +9,8 @@ #include #include +#include "configuration.h" + #ifndef PATH_MAX #define PATH_MAX 4096 #endif @@ -148,9 +149,12 @@ class MulticastMonitor final : public MonitorInterface { /** * @brief Multicast port number. */ - int MULTICAST_COMMIT_PORT; + int MULTICAST_COMMIT_PORT{}; + + int MULTICAST_HOME_NODE_PORT{}; - int MULTICAST_HOME_NODE_PORT; + ///@brief Delay in milliseconds before checking again for a status change + int MULTICAST_DELAY_MILLIS{}; /** * @brief Supported network command types for commit messages. @@ -204,13 +208,9 @@ class MulticastMonitor final : public MonitorInterface { /** * @brief Construct a multicast-based monitor. * - * @param commit_ip_addr Multicast commit group address to use. - * @param commit_ip_port Multicast commit port to use. - * @param home_node_ip_addr Multicast home node group address to use. - * @param home_node_ip_port Multicast home node port to use. + *@param config const reference to CAPIO-CL configuration */ - MulticastMonitor(const std::string &commit_ip_addr, int commit_ip_port, - const std::string &home_node_ip_addr, int home_node_ip_port); + MulticastMonitor(const capiocl::configuration::CapioClConfiguration &config); /** * @brief Destructor; stops listener thread and cleans resources. diff --git a/include/parser.h b/include/parser.h index 8327d54..6ebafee 100644 --- a/include/parser.h +++ b/include/parser.h @@ -42,6 +42,18 @@ class Parser final { static engine::Engine *parse_v1(const std::filesystem::path &source, const std::filesystem::path &resolve_prefix, bool store_only_in_memory); + + /** + * Parser for the V1.1 Specification of the CAPIO-CL language + * @param source Path of CAPIO-CL configuration file + * @param resolve_prefix Prefix to prepend to path if found to be relative + * @param store_only_in_memory Flag to set to returned instance of Engine if required to + * store all files in memory + * @return Parsed Engine. + */ + static engine::Engine *parse_v1_1(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, + bool store_only_in_memory); }; /** @@ -65,8 +77,9 @@ class Parser final { /** * Validate a CAPIO-CL configuration file according to the JSON schema of the language * @param doc The loaded CAPIO-CL configuration file + * @param str_schema Raw JSON schema to use */ - static void validate_json(const jsoncons::json &doc); + static void validate_json(const jsoncons::json &doc, const char *str_schema); /** * @brief Perform the parsing of the capio_server configuration file diff --git a/include/printer.h b/include/printer.h index 22a9bed..1069bf6 100644 --- a/include/printer.h +++ b/include/printer.h @@ -1,6 +1,6 @@ #ifndef CAPIO_CL_PRINTER_H #define CAPIO_CL_PRINTER_H -#include +#include #include #ifndef HOST_NAME_MAX diff --git a/include/serializer.h b/include/serializer.h index fad0c53..1c20ef7 100644 --- a/include/serializer.h +++ b/include/serializer.h @@ -1,7 +1,7 @@ #ifndef CAPIO_CL_SERIALIZER_H #define CAPIO_CL_SERIALIZER_H -#include +#include "capiocl.hpp" /// @brief Namespace containing the CAPIO-CL Serializer component namespace capiocl::serializer { @@ -41,6 +41,17 @@ class Serializer final { */ static void serialize_v1(const engine::Engine &engine, const std::filesystem::path &filename); + + /** + * @brief Dump the current configuration loaded into an instance of Engine to a CAPIO-CL + * VERSION 1.1 configuration file. + * + * @param engine instance of Engine to dump + * @param filename path of output file + * @throws SerializerException + */ + static void serialize_v1_1(const engine::Engine &engine, + const std::filesystem::path &filename); }; public: diff --git a/schema/v1.1.json b/schema/v1.1.json new file mode 100644 index 0000000..2daa8f3 --- /dev/null +++ b/schema/v1.1.json @@ -0,0 +1,232 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CAPIO-CL Configuration Schema", + "type": "object", + + "definitions": { + "streaming_name_rule": { + "type": "object", + "properties": { + "name": { + "type": "array", + "description": "Filenames to which the rule applies.", + "items": { "type": "string" } + }, + "committed": { + "type": "string", + "description": "Commit rule associated with the file.", + "pattern": "^(on_close(:[0-9]+)?|on_file|on_termination)$" + }, + "mode": { + "type": "string", + "description": "Firing rule associated with the files.", + "enum": ["update", "no_update"] + }, + "file_deps": { + "type": "array", + "description": "List of dependent files required when committed = on_file.", + "items": { "type": "string" } + }, + "n_files": { + "type": "integer", + "description": "Number of files expected when commit rule == n_files." + } + }, + "required": ["name"], + "additionalProperties": false, + + "if": { + "properties": { + "committed": { "pattern": "^on_file" } + }, + "required": ["committed"] + }, + "then": { + "required": ["file_deps"] + } + }, + + "streaming_dirname_rule": { + "type": "object", + "properties": { + "dirname": { + "type": "array", + "description": "Directory names to which the rule applies.", + "items": { "type": "string" } + }, + "committed": { + "type": "string", + "description": "Commit rule associated with the directory.", + "pattern": "^(on_close(:[0-9]+)?|on_file|on_n_files|on_termination)$" + }, + "mode": { + "type": "string", + "description": "Firing rule associated with the directory contents.", + "enum": ["update", "no_update"] + }, + "file_deps": { + "type": "array", + "description": "List of dependent files required when committed = on_file.", + "items": { "type": "string" } + }, + "n_files": { + "type": "integer", + "description": "Number of files expected when commit rule == n_files." + } + }, + "required": ["dirname"], + "additionalProperties": false, + + "allOf": [ + { + "if": { + "properties": { + "committed": { "pattern": "^on_file" } + }, + "required": ["committed"] + }, + "then": { + "required": ["file_deps"] + } + }, + { + "if": { + "properties": { + "committed": { "pattern": "^on_n_files" } + }, + "required": ["committed"] + }, + "then": { + "required": ["n_files"] + } + } + ] + } + }, + "properties": { + + "version": { + "type": "number", + "description": "Version of the CAPIO-CL configuration file", + "default": 1.0 + }, + + "configuration": { + "type": "string", + "description": "TOML configuration file for runtime CAPIO-CL parameters" + }, + + "name": { + "type": "string", + "description": "Identifies the application workflow." + }, + + "aliases": { + "type": "array", + "description": "Groups files or directories under a convenient name.", + "items": { + "type": "object", + "properties": { + "group_name": { "type": "string" }, + "files": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["group_name", "files"], + "additionalProperties": false + } + }, + + "IO_Graph": { + "type": "array", + "description": "Describes file data dependencies among application modules.", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + + "input_stream": { + "type": "array", + "items": { "type": "string" } + }, + + "output_stream": { + "type": "array", + "items": { "type": "string" } + }, + + "streaming": { + "type": "array", + "items": { + "oneOf": [ + { "$ref": "#/definitions/streaming_name_rule" }, + { "$ref": "#/definitions/streaming_dirname_rule" } + ] + } + } + }, + "required": ["name", "input_stream", "output_stream"], + "additionalProperties": false + } + }, + + "permanent": { + "type": "array", + "items": { "type": "string" } + }, + + "exclude": { + "type": "array", + "items": { "type": "string" } + }, + + "home_node_policies": { + "type": "object", + "properties": { + "create": { + "type": "array", + "items": { "type": "string" } + }, + "hashing": { + "type": "array", + "items": { "type": "string" } + }, + "manual": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "array", + "items": { "type": "string" } + }, + "app_node": { "type": "string" } + }, + "required": ["name", "app_node"], + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + + "storage": { + "type": "object", + "properties": { + "memory": { + "type": "array", + "items": { "type": "string" } + }, + "fs": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + } + }, + + "required": ["name", "IO_Graph", "version"], + "additionalProperties": false +} diff --git a/src/Engine.cpp b/src/Engine.cpp index 6055c40..5509ae2 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -3,6 +3,7 @@ #include #include "capiocl.hpp" +#include "include/configuration.h" #include "include/engine.h" #include "include/printer.h" @@ -125,7 +126,7 @@ void capiocl::engine::Engine::print() const { printer::print(printer::CLI_LEVEL_JSON, ""); } -capiocl::engine::Engine::Engine() { +capiocl::engine::Engine::Engine(const bool use_default_settings) { node_name = std::string(1024, '\0'); gethostname(node_name.data(), node_name.size()); node_name.resize(std::strlen(node_name.c_str())); @@ -136,10 +137,9 @@ capiocl::engine::Engine::Engine() { this->workflow_name = CAPIO_CL_DEFAULT_WF_NAME; } - // TODO: make selection of monitor available to user - monitor.registerMonitorBackend( - new monitor::MulticastMonitor("224.224.224.1", 12345, "224.224.224.2", 12345)); - monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); + if (use_default_settings) { + this->useDefaultConfiguration(); + } } void capiocl::engine::Engine::_newFile(const std::filesystem::path &path) const { @@ -201,12 +201,9 @@ void capiocl::engine::Engine::compute_directory_entry_count( } bool capiocl::engine::Engine::contains(const std::filesystem::path &file) const { - for (auto &[fst, snd] : _capio_cl_entries) { - if (fnmatch(fst.c_str(), file.c_str(), FNM_PATHNAME) == 0) { - return true; - } - } - return false; + return std::any_of(_capio_cl_entries.begin(), _capio_cl_entries.end(), [&](auto const &entry) { + return fnmatch(entry.first.c_str(), file.c_str(), FNM_PATHNAME) == 0; + }); } size_t capiocl::engine::Engine::size() const { return this->_capio_cl_entries.size(); } @@ -752,4 +749,17 @@ bool capiocl::engine::Engine::operator==(const capiocl::engine::Engine &other) c } } return true; +} +void capiocl::engine::Engine::loadConfiguration(const std::string &path) { + configuration.load(path); + + monitor.registerMonitorBackend(new monitor::MulticastMonitor(configuration)); + monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); +} +void capiocl::engine::Engine::useDefaultConfiguration() { + + const auto def_config = configuration::CapioClConfiguration(); + + monitor.registerMonitorBackend(new monitor::MulticastMonitor(def_config)); + monitor.registerMonitorBackend(new monitor::FileSystemMonitor()); } \ No newline at end of file diff --git a/src/Parser.cpp b/src/Parser.cpp index 2b80076..61951ea 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -1,7 +1,6 @@ #include #include -#include "capio_cl_json_schemas.hpp" #include "capiocl.hpp" #include "include/engine.h" #include "include/parser.h" @@ -11,6 +10,11 @@ capiocl::parser::ParserException::ParserException(const std::string &msg) : mess printer::print(printer::CLI_LEVEL_ERROR, msg); } +jsoncons::jsonschema::json_schema +capiocl::parser::Parser::loadSchema(const char *data) { + return jsoncons::jsonschema::make_json_schema(jsoncons::json::parse(data)); +} + std::filesystem::path capiocl::parser::Parser::resolve(std::filesystem::path path, const std::filesystem::path &prefix) { if (prefix.empty()) { @@ -28,13 +32,8 @@ std::filesystem::path capiocl::parser::Parser::resolve(std::filesystem::path pat return resolved; } -jsoncons::jsonschema::json_schema -capiocl::parser::Parser::loadSchema(const char *data) { - return jsoncons::jsonschema::make_json_schema(jsoncons::json::parse(data)); -} - -void capiocl::parser::Parser::validate_json(const jsoncons::json &doc) { - jsoncons::jsonschema::json_schema schema = loadSchema(schema_v1); +void capiocl::parser::Parser::validate_json(const jsoncons::json &doc, const char *str_schema) { + jsoncons::jsonschema::json_schema schema = loadSchema(str_schema); try { // throws jsoncons::jsonschema::validation_error on failure [[maybe_unused]] auto status = schema.validate(doc); @@ -72,6 +71,8 @@ capiocl::engine::Engine *capiocl::parser::Parser::parse(const std::filesystem::p if (capio_cl_release == CAPIO_CL_VERSION::V1) { return available_parsers::parse_v1(source, resolve_prefix, store_only_in_memory); + } else if (capio_cl_release == CAPIO_CL_VERSION::V1_1) { + return available_parsers::parse_v1_1(source, resolve_prefix, store_only_in_memory); } else { throw ParserException("Invalid CAPIO-CL specification version!"); } diff --git a/src/Serializer.cpp b/src/Serializer.cpp index fe65d7c..21de09b 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -13,6 +13,9 @@ void capiocl::serializer::Serializer::dump(const engine::Engine &engine, if (version == CAPIO_CL_VERSION::V1) { printer::print(printer::CLI_LEVEL_INFO, "Serializing engine with V1 specification"); available_serializers::serialize_v1(engine, filename); + } else if (version == CAPIO_CL_VERSION::V1_1) { + printer::print(printer::CLI_LEVEL_INFO, "Serializing engine with V1.1 specification"); + available_serializers::serialize_v1_1(engine, filename); } else { const auto message = "No serializer available for CAPIO-CL version: " + version; throw SerializerException(message); diff --git a/src/configuration.cpp b/src/configuration.cpp new file mode 100644 index 0000000..4859977 --- /dev/null +++ b/src/configuration.cpp @@ -0,0 +1,82 @@ +#include +#include + +#include "include/configuration.h" +#include "include/printer.h" +#include "toml++/toml.hpp" + +void flatten_table(const toml::table &tbl, std::unordered_map &map, + const std::string &prefix = "") { + for (const auto &[key, value] : tbl) { + std::string full_key; + if (prefix.empty()) { + full_key = std::string{key.str()}; + } else { + full_key = prefix + "." + std::string{key.str()}; + } + + if (value.is_table()) { + flatten_table(*value.as_table(), map, full_key); + } else { + if (value.is_string()) { + map[full_key] = value.as_string()->get(); + } else { + map[full_key] = std::to_string(value.as_integer()->get()); + } + } + } +} + +capiocl::configuration::CapioClConfiguration::CapioClConfiguration() { + this->set(defaults::DEFAULT_MONITOR_MCAST_IP); + this->set(defaults::DEFAULT_MONITOR_MCAST_PORT); + this->set(defaults::DEFAULT_MONITOR_HOMENODE_IP); + this->set(defaults::DEFAULT_MONITOR_HOMENODE_PORT); + this->set(defaults::DEFAULT_MONITOR_MCAST_DELAY); +} + +void capiocl::configuration::CapioClConfiguration::set(const std::string &key, std::string value) { + config[key] = std::move(value); +} + +void capiocl::configuration::CapioClConfiguration::set(const ConfigurationEntry &entry) { + this->set(entry.k, entry.v); +} + +void capiocl::configuration::CapioClConfiguration::load(const std::filesystem::path &path) { + if (path.empty()) { + throw CapioClConfigurationException("Empty pathname!"); + } + + toml::table tbl; + try { + tbl = toml::parse_file(path.string()); + } catch (const toml::parse_error &err) { + throw CapioClConfigurationException(err.what()); + } + flatten_table(tbl, config); +} + +void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, + int *value) const { + + if (config.find(key) != config.end()) { + *value = std::stoi(config.at(key)); + } else { + throw CapioClConfigurationException("Key " + key + " not found!"); + } +} + +void capiocl::configuration::CapioClConfiguration::getParameter(const std::string &key, + std::string *value) const { + if (config.find(key) != config.end()) { + *value = config.at(key); + } else { + throw CapioClConfigurationException("Key " + key + " not found!"); + } +} +capiocl::configuration::CapioClConfigurationException::CapioClConfigurationException( + const std::string &msg) + : message(msg) { + printer::print(printer::CLI_LEVEL_ERROR, msg); +} \ No newline at end of file diff --git a/src/defaults.cpp b/src/defaults.cpp new file mode 100644 index 0000000..3e833ce --- /dev/null +++ b/src/defaults.cpp @@ -0,0 +1,16 @@ +#include "include/configuration.h" + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_IP{ + "monitor.mcast.commit.ip", "224.224.224.1"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_PORT{ + "monitor.mcast.commit.port", "12345"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_MCAST_DELAY{ + "monitor.mcast.delay_ms", "300"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_HOMENODE_IP{ + "monitor.mcast.homenode.ip", "224.224.224.2"}; + +ConfigurationEntry capiocl::configuration::defaults::DEFAULT_MONITOR_HOMENODE_PORT{ + "monitor.mcast.homenode.port", "12345"}; diff --git a/src/monitors/FileSystem.cpp b/src/monitors/FileSystem.cpp index 4924e86..4358bf8 100644 --- a/src/monitors/FileSystem.cpp +++ b/src/monitors/FileSystem.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include "capiocl.hpp" #include "include/monitor.h" diff --git a/src/monitors/Multicast.cpp b/src/monitors/Multicast.cpp index acb0a21..e633fa0 100644 --- a/src/monitors/Multicast.cpp +++ b/src/monitors/Multicast.cpp @@ -183,14 +183,13 @@ void capiocl::monitor::MulticastMonitor::_send_message(const std::string &ip_add close(out_s); } -capiocl::monitor::MulticastMonitor::MulticastMonitor(const std::string &commit_ip_addr, - const int commit_ip_port, - const std::string &home_node_ip_addr, - int home_node_ip_port) { - MULTICAST_COMMIT_ADDR = commit_ip_addr; - MULTICAST_COMMIT_PORT = commit_ip_port; - MULTICAST_HOME_NODE_ADDR = home_node_ip_addr; - MULTICAST_HOME_NODE_PORT = home_node_ip_port; +capiocl::monitor::MulticastMonitor::MulticastMonitor( + const configuration::CapioClConfiguration &config) { + config.getParameter("monitor.mcast.commit.ip", &MULTICAST_COMMIT_ADDR); + config.getParameter("monitor.mcast.commit.port", &MULTICAST_COMMIT_PORT); + config.getParameter("monitor.mcast.homenode.ip", &MULTICAST_HOME_NODE_ADDR); + config.getParameter("monitor.mcast.homenode.port", &MULTICAST_HOME_NODE_PORT); + config.getParameter("monitor.mcast.delay_ms", &MULTICAST_DELAY_MILLIS); commit_thread = std::thread(&commit_listener, std::ref(_committed_files), std::ref(committed_lock), @@ -221,7 +220,7 @@ bool capiocl::monitor::MulticastMonitor::isCommitted(const std::filesystem::path } _send_message(MULTICAST_COMMIT_ADDR, MULTICAST_COMMIT_PORT, path, GET); - std::this_thread::sleep_for(std::chrono::milliseconds(300)); + std::this_thread::sleep_for(std::chrono::milliseconds(MULTICAST_DELAY_MILLIS)); { const std::lock_guard lg(committed_lock); @@ -258,7 +257,7 @@ capiocl::monitor::MulticastMonitor::getHomeNode(const std::filesystem::path &pat } _send_message(MULTICAST_HOME_NODE_ADDR, MULTICAST_HOME_NODE_PORT, path.string(), GET); - std::this_thread::sleep_for(std::chrono::milliseconds(300)); + std::this_thread::sleep_for(std::chrono::milliseconds(MULTICAST_DELAY_MILLIS)); const std::lock_guard lg(home_node_lock); if (const auto itm = _home_nodes.find(path); itm != _home_nodes.end()) { diff --git a/src/parsers/v1.1.cpp b/src/parsers/v1.1.cpp new file mode 100644 index 0000000..6c21a4a --- /dev/null +++ b/src/parsers/v1.1.cpp @@ -0,0 +1,194 @@ +#include + +#include "capio_cl_json_schemas.hpp" +#include "capiocl.hpp" +#include "include/engine.h" +#include "include/parser.h" +#include "include/printer.h" + +capiocl::engine::Engine * +capiocl::parser::Parser::available_parsers::parse_v1_1(const std::filesystem::path &source, + const std::filesystem::path &resolve_prefix, + bool store_only_in_memory) { + std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; + auto engine = new engine::Engine(false); + + // ---- Load JSON ---- + std::ifstream file(source); + + jsoncons::json doc = jsoncons::json::parse(file); + validate_json(doc, schema_v1_1); + + // ---- workflow name ---- + workflow_name = doc["name"].as(); + engine->setWorkflowName(workflow_name); + printer::print(printer::CLI_LEVEL_JSON, "Parsing configuration for workflow: " + workflow_name); + + // ---- CAPIO-CL TOML CONFIGURATION ---- + if (doc.contains("configuration")) { + auto toml_config_path = doc["configuration"].as(); + printer::print(printer::CLI_LEVEL_JSON, "Using configuration file : " + toml_config_path); + engine->loadConfiguration(toml_config_path); + } else { + engine->useDefaultConfiguration(); + } + + // ---- IO_Graph ---- + for (const auto &app : doc["IO_Graph"].array_range()) { + std::string app_name = app["name"].as(); + printer::print(printer::CLI_LEVEL_JSON, "Parsing config for app " + app_name); + + // ---- input_stream ---- + printer::print(printer::CLI_LEVEL_JSON, "Parsing input_stream for app " + app_name); + for (const auto &itm : app["input_stream"].array_range()) { + auto file_path = resolve(itm.as(), resolve_prefix); + engine->newFile(file_path); + engine->addConsumer(file_path, app_name); + } + + // ---- output_stream ---- + printer::print(printer::CLI_LEVEL_JSON, "Parsing output_stream for app " + app_name); + for (const auto &itm : app["output_stream"].array_range()) { + auto file_path = resolve(itm.as(), resolve_prefix); + engine->newFile(file_path); + engine->addProducer(file_path, app_name); + } + + // ---- streaming ---- + if (app.contains("streaming")) { + printer::print(printer::CLI_LEVEL_JSON, "Parsing streaming for app " + app_name); + for (const auto &stream_item : app["streaming"].array_range()) { + bool is_file = true; + std::vector streaming_names; + std::vector file_deps; + std::string commit_rule; + std::string fire_rule; + long int n_close = 0; + int64_t n_files = 0; + + if (stream_item.contains("name")) { + for (const auto &nm : stream_item["name"].array_range()) { + auto nm_resolved = resolve(nm.as(), resolve_prefix); + streaming_names.push_back(nm_resolved); + } + } else { + // At this point we have dirname, as either name or dirname is required + // This is checked by the JSON schema validation phase + is_file = false; + for (const auto &nm : stream_item["dirname"].array_range()) { + auto nm_resolved = resolve(nm.as(), resolve_prefix); + streaming_names.push_back(nm_resolved); + } + } + + // Commit rule (optional) + if (stream_item.contains("committed")) { + auto committed = stream_item["committed"].as(); + auto pos = committed.find(':'); + if (pos != std::string::npos) { + // If we reach here, we are certain that the commit rule is on_close + // as the json schema enforces the rule that the :n is allowed only for + // the on_close commit rule. + + size_t num_len; + std::string count_str = committed.substr(pos + 1); + n_close = std::stoi(count_str, &num_len); + + // clean up committed + committed = committed.substr(0, pos); + } + + commit_rule = committed; + + if (commit_rule == commitRules::ON_FILE) { + for (const auto &dep : stream_item["file_deps"].array_range()) { + auto dep_resolved = resolve(dep.as(), resolve_prefix); + file_deps.push_back(dep_resolved); + } + } + } else { + commit_rule = commitRules::ON_TERMINATION; + } + + // Firing rule (optional) + if (stream_item.contains("mode")) { + fire_rule = stream_item["mode"].as(); + } else { + fire_rule = fireRules::NO_UPDATE; + } + + // n_files (optional) + if (stream_item.contains("n_files") && !is_file) { + n_files = stream_item["n_files"].as(); + } + + for (auto &path : streaming_names) { + if (n_files != 0) { + engine->setDirectoryFileCount(path, n_files); + } + if (is_file) { + engine->setFile(path); + } else { + engine->setDirectory(path); + } + + engine->setCommitRule(path, commit_rule); + engine->setFireRule(path, fire_rule); + engine->setCommitedCloseNumber(path, n_close); + engine->setFileDeps(path, file_deps); + } + } + } + } + + // ---- permanent ---- + if (doc.contains("permanent")) { + for (const auto &item : doc["permanent"].array_range()) { + std::filesystem::path path = resolve(item.as(), resolve_prefix); + engine->newFile(path); + engine->setPermanent(path, true); + } + } + + // ---- exclude ---- + if (doc.contains("exclude")) { + for (const auto &item : doc["exclude"].array_range()) { + std::filesystem::path path = resolve(item.as(), resolve_prefix); + engine->newFile(path); + engine->setExclude(path, true); + } + } + + // ---- storage ---- + if (doc.contains("storage")) { + const auto &storage = doc["storage"]; + + if (storage.contains("memory")) { + for (const auto &f : storage["memory"].array_range()) { + std::string file_str = f.as(); + engine->setStoreFileInMemory(file_str); + } + } else { + printer::print(printer::CLI_LEVEL_INFO, "No MEM storage section found"); + } + + if (storage.contains("fs")) { + for (const auto &f : storage["fs"].array_range()) { + std::string file_str = f.as(); + engine->setStoreFileInFileSystem(file_str); + } + } else { + printer::print(printer::CLI_LEVEL_INFO, "No FS storage section found"); + } + } else { + printer::print(printer::CLI_LEVEL_INFO, "No storage section found"); + } + + // ---- Store only in memory ---- + if (store_only_in_memory) { + printer::print(printer::CLI_LEVEL_INFO, "Storing all files in memory"); + engine->setAllStoreInMemory(); + } + + return engine; +} \ No newline at end of file diff --git a/src/parsers/v1.cpp b/src/parsers/v1.cpp index d386e1a..4a892ca 100644 --- a/src/parsers/v1.cpp +++ b/src/parsers/v1.cpp @@ -1,5 +1,6 @@ #include +#include "capio_cl_json_schemas.hpp" #include "capiocl.hpp" #include "include/engine.h" #include "include/parser.h" @@ -10,13 +11,15 @@ capiocl::parser::Parser::available_parsers::parse_v1(const std::filesystem::path const std::filesystem::path &resolve_prefix, bool store_only_in_memory) { std::string workflow_name = CAPIO_CL_DEFAULT_WF_NAME; - auto engine = new engine::Engine(); + auto engine = new engine::Engine(true); + + engine->useDefaultConfiguration(); // ---- Load JSON ---- std::ifstream file(source); jsoncons::json doc = jsoncons::json::parse(file); - validate_json(doc); + validate_json(doc, schema_v1); // ---- workflow name ---- workflow_name = doc["name"].as(); diff --git a/src/serializers/v1.1.cpp b/src/serializers/v1.1.cpp new file mode 100644 index 0000000..eb75761 --- /dev/null +++ b/src/serializers/v1.1.cpp @@ -0,0 +1,128 @@ +#include + +#include "capiocl.hpp" +#include "include/engine.h" +#include "include/printer.h" +#include "include/serializer.h" + +void capiocl::serializer::Serializer::available_serializers::serialize_v1_1( + const engine::Engine &engine, const std::filesystem::path &filename) { + jsoncons::json doc; + doc["version"] = 1.1; + doc["name"] = engine.getWorkflowName(); + + const auto files = engine._capio_cl_entries; + + std::unordered_map> app_inputs; + std::unordered_map> app_outputs; + + std::vector permanent; + std::vector exclude; + std::vector memory_storage; + std::vector fs_storage; + + jsoncons::json storage = jsoncons::json::object(); + jsoncons::json io_graph = jsoncons::json::array(); + + for (const auto &[path, entry] : files) { + if (entry.permanent) { + permanent.push_back(path); + } + if (entry.excluded) { + exclude.push_back(path); + } + (entry.store_in_memory ? memory_storage : fs_storage).push_back(path); + + for (const auto &p : entry.producers) { + app_outputs[p].push_back(path); + } + for (const auto &c : entry.consumers) { + app_inputs[c].push_back(path); + } + } + + for (const auto &[app_name, outputs] : app_outputs) { + jsoncons::json app = jsoncons::json::object(); + jsoncons::json streaming = jsoncons::json::array(); + + for (const auto &path : outputs) { + const auto &entry = files.at(path); + + jsoncons::json streaming_item = jsoncons::json::object(); + std::string committed = entry.commit_rule; + const char *name_kind = entry.is_file ? "name" : "dirname"; + streaming_item[name_kind] = jsoncons::json::array({path}); // LCOV_EXCL_LINE + + if (entry.commit_on_close_count > 0) { + if (entry.commit_rule == commitRules::ON_CLOSE) { + const auto close_count = std::to_string(entry.commit_on_close_count); + streaming_item["committed"] = entry.commit_rule + ":" + close_count; + } else { + const auto msg = "Commit rule is not ON_CLOSE but close count > 0"; + printer::print(printer::CLI_LEVEL_WARNING, msg); + printer::print(printer::CLI_LEVEL_WARNING, "Setting commit rule = ON_CLOSE"); + streaming_item["committed"] = std::string(commitRules::ON_CLOSE) + ":" + + std::to_string(entry.commit_on_close_count); + } + } else { + streaming_item["committed"] = entry.commit_rule; + } + + if (!entry.is_file) { + streaming_item["n_files"] = entry.directory_children_count; + } + + // Convert std::vector -> std::vector + std::vector file_deps_str; + file_deps_str.reserve(entry.file_dependencies.size()); + for (const auto &p : entry.file_dependencies) { + file_deps_str.push_back(p.string()); + } + streaming_item["file_deps"] = file_deps_str; + + streaming_item["mode"] = entry.fire_rule; + + streaming.push_back(streaming_item); + } + + app["name"] = app_name; + app["input_stream"] = app_inputs[app_name]; + app["output_stream"] = outputs; + app["streaming"] = streaming; + + io_graph.push_back(app); + } + + // Ensure apps that only have inputs appear as well + for (const auto &[app_name, inputs] : app_inputs) { + bool contained = false; + for (const auto &entry : io_graph.array_range()) { + if (entry["name"].as() == app_name) { + contained = true; + break; + } + } + if (!contained) { + jsoncons::json app = jsoncons::json::object(); + app["name"] = app_name; + app["input_stream"] = inputs; + app["output_stream"] = jsoncons::json::array(); + io_graph.push_back(app); + } + } + + doc["IO_Graph"] = io_graph; + doc["permanent"] = permanent; + doc["exclude"] = exclude; + storage["memory"] = memory_storage; + storage["fs"] = fs_storage; + doc["storage"] = storage; + + std::ofstream out(filename); + if (!out.is_open()) { + throw SerializerException("Failed to open output file: " + filename.string()); + } + out << jsoncons::pretty_print(doc) << std::endl; + + printer::print(printer::CLI_LEVEL_INFO, "Configuration serialized to " + filename.string()); +} \ No newline at end of file diff --git a/src/serializers/v1.cpp b/src/serializers/v1.cpp index e730ab8..738f57d 100644 --- a/src/serializers/v1.cpp +++ b/src/serializers/v1.cpp @@ -1,5 +1,4 @@ #include -#include #include "capiocl.hpp" #include "include/engine.h" diff --git a/tests/cpp/main.cpp b/tests/cpp/main.cpp index e8ed6b5..248ec97 100644 --- a/tests/cpp/main.cpp +++ b/tests/cpp/main.cpp @@ -1,8 +1,12 @@ +#include "capiocl.hpp" #include #include #include #include +const std::vector CAPIO_CL_AVAIL_VERSIONS = {capiocl::CAPIO_CL_VERSION::V1, + capiocl::CAPIO_CL_VERSION::V1_1}; + template std::string demangled_name(const T &obj) { int status; const char *mangled = typeid(obj).name(); @@ -11,13 +15,13 @@ template std::string demangled_name(const T &obj) { return status == 0 ? demangled.get() : mangled; } -#include "capiocl.hpp" #include "include/engine.h" #include "include/monitor.h" #include "include/parser.h" #include "include/printer.h" #include "include/serializer.h" +#include "test_configuration.hpp" #include "test_engine.hpp" #include "test_exceptions.hpp" #include "test_monitor.hpp" diff --git a/tests/cpp/test_configuration.hpp b/tests/cpp/test_configuration.hpp new file mode 100644 index 0000000..e7948bb --- /dev/null +++ b/tests/cpp/test_configuration.hpp @@ -0,0 +1,41 @@ +#ifndef CAPIO_CL_TEST_CONFIGURATION_HPP +#define CAPIO_CL_TEST_CONFIGURATION_HPP + +#define CONFIGURATION_SUITE_NAME TestTOMLConfiguration +#include "include/configuration.h" + +TEST(CONFIGURATION_SUITE_NAME, TestLoadConfiguration) { + capiocl::engine::Engine engine; + engine.loadConfiguration("/tmp/capio_cl_tomls/sample1.toml"); + EXPECT_TRUE(true); +} + +TEST(CONFIGURATION_SUITE_NAME, TestLoadEmptyPath) { + capiocl::engine::Engine engine; + EXPECT_THROW(engine.loadConfiguration(""), + capiocl::configuration::CapioClConfigurationException); +} + +TEST(CONFIGURATION_SUITE_NAME, TestExceptions) { + capiocl::configuration::CapioClConfiguration config; + + EXPECT_THROW(config.getParameter("not.a.valid.key", static_cast(nullptr)), + capiocl::configuration::CapioClConfigurationException); + + EXPECT_THROW(config.getParameter("not.a.valid.key", static_cast(nullptr)), + capiocl::configuration::CapioClConfigurationException); + + try { + config.getParameter("not.a.valid.key", static_cast(nullptr)); + } catch (const capiocl::configuration::CapioClConfigurationException &err) { + EXPECT_GT(strlen(err.what()), 0); + } +} + +TEST(CONFIGURATION_SUITE_NAME, TestFailureParsingTOML) { + capiocl::configuration::CapioClConfiguration config; + EXPECT_THROW(config.load("/tmp/capio_cl_tomls/sample0.toml"), + capiocl::configuration::CapioClConfigurationException); +} + +#endif // CAPIO_CL_TEST_CONFIGURATION_HPP diff --git a/tests/cpp/test_exceptions.hpp b/tests/cpp/test_exceptions.hpp index cf08145..0021757 100644 --- a/tests/cpp/test_exceptions.hpp +++ b/tests/cpp/test_exceptions.hpp @@ -4,95 +4,85 @@ #define EXCEPTION_SUITE_NAME TestThrowExceptions #include "include/serializer.h" -TEST(EXCEPTION_SUITE_NAME, testFailedDump) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - bool exception_catched = false; +TEST(EXCEPTION_SUITE_NAME, testWhatMEthods) { + try { + capiocl::parser::Parser::parse(""); + } catch (const capiocl::parser::ParserException &e) { + EXPECT_TRUE(demangled_name(e) == "capiocl::parser::ParserException"); + EXPECT_GT(strlen(e.what()), 0); + } try { - capiocl::serializer::Serializer::dump(*engine, "/"); - } catch (std::exception &e) { - exception_catched = true; - auto demangled = demangled_name(e); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, - "Caught exception of type =" + demangled); - EXPECT_TRUE(demangled == "capiocl::serializer::SerializerException"); - EXPECT_GT(std::string(e.what()).size(), 0); + const auto engine = capiocl::engine::Engine(); + capiocl::serializer::Serializer::dump(engine, ""); + } catch (const capiocl::serializer::SerializerException &e) { + EXPECT_TRUE(demangled_name(e) == "capiocl::serializer::SerializerException"); + EXPECT_GT(strlen(e.what()), 0); } +} + +TEST(EXCEPTION_SUITE_NAME, testFailedDump) { + for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path source = "/tmp/capio_cl_jsons/V" + version + "/test24.json"; + auto engine = capiocl::parser::Parser::parse(source, "/tmp"); - EXPECT_TRUE(exception_catched); + EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "/"), + capiocl::serializer::SerializerException); + } } TEST(EXCEPTION_SUITE_NAME, testFailedserializeVersion) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - bool exception_catched = false; + for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path source = "/tmp/capio_cl_jsons/V" + version + "/test24.json"; + auto engine = capiocl::parser::Parser::parse(source, "/tmp"); - try { - capiocl::serializer::Serializer::dump(*engine, "test.json", "1234.5678"); - } catch (std::exception &e) { - exception_catched = true; - auto demangled = demangled_name(e); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, - "Caught exception of type =" + demangled); - EXPECT_TRUE(demangled == "capiocl::serializer::SerializerException"); - EXPECT_GT(std::string(e.what()).size(), 0); + EXPECT_THROW(capiocl::serializer::Serializer::dump(*engine, "test.json", "1234.5678"), + capiocl::serializer::SerializerException); } - - EXPECT_TRUE(exception_catched); } TEST(EXCEPTION_SUITE_NAME, testParserException) { - std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons"; + std::filesystem::path JSON_DIR = "/tmp/capio_cl_jsons/"; capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, "Loading jsons from " + JSON_DIR.string()); - bool exception_catched = false; std::vector test_filenames = { "", "ANonExistingFile", - JSON_DIR / "V1_test1.json", - JSON_DIR / "V1_test2.json", - JSON_DIR / "V1_test3.json", - JSON_DIR / "V1_test4.json", - JSON_DIR / "V1_test5.json", - JSON_DIR / "V1_test6.json", - JSON_DIR / "V1_test7.json", - JSON_DIR / "V1_test8.json", - JSON_DIR / "V1_test9.json", - JSON_DIR / "V1_test10.json", - JSON_DIR / "V1_test11.json", - JSON_DIR / "V1_test12.json", - JSON_DIR / "V1_test13.json", - JSON_DIR / "V1_test14.json", - JSON_DIR / "V1_test15.json", - JSON_DIR / "V1_test16.json", - JSON_DIR / "V1_test17.json", - JSON_DIR / "V1_test18.json", - JSON_DIR / "V1_test19.json", - JSON_DIR / "V1_test20.json", - JSON_DIR / "V1_test21.json", - JSON_DIR / "V1_test22.json", - JSON_DIR / "V1_test23.json", - JSON_DIR / "V1_test25.json", + "test1.json", + "test2.json", + "test3.json", + "test4.json", + "test5.json", + "test6.json", + "test7.json", + "test8.json", + "test9.json", + "test10.json", + "test11.json", + "test12.json", + "test13.json", + "test14.json", + "test15.json", + "test16.json", + "test17.json", + "test18.json", + "test19.json", + "test20.json", + "test21.json", + "test22.json", + "test23.json", + "test25.json", }; - - for (const auto &test : test_filenames) { - exception_catched = false; - try { + for (const auto &version : CAPIO_CL_AVAIL_VERSIONS) { + for (const auto &test : test_filenames) { + const auto test_file_path = test.empty() ? test : JSON_DIR / ("V" + version) / test; capiocl::printer::print(capiocl::printer::CLI_LEVEL_WARNING, - "Testing on file " + test.string()); - capiocl::parser::Parser::parse(test); - } catch (std::exception &e) { - exception_catched = true; - auto demangled = demangled_name(e); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, - "Caught exception of type =" + demangled); - EXPECT_TRUE(demangled == "capiocl::parser::ParserException"); - EXPECT_GT(std::string(e.what()).size(), 0); + "Testing on file " + test_file_path.string()); + + EXPECT_THROW(capiocl::parser::Parser::parse(test_file_path), + capiocl::parser::ParserException); } - EXPECT_TRUE(exception_catched); - capiocl::printer::print(capiocl::printer::CLI_LEVEL_INFO, "Test failed successfully\n\n"); } } diff --git a/tests/cpp/test_serialize_deserialize.hpp b/tests/cpp/test_serialize_deserialize.hpp index 5f49a2c..8c0cd1a 100644 --- a/tests/cpp/test_serialize_deserialize.hpp +++ b/tests/cpp/test_serialize_deserialize.hpp @@ -4,162 +4,178 @@ #define SERIALIZE_DESERIALIZE_SUITE_NAME TestSerializeAndDeserialize TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", - file_3_name = "my_command_history.txt", file_4_name = "/tmp"; - std::string producer_name = "_first", consumer_name = "_last", intermediate_name = "_middle"; - - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); - engine.addProducer(file_1_name, producer_name); - engine.addConsumer(file_1_name, intermediate_name); - engine.addProducer(file_2_name, intermediate_name); - engine.addConsumer(file_2_name, consumer_name); - engine.addConsumer(file_1_name, consumer_name); - - engine.setStoreFileInMemory(file_1_name); - engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); - engine.setCommitedCloseNumber(file_1_name, 3); - engine.setFireRule(file_1_name, capiocl::fireRules::UPDATE); - engine.setPermanent(file_1_name, true); - - engine.setStoreFileInFileSystem(file_2_name); - engine.setCommitRule(file_2_name, capiocl::commitRules::ON_TERMINATION); - engine.setFireRule(file_1_name, capiocl::fireRules::NO_UPDATE); - - engine.addProducer(file_3_name, producer_name); - engine.addProducer(file_3_name, consumer_name); - engine.addProducer(file_3_name, intermediate_name); - engine.setExclude(file_3_name, true); - - engine.setCommitRule(file_4_name, capiocl::commitRules::ON_N_FILES); - engine.setFireRule(file_4_name, capiocl::fireRules::NO_UPDATE); - engine.setDirectoryFileCount(file_4_name, 10); - engine.addProducer(file_4_name, intermediate_name); - - engine.print(); - - capiocl::serializer::Serializer::dump(engine, path); - - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); - - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - capiocl::printer::print("", ""); - EXPECT_TRUE(engine == *new_engine); - - auto new_engine1 = capiocl::parser::Parser::parse(path, resolve, true); - EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); - - std::filesystem::remove(path); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "my_command_history.txt", file_4_name = "/tmp"; + std::string producer_name = "_first", consumer_name = "_last", + intermediate_name = "_middle"; + + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, intermediate_name); + engine.addProducer(file_2_name, intermediate_name); + engine.addConsumer(file_2_name, consumer_name); + engine.addConsumer(file_1_name, consumer_name); + + engine.setStoreFileInMemory(file_1_name); + engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); + engine.setCommitedCloseNumber(file_1_name, 3); + engine.setFireRule(file_1_name, capiocl::fireRules::UPDATE); + engine.setPermanent(file_1_name, true); + + engine.setStoreFileInFileSystem(file_2_name); + engine.setCommitRule(file_2_name, capiocl::commitRules::ON_TERMINATION); + engine.setFireRule(file_1_name, capiocl::fireRules::NO_UPDATE); + + engine.addProducer(file_3_name, producer_name); + engine.addProducer(file_3_name, consumer_name); + engine.addProducer(file_3_name, intermediate_name); + engine.setExclude(file_3_name, true); + + engine.setCommitRule(file_4_name, capiocl::commitRules::ON_N_FILES); + engine.setFireRule(file_4_name, capiocl::fireRules::NO_UPDATE); + engine.setDirectoryFileCount(file_4_name, 10); + engine.addProducer(file_4_name, intermediate_name); + + engine.print(); + + capiocl::serializer::Serializer::dump(engine, path, _cl_version); + + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); + + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + capiocl::printer::print("", ""); + EXPECT_TRUE(engine == *new_engine); + + auto new_engine1 = capiocl::parser::Parser::parse(path, resolve, true); + EXPECT_EQ(new_engine1->getFileToStoreInMemory().size(), engine.size()); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1NcloseNfiles) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt"; - std::string producer_name = "_first", consumer_name = "_last"; + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt"; + std::string producer_name = "_first", consumer_name = "_last"; - engine.setDirectory(file_1_name); - engine.setDirectoryFileCount(file_1_name, 10); - engine.addProducer(file_1_name, producer_name); - engine.addConsumer(file_1_name, consumer_name); + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); - capiocl::serializer::Serializer::dump(engine, path); + engine.setDirectory(file_1_name); + engine.setDirectoryFileCount(file_1_name, 10); + engine.addProducer(file_1_name, producer_name); + engine.addConsumer(file_1_name, consumer_name); - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); + capiocl::serializer::Serializer::dump(engine, path, _cl_version); - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - capiocl::printer::print("", ""); - EXPECT_TRUE(engine == *new_engine); + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); - std::filesystem::remove(path); + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + capiocl::printer::print("", ""); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeParseCAPIOCLV1FileDeps) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", - file_3_name = "file3.txt"; - std::string producer_name = "_first", consumer_name = "_last"; - - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); - - engine.newFile(file_1_name); - engine.newFile(file_2_name); - engine.addProducer(file_1_name, producer_name); - engine.addProducer(file_2_name, producer_name); - - engine.newFile(file_3_name); - engine.addConsumer(file_3_name, consumer_name); - engine.addProducer(file_3_name, producer_name); - engine.setCommitRule(file_3_name, capiocl::commitRules::ON_FILE); - engine.setFileDeps(file_3_name, {file_1_name, file_2_name}); - - engine.print(); - capiocl::serializer::Serializer::dump(engine, path); - - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); - - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - capiocl::printer::print("", ""); - EXPECT_TRUE(engine == *new_engine); - - std::filesystem::remove(path); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt", file_2_name = "file2.txt", + file_3_name = "file3.txt"; + std::string producer_name = "_first", consumer_name = "_last"; + + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); + + engine.newFile(file_1_name); + engine.newFile(file_2_name); + engine.addProducer(file_1_name, producer_name); + engine.addProducer(file_2_name, producer_name); + + engine.newFile(file_3_name); + engine.addConsumer(file_3_name, consumer_name); + engine.addProducer(file_3_name, producer_name); + engine.setCommitRule(file_3_name, capiocl::commitRules::ON_FILE); + engine.setFileDeps(file_3_name, {file_1_name, file_2_name}); + + engine.print(); + capiocl::serializer::Serializer::dump(engine, path, _cl_version); + + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); + + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + capiocl::printer::print("", ""); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testSerializeCommitOnCloseCountNoCommitRule) { - const std::filesystem::path path("./config.json"); - const std::string workflow_name = "demo"; - const std::string file_1_name = "file1.txt"; - std::string producer_name = "_first", consumer_name = "_last"; - - capiocl::engine::Engine engine; - engine.setWorkflowName(workflow_name); - - engine.newFile(file_1_name); - engine.addProducer(file_1_name, producer_name); - engine.setCommitRule(file_1_name, capiocl::commitRules::ON_TERMINATION); - engine.setCommitedCloseNumber(file_1_name, 10); - - engine.print(); - capiocl::serializer::Serializer::dump(engine, path); - - std::filesystem::path resolve = ""; - auto new_engine = capiocl::parser::Parser::parse(path, resolve); - - EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); - EXPECT_FALSE(engine == *new_engine); - capiocl::printer::print("", ""); - engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); - EXPECT_TRUE(engine == *new_engine); - - std::filesystem::remove(path); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path path("./config.json"); + const std::string workflow_name = "demo"; + const std::string file_1_name = "file1.txt"; + std::string producer_name = "_first", consumer_name = "_last"; + + capiocl::engine::Engine engine; + engine.setWorkflowName(workflow_name); + + engine.newFile(file_1_name); + engine.addProducer(file_1_name, producer_name); + engine.setCommitRule(file_1_name, capiocl::commitRules::ON_TERMINATION); + engine.setCommitedCloseNumber(file_1_name, 10); + + engine.print(); + capiocl::serializer::Serializer::dump(engine, path, _cl_version); + + std::filesystem::path resolve = ""; + auto new_engine = capiocl::parser::Parser::parse(path, resolve); + + EXPECT_TRUE(new_engine->getWorkflowName() == workflow_name); + EXPECT_FALSE(engine == *new_engine); + capiocl::printer::print("", ""); + engine.setCommitRule(file_1_name, capiocl::commitRules::ON_CLOSE); + EXPECT_TRUE(engine == *new_engine); + + std::filesystem::remove(path); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testParserResolveAbsolute) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test0.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - EXPECT_TRUE(engine->getWorkflowName() == "test"); - EXPECT_TRUE(engine->contains("/tmp/file")); - EXPECT_TRUE(engine->contains("/tmp/file1")); - EXPECT_TRUE(engine->contains("/tmp/file2")); - EXPECT_TRUE(engine->contains("/tmp/file3")); + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V" + _cl_version + + "/test0.json"); + auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(engine->getWorkflowName() == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + EXPECT_TRUE(engine->contains("/tmp/file2")); + EXPECT_TRUE(engine->contains("/tmp/file3")); + } } TEST(SERIALIZE_DESERIALIZE_SUITE_NAME, testNoStorageSection) { - const std::filesystem::path json_path("/tmp/capio_cl_jsons/V1_test24.json"); - auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); - EXPECT_TRUE(engine->getWorkflowName() == "test"); - EXPECT_TRUE(engine->contains("/tmp/file")); - EXPECT_TRUE(engine->contains("/tmp/file1")); -} + for (const auto &_cl_version : CAPIO_CL_AVAIL_VERSIONS) { + const std::filesystem::path json_path("/tmp/capio_cl_jsons/V" + _cl_version + + "/test24.json"); + auto engine = capiocl::parser::Parser::parse(json_path, "/tmp"); + EXPECT_TRUE(engine->getWorkflowName() == "test"); + EXPECT_TRUE(engine->contains("/tmp/file")); + EXPECT_TRUE(engine->contains("/tmp/file1")); + } +} #endif // CAPIO_CL_TEST_SERIALIZE_DESERIALIZE_HPP \ No newline at end of file diff --git a/tests/jsons/V1_test0.json b/tests/jsons/V1.0/test0.json similarity index 100% rename from tests/jsons/V1_test0.json rename to tests/jsons/V1.0/test0.json diff --git a/tests/jsons/V1_test1.json b/tests/jsons/V1.0/test1.json similarity index 100% rename from tests/jsons/V1_test1.json rename to tests/jsons/V1.0/test1.json diff --git a/tests/jsons/V1_test10.json b/tests/jsons/V1.0/test10.json similarity index 100% rename from tests/jsons/V1_test10.json rename to tests/jsons/V1.0/test10.json diff --git a/tests/jsons/V1_test11.json b/tests/jsons/V1.0/test11.json similarity index 100% rename from tests/jsons/V1_test11.json rename to tests/jsons/V1.0/test11.json diff --git a/tests/jsons/V1_test12.json b/tests/jsons/V1.0/test12.json similarity index 100% rename from tests/jsons/V1_test12.json rename to tests/jsons/V1.0/test12.json diff --git a/tests/jsons/V1_test13.json b/tests/jsons/V1.0/test13.json similarity index 100% rename from tests/jsons/V1_test13.json rename to tests/jsons/V1.0/test13.json diff --git a/tests/jsons/V1_test14.json b/tests/jsons/V1.0/test14.json similarity index 100% rename from tests/jsons/V1_test14.json rename to tests/jsons/V1.0/test14.json diff --git a/tests/jsons/V1_test15.json b/tests/jsons/V1.0/test15.json similarity index 100% rename from tests/jsons/V1_test15.json rename to tests/jsons/V1.0/test15.json diff --git a/tests/jsons/V1_test16.json b/tests/jsons/V1.0/test16.json similarity index 100% rename from tests/jsons/V1_test16.json rename to tests/jsons/V1.0/test16.json diff --git a/tests/jsons/V1_test17.json b/tests/jsons/V1.0/test17.json similarity index 100% rename from tests/jsons/V1_test17.json rename to tests/jsons/V1.0/test17.json diff --git a/tests/jsons/V1_test18.json b/tests/jsons/V1.0/test18.json similarity index 100% rename from tests/jsons/V1_test18.json rename to tests/jsons/V1.0/test18.json diff --git a/tests/jsons/V1_test19.json b/tests/jsons/V1.0/test19.json similarity index 100% rename from tests/jsons/V1_test19.json rename to tests/jsons/V1.0/test19.json diff --git a/tests/jsons/V1_test2.json b/tests/jsons/V1.0/test2.json similarity index 100% rename from tests/jsons/V1_test2.json rename to tests/jsons/V1.0/test2.json diff --git a/tests/jsons/V1_test20.json b/tests/jsons/V1.0/test20.json similarity index 100% rename from tests/jsons/V1_test20.json rename to tests/jsons/V1.0/test20.json diff --git a/tests/jsons/V1_test21.json b/tests/jsons/V1.0/test21.json similarity index 100% rename from tests/jsons/V1_test21.json rename to tests/jsons/V1.0/test21.json diff --git a/tests/jsons/V1_test22.json b/tests/jsons/V1.0/test22.json similarity index 100% rename from tests/jsons/V1_test22.json rename to tests/jsons/V1.0/test22.json diff --git a/tests/jsons/V1_test23.json b/tests/jsons/V1.0/test23.json similarity index 100% rename from tests/jsons/V1_test23.json rename to tests/jsons/V1.0/test23.json diff --git a/tests/jsons/V1_test24.json b/tests/jsons/V1.0/test24.json similarity index 100% rename from tests/jsons/V1_test24.json rename to tests/jsons/V1.0/test24.json diff --git a/tests/jsons/V1_test25.json b/tests/jsons/V1.0/test25.json similarity index 100% rename from tests/jsons/V1_test25.json rename to tests/jsons/V1.0/test25.json diff --git a/tests/jsons/V1_test3.json b/tests/jsons/V1.0/test3.json similarity index 100% rename from tests/jsons/V1_test3.json rename to tests/jsons/V1.0/test3.json diff --git a/tests/jsons/V1_test4.json b/tests/jsons/V1.0/test4.json similarity index 100% rename from tests/jsons/V1_test4.json rename to tests/jsons/V1.0/test4.json diff --git a/tests/jsons/V1_test5.json b/tests/jsons/V1.0/test5.json similarity index 100% rename from tests/jsons/V1_test5.json rename to tests/jsons/V1.0/test5.json diff --git a/tests/jsons/V1_test6.json b/tests/jsons/V1.0/test6.json similarity index 100% rename from tests/jsons/V1_test6.json rename to tests/jsons/V1.0/test6.json diff --git a/tests/jsons/V1_test7.json b/tests/jsons/V1.0/test7.json similarity index 100% rename from tests/jsons/V1_test7.json rename to tests/jsons/V1.0/test7.json diff --git a/tests/jsons/V1_test8.json b/tests/jsons/V1.0/test8.json similarity index 100% rename from tests/jsons/V1_test8.json rename to tests/jsons/V1.0/test8.json diff --git a/tests/jsons/V1_test9.json b/tests/jsons/V1.0/test9.json similarity index 100% rename from tests/jsons/V1_test9.json rename to tests/jsons/V1.0/test9.json diff --git a/tests/jsons/V1.1/V1-1_test0.json b/tests/jsons/V1.1/V1-1_test0.json new file mode 100644 index 0000000..292ddcf --- /dev/null +++ b/tests/jsons/V1.1/V1-1_test0.json @@ -0,0 +1,32 @@ +{ + "version": 1.1, + "configuration" : "/tmp/capio_cl.toml", + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "/tmp/file", + "file3" + ], + "output_stream": [ + "file1", + "file2" + ], + "streaming": [ + { + "name": [ + "/tmp/file" + ], + "committed": "on_termination", + "mode": "no_update" + }, + { + "name": [ + "/tmp/file1" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test0.json b/tests/jsons/V1.1/test0.json new file mode 100644 index 0000000..efaca20 --- /dev/null +++ b/tests/jsons/V1.1/test0.json @@ -0,0 +1,32 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "/tmp/file", + "file3" + ], + "output_stream": [ + "file1", + "file2" + ], + "streaming": [ + { + "name": [ + "/tmp/file" + ], + "committed": "on_termination", + "mode": "no_update" + }, + { + "name": [ + "/tmp/file1" + ] + } + ] + } + ], + "storage": {} +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test1.json b/tests/jsons/V1.1/test1.json new file mode 100644 index 0000000..0394d23 --- /dev/null +++ b/tests/jsons/V1.1/test1.json @@ -0,0 +1,3 @@ +{ + "version": 1.1 +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test10.json b/tests/jsons/V1.1/test10.json new file mode 100644 index 0000000..248bfe6 --- /dev/null +++ b/tests/jsons/V1.1/test10.json @@ -0,0 +1,13 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test11.json b/tests/jsons/V1.1/test11.json new file mode 100644 index 0000000..d7bc9b8 --- /dev/null +++ b/tests/jsons/V1.1/test11.json @@ -0,0 +1,18 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + {} + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test12.json b/tests/jsons/V1.1/test12.json new file mode 100644 index 0000000..95cfe49 --- /dev/null +++ b/tests/jsons/V1.1/test12.json @@ -0,0 +1,20 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test13.json b/tests/jsons/V1.1/test13.json new file mode 100644 index 0000000..018310e --- /dev/null +++ b/tests/jsons/V1.1/test13.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test14.json b/tests/jsons/V1.1/test14.json new file mode 100644 index 0000000..3b07731 --- /dev/null +++ b/tests/jsons/V1.1/test14.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "a:b" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test15.json b/tests/jsons/V1.1/test15.json new file mode 100644 index 0000000..1555951 --- /dev/null +++ b/tests/jsons/V1.1/test15.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test16.json b/tests/jsons/V1.1/test16.json new file mode 100644 index 0000000..f123886 --- /dev/null +++ b/tests/jsons/V1.1/test16.json @@ -0,0 +1,23 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_file" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test17.json b/tests/jsons/V1.1/test17.json new file mode 100644 index 0000000..b7af2f2 --- /dev/null +++ b/tests/jsons/V1.1/test17.json @@ -0,0 +1,26 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": [ + "failMe" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test18.json b/tests/jsons/V1.1/test18.json new file mode 100644 index 0000000..da4a0b2 --- /dev/null +++ b/tests/jsons/V1.1/test18.json @@ -0,0 +1,24 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test19.json b/tests/jsons/V1.1/test19.json new file mode 100644 index 0000000..bd8ad57 --- /dev/null +++ b/tests/jsons/V1.1/test19.json @@ -0,0 +1,25 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": "failMe" +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test2.json b/tests/jsons/V1.1/test2.json new file mode 100644 index 0000000..7a14ed0 --- /dev/null +++ b/tests/jsons/V1.1/test2.json @@ -0,0 +1,4 @@ +{ + "version": 1.1, + "name": [] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test20.json b/tests/jsons/V1.1/test20.json new file mode 100644 index 0000000..d8c8b57 --- /dev/null +++ b/tests/jsons/V1.1/test20.json @@ -0,0 +1,27 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test21.json b/tests/jsons/V1.1/test21.json new file mode 100644 index 0000000..665cbba --- /dev/null +++ b/tests/jsons/V1.1/test21.json @@ -0,0 +1,28 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "name": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update" + } + ] + } + ], + "storage": { + "memory": [], + "fs": "failMe" + } +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test22.json b/tests/jsons/V1.1/test22.json new file mode 100644 index 0000000..348d200 --- /dev/null +++ b/tests/jsons/V1.1/test22.json @@ -0,0 +1,25 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "on_termination", + "mode": "no_update", + "n_files": "failMe" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test23.json b/tests/jsons/V1.1/test23.json new file mode 100644 index 0000000..e361dfc --- /dev/null +++ b/tests/jsons/V1.1/test23.json @@ -0,0 +1,24 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ], + "streaming": [ + { + "dirname": [ + "test" + ], + "committed": "n_files:failMe", + "mode": "no_update" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test24.json b/tests/jsons/V1.1/test24.json new file mode 100644 index 0000000..5dc8be5 --- /dev/null +++ b/tests/jsons/V1.1/test24.json @@ -0,0 +1,16 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ] + } + ], + "configuration" : "/tmp/capio_cl_tomls/sample1.toml" +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test25.json b/tests/jsons/V1.1/test25.json new file mode 100644 index 0000000..d1f98f0 --- /dev/null +++ b/tests/jsons/V1.1/test25.json @@ -0,0 +1,15 @@ +{ + "version": "1234.5678", + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ], + "output_stream": [ + "file1" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test3.json b/tests/jsons/V1.1/test3.json new file mode 100644 index 0000000..60fa1e8 --- /dev/null +++ b/tests/jsons/V1.1/test3.json @@ -0,0 +1,4 @@ +{ + "version": 1.1, + "name": "test" +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test4.json b/tests/jsons/V1.1/test4.json new file mode 100644 index 0000000..4dd0576 --- /dev/null +++ b/tests/jsons/V1.1/test4.json @@ -0,0 +1,5 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": "fail" +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test5.json b/tests/jsons/V1.1/test5.json new file mode 100644 index 0000000..1ccbca6 --- /dev/null +++ b/tests/jsons/V1.1/test5.json @@ -0,0 +1,7 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + {} + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test6.json b/tests/jsons/V1.1/test6.json new file mode 100644 index 0000000..15af65c --- /dev/null +++ b/tests/jsons/V1.1/test6.json @@ -0,0 +1,11 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": [ + "failMe" + ] + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test7.json b/tests/jsons/V1.1/test7.json new file mode 100644 index 0000000..59232eb --- /dev/null +++ b/tests/jsons/V1.1/test7.json @@ -0,0 +1,9 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test8.json b/tests/jsons/V1.1/test8.json new file mode 100644 index 0000000..516fd55 --- /dev/null +++ b/tests/jsons/V1.1/test8.json @@ -0,0 +1,10 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": "failMe" + } + ] +} \ No newline at end of file diff --git a/tests/jsons/V1.1/test9.json b/tests/jsons/V1.1/test9.json new file mode 100644 index 0000000..f85b1f3 --- /dev/null +++ b/tests/jsons/V1.1/test9.json @@ -0,0 +1,12 @@ +{ + "version": 1.1, + "name": "test", + "IO_Graph": [ + { + "name": "step", + "input_stream": [ + "file" + ] + } + ] +} \ No newline at end of file diff --git a/tests/python/test_parser_serializer.py b/tests/python/test_parser_serializer.py index 11793ea..6493392 100644 --- a/tests/python/test_parser_serializer.py +++ b/tests/python/test_parser_serializer.py @@ -68,7 +68,7 @@ def test_serialize_parse_py_capio_cl_v1(tmp_path): def test_parser_resolve_absolute(): - json_path = "/tmp/capio_cl_jsons/V1_test0.json" + json_path = "/tmp/capio_cl_jsons/V1.0/test0.json" engine = py_capio_cl.Parser.parse(str(json_path), "/tmp") assert engine.getWorkflowName() == "test" diff --git a/tests/tomls/sample0.toml b/tests/tomls/sample0.toml new file mode 100644 index 0000000..7b80d4c --- /dev/null +++ b/tests/tomls/sample0.toml @@ -0,0 +1 @@ +[a.wrong.formatted.table \ No newline at end of file diff --git a/tests/tomls/sample1.toml b/tests/tomls/sample1.toml new file mode 100644 index 0000000..b8e486f --- /dev/null +++ b/tests/tomls/sample1.toml @@ -0,0 +1,10 @@ +[monitor.mcast] +delay_ms = 300 + +[monitor.mcast.commit] +ip = "224.224.224.3" +port = 11223 + +[monitor.mcast.homenode] +ip = "224.224.224.4" +port = 22334