From 2f5f2924c69ff0e17744dcbfea46da2c77e22146 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 10:22:37 +0200 Subject: [PATCH 1/8] Began adding support to exclude capio-cl section --- capio-server/include/capio-cl-engine/capio_cl_engine.hpp | 1 + capio-server/src/capio-cl-engine/capio_cl_engine.cpp | 9 +++++++++ capio-server/src/client-manager/handlers/consent.cpp | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/capio-server/include/capio-cl-engine/capio_cl_engine.hpp b/capio-server/include/capio-cl-engine/capio_cl_engine.hpp index 6337c5dde..366208805 100644 --- a/capio-server/include/capio-cl-engine/capio_cl_engine.hpp +++ b/capio-server/include/capio-cl-engine/capio_cl_engine.hpp @@ -69,6 +69,7 @@ class CapioCLEngine { void setDirectory(const std::string &path); void setFile(const std::string &path); bool isFile(const std::string &path) const; + bool isExcluded(const std::string &path) const; bool isDirectory(const std::string &path) const; void setCommitedNumber(const std::string &path, int num); void setDirectoryFileCount(const std::string &path, long num); diff --git a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp index 4cb60859e..c7529cfc4 100644 --- a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp +++ b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp @@ -431,3 +431,12 @@ auto CapioCLEngine::get_home_node(const std::string &path) { } return capio_global_configuration->node_name; } + + +bool CapioCLEngine::isExcluded(const std::string &path) const { + START_LOG(gettid(), "call(path=%s)", path.c_str()); + if (const auto itm = _locations.find(path); itm != _locations.end()) { + return std::get<5>(itm->second); + } + return false; +} diff --git a/capio-server/src/client-manager/handlers/consent.cpp b/capio-server/src/client-manager/handlers/consent.cpp index 662d66cff..54132c7c2 100644 --- a/capio-server/src/client-manager/handlers/consent.cpp +++ b/capio-server/src/client-manager/handlers/consent.cpp @@ -17,6 +17,12 @@ void consent_to_proceed_handler(const char *const str) { return; } + if (capio_cl_engine->isExcluded(path)) { + LOG("File should NOT be handled by CAPIO as it is marked as EXCLUDED"); + client_manager->reply_to_client(tid, 1); + return; + } + if (capio_cl_engine->isProducer(path, tid)) { LOG("Application is producer. continuing"); client_manager->reply_to_client(tid, 1); From 79bb5614d98bad6530fbb2f1d5eefd0342cfd4e4 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 10:30:37 +0200 Subject: [PATCH 2/8] Added regex --- capio-server/src/capio-cl-engine/capio_cl_engine.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp index c7529cfc4..c9520eafd 100644 --- a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp +++ b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp @@ -432,11 +432,16 @@ auto CapioCLEngine::get_home_node(const std::string &path) { return capio_global_configuration->node_name; } - bool CapioCLEngine::isExcluded(const std::string &path) const { START_LOG(gettid(), "call(path=%s)", path.c_str()); if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<5>(itm->second); } - return false; + + return std::any_of(_locations.begin(), _locations.end(), [&](auto &itm) { + if (std::regex_match(path.c_str(), std::get<10>(itm.second))) { + return std::get<5>(itm.second); + } + return false; + }); } From d261da5321cd92df28691d833890810ec2e6e21e Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 10:33:33 +0200 Subject: [PATCH 3/8] log --- capio-server/src/capio-cl-engine/capio_cl_engine.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp index c9520eafd..ed45168fb 100644 --- a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp +++ b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp @@ -437,9 +437,11 @@ bool CapioCLEngine::isExcluded(const std::string &path) const { if (const auto itm = _locations.find(path); itm != _locations.end()) { return std::get<5>(itm->second); } - + LOG("Checking against REGEX"); return std::any_of(_locations.begin(), _locations.end(), [&](auto &itm) { + LOG("Checking against %s", itm.first.c_str()); if (std::regex_match(path.c_str(), std::get<10>(itm.second))) { + LOG("Found match. Is excluded: %s", std::get<5>(itm.second) ? "YES" : "NO"); return std::get<5>(itm.second); } return false; From 1abc142b863bc9c0f159cc5f568798a924d5de39 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 10:40:53 +0200 Subject: [PATCH 4/8] Parser fix --- capio-server/src/capio-cl-engine/json_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capio-server/src/capio-cl-engine/json_parser.cpp b/capio-server/src/capio-cl-engine/json_parser.cpp index e3b35083f..980b51876 100644 --- a/capio-server/src/capio-cl-engine/json_parser.cpp +++ b/capio-server/src/capio-cl-engine/json_parser.cpp @@ -385,7 +385,7 @@ CapioCLEngine *JsonParser::parse(const std::filesystem::path &source, " IS RELATIVE! using cwd() of server to compute abs path."); path = resolve_prexix / path; } - // TODO: check for globs + locations->newFile(path); locations->setExclude(path, true); } From fae93f472f28602f221908ae14c4a443bb0253a9 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 10:45:42 +0200 Subject: [PATCH 5/8] Read fix --- capio-server/src/client-manager/handlers/read.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/capio-server/src/client-manager/handlers/read.cpp b/capio-server/src/client-manager/handlers/read.cpp index 6dfe34b57..1c257acb0 100644 --- a/capio-server/src/client-manager/handlers/read.cpp +++ b/capio-server/src/client-manager/handlers/read.cpp @@ -15,6 +15,13 @@ void read_handler(const char *const str) { const std::filesystem::path path_fs(path); + + if (capio_cl_engine->isExcluded(path)) { + LOG("PATH IS MARKED AS EXCLUDE"); + client_manager->reply_to_client(tid, ULLONG_MAX); + return; + } + /** * If process is producer OR fire rule is no update and there is enough data, allow the process * to continue in its execution From 0fb9560bd0b976f1dad66c99663c5680f8069d70 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 11:33:05 +0200 Subject: [PATCH 6/8] wip --- capio-server/src/client-manager/handlers/posix_readdir.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/capio-server/src/client-manager/handlers/posix_readdir.cpp b/capio-server/src/client-manager/handlers/posix_readdir.cpp index 4ebc58a4a..c43226bbe 100644 --- a/capio-server/src/client-manager/handlers/posix_readdir.cpp +++ b/capio-server/src/client-manager/handlers/posix_readdir.cpp @@ -13,6 +13,12 @@ void posix_readdir_handler(const char *const str) { auto metadata_token = file_manager->getMetadataPath(path); LOG("sending to pid %ld token path of %s", pid, metadata_token.c_str()); + if (capio_cl_engine->isExcluded(path)) { + LOG("Path is excluded. Not sending token"); + client_manager->reply_to_client(pid, 0); + return; + } + client_manager->reply_to_client(pid, metadata_token.length()); storage_service->reply_to_client_raw(pid, metadata_token.c_str(), metadata_token.length()); } From 81db8ff9b2b31c70895126290e376b5b152a775b Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Fri, 12 Sep 2025 11:38:19 +0200 Subject: [PATCH 7/8] Fixes --- .../src/client-manager/handlers/posix_readdir.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/capio-server/src/client-manager/handlers/posix_readdir.cpp b/capio-server/src/client-manager/handlers/posix_readdir.cpp index c43226bbe..aaecc5caf 100644 --- a/capio-server/src/client-manager/handlers/posix_readdir.cpp +++ b/capio-server/src/client-manager/handlers/posix_readdir.cpp @@ -10,15 +10,14 @@ void posix_readdir_handler(const char *const str) { sscanf(str, "%d %s", &pid, path); START_LOG(gettid(), "call(pid=%d, path=%s", pid, path); - auto metadata_token = file_manager->getMetadataPath(path); - LOG("sending to pid %ld token path of %s", pid, metadata_token.c_str()); - if (capio_cl_engine->isExcluded(path)) { - LOG("Path is excluded. Not sending token"); - client_manager->reply_to_client(pid, 0); - return; + LOG("Path is excluded. Creating commit token to avoid starvation"); + CapioFileManager::setCommitted(path); } + const auto metadata_token = CapioFileManager::getMetadataPath(path); + LOG("sending to pid %ld token path of %s", pid, metadata_token.c_str()); + client_manager->reply_to_client(pid, metadata_token.length()); storage_service->reply_to_client_raw(pid, metadata_token.c_str(), metadata_token.length()); } From a5c6bb73352768021cb3c7578d64277f8707758c Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Sun, 14 Sep 2025 10:07:30 +0200 Subject: [PATCH 8/8] Minor cleanup of exclude section code handlers Now when a file that should be excluded is opened, the library skips the opening of the file, and the registration of the file descriptor as a capio one --- capio-posix/handlers/open.hpp | 16 ++++++++++++---- .../utils/cache/consent_request_cache.hpp | 2 +- capio-posix/utils/requests.hpp | 8 +++++--- .../src/capio-cl-engine/capio_cl_engine.cpp | 2 +- .../src/client-manager/handlers/open.cpp | 6 ++++++ .../src/client-manager/handlers/read.cpp | 7 ------- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/capio-posix/handlers/open.hpp b/capio-posix/handlers/open.hpp index 7abc5b732..301231297 100644 --- a/capio-posix/handlers/open.hpp +++ b/capio-posix/handlers/open.hpp @@ -87,10 +87,14 @@ int open_handler(long arg0, long arg1, long arg2, long arg3, long arg4, long arg create_request(-1, resolved_path.data(), tid); } else { LOG("not O_CREAT"); - open_request(-1, resolved_path.data(), tid); + if (open_request(-1, resolved_path.data(), tid) == 0) { + LOG("File is excluded! Skipping open of file!"); + return CAPIO_POSIX_SYSCALL_REQUEST_SKIP; + } } - int fd = static_cast(syscall_no_intercept(SYS_open, arg0, arg1, arg2, arg3, arg4, arg5)); + const int fd = + static_cast(syscall_no_intercept(SYS_open, arg0, arg1, arg2, arg3, arg4, arg5)); LOG("Adding capio path"); add_capio_fd(tid, resolved_path, fd, 0, (flags & O_CLOEXEC) == O_CLOEXEC); @@ -124,10 +128,14 @@ int openat_handler(long arg0, long arg1, long arg2, long arg3, long arg4, long a create_request(-1, resolved_path.data(), tid); } else { LOG("not O_CREAT"); - open_request(-1, resolved_path.data(), tid); + if (open_request(-1, resolved_path.data(), tid) == 0) { + LOG("File is excluded! Skipping open of file!"); + return CAPIO_POSIX_SYSCALL_REQUEST_SKIP; + } } - int fd = static_cast(syscall_no_intercept(SYS_openat, arg0, arg1, arg2, arg3, arg4, arg5)); + const int fd = + static_cast(syscall_no_intercept(SYS_openat, arg0, arg1, arg2, arg3, arg4, arg5)); LOG("fd=%d", fd); LOG("Adding resolved capio path (%s)", resolved_path.c_str()); diff --git a/capio-posix/utils/cache/consent_request_cache.hpp b/capio-posix/utils/cache/consent_request_cache.hpp index 0b2e4058c..4d0f7c49e 100644 --- a/capio-posix/utils/cache/consent_request_cache.hpp +++ b/capio-posix/utils/cache/consent_request_cache.hpp @@ -4,7 +4,7 @@ class ConsentRequestCache { std::unordered_map *available_consent; - // Block until server allows for proceeding to a generic request + // Block until the server allows for proceeding to a generic request static capio_off64_t _consent_to_proceed_request(const std::filesystem::path &path, const long tid, const std::string &source_func) { diff --git a/capio-posix/utils/requests.hpp b/capio-posix/utils/requests.hpp index b47f248f4..1e7dd531d 100644 --- a/capio-posix/utils/requests.hpp +++ b/capio-posix/utils/requests.hpp @@ -137,15 +137,17 @@ inline void exit_group_request(const long tid) { } // block until open is possible -inline void open_request(const int fd, const std::filesystem::path &path, const long tid) { +[[nodiscard]] inline capio_off64_t open_request(const int fd, const std::filesystem::path &path, + const long tid) { START_LOG(capio_syscall(SYS_gettid), "call(fd=%ld, path=%s, tid=%ld)", fd, path.c_str(), tid); write_request_cache_fs->flush(tid); char req[CAPIO_REQ_MAX_SIZE]; sprintf(req, "%04d %ld %d %s", CAPIO_REQUEST_OPEN, tid, fd, path.c_str()); buf_requests->write(req, CAPIO_REQ_MAX_SIZE); - capio_off64_t res = bufs_response->at(tid)->read(); - LOG("Obtained from server %llu", res); + const capio_off64_t res = bufs_response->at(tid)->read(); + LOG("Obtained from server %llu. File is %s exclude", res, res == 0 ? "" : "NOT"); + return res; } // non blocking diff --git a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp index ed45168fb..e4f7b83cc 100644 --- a/capio-server/src/capio-cl-engine/capio_cl_engine.cpp +++ b/capio-server/src/capio-cl-engine/capio_cl_engine.cpp @@ -97,7 +97,7 @@ void CapioCLEngine::print() const { if (i == 0) { std::string commit_rule = std::get<2>(itm.second), fire_rule = std::get<3>(itm.second); - bool exclude = std::get<4>(itm.second), permanent = std::get<5>(itm.second); + bool exclude = std::get<5>(itm.second), permanent = std::get<4>(itm.second); line << " " << commit_rule << std::setfill(' ') << std::setw(20 - commit_rule.length()) << " | " << fire_rule diff --git a/capio-server/src/client-manager/handlers/open.cpp b/capio-server/src/client-manager/handlers/open.cpp index c8993eba2..db6ed363d 100644 --- a/capio-server/src/client-manager/handlers/open.cpp +++ b/capio-server/src/client-manager/handlers/open.cpp @@ -12,6 +12,12 @@ void open_handler(const char *const str) { sscanf(str, "%d %d %s", &tid, &fd, path); START_LOG(gettid(), "call(tid=%d, fd=%d, path=%s", tid, fd, path); + if (capio_cl_engine->isExcluded(path)) { + LOG("File should not be handled as it is excluded!"); + client_manager->reply_to_client(tid, 0); + return; + } + if (capio_cl_engine->isProducer(path, tid)) { LOG("Thread is producer. allowing to continue with open"); client_manager->reply_to_client(tid, 1); diff --git a/capio-server/src/client-manager/handlers/read.cpp b/capio-server/src/client-manager/handlers/read.cpp index 1c257acb0..6dfe34b57 100644 --- a/capio-server/src/client-manager/handlers/read.cpp +++ b/capio-server/src/client-manager/handlers/read.cpp @@ -15,13 +15,6 @@ void read_handler(const char *const str) { const std::filesystem::path path_fs(path); - - if (capio_cl_engine->isExcluded(path)) { - LOG("PATH IS MARKED AS EXCLUDE"); - client_manager->reply_to_client(tid, ULLONG_MAX); - return; - } - /** * If process is producer OR fire rule is no update and there is enough data, allow the process * to continue in its execution