diff --git a/sources/agent/src/collectors/nvml_collector.cc b/sources/agent/src/collectors/nvml_collector.cc index 8a301de..fa0fc8a 100644 --- a/sources/agent/src/collectors/nvml_collector.cc +++ b/sources/agent/src/collectors/nvml_collector.cc @@ -3,6 +3,7 @@ #include #include #include +#include namespace volta { namespace agent { @@ -31,30 +32,47 @@ bool NvmlCollector::Init() { return false; } - result = nvml->DeviceGetHandleByIndex(0, &device_handle_); - if (result != NVML_SUCCESS) { - std::cerr << "Failed to get device handle: " << nvml->ErrorString(result) - << std::endl; + unsigned int device_count = 0; + result = nvml->DeviceGetCount(&device_count); + if (result != NVML_SUCCESS || device_count == 0) { + std::cerr << "Failed to get NVML device count: " + << nvml->ErrorString(result) << std::endl; nvml->Shutdown(); return false; } - nvmlPciInfo_t pci_info; - result = nvml->DeviceGetPciInfo(device_handle_, &pci_info); - if (result != NVML_SUCCESS) { - std::cerr << "Failed to get PCI info: " << nvml->ErrorString(result) - << std::endl; + devices_.clear(); + for (unsigned int i = 0; i < device_count; ++i) { + nvmlDevice_t device_handle; + result = nvml->DeviceGetHandleByIndex(i, &device_handle); + if (result != NVML_SUCCESS) { + std::cerr << "Failed to get device handle for GPU " << i << ": " + << nvml->ErrorString(result) << std::endl; + continue; + } + + nvmlPciInfo_t pci_info; + result = nvml->DeviceGetPciInfo(device_handle, &pci_info); + if (result != NVML_SUCCESS) { + std::cerr << "Failed to get PCI info for GPU " << i << ": " + << nvml->ErrorString(result) << std::endl; + continue; + } + + GpuID gpu_id; + gpu_id.set_pci_domain(pci_info.domain); + gpu_id.set_pci_bus(pci_info.bus); + gpu_id.set_pci_device(pci_info.device); + gpu_id.set_pci_function(0); + devices_.push_back({device_handle, std::move(gpu_id)}); + } + + if (devices_.empty()) { + std::cerr << "Failed to discover any NVML GPUs" << std::endl; nvml->Shutdown(); return false; } - GpuID gpu_id; - gpu_id.set_pci_domain(pci_info.domain); - gpu_id.set_pci_bus(pci_info.bus); - gpu_id.set_pci_device(pci_info.device); - gpu_id.set_pci_function(0); - gpu_id_ = std::move(gpu_id); - nvml_ = nvml; initialized_ = true; return true; @@ -95,7 +113,9 @@ std::vector NvmlCollector::Collect() { const auto& nvml = *nvml_; std::vector metrics; unsigned int power_mw = 0; - auto now = std::chrono::system_clock::now().time_since_epoch().count(); + auto now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); auto needs = [&](MetricType type) { return std::find(requested_metrics_.begin(), requested_metrics_.end(), @@ -103,49 +123,51 @@ std::vector NvmlCollector::Collect() { }; nvmlReturn_t result; - if (needs(MetricType::METRIC_TYPE_GPU_POWER)) { - result = nvml.DeviceGetPowerUsage(device_handle_, &power_mw); - if (result == NVML_SUCCESS) { - metrics.push_back({MetricType::METRIC_TYPE_GPU_POWER, gpu_id_, - static_cast(power_mw) / 1000.0, now}); + for (const auto& device : devices_) { + if (needs(MetricType::METRIC_TYPE_GPU_POWER)) { + result = nvml.DeviceGetPowerUsage(device.handle, &power_mw); + if (result == NVML_SUCCESS) { + metrics.push_back({MetricType::METRIC_TYPE_GPU_POWER, device.id, + static_cast(power_mw) / 1000.0, now}); + } } - } - if (needs(MetricType::METRIC_TYPE_GPU_TEMPERATURE)) { - unsigned int temp_c = 0; - result = nvml.DeviceGetTemperature(device_handle_, NVML_TEMPERATURE_GPU, - &temp_c); - if (result == NVML_SUCCESS) { - metrics.push_back({MetricType::METRIC_TYPE_GPU_TEMPERATURE, gpu_id_, - static_cast(temp_c), now}); + if (needs(MetricType::METRIC_TYPE_GPU_TEMPERATURE)) { + unsigned int temp_c = 0; + result = nvml.DeviceGetTemperature(device.handle, NVML_TEMPERATURE_GPU, + &temp_c); + if (result == NVML_SUCCESS) { + metrics.push_back({MetricType::METRIC_TYPE_GPU_TEMPERATURE, device.id, + static_cast(temp_c), now}); + } } - } - if (needs(MetricType::METRIC_TYPE_GPU_UTILIZATION) || - needs(MetricType::METRIC_TYPE_GPU_SHARED_MEMORY_UTILIZATION)) { - nvmlUtilization_t utilization; - result = nvml.DeviceGetUtilizationRates(device_handle_, &utilization); - if (result == NVML_SUCCESS) { - if (needs(MetricType::METRIC_TYPE_GPU_UTILIZATION)) { - metrics.push_back({MetricType::METRIC_TYPE_GPU_UTILIZATION, gpu_id_, - static_cast(utilization.gpu), now}); - } - if (needs(MetricType::METRIC_TYPE_GPU_SHARED_MEMORY_UTILIZATION)) { - metrics.push_back( - {MetricType::METRIC_TYPE_GPU_SHARED_MEMORY_UTILIZATION, gpu_id_, - static_cast(utilization.memory), now}); + if (needs(MetricType::METRIC_TYPE_GPU_UTILIZATION) || + needs(MetricType::METRIC_TYPE_GPU_SHARED_MEMORY_UTILIZATION)) { + nvmlUtilization_t utilization; + result = nvml.DeviceGetUtilizationRates(device.handle, &utilization); + if (result == NVML_SUCCESS) { + if (needs(MetricType::METRIC_TYPE_GPU_UTILIZATION)) { + metrics.push_back({MetricType::METRIC_TYPE_GPU_UTILIZATION, device.id, + static_cast(utilization.gpu), now}); + } + if (needs(MetricType::METRIC_TYPE_GPU_SHARED_MEMORY_UTILIZATION)) { + metrics.push_back( + {MetricType::METRIC_TYPE_GPU_SHARED_MEMORY_UTILIZATION, device.id, + static_cast(utilization.memory), now}); + } } } - } - if (needs(MetricType::METRIC_TYPE_GPU_VRAM_USED)) { - nvmlMemory_t memory; - result = nvml.DeviceGetMemoryInfo(device_handle_, &memory); - if (result == NVML_SUCCESS) { - metrics.push_back( - {MetricType::METRIC_TYPE_GPU_VRAM_USED, gpu_id_, - static_cast(memory.used) / static_cast(memory.total), - now}); + if (needs(MetricType::METRIC_TYPE_GPU_VRAM_USED)) { + nvmlMemory_t memory; + result = nvml.DeviceGetMemoryInfo(device.handle, &memory); + if (result == NVML_SUCCESS) { + metrics.push_back({MetricType::METRIC_TYPE_GPU_VRAM_USED, device.id, + static_cast(memory.used) / + static_cast(memory.total), + now}); + } } } diff --git a/sources/agent/src/collectors/nvml_collector.h b/sources/agent/src/collectors/nvml_collector.h index b0660c9..be1f7cf 100644 --- a/sources/agent/src/collectors/nvml_collector.h +++ b/sources/agent/src/collectors/nvml_collector.h @@ -25,11 +25,15 @@ class NvmlCollector final : public RegisteredCollector { void SetRequestedMetrics(const std::vector& metrics) override; private: + struct DeviceInfo { + nvmlDevice_t handle; + GpuID id; + }; + std::vector requested_metrics_; - std::optional gpu_id_; + std::vector devices_; const platform::NvmlApi* nvml_ = nullptr; - nvmlDevice_t device_handle_ = nullptr; bool initialized_ = false; }; diff --git a/sources/agent/src/collectors/proc_stat_collector.cc b/sources/agent/src/collectors/proc_stat_collector.cc index d774f10..5000edf 100644 --- a/sources/agent/src/collectors/proc_stat_collector.cc +++ b/sources/agent/src/collectors/proc_stat_collector.cc @@ -45,7 +45,9 @@ std::vector ProcStatCollector::Collect() { m.type = MetricType::METRIC_TYPE_CPU_UTILIZATION; m.devId = std::nullopt; m.value = usage_percent; - m.timestamp = std::chrono::system_clock::now().time_since_epoch().count(); + m.timestamp = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); return {m}; } diff --git a/sources/agent/src/collectors/ram_collector.cc b/sources/agent/src/collectors/ram_collector.cc index 4abb101..99214da 100644 --- a/sources/agent/src/collectors/ram_collector.cc +++ b/sources/agent/src/collectors/ram_collector.cc @@ -34,7 +34,9 @@ std::vector RamCollector::Collect() { uint64_t total = 0; ReadStats(used, total); - auto now = std::chrono::system_clock::now().time_since_epoch().count(); + auto now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); std::vector metrics; if (needs_total) { metrics.push_back( diff --git a/sources/agent/src/collectors/rapl_collector.cc b/sources/agent/src/collectors/rapl_collector.cc index 2f8c64a..121eb59 100644 --- a/sources/agent/src/collectors/rapl_collector.cc +++ b/sources/agent/src/collectors/rapl_collector.cc @@ -4,55 +4,182 @@ #include #include +#include +#include #include -#include #include +#include +#include +#include +#include +#include +#include namespace volta { namespace agent { namespace collectors { -RaplCollector::RaplCollector() = default; +namespace { -bool RaplCollector::Init() { - try { - OpenMSR(); - uint64_t readout = ReadMSR(0, MSR_RAPL::POWER_UNIT); - power_units_ = pow(0.5, (double)(readout & 0xf)); - energy_units_ = pow(0.5, (double)((readout >> 8) & 0x1f)); - time_units_ = pow(0.5, (double)((readout >> 16) & 0xf)); - readout = ReadMSR(0, MSR_RAPL::PKG::ENERGY_STATUS); - last_value = energy_units_ * readout; - initialized_ = true; - return true; - } catch (const MSR_Open_Exception&) { - return false; - } catch (const MSR_Read_Exception&) { +constexpr const char* kPowercapBase = "/sys/class/powercap"; +constexpr const char* kEnergyFile = "energy_uj"; +constexpr const char* kMaxEnergyRangeFile = "max_energy_range_uj"; +constexpr const char* kNameFile = "name"; + +std::string Trim(std::string value) { + auto begin = + std::find_if_not(value.begin(), value.end(), + [](unsigned char c) { return std::isspace(c); }); + auto end = + std::find_if_not(value.rbegin(), value.rend(), [](unsigned char c) { + return std::isspace(c); + }).base(); + if (begin >= end) return {}; + return std::string(begin, end); +} + +bool ReadFdText(int fd, std::string* out) { + char buffer[128]; + const ssize_t size = pread(fd, buffer, sizeof(buffer), 0); + if (size <= 0) return false; + out->assign(buffer, static_cast(size)); + return true; +} + +bool ParseUnsigned(const std::string& text, uint64_t* value) { + const std::string trimmed = Trim(text); + if (trimmed.empty()) return false; + + uint64_t parsed = 0; + const auto [ptr, ec] = + std::from_chars(trimmed.data(), trimmed.data() + trimmed.size(), parsed); + if (ec != std::errc() || ptr != trimmed.data() + trimmed.size()) { return false; } + *value = parsed; + return true; +} + +bool ReadUnsignedFile(const std::filesystem::path& path, uint64_t* value) { + const int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + + std::string text; + const bool ok = ReadFdText(fd, &text) && ParseUnsigned(text, value); + close(fd); + return ok; +} + +bool ReadTextFile(const std::filesystem::path& path, std::string* value) { + const int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + + const bool ok = ReadFdText(fd, value); + close(fd); + return ok; +} + +bool CanReadFile(const std::filesystem::path& path) { + return access(path.c_str(), R_OK) == 0; } -bool RaplCollector::IsSupported() { - const std::filesystem::path cpu_base = "/dev/cpu"; +std::optional ParseSocketIndex(const std::string& name) { + constexpr std::string_view kPrefix = "package-"; + if (!name.starts_with(kPrefix)) return std::nullopt; + + uint64_t index = 0; + const std::string suffix = name.substr(kPrefix.size()); + if (!ParseUnsigned(suffix, &index)) return std::nullopt; + if (index > std::numeric_limits::max()) return std::nullopt; + return static_cast(index); +} + +struct DiscoveredZone { + std::filesystem::path path; + uint32_t socket_index = 0; +}; + +std::vector DiscoverZones() { + std::vector zones; std::error_code ec; - if (!std::filesystem::exists(cpu_base, ec) || - !std::filesystem::is_directory(cpu_base, ec)) { - return false; + const std::filesystem::path base{kPowercapBase}; + if (!std::filesystem::exists(base, ec) || + !std::filesystem::is_directory(base, ec)) { + return zones; } - for (const auto& entry : std::filesystem::directory_iterator(cpu_base)) { - if (!entry.is_directory()) continue; + const auto options = + std::filesystem::directory_options::skip_permission_denied | + std::filesystem::directory_options::follow_directory_symlink; + for (std::filesystem::recursive_directory_iterator it(base, options, ec), end; + it != end && !ec; it.increment(ec)) { + const auto& entry = *it; + if (!entry.is_directory(ec) || ec) { + ec.clear(); + continue; + } + + const auto path = entry.path(); + const auto name_path = path / kNameFile; + const auto energy_path = path / kEnergyFile; + if (!std::filesystem::exists(name_path, ec) || + !std::filesystem::exists(energy_path, ec) || !CanReadFile(name_path) || + !CanReadFile(energy_path)) { + ec.clear(); + continue; + } - const auto msr_path = entry.path() / "msr"; - if (std::filesystem::exists(msr_path, ec) && - access(msr_path.c_str(), R_OK) == 0) { - return true; + std::string name; + if (!ReadTextFile(name_path, &name)) { + ec.clear(); + continue; } + name = Trim(name); + + const auto socket_index = ParseSocketIndex(name); + if (!socket_index.has_value()) { + ec.clear(); + continue; + } + + zones.push_back({path, socket_index.value()}); + } + + std::sort(zones.begin(), zones.end(), + [](const DiscoveredZone& lhs, const DiscoveredZone& rhs) { + if (lhs.socket_index != rhs.socket_index) + return lhs.socket_index < rhs.socket_index; + return lhs.path.string() < rhs.path.string(); + }); + + return zones; +} + +std::optional ReadEnergyUJ(int fd) { + std::string text; + uint64_t value = 0; + if (!ReadFdText(fd, &text) || !ParseUnsigned(text, &value)) { + return std::nullopt; } + return value; +} - return false; +} // namespace + +RaplCollector::RaplCollector() = default; + +bool RaplCollector::Init() { + ClosePowercap(); + if (!OpenPowercap()) { + initialized_ = false; + return false; + } + initialized_ = true; + return true; } +bool RaplCollector::IsSupported() { return !DiscoverZones().empty(); } + void RaplCollector::SetRequestedMetrics( const std::vector& metrics) { requested_metrics_ = metrics; @@ -66,77 +193,112 @@ std::vector RaplCollector::Collect() { return {}; } - uint64_t readout; - try { - readout = ReadMSR(0, MSR_RAPL::PKG::ENERGY_STATUS); - } catch (const MSR_Read_Exception&) { - return {}; + const auto wall_now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); + const auto steady_now = std::chrono::steady_clock::now(); + + std::vector metrics; + metrics.reserve(zones_.size()); + + for (auto& zone : zones_) { + if (zone.energy_fd < 0) continue; + + const auto current_energy_uj = ReadEnergyUJ(zone.energy_fd); + if (!current_energy_uj.has_value()) { + continue; + } + + uint64_t delta_energy_uj = 0; + if (current_energy_uj.value() >= zone.last_energy_uj) { + delta_energy_uj = current_energy_uj.value() - zone.last_energy_uj; + } else if (zone.max_energy_range_uj > 0) { + delta_energy_uj = current_energy_uj.value() + zone.max_energy_range_uj - + zone.last_energy_uj; + } else { + zone.last_energy_uj = current_energy_uj.value(); + zone.last_sample_time = steady_now; + continue; + } + + const auto delta_time_ns = + std::chrono::duration_cast( + steady_now - zone.last_sample_time) + .count(); + if (delta_time_ns <= 0) { + zone.last_energy_uj = current_energy_uj.value(); + zone.last_sample_time = steady_now; + continue; + } + + Metric metric; + metric.type = MetricType::METRIC_TYPE_CPU_POWER_PACKAGE; + CpuID cpu_id; + cpu_id.set_socket_index(zone.socket_index); + cpu_id.set_core_index(0); + metric.devId = DeviceId{std::move(cpu_id)}; + metric.value = static_cast(delta_energy_uj) * 1000.0 / + static_cast(delta_time_ns); + metric.timestamp = wall_now.count(); + metrics.push_back(std::move(metric)); + + zone.last_energy_uj = current_energy_uj.value(); + zone.last_sample_time = steady_now; } - double value = energy_units_ * readout; - Metric m; - m.type = MetricType::METRIC_TYPE_CPU_POWER_PACKAGE; - CpuID cpu_id; - cpu_id.set_socket_index(0); - cpu_id.set_core_index(0); - m.devId = DeviceId{std::move(cpu_id)}; - m.value = value - last_value; - m.timestamp = std::chrono::system_clock::now().time_since_epoch().count(); - last_value = value; - return {m}; + return metrics; } std::vector RaplCollector::Satisfiable() { return {MetricType::METRIC_TYPE_CPU_POWER_PACKAGE}; } -uint64_t RaplCollector::ReadMSR(uint8_t core, uint32_t offset) { - uint64_t data; - if (core + 1 > MSR_files_.size()) { - throw MSR_Read_Exception(); - } - if (pread(MSR_files_[core], &data, sizeof data, offset) != sizeof data) { - throw MSR_Read_Exception(); - } - return data; -} +bool RaplCollector::OpenPowercap() { + zones_.clear(); -void RaplCollector::OpenMSR() { - const std::filesystem::path cpu_base = "/dev/cpu"; - MSR_files_.clear(); - std::error_code ec; + for (const auto& zone : DiscoverZones()) { + PowercapZone opened_zone; + opened_zone.path = zone.path; + opened_zone.socket_index = zone.socket_index; - if (!std::filesystem::exists(cpu_base, ec)) { - throw MSR_Open_Exception(); - } - std::vector> cpu_entries; - for (const auto& entry : std::filesystem::directory_iterator(cpu_base)) { - if (!entry.is_directory()) continue; - const auto& dirname = entry.path().filename().string(); - if (!std::ranges::all_of(dirname, ::isdigit)) continue; - cpu_entries.emplace_back(std::stoi(dirname), entry.path()); - } + opened_zone.energy_fd = + open((opened_zone.path / kEnergyFile).c_str(), O_RDONLY | O_CLOEXEC); + if (opened_zone.energy_fd < 0) { + continue; + } - std::ranges::sort(cpu_entries); - for (const auto& [id, path] : cpu_entries) { - int fd = open((path / "msr").c_str(), O_RDONLY); - if (fd >= 0) { - MSR_files_.push_back(fd); + const auto initial_energy = ReadEnergyUJ(opened_zone.energy_fd); + if (!initial_energy.has_value()) { + close(opened_zone.energy_fd); + continue; } + + opened_zone.last_energy_uj = initial_energy.value(); + opened_zone.last_sample_time = std::chrono::steady_clock::now(); + + uint64_t max_range_uj = 0; + if (ReadUnsignedFile(opened_zone.path / kMaxEnergyRangeFile, + &max_range_uj)) { + opened_zone.max_energy_range_uj = max_range_uj; + } + + zones_.push_back(std::move(opened_zone)); } - if (MSR_files_.empty()) { - throw MSR_Open_Exception(); - } -} -void RaplCollector::CloseMSR(int fd) { close(fd); } + return !zones_.empty(); +} -RaplCollector::~RaplCollector() { - for (auto file : MSR_files_) { - CloseMSR(file); +void RaplCollector::ClosePowercap() { + for (auto& zone : zones_) { + if (zone.energy_fd >= 0) { + close(zone.energy_fd); + zone.energy_fd = -1; + } } + zones_.clear(); } +RaplCollector::~RaplCollector() { ClosePowercap(); } + } // namespace collectors } // namespace agent } // namespace volta diff --git a/sources/agent/src/collectors/rapl_collector.h b/sources/agent/src/collectors/rapl_collector.h index 9e9817c..1a5f22c 100644 --- a/sources/agent/src/collectors/rapl_collector.h +++ b/sources/agent/src/collectors/rapl_collector.h @@ -1,6 +1,11 @@ #ifndef VOLTA_AGENT_SRC_COLLECTORS_RAPL_COLLECTOR_H_ #define VOLTA_AGENT_SRC_COLLECTORS_RAPL_COLLECTOR_H_ +#include +#include +#include +#include + #include "collectors/collector.h" namespace volta { @@ -20,56 +25,21 @@ class RaplCollector final : public RegisteredCollector { ~RaplCollector(); private: - uint64_t ReadMSR(uint8_t core, uint32_t offset); - void OpenMSR(); - void CloseMSR(int fd); - std::vector requested_metrics_; - bool initialized_ = false; - double power_units_ = 0, energy_units_ = 0, time_units_ = 0; - std::vector MSR_files_; - double last_value = 0; - - class MSR_Read_Exception : std::exception {}; - class MSR_Open_Exception : std::exception {}; - - struct MSR_RAPL { - static constexpr uint32_t POWER_UNIT = 0x606; - struct Units { - static constexpr uint32_t POWER_UNIT_OFFSET = 0; - static constexpr uint32_t POWER_UNIT_MASK = 0x0F; - static constexpr uint32_t ENERGY_UNIT_OFFSET = 0x08; - static constexpr uint32_t ENERGY_UNIT_MASK = 0x1F00; - static constexpr uint32_t TIME_UNIT_OFFSET = 0x10; - static constexpr uint32_t TIME_UNIT_MASK = 0xF000; - }; - - struct PKG { - static constexpr uint32_t POWER_LIMIT = 0x610; - static constexpr uint32_t ENERGY_STATUS = 0x611; - static constexpr uint32_t PERF_STATUS = 0x613; - static constexpr uint32_t POWER_INFO = 0x614; - }; - - struct PP0 { - static constexpr uint32_t POWER_LIMIT = 0x638; - static constexpr uint32_t ENERGY_STATUS = 0x639; - static constexpr uint32_t POLICY = 0x63A; - static constexpr uint32_t PERF_STATUS = 0x63B; - }; + struct PowercapZone { + std::filesystem::path path; + int energy_fd = -1; + uint32_t socket_index = 0; + uint64_t max_energy_range_uj = 0; + uint64_t last_energy_uj = 0; + std::chrono::steady_clock::time_point last_sample_time{}; + }; - struct PP1 { - static constexpr uint32_t POWER_LIMIT = 0x640; - static constexpr uint32_t ENERGY_STATUS = 0x641; - static constexpr uint32_t POLICY = 0x642; - }; + bool OpenPowercap(); + void ClosePowercap(); - struct DRAM { - static constexpr uint32_t POWER_LIMIT = 0x618; - static constexpr uint32_t ENERGY_STATUS = 0x619; - static constexpr uint32_t PERF_STATUS = 0x61B; - static constexpr uint32_t POWER_INFO = 0x61C; - }; - }; + std::vector requested_metrics_; + bool initialized_ = false; + std::vector zones_; }; } // namespace collectors