diff --git a/include/silk/util/pmc.h b/include/silk/util/pmc.h new file mode 100644 index 0000000..d580e3b --- /dev/null +++ b/include/silk/util/pmc.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include + +namespace silk +{ + +/** + * Measure hardware/software performance counters over a measured window. + * + * Opens a per-CPU event group on every CPU in the calling process's affinity + * mask (so it respects taskset/cgroup pinning). Each group counts, on its CPU: + * + * - CPU cycles (PERF_TYPE_HARDWARE / PERF_COUNT_HW_CPU_CYCLES) + * - instructions (PERF_TYPE_HARDWARE / PERF_COUNT_HW_INSTRUCTIONS) + * - context switches (PERF_TYPE_SOFTWARE / PERF_COUNT_SW_CONTEXT_SWITCHES) + * + * Per-CPU scope (pid == -1) is what makes this honest for io_uring work: it + * captures kernel-side and async-offload (io-wq / sqpoll) cost that ran on the + * CPU, which a per-thread counter on the submitting thread would miss. The + * tradeoff is that it also counts anything else scheduled on those CPUs, so it + * is meant for benchmark box. + * + * The three events form one group (single group leader) so they are co-scheduled + * and the cycles/instructions ratio is internally consistent. Scaling for counter + * multiplexing is applied on read via TIME_ENABLED / TIME_RUNNING. + * + * Requires kernel.perf_event_paranoid <= 0 (per-CPU CPU-event access) or + * CAP_PERFMON. On failure the object stays stable: read() returns {valid=false} + * and the bracketing calls are no-ops, so callers can wire it in unconditionally. + * + * Lifecycle: open() -> [reset() enable() ... disable()] -> read() -> close(). + * Counters start disabled after open(). + */ +class Pmc +{ +public: + struct Counts + { + uint64_t cycles = 0; + uint64_t instructions = 0; + uint64_t contextSwitches = 0; + /** False if open() failed (no permission / no PMU); other fields are 0. */ + bool valid = false; + /** True if any CPU group was multiplexed and values were scaled up. */ + bool scaled = false; + }; + + Pmc() noexcept = default; + ~Pmc() noexcept { close(); } + + Pmc(const Pmc &) = delete; + Pmc & operator=(const Pmc &) = delete; + + /** + * Open per-CPU groups across the process affinity mask. Returns 0 on success, + * otherwise an errno (commonly EACCES when perf_event_paranoid is too high). + * On failure the object is inert and all other methods are safe no-ops. + */ + int open() noexcept; + + /** Zero all counters (group-wide). No-op if not open. */ + void reset() noexcept; + + /** Start counting (group-wide). No-op if not open. */ + void enable() noexcept; + + /** Stop counting (group-wide). No-op if not open. */ + void disable() noexcept; + + /** Sum scaled counts across every CPU group. Returns {valid=false} if not open. */ + Counts read() noexcept; + + /** Close all file descriptors. Idempotent. */ + void close() noexcept; + + /** True if at least one per-CPU group is open. */ + bool valid() const noexcept { return !groups.empty(); } + +private: + struct CpuGroup + { + int cpu = -1; + int leaderFd = -1; // cycles; group leader + int instrFd = -1; + int csFd = -1; + }; + + std::vector groups; +}; + +} // namespace silk diff --git a/src/perf/common.cpp b/src/perf/common.cpp index b6a13fc..550c4f5 100644 --- a/src/perf/common.cpp +++ b/src/perf/common.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -146,6 +147,32 @@ void printCounters() noexcept printf(" }\n"); } +void printPmc(const silk::Pmc::Counts & counts, uint64_t totalIos) noexcept +{ + printf(" \"pmc\": {\n"); + printf(" \"valid\": %s", counts.valid ? "true" : "false"); + if (counts.valid) + { + double ipc = counts.cycles ? static_cast(counts.instructions) / static_cast(counts.cycles) : 0.0; + double cyclesPerIo = totalIos ? static_cast(counts.cycles) / static_cast(totalIos) : 0.0; + double instrPerIo = totalIos ? static_cast(counts.instructions) / static_cast(totalIos) : 0.0; + + printf(",\n"); + printf(" \"scaled\": %s,\n", counts.scaled ? "true" : "false"); + printf(" \"cycles\": %lu,\n", counts.cycles); + printf(" \"instructions\": %lu,\n", counts.instructions); + printf(" \"context_switches\": %lu,\n", counts.contextSwitches); + printf(" \"ipc\": %.4f,\n", ipc); + printf(" \"cycles_per_io\": %.1f,\n", cyclesPerIo); + printf(" \"instructions_per_io\": %.1f\n", instrPerIo); + } + else + { + printf("\n"); + } + printf(" }\n"); +} + void printSchedulerLatency() noexcept { printf(" \"scheduler_latency\": {\n"); diff --git a/src/perf/common.h b/src/perf/common.h index f4df896..a017ff1 100644 --- a/src/perf/common.h +++ b/src/perf/common.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -37,6 +39,14 @@ void printSchedulerLatency() noexcept; */ void printCounters() noexcept; +/** + * Print the "pmc" JSON section: aggregate hardware/software counts over the + * measured window, plus per-IO and ratio derivations (totalIos is the number + * of completed, post-warmup IOs the run measured). + * Outputs no trailing comma. Emits "valid": false when counters could not be opened. + */ +void printPmc(const silk::Pmc::Counts & counts, uint64_t totalIos) noexcept; + /** * Parse a size string with optional k/m/g suffix (case-insensitive) into bytes. * Examples: "4k" -> 4096, "1g" -> 1073741824, "512" -> 512. diff --git a/src/perf/file-perf.cpp b/src/perf/file-perf.cpp index 0951c8c..8d1beda 100644 --- a/src/perf/file-perf.cpp +++ b/src/perf/file-perf.cpp @@ -132,6 +132,8 @@ struct ClientConfig bool printCounters = false; // use registered buffers (IORING_OP_READ_FIXED / IORING_OP_WRITE_FIXED). bool fixedBuffers = false; + // collect per-CPU hardware/software PMCs over the measurement window. + bool pmc = false; }; class Benchmark @@ -346,7 +348,7 @@ int Benchmark::workerFiberMain(WorkerFiberParams * params) noexcept return 0; } -static void printJson(std::vector & latNs, const ClientConfig & cfg) +static void printJson(std::vector & latNs, const ClientConfig & cfg, const silk::Pmc::Counts & pmcCounts) { uint64_t total = latNs.size(); double durationS = static_cast(cfg.durationNs) / 1e9; @@ -365,6 +367,11 @@ static void printJson(std::vector & latNs, const ClientConfig & cfg) printf(" \"rps\": %.1f,\n", rps); printf(" \"bw_bytes\": %.0f,\n", bwBytesS); printLatencyUs(latNs); + if (cfg.pmc) + { + printf(","); + printPmc(pmcCounts, total); + } if (cfg.printCounters) { printf(","); @@ -406,6 +413,7 @@ int main(int argc, char ** argv) ("direct", "use O_DIRECT (bypass page cache)", cxxopts::value(cfg.direct)) ("fixed-buffers", "use registered buffers (IORING_OP_READ_FIXED / WRITE_FIXED)", cxxopts::value(cfg.fixedBuffers)) ("print-counters", "enable per-CPU profiler and include counters in the JSON report", cxxopts::value(cfg.printCounters)) + ("pmc", "collect per-CPU HW/SW perf counters (cycles, instructions, ctx-switches)", cxxopts::value(cfg.pmc)) ("v,verbose", "enable debug logging", cxxopts::value(verbose)) ; // clang-format on @@ -490,10 +498,23 @@ int main(int argc, char ** argv) signalled = sigwaitFor(mask, cfg.warmupNs); } + // Bracket the measurement window with per-CPU PMCs, matching the + // post-warmup latency-collection window (see warmupEndCycles). + silk::Pmc pmc; + silk::Pmc::Counts pmcCounts; + if (cfg.pmc) + { + pmc.open(); + } + if (!signalled) { + pmc.reset(); + pmc.enable(); SILK_INFO("measuring for %s...", formatDuration(cfg.durationNs).c_str()); sigwaitFor(mask, cfg.durationNs); + pmc.disable(); + pmcCounts = pmc.read(); } pthread_sigmask(SIG_UNBLOCK, &mask, nullptr); @@ -502,7 +523,7 @@ int main(int argc, char ** argv) benchmark.stop(); std::vector allLat = benchmark.collectLatencies(); - printJson(allLat, cfg); + printJson(allLat, cfg, pmcCounts); silk::FiberScheduler::destroy(); silk::destroy(); diff --git a/src/util/pmc.cpp b/src/util/pmc.cpp new file mode 100644 index 0000000..351bed9 --- /dev/null +++ b/src/util/pmc.cpp @@ -0,0 +1,208 @@ +#include + +#include + +#include +#include + +#include +#include + +#include +#include +#include + +namespace silk +{ + +namespace +{ + +int perfEventOpen(perf_event_attr * attr, pid_t pid, int cpu, int groupFd, unsigned long flags) noexcept +{ + return static_cast(::syscall(__NR_perf_event_open, attr, pid, cpu, groupFd, flags)); +} + +// Open one event on (pid=-1, cpu). groupFd == -1 makes it a group leader. +// Leaders start disabled; members inherit the leader's enable state. +int openEvent(uint32_t type, uint64_t config, int cpu, int groupFd) noexcept +{ + perf_event_attr attr = {}; + attr.size = sizeof(attr); + attr.type = type; + attr.config = config; + attr.disabled = (groupFd == -1) ? 1 : 0; + attr.exclude_hv = 1; + // Group read: nr, time_enabled, time_running, then one value per member. + attr.read_format = PERF_FORMAT_GROUP | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; + + return perfEventOpen(&attr, /*pid=*/-1, cpu, groupFd, /*flags=*/0); +} + +} // namespace + +int Pmc::open() noexcept +{ + close(); + + cpu_set_t cpuSet; + CPU_ZERO(&cpuSet); + if (::sched_getaffinity(0, sizeof(cpuSet), &cpuSet) != 0) + { + int r = errno; + SILK_WARN("pmc: sched_getaffinity: %s", std::strerror(r)); + return r; + } + + int firstErrno = 0; + int numCpus = CPU_SETSIZE; + for (int cpu = 0; cpu < numCpus; ++cpu) + { + if (!CPU_ISSET(cpu, &cpuSet)) + { + continue; + } + + int leaderFd = openEvent(PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES, cpu, -1); + if (leaderFd < 0) + { + if (!firstErrno) + { + firstErrno = errno; + } + // Permission/PMU failures are the same on every CPU; report once. + continue; + } + + int instrFd = openEvent(PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, cpu, leaderFd); + int csFd = openEvent(PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CONTEXT_SWITCHES, cpu, leaderFd); + if (instrFd < 0 || csFd < 0) + { + if (!firstErrno) + { + firstErrno = errno; + } + if (instrFd >= 0) + { + ::close(instrFd); + } + if (csFd >= 0) + { + ::close(csFd); + } + ::close(leaderFd); + continue; + } + + groups.push_back({cpu, leaderFd, instrFd, csFd}); + } + + if (groups.empty()) + { + int r = firstErrno ? firstErrno : ENODEV; + SILK_WARN("pmc: could not open any per-CPU counters: %s (need perf_event_paranoid<=0 or CAP_PERFMON)", std::strerror(r)); + return r; + } + + return 0; +} + +void Pmc::reset() noexcept +{ + for (const CpuGroup & g : groups) + { + ::ioctl(g.leaderFd, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); + } +} + +void Pmc::enable() noexcept +{ + for (const CpuGroup & g : groups) + { + ::ioctl(g.leaderFd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); + } +} + +void Pmc::disable() noexcept +{ + for (const CpuGroup & g : groups) + { + ::ioctl(g.leaderFd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); + } +} + +Pmc::Counts Pmc::read() noexcept +{ + Counts out; + if (groups.empty()) + { + return out; + } + out.valid = true; + + // Group read layout for read_format = GROUP | TOTAL_TIME_ENABLED | TOTAL_TIME_RUNNING: + // u64 nr; u64 time_enabled; u64 time_running; u64 values[nr]; + // values are in the order events were added: cycles, instructions, context-switches. + uint64_t buf[3 + 3]; + + for (const CpuGroup & g : groups) + { + ssize_t n = ::read(g.leaderFd, buf, sizeof(buf)); + if (n < static_cast(sizeof(buf))) + { + continue; + } + + uint64_t nr = buf[0]; + uint64_t timeEnabled = buf[1]; + uint64_t timeRunning = buf[2]; + if (nr != 3 || timeRunning == 0) + { + continue; + } + + uint64_t cycles = buf[3]; + uint64_t instructions = buf[4]; + uint64_t contextSwitches = buf[5]; + + // Scale up if the group was multiplexed off the PMU part of the window. + if (timeRunning < timeEnabled) + { + out.scaled = true; + cycles = cycles * timeEnabled / timeRunning; + instructions = instructions * timeEnabled / timeRunning; + // Context switches is a software counter and is never multiplexed, + // but it rides the same group window; scale it the same way for + // consistency with the hardware values it is reported beside. + contextSwitches = contextSwitches * timeEnabled / timeRunning; + } + + out.cycles += cycles; + out.instructions += instructions; + out.contextSwitches += contextSwitches; + } + + return out; +} + +void Pmc::close() noexcept +{ + for (const CpuGroup & g : groups) + { + if (g.csFd >= 0) + { + ::close(g.csFd); + } + if (g.instrFd >= 0) + { + ::close(g.instrFd); + } + if (g.leaderFd >= 0) + { + ::close(g.leaderFd); + } + } + groups.clear(); +} + +} // namespace silk