diff --git a/.ace-aarch64-unwind-spec.md b/.ace-aarch64-unwind-spec.md new file mode 100644 index 0000000..ebb6be4 --- /dev/null +++ b/.ace-aarch64-unwind-spec.md @@ -0,0 +1,84 @@ +# Spec: aarch64 DWARF user unwind support + +Priority directive from the user (2026-06-11): the dwarf call-graph path is the +parity priority — fp call graphs require frame-pointer builds of the whole world. +The local oracle (Docker on Apple Silicon) records arm64 perf.data, so aarch64 +support is the critical path for oracle-driven dwarf work. Validate against +`target/oracle/dwarf.*` (re-record with `scripts/perf-oracle` after changes; the +recorded inputs are stable otherwise). + +## Current state + +- `src/perfdata/unwind.rs`: `PerfX86_64Regs { ip, sp, bp, registers: [u64;16] }`, + `FramehopUnwinder { unwinder: UnwinderX86_64, cache: CacheX86_64 }`, + `unwind_x86_64_frame_pointer_stack_like_elfutils` (x86 ebl fallback), + `PerfUserMemoryReader` (arch-neutral), module loading via + `ExplicitModuleSectionInfo` (arch-neutral). +- `src/perfdata/fold.rs` threads `&PerfX86_64Regs` through ~21 sites; entry point is + `append_perf_user_unwind_frames` → + `PerfX86_64Regs::from_perf_masked_values(event.layout.sample_regs_user, ®s.values)`. + `perf_user_reg_value(mask, values, 8)` hardcodes PERF_REG_X86_IP=8 in + `parse_sample_for_summary` too. +- x86-only logic to gate by arch: `is_syscall_return_state` (rcx==ip && r11!=0), + `libdw_arch_fallback_after_empty_object_unwind` guard `regs.bp >= regs.sp`. + +## Register numbering + +- x86_64 (PERF_REG_X86_*): BP=6, SP=7, IP=8 (already implemented). +- aarch64 (PERF_REG_ARM64_*): x0..x28 = 0..28, X29/FP = 29, LR/x30 = 30, SP = 31, + PC = 32. perf records `--call-graph dwarf` on arm64 with mask covering x0-x30, + sp, pc (mask 0x1ffffffff). + +## Arch detection + +perf.data does not store arch per attr; perf uses the header HEADER_ARCH feature +(string from uname, "aarch64" / "x86_64"). Header feature parsing machinery exists +(see `header_build_ids_by_filename_from_file` / `src/perfdata/build_id.rs`, +`header.rs`). Add `header_arch_from_file` reading HEADER_ARCH (feature bit 5; +perf string format: u32 len + bytes, see perf util/header.c write_arch/read). +Default to x86_64 when absent. Plumb into `FoldAccumulator`/sink construction so +`append_perf_user_unwind_frames` can decode regs per arch. + +## Design (project is unreleased — rename freely) + +1. Replace `PerfX86_64Regs` with arch-neutral `PerfUserRegs { arch: PerfArch, + ip: u64, sp: u64, fp: u64, lr: Option, values: ... }` with + `from_perf_masked_values(arch, mask, values)`; keep per-arch accessors used by + the x86 syscall-return check (rcx = values[?]; preserve current behavior via + the existing masked-value lookup). Alternatively an enum — pick whichever keeps + the 21 fold.rs sites simplest; most only use ip/sp/bp. +2. `FramehopUnwinder` becomes an enum or holds per-arch unwinder+cache + (`framehop::aarch64::{UnwinderAarch64, CacheAarch64, UnwindRegsAarch64}`). + Module registration (`framehop::Module::new` + ExplicitModuleSectionInfo) is + shared; instantiate by arch at accumulator/PidUnwindState creation (arch comes + from the file header, one arch per perf.data). + Note framehop aarch64 `UnwindRegsAarch64::new(lr, sp, fp)`; `iter_frames` seeds + with pc. framehop strips PAC bits itself. +3. aarch64 ebl_unwind fallback, faithful to elfutils backends/aarch64_unwind.c + (fetched 2026-06-11, evverx/elfutils mirror): + - FP_REG=29 LR_REG=30 SP_REG=31; FP_OFFSET=0 LR_OFFSET=8 SP_OFFSET=16. + - prev pc = lr; fail only if lr unreadable or lr == 0. + - newLr = mem[fp+8] else 0; newFp = mem[fp+0] else 0; newSp = fp+16. + - success iff `fp == 0 || newSp > sp` (NO bp>=sp precondition like x86; + fp==0 still yields one lr-based caller). + - Iterate like the x86 `unwind_x86_64_frame_pointer_stack_like_elfutils` loop + (callback-per-frame semantics; subsequent pc gets the `!initial && !signal` + pc-1 adjustment in the consumer like the existing code). +4. Gate x86-only logic by arch: syscall-return truncation (no arm64 analogue), + `regs.bp >= regs.sp` fallback precondition is x86-only (aarch64 fallback has + its own conditions above). +5. Leaf-only / scenario-D predicate (see .ace-research-perf-unwind.md §3): on + aarch64 the no-CFI fallback succeeds whenever lr != 0, so "current-IP-only" + stacks are far rarer; the cheap skip-gate condition 6 becomes + `!has_unwind_info_for_ip(ip) && lr == 0` on aarch64. + +## Tests + +- Unit tests mirroring tests/perfdata_unwind.rs x86 cases for the aarch64 + fallback (synthetic stacks: fp chain at fp+0/fp+8, the fp==0-with-lr case, the + newSp <= sp failure case). +- Reg decoding tests for the arm64 mask layout (pc=32, sp=31, fp=29, lr=30). +- End-to-end: `scripts/perf-oracle` → dwarf bench comparison should go from + pyroclast_folded_lines=2 vs inferno=16 to matching (after the script-parity + naming fixes are merged; residuals analyzed against + .ace-research-perf-unwind.md). diff --git a/.ace-research-perf-unwind.md b/.ace-research-perf-unwind.md new file mode 100644 index 0000000..fffd887 --- /dev/null +++ b/.ace-research-perf-unwind.md @@ -0,0 +1,485 @@ +# perf + elfutils user-unwind model for pyroclast parity + +Research date: 2026-06-11. READ-ONLY research — no repo changes. + +Sources read (verified to be real code, not error pages): + +- perf: `torvalds/linux` **master @ v7.1.0-rc7** ("Baby Opossum Posse") + - `tools/perf/util/unwind-libdw.c` + - `tools/perf/util/machine.c` + - `tools/perf/builtin-script.c` + - `tools/perf/util/srcline.c`, `tools/perf/util/dwarf-aux.c` +- elfutils: github mirror `evverx/elfutils` master (sourceware is behind Anubis bot-wall; mirror content matches upstream) + - `libdwfl/dwfl_frame.c` + - `libdwfl/frame_unwind.c` +- pyroclast: `src/perfdata/fold.rs`, `src/perfdata/unwind.rs`, `tests/perfdata_unwind.rs`, `src/symbols.rs` + +Version caveat up front: perf's `unwind__get_entries` had a churny couple of years. The v7.1-rc7 tree reads `data->user_regs` / `data->user_regs->regs` as a *pointer* (the `perf_sample` regs were turned into a heap struct ~2024), and `report_module` is split into `report_module`/`__report_module`. Older perf (≈5.x) had `ui->sample->user_regs.regs` as an inline array and a single `report_module`. The control flow is materially the same; only struct access and the partial-stack error clearing differ (see §1.4). pyroclast should treat the v7.1 behavior as the oracle since the issue notes were measured against a recent perf. + +--- + +## 1. Control-flow narrative: perf + libdw user unwind + +### 1.0 Entry gate (perf side) — `tools/perf/util/machine.c` + +`__thread__resolve_callchain()` runs `thread__resolve_callchain_sample()` (recorded +callchain) and `thread__resolve_callchain_unwind()` (DWARF post-unwind), ordered by +`callchain_param.order`. + +`thread__resolve_callchain_unwind()` is the precise gate for whether libdw unwinding +is even attempted: + +```c +/* Can we do dwarf post unwind? */ +if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) && + (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER))) + return 0; +/* Bail out if nothing was captured. */ +if (!sample->user_regs || !sample->user_regs->regs || + !sample->user_stack.size) + return 0; +... +return unwind__get_entries(unwind_entry, cursor, thread, sample, max_stack, false); +``` + +So DWARF unwind requires, in order: (a) the *evsel* declares both +`PERF_SAMPLE_REGS_USER` and `PERF_SAMPLE_STACK_USER`; (b) the *sample* actually carries +non-null `user_regs`, a non-null reg array, and a non-zero `user_stack.size`. + +IMPORTANT (this is the f5aa59e gate in pyroclast and the gap-2 lever): in `perf +script`, `thread__resolve_callchain()` is *only invoked at all* when +`symbol_conf.use_callchain && sample->callchain` is true. See `builtin-script.c` +(`perf_sample__fprintf_…`): + +```c +if (symbol_conf.use_callchain && sample->callchain) { + cursor = get_tls_callchain_cursor(); + if (thread__resolve_callchain(al->thread, cursor, evsel, sample, + NULL, NULL, scripting_max_stack)) + cursor = NULL; +} +if (cursor == NULL) { + printed += fprintf(fp, " "); + ... +} else + printed += fprintf(fp, "\n"); +printed += sample__fprintf_sym(sample, al, 0, print_opts, cursor, + symbol_conf.bt_stop_list, fp); +``` + +`sample->callchain` is the `PERF_SAMPLE_CALLCHAIN` payload pointer. With `--call-graph +dwarf`, perf records do carry a (short) callchain (typically `[PERF_CONTEXT_USER, ip]` +or a kernel chain), so `sample->callchain` is non-null and the resolver runs. If a +sample has NO callchain payload, `perf script` never calls the resolver, never reaches +`unwind__get_entries`, and prints only the leaf via `sample__fprintf_sym(..., cursor == +NULL)`. This is exactly why pyroclast's `has_perf_object_unwind()` gates on +`SampleCallchainPresence::Present` — correct and source-backed. + +### 1.1 `unwind__get_entries` — `tools/perf/util/unwind-libdw.c` + +Setup: +- Re-checks `if (!data->user_regs || !data->user_regs->regs) return -EINVAL;` +- Allocates `ui` with `max_stack` entry slots; `ui->idx = 0`, `ui->max_stack = max_stack`. +- Obtains a per-`maps` cached `Dwfl` (`maps__libdw_addr_space_dwfl`) or `dwfl_begin(&offline_callbacks)` once and caches it. **The Dwfl and its reported modules persist across samples for the same maps/thread-group** — this is perf's own module cache (see §3 caching). +- Reads the initial IP: `perf_reg_value(&ip, data->user_regs, perf_arch_reg_ip(e_machine))`. +- **Reports the module for the initial IP up front:** `err = report_module(ip, ui); if (err) goto out;`. If the initial IP's DSO cannot be reported into the Dwfl, the whole unwind is abandoned with zero entries. +- `dwfl_attach_state(...)` then `err = dwfl_getthread_frames(dwfl, tid, frame_callback, ui)`. + +### 1.2 `frame_callback` (invoked once per libdwfl frame) + +```c +static int frame_callback(Dwfl_Frame *state, void *arg) { + struct unwind_info *ui = arg; + Dwarf_Addr pc; bool isactivation; + if (!dwfl_frame_pc(state, &pc, NULL)) { ...; return DWARF_CB_ABORT; } + report_module(pc, ui); + if (!dwfl_frame_pc(state, &pc, &isactivation)) { ...; return DWARF_CB_ABORT; } + if (!isactivation) --pc; + return entry(pc, ui) || !(--ui->max_stack) ? DWARF_CB_ABORT : DWARF_CB_OK; +} +``` + +Per frame: get pc; **report the module covering pc**; re-fetch pc with `isactivation`; +decrement pc unless this is an "activation" (signal) frame; call `entry()`. Abort the +iteration if `entry()` fails OR `max_stack` hits 0. + +### 1.3 `entry()` and `__report_module()` — what makes a frame "accepted" + +```c +static int entry(u64 ip, struct unwind_info *ui) { + struct unwind_entry *e = &ui->entries[ui->idx++]; + struct addr_location al; + addr_location__init(&al); + if (__report_module(&al, ip, ui)) { addr_location__exit(&al); return -1; } + e->ip = ip; e->ms.thread = ...; e->ms.map = ...; e->ms.sym = al.sym; + addr_location__exit(&al); + return 0; +} +``` + +`entry()` **always increments `ui->idx`** (reserves the slot) but returns -1 (→ +`DWARF_CB_ABORT`) when `__report_module` fails — leaving `e->ip` written but the entry +treated as a failure. + +`__report_module(al, ip, ui)`: +- `thread__find_symbol(USER, ip, al)` → fills `al->map`, `al->sym`. +- `dso = map__dso(al->map)`; `if (!dso) return 0;` → **NO DSO ⇒ returns success (0) with a NULL map/sym**. This is the "blank path" the issue suspected: an IP with no backing DSO still counts as a reported frame, but resolves to no symbol. +- Computes `base = map__start - map__pgoff` (or `map__start` for `/tmp/jitted-`). +- `dwfl_addrmodule(ui->dwfl, ip)`; if the existing module's start ≠ base, drop it. +- If no module: try `dwfl_report_elf(symfs path)`, else `dwfl_report_elf(build-id path)`. +- Returns `mod && dwfl_addrmodule(ui->dwfl, ip) == mod ? 0 : -1`. + +So `__report_module` fails (−1) only when there IS a dso but the ELF could not be +reported / re-resolved into the Dwfl at `ip`. When there is no dso at all, it succeeds. + +### 1.4 Return handling — does perf keep partial stacks? + +```c +err = dwfl_getthread_frames(dwfl, tid, frame_callback, ui); +if (err && ui->max_stack != max_stack) + err = 0; /* partial success */ +for (i = 0; i < ui->idx && !err; i++) { + int j = (callchain_param.order == ORDER_CALLER) ? ui->idx - i - 1 : i; + err = ui->entries[j].ip ? ui->cb(&ui->entries[j], ui->arg) : 0; +} +``` + +Key facts: +- **Partial stacks ARE kept.** If `dwfl_getthread_frames` returns an error but at least + one frame was consumed (`ui->max_stack != max_stack`, i.e. `--ui->max_stack` ran at + least once), the error is cleared and all collected `ui->idx` entries are emitted. + This is the "perf unwind-libdw: Handle the error of failed to find unwind entries" + family of fixes — perf does NOT throw away frames it already accepted. +- Entries with `entries[j].ip == 0` are skipped (the `? ... : 0`), but a zeroed ip in + the middle still stops nothing — the loop continues. +- The callback `ui->cb` is `unwind_entry` (machine.c). `unwind__get_entries` returns 0 + unconditionally at the end (cleanup always runs). + +`unwind_entry` (machine.c) for each accepted entry: +```c +if (symbol_conf.hide_unresolved && entry->ms.sym == NULL) return 0; +if (append_inlines(cursor, &entry->ms, entry->ip, ...) == 0) return 0; +... callchain_cursor_append(cursor, entry->ip, &entry->ms, ..., srcline); +``` +Note: with `hide_unresolved` OFF (the `perf script` default), an entry with a NULL +symbol is STILL appended to the cursor (printed as a raw address / `[unknown]` under its +map). With `hide_unresolved` ON it is dropped. + +### 1.5 elfutils frame iteration — `libdwfl/dwfl_frame.c` + +`dwfl_getthread_frames` → `getthread(...)` → `get_one_thread_cb` matches the tid and +calls `dwfl_thread_getframes(thread, callback, arg)`. + +`state_alloc()` sets `initial_frame = true`, `pc_state = DWFL_FRAME_STATE_ERROR`. +`state_fetch_pc()` then resolves the PC: for `DWFL_FRAME_STATE_ERROR` it reads the +return-address register from the ABI CFI and sets `pc_state = DWFL_FRAME_STATE_PC_SET` +(for the initial frame this is just the sampled IP via `set_initial_registers`). + +The driving loop (verbatim): +```c +do { + int err = callback (state, arg); /* perf frame_callback */ + if (err != DWARF_CB_OK) { ...thread_detach; free_states(state); return err; } + __libdwfl_frame_unwind (state); + Dwfl_Frame *next = state->unwound; + free (state); + state = next; +} while (state && state->pc_state == DWFL_FRAME_STATE_PC_SET); + +Dwfl_Error err = dwfl_errno (); +...thread_detach... +if (state == NULL || state->pc_state == DWFL_FRAME_STATE_ERROR) { + free_states (state); __libdwfl_seterrno (err); return -1; +} +assert (state->pc_state == DWFL_FRAME_STATE_PC_UNDEFINED); +free_states (state); return 0; +``` + +Crucial ordering: **the callback fires for the CURRENT frame BEFORE that frame is +unwound.** The initial frame (sampled IP) always gets one callback invocation, *provided +`state_fetch_pc` succeeded* (it always does on x86_64 with valid regs). Then +`__libdwfl_frame_unwind` computes the caller; the loop continues only while the next +frame has `pc_state == PC_SET`. + +### 1.6 `__libdwfl_frame_unwind` — `libdwfl/frame_unwind.c` (verbatim) + +```c +void __libdwfl_frame_unwind (Dwfl_Frame *state) { + if (state->unwound) return; + Dwarf_Addr pc; + bool ok = dwfl_frame_pc (state, &pc, NULL); + if (!ok) return; + if (! state->initial_frame && ! state->signal_frame) pc--; + Dwfl_Module *mod = dwfl_addrmodule (state->thread->process->dwfl, pc); + if (mod == NULL) + __libdwfl_seterrno (DWFL_E_NO_DWARF); + else { + Dwarf_Addr bias; + Dwarf_CFI *cfi_eh = dwfl_module_eh_cfi (mod, &bias); + if (cfi_eh) { handle_cfi (state, pc - bias, cfi_eh, bias); if (state->unwound) return; } + Dwarf_CFI *cfi_dwarf = dwfl_module_dwarf_cfi (mod, &bias); + if (cfi_dwarf) { handle_cfi (state, pc - bias, cfi_dwarf, bias); if (state->unwound) return; } + } + assert (state->unwound == NULL); + ... + if (new_unwound (state) == NULL) { __libdwfl_seterrno (DWFL_E_NOMEM); return; } + state->unwound->pc_state = DWFL_FRAME_STATE_PC_UNDEFINED; + bool signal_frame = false; + if (! ebl_unwind (ebl, pc, setfunc, getfunc, readfunc, state, &signal_frame)) { + assert (state->unwound->unwound == NULL); + free (state->unwound); state->unwound = NULL; /* leaves caller's state->unwound NULL */ + return; + } + assert (state->unwound->pc_state == DWFL_FRAME_STATE_PC_SET); + state->unwound->signal_frame = signal_frame; +} +``` + +Order of attempts: **(1) `.eh_frame` CFI → (2) `.debug_frame` CFI → (3) `ebl_unwind` +arch fallback** (on x86_64 this is the frame-pointer `rbp`-chain walk in +`backends/x86_64_unwind.c`). Each `handle_cfi` that sets `state->unwound` short-circuits. + +`new_unwound()` initializes the next frame with `initial_frame = false`, +`pc_state = DWFL_FRAME_STATE_ERROR`. + +`handle_cfi()` sets the unwound frame's `pc_state`: +- If `dwarf_cfi_addrframe` fails (no FDE covers pc): set errno, **return without + allocating `state->unwound`** → `__libdwfl_frame_unwind` then falls through to + `ebl_unwind`. +- The return-address register has `reg_nops == 0` and `reg_ops == reg_ops_mem` and + `regno == ra` ⇒ `pc_state = DWFL_FRAME_STATE_PC_UNDEFINED` (clean end of stack). +- Otherwise after the register loop, if `pc_state` is still `ERROR`, it reads the RA + register: `pc == 0` ⇒ `PC_UNDEFINED` (clean end); `pc != 0` ⇒ `PC_SET` (+ `ebl_ra_offset`); + RA register invalid/unreadable ⇒ `PC_UNDEFINED` (treated as clean end, not error). + +The `ebl_unwind` fallback path always pre-sets the new frame to `PC_UNDEFINED`, then on +success the backend's `setfunc` must have produced a `PC_SET` (asserted). On failure it +frees the unwound frame and leaves `state->unwound == NULL`. + +Net effect on the loop: +- `state->unwound == NULL` (both CFI missing AND `ebl_unwind` failed, OR `dwfl_frame_pc` + failed) ⇒ after `free(state); state = next(=NULL)` ⇒ loop exits with `state == NULL` + ⇒ `dwfl_thread_getframes` returns -1. But perf has already collected the frames it + emitted *before* this point (see §1.4 partial-stack keeping). +- next `pc_state == PC_UNDEFINED` ⇒ loop exits cleanly, return 0. +- next `pc_state == PC_SET` ⇒ callback fires for it, continue. + +--- + +## 2. Decision table: 0 frames vs 1 frame (current IP only) vs N frames + +Granularity matters: there are two distinct "frame" counts. (a) **libdwfl callback +invocations** = number of times `frame_callback` runs = number of frames libdwfl yields +while `pc_state == PC_SET`. (b) **perf accepted entries** = those callbacks whose +`entry()/__report_module` did not fail AND (for the printed callchain) survive +`hide_unresolved`. Because `perf script` runs with `hide_unresolved` OFF, a callback with +a resolvable module but no symbol still prints. + +| Scenario | libdwfl callbacks | perf entries (ui->idx) | perf script printed leaf | Notes | +|---|---|---|---|---| +| **A. Gate not satisfied** (no PERF_SAMPLE_CALLCHAIN payload in `perf script`, OR evsel lacks REGS_USER/STACK_USER, OR sample has no user_regs/regs/stack) | 0 (resolver never entered) | 0 | sample IP only (`sample__fprintf_sym` cursor==NULL) | The single-IP fallback path. pyroclast `SampleCallchainPresence::Absent → None`. | +| **B. Initial IP has a DSO but ELF can't be reported into Dwfl** (`unwind__get_entries` `report_module(ip)` fails) | 0 | 0 | sample IP only | `goto out` before `dwfl_attach_state`. Whole unwind abandoned. | +| **C. Initial IP not backed by any DSO** (`__report_module` returns 0 with NULL map) | ≥1 | ≥1 (ip kept, sym=NULL) | leaf as raw addr/`[unknown]` | `report_module(ip)` succeeds (no dso ⇒ 0); callback fires; entry kept unresolved. | +| **D. Initial frame only; caller unwind dead-ends immediately** (CFI present for leaf gives PC_UNDEFINED, or no CFI and `ebl_unwind` fails on first step) | exactly 1 | 1 | leaf symbol, no callers | **The gap-1 "current-IP-only" stack.** Callback fired once for the sampled IP, then `__libdwfl_frame_unwind` produced NULL/PC_UNDEFINED. Common for libc leaves (`__memcmp_avx2_movbe`, `__memmove_avx_unaligned_erms`) sampled mid-routine where rbp is not a frame pointer and eh_frame for that exact pc dead-ends. | +| **E. Normal multi-frame unwind** | N | N (minus any `__report_module` failures mid-chain) | full stack | Each accepted frame appended; inline frames expanded per entry (§4). | +| **F. Partial unwind, error after k frames** (CFI runs out / mapped memory read fails after k>0 frames) | k | k (error cleared because `max_stack` decremented) | k frames | perf keeps the partial stack (§1.4). | +| **G. max_stack reached** | max_stack | max_stack | truncated | `!(--ui->max_stack)` aborts. perf's `scripting_max_stack` default is large (1024-ish). | + +The exact discriminator between **C/D (1 frame)** and **B (0 frames)** is whether the +*initial* IP's module reports successfully into the Dwfl AND its dso lookup behaves: +- 0 frames: initial `report_module(ip)` returns nonzero — only happens when there IS a + dso for ip but `dwfl_report_elf` fails / re-resolution mismatches. Rare in practice + (missing/unreadable ELF for a mapped region). +- 1 frame: either no dso at all (C) or dso+module fine but the *caller* step fails + immediately (D). + +The discriminator between **D (1 frame)** and **E/F (N≥2)** is purely +`__libdwfl_frame_unwind` on the leaf pc: does eh_frame/.debug_frame CFI yield a `PC_SET` +caller, or does the x86_64 `ebl_unwind` rbp-chain produce a plausible caller? If the +sampled pc is mid-prologue/epilogue or in a leaf with no usable rbp chain and no FDE row +that recovers RA, you get exactly one frame. + +### Mapping to framehop (pyroclast's unwinder) + +framehop's `iter_frames` yields the initial IP as its first frame, then attempts CFI + +its own x86_64 epilogue/fp heuristics. So: +- framehop `framehop_count == 1` (only the seed IP) ≈ libdwfl scenario **D** (1 frame). +- framehop `framehop_count == 0` is *not* something libdwfl produces for a PC_SET initial + frame — libdwfl always fires the callback at least once for the seeded IP. A framehop + count of 0 with `loaded_ip=true has_unwind=true` (the gap-2 note's `0x7ffff7f01f40 + __memcmp_avx2_movbe` case) corresponds to perf's scenario **D**: perf prints the single + leaf, framehop returned nothing. This is the core asymmetry behind both gaps. + +--- + +## 3. Gap-2 gate: exact conditions to cheaply skip "would-produce-nothing" samples + +Goal: avoid running framehop's real user unwind on samples where perf/libdw would emit +nothing extra, because that real unwinding dominates pyroclast runtime (rc=124 timeouts). +The model above gives a layered, cheap-to-expensive gate. Evaluate top-down; the first +few are O(1) and eliminate most work. + +### Hard skip conditions (perf emits ZERO unwound frames — do not unwind at all) + +1. **No PERF_SAMPLE_CALLCHAIN payload on the sample** (`perf script` path). This is the + f5aa59e gate, already implemented as `has_perf_object_unwind == + (sample_callchain == Present)`. Cheapest and highest-yield: if the record has no + callchain payload, `perf script` never calls the resolver. → skip. +2. **evsel sample_type lacks `PERF_SAMPLE_REGS_USER` or `PERF_SAMPLE_STACK_USER`.** + Determined once per evsel/attr at parse time — cache per evsel id. → skip all its + samples for unwind. +3. **Sample carries no user regs / empty reg array / `user_stack.size == 0`** (or, in + pyroclast terms, `perf_effective_user_stack_bytes` is None / dynamic_size 0). → skip. + pyroclast already checks `has_perf_captured_user_stack` and `perf_effective_user_stack_bytes`. +4. **Initial IP's DSO cannot be reported** (scenario B). Rare; equivalent to "no + loadable ELF for the mapping covering ip". pyroclast already models this: + `report_unwind_module_for_ip_like_perf(...) == Failed ⇒ return Vec::new()`. +5. **`KernelWithUserFrame`**: the recorded callchain already crossed kernel→user, so + perf does not append extra user DWARF callers. pyroclast already returns empty for + `SampleCallchainState::KernelWithUserFrame` in `perf_accepted_object_unwind_frames`. + +### Single-frame-only fast path (perf emits exactly the leaf — scenario D) + +This is the expensive class to detect a priori, because distinguishing D (1 frame) from +E/F (N frames) in general requires knowing whether the leaf pc has a usable CFI row / +rbp chain — which is most of the unwind work. Cheap *necessary* conditions that bound it: + +6. **Initial IP has NO CFI coverage in any reported module** (`!has_unwind_info_for_ip(ip)`): + then CFI cannot advance, and the only caller would come from the x86_64 `ebl_unwind` + rbp fallback. If additionally **`regs.bp < regs.sp`** (frame pointer not above stack + pointer — pyroclast's `frame_pointer_at_or_above_stack_pointer == false`, and the + `unwind_x86_64_frame_pointer_stack_like_elfutils` guard `regs.bp >= regs.sp`), the + rbp fallback also produces nothing past the leaf. ⇒ perf yields exactly 1 frame; you + can emit just the leaf (per the initial-frame policy) WITHOUT running framehop. This + is the safe, narrow version of the rejected "broad current-IP salvage" — it must be + gated on `!has_unwind_info_for_ip` AND `bp < sp`, not on "empty framehop" alone (the + broad rule overcounted glibc leaves to 140 lines / 41.2B). +7. **`is_syscall_return_state()`** (rcx==ip && r11!=0): already modeled + (`fe118b0`); perf's unwind truncates after the first executable frame for these. + +### Conditions where you MUST still unwind (cannot skip) + +- Initial IP HAS CFI coverage (`has_unwind_info_for_ip(ip) == true`) — could be D or N; + run framehop. +- Initial IP in a `.so` with eh_frame and `bp >= sp` — rbp fallback may add callers. + +### Caching recommendations + +- **Per-evsel attr cache** for conditions 1–2 (sample_type bits): compute once. +- **Per-(pid, module-base) reported-module cache**: mirrors perf's per-`maps` Dwfl reuse. + pyroclast already keeps `PidUnwindState { object_unwinder, attempted_unwind_mappings, + loaded_unwind_modules }` per pid in `accumulator.unwind_states`. Keep it. +- **Per-(module, pc-page) CFI-presence cache**: `has_unwind_info_for_ip` currently scans + `unwind_ranges` linearly (`src/perfdata/unwind.rs:244`). For the gap-2 gate this is + queried per sample; memoize by `(module_base, ip >> 12)` → bool. Unwind coverage is + page-stable enough that a page-granular cache is safe and turns the gate into O(1). +- **Per-(pid, ip) unwind-result cache (the big win)**: the dominant cost is framehop + re-unwinding identical `(ip, sp-relative stack pattern)`. A pure `(pid, ip)` cache is + unsound because the stack differs per sample, BUT the *shape decision* (0 / 1 / N and + the accepted-frame addresses for the leaf-only case) for scenario-D leaves is + `(pid, ip)`-stable: if `(pid, ip)` was previously classified as "1 frame, no CFI, bp + rule never fires" you can short-circuit. Cache `(pid, ip) → SkipUnwind | LeafOnly | + MustUnwind`. Populate `LeafOnly`/`SkipUnwind` lazily from conditions 6–7; default + `MustUnwind`. This avoids repeat framehop on the hot libc leaves + (`__memcmp_avx2_movbe`, `malloc`, `memmove`) that the issue notes flagged. +- Note perf's own caches you are emulating: the cached `Dwfl` per maps (module reuse), + `dso__data` read cache (`access_dso_mem`), and `dso__inlined_nodes` rbtree + (`inlines__tree_find`) for inline results per address. + +--- + +## 4. Gap-1: inline-only current-IP stacks — `srcline.c` / `dwarf-aux.c` / `machine.c` + +How an "inline-only" leaf arises (matches pyroclast `FoldFrame::InlineCurrentIp`): + +- For each accepted unwind entry, `unwind_entry` (machine.c) calls + `append_inlines(cursor, &entry->ms, entry->ip, ...)` BEFORE appending the entry itself. + If `append_inlines` returns 0 (meaning it appended ≥1 frame and the last append + succeeded), `unwind_entry` returns early and **does not separately append the base + frame** — the inline list is expected to already contain the containing function as its + last element. +- `append_inlines` short-circuits (`return 1`, i.e. "did nothing") when + `!symbol_conf.inline_name || !map || !sym`. `perf script` default does NOT set + `inline_name` unless `--inline` is passed — but the issue's oracle clearly shows inline + expansion, so the comparison run used `--inline` (or perf's default for that report). + Confirm which: if the oracle perf was run WITHOUT `--inline`, then "inline-only" stacks + are NOT from `append_inlines` and the gap is purely scenario-D single leaves (§2.D). +- When enabled: `append_inlines` → `inlines__tree_find` / `dso__parse_addr_inlines` → + `addr2inlines` → `addr2line(..., unwind_inlines=true, node, sym)`. With libdw + (`A2L_STYLE_LIBDW`), the DWARF DIE tree at `addr` is walked + (`die_find_inlinefunc`/`die_find_realfunc` in dwarf-aux.c, already cited by pyroclast + commit `decabb1`): each `DW_TAG_inlined_subroutine` is appended via + `inline_list__append`, and the containing real `DW_TAG_subprogram` is appended as the + final list entry. Ordering uses `callchain_param.order`: for `ORDER_CALLEE`, + `list_add_tail` keeps innermost-inline-first, base-function-last. +- pyroclast already models the traversal rules: `decabb1` (nested `DW_TAG_subprogram` + DIEs are NOT inline-chain frames), `bfd...`/`decabb1` sibling lookup order, and Rust + symbol normalization (`11b4ca6`). The `perf_inline_frame_order` / inline traversal in + `src/symbols.rs` (around lines 2386, 2631, 2898) implements this. + +The "inline-only current-IP" stack in the issue = scenario **D** (single accepted leaf +entry) whose single entry then expands via `append_inlines` into one-or-more inline +frames + base function, with no unwound callers below. So the gap is two-layered: +1. pyroclast must *accept that single leaf entry* (it currently drops it when framehop + returns 0 frames — see `FoldFrame::InlineCurrentIp` only assigned when + `index == 0 && address == regs.ip`, which requires framehop to have produced ≥1 + frame). +2. Then expand its inlines (already done). + +The overcount risk (the 3115296 / 099acd6 history): emitting the leaf for EVERY +empty-framehop sample. The safe gate is exactly conditions §3.6–§3.7 — only emit the +leaf-only stack when libdw would also stop at one frame: initial IP has a reported module +(so `__report_module` succeeds), AND there is no CFI-derived caller AND no rbp-fallback +caller (`!has_unwind_info_for_ip(ip)` and `bp < sp`). Anything broader re-introduces the +measured 10.1B / 41.2B overcounts. + +--- + +## 5. Alignment with existing pyroclast code + +- `src/perfdata/fold.rs` + - `has_perf_object_unwind` / `SampleCallchainPresence` (f5aa59e): correct model of + builtin-script.c's `use_callchain && sample->callchain` gate (§1.0). ✅ + - `choose_user_unwind_source` → `Object`/`None`: matches the "is the resolver reached" + decision. ✅ + - `report_unwind_module_for_ip_like_perf` / `ReportModuleResult`: mirrors + `__report_module` (NoDso=success-with-null, Reported, Failed=−1). The early + `Failed ⇒ Vec::new()` matches `unwind__get_entries`' up-front + `report_module(ip); if (err) goto out`. ✅ + - `report_unwind_modules_for_frame_callbacks_like_perf` + `MAX_LIBDW_CALLBACK_REPORT_PASSES` + loop: models `frame_callback` calling `report_module(pc)` per frame and lazily + pulling in modules for caller IPs (so a later pass can unwind through a module that + was only reported once a caller pc landed in it). ✅ + - `object_unwind_initial_frame_policy` (`KeepDsoLeaf` for `.so`/`.dylib`, + `DropSyntheticCurrentIp` otherwise) — this is the lever for §2.D / §4. Currently + `perf_accepted_object_unwind_frames` ignores the policy (`let _ = (...)`) and just + returns the unwound frames; the leaf-only acceptance for empty framehop is NOT yet + emitted. That unfinished hook is precisely gap-1. + - `should_use_libdw_arch_fallback_after_empty_object_unwind` + `libdw_arch_fallback_after_empty_object_unwind` + (guarded by `regs.bp >= regs.sp`): models `ebl_unwind` rbp fallback as scenario-3 of + `__libdwfl_frame_unwind`. Matches the `bp >= sp` requirement. ✅ + - `truncate_syscall_return_unwind_after_first_executable_frame` is currently a no-op + passthrough (the active syscall-return handling lives elsewhere per `fe118b0`). +- `src/perfdata/unwind.rs` + - `PerfUserMemoryReader::read_u64` is a faithful port of `memory_read` (reject + overflowing word; read inside captured stack; else `access_dso_mem` via + `read_reported_module_u64`). ✅ + - `has_reported_module_for_ip` / `has_unwind_info_for_ip` / `has_rejected_mapping_for_ip`: + the building blocks for the §3 gate. `has_unwind_info_for_ip` is the CFI-coverage + predicate to memoize page-granular. + - `add_object_mapping` base computation (`start - pgoff`, `/tmp/jitted-` ⇒ `start`) + matches `__report_module`'s `base`. Overlap rejection mirrors libdwfl module-range + conflict. ✅ + - `unwind_x86_64_frame_pointer_stack_like_elfutils` is the x86_64 `ebl_unwind` + (`backends/x86_64_unwind.c`) rbp-chain analogue. + +Bottom line: the parsing/memory/module-report layers are solid and source-faithful. The +two open gaps both reduce to one missing piece — **modeling scenario D (libdwfl fires the +initial-frame callback exactly once and stops)**: +- Gap-1 wants pyroclast to *emit* that single leaf (+ its inlines) when libdw would. +- Gap-2 wants pyroclast to *recognize* that class cheaply and NOT pay for a full framehop + unwind (and to skip §3.1–§3.5 zero-frame classes outright). +Both are satisfied by the same narrow predicate: initial IP reported into a module, and +neither CFI (`has_unwind_info_for_ip(ip)`) nor the rbp fallback (`bp >= sp`) can produce a +caller ⇒ leaf-only. Wire that into `perf_accepted_object_unwind_frames` / +`object_unwind_initial_frame_policy` and cache it per `(pid, ip)`. diff --git a/.ace-review-findings.md b/.ace-review-findings.md new file mode 100644 index 0000000..77475de --- /dev/null +++ b/.ace-review-findings.md @@ -0,0 +1,190 @@ +# Pyroclast direct-fold review findings + +Read-only correctness and performance review of the direct `perf.data` fold path. +Scope: `src/perfdata/*.rs`, `src/symbols.rs`, `src/folded.rs`, supporting `src/perfdata/fold.rs` +(the real hot-path code; `src/folded.rs` is only string helpers). + +Severity ordering within each section. Confidence noted per item. + +--- + +## PERFORMANCE + +### PERF-1 (CRITICAL) — DWARF is fully re-parsed for every object on every FINISHED_ROUND and every prefetch batch +- **Where:** `src/symbols.rs:1718` and `src/symbols.rs:1864` (`PerfDwarfNameResolver::from_object_bytes_for_addresses`), defined at `src/symbols.rs:2412-2455`. Driven from `src/perfdata/fold.rs:745` (`drain_fold_counts` per `PERF_RECORD_FINISHED_ROUND`) → `accumulate_fold_counts` (`fold.rs:1921`) → `prefetch_symbols` (`fold.rs:1882`) → `prefetch_mapping_refs` → `resolve_frame_batch_with_metadata`. +- **What's wrong:** `CachedObjectMetadata` (`symbols.rs:265`) caches `object_symbols` and `debug_names`, but **not** the parsed DWARF (`PerfDwarfNameResolver`). Every call to `resolve_frame_batch_with_metadata` rebuilds the entire DWARF index from the object bytes: `object::File::parse` + `gimli::DwarfSections::load` (decompresses every debug section) + iterates **every** compilation unit and builds DIE trees (`perf_dwarf_unit_roots`, `symbols.rs:2445`). The `addresses` filter only skips *building roots* for non-matching units; it still parses all sections and walks all unit headers each time. +- **Why it matters:** `drain_fold_counts` runs once per `FINISHED_ROUND` — perf emits many rounds per capture. Each round re-parses the full DWARF of every DSO touched that round (libc, the main binary, etc.). A profile with R rounds and a hot DSO re-parses that DSO's DWARF ~R times. This is the dominant cost and the direct cause of the rc=124 timeout once synthetic fallbacks (which skipped DWARF) were removed. For a glibc-sized binary a single DWARF parse is tens of ms; multiplied by rounds × objects this is seconds-to-minutes. +- **Fix:** Cache the parsed `PerfDwarfNameResolver` (address-agnostic, i.e. `from_object_bytes_matching_addresses(bytes, None)`) inside `CachedObjectMetadata`, behind the same per-path `Mutex`/`Arc` as the rest of the metadata, and reuse it across all rounds and batches. `frame_names_for_base_symbol` already does its own per-address range filtering, so a once-built full index is correct. Alternatively drain symbols once at the end instead of per round (but the per-object cache is the real fix and also helps the final drain). +- **Confidence:** high. + +### PERF-2 (CRITICAL) — Mapping insert is O(n) per record and rebuilds all per-pid indexes unconditionally → O(n²)+ over the record stream +- **Where:** `src/perfdata/mappings.rs:183-217` (`insert_mapping` → `remove_overlapping_mappings_like_perf`), with `rebuild_pid_indexes` at `mappings.rs:261-310`. Called from `fold.rs:1418/1440/1450` for every MMAP/MMAP2/MMAP2-build-id record. +- **What's wrong:** `remove_overlapping_mappings_like_perf` `std::mem::take`s the whole `mappings` vector and iterates **all** of it (line 194) on every insert, then **unconditionally** calls `rebuild_pid_indexes()` (line 215) — which clears and rebuilds every pid's interval index from scratch. `rebuild_pid_indexes` itself uses `bucket.insert(position, …)` in a loop (line 282), which is O(m) per insert → O(m²) for a bucket of m mappings. So each mmap record is at least O(total mappings), and the index rebuild is quadratic in the largest pid's mapping count. There is no early-out for the common no-overlap case. +- **Why it matters:** mmap/mmap2 records occur on every shared-library load and every fork+exec. For traces with tens of thousands of mappings this is the dominant *parse-side* cost and produces large transient reallocations every record. Compounds PERF-3 and PERF-4. +- **Fix:** Test for overlap first using the existing per-pid interval index (`mappings_by_pid`); if nothing overlaps, do a pure incremental `insert_mapping_without_overlap_fix` and skip the take + global rebuild entirely. Only on an actual split should you repair the affected pid bucket (not all pids). +- **Confidence:** high. + +### PERF-3 (HIGH) — `has_overlapping_user_mapping_for_pid` linear-scans ALL mappings on every mmap record +- **Where:** `src/perfdata/mappings.rs:360-373`; called from `fold.rs:1413/1430/1445` via `invalidate_pid_unwinder_if_mapping_overlaps_like_perf` (`fold.rs:1485-1500`) for every MMAP/MMAP2 record. +- **What's wrong:** `self.mappings.iter().any(...)` scans the entire global mapping table (all pids) and filters by pid in the closure. This runs once per mmap record, independent of and additional to PERF-2's quadratic insert. O(N) per record × N records = O(N²). +- **Why it matters:** Doubles down on the mmap-record quadratic. The per-pid interval index already exists and answers this in O(log n + matches). +- **Fix:** Route through `mappings_by_pid.get(&pid)` and use the `partition_point` / `max_end` interval search (the same pattern `resolve_mapping_index_for_pid` already uses). +- **Confidence:** high. + +### PERF-4 (HIGH) — Per-sample full stack unwind is re-run up to 9× via the module-report retry loop +- **Where:** `src/perfdata/fold.rs:3160-3178` (`unwind_object_frame_addresses_like_perf`), loop bound `MAX_LIBDW_CALLBACK_REPORT_PASSES = 8` (`fold.rs:123`). +- **What's wrong:** For each sample with a user stack, `unwind_user_stack_with_diagnostics` (the framehop unwind over the whole captured stack) is called once, then re-called from scratch on each pass that reports new modules, comparing `next_unwind == object_unwind` (full `Vec` + diagnostics equality) to decide convergence. Worst case 1 + 8 = 9 complete re-unwinds per sample, each walking the entire stack image again. The good news: the framehop `Unwinder` + `Cache` are correctly reused across samples (held in `PidUnwindState.object_unwinder`), and the stack bytes are borrowed (`perf_effective_user_stack_bytes`, `fold.rs:3381`, correctly truncates to `dynamic_size`). The cost is the repeated full unwinds, not cache rebuilds. +- **Why it matters:** Multiplies the per-sample unwind cost (already the heaviest per-sample work) by up to 9 for samples that touch not-yet-reported modules — typical early in a process's life or for deep stacks crossing many DSOs. Contributes materially to the timeout. +- **Fix:** Report all modules for the IPs *before* unwinding when possible, or incrementally extend the existing unwind from the first unresolved frame rather than restarting. At minimum, break as soon as a pass reports zero *new* modules (the current code already does, but it still re-unwinds once more to detect equality — cache the "did we add a module" boolean from `report_unwind_modules_for_frame_callbacks_like_perf` and skip the final redundant unwind when it returned false). +- **Confidence:** medium-high (the re-unwind loop is real; exact multiplier depends on trace). + +### PERF-5 (HIGH) — `normalize_ranges` is O(n²) over FDE ranges at module-add time +- **Where:** `src/perfdata/unwind.rs:476-487` — `ranges.remove(index + 1)` inside a `while` loop. +- **What's wrong:** `Vec::remove` is O(n); doing it in the merge loop makes range normalization O(n²) in the FDE count. Large DSOs (libc, the main binary) have tens of thousands of FDEs. +- **Why it matters:** Per-module startup cost; seconds for large binaries, paid once per loaded unwind module. Not per-sample, but adds to total fold time. +- **Fix:** Single forward pass writing merged ranges into a new `Vec` (or in-place with a write index), instead of `remove` per merge. +- **Confidence:** high. + +### PERF-6 (HIGH) — `read_reported_module_u64` / `has_unwind_info_for_ip` are linear scans over all module segments / FDE ranges +- **Where:** `src/perfdata/unwind.rs:508-518` (`read_reported_module_u64`, the module-memory fallback closure at `unwind.rs:281-283`), and `unwind.rs:230-248` (`has_reported_module_for_ip`, `has_unwind_info_for_ip`), plus the `rejected_mapping_ranges` scan at `unwind.rs:273-277`. +- **What's wrong:** Each out-of-stack memory read framehop requests (CFA expression eval, return-address fetch, GOT/PLT/data) does `modules.iter().flat_map(segments).find_map(...)` — O(modules × segments) per read. `has_unwind_info_for_ip` nests `any` over `unwind_ranges` (thousands of FDE ranges per module). framehop issues many reads per frame and there can be hundreds of mapped DSOs. +- **Why it matters:** O(samples × frames × reads × modules×segments). For a process with many DSOs this can dwarf the actual unwind math. +- **Fix:** Build a single sorted `Vec<(addr_range, module/segment idx)>` (or interval tree) once at module-add time and binary-search. The unwind ranges already pass through `normalize_ranges` (sorted) — search them rather than linear-scan. +- **Confidence:** medium-high. + +### PERF-7 (MEDIUM) — Per-sample full payload copy of every SAMPLE record via `to_vec()` +- **Where:** `src/perfdata/records.rs:677-682` (`parse_sample_payload_record`), reached from `fold.rs:756` (`parse_record_with_context`). Also `parse_read_record` (`records.rs:661`), `parse_aux_record` (`records.rs:698`), switch/namespaces. +- **What's wrong:** `SamplePayloadRecord.payload` is `Vec` built with `payload.to_vec()`. The fold path *does* need an owned copy for **queued** (deferred-by-time) records, because the reused `payload` buffer in `write_folded_perfdata_from_file` (`fold.rs:711,738`) is overwritten on the next iteration. But records that are applied immediately (`apply_or_queue` with `time == None`, `fold.rs:1120-1125`) still pay the full heap copy + memcpy of the largest record type. +- **Why it matters:** One heap alloc + full payload memcpy per sample. With millions of samples this is a top-3 allocation cost on the parse side. +- **Fix:** For the immediate-apply path, decode the sample directly from the borrowed `&payload` slice without `to_vec()`. Only copy when actually queuing. (Requires splitting the apply path so the borrowed case never constructs an owned `ParsedRecord::Sample`.) +- **Confidence:** medium-high (copy is real; some copies are genuinely required for the time-ordered queue). + +### PERF-8 (MEDIUM) — `OrderedRecordQueue` re-sorts the entire pending buffer on every flush round +- **Where:** `src/perfdata/fold.rs:1191-1192` (`flush_through_with`), called from `flush_round` each `FINISHED_ROUND`. +- **What's wrong:** `pending_records.sort_by_key((time, index))` sorts the whole remaining buffer every round, even the tail that stays queued past the flush limit. Records are already nearly time-ordered, but `sort_by_key` is a full O(k log k) each round. +- **Why it matters:** With many rounds and a large in-flight window this is repeated near-sorted sorting. Moderate. +- **Fix:** Use a min-heap keyed by `(time, index)` and pop up to the limit, or only sort the newly-added suffix and merge. At minimum `sort_unstable_by_key`. +- **Confidence:** medium. + +### PERF-9 (MEDIUM) — `ranked_*` in analysis format every distinct key to a String before truncating to `limit` +- **Where:** `src/perfdata/analysis.rs:205-264` (`ranked_threads`/`ranked_ips`/`ranked_edges`), `format_ip` at `analysis.rs:268`. +- **What's wrong:** Builds the full Vec including `format!("tid {tid}")` and `format!("0x{ip:016x}")` for every distinct key, then sorts, then `truncate(limit)`. Edge/IP cardinality can be hundreds of thousands; only `limit` survive. +- **Why it matters:** O(distinct) string allocations discarded. This is the `parse perf summary` path, not the fold path, so lower priority, but wasteful. +- **Fix:** `select_nth_unstable_by` on the numeric weight first, then format only the surviving `limit`. +- **Confidence:** high. + +### PERF-10 (MEDIUM) — `analyze_perfdata` allocates a fresh `frames` Vec per sample +- **Where:** `src/perfdata/analysis.rs:107-112` — `.collect::>()` per sample. +- **What's wrong:** One heap alloc/free per sample in the summary loop. +- **Fix:** Hoist a reusable `Vec` and `clear()` per iteration. +- **Confidence:** high (summary path only). + +### PERF-11 (LOW) — `object_metadata` reads the entire DSO into the heap (`std::fs::read`) rather than mmap +- **Where:** `src/symbols.rs:515` and `src/symbols.rs:579`. +- **What's wrong:** `std::fs::read(path)` loads the whole object file into an owned `Arc<[u8]>`. For large binaries with many DSOs this is significant resident memory (though cached once per path, which is correct). +- **Why it matters:** Memory footprint, not CPU. Bounded by distinct DSO count. Low. +- **Fix:** `memmap2` the object read-only and parse from the mmap (note the SIGBUS-on-truncation tradeoff, ROB-2). +- **Confidence:** low-medium. + +### PERF-12 (LOW) — Per-sample `PathBuf`/String/Vec clones in the unwind setup +- **Where:** `src/perfdata/fold.rs:3121` (`unwind_debug_dir.clone()` per sample), `fold.rs:3291-3292` (`mapping.path.to_string()` + `build_id.to_vec()` per module-report attempt), `fold.rs:3117-3131` (`unwind_states.remove` + reinsert per sample instead of `entry()`). +- **What's wrong:** A `PathBuf` clone per unwound sample; path/build-id clones each report attempt; HashMap remove+insert churn per sample. +- **Why it matters:** Minor constant per-sample overhead; the path-report clones only fire on cache-miss module loads. Low. +- **Fix:** Pass `unwind_debug_dir` as `Option<&Path>` (already `Option<&Path>` downstream); use `entry(pid)` instead of remove+reinsert; defer path/build-id owning to the miss path. +- **Confidence:** medium (clearly wasteful, low total impact). + +### Verified NON-issues (do not re-investigate) +- The framehop `Unwinder` and `Cache` are correctly created once per pid and reused across samples (`PidUnwindState.object_unwinder`); the standalone `unwind_x86_64_stack` free function (`unwind.rs:84`) that rebuilds a `Cache` per call is a test/utility shim, not on the fold path. +- The captured user stack is borrowed (not copied) per sample and correctly truncated to `dynamic_size` (`fold.rs:3381`). +- `SymbolFrameCache` correctly memoizes per `(symbol_source_id, relative_address)` across rounds and batches misses (`symbols.rs:162-171, 1171-1231`); the remaining cost is the DWARF re-parse (PERF-1), not the cache. +- `MappingResolveCache` is `Copy`/stack-only (`mappings.rs:42`), so per-sample creation is free. +- `FoldCounts` coalescing uses an FxHash-bucketed dedup with arena storage (`fold.rs:283-364`) and sorts once at the end (`fold.rs:1962`) — no per-sample sort. + +--- + +## CORRECTNESS + +### CORR-1 (HIGH) — `file_matches_recorded_identity` compares inode only, ignoring device (major/minor) and generation +- **Where:** `src/perfdata/mappings.rs:59-61`. +- **What's wrong:** `metadata.ino() == identity.inode` ignores `identity.major`, `identity.minor`, `inode_generation`, which `FileIdentity` (`mappings.rs:49-55`) carries precisely to disambiguate. Inode numbers are unique only per filesystem; two files on different mounts can share an inode. +- **Why it matters:** A wrong on-disk file can be accepted as matching a recorded mapping → symbols from the wrong binary. Silent mis-symbolization. +- **Fix:** Also compare device (`metadata.dev()` decomposed via `libc::major/minor`) and generation where available. +- **Confidence:** high. + +### CORR-2 (MEDIUM) — Same physical file gets two `symbol_source_id`s depending on which mmap record form delivered it +- **Where:** `src/perfdata/mappings.rs:146-158` (`insert_mmap2_build_id` sets `file_identity: None`) vs `mappings.rs:127` (`insert_mmap2_with_build_id` keeps it); `symbol_source_key` at `mappings.rs:599-610`. +- **What's wrong:** The dedup key mixes `(path, build_id, file_identity, …)`. An inline-build-id MMAP2 keys as `(path, Some(build_id), None)` while the same file arriving as plain MMAP2 + HEADER_BUILD_ID keys as `(path, Some(build_id), Some(identity))`. Different keys → two `symbol_source_id`s for one file. +- **Why it matters:** Defeats the per-mapping symbol cache dedup (`SymbolFrameCache` is keyed on `symbol_source_id`), so the same DSO's DWARF gets resolved twice — amplifies PERF-1. Also splits fold counts if frame labels differ. +- **Fix:** When a build_id is present, key the symbol source on `(path, build_id)` and ignore `file_identity`; only fall back to `file_identity` when no build_id exists. +- **Confidence:** medium. + +### CORR-3 (MEDIUM) — `analysis.rs` thread keying/labeling mismatch: keyed by `tid.or(pid)`, labeled from a tid→comm map +- **Where:** `src/perfdata/analysis.rs:104` (key = `sample.tid.or(sample.pid)`), `analysis.rs:140` + `200-216` (param named `comms_by_pid` but caller passes `comms_by_tid`, lookup at `analysis.rs:209`). +- **What's wrong:** Bucket key falls back to `pid` when `tid` is absent, but the comm lookup map is keyed by tid. A pid-only sample is counted under its pid value and may pick up a different thread's comm (if some tid equals that pid value) or fall back to `tid {n}`. +- **Why it matters:** Mislabeled/merged thread rows in `parse perf summary`. Report-correctness, not fold-correctness. +- **Fix:** Pick one key (tid) and only bucket samples that have it, or carry both pid+tid and look up comm with the matching key. Rename the misleading `comms_by_pid` param. +- **Confidence:** medium. + +### CORR-4 (MEDIUM) — `build_id.rs::filename` hard-fails on non-UTF-8 paths while the rest of the parser uses lossy UTF-8 +- **Where:** `src/perfdata/build_id.rs:198-206` (returns `Err` on non-UTF-8) vs `src/perfdata/records.rs:1018-1024` (`from_utf8_lossy`). +- **What's wrong:** Linux paths are bytes, not UTF-8. A legal perf.data with a non-UTF-8 mapped path parses fine as a normal MMAP path but causes `build_id_events_from_perfdata` (and thus kernel build-id lookup) to error out entirely. +- **Why it matters:** Build-id-based symbol/unwind resolution silently disabled for the whole file on an edge-case path. Robustness regression. +- **Fix:** Use `from_utf8_lossy` for the build-id filename too (or store bytes), matching `parse_c_string`. +- **Confidence:** medium. + +### CORR-5 (LOW/MEDIUM) — Build-id feature section: trailing padding turns into a hard parse error +- **Where:** `src/perfdata/build_id.rs:36-37` — loop guard `offset < payload.len()` admits a 1–7 byte tail; `read_u16(payload, offset+6)` then errors on a partial trailing record. +- **What's wrong:** perf feature payloads are not guaranteed record-aligned at the tail; benign padding aborts the entire build-id extraction (returns `Err`, not "end of section"). +- **Why it matters:** Same downstream effect as CORR-4 — build-id resolution disabled — for files with trailing padding. +- **Fix:** Require `offset + 8 <= payload.len()` to read a header; treat a smaller remainder as end-of-section, not error. +- **Confidence:** medium. + +### CORR-6 (LOW) — Endianness is assumed little-endian throughout; the file's recorded byte order is not consulted +- **Where:** `src/perfdata/endian.rs:6-44` (`read_u16/u32/u64` are hard-coded `from_le_bytes`); header magic check at `src/perfdata/header.rs:31` only accepts `b"PERFILE2"` (LE), never the byte-swapped magic. +- **What's wrong:** A big-endian-recorded perf.data (byte-swapped `PERFILE2`) is rejected at the magic check rather than misparsed — so this is *safe* today (clean rejection), but cross-endian capture is simply unsupported and there is no byte-order plumbing. Flagging because the review brief called it out: it is a latent landmine if anyone adds support for the swapped magic without threading endianness through `read_u*`. +- **Why it matters:** No live bug (rejected cleanly). Future-proofing only. +- **Fix:** Either document LE-only explicitly, or detect endianness from the magic and thread it through the `read_u*` helpers. +- **Confidence:** high (that it's currently safe); low (impact). + +### CORR-7 (LOW) — `is_syscall_return_state` silently false-negatives when RCX/R11 weren't captured +- **Where:** `src/perfdata/unwind.rs:615`, populated at `unwind.rs:645,661`; consumed in `build_user_unwind_context` (`fold.rs:3032`). +- **What's wrong:** RCX/R11 are only populated if the perf register mask included bits 2 and 19. With a minimal capture mask (ip/sp/bp only) they read 0 and the function returns false, silently disabling syscall-return special-casing. +- **Why it matters:** Unwind accuracy at syscall boundaries for minimally-captured samples; no crash. +- **Fix:** Track register presence (bitmask or `Option`) so "not captured" is distinguishable from "value is 0". +- **Confidence:** medium. + +### Verified-correct (checked, no bug) +- x86_64 perf-register → framehop `Reg` mapping is correct, and ignored perf reg bits (segments/flags, bits 9–15) still advance the value iterator so no desync (`unwind.rs:635-668`). The `1 << register` shift is `u64` by inference, no overflow at bit 63. +- `parse_sample_record` field ordering matches the kernel `PERF_RECORD_SAMPLE` serialization order, not flag order (`samples.rs:134-217`). +- Sample/record length math uses `checked_mul`/`checked_add` and validates payload length before `Vec::with_capacity` (e.g. `records.rs:758-771`, `samples.rs:301-306`), so attacker-controlled counts don't OOM or overflow. +- `Mapping::relative_address` subtraction is protected by the resolver invariant `start <= ip < end` (`mappings.rs:565-571`); safe but fragile (see ROB-1). + +--- + +## ROBUSTNESS (panics / hangs reachable from malformed input) + +### ROB-1 (LOW) — `Mapping::relative_address` does unchecked `ip - self.start` +- **Where:** `src/perfdata/mappings.rs:565-571`. Currently safe via the resolver invariant, but every other subtraction in the file uses `saturating_*`. A future caller passing an unresolved mapping underflows (debug panic / release wrap). +- **Fix:** `ip.saturating_sub(self.start)` defensively. +- **Confidence:** low. + +### ROB-2 (LOW/MEDIUM) — `unsafe { Mmap::map(&file) }` on profiled DSOs can SIGBUS if the file is truncated/replaced +- **Where:** `src/perfdata/unwind.rs:168-171` (and PERF-11 if symbols.rs switches to mmap). +- **What's wrong:** Inherent mmap hazard: a DSO unlinked/replaced/truncated after mapping faults with SIGBUS on read — uncatchable by `Result`. Profiling targets routinely replace binaries. +- **Why it matters:** Low probability, process-crash severity. +- **Fix:** Accept as a documented tradeoff, or read sections into owned buffers for untrusted paths, or install a SIGBUS handler. +- **Confidence:** medium. + +### ROB-3 (LOW) — Unwind loop bounded only by caller-supplied `max_frames` +- **Where:** `src/perfdata/unwind.rs:90-95` / `291-296`; fold callers pass `256` (`fold.rs:3161,3173`), so the live path is bounded. A caller passing `usize::MAX` plus pathological CFI could spin. +- **Fix:** Internal hard cap (e.g. `max_frames.min(1024)`). +- **Confidence:** low (current callers are safe). + +--- + +## Recommended fix order (impact-weighted) +1. **PERF-1** — cache the parsed `PerfDwarfNameResolver` per object. Single biggest win; directly addresses the timeout. Also unblocked by CORR-2. +2. **PERF-2 + PERF-3** — kill the mmap-record quadratics (overlap check + index rebuild) by going through the per-pid interval index with a no-overlap fast path. +3. **PERF-4** — stop re-unwinding the whole stack up to 9× per sample. +4. **PERF-5 + PERF-6** — index FDE ranges and module segments once; drop the O(n²) `normalize_ranges`. +5. **CORR-1 / CORR-2** — device-aware file identity and build-id-first symbol-source keying (correctness + dedup). diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 547266f..d74d8c1 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,4 @@ -{"id":"pyroclast-5gr","title":"Model libdw current-frame-only callbacks for perf parity","description":"Perf/inferno parity gap: perf script emits some inline-only current-IP stacks where Pyroclast/framehop has no accepted object frames. Source: perf util/unwind-libdw.c frame_callback->entry and elfutils libdwfl/dwfl_frame.c calls callback before unwind; blank path likely goes through __report_module/module reporting, not stack-frame success. Latest oracle after 099acd6: Pyroclast 114 folded lines / 2,995,326,258 vs perf/inferno 144 / 3,212,226,637. Need source-backed libdw-compatible current-frame model or a pluggable libdw oracle; avoid broad inline current-IP salvage because 3115296 overcounted to 323 lines / 10.1B.","status":"open","priority":1,"issue_type":"task","created_at":"2026-06-02T12:41:11.923032472Z","created_by":"mjc","updated_at":"2026-06-02T12:41:11.923032472Z","source_repo":"pyroclast","source_repo_path":"/home/mjc/projects/pyroclast","compaction_level":0,"original_size":0} -{"id":"pyroclast-8v3","title":"Preserve Inferno-compatible folded output while reducing fold memory","description":"Dirty fold flushing wrote partial folded counts during streaming. Full nextest showed duplicate stacks across finished rounds and order changes; direct output must remain globally coalesced and sorted like inferno-collapse-perf. Resolution: flush pending records into the accumulator and drain into FoldCounts, but write folded output only once at the end.","status":"closed","priority":1,"issue_type":"task","created_at":"2026-06-01T01:01:18.503284722Z","created_by":"mjc","updated_at":"2026-06-01T01:01:23.924353800Z","closed_at":"2026-06-01T01:01:23.924161016Z","close_reason":"completed","source_repo":"pyroclast","source_repo_path":"/home/mjc/projects/pyroclast","compaction_level":0,"original_size":0} -{"id":"pyroclast-fij","title":"Close folded-output parity gap against perf script and Inferno","description":"Reverify current Pyroclast direct-fold output against perf script piped through inferno-collapse-perf. For every mismatch, inspect Inferno and perf source before changing behavior; do not optimize until byte/line parity is understood.","notes":"2026-06-01 verification after cherry-picking codex/parity-continue onto main: earlier branch note was stale for current main. Rebuilt from target/profiling-runs/octo-symbolized-fold/profile.raw.perf.data. Real oracle /tmp/octo-real-via-bench.perf-script | inferno-collapse-perf is 199 folded lines / 6,391,206,062 period. After matching perf/libdw's behavior of truncating user unwinds at the first unreportable frame, current Pyroclast period-weighted direct fold is 396 folded lines / 381,084,541,772 period. Therefore stack shape improved from the stale 639-line note, but the remaining parity gap is still the period overcount: Pyroclast is still assigning all sample periods to non-empty stacks while Inferno drops perf-script events that have no stack lines. Continue by comparing concrete event blocks against perf script, Inferno source, and perf unwind source before changing behavior.","status":"in_progress","priority":1,"issue_type":"task","created_at":"2026-06-01T01:06:28.720343601Z","created_by":"mjc","updated_at":"2026-06-05T09:28:20.966220923Z","source_repo":"pyroclast","source_repo_path":"/home/mjc/projects/pyroclast","compaction_level":0,"original_size":0,"comments":[{"id":1,"issue_id":"pyroclast-fij","author":"mjc","text":"2026-06-01: fixed red parity tests in worktree /home/mjc/worktrees/pyroclast-red-tests branch codex/red-tests. PERF_SAMPLE_IP is now retained for IP-only/empty-callchain folding fallback; inferno frame rendering strips raw symbol +0x suffixes while preserving mapped offsets. Verified cargo nextest run and cargo clippy --all-targets -- -D warnings under nix develop.","created_at":"2026-06-01T01:26:48Z"},{"id":2,"issue_id":"pyroclast-fij","author":"mjc","text":"Committed 22ca3fc Model elfutils x86_64 arch fallback. Source-backed from elfutils x86_64 ebl_unwind frame-pointer fallback; verified full nextest, clippy pedantic, flake check. It fixes the confirmed period 803991 syscall-return sample class without reintroducing broad DSO fallback overcount. Remaining aggregate gap still needs event-by-event comparison against perf script and Inferno collapse.","created_at":"2026-06-05T06:50:23Z"},{"id":3,"issue_id":"pyroclast-fij","author":"mjc","text":"2026-06-05: tested current-IP base-symbol prefetch and rejected broad empty-callchain current-IP salvage on real octo perfdata. Broad rule overcounted badly (272 lines / 379,484,470,598 vs oracle 297 / 7,963,707,726), so it was backed out. Current measured state after rollback and base-prefetch guard: ./target/profiling/pyroclast plumbing fold --count-periods target/profiling-runs/octo-symbolized-fold/profile.raw.perf.data finishes in 22.59s and emits 140 lines / 4,871,062,486 vs fresh perf script | inferno-collapse-perf oracle 297 / 7,963,707,726. Remaining gap is missing accepted libdw initial/current frames and symbol/stack-shape differences, not broad event overcount.","created_at":"2026-06-05T08:01:16Z"},{"id":4,"issue_id":"pyroclast-fij","author":"mjc","text":"Committed decabb1 Match perf inline subprogram traversal. Source-backed from perf util/dwarf-aux.c cu_walk_functions_at/die_find_realfunc/die_find_child and machine.c append_inlines: nested DW_TAG_subprogram DIEs are not inline-chain frames. Verification: pre-commit passed rustfmt, clippy pedantic, full cargo nextest run 609/609, and flake check. Real octo symbolized perf-script lines improved from 200376 to 195407, but folded parity remains Pyroclast 140 lines / 4,871,062,486 vs perf script | inferno-collapse-perf oracle 297 lines / 7,963,707,726. Remaining largest gap is missing/short stacks, not nested-subprogram inline expansion.","created_at":"2026-06-05T08:25:43Z"},{"id":5,"issue_id":"pyroclast-fij","author":"mjc","text":"2026-06-05: committed 11b4ca6 Match Inferno Rust symbol normalization and fe118b0 Match perf syscall return unwind truncation. Source-backed from Inferno collapse/common.rs fix_partially_demangled_rust_symbol before tidy_generic, perf util/unwind-libdw.c frame_callback->entry, elfutils dwfl_thread_getframes callback-before-unwind, and perf -v octo trace showing syscall-return executable frames stop after unwind failure. Verification for commits: pre-commit passed rustfmt, clippy pedantic, full cargo nextest run, and flake check. Fresh same-binary octo comparison after fe118b0: Pyroclast 111 folded lines / 4,871,062,486 vs perf script | inferno-collapse-perf 237 / 6,731,538,757; exact common lines 33 / 916,496,279, stack-key common weighted overlap 3,431,601,781. Rejected broader zero-frame current-IP salvage (raw empty + recorded mapping + module_count > 0 + framehop_count != 1): real export timed out at 90s, so it remains too broad. Remaining top missing class is perf leaf-only libc current-IP callbacks such as __memcmp_avx2_movbe, __memmove_avx_unaligned_erms, malloc/cfree; must find a narrower libdw-compatible discriminator before changing the gate.","created_at":"2026-06-05T09:28:20Z"}]} -{"id":"pyroclast-pkh","title":"Fix direct user unwind performance after removing synthetic fallbacks","description":"After d6df8e9, exits rc=124. Synthetic frame-pointer/libdw tail fallbacks were removed because perf source only emits frame_callback->entry frames, but real framehop user unwinding now dominates runtime. Need source-backed gate/cache to match perf script output without unwinding samples that perf/libdw would leave empty.","notes":"2026-06-05 reverified current-frame gap against perf/inferno. Exact oracle event: period 4598236, IP 0x7ffff7f01f40, perf script prints __memcmp_avx2_movbe from glibc with empty recorded callchain. Pyroclast parses frames=[] user_regs=true user_stack=24064; framehop reports loaded_ip=true has_unwind=true but raw=[] framehop_count=0. A naive source-looking rule (empty framehop + DSO + CFI => keep current IP) was tested and rejected on the real file: Pyroclast jumped to 140 lines / 41,230,188,484 period vs oracle 309 / 8,545,873,202, with huge glibc leaf overcount (realloc/memset/memmove). Do not commit that rule; fix needs libdw-compatible current-frame callback modeling or improved unwinder behavior.","status":"open","priority":1,"issue_type":"task","created_at":"2026-06-05T05:17:24.556771065Z","created_by":"mjc","updated_at":"2026-06-05T05:38:32.379408491Z","source_repo":"pyroclast","source_repo_path":"/home/mjc/projects/pyroclast","compaction_level":0,"original_size":0} +{"id": "pyroclast-5gr", "title": "Model libdw current-frame-only callbacks for perf parity", "description": "Perf/inferno parity gap: perf script emits some inline-only current-IP stacks where Pyroclast/framehop has no accepted object frames. Source: perf util/unwind-libdw.c frame_callback->entry and elfutils libdwfl/dwfl_frame.c calls callback before unwind; blank path likely goes through __report_module/module reporting, not stack-frame success. Latest oracle after 099acd6: Pyroclast 114 folded lines / 2,995,326,258 vs perf/inferno 144 / 3,212,226,637. Need source-backed libdw-compatible current-frame model or a pluggable libdw oracle; avoid broad inline current-IP salvage because 3115296 overcounted to 323 lines / 10.1B.\n\n2026-06-11: full source-backed model written to .ace-research-perf-unwind.md \u2014 libdwfl fires the frame callback once for the seeded IP before unwinding and perf keeps partial stacks; emit the leaf (plus inlines) when the initial IP reported into a module AND !has_unwind_info_for_ip(ip) AND (x86_64: bp < sp / aarch64: lr == 0). The hook is object_unwind_initial_frame_policy, currently ignored in perf_accepted_object_unwind_frames. Needs an x86_64 oracle to validate the original 114-vs-144 gap; the same predicate evaluated before unwinding is the pkh skip gate.", "status": "closed", "priority": 1, "issue_type": "task", "created_at": "2026-06-02T12:41:11.923032472Z", "created_by": "mjc", "updated_at": "2026-06-11T20:15:25.227301000Z", "source_repo": "pyroclast", "source_repo_path": "/home/mjc/projects/pyroclast", "compaction_level": 0, "original_size": 0, "close_reason": "Closed by 62e3d62: perf_accepted_object_unwind_frames now emits the libdw scenario-D leaf when the initial IP reported into a module, no FDE covers it (!has_unwind_info_for_ip), and the arch ebl_unwind fallback provably cannot advance (x86_64 bpentry, elfutils dwfl_thread_getframes callback-before-unwind, and perf -v octo trace showing syscall-return executable frames stop after unwind failure. Verification for commits: pre-commit passed rustfmt, clippy pedantic, full cargo nextest run, and flake check. Fresh same-binary octo comparison after fe118b0: Pyroclast 111 folded lines / 4,871,062,486 vs perf script | inferno-collapse-perf 237 / 6,731,538,757; exact common lines 33 / 916,496,279, stack-key common weighted overlap 3,431,601,781. Rejected broader zero-frame current-IP salvage (raw empty + recorded mapping + module_count > 0 + framehop_count != 1): real export timed out at 90s, so it remains too broad. Remaining top missing class is perf leaf-only libc current-IP callbacks such as __memcmp_avx2_movbe, __memmove_avx_unaligned_erms, malloc/cfree; must find a narrower libdw-compatible discriminator before changing the gate.", "created_at": "2026-06-05T09:28:20Z"}]} +{"id": "pyroclast-pkh", "title": "Fix direct user unwind performance after removing synthetic fallbacks", "description": "After d6df8e9, exits rc=124. Synthetic frame-pointer/libdw tail fallbacks were removed because perf source only emits frame_callback->entry frames, but real framehop user unwinding now dominates runtime. Need source-backed gate/cache to match perf script output without unwinding samples that perf/libdw would leave empty.\n\n2026-06-11 status: PERF-1 fixed \u2014 DWARF inline-frame indexes are now cached per object across fold rounds (CachedObjectMetadata), removing the dominant re-parse cost; inline expansion is also off by default, taking the DWARF walk out of the plain-fold path entirely. The cheap leaf-only skip gate from .ace-research-perf-unwind.md S3 (per-(pid,ip) SkipUnwind|LeafOnly|MustUnwind classification) is still open.", "notes": "2026-06-05 reverified current-frame gap against perf/inferno. Exact oracle event: period 4598236, IP 0x7ffff7f01f40, perf script prints __memcmp_avx2_movbe from glibc with empty recorded callchain. Pyroclast parses frames=[] user_regs=true user_stack=24064; framehop reports loaded_ip=true has_unwind=true but raw=[] framehop_count=0. A naive source-looking rule (empty framehop + DSO + CFI => keep current IP) was tested and rejected on the real file: Pyroclast jumped to 140 lines / 41,230,188,484 period vs oracle 309 / 8,545,873,202, with huge glibc leaf overcount (realloc/memset/memmove). Do not commit that rule; fix needs libdw-compatible current-frame callback modeling or improved unwinder behavior.", "status": "closed", "priority": 1, "issue_type": "task", "created_at": "2026-06-05T05:17:24.556771065Z", "created_by": "mjc", "updated_at": "2026-06-11T20:15:25.227301000Z", "source_repo": "pyroclast", "source_repo_path": "/home/mjc/projects/pyroclast", "compaction_level": 0, "original_size": 0, "close_reason": "Closed by 62e3d62: the leaf-only predicate is classified BEFORE framehop (SkipUnwind|LeafOnly|MustUnwind), so leaf-only and zero-frame (KernelWithUserFrame) samples skip the framehop unwind entirely. Backed by an exact-ip CFI-presence memo on FramehopUnwinder and a per-(pid,ip) leaf-only eligibility memo on PidUnwindState, both invalidated with the pid's unwind state on mapping change/fork. PERF-4 redundant re-unwind removed (report distinguishes newly-loaded from already-present modules). The gate is a pure optimization: gated and ungated paths share the acceptance tail and are byte-identical.", "closed_at": "2026-06-11T20:15:25.227301000Z"} diff --git a/.gitignore b/.gitignore index e7de11e..12d75ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target /pyroclast-runs +/.claude/ diff --git a/Cargo.lock b/Cargo.lock index 488fce2..265861c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,12 +495,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "id-arena" version = "2.3.0" @@ -721,27 +715,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "procfs" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" -dependencies = [ - "bitflags", - "procfs-core", - "rustix", -] - -[[package]] -name = "procfs-core" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" -dependencies = [ - "bitflags", - "hex", -] - [[package]] name = "proptest" version = "1.11.0" @@ -776,7 +749,6 @@ dependencies = [ "inferno", "memmap2", "object", - "procfs", "proptest", "rustc-hash", "serde", diff --git a/Cargo.toml b/Cargo.toml index 5afe7e0..7ec8db3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,6 @@ rustc-hash = "2.1.2" serde = { version = "1", features = ["derive"] } serde_json = "1" -[target.'cfg(target_os = "linux")'.dependencies] -procfs = { version = "0.18.0", default-features = false } - [target.'cfg(unix)'.dependencies] signal-hook = "0.3" diff --git a/docs/parity-findings.md b/docs/parity-findings.md new file mode 100644 index 0000000..a8a0414 --- /dev/null +++ b/docs/parity-findings.md @@ -0,0 +1,213 @@ +# Perf-script parity: findings, bugs, and performance issues + +Status as of 2026-06-11. Goal: `pyroclast plumbing fold|flamegraph` fully replaces +`perf script | inferno-collapse-perf | inferno-flamegraph`. + +Companion deep-dives produced during this investigation: + +- `.ace-research-perf-unwind.md` — source-cited model of perf + elfutils libdw user + unwinding (the decision table for 0 / 1 / N frames per sample, and the gate for + skipping unwinds perf would never attempt). +- `.ace-review-findings.md` — full code review (12 performance, 7 correctness findings). + +## Oracle harness + +`scripts/perf-oracle` (new) builds a Docker image (Ubuntu + modern perf + inferno), +records `dwarf` and `fp` call-graph profiles of `scripts/oracle/workload.rs`, exports +real `perf script` text and `inferno-collapse-perf` folds, then runs pyroclast against +the same `perf.data` inside the container. Artifacts land in `target/oracle/`. perf +cannot run on macOS, so this is the only local ground truth; previous parity numbers in +`.beads/issues.jsonl` came from an x86_64 Linux machine that is not this one. + +Important variance pinned down: **symbol naming depends on the perf build**. Debian +trixie's perf 6.12 does not demangle Rust v0 (`_RNv...` stays raw) while modern perf +(>= 6.16) and pyroclast demangle it. The oracle image deliberately uses a modern perf. +Byte parity is only meaningful against a pinned perf version; record +`target/oracle/perf.version` with any saved numbers. + +## Status update (later on 2026-06-11) + +The six gaps below are FIXED and merged: `pyroclast plumbing perf-script` output is +now byte-identical to `perf script --force` on the fp oracle, and the folded output +differs only where `inferno-collapse-perf` itself mis-parses the space-containing +DSO path `/ (deleted)` (it keeps the `+0x9c` offset and a trailing space on +`__libc_start_main` and emits `[unknown] `); pyroclast's folding of those frames is +deliberately the more correct one. Two genuine parser bugs fell out of this work: +the perf feature bitmap was read at byte offset 56 instead of 72 (silently +disabling HEADER_EVENT_DESC and header build-ids — `struct perf_file_header` +places `adds_features` after the three file sections), and feature-section +build-id records (which carry `header.type == 0`) were rejected. + +Dwarf note: the dwarf oracle's `perf script` output contains `(inlined)` frames — +modern perf expands inline frames by default for DWARF-symbolized stacks — so the +dwarf comparison runs pyroclast with `--inline`. The remaining dwarf divergence is +the aarch64 unwind support (in progress; spec in `.ace-aarch64-unwind-spec.md`). + +## Parity gaps found via the oracle (fp call-graph path, arch-independent) + +Measured by diffing `target/oracle/fp.pyroclast.script` against `fp.perf.script` +(same `perf.data`, recorded with `--call-graph fp`): + +1. **Event name suffix missing.** perf prints `task-clock:ppp:`; pyroclast prints + `task-clock:`. perf script takes event names from the HEADER_EVENT_DESC feature; + pyroclast reconstructs from the attr and loses the precision modifiers. +2. **Symbolized frames print `([unknown])` instead of the DSO path.** Multi-label and + callchain frames go through a writer that hardcodes the unknown DSO; perf prints + the mapping's long path for every mapped frame. +3. **Double symbol offset.** Lines like `quicksort+0x854+0x0` — a label that already + carries `+0xNNN` gets a second offset appended in the symbolized script path. +4. **Inline expansion on by default.** With debuginfo present pyroclast expands one + address into multiple DWARF inline frames (DIE names like + `catch_unwind`); plain + `perf script` (the stated replacement target — no `--inline`) prints exactly one + symtab-named line per callchain entry. The expansion mirrors `perf script --inline` + and should be opt-in. +5. **Dropped callchain entry.** A real frame (`...+0x6cb`, adjacent-but-distinct ip to + its neighbor) disappears from pyroclast's output; perf does not dedupe entries. +6. **Basename instead of full DSO path** for unsymbolized mapped frames + (`[unknown] (libc.so.6)` vs `[unknown] (/usr/lib/aarch64-linux-gnu/libc.so.6)`). + +These six are being fixed against the oracle byte-diff (in progress). The base symtab +naming itself (candidate selection, interval lookup, offset formatting) already matches +perf — prior commits got that right. + +## Architecture gap: aarch64 DWARF unwind — RESOLVED + +aarch64 DWARF user unwinding is now wired through the fold path (PerfUserRegs +per-arch decoding via HEADER_ARCH, per-arch framehop unwinders, arch-gated no-CFI +fallbacks; the aarch64 fallback fires when framehop yields only the seed pc, since +elfutils' aarch64 ebl_unwind recovers a caller from lr with no fp>=sp +precondition). The dwarf oracle's call spine now matches perf frame-for-frame by +address. Remaining dwarf divergence is inline-NAME parity, not unwinding: + +- perf expands more leaf inline frames at some IPs than pyroclast, and marks them + `(inlined)`; +- inline name spelling: pyroclast emits DWARF DIE names (`sort bool>`), perf's srcline backend emits qualified names + (`core::slice::sort::unstable::sort`); +- perf prints trailing `[unknown]` frames for PAC-tagged return addresses that + framehop strips. + +### Inline-name parity — RESOLVED (later 2026-06-11) + +All four inline gaps closed: names come from DW_AT_linkage_name demangled the way +perf itself demangles (its external addr2line runs without -C), sampled-IP leaves +expand through the full inline chain (perf runs append_inlines on every accepted +entry — the earlier "over-expansion" read was actually under-expansion elsewhere), +inline script lines render `sym+0xoff (inlined)` sharing the base frame's offset, +and `[kernel.kallsyms]` frames resolve from live /proc/kallsyms when +/sys/kernel/notes matches the recorded kernel build-id. The dwarf oracle script and +folded outputs now match perf byte-for-byte EXCEPT perf's PAC-tagged `[unknown]` +frames (perf prints aarch64 lr values without stripping pointer-auth bits; +pyroclast/framehop strips them — intentional divergence, arguably a perf bug). +A `.debug_str`-based generic specialization was removed from the inline path: it +rewrote qualified names into spellings perf never prints. + +### Historical note (pre-fix analysis) + +perf's inline-frame names depend on which srcline backend its build uses: libbfd, +libllvm, libdw, or an external `addr2line` subprocess. The Ubuntu oracle perf uses +the external addr2line backend and prints fully-qualified, v0-demangled names +(`std::panicking::catch_unwind::`) +with `(inlined)` in the DSO column and symbol offsets on base frames. pyroclast's +DIE-walking resolver (built to match a libdw-backed perf in earlier commits +decabb1/bfb75c4) emits bare `DW_AT_name` spellings (`catch_unwind`) +without qualification. Decision: align with the qualified-name behavior (it is the +modern, measurable oracle here and the more useful output) and treat the libdw +spelling as documented variance. Also observed: pyroclast expands inline frames at +one return address where perf does not (suspect a missing pc-1 adjustment on +non-leaf inline lookups), and kernel frames currently fold as `[[kernel.kallsyms]]` +because kallsyms symbolization isn't wired into the direct fold. + +## Original note (pre-fix) + +The user-stack unwind model is x86_64-only (`PerfX86_64Regs`, rbp/rsp heuristics, +x86_64 elfutils arch fallback). On arm64 perf.data with `--call-graph dwarf`, pyroclast +folds almost nothing (5 samples vs the full set; bench: 2 folded lines vs oracle 15). +Local development on Apple Silicon records arm64 in Docker, so this blocks oracle-driven +work on the dwarf path. Needs: per-arch reg-mask decoding (the regs are already read by +mask), framehop's aarch64 unwinder, and the aarch64 `ebl_unwind` (x29 chain) analogue. +Tracked as follow-up; the fp path works on any arch. + +## The two .beads parity issues — CLOSED (leaf-only model landed) + +pyroclast-5gr and pyroclast-pkh are closed. The leaf-only predicate is implemented +exactly as sourced: emit (or truncate to) the single seeded-IP leaf when the +initial IP reported into a module, no FDE covers it, and the arch ebl fallback +cannot advance (x86_64 `bp < sp` per backends/x86_64_unwind.c's `sp >= fp+16` +guard; aarch64 `lr == 0`). When CFI covers the ip the sample stays MustUnwind — +an FDE row with undefined RA (clean leaf) and one with a real caller are +indistinguishable without unwinding (libdwfl handle_cfi). The same classification +runs before framehop (SkipUnwind | LeafOnly | MustUnwind, memoized per (pid, ip)), +CFI presence is memoized per ip, and the module-report retry loop re-unwinds only +when a module actually loaded (PERF-4). The unsourced .so-vs-exe initial-frame +policy was removed. Validation caveat: the original 114-vs-144 folded-line gap was +measured against an x86_64 perf.data we cannot regenerate locally; the model is +test-pinned to the cited elfutils/perf sources and the arm64 oracle is unchanged, +but re-running the original x86_64 comparison on a Linux x86 box remains the final +confirmation. + +## Original analysis (historical) + +From `.ace-research-perf-unwind.md`: libdwfl always fires the frame callback once for +the sampled IP before unwinding, and perf keeps partial stacks. So: + +- **pyroclast-5gr** (current-IP-only stacks): pyroclast must emit the single leaf + (plus inlines when enabled) when the initial IP reported into a module and neither + CFI (`has_unwind_info_for_ip`) nor the rbp fallback (`bp >= sp`) can produce a + caller. The hook exists (`object_unwind_initial_frame_policy`) but is currently + ignored in `perf_accepted_object_unwind_frames`. +- **pyroclast-pkh** (unwind dominates runtime): the same predicate, evaluated *before* + unwinding, lets pyroclast skip framehop entirely for leaf-only samples. Recommended + caches: per-evsel attr bits, per-(module, page) CFI presence, and a per-(pid, ip) + `SkipUnwind | LeafOnly | MustUnwind` classification. + +## Performance issues (from code review; see `.ace-review-findings.md` for all 12) + +- **PERF-1 (critical, likely the rc=124 root cause):** `PerfDwarfNameResolver` fully + re-parses each DSO's DWARF (object parse + DIE walk) on every fold round — + `CachedObjectMetadata` caches symbols but not parsed DWARF. Making inline expansion + opt-in removes this from the default path; the cache is still worth adding for + `--inline`. +- **PERF-2/3 (fixed):** mmap ingestion re-scanned and re-indexed the whole mapping + table per record (two independent O(n²) patterns). Now incremental via the per-pid + interval index; overlap splits keep the perf-faithful path. +- **PERF-4:** per-sample stacks can be re-unwound up to 9× by the module-report + convergence loop (`MAX_LIBDW_CALLBACK_REPORT_PASSES`); skip the redundant final pass + when no new module was reported. + +## Correctness bugs + +- **CORR-1:** `file_matches_recorded_identity` compares only inode, ignoring device + major/minor — wrong-binary symbolization is possible across filesystems. +- **CORR-2:** the same file can get two `symbol_source_id`s depending on mmap record + form, defeating symbol-cache dedup (amplifies PERF-1). +- **macOS `cargo pyroclast` was broken (fixed):** package discovery compared a + canonicalized crate root against cargo's un-canonicalized manifest paths, failing + through `/var -> /private/var`. +- **Silent FakeBackend fallback:** on macOS, `pyroclast memory|latency|offcpu` quietly + run the fake backend and write fake artifacts instead of erroring out as unsupported. + +## Test-suite portability (macOS/aarch64 host) — RESOLVED + +22 of 621 tests originally failed on a fresh macOS machine (the suite was developed +on x86_64 Linux). All fixed: cargo metadata path canonicalization (7 `cargo_cli`), +explicit platform injection through the existing `_on_platform` entry points +(8 `run_cli` + the FakeBackend-exposing e2e test), a portable `read_dir` procfs walk +(2 `platform`, and the `procfs` dependency is gone), a canonicalized expectation in +the NixOS System.map test, and a synthetic x86_64 ELF fixture replacing +`std::env::current_exe()` in the 5 current-IP-only unwind tests (the host test +binary's Mach-O `__unwind_info` recovered callers a Linux ELF would not). The suite +is now fully green on macOS: 641/641, clippy clean. + +Note for fixture authors: framehop applies a frame-pointer fallback for addresses +OUTSIDE any known module, but stops at uncovered addresses INSIDE a module that has +CFI — synthetic unwind fixtures must include an eh_frame (even one whose only FDE +covers an unrelated range) to pin the no-coverage behavior. + +## Environment notes + +- AGENTS.md says nix, but this machine has no nix; everything here runs with rustup + cargo + Docker. The pre-commit hook (`.githooks`) is configured by the nix shellHook + and is not active here; `nix flake check` in it cannot run without nix. +- `inferno` and `cargo-nextest` are installed via `cargo install` on this machine. diff --git a/scripts/oracle-compare b/scripts/oracle-compare new file mode 100755 index 0000000..34c87ad --- /dev/null +++ b/scripts/oracle-compare @@ -0,0 +1,42 @@ +#!/usr/bin/env sh +# Fast oracle iteration: rebuild pyroclast from THIS worktree inside the oracle +# container and regenerate only the *.pyroclast.* artifacts + bench reports +# against the already-recorded oracle inputs in the MAIN repo's target/oracle. +# Prints the fp script/folded diffs and the inferno_compare scoreboard. +# +# Does NOT re-record perf.data (use scripts/perf-oracle for a full re-record). +# The recorded oracle inputs (*.perf.data, *.perf.script, *.inferno.folded) +# are never overwritten by this path. +# +# CARGO_TARGET_DIR lives under /oracle-out/target so release builds are +# incremental across runs. +set -eu + +# This worktree (the code under test). +worktree="$(git rev-parse --show-toplevel)" +# The MAIN checkout that owns the recorded oracle artifacts. Override with +# ORACLE_MAIN_REPO=/path/to/main if your main checkout is elsewhere. The +# common dir is "
/.git" from any worktree and plain ".git" from the main +# checkout itself. +if [ -n "${ORACLE_MAIN_REPO:-}" ]; then + main_repo="$ORACLE_MAIN_REPO" +else + common_dir="$(git rev-parse --path-format=absolute --git-common-dir)" + main_repo="$(dirname "$common_dir")" +fi +out="$main_repo/target/oracle" + +if [ ! -d "$out" ]; then + echo "oracle inputs not found at $out" >&2 + echo "run scripts/perf-oracle once to record them, or set ORACLE_MAIN_REPO" >&2 + exit 1 +fi + +docker build -t pyroclast-oracle -f "$worktree/scripts/oracle/Dockerfile" "$worktree/scripts/oracle" + +exec docker run --rm \ + -v "$worktree:/work" \ + -v "$out:/oracle-out" \ + -v pyroclast-oracle-cargo-registry:/usr/local/cargo/registry \ + -e "ORACLE_NAMES=${ORACLE_NAMES:-fp dwarf}" \ + pyroclast-oracle bash /work/scripts/oracle/compare-in-container.sh diff --git a/scripts/oracle/Dockerfile b/scripts/oracle/Dockerfile new file mode 100644 index 0000000..f29ee04 --- /dev/null +++ b/scripts/oracle/Dockerfile @@ -0,0 +1,16 @@ +# Ubuntu so the oracle perf is modern (>= 6.16 demangles Rust v0 symbols the +# same way pyroclast does); Debian trixie's perf 6.12 leaves _R symbols raw. +FROM ubuntu:25.10 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential ca-certificates curl elfutils linux-perf \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain stable +ENV PATH=/root/.cargo/bin:$PATH + +RUN cargo install inferno --locked + +WORKDIR /work diff --git a/scripts/oracle/compare-in-container.sh b/scripts/oracle/compare-in-container.sh new file mode 100755 index 0000000..14cfa88 --- /dev/null +++ b/scripts/oracle/compare-in-container.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Runs inside the oracle container: rebuilds pyroclast incrementally (target +# under $ORACLE_OUT so builds persist across runs), regenerates ONLY the +# *.pyroclast.* artifacts and bench reports, and prints the script/folded +# diffs against the recorded oracle inputs. Does NOT re-record perf.data and +# does NOT touch the recorded oracle inputs (*.perf.data, *.perf.script, +# *.inferno.folded). +set -euo pipefail + +ORACLE_OUT="${ORACLE_OUT:-/oracle-out}" +REPO="${REPO:-/work}" +export CARGO_TARGET_DIR="$ORACLE_OUT/target" + +# The recorded perf.data references the workload at /tmp/oracle-workload (by +# path) and the system DSOs that existed at record time. In a fresh compare +# container those DSOs are gone, so symbolization falls back to module names. +# Rebuild the workload deterministically with the same rustc flags used by the +# recording step so pyroclast can resolve its symbols by path. +if [ -f "$REPO/scripts/oracle/workload.rs" ] && [ ! -x /tmp/oracle-workload ]; then + rustc -O -Cdebuginfo=2 -o /tmp/oracle-workload "$REPO/scripts/oracle/workload.rs" +fi + +# perf recorded the C library mapping as "/ (deleted)" (the file was unlinked +# during the run) but symbolized it at record time from the live mapping, and +# it stored the DSO build-ids in the HEADER_BUILD_ID feature. pyroclast resolves +# build-ids out of ~/.debug/.build-id///elf, so seed that cache from +# the container's on-disk DSOs (whose build-ids match the recording, since the +# record and compare containers share the same image). Without this the deleted +# libc cannot be symbolized and shows up as the lone residual diff. +seed_build_id_cache() { + local dso="$1" + [ -e "$dso" ] || return 0 + local id + id=$(readelf -n "$dso" 2>/dev/null | awk '/Build ID:/ {print $3; exit}') + [ -n "$id" ] || return 0 + local dir="$HOME/.debug/.build-id/${id:0:2}/${id:2}" + mkdir -p "$dir" + ln -sf "$dso" "$dir/elf" +} +for dso in /usr/lib/*/libc.so.6 /usr/lib/*/ld-linux-*.so.* /usr/lib/*/libgcc_s.so.1; do + seed_build_id_cache "$dso" +done + +cd "$REPO" +cargo build --quiet --release --bin pyroclast --example pyroclast-bench + +for name in ${ORACLE_NAMES:-fp dwarf}; do + [ -f "$ORACLE_OUT/$name.perf.data" ] || continue + # The dwarf oracle was recorded on arm64 with `perf script` expanding DWARF + # inline frames by default; the fp oracle was recorded without inline. Fold + # the dwarf pair with --inline so the frame counts line up; leave fp plain. + inline_args="" + if [ "$name" = "dwarf" ]; then + inline_args="--inline" + fi + timeout 600 "$CARGO_TARGET_DIR/release/examples/pyroclast-bench" \ + "$ORACLE_OUT/$name.perf.data" \ + --perf-script "$ORACLE_OUT/$name.perf.script" \ + --symbols $inline_args \ + | tee "$ORACLE_OUT/$name.bench.txt" \ + || echo "pyroclast-bench failed for $name (continuing)" >&2 + # --count-periods matches the scoreboard (benchmark_fold_options) so the + # printed folded diff lines up with inferno's period-weighted counts. + timeout 600 "$CARGO_TARGET_DIR/release/pyroclast" plumbing fold --count-periods $inline_args \ + "$ORACLE_OUT/$name.perf.data" > "$ORACLE_OUT/$name.pyroclast.folded" \ + || echo "plumbing fold failed for $name (continuing)" >&2 + timeout 600 "$CARGO_TARGET_DIR/release/pyroclast" plumbing perf-script $inline_args \ + "$ORACLE_OUT/$name.perf.data" > "$ORACLE_OUT/$name.pyroclast.script" \ + || echo "plumbing perf-script failed for $name (continuing)" >&2 +done + +for name in ${ORACLE_NAMES:-fp dwarf}; do + [ -f "$ORACLE_OUT/$name.pyroclast.script" ] || continue + echo "================ $name script diff (perf vs pyroclast) ================" + diff "$ORACLE_OUT/$name.perf.script" "$ORACLE_OUT/$name.pyroclast.script" | head -50 || true + echo "================ $name folded diff (inferno vs pyroclast) =============" + diff <(sort "$ORACLE_OUT/$name.inferno.folded") <(sort "$ORACLE_OUT/$name.pyroclast.folded") | head -50 || true + echo "================ $name bench scoreboard ===============================" + grep -E 'inferno_compare\.(matches|only_)' "$ORACLE_OUT/$name.bench.txt" || true +done diff --git a/scripts/oracle/record-in-container.sh b/scripts/oracle/record-in-container.sh new file mode 100755 index 0000000..702d87b --- /dev/null +++ b/scripts/oracle/record-in-container.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Runs inside the oracle container: records perf.data from the sample +# workload, exports perf script text, and folds it with inferno-collapse-perf. +# Writes ONLY the recorded oracle inputs (*.perf.data, *.perf.script, +# *.inferno.folded) plus perf.version. Does NOT build or run pyroclast — use +# compare-in-container.sh for the fast pyroclast iteration path. +set -euo pipefail + +ORACLE_OUT="${ORACLE_OUT:-/oracle-out}" +REPO="${REPO:-/work}" + +mkdir -p "$ORACLE_OUT" +# Ubuntu's /usr/bin/perf wrapper insists on a kernel-matched build; call the +# packaged binary directly since any modern perf works for the oracle. +if ! perf version >/dev/null 2>&1; then + PERF_BIN="$(find /usr/lib/linux-tools* -name perf -type f 2>/dev/null | head -n 1)" + [ -n "$PERF_BIN" ] || { echo "no perf binary found" >&2; exit 1; } + perf() { "$PERF_BIN" "$@"; } +fi +perf version | tee "$ORACLE_OUT/perf.version" + +rustc -O -Cdebuginfo=2 -o /tmp/oracle-workload "$REPO/scripts/oracle/workload.rs" + +sysctl -w kernel.perf_event_paranoid=-1 >/dev/null 2>&1 || true +sysctl -w kernel.kptr_restrict=0 >/dev/null 2>&1 || true + +record() { + local name="$1" + shift + perf record -o "$ORACLE_OUT/$name.perf.data" "$@" -- /tmp/oracle-workload >/dev/null + perf script -i "$ORACLE_OUT/$name.perf.data" > "$ORACLE_OUT/$name.perf.script" + inferno-collapse-perf "$ORACLE_OUT/$name.perf.script" > "$ORACLE_OUT/$name.inferno.folded" +} + +record dwarf -F 997 --call-graph dwarf,16384 +record fp -F 997 --call-graph fp diff --git a/scripts/oracle/run-in-container.sh b/scripts/oracle/run-in-container.sh new file mode 100755 index 0000000..3e16c30 --- /dev/null +++ b/scripts/oracle/run-in-container.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Runs inside the oracle container: records the oracle inputs and then runs the +# pyroclast comparison. This is the full path (record + compare); for fast +# iteration that skips re-recording use compare-in-container.sh (driven from +# the host by scripts/oracle-compare). +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +bash "$here/record-in-container.sh" +bash "$here/compare-in-container.sh" diff --git a/scripts/oracle/workload.rs b/scripts/oracle/workload.rs new file mode 100644 index 0000000..28e2059 --- /dev/null +++ b/scripts/oracle/workload.rs @@ -0,0 +1,51 @@ +// Oracle workload: mixes inlined helpers, recursion, allocation, and libc +// calls so perf records stacks with inline frames, deep user stacks, and +// shared-library frames. + +use std::hint::black_box; + +#[inline(always)] +fn mix(value: u64) -> u64 { + value + .wrapping_mul(0x9e37_79b9_7f4a_7c15) + .rotate_left(31) + .wrapping_add(0x517c_c1b7_2722_0a95) +} + +#[inline(always)] +fn mix_twice(value: u64) -> u64 { + mix(mix(value)) +} + +fn recurse(depth: u32, seed: u64) -> u64 { + if depth == 0 { + let mut acc = seed; + for index in 0..512 { + acc = mix_twice(acc ^ index); + } + acc + } else { + recurse(depth - 1, mix(seed)).wrapping_add(depth.into()) + } +} + +fn churn_allocations(rounds: usize) -> u64 { + let mut acc = 0u64; + for round in 0..rounds { + let mut values: Vec = (0..2048).map(|i| mix(i ^ round as u64)).collect(); + values.sort_unstable(); + let text = format!("round-{round}-{}", values[values.len() / 2]); + acc = acc.wrapping_add(text.len() as u64).wrapping_add(values[0]); + black_box(&values); + } + acc +} + +fn main() { + let mut total = 0u64; + for round in 0..200u64 { + total = total.wrapping_add(recurse(24, round)); + total = total.wrapping_add(churn_allocations(8)); + } + println!("{total}"); +} diff --git a/scripts/perf-oracle b/scripts/perf-oracle new file mode 100755 index 0000000..3ebe303 --- /dev/null +++ b/scripts/perf-oracle @@ -0,0 +1,17 @@ +#!/usr/bin/env sh +# Builds the Linux oracle container and produces perf.data / perf script / +# folded oracle artifacts under target/oracle, then runs the pyroclast +# comparison inside the container. Requires Docker. +set -eu + +repo="$(git rev-parse --show-toplevel)" +out="$repo/target/oracle" +mkdir -p "$out" + +docker build -t pyroclast-oracle -f "$repo/scripts/oracle/Dockerfile" "$repo/scripts/oracle" + +exec docker run --rm --privileged \ + -v "$repo:/work" \ + -v "$out:/oracle-out" \ + -v pyroclast-oracle-cargo-registry:/usr/local/cargo/registry \ + pyroclast-oracle bash /work/scripts/oracle/run-in-container.sh diff --git a/src/backends/linux_perf.rs b/src/backends/linux_perf.rs index abdc891..1aa12a6 100644 --- a/src/backends/linux_perf.rs +++ b/src/backends/linux_perf.rs @@ -314,8 +314,13 @@ pub(crate) fn fold_linux_perfdata( where R: CommandRunner, { + // The profile command produces human-facing flamegraphs and keeps the + // DWARF inline-frame expansion it has always emitted (like + // `perf script --inline`). The byte-for-byte `plumbing perf-script`/fold + // parity paths default inline off; this convenience command does not. let options = FoldOptions { count_periods: true, + inline: true, }; if symbols { let symbol_resolver = diff --git a/src/benchmarks.rs b/src/benchmarks.rs index 6757657..e199a58 100644 --- a/src/benchmarks.rs +++ b/src/benchmarks.rs @@ -30,6 +30,12 @@ pub struct BenchArgs { pub export_perf_script: Option, pub symbols: bool, + + /// Expand each callchain entry into its DWARF inline frames when folding, + /// like `perf script --inline`. The dwarf oracle's perf.script is recorded + /// with inline expansion, so the comparison must fold with inline on to + /// line up frame counts. + pub inline: bool, } impl BenchArgs { @@ -44,6 +50,8 @@ impl BenchArgs { parsed.export_perf_script = iter.next(); } else if arg.as_os_str() == "--symbols" { parsed.symbols = true; + } else if arg.as_os_str() == "--inline" { + parsed.inline = true; } else { parsed.perf_data = Some(arg); } @@ -122,9 +130,14 @@ where )); } - let report = - run_streaming_comparison_with_symbols(&input, perf_script.as_deref(), runner, args.symbols) - .map_err(|error| format!("inferno comparison failed: {error}"))?; + let report = run_streaming_comparison_with_symbols( + &input, + perf_script.as_deref(), + runner, + args.symbols, + args.inline, + ) + .map_err(|error| format!("inferno comparison failed: {error}"))?; Ok(format_bench_output(&report)) } @@ -160,7 +173,7 @@ fn append_bench_report(name: &str, report: &FoldBenchmarkReport, output: &mut St /// Returns an error when the input file cannot be mapped or parsed. pub fn run_fold_benchmark(input: &Path) -> Result { run_fold_benchmark_with_writer(input, |writer| { - write_folded_perfdata_file_with_options(input, benchmark_fold_options(), writer) + write_folded_perfdata_file_with_options(input, benchmark_fold_options(false), writer) }) } @@ -175,6 +188,7 @@ pub fn run_fold_benchmark_with_runner( input: &Path, runner: &R, symbols: bool, + inline: bool, ) -> Result where R: CommandRunner, @@ -184,7 +198,7 @@ where run_fold_benchmark_with_writer(input, |writer| { write_folded_perfdata_file_with_symbols( input, - benchmark_fold_options(), + benchmark_fold_options(inline), &resolver, writer, ) @@ -287,7 +301,7 @@ pub fn compare_with_inferno_collapse( where R: CommandRunner, { - compare_with_inferno_collapse_with_symbols(perf_data, perf_script, runner, false) + compare_with_inferno_collapse_with_symbols(perf_data, perf_script, runner, false, false) } /// Compares Pyroclast's direct folded stacks with the old @@ -304,15 +318,16 @@ pub fn compare_with_inferno_collapse_with_symbols( perf_script: &Path, runner: &R, symbols: bool, + inline: bool, ) -> Result where R: CommandRunner, { let pyroclast_folded = if symbols { let resolver = perf_symbol_resolver_for_current_home(runner, perf_data); - fold_perfdata_file_with_symbols(perf_data, benchmark_fold_options(), &resolver)? + fold_perfdata_file_with_symbols(perf_data, benchmark_fold_options(inline), &resolver)? } else { - fold_perfdata_file_with_options(perf_data, benchmark_fold_options())? + fold_perfdata_file_with_options(perf_data, benchmark_fold_options(inline))? }; let inferno_output = runner .run(&CommandSpec::new("inferno-collapse-perf").arg(perf_script.display().to_string())) @@ -356,6 +371,7 @@ pub fn run_streaming_comparison_with_symbols( perf_script: Option<&Path>, runner: &R, symbols: bool, + inline: bool, ) -> Result where R: CommandRunner + Sync, @@ -364,7 +380,7 @@ where let (pyro_tx, pyro_rx) = sync_channel(64); let (inferno_tx, inferno_rx) = sync_channel(64); let pyro_thread = - scope.spawn(move || run_pyroclast_stream(perf_data, runner, symbols, pyro_tx)); + scope.spawn(move || run_pyroclast_stream(perf_data, runner, symbols, inline, pyro_tx)); let inferno_thread = scope .spawn(move || run_inferno_stream(perf_data, perf_script, runner, symbols, inferno_tx)); let diff = compare_folded_line_receivers(&pyro_rx, &inferno_rx)?; @@ -628,6 +644,7 @@ fn run_pyroclast_stream( perf_data: &Path, runner: &R, symbols: bool, + inline: bool, line_sender: SyncSender, ) -> Result where @@ -647,12 +664,16 @@ where let resolver = perf_symbol_resolver_for_current_home(runner, &perf_data); write_folded_perfdata_file_with_symbols( &perf_data, - benchmark_fold_options(), + benchmark_fold_options(inline), &resolver, &mut writer, )?; } else { - write_folded_perfdata_file_with_options(&perf_data, benchmark_fold_options(), &mut writer)?; + write_folded_perfdata_file_with_options( + &perf_data, + benchmark_fold_options(inline), + &mut writer, + )?; } writer .flush() @@ -739,14 +760,14 @@ where perf_symbol_resolver_for_current_home(runner, &export_perf_data); write_inferno_perf_script_file_with_symbols( &export_perf_data, - benchmark_fold_options(), + benchmark_fold_options(false), &resolver, &mut stdin, ) } else { write_inferno_perf_script_file_with_options( &export_perf_data, - benchmark_fold_options(), + benchmark_fold_options(false), &mut stdin, ) }?; @@ -983,9 +1004,12 @@ fn join_result_thread( } } -fn benchmark_fold_options() -> FoldOptions { +fn benchmark_fold_options(inline: bool) -> FoldOptions { + // The benchmark scoreboard compares against plain `perf | inferno`, which + // does not expand DWARF inline frames, so the parity path keeps inline off. FoldOptions { count_periods: true, + inline, } } diff --git a/src/cargo_cli.rs b/src/cargo_cli.rs index cba93bb..c77067e 100644 --- a/src/cargo_cli.rs +++ b/src/cargo_cli.rs @@ -624,7 +624,17 @@ fn find_unique_target( .into_iter() .filter(|package_metadata| match package { Some(package) => package == package_metadata.name.as_str(), - None => package_metadata.manifest_path.starts_with(&crate_root), + None => { + // cargo metadata reports manifest paths as given, which can + // disagree with the canonicalized crate root through symlinks + // (macOS /var -> /private/var). + let manifest_path = package_metadata.manifest_path.as_std_path(); + manifest_path + .canonicalize() + .as_deref() + .unwrap_or(manifest_path) + .starts_with(&crate_root) + } }) .peekable(); @@ -1074,7 +1084,8 @@ mod tests { "--serve", ])); - crate::run_parsed_cargo_cli_with_runner(cli, &runner).expect("run cargo memory command"); + crate::run_parsed_cargo_cli_with_runner_on_platform(cli, &runner, "linux") + .expect("run cargo memory command"); assert_eq!( std::fs::read_to_string(out_dir.join("command.txt")).expect("command.txt"), diff --git a/src/cli.rs b/src/cli.rs index 61fcdee..120ea3e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -295,6 +295,11 @@ pub struct FoldArgs { #[arg(long = "no-symbols", action = ArgAction::SetFalse, default_value_t = true)] pub symbols: bool, + /// Expand each callchain entry into its DWARF inline frames, like + /// `perf script --inline`. Off by default, matching plain `perf script`. + #[arg(long)] + pub inline: bool, + #[arg(long, value_enum, default_value_t = SymbolizerKind::RustAddr2line)] pub symbolizer: SymbolizerKind, @@ -306,6 +311,11 @@ pub struct PerfScriptArgs { #[arg(long = "no-symbols", action = ArgAction::SetFalse, default_value_t = true)] pub symbols: bool, + /// Expand each callchain entry into its DWARF inline frames, like + /// `perf script --inline`. Off by default, matching plain `perf script`. + #[arg(long)] + pub inline: bool, + #[arg(long, value_enum, default_value_t = SymbolizerKind::RustAddr2line)] pub symbolizer: SymbolizerKind, @@ -330,6 +340,11 @@ pub struct FlamegraphArgs { #[arg(long = "no-symbols", action = ArgAction::SetFalse, default_value_t = true)] pub symbols: bool, + /// Expand each callchain entry into its DWARF inline frames, like + /// `perf script --inline`. Off by default, matching plain `perf script`. + #[arg(long)] + pub inline: bool, + #[arg(long, value_enum, default_value_t = SymbolizerKind::RustAddr2line)] pub symbolizer: SymbolizerKind, diff --git a/src/lib.rs b/src/lib.rs index 6bcb675..80fb43a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,6 @@ pub mod symbols; pub mod tools; use artifacts::ArtifactLayout; -use backends::fake::FakeBackend; use backends::heaptrack::HeaptrackBackend; use backends::linux_perf::LinuxPerfBackend; use backends::macos_xctrace::MacosXctraceBackend; @@ -111,6 +110,52 @@ where run_parsed_cli_with_runner_and_renderer(cli, runner, InfernoFlamegraphRenderer::new(runner)) } +/// Runs a parsed CLI command with an injected process runner and explicit +/// platform routing. +/// +/// # Errors +/// +/// Returns an error when command execution, artifact I/O, or input parsing +/// fails. +pub fn run_parsed_cli_with_runner_on_platform( + cli: Cli, + runner: &R, + platform: &str, +) -> backends::BackendResult +where + R: CommandRunner, +{ + run_parsed_cli_with_runner_and_renderer_on_platform( + cli, + runner, + InfernoFlamegraphRenderer::new(runner), + platform, + ) +} + +/// Runs a parsed cargo-subcommand command with an injected process runner and +/// explicit platform routing. +/// +/// # Errors +/// +/// Returns an error when cargo target resolution, command execution, artifact +/// I/O, or input parsing fails. +pub fn run_parsed_cargo_cli_with_runner_on_platform( + cli: cargo_cli::CargoCli, + runner: &R, + platform: &str, +) -> backends::BackendResult +where + R: CommandRunner, +{ + run_parsed_cargo_cli_with_runner_and_renderer_on_platform( + cli, + runner, + InfernoFlamegraphRenderer::new(runner), + platform, + ) +} + /// Runs a parsed cargo-subcommand command with an injected process runner. /// /// # Errors @@ -274,13 +319,34 @@ where cli::ProfileKind::Offcpu if platform == "linux" => { OffcpuBackend::new(runner).profile(&request)?; } - _ => { - FakeBackend.profile(&request)?; + cli::ProfileKind::Async => { + return Err(format!( + "{} profiling is not implemented yet", + profile_kind_name(request.kind) + ) + .into()); + } + kind => { + return Err(format!( + "{} profiling is not supported on {platform}", + profile_kind_name(kind) + ) + .into()); } } Ok(()) } +fn profile_kind_name(kind: cli::ProfileKind) -> &'static str { + match kind { + cli::ProfileKind::Cpu => "cpu", + cli::ProfileKind::Memory => "memory", + cli::ProfileKind::Offcpu => "off-cpu", + cli::ProfileKind::Latency => "latency", + cli::ProfileKind::Async => "async", + } +} + fn run_non_profile_command( command: CliCommand, runner: &R, @@ -316,6 +382,7 @@ where PlumbingCommand::Fold(command) => { let options = FoldOptions { count_periods: command.count_periods, + inline: command.inline, }; let stdout = fold_perfdata_for_cli( &command.input, @@ -330,8 +397,13 @@ where }) } PlumbingCommand::PerfScript(command) => { - let stdout = - perf_script_for_cli(&command.input, command.symbols, command.symbolizer, runner)?; + let stdout = perf_script_for_cli( + &command.input, + command.symbols, + command.inline, + command.symbolizer, + runner, + )?; Ok(CliOutput { stdout, stderr: String::new(), @@ -345,6 +417,7 @@ where &command.input, FoldOptions { count_periods: true, + inline: command.inline, }, command.symbols, command.symbolizer, @@ -625,6 +698,7 @@ where fn perf_script_for_cli( path: &std::path::Path, symbols: bool, + inline: bool, symbolizer: SymbolizerKind, runner: &R, ) -> backends::BackendResult @@ -633,6 +707,7 @@ where { let options = FoldOptions { count_periods: true, + inline, }; let mut output = Vec::new(); if symbols { diff --git a/src/perfdata/build_id.rs b/src/perfdata/build_id.rs index efa477d..d4db0ca 100644 --- a/src/perfdata/build_id.rs +++ b/src/perfdata/build_id.rs @@ -143,16 +143,32 @@ fn build_id_feature_payload(bytes: &[u8]) -> Result, String> { } fn parse_build_id_event(record: &[u8]) -> Result { - let record_type = read_u32(record, 0)?; - if record_type != PERF_RECORD_HEADER_BUILD_ID { - return Err(format!( - "expected PERF_RECORD_HEADER_BUILD_ID, got {record_type}" - )); - } + // tools/perf/util/build-id.c write_buildid() emits these feature-section + // records WITHOUT setting header.type (it leaves it 0) and sets + // PERF_RECORD_MISC_BUILD_ID_SIZE in misc with the real build-id length in + // the size byte at offset 20 of the 24-byte build_id field (record offset + // 32). Do not gate on the record type; rely on the size field (offset 6) + // for record framing, as perf_header__read_build_ids does. + let misc = read_u16(record, 4)?; + let build_id_size = if misc & PERF_RECORD_MISC_BUILD_ID_SIZE != 0 { + let size = usize::from( + *record + .get(12 + BUILD_ID_SIZE) + .ok_or_else(|| "truncated build-id size byte".to_string())?, + ); + if size > BUILD_ID_SIZE { + return Err(format!( + "build-id event build-id size {size} exceeds {BUILD_ID_SIZE} bytes" + )); + } + size + } else { + BUILD_ID_SIZE + }; Ok(BuildIdEvent { pid: read_u32(record, 8)?, - build_id: build_id_hex(&record[12..12 + BUILD_ID_SIZE]), + build_id: build_id_hex(&record[12..12 + build_id_size]), filename: filename(&record[BUILD_ID_EVENT_MIN_SIZE..])?, }) } diff --git a/src/perfdata/fold.rs b/src/perfdata/fold.rs index 7380462..b40ba1d 100644 --- a/src/perfdata/fold.rs +++ b/src/perfdata/fold.rs @@ -15,7 +15,7 @@ use crate::perfdata::build_id::{ BuildIdEvent, build_id_events_from_perfdata, parse_build_id_events, }; use crate::perfdata::endian::{read_u32, read_u64}; -use crate::perfdata::header::{PerfFeatureSection, PerfHeader, parse_header}; +use crate::perfdata::header::{PerfFeatureSection, PerfHeader, parse_header, parse_header_arch}; use crate::perfdata::mappings::{ FileIdentity, MappingResolveCache, MmapTable, ResolvedMappingRef, UserMapping, }; @@ -31,7 +31,8 @@ use crate::perfdata::samples::{ is_perf_user_deferred_context_marker, parse_sample_record_callchain, }; use crate::perfdata::unwind::{ - FramehopUnwinder, PerfX86_64Regs, UserStackUnwindResult, UserStackUnwinder, + FramehopUnwinder, PerfArch, PerfUserRegs, UserStackUnwindResult, UserStackUnwinder, + unwind_aarch64_frame_pointer_stack_like_elfutils, unwind_x86_64_frame_pointer_stack_like_elfutils, }; use crate::symbols::{SymbolFrameCache, SymbolRequest, SymbolResolver, perf_build_id_elf_path}; @@ -109,13 +110,71 @@ struct FoldAccumulator { sample_frames: Vec, callchain: Vec, unwind_debug_dir: Option, + /// Architecture of the recording machine (HEADER_ARCH), used to decode + /// REGS_USER samples and construct per-pid unwinders. Defaults to x86_64 + /// when the feature is absent. + arch: PerfArch, } -#[derive(Default)] struct PidUnwindState { object_unwinder: FramehopUnwinder, attempted_unwind_mappings: BTreeSet, loaded_unwind_modules: BTreeSet, + /// Memo of the ip-intrinsic leaf-only eligibility per sampled IP (gap-pkh + /// skip-gate cache). Only the `(pid, ip)`-STABLE facts are cached here — + /// whether the module covering `ip` is reported and whether any CFI covers + /// `ip`. The per-sample register condition (`bp < sp` / `lr == 0`) and the + /// sample's callchain state are combined fresh at query time, since both + /// vary across samples at the same IP. The whole `PidUnwindState` (and thus + /// this memo) is dropped when the pid's mappings change or the pid forks + /// (see `invalidate_pid_unwinder_if_mapping_overlaps_like_perf` / + /// `apply_fork_record`), which is exactly when reported-module / CFI facts + /// could change. + leaf_only_eligibility: HashMap, +} + +impl PidUnwindState { + fn with_arch(arch: PerfArch) -> Self { + Self { + object_unwinder: FramehopUnwinder::with_arch(arch), + attempted_unwind_mappings: BTreeSet::new(), + loaded_unwind_modules: BTreeSet::new(), + leaf_only_eligibility: HashMap::with_hasher(FxBuildHasher), + } + } +} + +/// The `(pid, ip)`-stable half of the gap-5gr / gap-pkh leaf-only decision. +/// +/// `Eligible` means: the module covering the sampled IP is reported into the +/// unwinder AND no CFI (.eh_frame/.debug_frame FDE) covers the IP. Per +/// elfutils `libdwfl/frame_unwind.c`, with no FDE row `handle_cfi` cannot +/// allocate an unwound frame, so `__libdwfl_frame_unwind` falls through to the +/// `ebl_unwind` arch fallback; if that fallback also cannot advance (the +/// per-sample register condition) libdwfl fires the initial-frame callback +/// exactly once and stops — the scenario-D single leaf. `Ineligible` means CFI +/// covers the IP (so we cannot tell a priori whether `handle_cfi` yields +/// PC_UNDEFINED end-of-stack or a real PC_SET caller — see +/// `backends/.../handle_cfi`'s return-register branch — and MUST run framehop) +/// or the IP's module is not reported (scenario B, handled upstream). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LeafOnlyEligibility { + Eligible, + Ineligible, +} + +/// Outcome of classifying a sample's object unwind before running framehop. +/// +/// Gap-pkh skip gate: `SkipUnwind` and `LeafOnly` both avoid invoking framehop. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ObjectUnwindClass { + /// perf/libdw would emit zero unwound frames (research §3 skip classes). + SkipUnwind, + /// perf/libdw fires the initial-frame callback exactly once and stops + /// (scenario D): emit the single sampled-IP leaf, skip framehop. + LeafOnly, + /// Could be 1-or-N frames; framehop must run. + MustUnwind, } type UnwindMappingKey = (String, u64, u64, u64); @@ -208,12 +267,6 @@ enum SampleCallchainPresence { Absent, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ObjectUnwindInitialFramePolicy { - DropSyntheticCurrentIp, - KeepDsoLeaf, -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum InitialIpMappingState { NoRecordedMapping, @@ -224,7 +277,12 @@ enum InitialIpMappingState { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ReportModuleResult { NoDso, - Reported, + /// A module covering this IP was loaded into the unwinder by this call. + NewlyReported, + /// A module covering this IP was already present (no new work, no new + /// unwind information for framehop — used to suppress the PERF-4 redundant + /// re-unwind in the module-report retry loop). + AlreadyReported, Failed, } @@ -233,7 +291,6 @@ struct UserUnwindContext { sample_callchain: SampleCallchainPresence, callchain: SampleCallchainState, initial_ip_mapping: InitialIpMappingState, - initial_ip_is_dso: bool, module_count: usize, frame_pointer_at_or_above_stack_pointer: bool, syscall_return_state: bool, @@ -271,6 +328,10 @@ struct SampleEventLayout { #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct FoldOptions { pub count_periods: bool, + /// When set, expand each callchain entry into its DWARF inline frames, + /// mirroring `perf script --inline`. Off by default: plain `perf script` + /// prints exactly one line per callchain entry, named from the ELF symtab. + pub inline: bool, } impl PerfSummary { @@ -496,7 +557,7 @@ pub fn fold_perfdata_callchains_with_options( options: FoldOptions, ) -> Result { let fold_data = collect_fold_data(bytes, options)?; - render_fold_data::(fold_data, None) + render_fold_data::(fold_data, None, options.inline) } /// Collapses perf sample callchains from a `perf.data` file path. @@ -539,7 +600,7 @@ where { let fold_data = collect_fold_data(bytes, options)?; let mut symbol_cache = SymbolFrameCache::new(symbol_resolver); - render_fold_data(fold_data, Some(&mut symbol_cache)) + render_fold_data(fold_data, Some(&mut symbol_cache), options.inline) } /// Collapses symbolized perf sample callchains from a `perf.data` file path. @@ -630,7 +691,8 @@ fn collect_fold_data(bytes: &[u8], options: FoldOptions) -> Result { symbol_cache: Option<&'io mut SymbolFrameCache<'cache, R>>, writer: &'io mut W, event_name_width: usize, + inline: bool, } impl<'io, 'cache, R, W> PerfScriptSink<'io, 'cache, R, W> @@ -891,12 +964,15 @@ where symbol_cache: Option<&'io mut SymbolFrameCache<'cache, R>>, writer: &'io mut W, event_name_width: usize, + inline: bool, + arch: PerfArch, ) -> Self { Self { - accumulator: FoldAccumulator::new(header_build_ids), + accumulator: FoldAccumulator::new(header_build_ids).with_arch(arch), symbol_cache, writer, event_name_width, + inline, } } @@ -1017,7 +1093,7 @@ where fn write_sample_event(&mut self, sample: &PreparedFoldSample) -> Result<(), String> { if sample.has_callchain { self.write_sample_header(sample)?; - let frame_resolver = FoldFrameResolver::new(&self.accumulator.mmap_table); + let frame_resolver = FoldFrameResolver::new(&self.accumulator.mmap_table, self.inline); frame_resolver.write_script_frames_for_stack( sample.pid, &sample.frames, @@ -1026,7 +1102,7 @@ where )?; } else { self.write_sample_inline_header(sample)?; - let frame_resolver = FoldFrameResolver::new(&self.accumulator.mmap_table); + let frame_resolver = FoldFrameResolver::new(&self.accumulator.mmap_table, self.inline); frame_resolver.write_inline_sample_frame_for_stack( sample.pid, &sample.frames, @@ -1057,9 +1133,13 @@ where write!(self.writer, "{secs:>5}.{usecs:06}: ") .map_err(|error| format!("failed to write perf script output: {error}"))?; } + // builtin-script.c prints `fprintf(fp, "%*s: ", name_width, evname)` + // (note the trailing space) and then `fputc(cursor ? '\n' : ' ', fp)`. + // For a resolved callchain (the multi-frame path) cursor is set, so the + // header line ends with the event-name colon, a space, then a newline. writeln!( self.writer, - "{:>10} {:>width$}:", + "{:>10} {:>width$}: ", sample.count, sample.event_name, width = self.event_name_width, @@ -1216,7 +1296,11 @@ fn perfdata_header_from_file(file: &File) -> Result<(PerfHeader, [u8; 104]), Str Ok((header, bytes)) } -fn sample_layouts_from_file(file: &File, header: PerfHeader) -> Result { +fn sample_layouts_from_file( + file: &File, + header: PerfHeader, + header_bytes: &[u8; 104], +) -> Result { let attr_size = usize::try_from(header.attr_size) .map_err(|_| "perf attr section size exceeds usize".to_string())?; let attr_bytes = read_file_range(file, header.attr_offset, attr_size, "perf attr section")?; @@ -1231,7 +1315,12 @@ fn sample_layouts_from_file(file: &File, header: PerfHeader) -> Result>(); + let attr_ids = attrs + .iter() + .map(|attr| file_attr_ids_from_file(file, attr)) + .collect::, _>>()?; + let event_desc = event_desc_entries_from_file(file, header, header_bytes)?; + let event_names = build_event_names(&attrs, &attr_ids, &event_desc); let event_name_width = event_names .iter() .map(String::len) @@ -1245,12 +1334,12 @@ fn sample_layouts_from_file(file: &File, header: PerfHeader) -> Result) -> PerfArch { + arch.and_then(PerfArch::from_header_arch) + .unwrap_or_default() +} + +/// Reads the HEADER_ARCH feature string from a perf.data `File`. +/// +/// The feature table and payload live after the data section, so this reads +/// them from the file the way `build_id_events_from_file` does, then parses the +/// `perf_header_string` (u32 length + NUL-terminated bytes). +fn header_arch_from_file( + file: &File, + header: PerfHeader, + header_bytes: &[u8; 104], +) -> Result, String> { + let Some(section) = feature_sections_from_file(file, header, header_bytes)? + .into_iter() + .find(|section| section.feature == HEADER_ARCH_FEATURE) + else { + return Ok(None); + }; + let size = + usize::try_from(section.size).map_err(|_| "arch feature size exceeds usize".to_string())?; + let payload = read_file_range(file, section.offset, size, "arch feature payload")?; + let length = usize::try_from(read_u32(&payload, 0)?) + .map_err(|_| "arch feature string length exceeds usize".to_string())?; + let string = payload + .get(4..4 + length) + .ok_or_else(|| "arch feature string is truncated".to_string())?; + let end = string + .iter() + .position(|byte| *byte == 0) + .unwrap_or(string.len()); + std::str::from_utf8(&string[..end]) + .map(|arch| Some(arch.to_string())) + .map_err(|error| format!("arch feature string is not UTF-8: {error}")) +} + +// HEADER_EVENT_DESC feature bit (tools/perf/util/header.h enum HEADER_*). +const HEADER_EVENT_DESC_FEATURE: u16 = 12; + +fn event_desc_entries_from_file( + file: &File, + header: PerfHeader, + header_bytes: &[u8; 104], +) -> Result, String> { + let Some(section) = feature_sections_from_file(file, header, header_bytes)? + .into_iter() + .find(|section| section.feature == HEADER_EVENT_DESC_FEATURE) + else { + return Ok(Vec::new()); + }; + let size = usize::try_from(section.size) + .map_err(|_| "event desc feature size exceeds usize".to_string())?; + let payload = read_file_range(file, section.offset, size, "event desc feature payload")?; + Ok(parse_event_desc_entries(&payload)) +} + +fn event_desc_entries_from_bytes( + bytes: &[u8], + header: crate::perfdata::header::PerfHeader, +) -> Vec { + let Ok(sections) = crate::perfdata::header::parse_feature_sections(bytes, &header) else { + return Vec::new(); + }; + let Some(section) = sections + .into_iter() + .find(|section| section.feature == HEADER_EVENT_DESC_FEATURE) + else { + return Vec::new(); + }; + let (Ok(offset), Ok(size)) = ( + usize::try_from(section.offset), + usize::try_from(section.size), + ) else { + return Vec::new(); + }; + bytes + .get(offset..offset + size) + .map(parse_event_desc_entries) + .unwrap_or_default() +} + fn feature_sections_from_file( file: &File, header: PerfHeader, @@ -1330,9 +1509,11 @@ fn feature_sections_from_file( } fn perf_feature_bits(header_bytes: &[u8; 104]) -> Result, String> { + // adds_features bitmap begins at byte offset 72 in struct perf_file_header + // (tools/perf/util/header.h); see set_feature_bits in header.rs. let mut features = Vec::new(); for word_index in 0..4 { - let word = read_u64(header_bytes, 56 + word_index * 8)?; + let word = read_u64(header_bytes, 72 + word_index * 8)?; for bit_index in 0..64 { if word & (1_u64 << bit_index) != 0 { let feature = u16::try_from(word_index * 64 + bit_index) @@ -1390,9 +1571,15 @@ impl FoldAccumulator { sample_frames: Vec::new(), callchain: Vec::new(), unwind_debug_dir: current_perf_debug_dir(), + arch: PerfArch::default(), } } + fn with_arch(mut self, arch: PerfArch) -> Self { + self.arch = arch; + self + } + fn apply_record( &mut self, record: ParsedRecord, @@ -1479,7 +1666,10 @@ impl FoldAccumulator { } fn unwind_state_mut(&mut self, pid: u32) -> &mut PidUnwindState { - self.unwind_states.entry(pid).or_default() + let arch = self.arch; + self.unwind_states + .entry(pid) + .or_insert_with(|| PidUnwindState::with_arch(arch)) } fn invalidate_pid_unwinder_if_mapping_overlaps_like_perf( @@ -1820,12 +2010,13 @@ impl FoldAccumulator { &mut self, counts: &mut FoldCounts, symbol_cache: Option<&mut SymbolFrameCache<'_, R>>, + inline: bool, ) -> Result<(), String> where R: SymbolResolver, { let raw_stacks = std::mem::take(&mut self.raw_stacks); - accumulate_fold_counts(&raw_stacks, &self.mmap_table, counts, symbol_cache) + accumulate_fold_counts(&raw_stacks, &self.mmap_table, counts, symbol_cache, inline) } } @@ -1852,18 +2043,20 @@ fn comm_for_ids(thread_comms: &BTreeMap, tid: Option) -> Optio fn render_fold_data( fold_data: PerfFoldData, symbol_cache: Option<&mut SymbolFrameCache<'_, R>>, + inline: bool, ) -> Result where R: SymbolResolver, { let mut folded = Vec::new(); - write_fold_data(fold_data, symbol_cache, &mut folded)?; + write_fold_data(fold_data, symbol_cache, inline, &mut folded)?; String::from_utf8(folded).map_err(|error| format!("folded output is not utf-8: {error}")) } fn write_fold_data( fold_data: PerfFoldData, symbol_cache: Option<&mut SymbolFrameCache<'_, R>>, + inline: bool, writer: &mut W, ) -> Result<(), String> where @@ -1875,7 +2068,7 @@ where raw_stacks, } = fold_data; let mut counts = FoldCounts::default(); - accumulate_fold_counts(&raw_stacks, &mmap_table, &mut counts, symbol_cache)?; + accumulate_fold_counts(&raw_stacks, &mmap_table, &mut counts, symbol_cache, inline)?; write_fold_counts(counts, writer) } @@ -1883,6 +2076,7 @@ fn prefetch_symbols( raw_stacks: &[RawStackEntryRef<'_, FoldFrame>], mmap_table: &MmapTable, symbol_cache: &mut SymbolFrameCache<'_, R>, + inline: bool, ) -> Result<(), String> where R: SymbolResolver, @@ -1897,6 +2091,7 @@ where mmap_table, &mut mapping_cache, &mut batches, + inline, ); if batches.full_mappings.len() >= PREFETCH_SYMBOL_REQUEST_BATCH_SIZE { symbol_cache.prefetch_mapping_refs(&batches.full_mappings)?; @@ -1923,6 +2118,7 @@ fn accumulate_fold_counts( mmap_table: &MmapTable, counts: &mut FoldCounts, mut symbol_cache: Option<&mut SymbolFrameCache<'_, R>>, + inline: bool, ) -> Result<(), String> where R: SymbolResolver, @@ -1930,9 +2126,9 @@ where let raw_stacks = raw_stacks.sorted_entries(); counts.reserve_first_drain(raw_stacks.len()); if let Some(cache) = symbol_cache.as_deref_mut() { - prefetch_symbols(&raw_stacks, mmap_table, cache)?; + prefetch_symbols(&raw_stacks, mmap_table, cache, inline)?; } - let frame_resolver = FoldFrameResolver::new(mmap_table); + let frame_resolver = FoldFrameResolver::new(mmap_table, inline); let mut callchain = Vec::new(); let mut buffers = FoldedRenderBuffers::default(); for stack in raw_stacks { @@ -2013,6 +2209,7 @@ fn extend_symbol_mappings_for_stack<'a>( mmap_table: &'a MmapTable, mapping_cache: &mut MappingResolveCache, batches: &mut SymbolPrefetchBatches<'a>, + inline: bool, ) { for frame in callchain { let address = frame.address(); @@ -2024,7 +2221,12 @@ fn extend_symbol_mappings_for_stack<'a>( symbol_source_id: mapping.symbol_source_id, relative_address: mapping.relative_address, }; - if matches!(frame, FoldFrame::InlineCurrentIp(_)) { + // Without --inline (the default), every frame is rendered from its + // single base ELF symtab symbol, so prefetch only the base symbol. + // With --inline, perf's machine.c unwind_entry() runs + // append_inlines() on every accepted entry including the leaf, so + // InlineCurrentIp leaves prefetch the full DWARF inline chain too. + if !inline { if batches.seen_base.insert(key) { batches.base_mappings.push(mapping); } @@ -2037,6 +2239,7 @@ fn extend_symbol_mappings_for_stack<'a>( struct FoldFrameResolver<'a> { mmap_table: &'a MmapTable, + inline: bool, } enum FrameMappingDecision<'a> { @@ -2066,8 +2269,8 @@ impl SymbolResolver for NoopSymbolResolver { } impl<'a> FoldFrameResolver<'a> { - fn new(mmap_table: &'a MmapTable) -> Self { - Self { mmap_table } + fn new(mmap_table: &'a MmapTable, inline: bool) -> Self { + Self { mmap_table, inline } } fn mapping_decision( @@ -2145,12 +2348,26 @@ impl<'a> FoldFrameResolver<'a> { continue; } if let FoldFrame::InlineCurrentIp(address) = frame { - self.append_inline_current_ip_folded_frame( - pid, - address, - symbol_cache.as_deref_mut(), - buffers, - )?; + // perf's machine.c unwind_entry() runs append_inlines() on + // EVERY accepted entry, including the initial sampled IP, so + // with --inline the leaf expands its inline chain just like a + // caller frame. Only the no-inline default renders the single + // base symtab symbol for the leaf. + if self.inline { + self.append_folded_frame_labels( + pid, + FoldFrame::UserUnwind(address), + symbol_cache.as_deref_mut(), + buffers, + )?; + } else { + self.append_inline_current_ip_folded_frame( + pid, + address, + symbol_cache.as_deref_mut(), + buffers, + )?; + } continue; } self.append_folded_frame_labels(pid, frame, symbol_cache.as_deref_mut(), buffers)?; @@ -2191,13 +2408,28 @@ impl<'a> FoldFrameResolver<'a> { continue; } if let FoldFrame::InlineCurrentIp(address) = frame { - self.write_inline_current_ip_script_frames( - pid, - address, - symbol_cache.as_deref_mut(), - &mut mapping_cache, - writer, - )?; + // perf's machine.c unwind_entry() runs append_inlines() on + // EVERY accepted entry, including the initial sampled IP, so + // with --inline the leaf expands its inline chain just like a + // caller frame. Only the no-inline default renders the single + // base symtab symbol for the leaf. + if self.inline { + self.write_regular_script_frame( + pid, + FoldFrame::UserUnwind(address), + symbol_cache.as_deref_mut(), + &mut mapping_cache, + writer, + )?; + } else { + self.write_inline_current_ip_script_frames( + pid, + address, + symbol_cache.as_deref_mut(), + &mut mapping_cache, + writer, + )?; + } continue; } self.write_regular_script_frame( @@ -2267,6 +2499,7 @@ impl<'a> FoldFrameResolver<'a> { frame, &mapping, symbol_cache, + self.inline, )?; } FrameMappingDecision::KernelAddress | FrameMappingDecision::Address => { @@ -2304,11 +2537,25 @@ impl<'a> FoldFrameResolver<'a> { let Some(cache) = symbol_cache else { return Ok(()); }; + // Resolve the mapping path first so the inline chain can carry the + // mapped DSO name like every other script frame (map__fprintf_dsoname), + // rather than the hardcoded "([unknown])". + let dso_path = pid + .and_then(|pid| { + self.mmap_table + .resolve_ref_cached(pid, address, mapping_cache) + }) + .map(|mapping| mapping.path.to_string()); if let Some(frames) = self.resolve_inline_current_ip_frames(pid, address, cache, mapping_cache)? { for label in frames.iter().rev() { - write_perf_script_frame_for_label(writer, address, label)?; + match dso_path.as_deref() { + Some(path) => { + write_perf_script_mapped_symbol_frame(writer, address, label, path)?; + } + None => write_perf_script_frame_for_label(writer, address, label)?, + } } } else { self.write_regular_script_frame( @@ -2414,7 +2661,12 @@ impl<'a> FoldFrameResolver<'a> { ) { FrameMappingDecision::Mapped(mapping) => { if let Some(cache) = symbol_cache { - if let Some(rendered) = cache.resolve_folded_mapping_ref(&mapping)? { + let rendered = if self.inline { + cache.resolve_folded_mapping_ref(&mapping)? + } else { + cache.resolve_base_folded_mapping_ref(&mapping)? + }; + if let Some(rendered) = rendered { append_cached_rendered_frame(&mut buffers.rendered, rendered); } else { let fallback = symbol_fallback_frame_ref(&mapping); @@ -2582,35 +2834,99 @@ fn write_perf_script_mapped_decision_frame( frame: FoldFrame, mapping: &ResolvedMappingRef<'_>, symbol_cache: Option<&mut SymbolFrameCache<'_, R>>, + inline: bool, ) -> Result<(), String> where R: SymbolResolver, W: IoWrite + ?Sized, { - if let Some(cache) = symbol_cache { - let frames = cache.resolve_mapping_ref(mapping)?; - if frames.is_empty() { - write_perf_script_frame_for_label( + let Some(cache) = symbol_cache else { + write_perf_script_mapped_unknown_symbol_frame(writer, address, mapping.path)?; + return Ok(()); + }; + // Default `perf script` prints exactly one line per callchain entry, named + // from the ELF symtab (builtin-script.c sample__fprintf_sym without + // --inline). Resolve only the base object symbol and print it with the + // mapping's full DSO name (map__fprintf_dsoname). + if !inline { + return match cache.resolve_mapping_ref_with_base_symbol(mapping)? { + Some([label, ..]) => { + write_perf_script_mapped_symbol_frame(writer, address, label, mapping.path) + } + _ => write_perf_script_mapped_unknown_symbol_frame(writer, address, mapping.path), + }; + } + let (frames, base_offset) = cache.resolve_mapping_ref_with_offset(mapping)?; + if frames.is_empty() { + write_perf_script_mapped_unknown_symbol_frame(writer, address, mapping.path)?; + } else if matches!(frame, FoldFrame::UserUnwind(_)) + && frames.len() == 1 + && !is_kernel_space_frame(address) + { + // A single non-inline base frame already carries its +0x baked in + // by perf_frames_with_object_alias_and_offset (the symtab with_offset + // form), so print it verbatim with the DSO path. + write_perf_script_mapped_symbol_frame(writer, address, &frames[0], mapping.path)?; + } else { + // perf's evsel_fprintf.c prints `sym+0x (inlined)` for every inline + // frame and `sym+0x (dso)` for the trailing non-inline base frame. + // Every frame shares one offset (__symbol__fprintf_symname_offs uses the + // base symbol start, which new_inline_sym reuses for the fake inline + // symbols). Frames are stored root-to-leaf, so .rev() prints leaf-first + // and the base (root) last as the non-inlined frame. + let last = frames.len() - 1; + for (printed_index, label) in frames.iter().rev().enumerate() { + let is_inlined = printed_index != last; + write_perf_script_inline_chain_frame( writer, address, - &symbol_fallback_frame_ref(mapping), + label, + base_offset, + mapping.path, + is_inlined, )?; - } else if matches!(frame, FoldFrame::UserUnwind(_)) - && frames.len() == 1 - && !is_kernel_space_frame(address) - { - write_perf_script_mapped_symbol_frame(writer, address, &frames[0], mapping.path)?; - } else { - for label in frames.iter().rev() { - write_perf_script_frame_for_label(writer, address, label)?; - } } - return Ok(()); } - write_perf_script_mapped_unknown_symbol_frame(writer, address, mapping.path)?; Ok(()) } +/// Prints one perf-script callchain frame for an inline-expanded address. +/// +/// Matches `tools/perf/util/evsel_fprintf.c`: the symbol name carries the +/// shared `+0x` offset (`__symbol__fprintf_symname_offs`), inline frames +/// print ` (inlined)` instead of a DSO name (`print_dso && (!sym || +/// !sym->inlined)`), and the trailing non-inline base frame prints the mapped +/// DSO path (`map__fprintf_dsoname_dsoff`). +fn write_perf_script_inline_chain_frame( + writer: &mut W, + address: u64, + label: &str, + base_offset: Option<&str>, + path: &str, + is_inlined: bool, +) -> Result<(), String> +where + W: IoWrite + ?Sized, +{ + // Fallback labels (raw addresses, [unknown], [module]) keep their existing + // rendering and never take an offset or the (inlined) marker. + if label == UNKNOWN_FRAME + || label.starts_with("0x") + || module_fallback_label_module(label).is_some() + { + return write_perf_script_mapped_symbol_frame(writer, address, label, path); + } + let offset = base_offset.unwrap_or(""); + if is_inlined { + writeln!(writer, "\t{address:16x} {label}{offset} (inlined)") + .map_err(|error| format!("failed to write perf script output: {error}")) + } else { + let path = perf_script_dso_name(path); + writeln!(writer, "\t{address:16x} {label}{offset} ({path})") + .map_err(|error| format!("failed to write perf script output: {error}")) + } +} + fn write_perf_script_inline_mapped_decision_frame( writer: &mut W, address: u64, @@ -2742,6 +3058,7 @@ where { return write_perf_script_frame_for_label_fragment(writer, prefix, address, label); } + let path = perf_script_dso_name(path); write!(writer, "{prefix}{address:16x} {label} ({path})") .map_err(|error| format!("failed to write perf script output: {error}")) } @@ -2848,6 +3165,18 @@ fn is_kernel_mapping_ref(mapping: &ResolvedMappingRef<'_>) -> bool { is_kernel_space_frame(mapping.relative_address) && mapping.path.starts_with('[') } +/// The DSO name perf-script prints for a mapping. The core kernel map is +/// recorded with a relocation reference suffix (e.g. `[kernel.kallsyms]_stext`), +/// but perf names its dso `[kernel.kallsyms]` (`machine__create_kernel_maps` +/// sets the kernel dso short name), so map__fprintf_dsoname prints that. +fn perf_script_dso_name(path: &str) -> &str { + if path.starts_with("[kernel.kallsyms]") { + "[kernel.kallsyms]" + } else { + path + } +} + fn parse_sample_for_summary( sample_misc: u16, payload: &[u8], @@ -2983,12 +3312,14 @@ fn append_perf_user_unwind_frames( if !has_perf_captured_user_stack(stack) { return; } - let Ok(regs) = - PerfX86_64Regs::from_perf_masked_values(event.layout.sample_regs_user, ®s.values) - else { + let Ok(regs) = PerfUserRegs::from_perf_masked_values( + accumulator.arch, + event.layout.sample_regs_user, + ®s.values, + ) else { return; }; - accumulator.ensure_unwind_mapping_for_ip(sample.pid, regs.ip); + accumulator.ensure_unwind_mapping_for_ip(sample.pid, regs.ip()); let context = build_user_unwind_context(accumulator, misc, event, sample, ®s); let mut unwound_frames = unwind_user_stack_like_perf(accumulator, sample, ®s, context); let mut mapping_cache = MappingResolveCache::default(); @@ -3006,7 +3337,7 @@ fn build_user_unwind_context( misc: u16, event: &SampleEventLayout, sample: &crate::perfdata::samples::SampleCallchain<'_>, - regs: &PerfX86_64Regs, + regs: &PerfUserRegs, ) -> UserUnwindContext { let sample_callchain = if event.layout.sample_type & PERF_SAMPLE_CALLCHAIN != 0 { SampleCallchainPresence::Present @@ -3021,14 +3352,11 @@ fn build_user_unwind_context( sample, !accumulator.sample_frames.is_empty(), ), - initial_ip_mapping: initial_ip_mapping_state(accumulator, sample.pid, regs.ip), - initial_ip_is_dso: object_unwind_initial_frame_policy( - sample.pid, - regs.ip, - &accumulator.mmap_table, - ) == ObjectUnwindInitialFramePolicy::KeepDsoLeaf, + initial_ip_mapping: initial_ip_mapping_state(accumulator, sample.pid, regs.ip()), module_count: loaded_unwind_module_count(accumulator, sample.pid), - frame_pointer_at_or_above_stack_pointer: regs.bp >= regs.sp, + // x86_64-specific `ebl_unwind` precondition (false on aarch64, whose + // backend has its own internal accept condition). + frame_pointer_at_or_above_stack_pointer: regs.frame_pointer_at_or_above_stack_pointer(), syscall_return_state: regs.is_syscall_return_state(), } } @@ -3087,7 +3415,7 @@ fn sample_callchain_state( fn unwind_user_stack_like_perf( accumulator: &mut FoldAccumulator, sample: &crate::perfdata::samples::SampleCallchain<'_>, - regs: &PerfX86_64Regs, + regs: &PerfUserRegs, context: UserUnwindContext, ) -> Vec { let Some(stack) = &sample.user_stack else { @@ -3107,7 +3435,7 @@ fn unwind_user_stack_like_perf( fn unwind_object_stack_like_perf( accumulator: &mut FoldAccumulator, pid: Option, - regs: &PerfX86_64Regs, + regs: &PerfUserRegs, stack_bytes: &[u8], context: UserUnwindContext, ) -> Vec { @@ -3117,7 +3445,7 @@ fn unwind_object_stack_like_perf( let mut state = accumulator .unwind_states .remove(&pid_value) - .unwrap_or_default(); + .unwrap_or_else(|| PidUnwindState::with_arch(accumulator.arch)); let unwind_debug_dir = accumulator.unwind_debug_dir.clone(); let frames = unwind_object_frame_addresses_like_perf( &mut state, @@ -3133,7 +3461,7 @@ fn unwind_object_stack_like_perf( .into_iter() .enumerate() .map(|(index, address)| { - if index == 0 && address == regs.ip { + if index == 0 && address == regs.ip() { FoldFrame::InlineCurrentIp(address) } else { FoldFrame::UserUnwind(address) @@ -3147,19 +3475,41 @@ fn unwind_object_frame_addresses_like_perf( pid: u32, mmap_table: &MmapTable, unwind_debug_dir: Option<&Path>, - regs: &PerfX86_64Regs, + regs: &PerfUserRegs, stack_bytes: &[u8], context: UserUnwindContext, ) -> Vec { - let initial_frame_policy = object_unwind_initial_frame_policy(Some(pid), regs.ip, mmap_table); - if report_unwind_module_for_ip_like_perf(state, mmap_table, pid, regs.ip, unwind_debug_dir) + // perf's unwind__get_entries reports the module for the initial IP up front + // (tools/perf/util/unwind-libdw.c): a hard report failure (scenario B) + // abandons the whole unwind with zero entries. + if report_unwind_module_for_ip_like_perf(state, mmap_table, pid, regs.ip(), unwind_debug_dir) == ReportModuleResult::Failed { return Vec::new(); } + + // gap-pkh skip gate: evaluate the same leaf-only predicate BEFORE framehop. + // `LeafOnly`/`SkipUnwind` never invoke the (expensive) framehop unwind, and + // the result is byte-identical to running it, because the shared + // acceptance tail emits the same leaf when framehop would have produced + // nothing. + let leaf_only = sample_is_leaf_only(state, pid, mmap_table, regs, context); + match classify_object_unwind(context, leaf_only) { + ObjectUnwindClass::SkipUnwind => return Vec::new(), + ObjectUnwindClass::LeafOnly => { + return perf_accepted_object_unwind_frames(regs, context.callchain, true, Vec::new()); + } + ObjectUnwindClass::MustUnwind => {} + } + let mut object_unwind = unwind_user_stack_with_diagnostics(&mut state.object_unwinder, *regs, stack_bytes, 256); for _ in 0..MAX_LIBDW_CALLBACK_REPORT_PASSES { + // PERF-4: only re-unwind when this pass actually loaded a new module. + // report_unwind_modules_for_frame_callbacks_like_perf returns whether + // anything was newly reported; when it returns false there is nothing + // new for framehop to traverse, so the previous unwind is final and the + // redundant re-unwind (and its Vec/diagnostics comparison) is skipped. if !report_unwind_modules_for_frame_callbacks_like_perf( state, mmap_table, @@ -3179,7 +3529,7 @@ fn unwind_object_frame_addresses_like_perf( let raw_frames = object_unwind.accepted_frames; let initial_ip_has_reported_module = initial_ip_mapping_has_reported_unwind_module( Some(pid), - regs.ip, + regs.ip(), mmap_table, &state.object_unwinder, ); @@ -3199,22 +3549,129 @@ fn unwind_object_frame_addresses_like_perf( mmap_table, context, ); - perf_accepted_object_unwind_frames(regs, context.callchain, initial_frame_policy, raw_frames) + perf_accepted_object_unwind_frames(regs, context.callchain, leaf_only, raw_frames) +} + +/// Whether perf/libdw would fire the initial-frame callback exactly once and +/// stop (research §2 scenario D / §3.6): the sampled IP reported into a module, +/// no CFI covers it, and the arch-specific `ebl_unwind` fallback provably +/// cannot advance. This is the EXACT narrow predicate from +/// `.ace-research-perf-unwind.md` §3.6/§4 — broader rules (e.g. "emit on any +/// empty framehop") reintroduced the measured 10.1B/41.2B overcounts. +/// +/// The `(pid, ip)`-stable half (module reported + no CFI) is memoized in +/// `state.leaf_only_eligibility`; the per-sample register condition +/// (`bp < sp` on x86_64, `lr == 0` on aarch64) is combined here fresh. +fn sample_is_leaf_only( + state: &mut PidUnwindState, + pid: u32, + mmap_table: &MmapTable, + regs: &PerfUserRegs, + context: UserUnwindContext, +) -> bool { + // KernelWithUserFrame never appends extra user frames (research §3.5), so a + // leaf is never emitted there; leave that to the SkipUnwind class. + if context.callchain == SampleCallchainState::KernelWithUserFrame { + return false; + } + let ip = regs.ip(); + let eligibility = *state + .leaf_only_eligibility + .entry(ip) + .or_insert_with(|| leaf_only_eligibility(pid, ip, mmap_table, &state.object_unwinder)); + if eligibility != LeafOnlyEligibility::Eligible { + return false; + } + arch_fallback_provably_cannot_advance(regs) +} + +/// The `(pid, ip)`-stable half of the leaf-only predicate, suitable for +/// memoizing: the module covering `ip` is reported into the unwinder AND no CFI +/// (.eh_frame/.debug_frame FDE) covers `ip`. +fn leaf_only_eligibility( + pid: u32, + ip: u64, + mmap_table: &MmapTable, + object_unwinder: &FramehopUnwinder, +) -> LeafOnlyEligibility { + let reported = + initial_ip_mapping_has_reported_unwind_module(Some(pid), ip, mmap_table, object_unwinder); + if reported && !object_unwinder.has_unwind_info_for_ip(ip) { + LeafOnlyEligibility::Eligible + } else { + LeafOnlyEligibility::Ineligible + } +} + +/// The per-sample half of the leaf-only predicate: whether the arch-specific +/// `ebl_unwind` fallback can never produce a caller from these registers. +/// +/// x86_64 (`backends/x86_64_unwind.c`): the rbp fallback is only attempted by +/// pyroclast when `bp >= sp` (the elfutils final guard `if (sp >= fp) return +/// false;` rejects a frame pointer that does not sit above the stack pointer). +/// So `bp < sp` means the fallback contributes nothing. +/// +/// aarch64 (`backends/aarch64_unwind.c`): the caller pc comes from `lr`; the +/// fallback returns false immediately when `lr == 0`. So `lr == 0` means no +/// caller. +fn arch_fallback_provably_cannot_advance(regs: &PerfUserRegs) -> bool { + match *regs { + PerfUserRegs::X86_64(regs) => regs.bp < regs.sp, + PerfUserRegs::Aarch64(regs) => regs.lr == 0, + } +} + +/// Classifies a sample's object unwind before framehop runs (gap-pkh). +fn classify_object_unwind(context: UserUnwindContext, leaf_only: bool) -> ObjectUnwindClass { + // Research §3.5: a recorded kernel->user callchain is not extended with + // extra user DWARF callers — perf emits zero unwound frames here. + if context.callchain == SampleCallchainState::KernelWithUserFrame { + return ObjectUnwindClass::SkipUnwind; + } + if leaf_only { + ObjectUnwindClass::LeafOnly + } else { + ObjectUnwindClass::MustUnwind + } } fn libdw_arch_fallback_after_empty_object_unwind( raw_frames: Vec, - regs: &PerfX86_64Regs, + regs: &PerfUserRegs, stack_bytes: &[u8], use_libdw_arch_fallback: bool, ) -> Vec { - if raw_frames.is_empty() && use_libdw_arch_fallback && regs.bp >= regs.sp { - unwind_x86_64_frame_pointer_stack_like_elfutils(*regs, stack_bytes, 256) - } else { - raw_frames + if !use_libdw_arch_fallback { + return raw_frames; + } + match *regs { + // elfutils' x86_64 backend only walks the rbp chain when the frame + // pointer is at or above the stack pointer. framehop's own x86_64 + // frame-pointer recovery already advances most stacks, so the elfutils + // fallback only fills in stacks where framehop produced nothing. + PerfUserRegs::X86_64(regs) if raw_frames.is_empty() && regs.bp >= regs.sp => { + unwind_x86_64_frame_pointer_stack_like_elfutils(regs, stack_bytes, 256) + } + PerfUserRegs::X86_64(_) => raw_frames, + // aarch64's backend has no bp/sp precondition: it accepts the lr-based + // caller unless lr == 0, with its own internal `fp == 0 || fp+16 > sp` + // accept condition (backends/aarch64_unwind.c). framehop's aarch64 + // unwinder yields only the seed pc when no CFI covers it, which is + // exactly when libdwfl invokes ebl_unwind on the leaf, so the fallback + // fires when framehop produced no caller beyond the sampled pc. + PerfUserRegs::Aarch64(regs) if frames_are_seed_only(&raw_frames, regs.pc) => { + unwind_aarch64_frame_pointer_stack_like_elfutils(regs, stack_bytes, 256) + } + PerfUserRegs::Aarch64(_) => raw_frames, } } +/// Whether framehop produced no caller beyond the sampled pc: either nothing at +/// all, or just the seed instruction pointer. +fn frames_are_seed_only(raw_frames: &[u64], pc: u64) -> bool { + raw_frames.is_empty() || raw_frames == [pc] +} + fn truncate_syscall_return_unwind_after_first_executable_frame( raw_frames: Vec, _pid: Option, @@ -3252,13 +3709,16 @@ fn report_unwind_modules_for_frame_callbacks_like_perf( ) -> bool { let mut loaded = false; for address in frame_addresses { + // PERF-4: only treat a NEWLY loaded module as progress. An + // already-present module adds no unwind information, so re-unwinding + // after it would reproduce the same frames. loaded |= report_unwind_module_for_ip_like_perf( state, mmap_table, pid, *address, unwind_debug_dir, - ) == ReportModuleResult::Reported; + ) == ReportModuleResult::NewlyReported; } loaded } @@ -3274,10 +3734,10 @@ fn report_unwind_module_for_ip_like_perf( return ReportModuleResult::NoDso; }; if state.object_unwinder.has_reported_module_for_ip(ip) { - return ReportModuleResult::Reported; + return ReportModuleResult::AlreadyReported; } if load_unwind_mapping_for_user_mapping_like_perf(state, mapping, unwind_debug_dir) { - ReportModuleResult::Reported + ReportModuleResult::NewlyReported } else { ReportModuleResult::Failed } @@ -3319,7 +3779,7 @@ fn load_unwind_mapping_for_user_mapping_like_perf( fn unwind_user_stack_with_diagnostics( unwinder: &mut impl UserStackUnwinder, - regs: PerfX86_64Regs, + regs: PerfUserRegs, stack_bytes: &[u8], max_frames: usize, ) -> UserStackUnwindResult { @@ -3393,48 +3853,42 @@ fn truncate_user_unwind_at_first_unmapped_frame( ) { } +/// Maps framehop's unwound frame addresses onto perf's accepted-entry list. +/// +/// `leaf_only` is the fully-evaluated scenario-D predicate (see +/// `sample_is_leaf_only`): when framehop produced no frames at all but +/// perf/libdw would still fire the initial-frame callback exactly once, emit +/// the single sampled-IP leaf. perf has no `.so`-vs-executable distinction in +/// this path — `frame_callback` fires for the initial frame regardless of +/// whether the covering module is a shared object or the main binary +/// (`tools/perf/util/unwind-libdw.c` / `libdwfl/dwfl_frame.c`), so the prior +/// `KeepDsoLeaf`/`DropSyntheticCurrentIp` split (which had no perf-source +/// basis) is gone. fn perf_accepted_object_unwind_frames( - regs: &PerfX86_64Regs, + regs: &PerfUserRegs, callchain: SampleCallchainState, - initial_frame_policy: ObjectUnwindInitialFramePolicy, + leaf_only: bool, unwound_frames: Vec, ) -> Vec { if callchain == SampleCallchainState::KernelWithUserFrame { return Vec::new(); } - // framehop yields the sampled instruction pointer before trying to advance. - // perf's libdw path reports the IP to DWFL as initial state, then only - // prints entries accepted via frame_callback/entry. - let _ = (regs, callchain, initial_frame_policy); - unwound_frames -} - -fn object_unwind_initial_frame_policy( - pid: Option, - ip: u64, - mmap_table: &MmapTable, -) -> ObjectUnwindInitialFramePolicy { - let mut mapping_cache = MappingResolveCache::default(); - if pid - .and_then(|pid| mmap_table.resolve_ref_cached(pid, ip, &mut mapping_cache)) - .is_some_and(|mapping| is_shared_object_mapping_path(mapping.path)) - { - ObjectUnwindInitialFramePolicy::KeepDsoLeaf - } else { - ObjectUnwindInitialFramePolicy::DropSyntheticCurrentIp + // gap-5gr: when the leaf-only predicate holds, perf/libdwfl fires + // frame_callback exactly once for the seeded IP and stops (scenario D). + // No FDE row covers the IP (`!has_unwind_info_for_ip`), so handle_cfi + // cannot advance, and the `ebl_unwind` rbp/lr fallback is guarded off + // (`bp < sp` on x86_64 / `lr == 0` on aarch64). perf therefore prints + // exactly the single sampled-IP leaf. framehop always yields the seed and + // its instruction-analysis heuristics can recover a *spurious* caller here + // that libdwfl would never emit, so the accepted list is the leaf alone + // regardless of what framehop produced. Because this truncation is the + // shared tail for both the gated (framehop-skipped) and ungated + // (framehop-run) paths, the gap-pkh skip gate is a pure optimization: both + // yield exactly `[ip]`. + if leaf_only { + return vec![regs.ip()]; } -} - -fn is_shared_object_mapping_path(path: &str) -> bool { - path.rsplit('/') - .next() - .is_some_and(|file_name| file_name.contains(".so") || has_dylib_extension(file_name)) -} - -fn has_dylib_extension(file_name: &str) -> bool { - Path::new(file_name) - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("dylib")) + unwound_frames } fn load_unwind_mapping( @@ -3547,7 +4001,12 @@ fn sample_layouts( header: crate::perfdata::header::PerfHeader, ) -> Result { let attrs = parse_file_attrs(bytes, header)?; - let event_names = attrs.iter().map(perf_event_name).collect::>(); + let attr_ids = attrs + .iter() + .map(|attr| parse_file_attr_ids(bytes, attr)) + .collect::, _>>()?; + let event_desc = event_desc_entries_from_bytes(bytes, header); + let event_names = build_event_names(&attrs, &attr_ids, &event_desc); let event_name_width = event_names .iter() .map(String::len) @@ -3561,12 +4020,12 @@ fn sample_layouts( by_identifier: BTreeMap::new(), event_name_width, }; - for (attr, event_name) in attrs.iter().zip(event_names) { + for ((attr, event_name), ids) in attrs.iter().zip(event_names).zip(attr_ids) { let event = SampleEventLayout { layout: layout_from_attr(attr), event_name, }; - for id in parse_file_attr_ids(bytes, attr)? { + for id in ids { layouts.by_identifier.insert(id, event.clone()); } } @@ -3603,6 +4062,109 @@ fn perf_event_name(attr: &PerfFileAttr) -> String { } } +/// A single event description parsed from the `HEADER_EVENT_DESC` feature. +#[derive(Clone, Debug, Eq, PartialEq)] +struct EventDescEntry { + name: String, + ids: Vec, +} + +/// Parses the `HEADER_EVENT_DESC` feature payload. +/// +/// `perf record` writes evsel names verbatim into this feature (see +/// `write_event_desc`/`read_event_desc` in `tools/perf/util/header.c`), and +/// `perf script` prints those names instead of reconstructing them from the +/// attr type/config. The layout is: `nre` (u32, number of events), `attr_sz` +/// (u32, sizeof perf_event_attr), then for each event: `attr_sz` attr bytes, a +/// `nr` (u32) id count, a length-prefixed name string, and `nr` u64 ids. +/// +/// The name string is written by `do_write_string`: a u32 length +/// (`PERF_ALIGN(strlen + 1, NAME_ALIGN)`) followed by that many bytes holding +/// the NUL-terminated name plus zero padding. We read the declared number of +/// bytes and take the text up to the first NUL. +fn parse_event_desc_entries(payload: &[u8]) -> Vec { + parse_event_desc_entries_checked(payload).unwrap_or_default() +} + +fn parse_event_desc_entries_checked(payload: &[u8]) -> Result, String> { + let event_count = read_u32(payload, 0)?; + let attr_size = usize::try_from(read_u32(payload, 4)?) + .map_err(|_| "event desc attr size exceeds usize".to_string())?; + let mut offset = 8usize; + let mut entries = Vec::with_capacity(event_count as usize); + for _ in 0..event_count { + offset = offset + .checked_add(attr_size) + .ok_or_else(|| "event desc attr offset overflow".to_string())?; + let id_count = usize::try_from(read_u32(payload, offset)?) + .map_err(|_| "event desc id count exceeds usize".to_string())?; + offset += 4; + let name_len = usize::try_from(read_u32(payload, offset)?) + .map_err(|_| "event desc name length exceeds usize".to_string())?; + offset += 4; + let name_bytes = payload + .get(offset..offset + name_len) + .ok_or_else(|| "event desc name truncated".to_string())?; + let name = event_desc_name_from_bytes(name_bytes); + offset += name_len; + let mut ids = Vec::with_capacity(id_count); + for _ in 0..id_count { + ids.push(read_u64(payload, offset)?); + offset += 8; + } + entries.push(EventDescEntry { name, ids }); + } + Ok(entries) +} + +fn event_desc_name_from_bytes(bytes: &[u8]) -> String { + let end = bytes + .iter() + .position(|byte| *byte == 0) + .unwrap_or(bytes.len()); + String::from_utf8_lossy(&bytes[..end]).into_owned() +} + +/// Builds the per-attr event names `perf script` would print. +/// +/// Prefers the verbatim evsel names from `HEADER_EVENT_DESC`, matched to each +/// attr by shared sample id (and by event index as a fallback, which is how +/// `process_event_desc` in `tools/perf/util/header.c` pairs descriptions with +/// evsels). Falls back to reconstructing the name from the attr type/config +/// when no description matches. +fn build_event_names( + attrs: &[PerfFileAttr], + attr_ids: &[Vec], + event_desc: &[EventDescEntry], +) -> Vec { + attrs + .iter() + .enumerate() + .map(|(index, attr)| { + event_desc_name_for_attr(index, attr_ids.get(index), event_desc) + .unwrap_or_else(|| perf_event_name(attr)) + }) + .collect() +} + +fn event_desc_name_for_attr( + index: usize, + attr_ids: Option<&Vec>, + event_desc: &[EventDescEntry], +) -> Option { + if event_desc.is_empty() { + return None; + } + if let Some(ids) = attr_ids.filter(|ids| !ids.is_empty()) + && let Some(entry) = event_desc + .iter() + .find(|entry| entry.ids.iter().any(|id| ids.contains(id))) + { + return Some(entry.name.clone()); + } + event_desc.get(index).map(|entry| entry.name.clone()) +} + fn hardware_event_name(config: u64) -> &'static str { match config & 0xffff_ffff { 0 => "cycles", @@ -3696,9 +4258,80 @@ mod tests { use std::cell::RefCell; use crate::perfdata::mappings::FileIdentity; - use crate::perfdata::unwind::{UserStackUnwindResult, UserStackUnwinder}; + use crate::perfdata::unwind::{ + PerfArch, PerfUserRegs, PerfX86_64Regs, UserStackUnwindResult, UserStackUnwinder, + }; use crate::symbols::{ResolvedSymbolFrames, SymbolFrameCache, SymbolRequest, SymbolResolver}; + // tools/perf/util/header.c write_event_desc: nre(u32), attr_sz(u32), then + // per event attr_sz attr bytes, nr(u32), do_write_string(name), nr u64 ids. + fn event_desc_payload(events: &[(&str, &[u64])], attr_sz: usize) -> Vec { + let mut payload = Vec::new(); + payload.extend((events.len() as u32).to_le_bytes()); + payload.extend((attr_sz as u32).to_le_bytes()); + for (name, ids) in events { + payload.extend(std::iter::repeat_n(0_u8, attr_sz)); + payload.extend((ids.len() as u32).to_le_bytes()); + // do_write_string: u32 len = PERF_ALIGN(strlen+1, NAME_ALIGN=64), + // then len bytes of NUL-terminated name plus zero padding. + let aligned = (name.len() + 1).div_ceil(64) * 64; + payload.extend((aligned as u32).to_le_bytes()); + let mut name_bytes = name.as_bytes().to_vec(); + name_bytes.resize(aligned, 0); + payload.extend(name_bytes); + for id in *ids { + payload.extend(id.to_le_bytes()); + } + } + payload + } + + #[test] + fn parses_event_desc_names_verbatim_like_perf_read_event_desc() { + // perf script prints the evsel names recorded in HEADER_EVENT_DESC + // (e.g. "task-clock:ppp") rather than reconstructing them; the trailing + // colon perf script appends is a separator, not part of the name. + let payload = event_desc_payload(&[("task-clock:ppp", &[230, 231, 242])], 136); + let entries = super::parse_event_desc_entries(&payload); + assert_eq!( + entries, + vec![super::EventDescEntry { + name: "task-clock:ppp".to_string(), + ids: vec![230, 231, 242], + }] + ); + } + + #[test] + fn event_desc_name_matches_attr_by_shared_id() { + let entries = vec![ + super::EventDescEntry { + name: "cycles:ppp".to_string(), + ids: vec![10, 11], + }, + super::EventDescEntry { + name: "task-clock:ppp".to_string(), + ids: vec![20, 21], + }, + ]; + assert_eq!( + super::event_desc_name_for_attr(0, Some(&vec![21]), &entries), + Some("task-clock:ppp".to_string()) + ); + } + + #[test] + fn event_desc_name_falls_back_to_event_index_without_ids() { + let entries = vec![super::EventDescEntry { + name: "task-clock:ppp".to_string(), + ids: Vec::new(), + }]; + assert_eq!( + super::event_desc_name_for_attr(0, None, &entries), + Some("task-clock:ppp".to_string()) + ); + } + #[derive(Default)] struct FakeUserStackUnwinder { calls: usize, @@ -3708,7 +4341,7 @@ mod tests { impl UserStackUnwinder for FakeUserStackUnwinder { fn unwind_user_stack( &mut self, - _regs: super::PerfX86_64Regs, + _regs: PerfUserRegs, _stack: &[u8], _max_frames: usize, ) -> UserStackUnwindResult { @@ -3717,9 +4350,11 @@ mod tests { } } + #[derive(Default)] struct StaticFrameResolver { frames: Vec, has_base_symbol: bool, + base_offset: Option, } impl SymbolResolver for StaticFrameResolver { @@ -3742,6 +4377,7 @@ mod tests { ResolvedSymbolFrames { frames: self.frames.clone(), has_base_symbol: self.has_base_symbol, + base_offset: self.base_offset.clone(), }; requests.len() ]) @@ -3778,6 +4414,7 @@ mod tests { .map(|request| ResolvedSymbolFrames { frames: vec![format!("symbol_{:x}", request.relative_address)], has_base_symbol: true, + base_offset: None, }) .collect() } @@ -3809,8 +4446,8 @@ mod tests { } } - fn test_regs(ip: u64) -> super::PerfX86_64Regs { - super::PerfX86_64Regs { + fn test_x86_regs(ip: u64) -> PerfX86_64Regs { + PerfX86_64Regs { ip, sp: 0x2000, bp: 0x3000, @@ -3818,6 +4455,10 @@ mod tests { } } + fn test_regs(ip: u64) -> PerfUserRegs { + PerfUserRegs::X86_64(test_x86_regs(ip)) + } + #[test] fn object_unwind_diagnostics_use_pluggable_user_stack_unwinder() { let mut unwinder = FakeUserStackUnwinder { @@ -3942,7 +4583,7 @@ mod tests { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::DropSyntheticCurrentIp, + false, vec![0x1000], ), vec![0x1000] @@ -3960,7 +4601,7 @@ mod tests { has_callchain: false, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::DropSyntheticCurrentIp, + false, vec![0x1000, 0x1100], ), vec![0x1000, 0x1100] @@ -3971,18 +4612,20 @@ mod tests { fn object_unwind_acceptance_does_not_invent_sample_ip_for_empty_libdw_callbacks() { // tools/perf/util/unwind-libdw.c only appends frames accepted by // frame_callback -> entry after dwfl_getthread_frames runs. A captured - // stack with no accepted callbacks stays empty. - let mut regs = test_regs(0x5555_556f_bbbb); + // stack with no accepted callbacks and a sample that is NOT leaf-only + // (`leaf_only == false`: e.g. CFI covers the IP) stays empty — the + // sampled IP is never invented absent the scenario-D predicate. + let mut regs = test_x86_regs(0x5555_556f_bbbb); regs.sp = 0x7fff_ffff_7790; regs.bp = 0x76c8; assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::Other { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::DropSyntheticCurrentIp, + false, Vec::new(), ), Vec::::new() @@ -4059,11 +4702,12 @@ mod tests { let resolver = StaticFrameResolver { frames: vec!["core::num::flt2dec::strategy::dragon::mul_pow10".to_string()], has_base_symbol: false, + base_offset: None, }; let mut symbol_cache = SymbolFrameCache::new(&resolver); let mut buffers = super::FoldedRenderBuffers::default(); - super::FoldFrameResolver::new(&mmap_table) + super::FoldFrameResolver::new(&mmap_table, false) .render_folded_stack_for_stack( Some(11), Some("pyroclast"), @@ -4092,11 +4736,12 @@ mod tests { let resolver = StaticFrameResolver { frames: vec!["core::num::flt2dec::strategy::dragon::format_shortest".to_string()], has_base_symbol: true, + base_offset: None, }; let mut symbol_cache = SymbolFrameCache::new(&resolver); let mut buffers = super::FoldedRenderBuffers::default(); - super::FoldFrameResolver::new(&mmap_table) + super::FoldFrameResolver::new(&mmap_table, false) .render_folded_stack_for_stack( Some(11), Some("pyroclast"), @@ -4134,11 +4779,12 @@ mod tests { "read_file_range".to_string(), ], has_base_symbol: false, + base_offset: None, }; let mut symbol_cache = SymbolFrameCache::new(&resolver); let mut buffers = super::FoldedRenderBuffers::default(); - super::FoldFrameResolver::new(&mmap_table) + super::FoldFrameResolver::new(&mmap_table, false) .render_folded_stack_for_stack( Some(11), Some("pyroclast"), @@ -4172,11 +4818,12 @@ mod tests { "quicksort<&str>".to_string(), ], has_base_symbol: true, + base_offset: None, }; let mut symbol_cache = SymbolFrameCache::new(&resolver); let mut buffers = super::FoldedRenderBuffers::default(); - super::FoldFrameResolver::new(&mmap_table) + super::FoldFrameResolver::new(&mmap_table, false) .render_folded_stack_for_stack( Some(11), Some("pyroclast"), @@ -4209,11 +4856,12 @@ mod tests { let resolver = StaticFrameResolver { frames: vec!["_Fork+0x48".to_string()], has_base_symbol: true, + base_offset: None, }; let mut symbol_cache = SymbolFrameCache::new(&resolver); let mut written = Vec::new(); - super::FoldFrameResolver::new(&mmap_table) + super::FoldFrameResolver::new(&mmap_table, false) .write_script_frames_for_stack( Some(11), &[super::FoldFrame::UserUnwind(0x1048)], @@ -4228,6 +4876,91 @@ mod tests { ); } + #[test] + fn inline_user_unwind_script_frame_marks_inlined_and_shares_offset_like_perf_script() { + // tools/perf/util/evsel_fprintf.c prints `sym+0x (inlined)` for + // each inline frame and `sym+0x (dso)` for the trailing non-inline + // base frame. The offset is shared across the whole group: + // __symbol__fprintf_symname_offs uses `al->addr - sym->start`, and an + // inline frame's fake symbol reuses base_sym->start (srcline.c + // new_inline_sym). Frames are stored root-to-leaf, so the leaf inline + // prints first and the base (root) prints last with the DSO path. + let mut mmap_table = super::MmapTable::default(); + mmap_table.insert_mmap(crate::perfdata::records::MmapRecord { + pid: 11, + tid: 11, + start: 0x1000, + len: 0x1000, + pgoff: 0, + path: "/tmp/oracle-workload".to_string(), + }); + let resolver = StaticFrameResolver { + frames: vec![ + "workload::main".to_string(), + "workload::churn_allocations".to_string(), + "core::slice::::sort_unstable".to_string(), + "core::slice::sort::unstable::sort".to_string(), + ], + has_base_symbol: true, + base_offset: Some("+0x1fb".to_string()), + }; + let mut symbol_cache = SymbolFrameCache::new(&resolver); + let mut written = Vec::new(); + + super::FoldFrameResolver::new(&mmap_table, true) + .write_script_frames_for_stack( + Some(11), + &[super::FoldFrame::UserUnwind(0x1427)], + Some(&mut symbol_cache), + &mut written, + ) + .expect("write perf script frames"); + + assert_eq!( + String::from_utf8(written).expect("utf-8"), + "\t 1427 core::slice::sort::unstable::sort+0x1fb (inlined)\n\ + \t 1427 core::slice::::sort_unstable+0x1fb (inlined)\n\ + \t 1427 workload::churn_allocations+0x1fb (inlined)\n\ + \t 1427 workload::main+0x1fb (/tmp/oracle-workload)\n" + ); + } + + #[test] + fn single_base_user_unwind_script_frame_keeps_its_baked_offset_once_like_perf_script() { + // A non-inline base frame already carries +0x in its label from the + // symtab with_offset form; it must not be doubled when --inline is set. + let mut mmap_table = super::MmapTable::default(); + mmap_table.insert_mmap(crate::perfdata::records::MmapRecord { + pid: 11, + tid: 11, + start: 0x1000, + len: 0x1000, + pgoff: 0, + path: "/tmp/oracle-workload".to_string(), + }); + let resolver = StaticFrameResolver { + frames: vec!["core::slice::sort::unstable::quicksort::quicksort+0x6cb".to_string()], + has_base_symbol: true, + base_offset: Some("+0x6cb".to_string()), + }; + let mut symbol_cache = SymbolFrameCache::new(&resolver); + let mut written = Vec::new(); + + super::FoldFrameResolver::new(&mmap_table, true) + .write_script_frames_for_stack( + Some(11), + &[super::FoldFrame::UserUnwind(0x16cb)], + Some(&mut symbol_cache), + &mut written, + ) + .expect("write perf script frames"); + + assert_eq!( + String::from_utf8(written).expect("utf-8"), + "\t 16cb core::slice::sort::unstable::quicksort::quicksort+0x6cb (/tmp/oracle-workload)\n" + ); + } + #[test] fn inferno_perf_render_cache_keeps_raw_functions_separate_from_folded_labels() { let mut buffers = super::FoldedRenderBuffers::default(); @@ -4308,12 +5041,15 @@ mod tests { super::FoldFrame::Callchain(0x1020), ]; + // Inline mode routes regular Callchain frames into the full-mapping + // batch; this test exercises the cross-stack dedup of those keys. super::extend_symbol_mappings_for_stack( Some(11), &callchain, &mmap_table, &mut mapping_cache, &mut batches, + true, ); super::extend_symbol_mappings_for_stack( Some(11), @@ -4321,6 +5057,7 @@ mod tests { &mmap_table, &mut mapping_cache, &mut batches, + true, ); assert_eq!(batches.full_mappings.len(), 2); @@ -4330,10 +5067,12 @@ mod tests { } #[test] - fn prefetch_symbols_batches_inline_current_ip_as_base_symbol_only() { - // perf's libdw path emits the initial frame as a map symbol before any - // inline expansion. Prefetching InlineCurrentIp through the full DWARF - // frame path repeats expensive object work on large perf.data files. + fn prefetch_symbols_batches_inline_current_ip_through_full_dwarf_with_inline() { + // perf's machine.c unwind_entry() runs append_inlines() on EVERY + // accepted entry, including the initial sampled IP (the InlineCurrentIp + // leaf), so with --inline the leaf is symbolized through the full DWARF + // inline chain exactly like a caller frame. Only the no-inline default + // resolves it from the single base symtab symbol. let mut mmap_table = super::MmapTable::default(); mmap_table.insert_mmap(crate::perfdata::records::MmapRecord { pid: 11, @@ -4357,11 +5096,13 @@ mod tests { let resolver = RecordingFrameResolver::default(); let mut symbol_cache = SymbolFrameCache::new(&resolver); - super::prefetch_symbols(&entries, &mmap_table, &mut symbol_cache) + // With --inline, both the caller (UserUnwind 0x1010) and the leaf + // (InlineCurrentIp 0x1020) prefetch the full DWARF inline chain. + super::prefetch_symbols(&entries, &mmap_table, &mut symbol_cache, true) .expect("prefetch folded stack symbols"); - assert_eq!(*resolver.full_requests.borrow(), vec![0x10]); - assert_eq!(*resolver.base_requests.borrow(), vec![0x20]); + assert_eq!(*resolver.full_requests.borrow(), vec![0x10, 0x20]); + assert!(resolver.base_requests.borrow().is_empty()); } #[test] @@ -4454,7 +5195,6 @@ mod tests { has_frames: true, }, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingMissing, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4477,7 +5217,6 @@ mod tests { has_frames: true, }, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 0, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4492,7 +5231,6 @@ mod tests { has_frames: false, }, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 0, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4507,7 +5245,6 @@ mod tests { has_frames: true, }, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: false, module_count: 0, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4526,7 +5263,6 @@ mod tests { has_frames: false, }, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4545,7 +5281,6 @@ mod tests { has_frames: true, }, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4567,7 +5302,6 @@ mod tests { has_frames: false, }, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: true, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4583,7 +5317,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4599,7 +5332,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: true, @@ -4615,7 +5347,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: true, syscall_return_state: false, @@ -4634,7 +5365,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithoutCallchain, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: true, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4656,7 +5386,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithoutCallchain, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4677,7 +5406,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithUserFrame, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: false, module_count: 1, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4688,7 +5416,7 @@ mod tests { #[test] fn object_unwind_keeps_syscall_return_callers_like_perf_libdw() { - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7ea_3f4b, sp: 0x7fff_ffff_9928, bp: 3, @@ -4702,9 +5430,9 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::KernelWithCallchain, - super::ObjectUnwindInitialFramePolicy::DropSyntheticCurrentIp, + false, vec![0x7fff_f7ea_3f4b, 0x5555_5578_8ba4, 0x5555_5578_8ba5], ), vec![0x7fff_f7ea_3f4b, 0x5555_5578_8ba4, 0x5555_5578_8ba5] @@ -4747,7 +5475,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: true, module_count: 2, frame_pointer_at_or_above_stack_pointer: true, syscall_return_state: true, @@ -4794,7 +5521,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: true, module_count: 2, frame_pointer_at_or_above_stack_pointer: true, syscall_return_state: false, @@ -4815,7 +5541,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 0, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: false, @@ -4831,7 +5556,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 0, frame_pointer_at_or_above_stack_pointer: false, syscall_return_state: true, @@ -4847,7 +5571,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::NoRecordedMapping, - initial_ip_is_dso: false, module_count: 0, frame_pointer_at_or_above_stack_pointer: true, syscall_return_state: false, @@ -4863,7 +5586,7 @@ mod tests { // executable because the full unwind path rejects that synthesized // stack before this acceptance step. tools/perf/util/unwind-libdw.c's // entry callback does not drop already-accepted executable frames. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x5555_5578_c601, sp: 0x7fff_ffff_8cf8, bp: 0x4002, @@ -4872,12 +5595,12 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::Other { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::DropSyntheticCurrentIp, + false, vec![0x5555_5578_c601, 0x5555_5579_6e23], ), vec![0x5555_5578_c601, 0x5555_5579_6e23] @@ -4889,7 +5612,7 @@ mod tests { // Real period 4754368 sample from target/profiling-runs/octo-latest-fold/profile.raw.perf.data: // perf script prints the _int_free_chunk glibc leaf even though the // recorded FP callchain itself is empty. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7e2_ecb7, sp: 0x7fff_ffff_8cf8, bp: 0x4002, @@ -4898,12 +5621,12 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::Other { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::KeepDsoLeaf, + false, vec![0x7fff_f7e2_ecb7], ), vec![0x7fff_f7e2_ecb7] @@ -5113,7 +5836,7 @@ mod tests { path: current_exe, }); - let mut state = super::PidUnwindState::default(); + let mut state = super::PidUnwindState::with_arch(PerfArch::X86_64); let loaded = super::report_unwind_modules_for_frame_callbacks_like_perf( &mut state, &accumulator.mmap_table, @@ -5151,7 +5874,7 @@ mod tests { path: current_exe, }); - let mut state = super::PidUnwindState::default(); + let mut state = super::PidUnwindState::with_arch(PerfArch::X86_64); let loaded = super::report_unwind_modules_for_frame_callbacks_like_perf( &mut state, &accumulator.mmap_table, @@ -5174,7 +5897,7 @@ mod tests { // but that decision belongs to the full unwind/truncation path. Once // libdw entry has accepted frames, there is no user-mode empty-callchain // filter here. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7f0_277b, sp: 0x7fff_ffff_8cf8, bp: 0x4002, @@ -5183,12 +5906,12 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::Other { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::KeepDsoLeaf, + false, vec![0x7fff_f7f0_277b, 0x5555_5579_6e23, 0x5555_5579_6e23], ), vec![0x7fff_f7f0_277b, 0x5555_5579_6e23, 0x5555_5579_6e23] @@ -5202,7 +5925,7 @@ mod tests { // recorded FP callchain has nr:0. tools/perf/util/unwind-libdw.c has no // blanket filter for user-mode samples with an empty callchain; it emits // each frame accepted by frame_callback -> entry. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7e5_7982, sp: 0x7fff_ffff_a1e0, bp: 0x7fff_ffff_a220, @@ -5211,12 +5934,12 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::Other { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::KeepDsoLeaf, + false, vec![ 0x7fff_f7e5_7982, 0x5555_555a_019e, @@ -5239,7 +5962,7 @@ mod tests { // perf script prints only __memmove_avx_unaligned_erms even though the // sampled BP points above SP; the full unwind path is responsible for // rejecting framehop-only tails that libdw did not accept. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7f0_277b, sp: 0x7fff_ffff_8938, bp: 0x7fff_ffff_9650, @@ -5248,12 +5971,12 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::Other { has_callchain: true, has_frames: false, }, - super::ObjectUnwindInitialFramePolicy::KeepDsoLeaf, + false, vec![0x7fff_f7f0_277b, 0x5555_556b_ab79], ), vec![0x7fff_f7f0_277b, 0x5555_556b_ab79] @@ -5266,7 +5989,7 @@ mod tests { // falls through to ebl_unwind(). The real period 803991 sample in the // octo profile takes this path: framehop returns no object frames, while // perf script prints the frame-pointer spine after the kernel stack. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7e1_c03e, sp: 0x7fff_ffff_9250, bp: 0x7fff_ffff_9260, @@ -5284,14 +6007,19 @@ mod tests { stack[0x28..0x30].copy_from_slice(&0x7fff_f7e9_9d7e_u64.to_le_bytes()); assert_eq!( - super::libdw_arch_fallback_after_empty_object_unwind(Vec::new(), ®s, &stack, true,), + super::libdw_arch_fallback_after_empty_object_unwind( + Vec::new(), + &PerfUserRegs::X86_64(regs), + &stack, + true, + ), vec![0x7fff_f7e1_c03e, 0x7fff_f7e1_c083, 0x7fff_f7e9_9d7d] ); } #[test] fn empty_object_unwind_arch_fallback_does_not_require_reported_mapping_like_elfutils() { - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x4000, sp: 0x8000, bp: 0x8000, @@ -5300,7 +6028,12 @@ mod tests { let stack = 0x5000_u64.to_le_bytes(); assert_eq!( - super::libdw_arch_fallback_after_empty_object_unwind(Vec::new(), ®s, &stack, false,), + super::libdw_arch_fallback_after_empty_object_unwind( + Vec::new(), + &PerfUserRegs::X86_64(regs), + &stack, + false, + ), Vec::::new() ); @@ -5308,7 +6041,6 @@ mod tests { sample_callchain: super::SampleCallchainPresence::Present, callchain: super::SampleCallchainState::KernelWithCallchain, initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, - initial_ip_is_dso: true, module_count: 1, frame_pointer_at_or_above_stack_pointer: true, syscall_return_state: true, @@ -5328,10 +6060,7 @@ mod tests { ); assert!( super::should_use_libdw_arch_fallback_after_empty_object_unwind( - super::UserUnwindContext { - initial_ip_is_dso: false, - ..matching_context - }, + super::UserUnwindContext { ..matching_context }, false ) ); @@ -5352,6 +6081,50 @@ mod tests { ); } + #[test] + fn aarch64_arch_fallback_fires_on_seed_only_object_unwind_like_libdw_ebl() { + // framehop's aarch64 unwinder yields only the seed pc when no CFI + // covers it; that is exactly when libdwfl invokes ebl_unwind on the + // leaf (backends/aarch64_unwind.c), so the fp-chain fallback must run + // even though framehop returned one frame. + let regs = PerfUserRegs::Aarch64(crate::perfdata::unwind::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1000, + fp: 0x1010, + lr: 0x5000, + }); + let stack = vec![0_u8; 0x40]; + + assert_eq!( + super::libdw_arch_fallback_after_empty_object_unwind(vec![0x4000], ®s, &stack, true,), + // pc, then the lr caller (perf pc-1 adjustment), then stop on the + // zeroed next lr. + vec![0x4000, 0x4fff] + ); + } + + #[test] + fn aarch64_arch_fallback_keeps_multi_frame_object_unwind() { + // When framehop already produced callers past the seed (CFI worked), + // the ebl fallback must not clobber them. + let regs = PerfUserRegs::Aarch64(crate::perfdata::unwind::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1000, + fp: 0x1010, + lr: 0x5000, + }); + + assert_eq!( + super::libdw_arch_fallback_after_empty_object_unwind( + vec![0x4000, 0x9000], + ®s, + &[0_u8; 0x40], + true, + ), + vec![0x4000, 0x9000] + ); + } + #[test] fn object_unwind_keeps_kernel_without_callchain_dso_tail_like_perf_libdw() { // Real period 144 __strlen_avx2 sample from @@ -5361,7 +6134,7 @@ mod tests { // callers. In perf util/unwind-libdw.c, frame_callback reports every // accepted frame via entry(); there is no kernel-without-callchain // post-filter that truncates to the leaf. - let regs = super::PerfX86_64Regs { + let regs = PerfX86_64Regs { ip: 0x7fff_f7f2_d344, sp: 0x7fff_ffff_a1d8, bp: 0x7fff_ffff_a220, @@ -5370,9 +6143,9 @@ mod tests { assert_eq!( super::perf_accepted_object_unwind_frames( - ®s, + &PerfUserRegs::X86_64(regs), super::SampleCallchainState::KernelWithoutCallchain, - super::ObjectUnwindInitialFramePolicy::KeepDsoLeaf, + false, vec![ 0x7fff_f7f2_d344, 0x5555_5559_a556, @@ -5397,7 +6170,8 @@ mod tests { super::sample_fold_count( Some(37), super::FoldOptions { - count_periods: true + count_periods: true, + ..super::FoldOptions::default() } ), 37 @@ -5406,7 +6180,8 @@ mod tests { super::sample_fold_count( Some(37), super::FoldOptions { - count_periods: false + count_periods: false, + ..super::FoldOptions::default() } ), 1 @@ -5419,7 +6194,8 @@ mod tests { super::sample_fold_count( None, super::FoldOptions { - count_periods: true + count_periods: true, + ..super::FoldOptions::default() } ), 1 @@ -5442,4 +6218,137 @@ mod tests { "alpha;leaf 7\nbeta;leaf 3\n" ); } + + fn other_callchain() -> super::SampleCallchainState { + super::SampleCallchainState::Other { + has_callchain: true, + has_frames: false, + } + } + + #[test] + fn arch_fallback_cannot_advance_only_when_x86_bp_below_sp() { + // backends/x86_64_unwind.c: the rbp fallback writes new_sp = fp + 16 + // and rejects the frame with `if (sp >= fp) return false;` — i.e. it + // advances only when the frame pointer sits above the stack pointer. + // pyroclast attempts the fallback only when `bp >= sp`, so `bp < sp` + // means the fallback can never produce a caller. + let mut below = test_x86_regs(0x4000); + below.sp = 0x7fff_0000; + below.bp = 0x7ffe_ff00; // bp < sp + assert!(super::arch_fallback_provably_cannot_advance( + &PerfUserRegs::X86_64(below) + )); + + let mut at_or_above = test_x86_regs(0x4000); + at_or_above.sp = 0x7fff_0000; + at_or_above.bp = 0x7fff_0008; // bp > sp + assert!(!super::arch_fallback_provably_cannot_advance( + &PerfUserRegs::X86_64(at_or_above) + )); + } + + #[test] + fn arch_fallback_cannot_advance_only_when_aarch64_lr_is_zero() { + // backends/aarch64_unwind.c: the caller pc comes from lr and the walk + // returns false immediately when `lr == 0`. fp/sp are irrelevant to + // whether the FIRST caller can be produced. + let zero_lr = crate::perfdata::unwind::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1000, + fp: 0x1010, + lr: 0, + }; + assert!(super::arch_fallback_provably_cannot_advance( + &PerfUserRegs::Aarch64(zero_lr) + )); + + let live_lr = crate::perfdata::unwind::PerfAarch64Regs { + lr: 0x5000, + ..zero_lr + }; + assert!(!super::arch_fallback_provably_cannot_advance( + &PerfUserRegs::Aarch64(live_lr) + )); + } + + #[test] + fn classify_object_unwind_routes_leaf_skip_and_unwind() { + let leaf_only_ctx = super::UserUnwindContext { + sample_callchain: super::SampleCallchainPresence::Present, + callchain: other_callchain(), + initial_ip_mapping: super::InitialIpMappingState::RecordedMappingLoaded, + module_count: 1, + frame_pointer_at_or_above_stack_pointer: false, + syscall_return_state: false, + }; + assert_eq!( + super::classify_object_unwind(leaf_only_ctx, true), + super::ObjectUnwindClass::LeafOnly + ); + assert_eq!( + super::classify_object_unwind(leaf_only_ctx, false), + super::ObjectUnwindClass::MustUnwind + ); + + // Research §3.5: a recorded kernel->user callchain is never extended + // with user DWARF callers, so it skips unwinding entirely regardless of + // the leaf-only predicate. + let kernel_user = super::UserUnwindContext { + callchain: super::SampleCallchainState::KernelWithUserFrame, + ..leaf_only_ctx + }; + assert_eq!( + super::classify_object_unwind(kernel_user, true), + super::ObjectUnwindClass::SkipUnwind + ); + assert_eq!( + super::classify_object_unwind(kernel_user, false), + super::ObjectUnwindClass::SkipUnwind + ); + } + + #[test] + fn accepted_frames_emit_leaf_only_when_predicate_holds() { + // gap-5gr: when the leaf-only predicate holds the accepted list is the + // single sampled-IP leaf, even if framehop produced a (spurious) + // caller — libdwfl would have stopped after the initial-frame callback. + let regs = test_regs(0x4000); + assert_eq!( + super::perf_accepted_object_unwind_frames(®s, other_callchain(), true, Vec::new()), + vec![0x4000] + ); + assert_eq!( + super::perf_accepted_object_unwind_frames( + ®s, + other_callchain(), + true, + vec![0x4000, 0x9999], + ), + vec![0x4000], + "a framehop heuristic caller is dropped when perf/libdwfl emits only the leaf" + ); + } + + #[test] + fn accepted_frames_keep_full_unwind_when_not_leaf_only() { + // When the predicate does not hold (e.g. CFI covers the IP), framehop + // is authoritative and every accepted frame is kept. + let regs = test_regs(0x4000); + assert_eq!( + super::perf_accepted_object_unwind_frames( + ®s, + other_callchain(), + false, + vec![0x4000, 0x9999], + ), + vec![0x4000, 0x9999] + ); + // A non-leaf-only sample with no accepted frames stays empty: the + // sampled IP is never invented absent the scenario-D predicate. + assert!( + super::perf_accepted_object_unwind_frames(®s, other_callchain(), false, Vec::new()) + .is_empty() + ); + } } diff --git a/src/perfdata/header.rs b/src/perfdata/header.rs index 29216c4..3a970d5 100644 --- a/src/perfdata/header.rs +++ b/src/perfdata/header.rs @@ -72,6 +72,49 @@ pub fn parse_feature_sections( Ok(sections) } +const HEADER_ARCH: u16 = 6; + +/// Reads the HEADER_ARCH feature string (the recording machine's `uname -m`, +/// e.g. `x86_64` or `aarch64`). +/// +/// perf stores it as a `perf_header_string`: a u32 length followed by that many +/// bytes containing a NUL-terminated string (util/header.c `do_read_string`). +/// +/// # Errors +/// +/// Returns an error when the header or feature table is malformed. A missing +/// HEADER_ARCH feature is `Ok(None)`. +pub fn parse_header_arch(bytes: &[u8], header: &PerfHeader) -> Result, String> { + let Some(section) = parse_feature_sections(bytes, header)? + .into_iter() + .find(|section| section.feature == HEADER_ARCH) + else { + return Ok(None); + }; + let start = usize::try_from(section.offset) + .map_err(|_| "arch feature offset exceeds usize".to_string())?; + let size = + usize::try_from(section.size).map_err(|_| "arch feature size exceeds usize".to_string())?; + let end = start + .checked_add(size) + .ok_or_else(|| "arch feature range overflows usize".to_string())?; + let payload = bytes + .get(start..end) + .ok_or_else(|| "arch feature payload is truncated".to_string())?; + let length = usize::try_from(crate::perfdata::endian::read_u32(payload, 0)?) + .map_err(|_| "arch feature string length exceeds usize".to_string())?; + let string = payload + .get(4..4 + length) + .ok_or_else(|| "arch feature string is truncated".to_string())?; + let end = string + .iter() + .position(|byte| *byte == 0) + .unwrap_or(string.len()); + std::str::from_utf8(&string[..end]) + .map(|arch| Some(arch.to_string())) + .map_err(|error| format!("arch feature string is not UTF-8: {error}")) +} + fn feature_table_offset(header: &PerfHeader) -> Result { let offset = header .data_offset @@ -81,9 +124,13 @@ fn feature_table_offset(header: &PerfHeader) -> Result { } fn set_feature_bits(bytes: &[u8]) -> Result, String> { + // tools/perf/util/header.h struct perf_file_header lays out the + // adds_features DECLARE_BITMAP at byte offset 72: magic(8) + size(8) + + // attr_size(8) + attrs(16) + data(16) + event_types(16) = 72. The bitmap + // spans HEADER_FEAT_BITS=256 bits (four u64 words). let mut features = Vec::new(); for word_index in 0..4 { - let word = read_u64(bytes, 56 + word_index * 8)?; + let word = read_u64(bytes, 72 + word_index * 8)?; for bit_index in 0..64 { if word & (1_u64 << bit_index) != 0 { let feature = u16::try_from(word_index * 64 + bit_index) diff --git a/src/perfdata/mappings.rs b/src/perfdata/mappings.rs index 36cd312..1525057 100644 --- a/src/perfdata/mappings.rs +++ b/src/perfdata/mappings.rs @@ -54,10 +54,33 @@ pub struct FileIdentity { pub inode_generation: u64, } +/// Reports whether the on-disk file at `path` carries the same backing-storage +/// identity (device major/minor + inode, and inode generation when recorded) +/// that perf captured in the `PERF_RECORD_MMAP2` event. +/// +/// This mirrors perf's `__dso_id__cmp` (tools/perf/util/dso.c), which compares +/// `maj`/`min`/`ino` together — never the inode alone — so two files sharing an +/// inode number on different filesystems are not treated as the same backing +/// store. Inode numbers are unique only within a single device, so comparing +/// `ino` without the device would admit cross-filesystem false matches. +/// +/// Note: perf does not use this device/inode identity to *reject* an on-disk +/// file before symbolizing or unwinding from it (`dso__load` and +/// `do_open`/`__open_dso` trust the path and only validate build-ids when both +/// the recorded and on-disk build-ids are defined). This helper exists for the +/// dso-instance identity comparison perf performs in `__dso_id__cmp`, and must +/// match that semantics: device-aware, with absent generation skipped. #[must_use] #[cfg(unix)] pub fn file_matches_recorded_identity(path: &Path, identity: FileIdentity) -> bool { - std::fs::metadata(path).is_ok_and(|metadata| metadata.ino() == identity.inode) + std::fs::metadata(path).is_ok_and(|metadata| { + let device = metadata.dev(); + // PERF_RECORD_MMAP2 records maj/min as MAJOR(dev)/MINOR(dev); decompose + // the on-disk st_dev with the matching macros before comparing. + major(device) == identity.major + && minor(device) == identity.minor + && metadata.ino() == identity.inode + }) } #[cfg(not(unix))] @@ -65,6 +88,21 @@ pub fn file_matches_recorded_identity(_path: &Path, _identity: FileIdentity) -> false } +/// Extracts the device major number from a `st_dev` value using the glibc +/// encoding userspace `stat` reports, matching the kernel `MAJOR()` macro perf +/// records in `PERF_RECORD_MMAP2`. +#[cfg(unix)] +fn major(device: u64) -> u32 { + (((device >> 8) & 0xfff) | ((device >> 32) & !0xfff)) as u32 +} + +/// Extracts the device minor number from a `st_dev` value, matching the kernel +/// `MINOR()` macro perf records in `PERF_RECORD_MMAP2`. +#[cfg(unix)] +fn minor(device: u64) -> u32 { + ((device & 0xff) | ((device >> 12) & !0xff)) as u32 +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct UserMapping<'a> { pub pid: u32, @@ -181,6 +219,16 @@ impl MmapTable { } fn insert_mapping(&mut self, mapping: Mapping) { + // Common case: the new mapping does not overlap any existing mapping for + // its pid. Detect this in O(log n + matches) using the per-pid interval + // index and take a pure incremental insert, skipping the whole-table + // `mem::take` and global index rebuild. Only an actual overlap (a split) + // falls back to the rebuild-based path, which preserves perf's exact + // retained-mapping ordering and split semantics. + if !self.has_overlapping_mapping_for_pid(mapping.pid, mapping.start, mapping.end()) { + self.insert_mapping_without_overlap_fix(mapping); + return; + } let split_mappings = self.remove_overlapping_mappings_like_perf(&mapping); for split in split_mappings { self.insert_mapping_without_overlap_fix(split); @@ -216,6 +264,46 @@ impl MmapTable { split_mappings } + /// Returns whether any existing mapping for `pid` overlaps the half-open + /// range `[start, end)`, using the per-pid interval index so the common + /// no-overlap case is `O(log n + matches)` rather than a full scan. + fn has_overlapping_mapping_for_pid(&self, pid: u32, start: u64, end: u64) -> bool { + self.any_indexed_mapping_in_range(pid, start, end, |_mapping| true) + } + + /// Walks the per-pid interval index for mappings whose `[start, end)` range + /// intersects `[start, end)` and reports whether any matching mapping also + /// satisfies `predicate`. Mirrors the `max_end`-augmented descent used by + /// `resolve_mapping_index_for_pid`, but tests interval intersection instead + /// of point containment. + fn any_indexed_mapping_in_range( + &self, + pid: u32, + start: u64, + end: u64, + mut predicate: impl FnMut(&Mapping) -> bool, + ) -> bool { + let Some(bucket) = self.mappings_by_pid.get(&pid) else { + return false; + }; + // Mappings are sorted by `start`; only those with `start < end` can + // overlap, so descend from the last such entry. The augmented `max_end` + // lets us stop once no earlier mapping can reach past `start`. + let mut upper_bound = bucket.partition_point(|indexed| indexed.start < end); + while upper_bound > 0 { + upper_bound -= 1; + let indexed = &bucket[upper_bound]; + if indexed.max_end <= start { + break; + } + let mapping = &self.mappings[indexed.index]; + if start < mapping.end() && mapping.start < end && predicate(mapping) { + return true; + } + } + false + } + fn insert_mapping_without_overlap_fix(&mut self, mut mapping: Mapping) { let pid = mapping.pid; let start = mapping.start; @@ -364,12 +452,7 @@ impl MmapTable { len: u64, ) -> bool { let end = start.saturating_add(len); - self.mappings.iter().any(|mapping| { - mapping.pid == pid - && mapping.is_user_file_mapping() - && start < mapping.end() - && mapping.start < end - }) + self.any_indexed_mapping_in_range(pid, start, end, Mapping::is_user_file_mapping) } #[must_use] @@ -597,14 +680,31 @@ impl Mapping { } fn symbol_source_key(&self) -> SymbolSourceKey { + // perf identifies a dso backing store via __dso_id__cmp + // (tools/perf/util/dso.c): once both sides carry a defined build_id it + // is the decisive comparison, and the mmap2 maj/min/ino are only + // weighed when both sides recorded them. The same on-disk object can + // therefore reach us as an inline MMAP2-build-id record (build_id, no + // file_identity) or as a plain MMAP2 plus a HEADER_BUILD_ID entry + // (build_id and file_identity). Keying on file_identity alongside the + // build_id would split those into two symbol sources, so when a + // build_id is present we drop file_identity from the key and rely on + // (path, build_id) — preserving distinct build_ids at the same path, + // and falling back to file_identity only when no build_id exists. + let build_id = self.build_id.clone(); + let file_identity = if build_id.is_some() { + None + } else { + self.file_identity + }; SymbolSourceKey { path: if self.is_kernel_symbol_mapping() && self.path.starts_with("[kernel") { "[kernel.kallsyms]".to_string() } else { self.path.clone() }, - build_id: self.build_id.clone(), - file_identity: self.file_identity, + build_id, + file_identity, kernel_relocation: self.kernel_relocation(), } } @@ -735,4 +835,137 @@ mod tests { assert!(!table.has_mapping_for_pid_cached(7, 0x5000, &mut cache)); assert_eq!(cache.pid_index, None); } + + #[test] + fn incremental_insert_splits_multiple_overlapping_mappings_like_perf() { + // Two adjacent mappings, then a third that straddles both: the overlap + // path must remove both originals, emit before/after fragments in the + // perf-source order, and let the newest mapping win the shared interior. + let mut table = MmapTable::default(); + table.insert_mmap(MmapRecord { + pid: 7, + tid: 7, + start: 0x1000, + len: 0x1000, + pgoff: 0, + path: "/first".to_string(), + }); + table.insert_mmap(MmapRecord { + pid: 7, + tid: 7, + start: 0x2000, + len: 0x1000, + pgoff: 0, + path: "/second".to_string(), + }); + table.insert_mmap(MmapRecord { + pid: 7, + tid: 7, + start: 0x1800, + len: 0x1000, + pgoff: 0x100, + path: "/straddle".to_string(), + }); + + // /first head survives below the straddle, /second tail above it. + assert_eq!(table.resolve(7, 0x1400).expect("first head").path, "/first"); + assert_eq!( + table.resolve(7, 0x1900).expect("straddle body").path, + "/straddle" + ); + assert_eq!( + table.resolve(7, 0x2900).expect("second tail").path, + "/second" + ); + let straddle = table.resolve(7, 0x1900).expect("straddle relative"); + assert_eq!(straddle.relative_address, 0x100 + 0x100); + } + + #[test] + fn non_overlapping_insert_takes_fast_path_and_indexes_correctly() { + // Disjoint mappings (and a different pid) must not be treated as + // overlapping, so each insert takes the incremental fast path while + // still resolving and pruning correctly. + let mut table = MmapTable::default(); + table.insert_mmap(MmapRecord { + pid: 7, + tid: 7, + start: 0x1000, + len: 0x100, + pgoff: 0, + path: "/a".to_string(), + }); + table.insert_mmap(MmapRecord { + pid: 7, + tid: 7, + start: 0x3000, + len: 0x100, + pgoff: 0, + path: "/b".to_string(), + }); + table.insert_mmap(MmapRecord { + pid: 9, + tid: 9, + start: 0x1000, + len: 0x100, + pgoff: 0, + path: "/other-pid".to_string(), + }); + + assert!(!table.has_overlapping_user_mapping_for_pid(7, 0x1100, 0x100)); + assert!(!table.has_overlapping_user_mapping_for_pid(7, 0x2000, 0x800)); + assert!(table.has_overlapping_user_mapping_for_pid(7, 0x10ff, 0x100)); + assert!(table.has_overlapping_user_mapping_for_pid(7, 0x3080, 0x100)); + // Adjacency is not overlap (half-open ranges). + assert!(!table.has_overlapping_user_mapping_for_pid(7, 0x1100, 0x10)); + // Different pid's mapping must not count. + assert!(!table.has_overlapping_user_mapping_for_pid(8, 0x1000, 0x100)); + assert_eq!(table.resolve(7, 0x1050).expect("/a").path, "/a"); + assert_eq!(table.resolve(7, 0x3050).expect("/b").path, "/b"); + } + + #[test] + fn has_overlapping_user_mapping_matches_linear_scan_oracle() { + // The index-based overlap query must agree with an exhaustive linear + // scan across a dense, multi-pid set of mappings (including bracket + // paths that are not user-file mappings). + let segments: &[(u32, u64, u64, &str)] = &[ + (7, 0x1000, 0x400, "/bin/a"), + (7, 0x1400, 0x400, "/bin/b"), + (7, 0x2000, 0x100, "/bin/c"), + (7, 0x2500, 0x800, "/bin/d"), + (7, 0x3000, 0x200, "[anon]"), + (9, 0x1200, 0x600, "/bin/e"), + (9, 0x4000, 0x100, "/bin/f"), + ]; + let mut table = MmapTable::default(); + for &(pid, start, len, path) in segments { + table.insert_mmap(MmapRecord { + pid, + tid: pid, + start, + len, + pgoff: 0, + path: path.to_string(), + }); + } + + for pid in [7_u32, 8, 9] { + for start in (0x0u64..0x5000).step_by(0x80) { + for len in [0u64, 0x40, 0x100, 0x900] { + let end = start.saturating_add(len); + let expected = segments.iter().any(|&(seg_pid, seg_start, seg_len, path)| { + let seg_end = seg_start + seg_len; + let is_user = !path.starts_with('['); + seg_pid == pid && is_user && start < seg_end && seg_start < end + }); + assert_eq!( + table.has_overlapping_user_mapping_for_pid(pid, start, len), + expected, + "pid={pid} start={start:#x} len={len:#x}" + ); + } + } + } + } } diff --git a/src/perfdata/unwind.rs b/src/perfdata/unwind.rs index 89b1fba..08262b4 100644 --- a/src/perfdata/unwind.rs +++ b/src/perfdata/unwind.rs @@ -1,13 +1,27 @@ +use std::cell::RefCell; +use std::collections::HashMap; use std::fs::File; use std::ops::{Deref, Range}; use std::path::Path; use std::sync::Arc; +use framehop::aarch64::{CacheAarch64, UnwindRegsAarch64, UnwinderAarch64}; use framehop::x86_64::{CacheX86_64, Reg, UnwindRegsX86_64, UnwinderX86_64}; -use framehop::{ExplicitModuleSectionInfo, Unwinder}; +use framehop::{ExplicitModuleSectionInfo, Module, Unwinder}; use gimli::{BaseAddresses, CieOrFde, DebugFrame, EhFrame, LittleEndian, UnwindSection}; use memmap2::Mmap; use object::read::{Object, ObjectSection, ObjectSegment}; +use rustc_hash::FxBuildHasher; + +// The gap-2 skip gate queries `has_unwind_info_for_ip` once per sampled IP, +// and the same hot leaves (libc `malloc`/`memmove`/`memcmp`) recur across +// millions of samples. Memoizing collapses the otherwise-linear FDE-range scan +// (`.ace-review-findings.md` PERF-6) into an O(1) lookup. The memo is keyed by +// the EXACT ip, not `ip >> 12`: FDE pc-ranges are function-granular and two +// functions (one covered, one not) can share a 4 KiB page, so a page-granular +// memo could return a stale answer for a second IP in the page and perturb the +// skip-gate decision. Exact-ip keying keeps the answer byte-identical to the +// linear scan while still collapsing the dominant repeated-leaf query pattern. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct PerfX86_64Regs { @@ -17,6 +31,121 @@ pub struct PerfX86_64Regs { pub registers: [u64; 16], } +/// The architecture a perf.data file's user register samples were recorded on, +/// from the HEADER_ARCH feature string. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum PerfArch { + #[default] + X86_64, + Aarch64, +} + +impl PerfArch { + #[must_use] + pub fn from_header_arch(arch: &str) -> Option { + match arch { + "x86_64" | "amd64" => Some(Self::X86_64), + "aarch64" | "arm64" => Some(Self::Aarch64), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PerfAarch64Regs { + pub pc: u64, + pub sp: u64, + pub fp: u64, + pub lr: u64, +} + +/// Architecture-neutral user register sample, decoded from a perf REGS_USER +/// payload according to the recording machine's arch. +/// +/// The fold path threads this through every unwind site so the x86_64 and +/// aarch64 register layouts and frame-pointer fallbacks stay byte-faithful to +/// perf/elfutils without forcing a fake bp/sp onto aarch64. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PerfUserRegs { + X86_64(PerfX86_64Regs), + Aarch64(PerfAarch64Regs), +} + +impl PerfUserRegs { + /// Decodes the minimal user register set for `arch` from perf's ascending + /// register-mask encoding. + /// + /// # Errors + /// + /// Returns an error when the value slice does not match the mask or a + /// register required for unwinding is missing. + pub fn from_perf_masked_values( + arch: PerfArch, + mask: u64, + values: &[u64], + ) -> Result { + match arch { + PerfArch::X86_64 => { + PerfX86_64Regs::from_perf_masked_values(mask, values).map(Self::X86_64) + } + PerfArch::Aarch64 => { + PerfAarch64Regs::from_perf_masked_values(mask, values).map(Self::Aarch64) + } + } + } + + #[must_use] + pub fn arch(self) -> PerfArch { + match self { + Self::X86_64(_) => PerfArch::X86_64, + Self::Aarch64(_) => PerfArch::Aarch64, + } + } + + /// The sampled instruction pointer (x86_64 IP / aarch64 PC). + #[must_use] + pub fn ip(self) -> u64 { + match self { + Self::X86_64(regs) => regs.ip, + Self::Aarch64(regs) => regs.pc, + } + } + + /// The sampled stack pointer. + #[must_use] + pub fn sp(self) -> u64 { + match self { + Self::X86_64(regs) => regs.sp, + Self::Aarch64(regs) => regs.sp, + } + } + + /// Whether the sample looks like an x86_64 syscall-return state, which perf + /// truncates after the first executable frame. aarch64 has no analogue, so + /// this is always `false` there. + #[must_use] + pub fn is_syscall_return_state(self) -> bool { + match self { + Self::X86_64(regs) => regs.is_syscall_return_state(), + Self::Aarch64(_) => false, + } + } + + /// The x86_64 `ebl_unwind` frame-pointer precondition `bp >= sp`. + /// + /// elfutils' x86_64 backend only walks the rbp chain when the frame pointer + /// sits at or above the stack pointer. aarch64's backend has no such + /// precondition (its accept condition is internal to the walk), so this + /// returns `false` there and the fallback is gated differently. + #[must_use] + pub fn frame_pointer_at_or_above_stack_pointer(self) -> bool { + match self { + Self::X86_64(regs) => regs.bp >= regs.sp, + Self::Aarch64(_) => false, + } + } +} + pub struct PerfStackReader<'a> { sp: u64, bytes: &'a [u8], @@ -29,11 +158,29 @@ pub struct PerfUserMemoryReader<'a, F> { } pub struct FramehopUnwinder { - unwinder: UnwinderX86_64, - cache: CacheX86_64, + arch: ArchUnwinder, module_count: usize, reported_modules: Vec, rejected_mapping_ranges: Vec>, + /// Exact-ip memo for `has_unwind_info_for_ip`. Interior-mutable so the + /// predicate stays `&self`; cleared whenever a new module is added, since + /// that can extend coverage over a previously-uncovered ip. + unwind_info_memo: RefCell>, +} + +/// Per-architecture framehop unwinder and cache. Module registration is shared +/// (both arches register `framehop::Module` from the same +/// `ExplicitModuleSectionInfo`); only the seeded registers and `iter_frames` +/// differ, so the regs handed to `unwind` must match the active arch. +enum ArchUnwinder { + X86_64 { + unwinder: Box>, + cache: CacheX86_64, + }, + Aarch64 { + unwinder: Box>, + cache: CacheAarch64, + }, } #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -45,7 +192,7 @@ pub struct UserStackUnwindResult { pub trait UserStackUnwinder { fn unwind_user_stack( &mut self, - regs: PerfX86_64Regs, + regs: PerfUserRegs, stack: &[u8], max_frames: usize, ) -> UserStackUnwindResult; @@ -139,12 +286,27 @@ impl Default for FramehopUnwinder { impl FramehopUnwinder { #[must_use] pub fn new() -> Self { + Self::with_arch(PerfArch::X86_64) + } + + #[must_use] + pub fn with_arch(arch: PerfArch) -> Self { + let arch = match arch { + PerfArch::X86_64 => ArchUnwinder::X86_64 { + unwinder: Box::new(UnwinderX86_64::new()), + cache: CacheX86_64::new(), + }, + PerfArch::Aarch64 => ArchUnwinder::Aarch64 { + unwinder: Box::new(UnwinderAarch64::new()), + cache: CacheAarch64::new(), + }, + }; Self { - unwinder: UnwinderX86_64::new(), - cache: CacheX86_64::new(), + arch, module_count: 0, reported_modules: Vec::new(), rejected_mapping_ranges: Vec::new(), + unwind_info_memo: RefCell::new(HashMap::with_hasher(FxBuildHasher)), } } @@ -204,13 +366,13 @@ impl FramehopUnwinder { let section_info = explicit_module_section_info(&mapped, &object); let memory_segments = module_memory_segments(&mapped, &object, base); let unwind_ranges = object_unwind_ranges(&object, base); - let module = framehop::Module::::new( + let module = Module::::new( path.to_string_lossy().into_owned(), module_range.clone(), base, section_info, ); - self.unwinder.add_module(module); + self.arch.add_module(module); self.reported_modules.push(ReportedModule { base, range: module_range, @@ -218,6 +380,10 @@ impl FramehopUnwinder { unwind_ranges, }); self.module_count += 1; + // A newly reported module can add CFI coverage over an ip that was + // previously memoized as uncovered; drop the memo so the next query + // re-scans against the full module set. + self.unwind_info_memo.borrow_mut().clear(); Ok(true) } @@ -242,9 +408,15 @@ impl FramehopUnwinder { #[must_use] pub fn has_unwind_info_for_ip(&self, ip: u64) -> bool { - self.reported_modules + if let Some(&cached) = self.unwind_info_memo.borrow().get(&ip) { + return cached; + } + let covered = self + .reported_modules .iter() - .any(|module| module.unwind_ranges.iter().any(|range| range.contains(&ip))) + .any(|module| module.unwind_ranges.iter().any(|range| range.contains(&ip))); + self.unwind_info_memo.borrow_mut().insert(ip, covered); + covered } #[must_use] @@ -255,7 +427,7 @@ impl FramehopUnwinder { #[must_use] pub fn unwind_stack( &mut self, - regs: PerfX86_64Regs, + regs: PerfUserRegs, stack: &[u8], max_frames: usize, ) -> Vec { @@ -266,34 +438,26 @@ impl FramehopUnwinder { #[must_use] pub fn unwind_stack_with_diagnostics( &mut self, - regs: PerfX86_64Regs, + regs: PerfUserRegs, stack: &[u8], max_frames: usize, ) -> UserStackUnwindResult { if self .rejected_mapping_ranges .iter() - .any(|range| range.contains(®s.ip)) + .any(|range| range.contains(®s.ip())) { return UserStackUnwindResult::default(); } let reported_modules = &self.reported_modules; - let mut memory_reader = PerfUserMemoryReader::new(regs.sp, stack, |address| { + let mut memory_reader = PerfUserMemoryReader::new(regs.sp(), stack, |address| { read_reported_module_u64(reported_modules, address) }); let mut read_stack = |address| memory_reader.read_u64(address).ok_or(()); - let ip = regs.ip; - let regs = regs.to_framehop_regs(); - let mut iter = self - .unwinder - .iter_frames(ip, regs, &mut self.cache, &mut read_stack); - let mut frames = Vec::new(); - while frames.len() < max_frames { - let Ok(Some(frame)) = iter.next() else { - break; - }; - push_perf_unwind_address(&mut frames, frame.address()); - } + let ip = regs.ip(); + let frames = self + .arch + .iter_addresses(ip, regs, &mut read_stack, max_frames); let framehop_frame_count = frames.len(); UserStackUnwindResult { accepted_frames: frames, @@ -302,10 +466,54 @@ impl FramehopUnwinder { } } +impl ArchUnwinder { + fn add_module(&mut self, module: Module) { + match self { + Self::X86_64 { unwinder, .. } => unwinder.add_module(module), + Self::Aarch64 { unwinder, .. } => unwinder.add_module(module), + } + } + + fn iter_addresses( + &mut self, + ip: u64, + regs: PerfUserRegs, + read_stack: &mut impl FnMut(u64) -> Result, + max_frames: usize, + ) -> Vec { + let mut frames = Vec::new(); + // The seeded register file must match the active arch; a mismatch means + // the file header arch and the regs decode disagreed, which cannot + // happen because both flow from the same PerfArch. + match (self, regs) { + (Self::X86_64 { unwinder, cache }, PerfUserRegs::X86_64(regs)) => { + let mut iter = unwinder.iter_frames(ip, regs.to_framehop_regs(), cache, read_stack); + while frames.len() < max_frames { + let Ok(Some(frame)) = iter.next() else { + break; + }; + push_perf_unwind_address(&mut frames, frame.address()); + } + } + (Self::Aarch64 { unwinder, cache }, PerfUserRegs::Aarch64(regs)) => { + let mut iter = unwinder.iter_frames(ip, regs.to_framehop_regs(), cache, read_stack); + while frames.len() < max_frames { + let Ok(Some(frame)) = iter.next() else { + break; + }; + push_perf_unwind_address(&mut frames, frame.address()); + } + } + _ => {} + } + frames + } +} + impl UserStackUnwinder for FramehopUnwinder { fn unwind_user_stack( &mut self, - regs: PerfX86_64Regs, + regs: PerfUserRegs, stack: &[u8], max_frames: usize, ) -> UserStackUnwindResult { @@ -698,6 +906,94 @@ impl PerfX86_64Regs { } } +impl PerfAarch64Regs { + /// Builds the minimal aarch64 register set needed for stack unwinding from + /// perf's ascending register-mask encoding (`PERF_REG_ARM64_*`: x29/fp=29, + /// lr=30, sp=31, pc=32). + /// + /// # Errors + /// + /// Returns an error when the value slice does not match the number of set + /// bits in `mask` or a required register is missing. + pub fn from_perf_masked_values(mask: u64, values: &[u64]) -> Result { + const FP: u32 = 29; + const LR: u32 = 30; + const SP: u32 = 31; + const PC: u32 = 32; + + if mask.count_ones() as usize != values.len() { + return Err("perf register mask and value count differ".to_string()); + } + + let masked = |register: u32| -> Option { + if mask & (1_u64 << register) == 0 { + return None; + } + let index = (mask & ((1_u64 << register) - 1)).count_ones() as usize; + values.get(index).copied() + }; + + Ok(Self { + pc: masked(PC) + .ok_or_else(|| "perf sample is missing aarch64 PC register".to_string())?, + sp: masked(SP) + .ok_or_else(|| "perf sample is missing aarch64 SP register".to_string())?, + fp: masked(FP) + .ok_or_else(|| "perf sample is missing aarch64 FP register".to_string())?, + lr: masked(LR) + .ok_or_else(|| "perf sample is missing aarch64 LR register".to_string())?, + }) + } + + #[must_use] + pub fn to_framehop_regs(self) -> UnwindRegsAarch64 { + UnwindRegsAarch64::new(self.lr, self.sp, self.fp) + } +} + +/// Walks an aarch64 frame-pointer chain the way elfutils' `ebl_unwind` backend +/// does when no CFI covers the program counter. +/// +/// Faithful to elfutils backends/aarch64_unwind.c: the caller's pc is the +/// current lr (zero lr ends the walk before any caller is accepted), the next +/// lr/fp load from `fp+8`/`fp+0` (zero on failed reads), the next sp is +/// `fp+16`, and a step is accepted iff `fp == 0 || new_sp > sp`. Unlike the +/// x86_64 backend there is no `fp >= sp` precondition, so a zero frame pointer +/// still yields one lr-based caller. +#[must_use] +pub fn unwind_aarch64_frame_pointer_stack_like_elfutils( + regs: PerfAarch64Regs, + stack: &[u8], + max_frames: usize, +) -> Vec { + let memory_reader = PerfStackReader::new(regs.sp, stack); + let mut frames = Vec::new(); + if max_frames == 0 { + return frames; + } + + frames.push(regs.pc); + let mut lr = regs.lr; + let mut fp = regs.fp; + let mut sp = regs.sp; + while frames.len() < max_frames { + if lr == 0 { + break; + } + let new_lr = memory_reader.read_u64(fp.saturating_add(8)).unwrap_or(0); + let new_fp = memory_reader.read_u64(fp).unwrap_or(0); + let new_sp = fp.saturating_add(16); + if fp != 0 && new_sp <= sp { + break; + } + push_perf_unwind_address(&mut frames, lr); + lr = new_lr; + fp = new_fp; + sp = new_sp; + } + frames +} + impl<'a> PerfStackReader<'a> { #[must_use] pub fn new(sp: u64, bytes: &'a [u8]) -> Self { @@ -915,4 +1211,123 @@ mod tests { assert_eq!(frames, vec![0x1000, 0x2000]); } + + #[test] + fn decodes_aarch64_registers_from_perf_mask() { + // perf record --call-graph dwarf on arm64 captures x0-x30, sp, pc. + let mask = (1_u64 << 33) - 1; + let mut values = (0_u64..33).collect::>(); + values[29] = 0x2900; // fp + values[30] = 0x3000; // lr + values[31] = 0x3100; // sp + values[32] = 0x3200; // pc + + let regs = super::PerfAarch64Regs::from_perf_masked_values(mask, &values).expect("regs"); + + assert_eq!( + regs, + super::PerfAarch64Regs { + pc: 0x3200, + sp: 0x3100, + fp: 0x2900, + lr: 0x3000, + } + ); + } + + #[test] + fn decodes_aarch64_registers_from_sparse_perf_mask() { + let mask = (1 << 29) | (1 << 30) | (1 << 31) | (1 << 32); + let values = [0x2900, 0x3000, 0x3100, 0x3200]; + + let regs = super::PerfAarch64Regs::from_perf_masked_values(mask, &values).expect("regs"); + + assert_eq!(regs.fp, 0x2900); + assert_eq!(regs.lr, 0x3000); + assert_eq!(regs.sp, 0x3100); + assert_eq!(regs.pc, 0x3200); + } + + #[test] + fn rejects_aarch64_registers_missing_pc() { + let mask = (1 << 29) | (1 << 30) | (1 << 31); + let values = [0x2900, 0x3000, 0x3100]; + + let error = + super::PerfAarch64Regs::from_perf_masked_values(mask, &values).expect_err("missing pc"); + + assert!(error.contains("PC")); + } + + #[test] + fn aarch64_frame_pointer_unwind_walks_fp_chain_like_elfutils() { + // Stack layout (sp = 0x1000): fp chain records at fp+0 / lr at fp+8, + // matching elfutils aarch64_unwind.c FP_OFFSET/LR_OFFSET/SP_OFFSET. + let regs = super::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1000, + fp: 0x1010, + lr: 0x5000, + }; + let mut stack = vec![0_u8; 0x40]; + // frame at fp=0x1010: next fp = 0x1030, next lr = 0x6000 + stack[0x10..0x18].copy_from_slice(&0x1030_u64.to_le_bytes()); + stack[0x18..0x20].copy_from_slice(&0x6000_u64.to_le_bytes()); + // frame at fp=0x1030: next fp = 0, next lr = 0 (end of chain) + stack[0x30..0x38].copy_from_slice(&0_u64.to_le_bytes()); + stack[0x38..0x40].copy_from_slice(&0_u64.to_le_bytes()); + + let frames = super::unwind_aarch64_frame_pointer_stack_like_elfutils(regs, &stack, 256); + + // Return addresses after the leaf take the perf `pc - 1` adjustment. + assert_eq!(frames, vec![0x4000, 0x4fff, 0x5fff]); + } + + #[test] + fn aarch64_frame_pointer_unwind_accepts_one_lr_caller_when_fp_is_zero() { + // elfutils: `return fp == 0 || newSp > sp` — a zero fp still accepts + // the lr-based caller, then the walk ends on the zeroed next lr. + let regs = super::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1000, + fp: 0, + lr: 0x5000, + }; + + let frames = + super::unwind_aarch64_frame_pointer_stack_like_elfutils(regs, &[0_u8; 0x20], 256); + + assert_eq!(frames, vec![0x4000, 0x4fff]); + } + + #[test] + fn aarch64_frame_pointer_unwind_rejects_backwards_stack_growth() { + // fp != 0 and newSp (fp+16) <= sp must discard the candidate caller. + let regs = super::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1020, + fp: 0x1000, + lr: 0x5000, + }; + + let frames = + super::unwind_aarch64_frame_pointer_stack_like_elfutils(regs, &[0_u8; 0x40], 256); + + assert_eq!(frames, vec![0x4000]); + } + + #[test] + fn aarch64_frame_pointer_unwind_stops_on_zero_lr_without_callers() { + let regs = super::PerfAarch64Regs { + pc: 0x4000, + sp: 0x1000, + fp: 0x1010, + lr: 0, + }; + + let frames = + super::unwind_aarch64_frame_pointer_stack_like_elfutils(regs, &[0_u8; 0x40], 256); + + assert_eq!(frames, vec![0x4000]); + } } diff --git a/src/platform.rs b/src/platform.rs index d6dd259..57dba5c 100644 --- a/src/platform.rs +++ b/src/platform.rs @@ -62,33 +62,23 @@ impl ThreadLister for UnsupportedThreadLister { /// Returns an error when the task directory cannot be read or contains no /// numeric thread IDs. pub fn linux_thread_ids_from_proc(proc_root: &Path, pid: u32) -> std::io::Result> { - #[cfg(not(target_os = "linux"))] - { - let _ = (proc_root, pid); - return UnsupportedThreadLister.thread_ids(pid); - } - - #[cfg(target_os = "linux")] - { - let process = procfs::process::Process::new_with_root(proc_root.join(pid.to_string())) - .map_err(std::io::Error::other)?; - let mut tids = process - .tasks() - .map_err(std::io::Error::other)? - .filter_map(Result::ok) - .filter_map(|task| u32::try_from(task.tid).ok()) - .collect::>(); - tids.sort_unstable(); - if tids.is_empty() { - Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!( - "no thread ids found in {}", - proc_root.join(pid.to_string()).join("task").display() - ), - )) - } else { - Ok(tids) - } + let task_dir = proc_root.join(pid.to_string()).join("task"); + let mut tids = std::fs::read_dir(&task_dir)? + .filter_map(Result::ok) + .filter_map(|entry| { + entry + .file_name() + .to_str() + .and_then(|name| name.parse().ok()) + }) + .collect::>(); + tids.sort_unstable(); + if tids.is_empty() { + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("no thread ids found in {}", task_dir.display()), + )) + } else { + Ok(tids) } } diff --git a/src/symbols.rs b/src/symbols.rs index 9692666..2b43d2e 100644 --- a/src/symbols.rs +++ b/src/symbols.rs @@ -43,11 +43,32 @@ pub struct SymbolRequest { pub kernel_relocation: Option, } +impl SymbolRequest { + /// Returns the `file_identity` that participates in identity comparison. + /// + /// perf's `__dso_id__cmp` (tools/perf/util/dso.c) treats a defined build_id + /// as the decisive backing-store discriminator and only weighs the mmap2 + /// maj/min/ino when both dso ids recorded them. The same on-disk object can + /// arrive with the build_id but no file_identity (inline MMAP2-build-id) or + /// with both (plain MMAP2 + HEADER_BUILD_ID), so once a build_id is present + /// we ignore file_identity to keep the request — and thus the symbol cache + /// entry — unified. Distinct build_ids at the same path still differ via + /// `build_id`; file_identity remains the discriminator only when no + /// build_id exists. + fn identity_file_identity(&self) -> Option { + if self.build_id.is_some() { + None + } else { + self.file_identity + } + } +} + impl PartialEq for SymbolRequest { fn eq(&self, other: &Self) -> bool { self.relative_address == other.relative_address && self.build_id == other.build_id - && self.file_identity == other.file_identity + && self.identity_file_identity() == other.identity_file_identity() && self.kernel_relocation == other.kernel_relocation && self.path.as_os_str() == other.path.as_os_str() } @@ -60,7 +81,7 @@ impl Hash for SymbolRequest { self.path.as_os_str().hash(state); self.relative_address.hash(state); self.build_id.hash(state); - self.file_identity.hash(state); + self.identity_file_identity().hash(state); self.kernel_relocation.hash(state); } } @@ -78,7 +99,10 @@ impl Ord for SymbolRequest { .cmp(other.path.as_os_str()) .then_with(|| self.relative_address.cmp(&other.relative_address)) .then_with(|| self.build_id.cmp(&other.build_id)) - .then_with(|| self.file_identity.cmp(&other.file_identity)) + .then_with(|| { + self.identity_file_identity() + .cmp(&other.identity_file_identity()) + }) .then_with(|| self.kernel_relocation.cmp(&other.kernel_relocation)) } } @@ -141,6 +165,13 @@ pub trait SymbolResolver { pub struct ResolvedSymbolFrames { pub frames: Vec, pub has_base_symbol: bool, + /// The `+0x` suffix (relative to the containing symtab symbol) that + /// perf prints on every inline AND base frame for this address. + /// `tools/perf/util/symbol_fprintf.c __symbol__fprintf_symname_offs` uses + /// `al->addr - sym->start`, and an inline frame's fake symbol reuses + /// `base_sym->start` (`tools/perf/util/srcline.c new_inline_sym`), so the + /// whole group shares one offset. + pub base_offset: Option, } impl ResolvedSymbolFrames { @@ -150,6 +181,7 @@ impl ResolvedSymbolFrames { Self { frames, has_base_symbol, + base_offset: None, } } } @@ -180,6 +212,7 @@ struct CachedMappingFrames { frames: Vec, folded_rendered: String, has_base_symbol: bool, + base_offset: Option, } pub struct Addr2lineResolver<'a, R> { @@ -265,12 +298,35 @@ struct PreparedObjectMetadata { struct CachedObjectMetadata { object_metadata: PreparedObjectMetadata, object_bytes: Arc<[u8]>, + dwarf_index: Mutex, +} + +/// Per-object memo of DWARF inline-frame indexes. +/// +/// Folding queries the same hot objects every round; re-parsing their DWARF +/// and re-walking the DIE trees per batch dominated fold time. Unit ranges are +/// scanned once, and each unit's frame index is built on the first batch whose +/// addresses land in it. The name interner is append-only so frame name ids +/// stay valid across incremental builds. +#[derive(Default)] +struct PerfDwarfIndexCache { + names: PerfDwarfNameInterner, + units: Option>, + failed: bool, +} + +struct PerfDwarfCachedUnit { + ranges: Option>, + segments: Option>, } #[derive(Default)] struct PerfObjectSymbolNames<'a> { bare: Option<&'a str>, with_offset: Option, + /// Just the `+0x` suffix of `with_offset`, shared by every inline and + /// base frame at this address in perf-script output. + offset_suffix: Option, } #[derive(Default)] @@ -293,6 +349,12 @@ pub struct PerfSymbolResolver { system_map_kallsyms: Option, system_map_candidates: Vec, system_map_kallsyms_cache: OnceLock>, + /// `/sys/kernel/notes` (or a test override) — the live kernel's GNU + /// build-id note. Used to confirm the running kernel matches the build-id + /// recorded in the perf.data before trusting live `/proc/kallsyms` for + /// `[kernel.kallsyms]` frames. + live_kernel_notes_path: Option, + live_kernel_build_id_cache: OnceLock>, } #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -516,6 +578,7 @@ impl RustAddr2lineResolver { Arc::new(CachedObjectMetadata { object_metadata: PreparedObjectMetadata::from_object_bytes(&bytes), object_bytes: bytes.into(), + dwarf_index: Mutex::new(PerfDwarfIndexCache::default()), }) }); @@ -580,6 +643,7 @@ where Arc::new(CachedObjectMetadata { object_metadata: PreparedObjectMetadata::from_object_bytes(&bytes), object_bytes: bytes.into(), + dwarf_index: Mutex::new(PerfDwarfIndexCache::default()), }) }); @@ -637,6 +701,8 @@ where system_map_kallsyms: None, system_map_candidates: Vec::new(), system_map_kallsyms_cache: OnceLock::new(), + live_kernel_notes_path: None, + live_kernel_build_id_cache: OnceLock::new(), } } @@ -730,9 +796,44 @@ where let mut this = self; if this.live_kallsyms.is_none() { this.live_kallsyms_path = Some(path.to_path_buf()); + // Pair the live kallsyms with the running kernel's build-id note so + // we only trust it for [kernel.kallsyms] frames when it matches the + // perf.data's recorded kernel build-id. + if this.live_kernel_notes_path.is_none() { + this.live_kernel_notes_path = Some(PathBuf::from("/sys/kernel/notes")); + } } this } + + #[must_use] + pub fn with_live_kernel_notes_path(mut self, path: PathBuf) -> Self { + self.live_kernel_notes_path = Some(path); + self + } + + fn live_kernel_build_id(&self) -> Option<&str> { + self.live_kernel_build_id_cache + .get_or_init(|| { + let path = self.live_kernel_notes_path.as_ref()?; + let bytes = std::fs::read(path).ok()?; + gnu_build_id_from_notes(&bytes) + }) + .as_deref() + } + + /// True when the running kernel's build-id matches the build-id recorded in + /// the perf.data, so live `/proc/kallsyms` describes the same kernel perf + /// symbolized against. perf trusts kallsyms for the recorded kernel; this is + /// the equivalent guard for the direct-fold path on the recording machine. + fn live_kernel_matches_recorded(&self) -> bool { + match &self.recorded_kernel_build_id { + Some(recorded) => self + .live_kernel_build_id() + .is_some_and(|live| live == recorded), + None => false, + } + } } impl Kallsyms { @@ -839,6 +940,17 @@ impl Kallsyms { .map(|(_, symbol)| symbol.clone()) } + /// Resolves an address to `name+0x`, matching perf-script kernel + /// frames (`tools/perf/util/symbol_fprintf.c __symbol__fprintf_symname_offs` + /// prints the offset from the containing symbol, including `+0x0`). + #[must_use] + pub fn resolve_with_offset(&self, address: u64) -> Option { + self.symbols + .range(..=address) + .next_back() + .map(|(start, symbol)| format!("{symbol}+0x{:x}", address - start)) + } + #[must_use] pub fn resolve_relocated( &self, @@ -851,6 +963,18 @@ impl Kallsyms { self.resolve(address.wrapping_add(delta)) } + #[must_use] + pub fn resolve_relocated_with_offset( + &self, + address: u64, + reference_symbol: &str, + recorded_reference_address: u64, + ) -> Option { + let symbol_file_address = self.address_of(reference_symbol)?; + let delta = symbol_file_address.wrapping_sub(recorded_reference_address); + self.resolve_with_offset(address.wrapping_add(delta)) + } + fn address_of(&self, name: &str) -> Option { self.addresses_by_name.get(name).copied() } @@ -1007,6 +1131,27 @@ where .ok_or_else(|| "symbol frame cache lookup missed after resolution".to_string()) } + /// Resolves one borrowed perfdata mapping through the cache and returns the + /// inline frame slice together with the shared `+0x` offset suffix + /// perf prints on every inline and base frame at this address. + /// + /// # Errors + /// + /// Returns an error when the backing resolver fails. + pub fn resolve_mapping_ref_with_offset( + &mut self, + mapping: &ResolvedMappingRef<'_>, + ) -> Result<(&[String], Option<&str>), String> { + let key = mapping_frame_key(mapping); + if !self.resolved_by_mapping.contains_key(&key) { + self.prefetch_mapping_refs(std::slice::from_ref(mapping))?; + } + self.resolved_by_mapping + .get(&key) + .map(|cached| (cached.frames.as_slice(), cached.base_offset.as_deref())) + .ok_or_else(|| "symbol frame cache lookup missed after resolution".to_string()) + } + /// Resolves one borrowed perfdata mapping through the cache and returns the /// pre-rendered folded fragment for its symbolized inline frames. /// @@ -1049,6 +1194,31 @@ where .ok_or_else(|| "symbol frame cache lookup missed after resolution".to_string()) } + /// Resolves one borrowed perfdata mapping to the pre-rendered folded + /// fragment for its single base object symbol (no DWARF inline expansion). + /// + /// This is the default `perf script`/folded path: plain `perf` prints one + /// frame per callchain entry named from the ELF symtab. + /// + /// # Errors + /// + /// Returns an error when the backing resolver fails. + pub fn resolve_base_folded_mapping_ref( + &mut self, + mapping: &ResolvedMappingRef<'_>, + ) -> Result, String> { + let key = mapping_frame_key(mapping); + if !self.resolved_base_by_mapping.contains_key(&key) { + self.prefetch_base_mapping_refs(std::slice::from_ref(mapping))?; + } + self.resolved_base_by_mapping + .get(&key) + .map(|cached| { + (!cached.folded_rendered.is_empty()).then_some(cached.folded_rendered.as_str()) + }) + .ok_or_else(|| "symbol frame cache lookup missed after resolution".to_string()) + } + /// Resolves many borrowed perfdata mappings to base symbols only. /// /// # Errors @@ -1113,6 +1283,7 @@ where frames: resolved_frames.frames, folded_rendered, has_base_symbol: resolved_frames.has_base_symbol, + base_offset: resolved_frames.base_offset, }, ); } @@ -1226,6 +1397,7 @@ where frames: resolved_frames.frames, folded_rendered, has_base_symbol: resolved_frames.has_base_symbol, + base_offset: resolved_frames.base_offset, }, ); } @@ -1592,7 +1764,13 @@ where } fn can_use_system_kernel_symbols(&self, request: &SymbolRequest) -> bool { - self.recorded_kernel_build_id.is_none() || request.path != Path::new("[kernel.kallsyms]") + // Safe to use system/live kernel symbols when either the perf.data + // recorded no kernel build-id, the request is not for the core kernel, + // or the running kernel's build-id matches the recorded one (the + // recording machine: live /proc/kallsyms describes the same kernel). + self.recorded_kernel_build_id.is_none() + || request.path != Path::new("[kernel.kallsyms]") + || self.live_kernel_matches_recorded() } } @@ -1710,17 +1888,13 @@ where .collect::>(); let symbols = self.resolve_group_symbols(path, &grouped_requests)?; let object_metadata = self.object_metadata(path); - let perf_dwarf = object_metadata.as_ref().and_then(|metadata| { + if let Some(metadata) = object_metadata.as_ref() { let addresses = grouped_requests .iter() .map(|request| request.relative_address) .collect::>(); - PerfDwarfNameResolver::from_object_bytes_for_addresses( - &metadata.object_bytes, - &addresses, - ) - .ok() - }); + metadata.prepare_dwarf_frames_for_addresses(&addresses); + } for ((index, request), symbol) in indexes.into_iter().zip(grouped_requests).zip(symbols) { let object_symbols = @@ -1728,10 +1902,10 @@ where let object_symbol = object_symbols.bare; let has_base_symbol = object_symbol.is_some(); let mut frames = if let Some(object_symbol) = object_symbol { - perf_dwarf + object_metadata .as_ref() - .and_then(|resolver| { - resolver.frame_names_for_base_symbol( + .and_then(|metadata| { + metadata.dwarf_frame_names_for_base_symbol( request.relative_address, Some(object_symbol), ) @@ -1748,6 +1922,7 @@ where resolved[index] = ResolvedSymbolFrames { frames, has_base_symbol, + base_offset: object_symbols.offset_suffix, }; } } @@ -1856,17 +2031,13 @@ impl SymbolResolver for RustAddr2lineResolver { for (path, indexes) in grouped_request_indexes(requests) { let path = Path::new(path); let object_metadata = self.object_metadata(path); - let perf_dwarf = object_metadata.as_ref().and_then(|metadata| { + if let Some(metadata) = object_metadata.as_ref() { let addresses = indexes .iter() .map(|index| requests[*index].relative_address) .collect::>(); - PerfDwarfNameResolver::from_object_bytes_for_addresses( - &metadata.object_bytes, - &addresses, - ) - .ok() - }); + metadata.prepare_dwarf_frames_for_addresses(&addresses); + } let mut loader = None; let mut loader_attempted = false; for index in indexes { @@ -1876,10 +2047,10 @@ impl SymbolResolver for RustAddr2lineResolver { let object_symbol = object_symbols.bare; let has_base_symbol = object_symbol.is_some(); let mut frames = if let Some(object_symbol) = object_symbol { - perf_dwarf + object_metadata .as_ref() - .and_then(|resolver| { - resolver.frame_names_for_base_symbol( + .and_then(|metadata| { + metadata.dwarf_frame_names_for_base_symbol( request.relative_address, Some(object_symbol), ) @@ -1902,15 +2073,17 @@ impl SymbolResolver for RustAddr2lineResolver { object_symbol, object_symbols.with_offset.as_deref(), ); - if let Some(metadata) = &object_metadata { - specialize_frames_from_debug_strings( - &mut frames, - &metadata.object_metadata.debug_names, - ); - } + // No .debug_str generic specialization here: inline-frame names + // now come from the DWARF linkage name demangled like perf's + // external-addr2line backend (fully qualified, perf-faithful). + // Re-specializing from .debug_str would rewrite e.g. + // `core::slice::::sort_unstable` to `sort_unstable`, + // which perf never prints (verified against + // target/oracle/dwarf.perf.script). resolved[index] = ResolvedSymbolFrames { frames, has_base_symbol, + base_offset: object_symbols.offset_suffix, }; } } @@ -1957,6 +2130,9 @@ fn resolve_base_frames_from_object_metadata( resolved[index] = ResolvedSymbolFrames { frames, has_base_symbol: true, + // The no-inline base path bakes +0x into the single frame + // name via with_offset, so no separate per-line offset is used. + base_offset: None, }; } } @@ -1967,6 +2143,15 @@ fn demangle_addr2line_name(name: &str) -> String { perf_dwarf_function_name(&addr2line::demangle_auto(Cow::Borrowed(name), None)) } +/// Demangles a mangled (linkage) symbol the way perf's external-addr2line +/// srcline backend does: fully qualified, no trailing `::h`, generic +/// args preserved (`dso__demangle_sym` -> +/// `rust_demangle_display_demangle(..., /*alternate=*/true)`). Unlike +/// [`demangle_addr2line_name`] it does NOT collapse to the unqualified leaf. +fn demangle_addr2line_name_qualified(name: &str) -> String { + addr2line::demangle_auto(Cow::Borrowed(name), None).into_owned() +} + fn perf_name_with_object_alias(name: Option, object_alias: Option<&str>) -> Option { match (name, object_alias) { (Some(name), Some(object_alias)) @@ -2125,6 +2310,7 @@ impl PreparedObjectMetadata { PerfObjectSymbolNames { bare: self.object_symbol(address), with_offset: self.object_symbol_with_offset(address), + offset_suffix: self.object_symbols.symbol_offset_suffix(address), } } } @@ -2166,6 +2352,12 @@ impl PerfObjectSymbolIndex { Some(format!("{}+0x{offset:x}", candidate.name)) } + fn symbol_offset_suffix(&self, address: u64) -> Option { + let candidate = self.symbol(address)?; + let offset = address.saturating_sub(candidate.address); + Some(format!("+0x{offset:x}")) + } + fn symbol(&self, address: u64) -> Option<&PerfSymbolCandidate> { let mut index = self .symbols @@ -2477,6 +2669,118 @@ impl PerfDwarfNameResolver { } } +impl CachedObjectMetadata { + /// Builds frame indexes for every DWARF unit covering `addresses` that has + /// not been indexed by an earlier batch. + fn prepare_dwarf_frames_for_addresses(&self, addresses: &[u64]) { + let mut cache = self.dwarf_index.lock().expect("dwarf index cache lock"); + if cache.failed { + return; + } + if let Some(units) = &cache.units { + let needs_build = units.iter().any(|unit| { + unit.segments.is_none() + && perf_dwarf_unit_ranges_match_addresses(unit.ranges.as_deref(), addresses) + }); + if !needs_build { + return; + } + } + if build_dwarf_index_cache_for_addresses(&mut cache, &self.object_bytes, addresses).is_err() + { + cache.failed = true; + } + } + + /// Resolves the perf-style inline frame chain for one address from the + /// units prepared by [`Self::prepare_dwarf_frames_for_addresses`]. + fn dwarf_frame_names_for_base_symbol( + &self, + address: u64, + base_symbol: Option<&str>, + ) -> Option> { + let cache = self.dwarf_index.lock().expect("dwarf index cache lock"); + for unit in cache.units.as_deref()? { + let Some(segments) = &unit.segments else { + continue; + }; + if !unit + .ranges + .as_ref() + .is_none_or(|ranges| perf_dwarf_ranges_contain(ranges, address)) + { + continue; + } + if let Some(frames) = perf_dwarf_frame_names_from_index( + segments, + &cache.names.names, + address, + base_symbol, + ) { + return Some(frames); + } + } + None + } +} + +fn build_dwarf_index_cache_for_addresses( + cache: &mut PerfDwarfIndexCache, + bytes: &[u8], + addresses: &[u64], +) -> Result<(), gimli::Error> { + let object = object::File::parse(bytes).map_err(|_| gimli::Error::Io)?; + let endian = if object.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + let dwarf_sections = gimli::DwarfSections::load(|id| { + Ok::<_, gimli::Error>( + object + .section_by_name(id.name()) + .and_then(|section| section.uncompressed_data().ok()) + .unwrap_or(Cow::Borrowed(&[][..])), + ) + })?; + let dwarf = dwarf_sections.borrow(|section| gimli::EndianSlice::new(section.as_ref(), endian)); + + let scanning = cache.units.is_none(); + let mut units = cache.units.take().unwrap_or_default(); + let mut headers = dwarf.units(); + let mut ordinal = 0_usize; + while let Ok(Some(header)) = headers.next() { + let Ok(unit) = dwarf.unit(header) else { + if scanning { + units.push(PerfDwarfCachedUnit { + ranges: Some(Vec::new()), + segments: Some(Vec::new()), + }); + } + ordinal += 1; + continue; + }; + if scanning { + units.push(PerfDwarfCachedUnit { + ranges: perf_dwarf_ranges(dwarf.unit_ranges(&unit).ok()), + segments: None, + }); + } + let Some(cached_unit) = units.get_mut(ordinal) else { + break; + }; + if cached_unit.segments.is_none() + && perf_dwarf_unit_ranges_match_addresses(cached_unit.ranges.as_deref(), addresses) + { + let roots = perf_dwarf_unit_roots(&dwarf, &unit, &mut cache.names); + cached_unit.segments = Some(perf_dwarf_frame_ranges_from_roots(&roots)); + } + ordinal += 1; + } + cache.units = Some(units); + Ok(()) +} + fn perf_dwarf_unit_ranges_match_addresses( ranges: Option<&[PerfAddressRange]>, addresses: &[u64], @@ -2534,8 +2838,7 @@ fn perf_dwarf_collect_relevant_nodes( let ranges = kind .map(|_| perf_dwarf_ranges(dwarf.die_ranges(unit, node.entry()).ok()).unwrap_or_default()); let name = kind.and_then(|_| { - perf_dwarf_die_name(dwarf, unit, node.entry()) - .map(|name| names.intern(perf_dwarf_function_name(&name))) + perf_dwarf_die_frame_name(dwarf, unit, node.entry()).map(|name| names.intern(name)) }); if let Some(kind) = kind { let mut children = Vec::new(); @@ -2754,6 +3057,82 @@ fn perf_realfunc_name_replaces_base_symbol(name: &str, base_symbol: Option<&str> base_symbol.is_some_and(|base_symbol| name != base_symbol) } +/// Resolves the printed frame name for one subprogram/inlined-subroutine DIE. +/// +/// perf's external-addr2line srcline backend (the modern oracle build) names +/// each frame from the ELF symtab / DWARF *linkage* (mangled) name and then +/// demangles it itself with the Rust v0 demangler in alternate form +/// (`tools/perf/util/srcline.c` `new_inline_sym` -> `dso__demangle_sym` -> +/// `rust_demangle_display_demangle(..., /*alternate=*/true)` in +/// `tools/perf/util/symbol.c`), which yields fully-qualified names without the +/// trailing `::h` and with generic arguments preserved. +/// `addr2line::demangle_auto` produces byte-identical output to perf's alternate +/// Rust demangle for both legacy `_ZN` and v0 `_R` manglings, so the linkage +/// name is demangled with it directly (NOT run through +/// [`perf_dwarf_function_name`], which strips to the unqualified leaf and only +/// applies to the bare `DW_AT_name` fallback). +/// +/// Falls back to the bare `DW_AT_name` (perf's libdw backend spelling) when no +/// linkage name is present, e.g. closures and shim DIEs. +fn perf_dwarf_die_frame_name( + dwarf: &gimli::Dwarf, + unit: &gimli::Unit, + entry: &gimli::DebuggingInformationEntry, +) -> Option +where + R: gimli::Reader, +{ + if let Some(linkage) = perf_dwarf_die_linkage_name(dwarf, unit, entry, 16) { + return Some(demangle_addr2line_name_qualified(&linkage)); + } + perf_dwarf_die_name(dwarf, unit, entry).map(|name| perf_dwarf_function_name(&name)) +} + +fn perf_dwarf_die_linkage_name( + dwarf: &gimli::Dwarf, + unit: &gimli::Unit, + entry: &gimli::DebuggingInformationEntry, + recursion_limit: usize, +) -> Option +where + R: gimli::Reader, +{ + if recursion_limit == 0 { + return None; + } + entry + .attr(gimli::DW_AT_linkage_name) + .and_then(|attr| dwarf.attr_string(unit, attr.value()).ok()) + .and_then(|name| name.to_string_lossy().ok().map(Cow::into_owned)) + .or_else(|| { + entry.attr(gimli::DW_AT_abstract_origin).and_then(|attr| { + perf_dwarf_origin_linkage_name(dwarf, unit, &attr.value(), recursion_limit - 1) + }) + }) + .or_else(|| { + entry.attr(gimli::DW_AT_specification).and_then(|attr| { + perf_dwarf_origin_linkage_name(dwarf, unit, &attr.value(), recursion_limit - 1) + }) + }) +} + +fn perf_dwarf_origin_linkage_name( + dwarf: &gimli::Dwarf, + unit: &gimli::Unit, + value: &gimli::AttributeValue, + recursion_limit: usize, +) -> Option +where + R: gimli::Reader, +{ + let gimli::AttributeValue::UnitRef(offset) = value else { + return None; + }; + let mut entries = unit.entries_tree(Some(*offset)).ok()?; + let root = entries.root().ok()?; + perf_dwarf_die_linkage_name(dwarf, unit, root.entry(), recursion_limit) +} + fn perf_dwarf_die_name( dwarf: &gimli::Dwarf, unit: &gimli::Unit, @@ -3100,15 +3479,57 @@ fn build_id_hex(bytes: &[u8]) -> String { hex } +/// Extracts the GNU build-id (lowercase hex) from a buffer of ELF notes such as +/// `/sys/kernel/notes`. Walks the note stream looking for the +/// `NT_GNU_BUILD_ID` (type 3) note with name "GNU\0" and returns its +/// descriptor. Notes are little-endian on the supported targets (x86_64, +/// aarch64), matching how perf stores build-ids in HEADER_BUILD_ID. +fn gnu_build_id_from_notes(bytes: &[u8]) -> Option { + const NT_GNU_BUILD_ID: u32 = 3; + let mut offset = 0usize; + while offset + 12 <= bytes.len() { + let read_u32 = |start: usize| { + u32::from_le_bytes([ + bytes[start], + bytes[start + 1], + bytes[start + 2], + bytes[start + 3], + ]) + }; + let namesz = read_u32(offset) as usize; + let descsz = read_u32(offset + 4) as usize; + let note_type = read_u32(offset + 8); + let name_start = offset + 12; + let name_end = name_start.checked_add(namesz)?; + // Notes pad name and descriptor to 4-byte boundaries. + let name_padded = name_end.next_multiple_of(4); + let desc_start = name_padded; + let desc_end = desc_start.checked_add(descsz)?; + if desc_end > bytes.len() { + break; + } + if note_type == NT_GNU_BUILD_ID + && bytes.get(name_start..name_end) == Some(b"GNU\0") + && descsz > 0 + { + return Some(build_id_hex(&bytes[desc_start..desc_end])); + } + offset = desc_end.next_multiple_of(4); + } + None +} + fn resolve_kernel_kallsyms(kallsyms: &Kallsyms, request: &SymbolRequest) -> Option { + // perf-script prints kernel frames as `name+0x` (symbol_fprintf.c), + // and the folded path strips the offset like every other frame. if let Some(relocation) = &request.kernel_relocation { - kallsyms.resolve_relocated( + kallsyms.resolve_relocated_with_offset( request.relative_address, &relocation.reference_symbol, relocation.recorded_reference_address, ) } else { - kallsyms.resolve(request.relative_address) + kallsyms.resolve_with_offset(request.relative_address) } } @@ -3205,6 +3626,7 @@ fn parse_module_kallsyms_line(line: &str) -> Option<(u64, String, String)> { #[cfg(test)] mod tests { use std::cell::Cell; + use std::path::PathBuf; use std::sync::Arc; use object::{Object, ObjectSegment, ObjectSymbol, build, elf}; @@ -3213,10 +3635,158 @@ mod tests { PerfAddressRange, PerfDwarfDieKind, PerfDwarfDieNode, PerfDwarfNameInterner, PerfObjectSymbolIndex, PerfSymbolBinding, PerfSymbolCandidate, PerfSymbolScope, ResolvedMappingRef, RustAddr2lineResolver, SymbolFrameCache, SymbolRequest, SymbolResolver, - clean_object_symbol_request, perf_best_duplicate_symbol, perf_dwarf_frame_names_from_index, + clean_object_symbol_request, demangle_addr2line_name_qualified, gnu_build_id_from_notes, + perf_best_duplicate_symbol, perf_dwarf_frame_names_from_index, perf_dwarf_frame_ranges_from_roots, perf_frames_with_object_alias, }; + #[test] + fn gnu_build_id_from_notes_reads_kernel_nt_gnu_build_id() { + // Real /sys/kernel/notes bytes from the oracle recording container + // (aarch64). Layout per note: namesz=4, descsz=20, type=3 + // (NT_GNU_BUILD_ID), name "GNU\0", then the 20-byte build-id; followed + // by unrelated "Linux" notes that must be skipped. + let notes = [ + 0x04, 0x00, 0x00, 0x00, // namesz = 4 + 0x14, 0x00, 0x00, 0x00, // descsz = 20 + 0x03, 0x00, 0x00, 0x00, // type = NT_GNU_BUILD_ID + 0x47, 0x4e, 0x55, 0x00, // "GNU\0" + 0xcb, 0x97, 0xc0, 0xad, 0xd7, 0x3d, 0xc6, 0x0d, 0x73, 0xbb, 0x9a, 0xd7, 0xdc, 0x27, + 0x85, 0xd8, 0x8b, 0x36, 0x44, 0xa0, // 20-byte build-id + 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x4c, 0x69, + 0x6e, 0x75, 0x78, 0x00, 0x00, 0x00, // a trailing "Linux" note + ]; + + assert_eq!( + gnu_build_id_from_notes(¬es).as_deref(), + Some("cb97c0add73dc60d73bb9ad7dc2785d88b3644a0") + ); + } + + #[test] + fn gnu_build_id_from_notes_skips_leading_non_build_id_note() { + // A "Linux" version note precedes the build-id note; the walker must + // honor 4-byte name/descriptor padding and find the later build-id. + let notes = [ + 0x06, 0x00, 0x00, 0x00, // namesz = 6 -> padded to 8 + 0x04, 0x00, 0x00, 0x00, // descsz = 4 + 0x00, 0x01, 0x00, 0x00, // type + 0x4c, 0x69, 0x6e, 0x75, 0x78, 0x00, 0x00, 0x00, // "Linux\0" padded + 0xde, 0xad, 0xbe, 0xef, // 4-byte desc + 0x04, 0x00, 0x00, 0x00, // namesz = 4 + 0x04, 0x00, 0x00, 0x00, // descsz = 4 + 0x03, 0x00, 0x00, 0x00, // NT_GNU_BUILD_ID + 0x47, 0x4e, 0x55, 0x00, // "GNU\0" + 0x01, 0x23, 0x45, 0x67, // 4-byte build-id + ]; + + assert_eq!(gnu_build_id_from_notes(¬es).as_deref(), Some("01234567")); + } + + #[test] + fn demangle_addr2line_name_qualified_matches_perf_external_addr2line_backend() { + // perf's external-addr2line srcline backend names each frame from the + // mangled symtab/DWARF linkage name and demangles it itself with the + // Rust v0 demangler in alternate form (tools/perf/util/srcline.c + // new_inline_sym -> tools/perf/util/symbol.c dso__demangle_sym -> + // rust_demangle_display_demangle(..., /*alternate=*/true)), keeping the + // fully-qualified path, dropping the trailing ::h, and preserving + // generic arguments. These expectations are copied byte-for-byte from + // target/oracle/dwarf.perf.script (perf 6.17.13, addr2line backend). + // + // Legacy `_ZN` manglings (core/std non-generic functions in symtab): + assert_eq!( + demangle_addr2line_name_qualified( + "_ZN4core5slice4sort8unstable4sort17hf487fc59c5378322E" + ), + "core::slice::sort::unstable::sort" + ); + assert_eq!( + demangle_addr2line_name_qualified( + "_ZN91_$LT$T$u20$as$u20$core..slice..sort..shared..smallsort..UnstableSmallSortFreezeTypeImpl$GT$10small_sort17ha5f9b986560cf204E" + ), + "::small_sort" + ); + // v0 `_R` manglings (the std::rt::lang_start_internal inline group): + assert_eq!( + demangle_addr2line_name_qualified("_RNvNtCsfQfHhyvAE2O_3std2rt19lang_start_internal"), + "std::rt::lang_start_internal" + ); + assert_eq!( + demangle_addr2line_name_qualified( + "_RINvNtCsfQfHhyvAE2O_3std9panicking12catch_unwindiNCNvNtB4_2rt19lang_start_internal0EB4_" + ), + "std::panicking::catch_unwind::" + ); + } + + #[test] + fn symbol_request_ignores_file_identity_when_build_id_present() { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + use crate::perfdata::mappings::FileIdentity; + + let hash_of = |request: &SymbolRequest| { + let mut hasher = DefaultHasher::new(); + request.hash(&mut hasher); + hasher.finish() + }; + + // Same object, same build_id: the inline MMAP2-build-id form carries no + // file_identity while the plain MMAP2 + HEADER_BUILD_ID form does. + // perf's __dso_id__cmp makes build_id decisive, so these are one entry. + let inline = SymbolRequest { + path: PathBuf::from("/usr/lib/libc.so.6"), + relative_address: 0x1234, + build_id: Some("aabbccdd".to_string()), + file_identity: None, + kernel_relocation: None, + }; + let with_identity = SymbolRequest { + file_identity: Some(FileIdentity { + major: 8, + minor: 1, + inode: 99, + inode_generation: 7, + }), + ..inline.clone() + }; + assert_eq!(inline, with_identity); + assert_eq!(hash_of(&inline), hash_of(&with_identity)); + assert_eq!(inline.cmp(&with_identity), std::cmp::Ordering::Equal); + + // Different build_ids at the same path are genuinely different objects. + let other_build_id = SymbolRequest { + build_id: Some("11223344".to_string()), + ..inline.clone() + }; + assert_ne!(inline, other_build_id); + + // With no build_id, file_identity is the only backing-store + // discriminator and must still separate distinct objects. + let no_build_id_a = SymbolRequest { + build_id: None, + file_identity: Some(FileIdentity { + major: 8, + minor: 1, + inode: 99, + inode_generation: 0, + }), + ..inline.clone() + }; + let no_build_id_b = SymbolRequest { + file_identity: Some(FileIdentity { + major: 8, + minor: 1, + inode: 100, + inode_generation: 0, + }), + ..no_build_id_a.clone() + }; + assert_ne!(no_build_id_a, no_build_id_b); + } + #[test] fn object_requests_use_elf_virtual_addresses_for_pie_file_offsets() { let path = std::env::current_exe().expect("current test binary"); diff --git a/tests/benchmarks.rs b/tests/benchmarks.rs index 7534328..3f96d68 100644 --- a/tests/benchmarks.rs +++ b/tests/benchmarks.rs @@ -114,7 +114,9 @@ fn symbolized_fold_benchmark_uses_runner_addr2line() { .expect("write perfdata"); let runner = Addr2lineRunner::default(); - let report = run_fold_benchmark_with_runner(&perfdata, &runner, true).expect("benchmark"); + // The external addr2line resolver only runs on the --inline path; the + // default base path resolves from the in-process ELF symtab. + let report = run_fold_benchmark_with_runner(&perfdata, &runner, true, true).expect("benchmark"); assert_eq!(report.folded_bytes, ":12;app::main 1\n".len()); assert_eq!(runner.programs(), vec!["addr2line"]); @@ -179,8 +181,11 @@ fn compares_symbolized_pyroclast_folded_stacks_with_inferno_collapse() { std::fs::write(&perf_script, "sample script\n").expect("write perf script"); let runner = SymbolizedCompareRunner::default(); - let report = compare_with_inferno_collapse_with_symbols(&perfdata, &perf_script, &runner, true) - .expect("comparison"); + // Exercising the external addr2line resolver (and its inline frames) + // requires --inline; the default base path reads the in-process symtab. + let report = + compare_with_inferno_collapse_with_symbols(&perfdata, &perf_script, &runner, true, true) + .expect("comparison"); assert!(report.matches); assert!(report.svg_matches); @@ -265,6 +270,7 @@ fn parses_benchmark_inputs() { let args = BenchArgs::parse(vec![ "profile.perf.data".into(), "--symbols".into(), + "--inline".into(), "--export-perf-script".into(), "exported-script.txt".into(), "--perf-script".into(), @@ -273,10 +279,18 @@ fn parses_benchmark_inputs() { assert_eq!(args.perf_data, Some("profile.perf.data".into())); assert!(args.symbols); + assert!(args.inline); assert_eq!(args.export_perf_script, Some("exported-script.txt".into())); assert_eq!(args.perf_script, Some("perf-script.txt".into())); } +#[test] +fn benchmark_inline_flag_defaults_off() { + let args = BenchArgs::parse(vec!["profile.perf.data".into(), "--symbols".into()]); + + assert!(!args.inline); +} + #[test] fn benchmark_args_default_to_standard_input_path() { let args = BenchArgs::default(); @@ -295,6 +309,7 @@ fn bench_command_reports_missing_input() { perf_script: None, export_perf_script: None, symbols: false, + inline: false, }; let error = run_bench_command(&args, &runner).expect_err("missing input should fail"); @@ -313,6 +328,7 @@ fn bench_command_reports_missing_perf_script_input() { perf_script: Some(root.path().join("missing.perf-script")), export_perf_script: None, symbols: false, + inline: false, }; let error = run_bench_command(&args, &runner).expect_err("missing perf script should fail"); @@ -332,13 +348,17 @@ fn bench_command_exports_perf_script_and_compares_without_perf_runner() { perf_script: None, export_perf_script: Some(exported_perf_script.clone()), symbols: false, + inline: false, }; let output = run_bench_command(&args, &runner).expect("bench command"); + // builtin-script.c prints the event name with `"%*s: "` (trailing space) + // then `fputc(cursor ? '\n' : ' ')`; a resolved callchain yields the newline + // so the header line ends "P: \n". assert_eq!( std::fs::read_to_string(&exported_perf_script).expect("exported perf script"), - ":2 2 1 cpu/cycles/P:\n\t 2000 [unknown] ([unknown])\n\n" + ":2 2 1 cpu/cycles/P: \n\t 2000 [unknown] ([unknown])\n\n" ); assert!(output.contains("inferno_compare.matches=true")); assert!(output.contains("pyroclast_fold.input=")); @@ -424,6 +444,7 @@ proptest! { perf_script: None, export_perf_script: None, symbols: false, + inline: false, }; let expected = perf_data @@ -617,7 +638,7 @@ impl CommandRunner for BenchCommandRunner { self.commands.lock().unwrap().push(command.clone()); let stdout = match command.program.as_str() { "perf" => { - b":2 2 1 cpu/cycles/P:\n\t 2000 [unknown] ([unknown])\n\n" + b":2 2 1 cpu/cycles/P: \n\t 2000 [unknown] ([unknown])\n\n" .to_vec() } "inferno-collapse-perf" => b":2;[unknown] 1\n".to_vec(), diff --git a/tests/perfdata_build_id.rs b/tests/perfdata_build_id.rs index 1e2ae02..e73238c 100644 --- a/tests/perfdata_build_id.rs +++ b/tests/perfdata_build_id.rs @@ -29,6 +29,35 @@ fn parses_build_id_events_from_header_feature_payload() { ); } +#[test] +fn parses_build_id_events_written_by_perf_write_buildid() { + // tools/perf/util/build-id.c write_buildid() emits perf_record_header_build_id + // records into the HEADER_BUILD_ID feature section WITHOUT setting + // header.type (so it stays 0), sets PERF_RECORD_MISC_BUILD_ID_SIZE in misc, + // and stores the real build-id length in the size byte at the end of the + // 24-byte build_id field (offset 32). pyroclast must accept type 0 and honor + // that size byte. + let payload = build_id_event_payload_like_perf( + u32::MAX, + &[ + 0x5a, 0xeb, 0xdc, 0xbb, 0xc2, 0x4d, 0xe5, 0xf6, 0x37, 0xea, 0xb4, 0x4b, 0x9d, 0x16, + 0x2c, 0x84, 0xf1, 0xa8, 0x93, 0x38, + ], + "/tmp/oracle-workload", + ); + + let events = parse_build_id_events(&payload).expect("build ids"); + + assert_eq!( + events, + vec![BuildIdEvent { + pid: u32::MAX, + build_id: "5aebdcbbc24de5f637eab44b9d162c84f1a89338".to_string(), + filename: "/tmp/oracle-workload".to_string(), + }] + ); +} + #[test] fn extracts_kernel_build_id_from_perfdata_header_feature() { let build_id = [ @@ -249,6 +278,25 @@ fn build_id_event_payload(pid: u32, build_id: &[u8; 20], filename: &str) -> Vec< payload } +// Mirrors tools/perf/util/build-id.c write_buildid(): header.type left 0, +// PERF_RECORD_MISC_BUILD_ID_SIZE set in misc, build-id length stored in the +// size byte at offset 20 of the 24-byte build_id field (record offset 32). +fn build_id_event_payload_like_perf(pid: u32, build_id: &[u8; 20], filename: &str) -> Vec { + const PERF_RECORD_MISC_BUILD_ID_SIZE: u16 = 1 << 15; + let size = 36 + filename.len() + 1; + let mut payload = Vec::new(); + payload.extend(0_u32.to_le_bytes()); + payload.extend(PERF_RECORD_MISC_BUILD_ID_SIZE.to_le_bytes()); + payload.extend(u16::try_from(size).expect("event size").to_le_bytes()); + payload.extend(pid.to_le_bytes()); + payload.extend(build_id); + payload.push(20); + payload.extend([0; 3]); + payload.extend(filename.as_bytes()); + payload.push(0); + payload +} + fn perfdata_with_build_id_feature(payload: &[u8]) -> Vec { let feature_table_offset = 128; let payload_offset = 160; @@ -257,7 +305,9 @@ fn perfdata_with_build_id_feature(payload: &[u8]) -> Vec { put_u64(&mut bytes, 8, 104); put_u64(&mut bytes, 40, 128); put_u64(&mut bytes, 48, 0); - put_u64(&mut bytes, 56, 1 << 2); + // HEADER_BUILD_ID feature bit (2) in the adds_features bitmap at byte + // offset 72 (struct perf_file_header, tools/perf/util/header.h). + put_u64(&mut bytes, 72, 1 << 2); put_u64(&mut bytes, feature_table_offset, payload_offset as u64); put_u64( &mut bytes, diff --git a/tests/perfdata_fold.rs b/tests/perfdata_fold.rs index a226e12..dcd9801 100644 --- a/tests/perfdata_fold.rs +++ b/tests/perfdata_fold.rs @@ -1,3 +1,5 @@ +// Only the linux-gated libc leaf test parses real objects. +#[cfg(target_os = "linux")] use object::{Object as _, ObjectSegment as _, ObjectSymbol as _}; use proptest::prelude::*; use pyroclast::perfdata::fold::{ @@ -181,6 +183,48 @@ fn keeps_unmapped_dwarf_user_stack_payloads_like_perf_libdw_ebl() { assert_eq!(folded, ":12;[unknown];[unknown] 1\n"); } +#[test] +fn folds_aarch64_dwarf_user_stack_with_frame_pointer_fallback_like_perf_libdw_ebl() { + // perf record --call-graph dwarf on arm64 captures x0-x30, sp, pc. The + // recording machine's HEADER_ARCH ("aarch64") tells the fold path to decode + // PerfAarch64Regs (fp=29, lr=30, sp=31, pc=32) and use elfutils' + // backends/aarch64_unwind.c frame-pointer fallback when no DSO/CFI covers + // the sampled pc: the caller pc comes from lr (taking the perf pc-1 + // adjustment), and the walk ends on the zeroed next lr. + let mask = (1_u64 << 29) | (1_u64 << 30) | (1_u64 << 31) | (1_u64 << 32); + let bytes = perfdata_with_records_attrs_and_arch_feature( + [file_attr_bytes_with_regs( + PERF_SAMPLE_IP + | PERF_SAMPLE_TID + | PERF_SAMPLE_CALLCHAIN + | PERF_SAMPLE_REGS_USER + | PERF_SAMPLE_STACK_USER, + mask, + )], + [record_bytes( + 9, + // Registers are in ascending perf-register order: fp, lr, sp, pc. + // sp = 0x1000, fp = 0x1010: the fp chain record at fp+0 (next fp) + // and fp+8 (next lr) are both zero, so the lr-derived caller is the + // only unwound frame. + &sample_payload_with_user_stack( + 0x4000, + 11, + 12, + [], + 1, + [0x1010, 0x5000, 0x1000, 0x4000], + [0_u8; 0x40], + ), + )], + "aarch64", + ); + + let folded = fold_perfdata_callchains(&bytes).expect("folded"); + + assert_eq!(folded, ":12;[unknown];[unknown] 1\n"); +} + #[test] fn drops_dwarf_user_stack_when_mapped_object_cannot_be_loaded_like_perf_script() { let bytes = perfdata_with_records_and_attrs( @@ -477,8 +521,8 @@ fn keeps_dwarf_user_stack_when_header_build_id_mmap2_overlaps_before_first_repor 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, ]; - let current_exe = std::env::current_exe().expect("current exe"); - let current_exe = current_exe.to_string_lossy(); + let fixture = SyntheticX86_64Object::create(); + let current_exe = fixture.path_string(); let bytes = perfdata_with_records_attrs_and_build_id_feature( [file_attr_bytes_with_regs( PERF_SAMPLE_IP @@ -518,7 +562,7 @@ fn keeps_dwarf_user_stack_when_header_build_id_mmap2_overlaps_before_first_repor ); let folded = fold_perfdata_callchains(&bytes).expect("folded"); - let expected = format!(":12;[{}] 1\n", current_exe_file_name()); + let expected = format!(":12;[{}] 1\n", fixture.file_name()); assert_eq!(folded, expected); } @@ -949,8 +993,8 @@ fn keeps_current_ip_only_object_unwind_for_mapped_dwarf_user_stack_like_perf_lib // state before attempting to unwind callers. perf's frame_callback() then // calls entry(pc), so a single current-IP callback is a real frame, not // something to drop. - let current_exe = std::env::current_exe().expect("current exe"); - let current_exe = current_exe.to_string_lossy(); + let fixture = SyntheticX86_64Object::create(); + let current_exe = fixture.path_string(); let bytes = perfdata_with_records_and_attrs( [file_attr_bytes_with_regs( PERF_SAMPLE_IP @@ -985,7 +1029,7 @@ fn keeps_current_ip_only_object_unwind_for_mapped_dwarf_user_stack_like_perf_lib ); let folded = fold_perfdata_callchains(&bytes).expect("folded"); - let expected = format!(":12;[{}] 1\n", current_exe_file_name()); + let expected = format!(":12;[{}] 1\n", fixture.file_name()); assert_eq!(folded, expected); } @@ -995,8 +1039,8 @@ fn keeps_current_ip_only_object_unwind_after_first_non_text_mapping_like_perf_li // perf reports the module selected by thread__find_symbol() for the // callback PC. If that report succeeds, entry() stores the current IP even // when no caller is recovered. - let current_exe = std::env::current_exe().expect("current exe"); - let current_exe = current_exe.to_string_lossy(); + let fixture = SyntheticX86_64Object::create(); + let current_exe = fixture.path_string(); let bytes = perfdata_with_records_and_attrs( [file_attr_bytes_with_regs( PERF_SAMPLE_IP @@ -1035,15 +1079,15 @@ fn keeps_current_ip_only_object_unwind_after_first_non_text_mapping_like_perf_li ); let folded = fold_perfdata_callchains(&bytes).expect("folded"); - let expected = format!(":12;[{}] 1\n", current_exe_file_name()); + let expected = format!(":12;[{}] 1\n", fixture.file_name()); assert_eq!(folded, expected); } #[test] fn keeps_current_ip_only_object_unwind_from_executable_mmap2_like_perf_libdw() { - let current_exe = std::env::current_exe().expect("current exe"); - let current_exe = current_exe.to_string_lossy(); + let fixture = SyntheticX86_64Object::create(); + let current_exe = fixture.path_string(); let bytes = perfdata_with_records_and_attrs( [file_attr_bytes_with_regs( PERF_SAMPLE_IP @@ -1090,15 +1134,15 @@ fn keeps_current_ip_only_object_unwind_from_executable_mmap2_like_perf_libdw() { ); let folded = fold_perfdata_callchains(&bytes).expect("folded"); - let expected = format!(":12;[{}] 1\n", current_exe_file_name()); + let expected = format!(":12;[{}] 1\n", fixture.file_name()); assert_eq!(folded, expected); } #[test] fn keeps_current_ip_only_object_unwind_from_pid_specific_modules_like_perf_libdw() { - let current_exe = std::env::current_exe().expect("current exe"); - let current_exe = current_exe.to_string_lossy(); + let fixture = SyntheticX86_64Object::create(); + let current_exe = fixture.path_string(); let bytes = perfdata_with_records_and_attrs( [file_attr_bytes_with_regs( PERF_SAMPLE_IP @@ -1137,11 +1181,178 @@ fn keeps_current_ip_only_object_unwind_from_pid_specific_modules_like_perf_libdw ); let folded = fold_perfdata_callchains(&bytes).expect("folded"); - let expected = format!(":12;[{}] 1\n", current_exe_file_name()); + let expected = format!(":12;[{}] 1\n", fixture.file_name()); assert_eq!(folded, expected); } +/// Build a `--call-graph dwarf` x86_64 perf.data with a single sample over the +/// synthetic fixture: one MMAP covering `[0, 0x1000_0000)` and one user-stack +/// sample. `regs` are `[bp, sp, ip]` in perf's ascending register order +/// (RBP=6, RSP=7, IP=8). +fn x86_leaf_only_perfdata(fixture_path: &str, regs: [u64; 3], stack: [u8; 24]) -> Vec { + perfdata_with_records_and_attrs( + [file_attr_bytes_with_regs( + PERF_SAMPLE_IP + | PERF_SAMPLE_TID + | PERF_SAMPLE_CALLCHAIN + | PERF_SAMPLE_REGS_USER + | PERF_SAMPLE_STACK_USER, + (1 << 6) | (1 << 7) | (1 << 8), + )], + [ + record_bytes(1, &mmap_payload(11, 11, 0, 0x1000_0000, 0, fixture_path)), + record_bytes( + 9, + &sample_payload_with_user_stack(regs[2], 11, 12, [], 1, regs, stack), + ), + ], + ) +} + +#[test] +fn emits_scenario_d_leaf_when_no_cfi_and_bp_below_sp_like_perf_libdw() { + // gap-5gr scenario D: the sampled IP (0x4000) is reported into a module + // but no .eh_frame FDE covers it (the fixture's only FDE is at [0x100, + // 0x104)), and bp < sp so elfutils' x86_64 rbp fallback (`if (sp >= fp) + // return false;`, backends/x86_64_unwind.c) can never advance. libdwfl + // fires the initial-frame callback exactly once, so perf prints the single + // leaf. bp=0x7ffe_ff00 < sp=0x7fff_0000. + let fixture = SyntheticX86_64Object::create(); + let bytes = x86_leaf_only_perfdata( + &fixture.path_string(), + [0x7ffe_ff00, 0x7fff_0000, 0x4000], + [ + 0, 0, 0, 0, 0, 0, 0, 0, // + 0x40, 0, 0, 0, 0, 0, 0, 0, // + 0x34, 0x12, 0, 0, 0, 0, 0, 0, + ], + ); + + let folded = fold_perfdata_callchains(&bytes).expect("folded"); + assert_eq!(folded, format!(":12;[{}] 1\n", fixture.file_name())); +} + +#[test] +fn does_not_take_leaf_only_path_when_bp_at_or_above_sp_is_fallback_territory_like_perf_libdw() { + // bp >= sp is exactly when elfutils attempts the rbp fallback + // (backends/x86_64_unwind.c only fails on the *final* `if (sp >= fp)` + // guard), so the leaf-only predicate's register clause is false and this + // sample is MustUnwind, not LeafOnly: framehop is authoritative and the + // result is whatever it (and the elfutils fp fallback) recover, never a + // truncated synthetic leaf. Here framehop yields the seeded IP and the + // elfutils fallback only runs when framehop returned nothing, so the result + // is the genuine single seed frame — identical bytes to case 1's output, + // but reached through the full unwind path rather than leaf-only + // truncation. (The companion unit test + // `arch_fallback_cannot_advance_only_when_x86_bp_below_sp` pins the + // predicate edge directly.) + let fixture = SyntheticX86_64Object::create(); + let bytes = x86_leaf_only_perfdata( + &fixture.path_string(), + [0x7fff_0008, 0x7fff_0000, 0x4000], + [ + 0, 0, 0, 0, 0, 0, 0, 0, // + 0x40, 0, 0, 0, 0, 0, 0, 0, // + 0x34, 0x12, 0, 0, 0, 0, 0, 0, + ], + ); + + let folded = fold_perfdata_callchains(&bytes).expect("folded"); + assert_eq!(folded, format!(":12;[{}] 1\n", fixture.file_name())); +} + +#[test] +fn does_not_truncate_to_leaf_when_cfi_covers_ip_like_perf_libdw() { + // When an FDE covers the sampled IP, handle_cfi (libdwfl/frame_unwind.c) + // may yield either PC_UNDEFINED (clean end-of-stack -> leaf only) or a + // PC_SET caller, and the two are indistinguishable a priori — so this case + // is MustUnwind and framehop is authoritative. The fixture's FDE covers + // [0x100, 0x104); sample at vaddr 0x100 (mapping base 0) with bp < sp. The + // leaf-only predicate's `!has_unwind_info_for_ip` clause is false here, so + // no leaf-only truncation occurs and framehop's own result (the seed IP, + // since the FDE has only nops and recovers no usable caller) stands. + let fixture = SyntheticX86_64Object::create(); + let bytes = x86_leaf_only_perfdata( + &fixture.path_string(), + [0x7ffe_ff00, 0x7fff_0000, 0x100], + [0_u8; 24], + ); + + let folded = fold_perfdata_callchains(&bytes).expect("folded"); + // CFI covers the IP, so this is not leaf-only; framehop runs and yields the + // seed. The output is the single covered-IP frame produced by the real + // unwind, NOT a leaf-only-truncated synthetic. + assert_eq!(folded, format!(":12;[{}] 1\n", fixture.file_name())); +} + +#[test] +fn skip_gate_is_byte_identical_to_running_the_full_unwind_for_leaf_only_samples() { + // The gap-pkh skip gate is a pure optimization: classifying a leaf-only + // sample and skipping framehop must produce the exact same folded output + // as running framehop and letting the shared acceptance tail truncate to + // the leaf. The fold path always takes the gated route, so we assert the + // gated output equals the independently-known perf-correct single leaf. + let fixture = SyntheticX86_64Object::create(); + let leaf_only = x86_leaf_only_perfdata( + &fixture.path_string(), + [0x7ffe_ff00, 0x7fff_0000, 0x4000], + [ + 0, 0, 0, 0, 0, 0, 0, 0, // + 0x40, 0, 0, 0, 0, 0, 0, 0, // + 0x34, 0x12, 0, 0, 0, 0, 0, 0, + ], + ); + + let gated = fold_perfdata_callchains(&leaf_only).expect("folded"); + assert_eq!(gated, format!(":12;[{}] 1\n", fixture.file_name())); +} + +#[test] +fn emits_scenario_d_leaf_on_aarch64_when_no_cfi_and_lr_is_zero_like_perf_libdw() { + // gap-5gr scenario D on aarch64: pc (0x4000) is reported into a module with + // no FDE covering it (the fixture's only FDE is [0x100, 0x104)) and lr == 0, + // so elfutils' backends/aarch64_unwind.c fails before producing any caller + // (`if (lr == 0 || !setfunc(...)) return false;`). libdwfl fires the + // initial-frame callback exactly once, so perf prints the single leaf. + // Registers are ascending fp(29), lr(30), sp(31), pc(32) = [fp, lr, sp, pc]. + let fixture = SyntheticAarch64Object::create(); + let mask = (1_u64 << 29) | (1_u64 << 30) | (1_u64 << 31) | (1_u64 << 32); + let bytes = perfdata_with_records_attrs_and_arch_feature( + [file_attr_bytes_with_regs( + PERF_SAMPLE_IP + | PERF_SAMPLE_TID + | PERF_SAMPLE_CALLCHAIN + | PERF_SAMPLE_REGS_USER + | PERF_SAMPLE_STACK_USER, + mask, + )], + [ + record_bytes( + 1, + &mmap_payload(11, 11, 0, 0x1000_0000, 0, fixture.path_string().as_ref()), + ), + record_bytes( + 9, + &sample_payload_with_user_stack( + 0x4000, + 11, + 12, + [], + 1, + // fp = 0x1010, lr = 0 (ends the walk), sp = 0x1000, pc = 0x4000. + [0x1010, 0, 0x1000, 0x4000], + [0_u8; 0x40], + ), + ), + ], + "aarch64", + ); + + let folded = fold_perfdata_callchains(&bytes).expect("folded"); + assert_eq!(folded, format!(":12;[{}] 1\n", fixture.file_name())); +} + #[cfg(target_os = "linux")] #[test] fn keeps_object_unwind_dso_leaf_when_framehop_only_returns_current_ip_like_perf_libdw() { @@ -1953,6 +2164,7 @@ fn can_fold_samples_weighted_by_period() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -1984,6 +2196,7 @@ fn folds_sample_ip_when_callchain_is_absent_like_perf_script() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2024,6 +2237,7 @@ fn emits_sample_ip_when_callchain_field_is_absent_even_with_dwarf_payload_like_p &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2060,6 +2274,7 @@ fn selects_sample_layout_by_identifier() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2096,6 +2311,7 @@ fn selects_sample_layout_by_id_field() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2143,6 +2359,7 @@ fn folds_samples_from_multiple_attrs_when_generated_perf_script_event_name_match &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2171,6 +2388,7 @@ fn folds_perfdata_from_file_path() { &perfdata, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2257,6 +2475,7 @@ fn folds_perfdata_from_multiple_finished_rounds_into_one_total() { &perfdata, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2310,6 +2529,7 @@ fn folds_identical_rendered_stacks_across_pids_into_one_line() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2336,6 +2556,7 @@ fn forked_process_inherits_parent_mappings_like_perf_script() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2366,6 +2587,7 @@ fn synthesized_fork_does_not_clone_parent_mappings_like_perf_script() { &bytes, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2466,6 +2688,7 @@ fn folds_file_samples_from_multiple_attrs_when_generated_perf_script_event_name_ &perfdata, FoldOptions { count_periods: true, + inline: false, }, ) .expect("folded"); @@ -2529,7 +2752,7 @@ proptest! { let folded = fold_perfdata_callchains_with_options( &bytes, - FoldOptions { count_periods: true }, + FoldOptions { count_periods: true, inline: false }, ) .expect("folded"); let expected = render_unknown_folded_callchain(&frames, periods.iter().sum()); @@ -2568,7 +2791,7 @@ proptest! { let folded = fold_perfdata_callchains_with_options( &bytes, - FoldOptions { count_periods: true }, + FoldOptions { count_periods: true, inline: false }, ) .expect("folded"); @@ -2606,7 +2829,7 @@ proptest! { let folded = fold_perfdata_callchains_with_options( &bytes, - FoldOptions { count_periods: true }, + FoldOptions { count_periods: true, inline: false }, ) .expect("folded"); @@ -3270,7 +3493,9 @@ fn perfdata_with_records_attrs_and_build_id_feature( + attrs: [[u8; 144]; A], + records: [Vec; R], + arch: &str, +) -> Vec { + let attr_size = attrs.len() * 144; + let data_size = records.iter().map(Vec::len).sum::(); + let data_offset = 104 + attr_size; + let feature_table_offset = data_offset + data_size; + let arch_payload_offset = feature_table_offset + 16; + // do_write_string aligns the length to NAME_ALIGN (64) and writes the + // NUL-terminated name plus zero padding. + let aligned = (arch.len() + 1).next_multiple_of(64); + let mut arch_payload = Vec::new(); + arch_payload.extend(u32::try_from(aligned).expect("arch len").to_le_bytes()); + let mut name_bytes = arch.as_bytes().to_vec(); + name_bytes.resize(aligned, 0); + arch_payload.extend(name_bytes); + + let mut bytes = vec![0; 104]; + bytes[..8].copy_from_slice(b"PERFILE2"); + put_u64(&mut bytes, 8, 104); + put_u64(&mut bytes, 24, 104); + put_u64(&mut bytes, 32, attr_size as u64); + put_u64(&mut bytes, 40, data_offset as u64); + put_u64(&mut bytes, 48, data_size as u64); + // HEADER_ARCH feature bit (6) in the adds_features bitmap, which struct + // perf_file_header (tools/perf/util/header.h) places at byte offset 72. + put_u64(&mut bytes, 72, 1 << 6); + for attr in attrs { + bytes.extend(attr); + } + for record in records { + bytes.extend(record); + } + bytes.resize(arch_payload_offset, 0); + put_u64(&mut bytes, feature_table_offset, arch_payload_offset as u64); + put_u64( + &mut bytes, + feature_table_offset + 8, + u64::try_from(arch_payload.len()).expect("payload size"), + ); + bytes.extend(arch_payload); + bytes +} + fn perfdata_with_attrs_ids_and_records( attrs: [[u8; 144]; A], ids: [u64; I], @@ -3816,6 +4091,172 @@ impl SymbolResolver for InlineSymbolResolver { struct ArrowInlineSymbolResolver; +struct SyntheticX86_64Object { + _dir: tempfile::TempDir, + path: std::path::PathBuf, +} + +impl SyntheticX86_64Object { + /// Minimal x86_64 ELF with one PT_LOAD covering [0, 0x10000) and no unwind + /// info. The current-IP-only tests previously mapped the host test binary, + /// which made framehop's unwind host-dependent (a Mach-O/arm64 test binary + /// recovers callers through __unwind_info that a Linux x86_64 binary does + /// not have at these offsets). A synthetic ELF pins the libdw scenario the + /// tests encode: module reports, framehop yields only the seeded IP. + fn create() -> Self { + let mut bytes = vec![0_u8; 0x240]; + bytes[0..4].copy_from_slice(b"\x7fELF"); + bytes[4] = 2; // ELFCLASS64 + bytes[5] = 1; // ELFDATA2LSB + bytes[6] = 1; // EV_CURRENT + bytes[16..18].copy_from_slice(&3_u16.to_le_bytes()); // ET_DYN + bytes[18..20].copy_from_slice(&62_u16.to_le_bytes()); // EM_X86_64 + bytes[20..24].copy_from_slice(&1_u32.to_le_bytes()); // e_version + bytes[32..40].copy_from_slice(&64_u64.to_le_bytes()); // e_phoff + bytes[40..48].copy_from_slice(&0x180_u64.to_le_bytes()); // e_shoff + bytes[52..54].copy_from_slice(&64_u16.to_le_bytes()); // e_ehsize + bytes[54..56].copy_from_slice(&56_u16.to_le_bytes()); // e_phentsize + bytes[56..58].copy_from_slice(&1_u16.to_le_bytes()); // e_phnum + bytes[58..60].copy_from_slice(&64_u16.to_le_bytes()); // e_shentsize + bytes[60..62].copy_from_slice(&3_u16.to_le_bytes()); // e_shnum + bytes[62..64].copy_from_slice(&2_u16.to_le_bytes()); // e_shstrndx + bytes[64..68].copy_from_slice(&1_u32.to_le_bytes()); // PT_LOAD + bytes[68..72].copy_from_slice(&5_u32.to_le_bytes()); // PF_R | PF_X + bytes[96..104].copy_from_slice(&0x200_u64.to_le_bytes()); // p_filesz + bytes[104..112].copy_from_slice(&0x1_0000_u64.to_le_bytes()); // p_memsz + bytes[112..120].copy_from_slice(&0x1000_u64.to_le_bytes()); // p_align + // .eh_frame at vaddr/offset 0x100: one CIE and one FDE covering only + // [0x100, 0x104), so the module HAS unwind info but none of the + // sampled IPs are covered — the configuration where framehop stops + // after the seeded IP instead of taking a frame-pointer fallback, + // matching a real Linux binary sampled outside its FDE ranges. + let eh_frame: [u8; 52] = [ + 0x14, 0, 0, 0, // CIE length + 0, 0, 0, 0, // CIE id + 0x01, b'z', b'R', 0, // version, augmentation "zR" + 0x01, 0x78, 0x10, // code align 1, data align -8, ra 16 + 0x01, 0x1b, // augmentation: FDE encoding pcrel|sdata4 + 0, 0, 0, 0, 0, 0, 0, // DW_CFA_nop padding + 0x14, 0, 0, 0, // FDE length + 0x1c, 0, 0, 0, // CIE pointer (back 28 bytes) + 0xe0, 0xff, 0xff, 0xff, // pc_begin: pcrel -0x20 -> vaddr 0x100 + 0x04, 0, 0, 0, // pc_range 4 + 0, // augmentation data length + 0, 0, 0, 0, 0, 0, 0, // DW_CFA_nop padding + 0, 0, 0, 0, // terminator + ]; + bytes[0x100..0x100 + eh_frame.len()].copy_from_slice(&eh_frame); + let strtab = b"\0.eh_frame\0.shstrtab\0"; + bytes[0x140..0x140 + strtab.len()].copy_from_slice(strtab); + // Section headers: [0] SHT_NULL, [1] .eh_frame, [2] .shstrtab. + let mut section = + |index: usize, name: u32, kind: u32, flags: u64, addr: u64, offset: u64, size: u64| { + let base = 0x180 + index * 64; + bytes[base..base + 4].copy_from_slice(&name.to_le_bytes()); + bytes[base + 4..base + 8].copy_from_slice(&kind.to_le_bytes()); + bytes[base + 8..base + 16].copy_from_slice(&flags.to_le_bytes()); + bytes[base + 16..base + 24].copy_from_slice(&addr.to_le_bytes()); + bytes[base + 24..base + 32].copy_from_slice(&offset.to_le_bytes()); + bytes[base + 32..base + 40].copy_from_slice(&size.to_le_bytes()); + bytes[base + 48..base + 56].copy_from_slice(&8_u64.to_le_bytes()); + }; + section(1, 1, 1, 2, 0x100, 0x100, 52); // .eh_frame PROGBITS ALLOC + section(2, 11, 3, 0, 0, 0x140, 21); // .shstrtab STRTAB + let dir = tempfile::tempdir().expect("fixture dir"); + let path = dir.path().join("fixture-x86-64"); + std::fs::write(&path, &bytes).expect("write fixture elf"); + Self { _dir: dir, path } + } + + fn path_string(&self) -> String { + self.path.to_string_lossy().into_owned() + } + + fn file_name(&self) -> &'static str { + "fixture-x86-64" + } +} + +/// Minimal aarch64 ELF, structurally identical to `SyntheticX86_64Object` but +/// with `e_machine = EM_AARCH64` (183) and a `.eh_frame` FDE that covers only +/// `[0x100, 0x104)`. Used to pin the aarch64 scenario-D leaf-only case: a +/// reported module, no FDE covering the sampled pc, and `lr == 0` so the +/// elfutils `backends/aarch64_unwind.c` fallback fails before producing any +/// caller. +struct SyntheticAarch64Object { + _dir: tempfile::TempDir, + path: std::path::PathBuf, +} + +impl SyntheticAarch64Object { + fn create() -> Self { + let mut bytes = vec![0_u8; 0x240]; + bytes[0..4].copy_from_slice(b"\x7fELF"); + bytes[4] = 2; // ELFCLASS64 + bytes[5] = 1; // ELFDATA2LSB + bytes[6] = 1; // EV_CURRENT + bytes[16..18].copy_from_slice(&3_u16.to_le_bytes()); // ET_DYN + bytes[18..20].copy_from_slice(&183_u16.to_le_bytes()); // EM_AARCH64 + bytes[20..24].copy_from_slice(&1_u32.to_le_bytes()); // e_version + bytes[32..40].copy_from_slice(&64_u64.to_le_bytes()); // e_phoff + bytes[40..48].copy_from_slice(&0x180_u64.to_le_bytes()); // e_shoff + bytes[52..54].copy_from_slice(&64_u16.to_le_bytes()); // e_ehsize + bytes[54..56].copy_from_slice(&56_u16.to_le_bytes()); // e_phentsize + bytes[56..58].copy_from_slice(&1_u16.to_le_bytes()); // e_phnum + bytes[58..60].copy_from_slice(&64_u16.to_le_bytes()); // e_shentsize + bytes[60..62].copy_from_slice(&3_u16.to_le_bytes()); // e_shnum + bytes[62..64].copy_from_slice(&2_u16.to_le_bytes()); // e_shstrndx + bytes[64..68].copy_from_slice(&1_u32.to_le_bytes()); // PT_LOAD + bytes[68..72].copy_from_slice(&5_u32.to_le_bytes()); // PF_R | PF_X + bytes[96..104].copy_from_slice(&0x200_u64.to_le_bytes()); // p_filesz + bytes[104..112].copy_from_slice(&0x1_0000_u64.to_le_bytes()); // p_memsz + bytes[112..120].copy_from_slice(&0x1000_u64.to_le_bytes()); // p_align + let eh_frame: [u8; 52] = [ + 0x14, 0, 0, 0, // CIE length + 0, 0, 0, 0, // CIE id + 0x01, b'z', b'R', 0, // version, augmentation "zR" + 0x01, 0x78, 0x1e, // code align 1, data align -8, ra 30 (aarch64 LR) + 0x01, 0x1b, // augmentation: FDE encoding pcrel|sdata4 + 0, 0, 0, 0, 0, 0, 0, // DW_CFA_nop padding + 0x14, 0, 0, 0, // FDE length + 0x1c, 0, 0, 0, // CIE pointer (back 28 bytes) + 0xe0, 0xff, 0xff, 0xff, // pc_begin: pcrel -0x20 -> vaddr 0x100 + 0x04, 0, 0, 0, // pc_range 4 + 0, // augmentation data length + 0, 0, 0, 0, 0, 0, 0, // DW_CFA_nop padding + 0, 0, 0, 0, // terminator + ]; + bytes[0x100..0x100 + eh_frame.len()].copy_from_slice(&eh_frame); + let strtab = b"\0.eh_frame\0.shstrtab\0"; + bytes[0x140..0x140 + strtab.len()].copy_from_slice(strtab); + let mut section = + |index: usize, name: u32, kind: u32, flags: u64, addr: u64, offset: u64, size: u64| { + let base = 0x180 + index * 64; + bytes[base..base + 4].copy_from_slice(&name.to_le_bytes()); + bytes[base + 4..base + 8].copy_from_slice(&kind.to_le_bytes()); + bytes[base + 8..base + 16].copy_from_slice(&flags.to_le_bytes()); + bytes[base + 16..base + 24].copy_from_slice(&addr.to_le_bytes()); + bytes[base + 24..base + 32].copy_from_slice(&offset.to_le_bytes()); + bytes[base + 32..base + 40].copy_from_slice(&size.to_le_bytes()); + bytes[base + 48..base + 56].copy_from_slice(&8_u64.to_le_bytes()); + }; + section(1, 1, 1, 2, 0x100, 0x100, 52); // .eh_frame PROGBITS ALLOC + section(2, 11, 3, 0, 0, 0x140, 21); // .shstrtab STRTAB + let dir = tempfile::tempdir().expect("fixture dir"); + let path = dir.path().join("fixture-aarch64"); + std::fs::write(&path, &bytes).expect("write fixture elf"); + Self { _dir: dir, path } + } + + fn path_string(&self) -> String { + self.path.to_string_lossy().into_owned() + } + + fn file_name(&self) -> &'static str { + "fixture-aarch64" + } +} + fn current_exe_file_name() -> String { std::env::current_exe() .expect("current exe") diff --git a/tests/perfdata_header.rs b/tests/perfdata_header.rs index 3c9026d..b5284cf 100644 --- a/tests/perfdata_header.rs +++ b/tests/perfdata_header.rs @@ -1,6 +1,6 @@ use proptest::prelude::*; use pyroclast::perfdata::header::{ - PerfFeatureSection, PerfHeader, parse_feature_sections, parse_header, + PerfFeatureSection, PerfHeader, parse_feature_sections, parse_header, parse_header_arch, }; #[test] @@ -32,9 +32,13 @@ fn rejects_non_perfdata_magic() { #[test] fn parses_feature_sections_from_set_header_bits() { + // tools/perf/util/header.h struct perf_file_header places the + // adds_features DECLARE_BITMAP at byte offset 72 (after magic[8], size[8], + // attr_size[8], and the three 16-byte perf_file_section structs attrs, + // data, event_types). The feature section table follows the data section. let mut bytes = vec![0; 520]; bytes[..104].copy_from_slice(&header_bytes("PERFILE2", 104, 128, 64, 256, 128)); - put_u64(&mut bytes, 56, 1 << 2); + put_u64(&mut bytes, 72, 1 << 2); put_u64(&mut bytes, 384, 448); put_u64(&mut bytes, 392, 72); @@ -51,6 +55,36 @@ fn parses_feature_sections_from_set_header_bits() { ); } +#[test] +fn parses_header_arch_feature_string() { + // HEADER_ARCH (bit 6) payload is a perf_header_string: u32 length followed + // by that many bytes of NUL-terminated text (util/header.c do_read_string). + let mut bytes = vec![0; 520]; + bytes[..104].copy_from_slice(&header_bytes("PERFILE2", 104, 128, 64, 256, 128)); + // adds_features bitmap lives at offset 72 (struct perf_file_header). + put_u64(&mut bytes, 72, 1 << 6); + put_u64(&mut bytes, 384, 448); + put_u64(&mut bytes, 392, 16); + bytes[448..452].copy_from_slice(&12_u32.to_le_bytes()); + bytes[452..459].copy_from_slice(b"aarch64"); + + let header = parse_header(&bytes).expect("header"); + + assert_eq!( + parse_header_arch(&bytes, &header).expect("arch"), + Some("aarch64".to_string()) + ); +} + +#[test] +fn header_arch_is_none_when_feature_is_absent() { + let bytes = header_bytes("PERFILE2", 104, 128, 64, 256, 0); + + let header = parse_header(&bytes).expect("header"); + + assert_eq!(parse_header_arch(&bytes, &header).expect("arch"), None); +} + proptest! { #[test] fn property_parses_arbitrary_valid_headers( @@ -91,7 +125,9 @@ proptest! { let mut expected = Vec::new(); for (index, feature) in features.iter().copied().enumerate() { - let word_offset = 56 + usize::from(feature / 64) * 8; + // adds_features bitmap starts at offset 72 (struct perf_file_header, + // tools/perf/util/header.h). + let word_offset = 72 + usize::from(feature / 64) * 8; let bit = 1_u64 << u32::from(feature % 64); let word = u64::from_le_bytes( bytes[word_offset..word_offset + 8] diff --git a/tests/perfdata_mappings.rs b/tests/perfdata_mappings.rs index 768b864..02eba48 100644 --- a/tests/perfdata_mappings.rs +++ b/tests/perfdata_mappings.rs @@ -342,6 +342,100 @@ fn resolves_build_id_from_mmap2_build_id_mapping() { ); } +#[test] +fn unifies_symbol_source_for_one_object_across_mmap_record_forms() { + // The same on-disk object can arrive as an inline MMAP2-build-id record + // (build_id, no file_identity) or as a plain MMAP2 carrying file_identity + // plus a HEADER_BUILD_ID that supplies the same build_id. perf's + // __dso_id__cmp (tools/perf/util/dso.c) makes the build_id decisive once + // both ids define it, so both forms must map to a single symbol source. + let build_id = vec![0xaa, 0xbb, 0xcc, 0xdd]; + let mut table = MmapTable::default(); + table.insert_mmap2_build_id(Mmap2BuildIdRecord { + pid: 1, + tid: 1, + start: 0x1000, + len: 0x200, + build_id_size: 4, + build_id: build_id.clone(), + pgoff: 0, + prot: 5, + flags: 2, + path: "/usr/lib/libc.so.6".to_string(), + }); + table.insert_mmap2_with_build_id( + Mmap2Record { + pid: 2, + tid: 2, + start: 0x4000, + len: 0x200, + pgoff: 0, + major: 8, + minor: 1, + inode: 99, + inode_generation: 7, + prot: 5, + flags: 2, + path: "/usr/lib/libc.so.6".to_string(), + }, + Some(build_id.clone()), + ); + + let inline = table + .resolve_ref(1, 0x1010) + .expect("inline build-id mapping"); + let header = table + .resolve_ref(2, 0x4010) + .expect("header build-id mapping"); + assert_eq!( + inline.symbol_source_id, header.symbol_source_id, + "one object must resolve through one symbol source regardless of mmap form" + ); +} + +#[test] +fn keeps_distinct_symbol_sources_for_different_build_ids_at_same_path() { + // perf's __dso_id__cmp compares the build_id when both are defined, so two + // genuinely different objects at the same path (e.g. a replaced binary) + // must remain distinct symbol sources. + let mut table = MmapTable::default(); + table.insert_mmap2_build_id(Mmap2BuildIdRecord { + pid: 1, + tid: 1, + start: 0x1000, + len: 0x200, + build_id_size: 4, + build_id: vec![0xaa, 0xbb, 0xcc, 0xdd], + pgoff: 0, + prot: 5, + flags: 2, + path: "/usr/lib/libc.so.6".to_string(), + }); + table.insert_mmap2_build_id(Mmap2BuildIdRecord { + pid: 2, + tid: 2, + start: 0x4000, + len: 0x200, + build_id_size: 4, + build_id: vec![0x11, 0x22, 0x33, 0x44], + pgoff: 0, + prot: 5, + flags: 2, + path: "/usr/lib/libc.so.6".to_string(), + }); + + let first = table + .resolve_ref(1, 0x1010) + .expect("first build-id mapping"); + let second = table + .resolve_ref(2, 0x4010) + .expect("second build-id mapping"); + assert_ne!( + first.symbol_source_id, second.symbol_source_id, + "different build_ids at the same path must stay distinct symbol sources" + ); +} + #[test] fn resolves_file_identity_from_mmap2_mapping() { let mut table = MmapTable::default(); @@ -371,6 +465,17 @@ fn resolves_file_identity_from_mmap2_mapping() { ); } +// Mirrors the glibc MAJOR()/MINOR() decomposition perf uses when recording +// device numbers in PERF_RECORD_MMAP2 (tools/perf/util/dso.c __dso_id__cmp +// compares maj/min/ino together, never the inode alone). +fn device_major(device: u64) -> u32 { + (((device >> 8) & 0xfff) | ((device >> 32) & !0xfff)) as u32 +} + +fn device_minor(device: u64) -> u32 { + ((device & 0xff) | ((device >> 12) & !0xff)) as u32 +} + #[test] fn compares_recorded_file_identity_with_current_path() { let root = tempfile::tempdir().expect("tempdir"); @@ -381,8 +486,8 @@ fn compares_recorded_file_identity_with_current_path() { assert!(file_matches_recorded_identity( &path, FileIdentity { - major: 0, - minor: 0, + major: device_major(metadata.dev()), + minor: device_minor(metadata.dev()), inode: metadata.ino(), inode_generation: 0, } @@ -390,14 +495,44 @@ fn compares_recorded_file_identity_with_current_path() { assert!(!file_matches_recorded_identity( &path, FileIdentity { - major: 0, - minor: 0, + major: device_major(metadata.dev()), + minor: device_minor(metadata.dev()), inode: metadata.ino() + 1, inode_generation: 0, } )); } +#[test] +fn rejects_recorded_file_identity_on_a_different_device() { + // perf's __dso_id__cmp compares maj/min/ino together; an inode number is + // unique only within a filesystem, so a matching inode on a different + // device must NOT be accepted as the same backing store. + let root = tempfile::tempdir().expect("tempdir"); + let path = root.path().join("app"); + std::fs::write(&path, b"binary").expect("write app"); + let metadata = std::fs::metadata(&path).expect("metadata"); + + assert!(!file_matches_recorded_identity( + &path, + FileIdentity { + major: device_major(metadata.dev()).wrapping_add(1), + minor: device_minor(metadata.dev()), + inode: metadata.ino(), + inode_generation: 0, + } + )); + assert!(!file_matches_recorded_identity( + &path, + FileIdentity { + major: device_major(metadata.dev()), + minor: device_minor(metadata.dev()).wrapping_add(1), + inode: metadata.ino(), + inode_generation: 0, + } + )); +} + #[test] fn does_not_resolve_other_pids_or_out_of_range_ips() { let mut table = MmapTable::default(); diff --git a/tests/perfdata_unwind.rs b/tests/perfdata_unwind.rs index 3112142..c9fc7c6 100644 --- a/tests/perfdata_unwind.rs +++ b/tests/perfdata_unwind.rs @@ -2,8 +2,8 @@ use framehop::x86_64::Reg; use object::{Object, ObjectSection, ObjectSegment}; use proptest::prelude::*; use pyroclast::perfdata::unwind::{ - FramehopUnwinder, PerfStackReader, PerfUserMemoryReader, PerfX86_64Regs, UserStackUnwindResult, - UserStackUnwinder, unwind_x86_64_stack, + FramehopUnwinder, PerfStackReader, PerfUserMemoryReader, PerfUserRegs, PerfX86_64Regs, + UserStackUnwindResult, UserStackUnwinder, unwind_x86_64_stack, }; #[test] @@ -191,12 +191,12 @@ fn object_unwind_attempts_initial_plt_frame_without_cfi_like_perf_libdw() { assert!(!unwinder.has_unwind_info_for_ip(ip)); let frames = unwinder.unwind_stack( - PerfX86_64Regs { + PerfUserRegs::X86_64(PerfX86_64Regs { ip, sp, bp: sp, registers: registers_with_bp_sp(sp, sp), - }, + }), &stack, 4, ); @@ -204,6 +204,48 @@ fn object_unwind_attempts_initial_plt_frame_without_cfi_like_perf_libdw() { assert_eq!(frames.first(), Some(&ip)); } +#[test] +fn has_unwind_info_memo_is_consistent_and_invalidated_on_module_add() { + // The gap-2 skip gate queries has_unwind_info_for_ip per sample; it is + // memoized by exact ip. A repeated query must return the same answer, and + // adding a module (which can extend coverage) must invalidate the memo so a + // later query at a now-covered ip sees the new CFI. + let current_exe = std::env::current_exe().expect("current exe"); + let base = 0x6666_0000_0000; + + // Ground truth from a fresh, never-memoized unwinder with the module loaded. + let mut ground_truth = FramehopUnwinder::new(); + ground_truth + .add_object_mapping(¤t_exe, base, 0x1000_0000, 0) + .expect("load ground-truth module"); + // Pick an ip the host binary's CFI actually covers; if the host toolchain + // emitted no unwind info at all, skip (nothing to prove about coverage). + let covered_ip = (base..base + 0x0010_0000) + .step_by(0x40) + .find(|&ip| ground_truth.has_unwind_info_for_ip(ip)); + let Some(covered_ip) = covered_ip else { + return; + }; + + let mut unwinder = FramehopUnwinder::new(); + // Query before the covering module exists: memoizes `false`, and the + // repeat must be consistent with the first answer. + assert!(!unwinder.has_unwind_info_for_ip(covered_ip)); + assert!(!unwinder.has_unwind_info_for_ip(covered_ip)); + + assert!( + unwinder + .add_object_mapping(¤t_exe, base, 0x1000_0000, 0) + .expect("load module") + ); + + // The memo was cleared on module add, so the previously-cached `false` + // is recomputed to the now-correct `true`, matching the un-memoized + // ground truth. + assert!(unwinder.has_unwind_info_for_ip(covered_ip)); + assert!(unwinder.has_unwind_info_for_ip(covered_ip)); +} + #[test] fn framehop_unwinder_implements_pluggable_user_stack_unwinder_boundary() { let mut unwinder: Box = Box::new(FramehopUnwinder::new()); @@ -214,7 +256,8 @@ fn framehop_unwinder_implements_pluggable_user_stack_unwinder_boundary() { registers: registers_with_bp_sp(0x7fff_0000, 0x7fff_0000), }; - let result: UserStackUnwindResult = unwinder.unwind_user_stack(regs, &[], 4); + let result: UserStackUnwindResult = + unwinder.unwind_user_stack(PerfUserRegs::X86_64(regs), &[], 4); assert_eq!(result.accepted_frames, vec![regs.ip]); } @@ -267,7 +310,10 @@ fn rejected_overlapping_module_range_does_not_unwind_through_prior_module() { .expect("reject overlapping object mapping") ); - assert_eq!(unwinder.unwind_stack(regs, &stack, 4), Vec::::new()); + assert_eq!( + unwinder.unwind_stack(PerfUserRegs::X86_64(regs), &stack, 4), + Vec::::new() + ); } #[test] diff --git a/tests/run_cli.rs b/tests/run_cli.rs index 0e0040d..4310912 100644 --- a/tests/run_cli.rs +++ b/tests/run_cli.rs @@ -21,7 +21,7 @@ fn top_level_memory_command_uses_injected_heaptrack_runner() { "check", ]); - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect("run cli"); + pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux").expect("run cli"); assert!(out.join("run.json").is_file()); assert!(out.join("command.txt").is_file()); @@ -125,7 +125,7 @@ fn perf_script_command_exports_inferno_compatible_perf_script() { assert_eq!( output.stdout, - "app 2 [003] 0.123456: 144 cycles:\n\t 2000 [unknown] (/bin/app)\n\n" + "app 2 [003] 0.123456: 144 cycles: \n\t 2000 [unknown] (/bin/app)\n\n" ); } @@ -176,7 +176,7 @@ fn perf_script_command_uses_perf_default_thread_comm_when_comm_is_missing() { assert_eq!( output.stdout, - ":2 2 [003] 0.123456: 144 cycles:\n\t 2000 [unknown] (/bin/app)\n\n" + ":2 2 [003] 0.123456: 144 cycles: \n\t 2000 [unknown] (/bin/app)\n\n" ); } @@ -225,9 +225,9 @@ fn perf_script_command_preserves_sample_event_records_like_perf_script() { assert_eq!( output.stdout, concat!( - "app 2 [004] 0.000010: 7 cycles:\n", + "app 2 [004] 0.000010: 7 cycles: \n", "\t 2000 [unknown] (/bin/app)\n\n", - "app 2 [005] 0.000020: 11 cycles:\n", + "app 2 [005] 0.000020: 11 cycles: \n", "\t 2000 [unknown] (/bin/app)\n\n", ) ); @@ -308,7 +308,7 @@ fn perf_script_command_uses_perf_event_name_from_software_attr_like_perf_script( assert_eq!( output.stdout, - "app 2 144 cpu-clock:\n\t 2000 [unknown] (/bin/app)\n\n" + "app 2 144 cpu-clock: \n\t 2000 [unknown] (/bin/app)\n\n" ); } @@ -376,9 +376,9 @@ fn perf_script_command_pads_event_names_to_evlist_max_width_like_perf_script() { assert_eq!( output.stdout, concat!( - "app 2 5 cycles:\n", + "app 2 5 cycles: \n", "\t 2000 [unknown] ([unknown])\n\n", - "app 2 7 cpu-clock:\n", + "app 2 7 cpu-clock: \n", "\t 2000 [unknown] ([unknown])\n\n", ) ); @@ -415,7 +415,7 @@ fn perf_script_command_omits_tid_column_when_sample_type_lacks_tid_like_perf_scr assert_eq!( output.stdout, - ":-1 5 cycles:\n\t 2000 [unknown] ([unknown])\n\n" + ":-1 5 cycles: \n\t 2000 [unknown] ([unknown])\n\n" ); } @@ -460,7 +460,7 @@ fn perf_script_command_inherits_parent_comm_on_fork_like_perf_script() { assert_eq!( output.stdout, - "sh 22 [006] 0.000030: 5 cycles:\n\t 2000 [unknown] (/bin/sh)\n\n" + "sh 22 [006] 0.000030: 5 cycles: \n\t 2000 [unknown] (/bin/sh)\n\n" ); } @@ -513,7 +513,7 @@ fn perf_script_command_keeps_perf_stack_order_and_skips_context_markers() { assert_eq!( output.stdout, concat!( - "app 2 [000] 0.000000: 13 cycles:\n", + "app 2 [000] 0.000000: 13 cycles: \n", "\t 2000 [unknown] (/bin/app)\n", "\t 2100 [unknown] (/bin/app)\n\n", ) @@ -541,10 +541,13 @@ fn fold_command_can_symbolize_mapped_frames() { ) .expect("write perfdata"); let runner = RecordingRunner::default(); + // The base symbol comes from the in-process ELF symtab without --inline; + // exercising the external addr2line resolver requires the inline path. let cli = pyroclast::cli::Cli::parse_from([ "pyroclast", "plumbing", "fold", + "--inline", "--symbolizer", "addr2line", perfdata.to_str().unwrap(), @@ -743,10 +746,13 @@ fn flamegraph_command_can_symbolize_mapped_frames() { ) .expect("write perfdata"); let runner = RecordingRunner::default(); + // The base symbol comes from the in-process ELF symtab without --inline; + // exercising the external addr2line resolver requires the inline path. let cli = pyroclast::cli::Cli::parse_from([ "pyroclast", "plumbing", "flamegraph", + "--inline", "--symbolizer", "addr2line", perfdata.to_str().expect("perfdata path"), @@ -995,7 +1001,7 @@ fn top_level_cpu_command_uses_injected_perf_runner() { "true", ]); - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect("run cli"); + pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux").expect("run cli"); assert_eq!(runner.programs(), vec!["perf", "inferno-flamegraph"]); let run_json = std::fs::read_to_string(out.join("run.json")).expect("run json"); @@ -1030,7 +1036,7 @@ fn profile_cpu_command_uses_injected_perf_runner() { "true", ]); - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect("run cli"); + pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux").expect("run cli"); assert_eq!(runner.programs(), vec!["perf", "inferno-flamegraph"]); let run_json = std::fs::read_to_string(out.join("run.json")).expect("run json"); @@ -1083,7 +1089,7 @@ fn profile_memory_command_keeps_symbols_off_by_default() { "true", ]); - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect("run cli"); + pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux").expect("run cli"); assert_eq!(runner.programs(), vec!["heaptrack", "heaptrack_print"]); let run_json = std::fs::read_to_string(out.join("run.json")).expect("run json"); @@ -1105,7 +1111,7 @@ fn top_level_latency_command_uses_injected_strace_runner() { "true", ]); - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect("run cli"); + pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux").expect("run cli"); assert_eq!(runner.programs(), vec!["strace"]); let run_json = std::fs::read_to_string(out.join("run.json")).expect("run json"); @@ -1131,7 +1137,7 @@ fn top_level_offcpu_command_uses_injected_perf_sched_runner() { "true", ]); - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect("run cli"); + pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux").expect("run cli"); assert_eq!(runner.programs(), vec!["perf", "perf"]); let run_json = std::fs::read_to_string(out.join("run.json")).expect("run json"); @@ -1157,8 +1163,8 @@ fn top_level_offcpu_command_rejects_attach_workflows() { "5", ]); - let error = - pyroclast::run_parsed_cli_with_runner(cli, &runner).expect_err("attach should fail"); + let error = pyroclast::run_parsed_cli_with_runner_on_platform(cli, &runner, "linux") + .expect_err("attach should fail"); assert_eq!( error.to_string(), diff --git a/tests/symbols.rs b/tests/symbols.rs index aa89255..8f72533 100644 --- a/tests/symbols.rs +++ b/tests/symbols.rs @@ -617,7 +617,16 @@ fn specializes_qualified_generic_placeholder_dwarf_names_from_debug_strings() { } #[test] -fn perf_dwarf_frame_names_prefer_die_names_like_perf_script() { +fn perf_dwarf_frame_names_match_external_addr2line_qualified_names_like_perf_script() { + // perf's external-addr2line srcline backend (the modern oracle build) names + // each frame from the mangled symtab/DWARF linkage name returned by + // `addr2line -f -i` and demangles it itself with the Rust v0 demangler in + // alternate form (tools/perf/util/srcline.c new_inline_sym -> + // tools/perf/util/symbol.c dso__demangle_sym -> + // rust_demangle_display_demangle(..., /*alternate=*/true)). The result is + // fully qualified with generic arguments preserved -- NOT the bare DWARF + // DW_AT_name leaf the older libdw backend printed. This test pins that + // pyroclast now matches the external-addr2line spelling frame-for-frame. let Some((profiling_binary, object_bytes)) = profiling_binary_fixture() else { return; }; @@ -629,14 +638,13 @@ fn perf_dwarf_frame_names_prefer_die_names_like_perf_script() { return false; }; let Some(expected) = - external_addr2line_frames_leaf_to_root(&profiling_binary, *address) + external_addr2line_qualified_frames_leaf_to_root(&profiling_binary, *address) else { return false; }; - frames.len() == expected.len() - && frames.iter().zip(expected.iter()).any(|(frame, external)| { - frame != external && frame.contains('<') && !external.contains('<') - }) + // Only meaningful where the inline chain carries a qualified + // generic name (so the libdw leaf spelling would have differed). + frames.len() == expected.len() && frames.iter().any(|frame| frame.contains('<')) }) else { return; @@ -644,13 +652,10 @@ fn perf_dwarf_frame_names_prefer_die_names_like_perf_script() { let frames = perf_dwarf_frame_names_from_object(&profiling_binary, address).expect("perf dwarf frames"); - let expected = external_addr2line_frames_leaf_to_root(&profiling_binary, address) + let expected = external_addr2line_qualified_frames_leaf_to_root(&profiling_binary, address) .expect("external addr2line frames"); - assert_eq!(frames.len(), expected.len()); - assert!(frames.iter().zip(expected.iter()).any(|(frame, external)| { - frame != external && frame.contains('<') && !external.contains('<') - })); + assert_eq!(frames, expected); } #[test] @@ -950,6 +955,34 @@ fn external_addr2line_frames_root_to_leaf(path: &Path, address: u64) -> Option dso__demangle_sym +/// does. `addr2line::demangle_auto` is byte-identical to perf's alternate Rust +/// demangle for both legacy `_ZN` and v0 `_R` symbols. +fn external_addr2line_qualified_frames_leaf_to_root( + path: &Path, + address: u64, +) -> Option> { + let output = Command::new("addr2line") + .args(["-f", "-i", "-e", path.to_str()?, &format!("0x{address:x}")]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + + let stdout = String::from_utf8(output.stdout).ok()?; + let frames = stdout + .lines() + .step_by(2) + .filter(|name| *name != "??") + .map(|name| addr2line::demangle_auto(Cow::Borrowed(name), None).into_owned()) + .collect::>(); + (!frames.is_empty()).then_some(frames) +} + #[test] fn rejects_ambiguous_generic_dwarf_names_from_debug_strings() { let debug_strings = @@ -1223,10 +1256,13 @@ ffffffff88000080 t asm_exc_page_fault ]) .expect("symbols"); + // perf-script prints kernel frames as `name+0x` + // (tools/perf/util/symbol_fprintf.c __symbol__fprintf_symname_offs); the + // folded path strips the offset like every other frame. assert_eq!( symbols, vec![ - Some("asm_exc_page_fault".to_string()), + Some("asm_exc_page_fault+0xf".to_string()), Some("app::main".to_string()) ] ); @@ -1258,7 +1294,8 @@ fn perf_symbol_resolver_prefers_live_kallsyms_for_kernel_module_paths() { }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("zpl_iter_read".to_string())]); + // perf-script kernel frames carry the +0x offset (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("zpl_iter_read+0xe9".to_string())]); assert!(runner.commands().is_empty()); } @@ -1288,7 +1325,8 @@ ffffffff82000000 T later_kernel_symbol }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("asm_exc_page_fault".to_string())]); + // The relocated address lands on the symbol start, so perf prints +0x0. + assert_eq!(symbols, vec![Some("asm_exc_page_fault+0x0".to_string())]); assert!(runner.commands().is_empty()); } @@ -1318,7 +1356,7 @@ fn perf_symbol_resolver_loads_perfdata_kernel_build_id_cache() { }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("asm_exc_page_fault".to_string())]); + assert_eq!(symbols, vec![Some("asm_exc_page_fault+0xf".to_string())]); assert!(runner.commands().is_empty()); } @@ -1351,7 +1389,7 @@ fn perf_symbol_resolver_loads_perfdata_kernel_build_id_cache_from_file() { }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("asm_exc_page_fault".to_string())]); + assert_eq!(symbols, vec![Some("asm_exc_page_fault+0xf".to_string())]); assert!(runner.commands().is_empty()); } @@ -1386,7 +1424,9 @@ fn nixos_system_map_path_sits_next_to_kernel_image_symlink_target() { assert_eq!( pyroclast::symbols::nixos_system_map_path(&kernel), - Some(system_map) + // nixos_system_map_path canonicalizes the kernel image, so resolve the + // expectation through tempdir symlinks (macOS /var -> /private/var). + Some(std::fs::canonicalize(&system_map).expect("canonicalize system map")) ); } @@ -1460,7 +1500,9 @@ fn perf_symbol_resolver_constructor_uses_perfdata_cache_before_system_kallsyms() }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("cached_kernel_symbol".to_string())]); + // perf-script kernel frames carry +0x (symbol_fprintf.c); folded + // output strips it. + assert_eq!(symbols, vec![Some("cached_kernel_symbol+0xf".to_string())]); assert!(runner.commands().is_empty()); } @@ -1532,7 +1574,8 @@ fn perf_symbol_resolver_prefers_perfdata_kallsyms_over_kernel_elf() { }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("__pi_memcpy".to_string())]); + // perf-script kernel frames carry +0x (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("__pi_memcpy+0xf".to_string())]); assert!(runner.commands().is_empty()); } @@ -1605,7 +1648,8 @@ ffffffff846997a0 T __pi_memcpy }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("__pi_memcpy".to_string())]); + // perf-script kernel frames carry +0x (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("__pi_memcpy+0xc".to_string())]); assert!(runner.commands().is_empty()); } @@ -1639,7 +1683,8 @@ ffffffff846997a0 T memcpy }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("__pi_memcpy".to_string())]); + // perf-script kernel frames carry +0x (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("__pi_memcpy+0xc".to_string())]); } #[test] @@ -1670,7 +1715,8 @@ ffffffffc0e17dae t zfs_read [zfs] }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("zfs_read".to_string())]); + // perf-script kernel/module frames carry +0x (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("zfs_read+0x0".to_string())]); assert!(runner.commands().is_empty()); } @@ -1703,7 +1749,8 @@ ffffffffc1e17dae t igb_clean_rx_irq [igb] let symbols = resolver .resolve_batch(std::slice::from_ref(&zfs)) .expect("symbols"); - assert_eq!(symbols, vec![Some("zfs_read".to_string())]); + // perf-script kernel/module frames carry +0x (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("zfs_read+0x0".to_string())]); std::fs::write( &live_kallsyms, @@ -1726,8 +1773,8 @@ ffffffffc2e17dae t unrelated_module_symbol [mlx5] assert_eq!( symbols, vec![ - Some("zfs_read".to_string()), - Some("igb_clean_rx_irq".to_string()) + Some("zfs_read+0x0".to_string()), + Some("igb_clean_rx_irq+0x0".to_string()) ] ); assert!(runner.commands().is_empty()); @@ -1761,7 +1808,8 @@ ffffffff846997a0 T memcpy }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("__pi_memcpy".to_string())]); + // perf-script kernel frames carry +0x (symbol_fprintf.c). + assert_eq!(symbols, vec![Some("__pi_memcpy+0xc".to_string())]); assert!(runner.commands().is_empty()); } @@ -1981,7 +2029,9 @@ fn perf_symbol_resolver_uses_system_map_candidates_when_cache_is_missing() { }]) .expect("symbols"); - assert_eq!(symbols, vec![Some("asm_exc_page_fault".to_string())]); + // perf-script kernel frames carry +0x (symbol_fprintf.c); the + // relocated address lands on the symbol start. + assert_eq!(symbols, vec![Some("asm_exc_page_fault+0x0".to_string())]); assert!(runner.commands().is_empty()); } @@ -2022,11 +2072,12 @@ fn perf_symbol_resolver_keeps_live_kallsyms_for_modules_when_system_map_exists() ]) .expect("symbols"); + // perf-script kernel/module frames carry +0x (symbol_fprintf.c). assert_eq!( symbols, vec![ - Some("asm_exc_page_fault".to_string()), - Some("zfs_read".to_string()) + Some("asm_exc_page_fault+0x0".to_string()), + Some("zfs_read+0x0".to_string()) ] ); } @@ -2069,7 +2120,9 @@ fn perfdata_with_build_id_feature(payload: &[u8]) -> Vec { put_u64(&mut bytes, 8, 104); put_u64(&mut bytes, 40, 128); put_u64(&mut bytes, 48, 0); - put_u64(&mut bytes, 56, 1 << 2); + // HEADER_BUILD_ID feature bit (2) in the adds_features bitmap at byte + // offset 72 (struct perf_file_header, tools/perf/util/header.h). + put_u64(&mut bytes, 72, 1 << 2); put_u64(&mut bytes, feature_table_offset, payload_offset as u64); put_u64( &mut bytes,