From a6e65a7bd37bf72aac3c9bdd9827db2d05c74279 Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Wed, 30 Sep 2026 23:23:44 +0100 Subject: [PATCH] tp: add cold sorts by key for sorts which are not hot std::sort and std::stable_sort are compiled for every element type and comparator, at 3-4KB each, even where the sort runs once per query or per trace on a handful of elements. ColdSortByKey and ColdStableSortByKey sort by an integer key through one std::sort of {key, index} pairs, compiled once. Each caller adds a few hundred bytes. Comparing integers inline keeps them within 2x of std::sort, and faster where the comparator did lookups or copies. Moves the sorts in query planning, the PerfettoSQL parser, SQL rewrites, trace type detection, system probes, perf feature sections, CPU per UID, the ninja log, the symbolizer, metatrace args, SurfaceFlinger layers, interval intersect's operand order, JSON export, Android logs and the memory graph over. This saves 110KB of trace_processor_shell's code on Linux and 147KB of the Wasm module. --- Android.bp | 14 ++ BUILD | 24 ++++ src/trace_processor/BUILD.gn | 1 + src/trace_processor/core/dataframe/BUILD.gn | 1 + .../core/dataframe/logical_plan.cc | 11 +- src/trace_processor/core/exec/BUILD.gn | 1 + .../core/exec/interval_intersect.cc | 6 +- src/trace_processor/export_json.cc | 11 +- .../importers/android_bugreport/BUILD.gn | 1 + .../android_bugreport/android_log_reader.cc | 5 +- .../importers/memory_tracker/BUILD.gn | 5 +- .../memory_tracker/graph_processor.cc | 12 +- src/trace_processor/importers/ninja/BUILD.gn | 1 + .../importers/ninja/ninja_log_parser.cc | 5 +- src/trace_processor/importers/perf/BUILD.gn | 1 + .../importers/perf/perf_data_tokenizer.cc | 25 ++-- src/trace_processor/importers/proto/BUILD.gn | 2 + .../proto/android_cpu_per_uid_module.cc | 5 +- .../proto/proto_trace_parser_impl.cc | 8 +- .../importers/proto/system_probes_parser.cc | 10 +- .../perfetto_sql/parser/BUILD.gn | 1 + .../parser/perfetto_sql_parser.cc | 7 +- .../plugins/interval_intersect/BUILD.gn | 1 + .../interval_intersect/interval_intersect.cc | 11 +- .../plugins/winscope_importer/BUILD.gn | 1 + .../surfaceflinger_layers_extractor.cc | 17 ++- src/trace_processor/sqlite/BUILD.gn | 1 + src/trace_processor/sqlite/sql_source.cc | 9 +- src/trace_processor/util/BUILD.gn | 16 +++ src/trace_processor/util/cold_sort.cc | 29 +++++ src/trace_processor/util/cold_sort.h | 123 ++++++++++++++++++ .../util/cold_sort_unittest.cc | 90 +++++++++++++ src/trace_processor/util/symbolizer/BUILD.gn | 1 + .../util/symbolizer/symbolize_database.cc | 8 +- src/trace_processor/util/trace_type.cc | 6 +- 35 files changed, 393 insertions(+), 77 deletions(-) create mode 100644 src/trace_processor/util/cold_sort.cc create mode 100644 src/trace_processor/util/cold_sort.h create mode 100644 src/trace_processor/util/cold_sort_unittest.cc diff --git a/Android.bp b/Android.bp index 3ab8fcfb7d6..3cefce5f462 100644 --- a/Android.bp +++ b/Android.bp @@ -3265,6 +3265,7 @@ cc_test { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_descriptors", @@ -21665,6 +21666,7 @@ cc_library_static { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", @@ -22064,6 +22066,14 @@ filegroup { ], } +// GN: //src/trace_processor/util:cold_sort +filegroup { + name: "perfetto_src_trace_processor_util_cold_sort", + srcs: [ + "src/trace_processor/util/cold_sort.cc", + ], +} + // GN: //src/trace_processor/util:compressor filegroup { name: "perfetto_src_trace_processor_util_compressor", @@ -22347,6 +22357,7 @@ filegroup { name: "perfetto_src_trace_processor_util_unittests", srcs: [ "src/trace_processor/util/bump_allocator_unittest.cc", + "src/trace_processor/util/cold_sort_unittest.cc", "src/trace_processor/util/concurrency_stress_test_util_unittest.cc", "src/trace_processor/util/descriptors_unittest.cc", "src/trace_processor/util/flatbuffer_reader_unittest.cc", @@ -24739,6 +24750,7 @@ cc_test { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_concurrency_stress_test_util", ":perfetto_src_trace_processor_util_decompressor", @@ -25868,6 +25880,7 @@ cc_library_static { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_descriptors", @@ -26587,6 +26600,7 @@ cc_binary_host { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", diff --git a/BUILD b/BUILD index 5845c08bf9d..584c048e88d 100644 --- a/BUILD +++ b/BUILD @@ -724,6 +724,7 @@ perfetto_cc_library( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_simple_json_parser", ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + @@ -1068,6 +1069,7 @@ perfetto_cc_library( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_deobfuscation_deobfuscator", ":src_trace_processor_util_simple_json_parser", ":src_trace_processor_util_symbolizer_symbolize_database", @@ -6129,6 +6131,7 @@ perfetto_cc_library( ":protos_perfetto_trace_track_event_zero", ":protos_perfetto_trace_translation_zero", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_symbolizer_symbolizer", ], linkstatic = True, @@ -6225,6 +6228,25 @@ perfetto_filegroup( ], ) +# GN target: //src/trace_processor/util:cold_sort +perfetto_cc_library( + name = "src_trace_processor_util_cold_sort", + srcs = [ + "src/trace_processor/util/cold_sort.cc", + ], + hdrs = [ + ":include_perfetto_base_base", + ":include_perfetto_ext_base_base", + ":include_perfetto_public_abi_base", + ":include_perfetto_public_base", + "src/trace_processor/util/cold_sort.h", + ], + deps = [ + ":src_base_base", + ], + linkstatic = True, +) + # GN target: //src/trace_processor/util:compressor perfetto_filegroup( name = "src_trace_processor_util_compressor", @@ -12197,6 +12219,7 @@ perfetto_cc_library( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_simple_json_parser", ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + @@ -12543,6 +12566,7 @@ perfetto_cc_binary( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_deobfuscation_deobfuscator", ":src_trace_processor_util_simple_json_parser", ":src_trace_processor_util_symbolizer_symbolize_database", diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn index 79237bb49da..39dca8a0dd3 100644 --- a/src/trace_processor/BUILD.gn +++ b/src/trace_processor/BUILD.gn @@ -70,6 +70,7 @@ source_set("export_json") { "tables", "types", "util:args_utils", + "util:cold_sort", "util:json_value", ] public_deps = [ "../../include/perfetto/ext/trace_processor:export_json" ] diff --git a/src/trace_processor/core/dataframe/BUILD.gn b/src/trace_processor/core/dataframe/BUILD.gn index 590bf7fc74c..62c8cf5e5c8 100644 --- a/src/trace_processor/core/dataframe/BUILD.gn +++ b/src/trace_processor/core/dataframe/BUILD.gn @@ -46,6 +46,7 @@ source_set("dataframe") { "../../../base", "../../../base/regex", "../../containers", + "../../util:cold_sort", "../../util:flatbuffer_reader", "../../util:flatbuffer_writer", "../../util:glob", diff --git a/src/trace_processor/core/dataframe/logical_plan.cc b/src/trace_processor/core/dataframe/logical_plan.cc index d3c5b172b76..917403fdb5c 100644 --- a/src/trace_processor/core/dataframe/logical_plan.cc +++ b/src/trace_processor/core/dataframe/logical_plan.cc @@ -37,6 +37,7 @@ #include "src/trace_processor/core/common/storage_types.h" #include "src/trace_processor/core/dataframe/specs.h" #include "src/trace_processor/core/dataframe/types.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::core::dataframe { @@ -204,13 +205,9 @@ class Planner { base::Status Planner::Filter(std::vector& specs) { // Sort filters by efficiency (most selective/cheapest first) - std::stable_sort(specs.begin(), specs.end(), - [this](const FilterSpec& a, const FilterSpec& b) { - const auto& a_col = GetColumn(a.col); - const auto& b_col = GetColumn(b.col); - return FilterPreference(a, a_col) < - FilterPreference(b, b_col); - }); + ColdStableSortByKey(specs.begin(), specs.end(), [this](const FilterSpec& fs) { + return FilterPreference(fs, GetColumn(fs.col)); + }); std::vector specs_handled(specs.size(), false); diff --git a/src/trace_processor/core/exec/BUILD.gn b/src/trace_processor/core/exec/BUILD.gn index ea73aee06a4..80c66948819 100644 --- a/src/trace_processor/core/exec/BUILD.gn +++ b/src/trace_processor/core/exec/BUILD.gn @@ -65,6 +65,7 @@ source_set("exec") { "../../../base", "../../containers", "../../util:bump_allocator", + "../../util:cold_sort", "../common", "../dataframe", "../util", diff --git a/src/trace_processor/core/exec/interval_intersect.cc b/src/trace_processor/core/exec/interval_intersect.cc index bb3ae5663b5..0a7b94a86a8 100644 --- a/src/trace_processor/core/exec/interval_intersect.cc +++ b/src/trace_processor/core/exec/interval_intersect.cc @@ -44,6 +44,7 @@ #include "src/trace_processor/core/exec/row_store.h" #include "src/trace_processor/core/util/bit_vector.h" #include "src/trace_processor/core/util/span.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::core::exec { namespace { @@ -276,9 +277,8 @@ void NarrowKey(const std::vector& groups, auto count = static_cast(groups.size()); order.resize(count); std::iota(order.begin(), order.end(), 0u); - std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) { - return groups[a]->intervals.size() < groups[b]->intervals.size(); - }); + ColdSortByKey(order.begin(), order.end(), + [&](uint32_t i) { return groups[i]->intervals.size(); }); running.Clear(); for (const Interval& interval : groups[order.front()]->intervals) { diff --git a/src/trace_processor/export_json.cc b/src/trace_processor/export_json.cc index 210de3e4fa9..950aa7e13a8 100644 --- a/src/trace_processor/export_json.cc +++ b/src/trace_processor/export_json.cc @@ -54,6 +54,7 @@ #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/args_utils.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/json_value.h" namespace perfetto::trace_processor::json { @@ -212,13 +213,9 @@ class JsonExporter { // is emitted before its parent's end event, even if both end events have // the same timestamp. To accomplish this, we perform a stable sort in // descending order and later iterate via reverse iterators. - struct { - bool operator()(const Dom& a, const Dom& b) const { - return a["ts"].AsInt64() > b["ts"].AsInt64(); - } - } CompareEvents; - std::stable_sort(async_end_events_.begin(), async_end_events_.end(), - CompareEvents); + ColdStableSortByKeyDescending( + async_end_events_.begin(), async_end_events_.end(), + [](const Dom& event) { return event["ts"].AsInt64(); }); // Merge sort by timestamp. If events share the same timestamp, prefer // instant events, then end events, so that old slices close before new diff --git a/src/trace_processor/importers/android_bugreport/BUILD.gn b/src/trace_processor/importers/android_bugreport/BUILD.gn index 091a421f4dc..5848a5935f5 100644 --- a/src/trace_processor/importers/android_bugreport/BUILD.gn +++ b/src/trace_processor/importers/android_bugreport/BUILD.gn @@ -68,6 +68,7 @@ source_set("android_bugreport") { "../../storage", "../../tables:tables_python", "../../types", + "../../util:cold_sort", "../../util:trace_type", "../../util:zip_reader", "../common", diff --git a/src/trace_processor/importers/android_bugreport/android_log_reader.cc b/src/trace_processor/importers/android_bugreport/android_log_reader.cc index 22d5b481b08..348c2faeb5e 100644 --- a/src/trace_processor/importers/android_bugreport/android_log_reader.cc +++ b/src/trace_processor/importers/android_bugreport/android_log_reader.cc @@ -43,6 +43,7 @@ #include "src/trace_processor/sorter/trace_sorter.h" #include "src/trace_processor/storage/stats.h" #include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/trace_type.h" namespace perfetto::trace_processor { @@ -350,7 +351,9 @@ DedupingAndroidLogReader::DedupingAndroidLogReader( std::vector events) : AndroidLogReader(context, year, std::move(stream), wait_for_tz), events_(std::move(events)) { - std::sort(events_.begin(), events_.end()); + ColdSortByKey( + events_.begin(), events_.end(), + [](const TimestampedAndroidLogEvent& e) { return e.ts.count(); }); } DedupingAndroidLogReader::~DedupingAndroidLogReader() {} diff --git a/src/trace_processor/importers/memory_tracker/BUILD.gn b/src/trace_processor/importers/memory_tracker/BUILD.gn index 0fc30b806f2..62f36958c9e 100644 --- a/src/trace_processor/importers/memory_tracker/BUILD.gn +++ b/src/trace_processor/importers/memory_tracker/BUILD.gn @@ -16,7 +16,10 @@ import("../../../../gn/perfetto.gni") import("../../../../gn/test.gni") source_set("graph_processor") { - deps = [ "../../../../gn:default_deps" ] + deps = [ + "../../../../gn:default_deps", + "../../util:cold_sort", + ] public_deps = [ "../../../../include/perfetto/base", "../../../../include/perfetto/ext/base", diff --git a/src/trace_processor/importers/memory_tracker/graph_processor.cc b/src/trace_processor/importers/memory_tracker/graph_processor.cc index 77e40a9bb76..7afe2fef693 100644 --- a/src/trace_processor/importers/memory_tracker/graph_processor.cc +++ b/src/trace_processor/importers/memory_tracker/graph_processor.cc @@ -18,6 +18,8 @@ #include +#include "src/trace_processor/util/cold_sort.h" + namespace perfetto { namespace trace_processor { @@ -680,13 +682,11 @@ void GraphProcessor::CalculateNodeOwnershipCoefficient(Node* node) { // Sort the owners in decreasing order of ownership priority and // increasing order of not-owning sub-size (in case of equal priority). std::vector owners = *node->owned_by_edges(); - std::sort(owners.begin(), owners.end(), [](Edge* a, Edge* b) { - if (a->priority() == b->priority()) { - return a->source()->not_owning_sub_size() < - b->source()->not_owning_sub_size(); - } - return b->priority() < a->priority(); + ColdSortByKey(owners.begin(), owners.end(), [](Edge* edge) { + return edge->source()->not_owning_sub_size(); }); + ColdStableSortByKeyDescending(owners.begin(), owners.end(), + [](Edge* edge) { return edge->priority(); }); // Loop over the list of owners and distribute the owned node's not-owned // sub-size among them according to their ownership priority and diff --git a/src/trace_processor/importers/ninja/BUILD.gn b/src/trace_processor/importers/ninja/BUILD.gn index c1748900da8..7da1c8b9a0a 100644 --- a/src/trace_processor/importers/ninja/BUILD.gn +++ b/src/trace_processor/importers/ninja/BUILD.gn @@ -22,6 +22,7 @@ source_set("ninja") { "../../importers/common", "../../sorter", "../../storage", + "../../util:cold_sort", "../../util:trace_type", ] } diff --git a/src/trace_processor/importers/ninja/ninja_log_parser.cc b/src/trace_processor/importers/ninja/ninja_log_parser.cc index b60100a719d..83191adff83 100644 --- a/src/trace_processor/importers/ninja/ninja_log_parser.cc +++ b/src/trace_processor/importers/ninja/ninja_log_parser.cc @@ -35,6 +35,7 @@ #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/storage/stats.h" #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/trace_type.h" namespace perfetto::trace_processor { @@ -126,8 +127,8 @@ base::Status NinjaLogParser::Parse(TraceBlobView blob) { // been populated. base::Status NinjaLogParser::OnPushDataToSorter() { // Phase 1: Sort jobs and write slices directly to storage - std::sort(jobs_.begin(), jobs_.end(), - [](const Job& x, const Job& y) { return x.start_ms < y.start_ms; }); + ColdSortByKey(jobs_.begin(), jobs_.end(), + [](const Job& job) { return job.start_ms; }); // Now we need to work out the job parallelism. There's no direct indication // of that in the ninja logs, so it must be inferred by observing overlapping diff --git a/src/trace_processor/importers/perf/BUILD.gn b/src/trace_processor/importers/perf/BUILD.gn index 4470fc85006..eddd12a7eb0 100644 --- a/src/trace_processor/importers/perf/BUILD.gn +++ b/src/trace_processor/importers/perf/BUILD.gn @@ -92,6 +92,7 @@ source_set("perf") { "../../tables:tables_python", "../../types", "../../util:build_id", + "../../util:cold_sort", "../../util:trace_blob_view_reader", "../../util:trace_type", "../common", diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.cc b/src/trace_processor/importers/perf/perf_data_tokenizer.cc index 5035c09875b..878bab1cc3e 100644 --- a/src/trace_processor/importers/perf/perf_data_tokenizer.cc +++ b/src/trace_processor/importers/perf/perf_data_tokenizer.cc @@ -68,6 +68,7 @@ #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/build_id.h" #include "src/trace_processor/util/clock_synchronizer.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/trace_blob_view_reader.h" #include "src/trace_processor/util/trace_type.h" @@ -437,17 +438,19 @@ PerfDataTokenizer::ParseFeatureSections() { PERFETTO_CHECK(reader.Read(feature_sections_.back().second)); } - std::sort(feature_sections_.begin(), feature_sections_.end(), - [](const std::pair& lhs, - const std::pair& rhs) { - if (lhs.second.offset == rhs.second.offset) { - // Some sections have 0 length and thus there can be offset - // collisions. To make sure we parse sections by increasing - // offset parse empty sections first. - return lhs.second.size > rhs.second.size; - } - return lhs.second.offset > rhs.second.offset; - }); + // Sorts by offset, then size, both descending: a stable sort by size, then + // by offset. Some sections have 0 length and thus there can be offset + // collisions. To make sure we parse sections by increasing offset parse + // empty sections first. + ColdSortByKeyDescending(feature_sections_.begin(), feature_sections_.end(), + [](const std::pair& s) { + return s.second.size; + }); + ColdStableSortByKeyDescending( + feature_sections_.begin(), feature_sections_.end(), + [](const std::pair& s) { + return s.second.offset; + }); buffer_.PopFrontUntil(feature_headers_section_.end()); parsing_state_ = feature_sections_.empty() ? ParsingState::kDone diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn index 48e1a55aebc..f281757e7b5 100644 --- a/src/trace_processor/importers/proto/BUILD.gn +++ b/src/trace_processor/importers/proto/BUILD.gn @@ -123,6 +123,7 @@ source_set("minimal") { "../../tables", "../../types", "../../util:build_id", + "../../util:cold_sort", "../../util:decompressor", "../../util:json_args", "../../util:json_parser", @@ -251,6 +252,7 @@ source_set("full") { "../../tables", "../../types", "../../util:build_id", + "../../util:cold_sort", "../../util:descriptors", "../../util:profiler_util", "../../util:proto_profiler", diff --git a/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc b/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc index a9193409faf..7395ab0fc80 100644 --- a/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc +++ b/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc @@ -36,6 +36,7 @@ #include "src/trace_processor/importers/proto/v8_module.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/tables/android_tables_py.h" +#include "src/trace_processor/util/cold_sort.h" #include "protos/perfetto/trace/android/cpu_per_uid_data.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" @@ -179,8 +180,8 @@ void AndroidCpuPerUidModule::OnEventsFullyExtracted() { rows.push_back(row); } - std::sort(rows.begin(), rows.end(), - [](auto& a, auto& b) { return a.track_id < b.track_id; }); + ColdSortByKey(rows.begin(), rows.end(), + [](const auto& row) { return row.track_id.value; }); for (const auto& row : rows) { context_->storage->mutable_android_cpu_per_uid_track_table()->Insert(row); diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc index 28874e770d6..1b8f59b2b31 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc @@ -16,7 +16,6 @@ #include "src/trace_processor/importers/proto/proto_trace_parser_impl.h" -#include #include #include #include @@ -55,6 +54,7 @@ #include "src/trace_processor/tables/metadata_tables_py.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" +#include "src/trace_processor/util/cold_sort.h" #include "protos/perfetto/config/trace_config.pbzero.h" #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h" @@ -290,10 +290,8 @@ void ProtoTraceParserImpl::ParseMetatraceEvent(int64_t ts, ConstBytes blob) { // We stable sort instead of sorting here to avoid changing the order of the // args in arrays. - std::stable_sort(interned.begin(), interned.end(), - [](const Arg& a, const Arg& b) { - return a.first.raw_id() < b.first.raw_id(); - }); + ColdStableSortByKey(interned.begin(), interned.end(), + [](const Arg& arg) { return arg.first.raw_id(); }); // Compute the correct key for each arg, possibly adding an index to // the end of the key if needed. diff --git a/src/trace_processor/importers/proto/system_probes_parser.cc b/src/trace_processor/importers/proto/system_probes_parser.cc index e318eed8224..8bd3d044d85 100644 --- a/src/trace_processor/importers/proto/system_probes_parser.cc +++ b/src/trace_processor/importers/proto/system_probes_parser.cc @@ -60,6 +60,7 @@ #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/clock_synchronizer.h" +#include "src/trace_processor/util/cold_sort.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" #include "protos/perfetto/common/system_info.pbzero.h" @@ -1189,8 +1190,9 @@ void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { uint32_t cluster_id = 0; if (valid_capacities) { - std::sort(cpu_infos.begin(), cpu_infos.end(), - [](auto a, auto b) { return a.capacity < b.capacity; }); + ColdSortByKey(cpu_infos.begin(), cpu_infos.end(), [](const CpuInfo& info) { + return info.capacity ? uint64_t{*info.capacity} + 1 : 0; + }); uint32_t previous_capacity = *cpu_infos[0].capacity; for (CpuInfo& cpu_info : cpu_infos) { uint32_t capacity = *cpu_info.capacity; @@ -1210,8 +1212,8 @@ void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { {info.cpu, *std::max_element(info.frequencies.begin(), info.frequencies.end())}); } - std::sort(cpu_max_freqs.begin(), cpu_max_freqs.end(), - [](auto a, auto b) { return a.max_frequency < b.max_frequency; }); + ColdSortByKey(cpu_max_freqs.begin(), cpu_max_freqs.end(), + [](const CpuMaxFrequency& f) { return f.max_frequency; }); uint32_t previous_max_freq = cpu_max_freqs[0].max_frequency; for (CpuMaxFrequency& cpu_max_freq : cpu_max_freqs) { diff --git a/src/trace_processor/perfetto_sql/parser/BUILD.gn b/src/trace_processor/perfetto_sql/parser/BUILD.gn index 5dc482727a0..caef3381b9c 100644 --- a/src/trace_processor/perfetto_sql/parser/BUILD.gn +++ b/src/trace_processor/perfetto_sql/parser/BUILD.gn @@ -31,6 +31,7 @@ source_set("parser") { "../../../perfetto_sql:intrinsic_macro_expansion", "../../../perfetto_sql/syntaqlite", "../../sqlite", + "../../util:cold_sort", "../../util:sql_argument", "../pipeline", "../pipeline:plan", diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc index 94a1367c624..45856438156 100644 --- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc +++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc @@ -40,6 +40,7 @@ #include "src/trace_processor/perfetto_sql/pipeline/compiler.h" #include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" #include "src/trace_processor/sqlite/sql_source.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/sql_argument.h" namespace perfetto::trace_processor { @@ -247,10 +248,8 @@ class RewriteTree { // This is the only ApplyRewrites caller whose items aren't already in // `start` order: we concatenated literal body calls and $param segments, // each internally sorted but interleaved relative to each other. - std::sort(items.begin(), items.end(), - [](const RewriteItem& a, const RewriteItem& b) { - return a.start < b.start; - }); + ColdSortByKey(items.begin(), items.end(), + [](const RewriteItem& item) { return item.start; }); return ApplyRewrites(m.sql, std::move(items)); } diff --git a/src/trace_processor/plugins/interval_intersect/BUILD.gn b/src/trace_processor/plugins/interval_intersect/BUILD.gn index b08c60325b0..c1a96776d60 100644 --- a/src/trace_processor/plugins/interval_intersect/BUILD.gn +++ b/src/trace_processor/plugins/interval_intersect/BUILD.gn @@ -34,5 +34,6 @@ source_set("interval_intersect") { "../../sqlite", "../../storage", "../../types", + "../../util:cold_sort", ] } diff --git a/src/trace_processor/plugins/interval_intersect/interval_intersect.cc b/src/trace_processor/plugins/interval_intersect/interval_intersect.cc index b6f1c6d7683..98c0adc4c9d 100644 --- a/src/trace_processor/plugins/interval_intersect/interval_intersect.cc +++ b/src/trace_processor/plugins/interval_intersect/interval_intersect.cc @@ -53,6 +53,7 @@ #include "src/trace_processor/sqlite/sqlite_utils.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::interval_intersect { namespace { @@ -164,12 +165,10 @@ base::StatusOr PushPartition( // Sort `tables_order` from the smallest to the biggest. std::vector tables_order(tables_count); std::iota(tables_order.begin(), tables_order.end(), 0); - std::sort( - tables_order.begin(), tables_order.end(), - [&intervals_in_table](const uint32_t idx_a, const uint32_t idx_b) { - return intervals_in_table[idx_a]->intervals.size() < - intervals_in_table[idx_b]->intervals.size(); - }); + ColdSortByKey(tables_order.begin(), tables_order.end(), + [&intervals_in_table](uint32_t idx) { + return intervals_in_table[idx]->intervals.size(); + }); uint32_t idx_of_smallest_part = tables_order.front(); PERFETTO_DCHECK( !intervals_in_table[idx_of_smallest_part]->intervals.empty()); diff --git a/src/trace_processor/plugins/winscope_importer/BUILD.gn b/src/trace_processor/plugins/winscope_importer/BUILD.gn index eb48d873b91..d4df29756f5 100644 --- a/src/trace_processor/plugins/winscope_importer/BUILD.gn +++ b/src/trace_processor/plugins/winscope_importer/BUILD.gn @@ -95,6 +95,7 @@ source_set("winscope_importer") { "../../storage", "../../tables", "../../types", + "../../util:cold_sort", ] } diff --git a/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc b/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc index 3c7eeeadf25..5afcfe23e11 100644 --- a/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc +++ b/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc @@ -19,6 +19,7 @@ #include #include #include "src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_utils.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::winscope::surfaceflinger_layers { @@ -29,13 +30,15 @@ namespace { void SortByZThenLayerId( std::vector& layer_ids, const std::unordered_map& layers_by_id) { - std::sort(layer_ids.begin(), layer_ids.end(), - [&layers_by_id](int32_t a, int32_t b) { - const auto& layer_a = layers_by_id.at(a); - const auto& layer_b = layers_by_id.at(b); - return std::make_tuple(layer_a.z(), layer_a.id()) < - std::make_tuple(layer_b.z(), layer_b.id()); - }); + // (z, id) packed into one key, each with its sign bit flipped to order + // negative values first. + ColdSortByKey(layer_ids.begin(), layer_ids.end(), + [&layers_by_id](int32_t layer_id) { + const auto& layer = layers_by_id.at(layer_id); + auto z = static_cast(layer.z()) ^ 0x80000000u; + auto id = static_cast(layer.id()) ^ 0x80000000u; + return (uint64_t{z} << 32) | id; + }); } // Extract layers bottom-to-top according to layer drawing order from diff --git a/src/trace_processor/sqlite/BUILD.gn b/src/trace_processor/sqlite/BUILD.gn index 8022b269701..df16cf0f8d1 100644 --- a/src/trace_processor/sqlite/BUILD.gn +++ b/src/trace_processor/sqlite/BUILD.gn @@ -51,6 +51,7 @@ source_set("sqlite") { "../importers/ftrace:ftrace_descriptors", "../storage", "../types", + "../util:cold_sort", "../util:profile_builder", ] public_deps = [ "bindings" ] diff --git a/src/trace_processor/sqlite/sql_source.cc b/src/trace_processor/sqlite/sql_source.cc index 9d489d2c594..552bf4a8d41 100644 --- a/src/trace_processor/sqlite/sql_source.cc +++ b/src/trace_processor/sqlite/sql_source.cc @@ -15,6 +15,7 @@ */ #include "src/trace_processor/sqlite/sql_source.h" +#include "src/trace_processor/util/cold_sort.h" #include #include @@ -370,10 +371,10 @@ SqlSource SqlSource::Rewriter::Build() && { // Phase 2: sort the new rewrite vector by original offset and verify that the // original offsets are monotonic and non-overlapping. - std::sort(all_rewrites.begin(), all_rewrites.end(), - [](const SqlSource::Rewrite& a, const SqlSource::Rewrite& b) { - return a.original_sql_start < b.original_sql_start; - }); + ColdSortByKey(all_rewrites.begin(), all_rewrites.end(), + [](const SqlSource::Rewrite& rewrite) { + return rewrite.original_sql_start; + }); for (uint32_t i = 1; i < all_rewrites.size(); ++i) { PERFETTO_CHECK(all_rewrites[i - 1].original_sql_end <= all_rewrites[i].original_sql_start); diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn index 141c5e1d5c0..17b73c110cb 100644 --- a/src/trace_processor/util/BUILD.gn +++ b/src/trace_processor/util/BUILD.gn @@ -249,6 +249,19 @@ source_set("zip_reader") { } } +perfetto_component("cold_sort") { + sources = [ + "cold_sort.cc", + "cold_sort.h", + ] + deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/base", + "../../../include/perfetto/ext/base", + "../../base", + ] +} + source_set("galloping_search") { sources = [ "galloping_search.h" ] deps = [ "../../../gn:default_deps" ] @@ -447,6 +460,7 @@ source_set("trace_type") { "trace_type.h", ] deps = [ + ":cold_sort", ":decompressor", "../../../gn:default_deps", "../../../include/perfetto/ext/base", @@ -492,6 +506,7 @@ source_set("flatbuffer_reader") { source_set("unittests") { sources = [ "bump_allocator_unittest.cc", + "cold_sort_unittest.cc", "concurrency_stress_test_util_unittest.cc", "descriptors_unittest.cc", "flatbuffer_reader_unittest.cc", @@ -516,6 +531,7 @@ source_set("unittests") { testonly = true deps = [ ":bump_allocator", + ":cold_sort", ":concurrency_stress_test_util", ":decompressor", ":descriptors", diff --git a/src/trace_processor/util/cold_sort.cc b/src/trace_processor/util/cold_sort.cc new file mode 100644 index 00000000000..48635b6ad19 --- /dev/null +++ b/src/trace_processor/util/cold_sort.cc @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/cold_sort.h" + +#include + +namespace perfetto::trace_processor::cold_sort_internal { + +void SortKeyedIndices(KeyedIndex* begin, KeyedIndex* end) { + std::sort(begin, end, [](const KeyedIndex& a, const KeyedIndex& b) { + return a.key < b.key || (a.key == b.key && a.index < b.index); + }); +} + +} // namespace perfetto::trace_processor::cold_sort_internal diff --git a/src/trace_processor/util/cold_sort.h b/src/trace_processor/util/cold_sort.h new file mode 100644 index 00000000000..7e062bc8ab7 --- /dev/null +++ b/src/trace_processor/util/cold_sort.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_COLD_SORT_H_ +#define SRC_TRACE_PROCESSOR_UTIL_COLD_SORT_H_ + +#include +#include +#include +#include +#include + +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/small_vector.h" + +// Replacements for std::sort() and std::stable_sort() for code which is not +// hot. Those are compiled for every element type and comparator, at 3-4KB +// each. These sort by an integer key through one sort compiled once, so each +// caller adds only a few hundred bytes. They are up to 2x slower, and need 16 +// bytes per element of extra memory, only heap allocated above 64 elements. + +namespace perfetto::trace_processor { +namespace cold_sort_internal { + +struct KeyedIndex { + uint64_t key; + uint32_t index; +}; +using KeyedIndices = base::SmallVector; + +// Sorts by key, then index. +void SortKeyedIndices(KeyedIndex* begin, KeyedIndex* end); + +// Maps `key` to a uint64_t with the same order. +template +uint64_t ToOrderedKey(K key) { + static_assert(std::is_integral_v, "Cold sorts need integer keys"); + if constexpr (std::is_signed_v) { + return static_cast(static_cast(key)) ^ (1ull << 63); + } else { + return static_cast(key); + } +} + +// Sorts [begin, end) by key, ties in their original order. +template +void SortByKey(It begin, It end, Key key) { + using T = typename std::iterator_traits::value_type; + PERFETTO_DCHECK(end - begin <= std::numeric_limits::max()); + auto size = static_cast(end - begin); + if (size < 2) { + return; + } + KeyedIndices keys; + for (uint32_t i = 0; i < size; ++i) { + keys.emplace_back(KeyedIndex{ToOrderedKey(key(begin[i])), i}); + } + SortKeyedIndices(keys.data(), keys.data() + size); + // Move each element to its sorted position by following the permutation's + // cycles, so only one element is ever held outside the range. + for (uint32_t i = 0; i < size; ++i) { + if (keys[i].index == i) { + continue; + } + T held = std::move(begin[i]); + uint32_t j = i; + for (uint32_t k = keys[j].index; k != i; k = keys[j].index) { + begin[j] = std::move(begin[k]); + keys[j].index = j; + j = k; + } + begin[j] = std::move(held); + keys[j].index = j; + } +} + +} // namespace cold_sort_internal + +// Sorts [begin, end) by the integer `key(element)`, ascending. Elements with +// equal keys end up in an unspecified order. +template +void ColdSortByKey(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, std::move(key)); +} + +// As ColdSortByKey(), but descending. +template +void ColdSortByKeyDescending(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, [&key](const auto& element) { + return ~cold_sort_internal::ToOrderedKey(key(element)); + }); +} + +// As ColdSortByKey(), but elements with equal keys keep their order. +template +void ColdStableSortByKey(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, std::move(key)); +} + +// As ColdStableSortByKey(), but descending. +template +void ColdStableSortByKeyDescending(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, [&key](const auto& element) { + return ~cold_sort_internal::ToOrderedKey(key(element)); + }); +} + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_UTIL_COLD_SORT_H_ diff --git a/src/trace_processor/util/cold_sort_unittest.cc b/src/trace_processor/util/cold_sort_unittest.cc new file mode 100644 index 00000000000..52e46ea75a5 --- /dev/null +++ b/src/trace_processor/util/cold_sort_unittest.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/cold_sort.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test/gtest_and_gmock.h" + +namespace perfetto::trace_processor { +namespace { + +TEST(ColdSortTest, MatchesStdStableSort) { + std::minstd_rand0 rnd(0); + for (uint32_t size : {0u, 1u, 2u, 3u, 16u, 17u, 64u, 65u, 1000u, 100000u}) { + std::vector> data; + for (uint32_t i = 0; i < size; ++i) { + data.emplace_back(static_cast(rnd() % 50) - 25, + std::to_string(i)); + } + auto expected = data; + std::stable_sort( + expected.begin(), expected.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + ColdStableSortByKey(data.begin(), data.end(), + [](const auto& x) { return x.first; }); + ASSERT_EQ(data, expected); + } +} + +TEST(ColdSortTest, KeyExtremes) { + std::vector data = {0, -1, std::numeric_limits::max(), + std::numeric_limits::min(), 1}; + ColdSortByKey(data.begin(), data.end(), [](int64_t x) { return x; }); + ASSERT_EQ(data, + (std::vector{std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()})); +} + +TEST(ColdSortTest, Descending) { + std::vector> data = { + {2, 0}, + {-5, 1}, + {2, 2}, + {std::numeric_limits::min(), 3}, + {7, 4}}; + ColdStableSortByKeyDescending(data.begin(), data.end(), + [](const auto& x) { return x.first; }); + ASSERT_EQ(data, (std::vector>{ + {7, 4}, + {2, 0}, + {2, 2}, + {-5, 1}, + {std::numeric_limits::min(), 3}})); +} + +TEST(ColdSortTest, MoveOnly) { + std::vector> data; + for (int v : {3, 1, 2}) { + data.push_back(std::make_unique(v)); + } + ColdSortByKey(data.begin(), data.end(), + [](const std::unique_ptr& p) { return *p; }); + ASSERT_EQ(*data[0], 1); + ASSERT_EQ(*data[1], 2); + ASSERT_EQ(*data[2], 3); +} + +} // namespace +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/util/symbolizer/BUILD.gn b/src/trace_processor/util/symbolizer/BUILD.gn index 0e016dc8277..d7b4ad9b9be 100644 --- a/src/trace_processor/util/symbolizer/BUILD.gn +++ b/src/trace_processor/util/symbolizer/BUILD.gn @@ -48,6 +48,7 @@ if (enable_perfetto_trace_processor) { ] deps = [ "../:build_id", + "../:cold_sort", "../../../../gn:default_deps", "../../../../include/perfetto/protozero", "../../../../include/perfetto/trace_processor", diff --git a/src/trace_processor/util/symbolizer/symbolize_database.cc b/src/trace_processor/util/symbolizer/symbolize_database.cc index f8f67491701..8a577e8fc6d 100644 --- a/src/trace_processor/util/symbolizer/symbolize_database.cc +++ b/src/trace_processor/util/symbolizer/symbolize_database.cc @@ -41,6 +41,7 @@ #include "perfetto/trace_processor/iterator.h" #include "perfetto/trace_processor/trace_processor.h" #include "src/trace_processor/util/build_id.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/symbolizer/breakpad_symbolizer.h" #include "src/trace_processor/util/symbolizer/local_symbolizer.h" #include "src/trace_processor/util/symbolizer/symbolizer.h" @@ -52,6 +53,7 @@ namespace perfetto::profiling { namespace { +using trace_processor::ColdStableSortByKeyDescending; using trace_processor::Iterator; // Runtime mapping start/end addresses do not affect symbolization. Group by @@ -462,11 +464,9 @@ std::vector GroupUnresolved(const SymbolizerResult& result) { } // Largest first, so the short report names the mappings that matter most. for (auto& group : groups) { - std::stable_sort( + ColdStableSortByKeyDescending( group.mappings.begin(), group.mappings.end(), - [](const UnresolvedMapping& a, const UnresolvedMapping& b) { - return a.frame_count > b.frame_count; - }); + [](const UnresolvedMapping& m) { return m.frame_count; }); } return groups; } diff --git a/src/trace_processor/util/trace_type.cc b/src/trace_processor/util/trace_type.cc index 0fdca4f85ef..84a77999efe 100644 --- a/src/trace_processor/util/trace_type.cc +++ b/src/trace_processor/util/trace_type.cc @@ -15,6 +15,7 @@ */ #include "src/trace_processor/util/trace_type.h" +#include "src/trace_processor/util/cold_sort.h" #include #include @@ -81,9 +82,8 @@ TraceImporterId TraceImporterRegistry::Guess(const uint8_t* data, for (auto it = importers_.GetIterator(); it; ++it) { entries.push_back({it.key(), it.value().get()}); } - std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) { - return a.importer->descriptor().detection_priority < - b.importer->descriptor().detection_priority; + ColdSortByKey(entries.begin(), entries.end(), [](const Entry& e) { + return e.importer->descriptor().detection_priority; }); for (const Entry& e : entries) { if (e.importer->Sniff(data, size)) {