diff --git a/Android.bp b/Android.bp index 3ab8fcfb7d6..3cefce5f462 100644 --- a/Android.bp +++ b/Android.bp @@ -3265,6 +3265,7 @@ cc_test { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_descriptors", @@ -21665,6 +21666,7 @@ cc_library_static { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", @@ -22064,6 +22066,14 @@ filegroup { ], } +// GN: //src/trace_processor/util:cold_sort +filegroup { + name: "perfetto_src_trace_processor_util_cold_sort", + srcs: [ + "src/trace_processor/util/cold_sort.cc", + ], +} + // GN: //src/trace_processor/util:compressor filegroup { name: "perfetto_src_trace_processor_util_compressor", @@ -22347,6 +22357,7 @@ filegroup { name: "perfetto_src_trace_processor_util_unittests", srcs: [ "src/trace_processor/util/bump_allocator_unittest.cc", + "src/trace_processor/util/cold_sort_unittest.cc", "src/trace_processor/util/concurrency_stress_test_util_unittest.cc", "src/trace_processor/util/descriptors_unittest.cc", "src/trace_processor/util/flatbuffer_reader_unittest.cc", @@ -24739,6 +24750,7 @@ cc_test { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_concurrency_stress_test_util", ":perfetto_src_trace_processor_util_decompressor", @@ -25868,6 +25880,7 @@ cc_library_static { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_descriptors", @@ -26587,6 +26600,7 @@ cc_binary_host { ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_cold_sort", ":perfetto_src_trace_processor_util_compressor", ":perfetto_src_trace_processor_util_decompressor", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", diff --git a/BUILD b/BUILD index 5845c08bf9d..584c048e88d 100644 --- a/BUILD +++ b/BUILD @@ -724,6 +724,7 @@ perfetto_cc_library( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_simple_json_parser", ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + @@ -1068,6 +1069,7 @@ perfetto_cc_library( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_deobfuscation_deobfuscator", ":src_trace_processor_util_simple_json_parser", ":src_trace_processor_util_symbolizer_symbolize_database", @@ -6129,6 +6131,7 @@ perfetto_cc_library( ":protos_perfetto_trace_track_event_zero", ":protos_perfetto_trace_translation_zero", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_symbolizer_symbolizer", ], linkstatic = True, @@ -6225,6 +6228,25 @@ perfetto_filegroup( ], ) +# GN target: //src/trace_processor/util:cold_sort +perfetto_cc_library( + name = "src_trace_processor_util_cold_sort", + srcs = [ + "src/trace_processor/util/cold_sort.cc", + ], + hdrs = [ + ":include_perfetto_base_base", + ":include_perfetto_ext_base_base", + ":include_perfetto_public_abi_base", + ":include_perfetto_public_base", + "src/trace_processor/util/cold_sort.h", + ], + deps = [ + ":src_base_base", + ], + linkstatic = True, +) + # GN target: //src/trace_processor/util:compressor perfetto_filegroup( name = "src_trace_processor_util_compressor", @@ -12197,6 +12219,7 @@ perfetto_cc_library( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_simple_json_parser", ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + @@ -12543,6 +12566,7 @@ perfetto_cc_binary( ":src_trace_processor_plugins_wattson_gen_tpu_curves", ":src_trace_processor_trace_summary_gen_cc_trace_summary_descriptor", ":src_trace_processor_util_build_id", + ":src_trace_processor_util_cold_sort", ":src_trace_processor_util_deobfuscation_deobfuscator", ":src_trace_processor_util_simple_json_parser", ":src_trace_processor_util_symbolizer_symbolize_database", diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn index 79237bb49da..39dca8a0dd3 100644 --- a/src/trace_processor/BUILD.gn +++ b/src/trace_processor/BUILD.gn @@ -70,6 +70,7 @@ source_set("export_json") { "tables", "types", "util:args_utils", + "util:cold_sort", "util:json_value", ] public_deps = [ "../../include/perfetto/ext/trace_processor:export_json" ] diff --git a/src/trace_processor/core/dataframe/BUILD.gn b/src/trace_processor/core/dataframe/BUILD.gn index 590bf7fc74c..62c8cf5e5c8 100644 --- a/src/trace_processor/core/dataframe/BUILD.gn +++ b/src/trace_processor/core/dataframe/BUILD.gn @@ -46,6 +46,7 @@ source_set("dataframe") { "../../../base", "../../../base/regex", "../../containers", + "../../util:cold_sort", "../../util:flatbuffer_reader", "../../util:flatbuffer_writer", "../../util:glob", diff --git a/src/trace_processor/core/dataframe/logical_plan.cc b/src/trace_processor/core/dataframe/logical_plan.cc index d3c5b172b76..917403fdb5c 100644 --- a/src/trace_processor/core/dataframe/logical_plan.cc +++ b/src/trace_processor/core/dataframe/logical_plan.cc @@ -37,6 +37,7 @@ #include "src/trace_processor/core/common/storage_types.h" #include "src/trace_processor/core/dataframe/specs.h" #include "src/trace_processor/core/dataframe/types.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::core::dataframe { @@ -204,13 +205,9 @@ class Planner { base::Status Planner::Filter(std::vector& specs) { // Sort filters by efficiency (most selective/cheapest first) - std::stable_sort(specs.begin(), specs.end(), - [this](const FilterSpec& a, const FilterSpec& b) { - const auto& a_col = GetColumn(a.col); - const auto& b_col = GetColumn(b.col); - return FilterPreference(a, a_col) < - FilterPreference(b, b_col); - }); + ColdStableSortByKey(specs.begin(), specs.end(), [this](const FilterSpec& fs) { + return FilterPreference(fs, GetColumn(fs.col)); + }); std::vector specs_handled(specs.size(), false); diff --git a/src/trace_processor/core/exec/BUILD.gn b/src/trace_processor/core/exec/BUILD.gn index ea73aee06a4..80c66948819 100644 --- a/src/trace_processor/core/exec/BUILD.gn +++ b/src/trace_processor/core/exec/BUILD.gn @@ -65,6 +65,7 @@ source_set("exec") { "../../../base", "../../containers", "../../util:bump_allocator", + "../../util:cold_sort", "../common", "../dataframe", "../util", diff --git a/src/trace_processor/core/exec/interval_intersect.cc b/src/trace_processor/core/exec/interval_intersect.cc index bb3ae5663b5..0a7b94a86a8 100644 --- a/src/trace_processor/core/exec/interval_intersect.cc +++ b/src/trace_processor/core/exec/interval_intersect.cc @@ -44,6 +44,7 @@ #include "src/trace_processor/core/exec/row_store.h" #include "src/trace_processor/core/util/bit_vector.h" #include "src/trace_processor/core/util/span.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::core::exec { namespace { @@ -276,9 +277,8 @@ void NarrowKey(const std::vector& groups, auto count = static_cast(groups.size()); order.resize(count); std::iota(order.begin(), order.end(), 0u); - std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) { - return groups[a]->intervals.size() < groups[b]->intervals.size(); - }); + ColdSortByKey(order.begin(), order.end(), + [&](uint32_t i) { return groups[i]->intervals.size(); }); running.Clear(); for (const Interval& interval : groups[order.front()]->intervals) { diff --git a/src/trace_processor/export_json.cc b/src/trace_processor/export_json.cc index 210de3e4fa9..950aa7e13a8 100644 --- a/src/trace_processor/export_json.cc +++ b/src/trace_processor/export_json.cc @@ -54,6 +54,7 @@ #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/args_utils.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/json_value.h" namespace perfetto::trace_processor::json { @@ -212,13 +213,9 @@ class JsonExporter { // is emitted before its parent's end event, even if both end events have // the same timestamp. To accomplish this, we perform a stable sort in // descending order and later iterate via reverse iterators. - struct { - bool operator()(const Dom& a, const Dom& b) const { - return a["ts"].AsInt64() > b["ts"].AsInt64(); - } - } CompareEvents; - std::stable_sort(async_end_events_.begin(), async_end_events_.end(), - CompareEvents); + ColdStableSortByKeyDescending( + async_end_events_.begin(), async_end_events_.end(), + [](const Dom& event) { return event["ts"].AsInt64(); }); // Merge sort by timestamp. If events share the same timestamp, prefer // instant events, then end events, so that old slices close before new diff --git a/src/trace_processor/importers/android_bugreport/BUILD.gn b/src/trace_processor/importers/android_bugreport/BUILD.gn index 091a421f4dc..5848a5935f5 100644 --- a/src/trace_processor/importers/android_bugreport/BUILD.gn +++ b/src/trace_processor/importers/android_bugreport/BUILD.gn @@ -68,6 +68,7 @@ source_set("android_bugreport") { "../../storage", "../../tables:tables_python", "../../types", + "../../util:cold_sort", "../../util:trace_type", "../../util:zip_reader", "../common", diff --git a/src/trace_processor/importers/android_bugreport/android_log_reader.cc b/src/trace_processor/importers/android_bugreport/android_log_reader.cc index 22d5b481b08..348c2faeb5e 100644 --- a/src/trace_processor/importers/android_bugreport/android_log_reader.cc +++ b/src/trace_processor/importers/android_bugreport/android_log_reader.cc @@ -43,6 +43,7 @@ #include "src/trace_processor/sorter/trace_sorter.h" #include "src/trace_processor/storage/stats.h" #include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/trace_type.h" namespace perfetto::trace_processor { @@ -350,7 +351,9 @@ DedupingAndroidLogReader::DedupingAndroidLogReader( std::vector events) : AndroidLogReader(context, year, std::move(stream), wait_for_tz), events_(std::move(events)) { - std::sort(events_.begin(), events_.end()); + ColdSortByKey( + events_.begin(), events_.end(), + [](const TimestampedAndroidLogEvent& e) { return e.ts.count(); }); } DedupingAndroidLogReader::~DedupingAndroidLogReader() {} diff --git a/src/trace_processor/importers/memory_tracker/BUILD.gn b/src/trace_processor/importers/memory_tracker/BUILD.gn index 0fc30b806f2..62f36958c9e 100644 --- a/src/trace_processor/importers/memory_tracker/BUILD.gn +++ b/src/trace_processor/importers/memory_tracker/BUILD.gn @@ -16,7 +16,10 @@ import("../../../../gn/perfetto.gni") import("../../../../gn/test.gni") source_set("graph_processor") { - deps = [ "../../../../gn:default_deps" ] + deps = [ + "../../../../gn:default_deps", + "../../util:cold_sort", + ] public_deps = [ "../../../../include/perfetto/base", "../../../../include/perfetto/ext/base", diff --git a/src/trace_processor/importers/memory_tracker/graph_processor.cc b/src/trace_processor/importers/memory_tracker/graph_processor.cc index 77e40a9bb76..7afe2fef693 100644 --- a/src/trace_processor/importers/memory_tracker/graph_processor.cc +++ b/src/trace_processor/importers/memory_tracker/graph_processor.cc @@ -18,6 +18,8 @@ #include +#include "src/trace_processor/util/cold_sort.h" + namespace perfetto { namespace trace_processor { @@ -680,13 +682,11 @@ void GraphProcessor::CalculateNodeOwnershipCoefficient(Node* node) { // Sort the owners in decreasing order of ownership priority and // increasing order of not-owning sub-size (in case of equal priority). std::vector owners = *node->owned_by_edges(); - std::sort(owners.begin(), owners.end(), [](Edge* a, Edge* b) { - if (a->priority() == b->priority()) { - return a->source()->not_owning_sub_size() < - b->source()->not_owning_sub_size(); - } - return b->priority() < a->priority(); + ColdSortByKey(owners.begin(), owners.end(), [](Edge* edge) { + return edge->source()->not_owning_sub_size(); }); + ColdStableSortByKeyDescending(owners.begin(), owners.end(), + [](Edge* edge) { return edge->priority(); }); // Loop over the list of owners and distribute the owned node's not-owned // sub-size among them according to their ownership priority and diff --git a/src/trace_processor/importers/ninja/BUILD.gn b/src/trace_processor/importers/ninja/BUILD.gn index c1748900da8..7da1c8b9a0a 100644 --- a/src/trace_processor/importers/ninja/BUILD.gn +++ b/src/trace_processor/importers/ninja/BUILD.gn @@ -22,6 +22,7 @@ source_set("ninja") { "../../importers/common", "../../sorter", "../../storage", + "../../util:cold_sort", "../../util:trace_type", ] } diff --git a/src/trace_processor/importers/ninja/ninja_log_parser.cc b/src/trace_processor/importers/ninja/ninja_log_parser.cc index b60100a719d..83191adff83 100644 --- a/src/trace_processor/importers/ninja/ninja_log_parser.cc +++ b/src/trace_processor/importers/ninja/ninja_log_parser.cc @@ -35,6 +35,7 @@ #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/storage/stats.h" #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/trace_type.h" namespace perfetto::trace_processor { @@ -126,8 +127,8 @@ base::Status NinjaLogParser::Parse(TraceBlobView blob) { // been populated. base::Status NinjaLogParser::OnPushDataToSorter() { // Phase 1: Sort jobs and write slices directly to storage - std::sort(jobs_.begin(), jobs_.end(), - [](const Job& x, const Job& y) { return x.start_ms < y.start_ms; }); + ColdSortByKey(jobs_.begin(), jobs_.end(), + [](const Job& job) { return job.start_ms; }); // Now we need to work out the job parallelism. There's no direct indication // of that in the ninja logs, so it must be inferred by observing overlapping diff --git a/src/trace_processor/importers/perf/BUILD.gn b/src/trace_processor/importers/perf/BUILD.gn index 4470fc85006..eddd12a7eb0 100644 --- a/src/trace_processor/importers/perf/BUILD.gn +++ b/src/trace_processor/importers/perf/BUILD.gn @@ -92,6 +92,7 @@ source_set("perf") { "../../tables:tables_python", "../../types", "../../util:build_id", + "../../util:cold_sort", "../../util:trace_blob_view_reader", "../../util:trace_type", "../common", diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.cc b/src/trace_processor/importers/perf/perf_data_tokenizer.cc index 5035c09875b..878bab1cc3e 100644 --- a/src/trace_processor/importers/perf/perf_data_tokenizer.cc +++ b/src/trace_processor/importers/perf/perf_data_tokenizer.cc @@ -68,6 +68,7 @@ #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/build_id.h" #include "src/trace_processor/util/clock_synchronizer.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/trace_blob_view_reader.h" #include "src/trace_processor/util/trace_type.h" @@ -437,17 +438,19 @@ PerfDataTokenizer::ParseFeatureSections() { PERFETTO_CHECK(reader.Read(feature_sections_.back().second)); } - std::sort(feature_sections_.begin(), feature_sections_.end(), - [](const std::pair& lhs, - const std::pair& rhs) { - if (lhs.second.offset == rhs.second.offset) { - // Some sections have 0 length and thus there can be offset - // collisions. To make sure we parse sections by increasing - // offset parse empty sections first. - return lhs.second.size > rhs.second.size; - } - return lhs.second.offset > rhs.second.offset; - }); + // Sorts by offset, then size, both descending: a stable sort by size, then + // by offset. Some sections have 0 length and thus there can be offset + // collisions. To make sure we parse sections by increasing offset parse + // empty sections first. + ColdSortByKeyDescending(feature_sections_.begin(), feature_sections_.end(), + [](const std::pair& s) { + return s.second.size; + }); + ColdStableSortByKeyDescending( + feature_sections_.begin(), feature_sections_.end(), + [](const std::pair& s) { + return s.second.offset; + }); buffer_.PopFrontUntil(feature_headers_section_.end()); parsing_state_ = feature_sections_.empty() ? ParsingState::kDone diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn index 48e1a55aebc..f281757e7b5 100644 --- a/src/trace_processor/importers/proto/BUILD.gn +++ b/src/trace_processor/importers/proto/BUILD.gn @@ -123,6 +123,7 @@ source_set("minimal") { "../../tables", "../../types", "../../util:build_id", + "../../util:cold_sort", "../../util:decompressor", "../../util:json_args", "../../util:json_parser", @@ -251,6 +252,7 @@ source_set("full") { "../../tables", "../../types", "../../util:build_id", + "../../util:cold_sort", "../../util:descriptors", "../../util:profiler_util", "../../util:proto_profiler", diff --git a/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc b/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc index a9193409faf..7395ab0fc80 100644 --- a/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc +++ b/src/trace_processor/importers/proto/android_cpu_per_uid_module.cc @@ -36,6 +36,7 @@ #include "src/trace_processor/importers/proto/v8_module.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/tables/android_tables_py.h" +#include "src/trace_processor/util/cold_sort.h" #include "protos/perfetto/trace/android/cpu_per_uid_data.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" @@ -179,8 +180,8 @@ void AndroidCpuPerUidModule::OnEventsFullyExtracted() { rows.push_back(row); } - std::sort(rows.begin(), rows.end(), - [](auto& a, auto& b) { return a.track_id < b.track_id; }); + ColdSortByKey(rows.begin(), rows.end(), + [](const auto& row) { return row.track_id.value; }); for (const auto& row : rows) { context_->storage->mutable_android_cpu_per_uid_track_table()->Insert(row); diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc index 28874e770d6..1b8f59b2b31 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc @@ -16,7 +16,6 @@ #include "src/trace_processor/importers/proto/proto_trace_parser_impl.h" -#include #include #include #include @@ -55,6 +54,7 @@ #include "src/trace_processor/tables/metadata_tables_py.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" +#include "src/trace_processor/util/cold_sort.h" #include "protos/perfetto/config/trace_config.pbzero.h" #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h" @@ -290,10 +290,8 @@ void ProtoTraceParserImpl::ParseMetatraceEvent(int64_t ts, ConstBytes blob) { // We stable sort instead of sorting here to avoid changing the order of the // args in arrays. - std::stable_sort(interned.begin(), interned.end(), - [](const Arg& a, const Arg& b) { - return a.first.raw_id() < b.first.raw_id(); - }); + ColdStableSortByKey(interned.begin(), interned.end(), + [](const Arg& arg) { return arg.first.raw_id(); }); // Compute the correct key for each arg, possibly adding an index to // the end of the key if needed. diff --git a/src/trace_processor/importers/proto/system_probes_parser.cc b/src/trace_processor/importers/proto/system_probes_parser.cc index e318eed8224..8bd3d044d85 100644 --- a/src/trace_processor/importers/proto/system_probes_parser.cc +++ b/src/trace_processor/importers/proto/system_probes_parser.cc @@ -60,6 +60,7 @@ #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" #include "src/trace_processor/util/clock_synchronizer.h" +#include "src/trace_processor/util/cold_sort.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" #include "protos/perfetto/common/system_info.pbzero.h" @@ -1189,8 +1190,9 @@ void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { uint32_t cluster_id = 0; if (valid_capacities) { - std::sort(cpu_infos.begin(), cpu_infos.end(), - [](auto a, auto b) { return a.capacity < b.capacity; }); + ColdSortByKey(cpu_infos.begin(), cpu_infos.end(), [](const CpuInfo& info) { + return info.capacity ? uint64_t{*info.capacity} + 1 : 0; + }); uint32_t previous_capacity = *cpu_infos[0].capacity; for (CpuInfo& cpu_info : cpu_infos) { uint32_t capacity = *cpu_info.capacity; @@ -1210,8 +1212,8 @@ void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { {info.cpu, *std::max_element(info.frequencies.begin(), info.frequencies.end())}); } - std::sort(cpu_max_freqs.begin(), cpu_max_freqs.end(), - [](auto a, auto b) { return a.max_frequency < b.max_frequency; }); + ColdSortByKey(cpu_max_freqs.begin(), cpu_max_freqs.end(), + [](const CpuMaxFrequency& f) { return f.max_frequency; }); uint32_t previous_max_freq = cpu_max_freqs[0].max_frequency; for (CpuMaxFrequency& cpu_max_freq : cpu_max_freqs) { diff --git a/src/trace_processor/perfetto_sql/parser/BUILD.gn b/src/trace_processor/perfetto_sql/parser/BUILD.gn index 5dc482727a0..caef3381b9c 100644 --- a/src/trace_processor/perfetto_sql/parser/BUILD.gn +++ b/src/trace_processor/perfetto_sql/parser/BUILD.gn @@ -31,6 +31,7 @@ source_set("parser") { "../../../perfetto_sql:intrinsic_macro_expansion", "../../../perfetto_sql/syntaqlite", "../../sqlite", + "../../util:cold_sort", "../../util:sql_argument", "../pipeline", "../pipeline:plan", diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc index 94a1367c624..45856438156 100644 --- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc +++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc @@ -40,6 +40,7 @@ #include "src/trace_processor/perfetto_sql/pipeline/compiler.h" #include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" #include "src/trace_processor/sqlite/sql_source.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/sql_argument.h" namespace perfetto::trace_processor { @@ -247,10 +248,8 @@ class RewriteTree { // This is the only ApplyRewrites caller whose items aren't already in // `start` order: we concatenated literal body calls and $param segments, // each internally sorted but interleaved relative to each other. - std::sort(items.begin(), items.end(), - [](const RewriteItem& a, const RewriteItem& b) { - return a.start < b.start; - }); + ColdSortByKey(items.begin(), items.end(), + [](const RewriteItem& item) { return item.start; }); return ApplyRewrites(m.sql, std::move(items)); } diff --git a/src/trace_processor/plugins/interval_intersect/BUILD.gn b/src/trace_processor/plugins/interval_intersect/BUILD.gn index b08c60325b0..c1a96776d60 100644 --- a/src/trace_processor/plugins/interval_intersect/BUILD.gn +++ b/src/trace_processor/plugins/interval_intersect/BUILD.gn @@ -34,5 +34,6 @@ source_set("interval_intersect") { "../../sqlite", "../../storage", "../../types", + "../../util:cold_sort", ] } diff --git a/src/trace_processor/plugins/interval_intersect/interval_intersect.cc b/src/trace_processor/plugins/interval_intersect/interval_intersect.cc index b6f1c6d7683..98c0adc4c9d 100644 --- a/src/trace_processor/plugins/interval_intersect/interval_intersect.cc +++ b/src/trace_processor/plugins/interval_intersect/interval_intersect.cc @@ -53,6 +53,7 @@ #include "src/trace_processor/sqlite/sqlite_utils.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::interval_intersect { namespace { @@ -164,12 +165,10 @@ base::StatusOr PushPartition( // Sort `tables_order` from the smallest to the biggest. std::vector tables_order(tables_count); std::iota(tables_order.begin(), tables_order.end(), 0); - std::sort( - tables_order.begin(), tables_order.end(), - [&intervals_in_table](const uint32_t idx_a, const uint32_t idx_b) { - return intervals_in_table[idx_a]->intervals.size() < - intervals_in_table[idx_b]->intervals.size(); - }); + ColdSortByKey(tables_order.begin(), tables_order.end(), + [&intervals_in_table](uint32_t idx) { + return intervals_in_table[idx]->intervals.size(); + }); uint32_t idx_of_smallest_part = tables_order.front(); PERFETTO_DCHECK( !intervals_in_table[idx_of_smallest_part]->intervals.empty()); diff --git a/src/trace_processor/plugins/winscope_importer/BUILD.gn b/src/trace_processor/plugins/winscope_importer/BUILD.gn index eb48d873b91..d4df29756f5 100644 --- a/src/trace_processor/plugins/winscope_importer/BUILD.gn +++ b/src/trace_processor/plugins/winscope_importer/BUILD.gn @@ -95,6 +95,7 @@ source_set("winscope_importer") { "../../storage", "../../tables", "../../types", + "../../util:cold_sort", ] } diff --git a/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc b/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc index 3c7eeeadf25..5afcfe23e11 100644 --- a/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc +++ b/src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_extractor.cc @@ -19,6 +19,7 @@ #include #include #include "src/trace_processor/plugins/winscope_importer/surfaceflinger_layers_utils.h" +#include "src/trace_processor/util/cold_sort.h" namespace perfetto::trace_processor::winscope::surfaceflinger_layers { @@ -29,13 +30,15 @@ namespace { void SortByZThenLayerId( std::vector& layer_ids, const std::unordered_map& layers_by_id) { - std::sort(layer_ids.begin(), layer_ids.end(), - [&layers_by_id](int32_t a, int32_t b) { - const auto& layer_a = layers_by_id.at(a); - const auto& layer_b = layers_by_id.at(b); - return std::make_tuple(layer_a.z(), layer_a.id()) < - std::make_tuple(layer_b.z(), layer_b.id()); - }); + // (z, id) packed into one key, each with its sign bit flipped to order + // negative values first. + ColdSortByKey(layer_ids.begin(), layer_ids.end(), + [&layers_by_id](int32_t layer_id) { + const auto& layer = layers_by_id.at(layer_id); + auto z = static_cast(layer.z()) ^ 0x80000000u; + auto id = static_cast(layer.id()) ^ 0x80000000u; + return (uint64_t{z} << 32) | id; + }); } // Extract layers bottom-to-top according to layer drawing order from diff --git a/src/trace_processor/sqlite/BUILD.gn b/src/trace_processor/sqlite/BUILD.gn index 8022b269701..df16cf0f8d1 100644 --- a/src/trace_processor/sqlite/BUILD.gn +++ b/src/trace_processor/sqlite/BUILD.gn @@ -51,6 +51,7 @@ source_set("sqlite") { "../importers/ftrace:ftrace_descriptors", "../storage", "../types", + "../util:cold_sort", "../util:profile_builder", ] public_deps = [ "bindings" ] diff --git a/src/trace_processor/sqlite/sql_source.cc b/src/trace_processor/sqlite/sql_source.cc index 9d489d2c594..552bf4a8d41 100644 --- a/src/trace_processor/sqlite/sql_source.cc +++ b/src/trace_processor/sqlite/sql_source.cc @@ -15,6 +15,7 @@ */ #include "src/trace_processor/sqlite/sql_source.h" +#include "src/trace_processor/util/cold_sort.h" #include #include @@ -370,10 +371,10 @@ SqlSource SqlSource::Rewriter::Build() && { // Phase 2: sort the new rewrite vector by original offset and verify that the // original offsets are monotonic and non-overlapping. - std::sort(all_rewrites.begin(), all_rewrites.end(), - [](const SqlSource::Rewrite& a, const SqlSource::Rewrite& b) { - return a.original_sql_start < b.original_sql_start; - }); + ColdSortByKey(all_rewrites.begin(), all_rewrites.end(), + [](const SqlSource::Rewrite& rewrite) { + return rewrite.original_sql_start; + }); for (uint32_t i = 1; i < all_rewrites.size(); ++i) { PERFETTO_CHECK(all_rewrites[i - 1].original_sql_end <= all_rewrites[i].original_sql_start); diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn index 141c5e1d5c0..17b73c110cb 100644 --- a/src/trace_processor/util/BUILD.gn +++ b/src/trace_processor/util/BUILD.gn @@ -249,6 +249,19 @@ source_set("zip_reader") { } } +perfetto_component("cold_sort") { + sources = [ + "cold_sort.cc", + "cold_sort.h", + ] + deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/base", + "../../../include/perfetto/ext/base", + "../../base", + ] +} + source_set("galloping_search") { sources = [ "galloping_search.h" ] deps = [ "../../../gn:default_deps" ] @@ -447,6 +460,7 @@ source_set("trace_type") { "trace_type.h", ] deps = [ + ":cold_sort", ":decompressor", "../../../gn:default_deps", "../../../include/perfetto/ext/base", @@ -492,6 +506,7 @@ source_set("flatbuffer_reader") { source_set("unittests") { sources = [ "bump_allocator_unittest.cc", + "cold_sort_unittest.cc", "concurrency_stress_test_util_unittest.cc", "descriptors_unittest.cc", "flatbuffer_reader_unittest.cc", @@ -516,6 +531,7 @@ source_set("unittests") { testonly = true deps = [ ":bump_allocator", + ":cold_sort", ":concurrency_stress_test_util", ":decompressor", ":descriptors", diff --git a/src/trace_processor/util/cold_sort.cc b/src/trace_processor/util/cold_sort.cc new file mode 100644 index 00000000000..48635b6ad19 --- /dev/null +++ b/src/trace_processor/util/cold_sort.cc @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/cold_sort.h" + +#include + +namespace perfetto::trace_processor::cold_sort_internal { + +void SortKeyedIndices(KeyedIndex* begin, KeyedIndex* end) { + std::sort(begin, end, [](const KeyedIndex& a, const KeyedIndex& b) { + return a.key < b.key || (a.key == b.key && a.index < b.index); + }); +} + +} // namespace perfetto::trace_processor::cold_sort_internal diff --git a/src/trace_processor/util/cold_sort.h b/src/trace_processor/util/cold_sort.h new file mode 100644 index 00000000000..7e062bc8ab7 --- /dev/null +++ b/src/trace_processor/util/cold_sort.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_COLD_SORT_H_ +#define SRC_TRACE_PROCESSOR_UTIL_COLD_SORT_H_ + +#include +#include +#include +#include +#include + +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/small_vector.h" + +// Replacements for std::sort() and std::stable_sort() for code which is not +// hot. Those are compiled for every element type and comparator, at 3-4KB +// each. These sort by an integer key through one sort compiled once, so each +// caller adds only a few hundred bytes. They are up to 2x slower, and need 16 +// bytes per element of extra memory, only heap allocated above 64 elements. + +namespace perfetto::trace_processor { +namespace cold_sort_internal { + +struct KeyedIndex { + uint64_t key; + uint32_t index; +}; +using KeyedIndices = base::SmallVector; + +// Sorts by key, then index. +void SortKeyedIndices(KeyedIndex* begin, KeyedIndex* end); + +// Maps `key` to a uint64_t with the same order. +template +uint64_t ToOrderedKey(K key) { + static_assert(std::is_integral_v, "Cold sorts need integer keys"); + if constexpr (std::is_signed_v) { + return static_cast(static_cast(key)) ^ (1ull << 63); + } else { + return static_cast(key); + } +} + +// Sorts [begin, end) by key, ties in their original order. +template +void SortByKey(It begin, It end, Key key) { + using T = typename std::iterator_traits::value_type; + PERFETTO_DCHECK(end - begin <= std::numeric_limits::max()); + auto size = static_cast(end - begin); + if (size < 2) { + return; + } + KeyedIndices keys; + for (uint32_t i = 0; i < size; ++i) { + keys.emplace_back(KeyedIndex{ToOrderedKey(key(begin[i])), i}); + } + SortKeyedIndices(keys.data(), keys.data() + size); + // Move each element to its sorted position by following the permutation's + // cycles, so only one element is ever held outside the range. + for (uint32_t i = 0; i < size; ++i) { + if (keys[i].index == i) { + continue; + } + T held = std::move(begin[i]); + uint32_t j = i; + for (uint32_t k = keys[j].index; k != i; k = keys[j].index) { + begin[j] = std::move(begin[k]); + keys[j].index = j; + j = k; + } + begin[j] = std::move(held); + keys[j].index = j; + } +} + +} // namespace cold_sort_internal + +// Sorts [begin, end) by the integer `key(element)`, ascending. Elements with +// equal keys end up in an unspecified order. +template +void ColdSortByKey(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, std::move(key)); +} + +// As ColdSortByKey(), but descending. +template +void ColdSortByKeyDescending(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, [&key](const auto& element) { + return ~cold_sort_internal::ToOrderedKey(key(element)); + }); +} + +// As ColdSortByKey(), but elements with equal keys keep their order. +template +void ColdStableSortByKey(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, std::move(key)); +} + +// As ColdStableSortByKey(), but descending. +template +void ColdStableSortByKeyDescending(It begin, It end, Key key) { + cold_sort_internal::SortByKey(begin, end, [&key](const auto& element) { + return ~cold_sort_internal::ToOrderedKey(key(element)); + }); +} + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_UTIL_COLD_SORT_H_ diff --git a/src/trace_processor/util/cold_sort_unittest.cc b/src/trace_processor/util/cold_sort_unittest.cc new file mode 100644 index 00000000000..52e46ea75a5 --- /dev/null +++ b/src/trace_processor/util/cold_sort_unittest.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/cold_sort.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test/gtest_and_gmock.h" + +namespace perfetto::trace_processor { +namespace { + +TEST(ColdSortTest, MatchesStdStableSort) { + std::minstd_rand0 rnd(0); + for (uint32_t size : {0u, 1u, 2u, 3u, 16u, 17u, 64u, 65u, 1000u, 100000u}) { + std::vector> data; + for (uint32_t i = 0; i < size; ++i) { + data.emplace_back(static_cast(rnd() % 50) - 25, + std::to_string(i)); + } + auto expected = data; + std::stable_sort( + expected.begin(), expected.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + ColdStableSortByKey(data.begin(), data.end(), + [](const auto& x) { return x.first; }); + ASSERT_EQ(data, expected); + } +} + +TEST(ColdSortTest, KeyExtremes) { + std::vector data = {0, -1, std::numeric_limits::max(), + std::numeric_limits::min(), 1}; + ColdSortByKey(data.begin(), data.end(), [](int64_t x) { return x; }); + ASSERT_EQ(data, + (std::vector{std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()})); +} + +TEST(ColdSortTest, Descending) { + std::vector> data = { + {2, 0}, + {-5, 1}, + {2, 2}, + {std::numeric_limits::min(), 3}, + {7, 4}}; + ColdStableSortByKeyDescending(data.begin(), data.end(), + [](const auto& x) { return x.first; }); + ASSERT_EQ(data, (std::vector>{ + {7, 4}, + {2, 0}, + {2, 2}, + {-5, 1}, + {std::numeric_limits::min(), 3}})); +} + +TEST(ColdSortTest, MoveOnly) { + std::vector> data; + for (int v : {3, 1, 2}) { + data.push_back(std::make_unique(v)); + } + ColdSortByKey(data.begin(), data.end(), + [](const std::unique_ptr& p) { return *p; }); + ASSERT_EQ(*data[0], 1); + ASSERT_EQ(*data[1], 2); + ASSERT_EQ(*data[2], 3); +} + +} // namespace +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/util/symbolizer/BUILD.gn b/src/trace_processor/util/symbolizer/BUILD.gn index 0e016dc8277..d7b4ad9b9be 100644 --- a/src/trace_processor/util/symbolizer/BUILD.gn +++ b/src/trace_processor/util/symbolizer/BUILD.gn @@ -48,6 +48,7 @@ if (enable_perfetto_trace_processor) { ] deps = [ "../:build_id", + "../:cold_sort", "../../../../gn:default_deps", "../../../../include/perfetto/protozero", "../../../../include/perfetto/trace_processor", diff --git a/src/trace_processor/util/symbolizer/symbolize_database.cc b/src/trace_processor/util/symbolizer/symbolize_database.cc index f8f67491701..8a577e8fc6d 100644 --- a/src/trace_processor/util/symbolizer/symbolize_database.cc +++ b/src/trace_processor/util/symbolizer/symbolize_database.cc @@ -41,6 +41,7 @@ #include "perfetto/trace_processor/iterator.h" #include "perfetto/trace_processor/trace_processor.h" #include "src/trace_processor/util/build_id.h" +#include "src/trace_processor/util/cold_sort.h" #include "src/trace_processor/util/symbolizer/breakpad_symbolizer.h" #include "src/trace_processor/util/symbolizer/local_symbolizer.h" #include "src/trace_processor/util/symbolizer/symbolizer.h" @@ -52,6 +53,7 @@ namespace perfetto::profiling { namespace { +using trace_processor::ColdStableSortByKeyDescending; using trace_processor::Iterator; // Runtime mapping start/end addresses do not affect symbolization. Group by @@ -462,11 +464,9 @@ std::vector GroupUnresolved(const SymbolizerResult& result) { } // Largest first, so the short report names the mappings that matter most. for (auto& group : groups) { - std::stable_sort( + ColdStableSortByKeyDescending( group.mappings.begin(), group.mappings.end(), - [](const UnresolvedMapping& a, const UnresolvedMapping& b) { - return a.frame_count > b.frame_count; - }); + [](const UnresolvedMapping& m) { return m.frame_count; }); } return groups; } diff --git a/src/trace_processor/util/trace_type.cc b/src/trace_processor/util/trace_type.cc index 0fdca4f85ef..84a77999efe 100644 --- a/src/trace_processor/util/trace_type.cc +++ b/src/trace_processor/util/trace_type.cc @@ -15,6 +15,7 @@ */ #include "src/trace_processor/util/trace_type.h" +#include "src/trace_processor/util/cold_sort.h" #include #include @@ -81,9 +82,8 @@ TraceImporterId TraceImporterRegistry::Guess(const uint8_t* data, for (auto it = importers_.GetIterator(); it; ++it) { entries.push_back({it.key(), it.value().get()}); } - std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) { - return a.importer->descriptor().detection_priority < - b.importer->descriptor().detection_priority; + ColdSortByKey(entries.begin(), entries.end(), [](const Entry& e) { + return e.importer->descriptor().detection_priority; }); for (const Entry& e : entries) { if (e.importer->Sniff(data, size)) {