diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto index d2932bb014d..3dd4b481d2c 100644 --- a/protos/perfetto/trace/perfetto_trace.proto +++ b/protos/perfetto/trace/perfetto_trace.proto @@ -16515,6 +16515,10 @@ message ObfuscatedClass { message MergedClasses { optional string class_id_field_name = 1; repeated MergedClass merged_classes = 2; + // Set to true if other unknown classes may have been merged into this + // class by optimization (e.g. when using older R8 mapping versions that + // do not emit mergedClasses metadata). + optional bool unknown_merged_classes = 3; } optional MergedClasses merged_classes = 5; diff --git a/protos/perfetto/trace/profiling/deobfuscation.proto b/protos/perfetto/trace/profiling/deobfuscation.proto index 76b711a163f..212f802567e 100644 --- a/protos/perfetto/trace/profiling/deobfuscation.proto +++ b/protos/perfetto/trace/profiling/deobfuscation.proto @@ -63,6 +63,10 @@ message ObfuscatedClass { message MergedClasses { optional string class_id_field_name = 1; repeated MergedClass merged_classes = 2; + // Set to true if other unknown classes may have been merged into this + // class by optimization (e.g. when using older R8 mapping versions that + // do not emit mergedClasses metadata). + optional bool unknown_merged_classes = 3; } optional MergedClasses merged_classes = 5; diff --git a/src/trace_processor/importers/proto/deobfuscation_tracker.cc b/src/trace_processor/importers/proto/deobfuscation_tracker.cc index f6b9222d21b..27a83fff65a 100644 --- a/src/trace_processor/importers/proto/deobfuscation_tracker.cc +++ b/src/trace_processor/importers/proto/deobfuscation_tracker.cc @@ -397,6 +397,12 @@ void DeobfuscationTracker::DeobfuscateHeapGraphClass( const std::vector& rows, const ObfuscatedClass::Decoder& cls) { auto* class_table = context_->storage->mutable_heap_graph_class_table(); + bool potentially_merged = false; + if (cls.has_merged_classes()) { + ObfuscatedClass::MergedClasses::Decoder mcs(cls.merged_classes()); + potentially_merged = + mcs.unknown_merged_classes() || static_cast(mcs.merged_classes()); + } for (tables::HeapGraphClassTable::RowNumber row : rows) { auto class_ref = row.ToRowReference(class_table); const base::StringView obfuscated_type_name = @@ -406,6 +412,9 @@ void DeobfuscationTracker::DeobfuscateHeapGraphClass( DenormalizeTypeName(normalized_type, cls.deobfuscated_name()); class_ref.set_deobfuscated_name(context_->storage->InternString( base::StringView(deobfuscated_type_name))); + if (potentially_merged) { + class_ref.set_potentially_merged(true); + } } } diff --git a/src/trace_processor/perfetto_sql/stdlib/prelude/after_eof/memory.sql b/src/trace_processor/perfetto_sql/stdlib/prelude/after_eof/memory.sql index 349085faf50..2f3ad1f50dc 100644 --- a/src/trace_processor/perfetto_sql/stdlib/prelude/after_eof/memory.sql +++ b/src/trace_processor/perfetto_sql/stdlib/prelude/after_eof/memory.sql @@ -40,7 +40,10 @@ CREATE PERFETTO VIEW heap_graph_class( -- The classloader that loaded this class. classloader_id LONG, -- The kind of class. - kind STRING + kind STRING, + -- True if other classes may have been merged into this class by optimization + -- (e.g. R8 class merging). + potentially_merged BOOL ) AS SELECT @@ -50,7 +53,8 @@ SELECT location, superclass_id, classloader_id, - kind + kind, + potentially_merged FROM __intrinsic_heap_graph_class; -- The objects on the Dalvik heap. diff --git a/src/trace_processor/tables/profiler_tables.py b/src/trace_processor/tables/profiler_tables.py index ca8745cb594..ef0bfca66fc 100644 --- a/src/trace_processor/tables/profiler_tables.py +++ b/src/trace_processor/tables/profiler_tables.py @@ -1056,6 +1056,12 @@ cpp_access=CppAccess.READ_AND_LOW_PERF_WRITE, cpp_access_duration=CppAccessDuration.POST_FINALIZATION, ), + C( + 'potentially_merged', + CppBool(), + cpp_access=CppAccess.READ_AND_LOW_PERF_WRITE, + cpp_access_duration=CppAccessDuration.POST_FINALIZATION, + ), ], tabledoc=TableDoc( doc='''''', @@ -1074,7 +1080,10 @@ 'classloader_id': '''''', 'kind': - '''''' + '''''', + 'potentially_merged': + '''True if other classes may have been merged into this class + by optimization (e.g. R8 class merging).''', })) HEAP_GRAPH_OBJECT_TABLE = Table( diff --git a/src/trace_processor/util/deobfuscation/deobfuscator.cc b/src/trace_processor/util/deobfuscation/deobfuscator.cc index dc71037931f..6a7e3812781 100644 --- a/src/trace_processor/util/deobfuscation/deobfuscator.cc +++ b/src/trace_processor/util/deobfuscation/deobfuscator.cc @@ -42,15 +42,65 @@ namespace { using SimpleJsonParser = perfetto::trace_processor::json::SimpleJsonParser; using FieldResult = perfetto::trace_processor::json::FieldResult; -// Json keys used to parse merged classes. +// Json keys used to parse merged classes and mapping header. constexpr std::string_view kMergedClassesId = "com.android.tools.r8.mergedClasses"; +constexpr std::string_view kMappingHeaderId = "com.android.tools.r8.mapping"; constexpr std::string_view kIdKey = "id"; +constexpr std::string_view kVersionKey = "version"; constexpr std::string_view kClassNameKey = "name"; constexpr std::string_view kClassIdKey = "classId"; constexpr std::string_view kClassIdFieldKey = "classIdField"; constexpr std::string_view kMergedClassesKey = "mergedClasses"; +// Parses R8 `# +// {"id":"com.android.tools.r8.mapping","version":"."}` header +// comment and sets `supports_merged_classes` to true if version >= 2.3. +void ParseMappingHeaderComment(std::string_view json_str, + bool* supports_merged_classes) { + if (json_str.find(kMappingHeaderId) == std::string_view::npos) { + return; + } + SimpleJsonParser parser(json_str); + if (!parser.Parse().ok()) { + return; + } + bool is_mapping_header = false; + std::string version; + base::Status s = + parser.ForEachField([&](std::string_view key) -> FieldResult { + if (key == kIdKey) { + if (auto val = parser.GetString(); val && *val == kMappingHeaderId) { + is_mapping_header = true; + } + return FieldResult::Handled{}; + } + if (key == kVersionKey) { + if (auto val = parser.GetString()) { + version = std::string(*val); + } + return FieldResult::Handled{}; + } + return FieldResult::Skip{}; + }); + if (!s.ok() || !is_mapping_header || version.empty()) { + return; + } + size_t dot = version.find('.'); + if (dot == std::string::npos) { + return; + } + auto major = + base::StringViewToInt32(std::string_view(version).substr(0, dot)); + auto minor = + base::StringViewToInt32(std::string_view(version).substr(dot + 1)); + if (major.has_value() && minor.has_value()) { + if (*major > 2 || (*major == 2 && *minor >= 3)) { + *supports_merged_classes = true; + } + } +} + struct ProguardClass { std::string obfuscated_name; std::string deobfuscated_name; @@ -162,7 +212,8 @@ base::Status ParseMergedClass(SimpleJsonParser& parser, // Parses R8 `com.android.tools.r8.mergedClasses` JSON comment string in // Proguard mapping. base::Status ParseMergedClassesComment(std::string_view json_str, - ObfuscatedClass& target_class) { + ObfuscatedClass& target_class, + bool* supports_merged_classes) { if (json_str.find(kMergedClassesKey) == std::string_view::npos) { // Avoid full parsing if possible. return base::OkStatus(); @@ -188,6 +239,7 @@ base::Status ParseMergedClassesComment(std::string_view json_str, RETURN_IF_ERROR(s); if (is_merged_classes_id) { + *supports_merged_classes = true; *target_class.mutable_merged_classes() = std::move(mcs); } return base::OkStatus(); @@ -451,17 +503,20 @@ base::Status ProguardParser::AddLine(std::string line) { return base::Status(); if (line[first_ch_pos] == '#') { + size_t json_start = line.find('{'); + if (json_start == std::string::npos) { + return base::Status(); + } + std::string_view json_sv = std::string_view(line).substr(json_start); if (current_class_ == nullptr) { + ParseMappingHeaderComment(json_sv, &supports_merged_classes_); return base::Status(); } - size_t json_start = line.find('{'); - if (json_start != std::string::npos) { - std::string_view json_sv = std::string_view(line).substr(json_start); - base::Status s = ParseMergedClassesComment(json_sv, *current_class_); - if (!s.ok()) { - PERFETTO_ELOG("Failed to parse merged classes comment: %s\non line %s", - s.message().c_str(), line.c_str()); - } + base::Status s = ParseMergedClassesComment(json_sv, *current_class_, + &supports_merged_classes_); + if (!s.ok()) { + PERFETTO_ELOG("Failed to parse merged classes comment: %s\non line %s", + s.message().c_str(), line.c_str()); } return base::Status(); } @@ -536,12 +591,26 @@ bool ProguardParser::AddLines(std::string contents) { return true; } +std::map ProguardParser::ConsumeMapping() { + if (!supports_merged_classes_) { + for (auto& [obfuscated_name, cls] : mapping_) { + if (obfuscated_name != cls.deobfuscated_name()) { + cls.mutable_merged_classes()->unknown_merged_classes = true; + } + } + } + return std::move(mapping_); +} + static void SerializeMergedClasses( const profiling::ObfuscatedClass::MergedClasses& src, perfetto::protos::pbzero::ObfuscatedClass::MergedClasses* dest) { if (!src.class_id_field_name.empty()) { dest->set_class_id_field_name(src.class_id_field_name); } + if (src.unknown_merged_classes) { + dest->set_unknown_merged_classes(true); + } for (const auto& mc : src.merged_classes) { auto* dest_mc = dest->add_merged_classes(); if (!mc.name.empty()) { @@ -550,7 +619,8 @@ static void SerializeMergedClasses( if (mc.class_id.has_value()) { dest_mc->set_class_id(*mc.class_id); } - if (!mc.nested_merged_classes.merged_classes.empty()) { + if (!mc.nested_merged_classes.merged_classes.empty() || + mc.nested_merged_classes.unknown_merged_classes) { SerializeMergedClasses(mc.nested_merged_classes, dest_mc->set_merged_classes()); } @@ -560,7 +630,7 @@ static void SerializeMergedClasses( static void SerializeTopLevelMergedClasses( const profiling::ObfuscatedClass::MergedClasses& src, perfetto::protos::pbzero::ObfuscatedClass* dest) { - if (!src.merged_classes.empty()) { + if (!src.merged_classes.empty() || src.unknown_merged_classes) { SerializeMergedClasses(src, dest->set_merged_classes()); } } diff --git a/src/trace_processor/util/deobfuscation/deobfuscator.h b/src/trace_processor/util/deobfuscation/deobfuscator.h index 9334e645304..da9bf89fbf5 100644 --- a/src/trace_processor/util/deobfuscation/deobfuscator.h +++ b/src/trace_processor/util/deobfuscation/deobfuscator.h @@ -52,6 +52,7 @@ class ObfuscatedClass { struct MergedClasses { std::string class_id_field_name; std::vector merged_classes; + bool unknown_merged_classes = false; }; // Represents an individual class merged by R8 into an obfuscated target @@ -107,13 +108,14 @@ class ProguardParser { base::Status AddLine(std::string line); bool AddLines(std::string contents); - std::map ConsumeMapping() { - return std::move(mapping_); - } + std::map ConsumeMapping(); private: std::map mapping_; ObfuscatedClass* current_class_ = nullptr; + // Set to true if the mapping file declares R8 mapping format version >= 2.3 + // or contains any `com.android.tools.r8.mergedClasses` comment. + bool supports_merged_classes_ = false; }; struct ProguardMap { diff --git a/src/trace_processor/util/deobfuscation/deobfuscator_unittest.cc b/src/trace_processor/util/deobfuscation/deobfuscator_unittest.cc index aa2e662265d..84bc1822d54 100644 --- a/src/trace_processor/util/deobfuscation/deobfuscator_unittest.cc +++ b/src/trace_processor/util/deobfuscation/deobfuscator_unittest.cc @@ -472,6 +472,35 @@ com.example.ClassE -> e: mapping.find("e")->second.merged_classes().merged_classes.empty()); } +TEST(ProguardParserTest, + MappingVersionPre2_3MarksObfuscatedClassesUnknownMerged) { + ProguardParser p; + const char input[] = R"(# compiler: R8 +# {"id":"com.android.tools.r8.mapping","version":"2.2"} +com.example.KeptClass -> com.example.KeptClass: +com.example.ObfuscatedClass -> a.b: +)"; + ASSERT_TRUE(p.AddLines(std::string(input))); + auto mapping = p.ConsumeMapping(); + ASSERT_EQ(mapping.size(), 2u); + EXPECT_FALSE(mapping.at("com.example.KeptClass") + .merged_classes() + .unknown_merged_classes); + EXPECT_TRUE(mapping.at("a.b").merged_classes().unknown_merged_classes); +} + +TEST(ProguardParserTest, MappingVersion2_3DoesNotMarkUnmergedClasses) { + ProguardParser p; + const char input[] = R"(# compiler: R8 +# {"id":"com.android.tools.r8.mapping","version":"2.3"} +com.example.ObfuscatedClass -> a.b: +)"; + ASSERT_TRUE(p.AddLines(std::string(input))); + auto mapping = p.ConsumeMapping(); + ASSERT_EQ(mapping.size(), 1u); + EXPECT_FALSE(mapping.at("a.b").merged_classes().unknown_merged_classes); +} + // ============================================================================= // R8 Retrace Compatibility Tests // diff --git a/test/trace_processor/diff_tests/parser/profiling/tests_heap_graph.py b/test/trace_processor/diff_tests/parser/profiling/tests_heap_graph.py index 14800374baa..8b749334d66 100644 --- a/test/trace_processor/diff_tests/parser/profiling/tests_heap_graph.py +++ b/test/trace_processor/diff_tests/parser/profiling/tests_heap_graph.py @@ -807,3 +807,104 @@ def test_heap_graph_heap_size(self): "ts","heap_size" 10,100000 """)) + + def test_heap_graph_potentially_merged(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + process_tree { + processes { + pid: 2 + ppid: 1 + cmdline: "system_server" + uid: 1000 + } + } + } + packet { + trusted_packet_sequence_id: 999 + timestamp: 10 + [com.android.art.tracing.ArtHeapGraphTracePacket.heap_graph] { + pid: 2 + location_names { + iid: 1 + str: "/data/app/~~ASDFG==/invalid.test.android-SDASD/test.apk" + } + types { + id: 1 + class_name: "java.lang.Object" + location_id: 1 + } + types { + id: 2 + class_name: "a" + location_id: 1 + } + types { + id: 3 + class_name: "b" + location_id: 1 + } + types { + id: 4 + class_name: "b[]" + location_id: 1 + } + types { + id: 5 + class_name: "java.lang.Class" + location_id: 1 + } + types { + id: 6 + class_name: "c" + location_id: 1 + } + objects { + id: 1 + type_id: 1 + self_size: 64 + } + continued: false + index: 0 + } + } + packet { + deobfuscation_mapping { + obfuscated_classes { + obfuscated_name: "a" + deobfuscated_name: "DeobfuscatedA" + } + obfuscated_classes { + obfuscated_name: "b" + deobfuscated_name: "DeobfuscatedB" + merged_classes { + unknown_merged_classes: true + } + } + obfuscated_classes { + obfuscated_name: "c" + deobfuscated_name: "DeobfuscatedC" + merged_classes { + merged_classes { + name: "MergedIntoC" + } + } + } + } + } + """), + query=""" + SELECT name, deobfuscated_name, potentially_merged + FROM heap_graph_class + ORDER BY name; + """, + out=Csv(""" + "name","deobfuscated_name","potentially_merged" + "a","DeobfuscatedA",0 + "b","DeobfuscatedB",1 + "b[]","DeobfuscatedB[]",1 + "c","DeobfuscatedC",1 + "java.lang.Class","java.lang.Class",1 + "java.lang.Object","[NULL]",0 + """))