[crashtracker] Experiment with compressing proc/self/maps

libdd-crashtracker/Cargo.toml

@@ -60,11 +60,13 @@ rand = "0.8.5"
 schemars = "0.8.21"
 serde = {version = "1.0", features = ["derive"]}
 serde_json = {version = "1.0"}
+serde_bytes = "0.11"
 symbolic-demangle = { version = "12.8.0", default-features = false, features = ["rust", "cpp", "msvc"] }
 symbolic-common = { version = "12.8.0", default-features = false }
 tokio = { version = "1.23", features = ["rt", "macros", "io-std", "io-util"] }
 uuid = { version = "1.4.1", features = ["v4", "serde"] }
 thiserror = "1.0"
+zstd = { version = "0.13", default-features = false }
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.59.0", features = ["Win32_System_Diagnostics_Debug", "Win32_System_Diagnostics_ToolHelp", "Win32_System_ErrorReporting", "Win32_System_Kernel", "Win32_System_ProcessStatus", "Win32_System_Registry", "Win32_System_SystemInformation", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_Security"] }

libdd-crashtracker/src/crash_info/builder.rs

@@ -6,12 +6,16 @@ use crate::runtime_callback::RuntimeStack;
 use chrono::{DateTime, Utc};
 use error_data::ThreadData;
 use stacktrace::StackTrace;
-use std::io::{BufRead, BufReader};
+use std::fs;
 use unknown_value::UnknownValue;
 use uuid::Uuid;
+use zstd::bulk;
 
 use super::*;
 
+const PROC_SELF_MAPS_PATH: &str = "/proc/self/maps";
+const PROC_SELF_MAPS_ZSTD_LEVEL: i32 = 3;
+
 #[derive(Debug, Default, PartialEq)]
 pub struct ErrorDataBuilder {
     pub kind: Option<ErrorKind>,
@@ -96,7 +100,7 @@ pub struct CrashInfoBuilder {
     pub counters: Option<HashMap<String, i64>>,
     pub error: ErrorDataBuilder,
     pub experimental: Option<Experimental>,
-    pub files: Option<HashMap<String, Vec<String>>>,
+    pub files: Option<HashMap<String, CrashFile>>,
     pub fingerprint: Option<String>,
     pub incomplete: Option<bool>,
     pub log_messages: Option<Vec<String>>,
@@ -232,10 +236,8 @@ impl CrashInfoBuilder {
     }
 
     pub fn with_file(&mut self, filename: String) -> anyhow::Result<()> {
-        let file = File::open(&filename).with_context(|| format!("filename: {filename}"))?;
-        let lines: std::io::Result<Vec<_>> = BufReader::new(file).lines().collect();
-        self.with_file_and_contents(filename, lines?)?;
-        Ok(())
+        let data = fs::read(&filename).with_context(|| format!("filename: {filename}"))?;
+        self.with_file_bytes(filename, data)
     }
 
     /// Appends the given file to the current set of files in the builder.
@@ -244,20 +246,36 @@ impl CrashInfoBuilder {
         filename: String,
         contents: Vec<String>,
     ) -> anyhow::Result<()> {
-        if let Some(ref mut files) = &mut self.files {
-            files.insert(filename, contents);
-        } else {
-            self.files = Some(HashMap::from([(filename, contents)]));
+        let mut data = Vec::new();
+        for line in contents {
+            data.extend_from_slice(line.as_bytes());
+            data.push(b'\n');
         }
-        Ok(())
+        self.with_file_bytes(filename, data)
     }
 
     /// Sets the current set of files in the builder.
-    pub fn with_files(&mut self, files: HashMap<String, Vec<String>>) -> anyhow::Result<()> {
+    pub fn with_files(&mut self, files: HashMap<String, CrashFile>) -> anyhow::Result<()> {
         self.files = Some(files);
         Ok(())
     }
 
+    pub fn with_file_bytes(&mut self, filename: String, contents: Vec<u8>) -> anyhow::Result<()> {
+        let file = if filename == PROC_SELF_MAPS_PATH {
+            let compressed = bulk::compress(&contents, PROC_SELF_MAPS_ZSTD_LEVEL)
+                .context("unable to zstd encode /proc/self/maps")?;
+            CrashFile::with_encoding(compressed, FileEncoding::Zstd)
+        } else {
+            CrashFile::new(contents)
+        };
+        if let Some(ref mut files) = &mut self.files {
+            files.insert(filename, file);
+        } else {
+            self.files = Some(HashMap::from([(filename, file)]));
+        }
+        Ok(())
+    }
+
     pub fn with_fingerprint(&mut self, fingerprint: String) -> anyhow::Result<()> {
         anyhow::ensure!(!fingerprint.is_empty(), "Expect non-empty fingerprint");
         self.fingerprint = Some(fingerprint);

libdd-crashtracker/src/crash_info/mod.rs

@@ -42,6 +42,36 @@ pub fn build_crash_ping_message(sig_info: &SigInfo) -> String {
     )
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "lowercase")]
+pub enum FileEncoding {
+    Zstd,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct CrashFile {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub encoding: Option<FileEncoding>,
+    #[serde(default)]
+    pub data: Vec<u8>,
+}
+
+impl CrashFile {
+    pub fn new(data: Vec<u8>) -> Self {
+        Self {
+            encoding: None,
+            data,
+        }
+    }
+
+    pub fn with_encoding(data: Vec<u8>, encoding: FileEncoding) -> Self {
+        Self {
+            encoding: Some(encoding),
+            data,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct CrashInfo {
     #[serde(default, skip_serializing_if = "HashMap::is_empty")]
@@ -51,7 +81,9 @@ pub struct CrashInfo {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub experimental: Option<Experimental>,
     #[serde(default, skip_serializing_if = "HashMap::is_empty")]
-    pub files: HashMap<String, Vec<String>>,
+    /// Supplemental files attached to the crash report. Each entry stores the raw
+    /// bytes plus optional encoding metadata (e.g. `/proc/self/maps` is zstd-compressed).
+    pub files: HashMap<String, CrashFile>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub fingerprint: Option<String>,
     pub incomplete: bool,

libdd-crashtracker/src/receiver/receive_report.rs

@@ -86,6 +86,20 @@ fn process_line(
     state: StdinState,
 ) -> anyhow::Result<StdinState> {
     let next = match state {
+        StdinState::File(filename, contents) if line.starts_with(DD_CRASHTRACK_END_FILE) => {
+            let mut data = Vec::new();
+            for entry in contents {
+                data.extend_from_slice(entry.as_bytes());
+                data.push(b'\n');
+            }
+            builder.with_file_bytes(filename, data)?;
+            StdinState::Waiting
+        }
+        StdinState::File(name, mut contents) => {
+            contents.push(line.to_string());
+            StdinState::File(name, contents)
+        }
+
         StdinState::AdditionalTags if line.starts_with(DD_CRASHTRACK_END_ADDITIONAL_TAGS) => {
             StdinState::Waiting
         }
@@ -127,15 +141,6 @@ fn process_line(
             StdinState::Done
         }
 
-        StdinState::File(filename, lines) if line.starts_with(DD_CRASHTRACK_END_FILE) => {
-            builder.with_file_and_contents(filename, lines)?;
-            StdinState::Waiting
-        }
-        StdinState::File(name, mut contents) => {
-            contents.push(line.to_string());
-            StdinState::File(name, contents)
-        }
-
         StdinState::Metadata if line.starts_with(DD_CRASHTRACK_END_METADATA) => StdinState::Waiting,
         StdinState::Metadata => {
             let metadata = serde_json::from_str(line)?;
@@ -330,7 +335,9 @@ pub(crate) async fn receive_report_from_stream(
             builder.with_log_message(format!("IO Error: {next_line:?}"), true)?;
             break;
         };
-        let Some(next_line) = next_line else { break };
+        let Some(next_line) = next_line else {
+            break;
+        };
 
         match process_line(&mut builder, &mut config, &next_line, stdin_state) {
             Ok(next_state) => {