diff --git a/crates/goose-providers/src/json.rs b/crates/goose-providers/src/json.rs new file mode 100644 index 000000000000..a0f8c1fc035e --- /dev/null +++ b/crates/goose-providers/src/json.rs @@ -0,0 +1,255 @@ +use serde_json::Value; + +pub fn unescape_json_values(value: &Value) -> Value { + let mut cloned = value.clone(); + unescape_json_values_in_place(&mut cloned); + cloned +} + +fn unescape_json_values_in_place(value: &mut Value) { + match value { + Value::Object(map) => { + for value in map.values_mut() { + unescape_json_values_in_place(value); + } + } + Value::Array(values) => { + for value in values.iter_mut() { + unescape_json_values_in_place(value); + } + } + Value::String(text) => { + if text.contains('\\') { + *text = text + .replace("\\\\n", "\n") + .replace("\\\\t", "\t") + .replace("\\\\r", "\r") + .replace("\\\\\"", "\"") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r") + .replace("\\\"", "\""); + } + } + _ => {} + } +} + +pub fn safely_parse_json(input: &str) -> Result { + match serde_json::from_str(input) { + Ok(value) => Ok(value), + Err(_) => { + for candidate in [ + repair_truncated_json(input), + json_escape_control_chars_in_string(input), + ] { + if let Ok(value) = serde_json::from_str(&candidate) { + return Ok(value); + } + } + + let repaired = repair_truncated_json(&json_escape_control_chars_in_string(input)); + serde_json::from_str(&repaired) + } + } +} + +fn repair_truncated_json(input: &str) -> String { + let mut repaired = String::with_capacity(input.len() + 8); + let mut in_string = false; + let mut escape_next = false; + let mut closers = Vec::new(); + + for character in input.chars() { + repaired.push(character); + + if in_string { + if escape_next { + escape_next = false; + continue; + } + + match character { + '\\' => escape_next = true, + '"' => in_string = false, + _ => {} + } + continue; + } + + match character { + '"' => in_string = true, + '{' => closers.push('}'), + '[' => closers.push(']'), + '}' | ']' => { + if closers.last() == Some(&character) { + closers.pop(); + } + } + _ => {} + } + } + + if in_string { + if escape_next { + repaired.push('\\'); + } + repaired.push('"'); + } + + while let Some(closer) = closers.pop() { + repaired.push(closer); + } + + repaired +} + +pub fn json_escape_control_chars_in_string(input: &str) -> String { + let mut escaped = String::with_capacity(input.len()); + for character in input.chars() { + match character { + '\u{0000}'..='\u{001F}' => match character { + '\u{0008}' => escaped.push_str("\\b"), + '\u{000C}' => escaped.push_str("\\f"), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + _ => { + escaped.push_str(&format!("\\u{:04x}", character as u32)); + } + }, + _ => escaped.push(character), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn unescape_json_values_with_object() { + let value = json!({"text": "Hello\\nWorld"}); + let unescaped_value = unescape_json_values(&value); + assert_eq!(unescaped_value, json!({"text": "Hello\nWorld"})); + } + + #[test] + fn unescape_json_values_with_array() { + let value = json!(["Hello\\nWorld", "Goodbye\\tWorld"]); + let unescaped_value = unescape_json_values(&value); + assert_eq!(unescaped_value, json!(["Hello\nWorld", "Goodbye\tWorld"])); + } + + #[test] + fn unescape_json_values_with_string() { + let value = json!("Hello\\nWorld"); + let unescaped_value = unescape_json_values(&value); + assert_eq!(unescaped_value, json!("Hello\nWorld")); + } + + #[test] + fn unescape_json_values_with_mixed_content() { + let value = json!({ + "text": "Hello\\nWorld\\\\n!", + "array": ["Goodbye\\tWorld", "See you\\rlater"], + "nested": { + "inner_text": "Inner\\\"Quote\\\"" + } + }); + let unescaped_value = unescape_json_values(&value); + assert_eq!( + unescaped_value, + json!({ + "text": "Hello\nWorld\n!", + "array": ["Goodbye\tWorld", "See you\rlater"], + "nested": { + "inner_text": "Inner\"Quote\"" + } + }) + ); + } + + #[test] + fn unescape_json_values_with_no_escapes() { + let value = json!({"text": "Hello World"}); + let unescaped_value = unescape_json_values(&value); + assert_eq!(unescaped_value, json!({"text": "Hello World"})); + } + + #[test] + fn safely_parse_json_repairs_common_malformed_json() { + let valid_json = r#"{"key1": "value1","key2": "value2"}"#; + let result = safely_parse_json(valid_json).unwrap(); + assert_eq!(result["key1"], "value1"); + assert_eq!(result["key2"], "value2"); + + let invalid_json = "{\"key1\": \"value1\n\",\"key2\": \"value2\"}"; + let result = safely_parse_json(invalid_json).unwrap(); + assert_eq!(result["key1"], "value1\n"); + assert_eq!(result["key2"], "value2"); + + let good_json = r#"{"test": "value"}"#; + let result = safely_parse_json(good_json).unwrap(); + assert_eq!(result["test"], "value"); + + let truncated_json = r#"{"key": "unclosed_string","nested": {"items": [1, 2, 3"#; + let result = safely_parse_json(truncated_json).unwrap(); + assert_eq!(result["key"], "unclosed_string"); + assert_eq!(result["nested"]["items"], json!([1, 2, 3])); + + let dangling_escape_json = String::from(r#"{"path":"abc\"#); + let result = safely_parse_json(&dangling_escape_json).unwrap(); + assert_eq!(result["path"], "abc\\"); + + let empty_json = "{}"; + let result = safely_parse_json(empty_json).unwrap(); + assert!(result.as_object().unwrap().is_empty()); + + let escaped_json = r#"{"key": "value with\nnewline"}"#; + let result = safely_parse_json(escaped_json).unwrap(); + assert_eq!(result["key"], "value with\nnewline"); + } + + #[test] + fn json_escape_control_chars_in_string_escapes_control_characters() { + assert_eq!( + json_escape_control_chars_in_string("Hello\nWorld"), + "Hello\\nWorld" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello\tWorld"), + "Hello\\tWorld" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello\rWorld"), + "Hello\\rWorld" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello\n\tWorld\r"), + "Hello\\n\\tWorld\\r" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello \"World\""), + "Hello \"World\"" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello\\World"), + "Hello\\World" + ); + assert_eq!( + json_escape_control_chars_in_string("{\"message\": \"Hello\nWorld\"}"), + "{\"message\": \"Hello\\nWorld\"}" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello World"), + "Hello World" + ); + assert_eq!( + json_escape_control_chars_in_string("Hello\u{0001}World"), + "Hello\\u0001World" + ); + } +} diff --git a/crates/goose-providers/src/lib.rs b/crates/goose-providers/src/lib.rs index 7e609f3961d7..a4939b6fd1ea 100644 --- a/crates/goose-providers/src/lib.rs +++ b/crates/goose-providers/src/lib.rs @@ -4,6 +4,7 @@ pub mod canonical; pub mod errors; pub mod function_name; pub mod http_status; +pub mod json; pub mod metadata; pub mod models; pub mod provider; diff --git a/crates/goose/src/providers/utils.rs b/crates/goose/src/providers/utils.rs index 4a6366cc8fd1..82cbcd832614 100644 --- a/crates/goose/src/providers/utils.rs +++ b/crates/goose/src/providers/utils.rs @@ -5,6 +5,9 @@ use crate::providers::errors::ProviderError; use anyhow::Result; use base64::Engine; pub use goose_providers::function_name::{is_valid_function_name, sanitize_function_name}; +pub use goose_providers::json::{ + json_escape_control_chars_in_string, safely_parse_json, unescape_json_values, +}; use goose_types::ModelConfig; pub use goose_types::{ extract_reasoning_effort, is_openai_responses_model, openai_reasoning_effort_for_thinking, @@ -294,41 +297,6 @@ pub fn load_image_file(path: &str) -> Result { .no_annotation()) } -pub fn unescape_json_values(value: &Value) -> Value { - let mut cloned = value.clone(); - unescape_json_values_in_place(&mut cloned); - cloned -} - -fn unescape_json_values_in_place(value: &mut Value) { - match value { - Value::Object(map) => { - for v in map.values_mut() { - unescape_json_values_in_place(v); - } - } - Value::Array(arr) => { - for v in arr.iter_mut() { - unescape_json_values_in_place(v); - } - } - Value::String(s) => { - if s.contains('\\') { - *s = s - .replace("\\\\n", "\n") - .replace("\\\\t", "\t") - .replace("\\\\r", "\r") - .replace("\\\\\"", "\"") - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\r", "\r") - .replace("\\\"", "\""); - } - } - _ => {} - } -} - pub use goose_providers::request_log::LOGS_TO_KEEP; pub struct RequestLog(goose_providers::request_log::RequestLog); @@ -360,130 +328,6 @@ impl RequestLog { } } -/// Safely parse a JSON string that may contain doubly-encoded or malformed JSON. -/// This function first attempts to parse the input string as-is. If that fails, -/// it applies control character escaping and truncated JSON repair and tries again. -/// -/// This approach preserves valid JSON like `{"key1": "value1",\n"key2": "value"}` -/// (which contains a literal \n but is perfectly valid JSON) while still fixing -/// broken JSON like `{"key1": "value1\n","key2": "value"}` (which contains an -/// unescaped newline character). -pub fn safely_parse_json(s: &str) -> Result { - // First, try parsing the string as-is - match serde_json::from_str(s) { - Ok(value) => Ok(value), - Err(_) => { - for candidate in [ - repair_truncated_json(s), - json_escape_control_chars_in_string(s), - ] { - if let Ok(value) = serde_json::from_str(&candidate) { - return Ok(value); - } - } - - let repaired = repair_truncated_json(&json_escape_control_chars_in_string(s)); - serde_json::from_str(&repaired) - } - } -} - -fn repair_truncated_json(s: &str) -> String { - let mut repaired = String::with_capacity(s.len() + 8); - let mut in_string = false; - let mut escape_next = false; - let mut closers = Vec::new(); - - for c in s.chars() { - repaired.push(c); - - if in_string { - if escape_next { - escape_next = false; - continue; - } - - match c { - '\\' => escape_next = true, - '"' => in_string = false, - _ => {} - } - continue; - } - - match c { - '"' => in_string = true, - '{' => closers.push('}'), - '[' => closers.push(']'), - '}' | ']' => { - if closers.last() == Some(&c) { - closers.pop(); - } - } - _ => {} - } - } - - if in_string { - if escape_next { - repaired.push('\\'); - } - repaired.push('"'); - } - - while let Some(closer) = closers.pop() { - repaired.push(closer); - } - - repaired -} - -/// Helper to escape control characters in a string that is supposed to be a JSON document. -/// This function iterates through the input string `s` and replaces any literal -/// control characters (U+0000 to U+001F) with their JSON-escaped equivalents -/// (e.g., '\n' becomes "\\n", '\u0001' becomes "\\u0001"). -/// -/// It does NOT escape quotes (") or backslashes (\) because it assumes `s` is a -/// full JSON document, and these characters might be structural (e.g., object delimiters, -/// existing valid escape sequences). The goal is to fix common LLM errors where -/// control characters are emitted raw into what should be JSON string values, -/// making the overall JSON structure unparsable. -/// -/// If the input string `s` has other JSON syntax errors (e.g., an unescaped quote -/// *within* a string value like `{"key": "string with " quote"}`), this function -/// will not fix them. It specifically targets unescaped control characters. -pub fn json_escape_control_chars_in_string(s: &str) -> String { - let mut r = String::with_capacity(s.len()); // Pre-allocate for efficiency - for c in s.chars() { - match c { - // ASCII Control characters (U+0000 to U+001F) - '\u{0000}'..='\u{001F}' => { - match c { - '\u{0008}' => r.push_str("\\b"), // Backspace - '\u{000C}' => r.push_str("\\f"), // Form feed - '\n' => r.push_str("\\n"), // Line feed - '\r' => r.push_str("\\r"), // Carriage return - '\t' => r.push_str("\\t"), // Tab - // Other control characters (e.g., NUL, SOH, VT, etc.) - // that don't have a specific short escape sequence. - _ => { - r.push_str(&format!("\\u{:04x}", c as u32)); - } - } - } - // Other characters are passed through. - // This includes quotes (") and backslashes (\). If these are part of the - // JSON structure (e.g. {"key": "value"}) or part of an already correctly - // escaped sequence within a string value (e.g. "string with \\\" quote"), - // they are preserved as is. This function does not attempt to fix - // malformed quote or backslash usage *within* string values if the LLM - // generates them incorrectly (e.g. {"key": "unescaped " quote in string"}). - _ => r.push(c), - } - } - r -} - #[cfg(test)] mod tests { use super::*;