diff --git a/src/admin.rs b/src/admin.rs index 2a3c7ee..0dc6db0 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -660,7 +660,7 @@ impl AdminConfig { if !path.exists() { return Ok(Self::default()); } - let text = std::fs::read_to_string(&path) + let text = crate::text::read_authored(&path) .map_err(|e| format!("could not read {}: {e}", path.display()))?; if text.trim().is_empty() { return Ok(Self::default()); @@ -1490,6 +1490,21 @@ mod tests { assert!(error.contains("not valid admin settings"), "got: {error}"); } + /// **Settings saved with a byte-order mark load**, rather than being + /// refused as "not valid admin settings": Windows PowerShell 5.1 writes + /// the mark with `Set-Content -Encoding utf8`. + #[test] + fn a_file_with_a_byte_order_mark_loads() { + let dir = tempfile::tempdir().expect("temp dir"); + std::fs::write( + AdminConfig::path_in(dir.path()), + b"\xEF\xBB\xBF{\"notify\": \"bell\"}", + ) + .expect("write"); + let admin = AdminConfig::load(dir.path()).expect("loads"); + assert_eq!(admin.notify.as_deref(), Some("bell")); + } + #[test] fn a_missing_or_empty_file_behaves_as_no_opinion() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/src/brief.rs b/src/brief.rs index a6dbb06..6c64b86 100644 --- a/src/brief.rs +++ b/src/brief.rs @@ -348,7 +348,7 @@ const EXCLUDED_HEADINGS: [&str; 5] = ["non-goal", "non goal", "assumption", "ris /// nobody wrote is worse than none. #[must_use] pub fn from_docs(workspace: &Path) -> BriefUpdate { - let Ok(prd) = std::fs::read_to_string(workspace.join(PRD_PATH)) else { + let Ok(prd) = crate::text::read_authored(workspace.join(PRD_PATH)) else { return BriefUpdate::default(); }; if prd.trim().is_empty() { diff --git a/src/cli/keys.rs b/src/cli/keys.rs index aac6965..ccb579a 100644 --- a/src/cli/keys.rs +++ b/src/cli/keys.rs @@ -186,7 +186,7 @@ impl Keymap { /// Read the bindings file in `data_dir`; empty when there is none. #[must_use] pub fn load(data_dir: &std::path::Path) -> (Self, Vec) { - match std::fs::read_to_string(data_dir.join(KEYBINDINGS_FILE)) { + match crate::text::read_authored(data_dir.join(KEYBINDINGS_FILE)) { Ok(text) => Self::parse(&text), Err(_) => (Self::default(), Vec::new()), } @@ -534,6 +534,24 @@ mod tests { assert!(problems[1].contains("line 6")); } + /// **The first binding counts when the file starts with a byte-order + /// mark**; it was reported as an unknown key and dropped. + #[test] + fn a_byte_order_mark_does_not_spoil_the_first_binding() { + let dir = tempfile::tempdir().expect("dir"); + std::fs::write( + dir.path().join(KEYBINDINGS_FILE), + b"\xEF\xBB\xBF\"ctrl-l\" = \"clear-line\"\r\n", + ) + .expect("write"); + let (map, problems) = Keymap::load(dir.path()); + assert!(problems.is_empty(), "{problems:?}"); + assert_eq!( + map.lookup(KeyCode::Char('l'), KeyModifiers::CONTROL), + Some(Action::ClearLine) + ); + } + #[test] fn a_missing_bindings_file_is_no_bindings() { let (map, problems) = Keymap::load(std::path::Path::new("/definitely/not/here")); diff --git a/src/contracts.rs b/src/contracts.rs index 2c55f57..c618dbc 100644 --- a/src/contracts.rs +++ b/src/contracts.rs @@ -168,7 +168,7 @@ impl Contracts { if !path.exists() { return Ok(Self::default()); } - let text = std::fs::read_to_string(&path) + let text = crate::text::read_authored(&path) .map_err(|e| format!("could not read {}: {e}", path.display()))?; if text.trim().is_empty() { return Ok(Self::default()); diff --git a/src/custom_commands.rs b/src/custom_commands.rs index 3a4325e..8cc7f67 100644 --- a/src/custom_commands.rs +++ b/src/custom_commands.rs @@ -446,7 +446,7 @@ fn read_command(path: &Path, name: String) -> Result { path.display() )); } - let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let text = crate::text::read_authored(path).map_err(|e| format!("{}: {e}", path.display()))?; let mut command = parse(&text).map_err(|e| format!("{}: {e}", path.display()))?; command.name = name; command.path = path.to_path_buf(); @@ -765,6 +765,25 @@ mod tests { assert_eq!(c.body, "Test $1"); } + /// **A command file that starts with a byte-order mark keeps its + /// `allowed-tools`.** The mark hid the opening `---`, so the front matter + /// was taken as body and the command ran with every tool, saying nothing. + #[test] + fn a_byte_order_mark_does_not_hide_the_front_matter() { + let dir = tempfile::tempdir().expect("dir"); + std::fs::write( + dir.path().join("review.md"), + b"\xEF\xBB\xBF---\r\ndescription: Read only\r\nallowed-tools: Read\r\n---\r\nLook at $1\r\n", + ) + .expect("write"); + let (found, problems) = scan_dir(dir.path(), None); + assert!(problems.is_empty(), "{problems:?}"); + assert_eq!(found.len(), 1); + assert_eq!(found[0].allowed_tools, vec!["Read"]); + assert_eq!(found[0].description, "Read only"); + assert_eq!(found[0].body, "Look at $1"); + } + #[test] fn a_hint_in_brackets_alone_is_kept_literally() { let c = parse("---\nargument-hint: [file]\nallowed-tools: [Read, Grep]\n---\nx") diff --git a/src/folders.rs b/src/folders.rs index b29f5be..64b96a0 100644 --- a/src/folders.rs +++ b/src/folders.rs @@ -121,7 +121,7 @@ pub fn ensure_gitignore(incode_dir: &Path) -> std::io::Result { Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} Err(e) => return Err(e), } - let text = std::fs::read_to_string(&path)?; + let text = crate::text::read_authored(&path)?; let missing = missing_from(&text); if missing.is_empty() { return Ok(false); diff --git a/src/goal.rs b/src/goal.rs index a857795..f3e3111 100644 --- a/src/goal.rs +++ b/src/goal.rs @@ -126,7 +126,7 @@ impl Standing { /// over it would make this feature a gate rather than a report. #[must_use] pub fn checks_from_docs(workspace: &Path) -> Vec { - let Ok(text) = std::fs::read_to_string(workspace.join(VERIFICATION_PATH)) else { + let Ok(text) = crate::text::read_authored(workspace.join(VERIFICATION_PATH)) else { return Vec::new(); }; parse_checks(&text) @@ -469,6 +469,25 @@ mod tests { ); } + /// **A check on the first line counts when the document starts with a + /// byte-order mark**; the mark kept the line from reading as a bullet, so + /// the check was silently not run. + #[test] + fn a_byte_order_mark_does_not_hide_the_first_check() { + let dir = tempfile::tempdir().expect("dir"); + std::fs::create_dir_all(dir.path().join("docs")).expect("mkdir"); + std::fs::write( + dir.path().join(VERIFICATION_PATH), + "\u{feff}- **b1 builds** — `make`\r\n- **b2 tests** — `make test`\r\n", + ) + .expect("write"); + let ids: Vec = checks_from_docs(dir.path()) + .into_iter() + .map(|c| c.roadmap_id) + .collect(); + assert_eq!(ids, vec!["b1", "b2"]); + } + /// The three shapes a single backticked token can take. #[test] fn a_bare_word_runs_an_explicit_path_runs_a_filename_does_not() { diff --git a/src/guards.rs b/src/guards.rs index aa08785..667d602 100644 --- a/src/guards.rs +++ b/src/guards.rs @@ -124,7 +124,7 @@ impl Guards { if !path.exists() { return Ok(Self::default()); } - let text = std::fs::read_to_string(&path) + let text = crate::text::read_authored(&path) .map_err(|e| format!("could not read {}: {e}", path.display()))?; if text.trim().is_empty() { return Ok(Self::default()); diff --git a/src/harness.rs b/src/harness.rs index a1df243..6c3db5d 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -57,7 +57,7 @@ pub struct Routing { /// The routing in `dir`; `Ok(None)` when it has no routing file. pub fn routing_in(dir: &Path) -> Result, String> { let path = dir.join(ROUTING_FILE); - match std::fs::read_to_string(&path) { + match crate::text::read_authored(&path) { Ok(text) => serde_json::from_str(&text) .map(Some) .map_err(|e| format!("{}: {e}", path.display())), diff --git a/src/mcp_config.rs b/src/mcp_config.rs index e61b351..6c5204b 100644 --- a/src/mcp_config.rs +++ b/src/mcp_config.rs @@ -189,7 +189,7 @@ pub fn parse(text: &str) -> Result { /// /// It could not be read, or is not valid. pub fn read(path: &Path) -> Result { - match std::fs::read_to_string(path) { + match crate::text::read_authored(path) { Ok(text) => parse(&text).map_err(|e| format!("{} is not valid: {e}", path.display())), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(McpFile::default()), Err(e) => Err(format!("could not read {}: {e}", path.display())), diff --git a/src/mcp_config_tests.rs b/src/mcp_config_tests.rs index 2f1d935..d81dea2 100644 --- a/src/mcp_config_tests.rs +++ b/src/mcp_config_tests.rs @@ -43,6 +43,20 @@ fn a_dot_mcp_json_parses_stdio_and_http_servers() { assert!(parse("{\"mcpServers\": []}").is_err()); } +/// **A `.mcp.json` saved with a byte-order mark is read**, not refused as +/// "not valid": its servers are all there. +#[test] +fn a_file_with_a_byte_order_mark_is_read() { + let dir = tempfile::tempdir().expect("dir"); + let path = dir.path().join(".mcp.json"); + let mut bytes = b"\xEF\xBB\xBF".to_vec(); + bytes.extend_from_slice(SAMPLE.as_bytes()); + std::fs::write(&path, bytes).expect("write"); + let file = read(&path).expect("reads"); + assert_eq!(file.servers.len(), 4); + assert_eq!(file.servers["github"].command.as_deref(), Some("npx")); +} + #[test] fn entries_become_specs_with_variables_expanded() { let file = parse(SAMPLE).expect("parses"); diff --git a/src/permissions/managed.rs b/src/permissions/managed.rs index 619eb87..b174881 100644 --- a/src/permissions/managed.rs +++ b/src/permissions/managed.rs @@ -206,7 +206,7 @@ pub fn load() -> Managed { return Managed::Absent; } let path = managed_path(); - match std::fs::read_to_string(&path) { + match crate::text::read_authored(&path) { Ok(text) => match parse(&text) { Ok(settings) => Managed::Loaded(settings), Err(why) => Managed::Broken(format!("{}: {why}", path.display())), diff --git a/src/permissions/rules/set.rs b/src/permissions/rules/set.rs index 993fb6b..4cb4a7c 100644 --- a/src/permissions/rules/set.rs +++ b/src/permissions/rules/set.rs @@ -318,7 +318,7 @@ fn read_project(workspace: &Path) -> Result, String> { permissions: Option, } let path = workspace.join(PROJECT_SETTINGS); - let text = match std::fs::read_to_string(&path) { + let text = match crate::text::read_authored(&path) { Ok(text) => text, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(e) => return Err(format!("could not read {}: {e}", path.display())), diff --git a/src/permissions/rules/tests/layers.rs b/src/permissions/rules/tests/layers.rs index f6cb4b9..f9c249e 100644 --- a/src/permissions/rules/tests/layers.rs +++ b/src/permissions/rules/tests/layers.rs @@ -91,6 +91,32 @@ fn project_settings_are_read_from_the_workspace() { .is_some()); } +/// **A project's deny rule still denies when its settings file starts with a +/// byte-order mark**, as Windows PowerShell 5.1 writes it. Unread, the file +/// was a warning and the rule was simply not there. +#[test] +fn project_settings_with_a_byte_order_mark_still_deny() { + let dir = tempfile::tempdir().expect("dir"); + std::fs::create_dir_all(dir.path().join(".incode")).expect("mkdir"); + std::fs::write( + dir.path().join(PROJECT_SETTINGS), + b"\xEF\xBB\xBF{\"permissions\": {\"deny\": [\"Read(.env)\"]}}", + ) + .expect("write"); + let rules = PermissionRules::load(dir.path()); + assert!(rules.warnings().is_empty(), "{:?}", rules.warnings()); + let ws = PathContext { + workspace: dir.path().to_path_buf(), + home: None, + }; + assert_eq!( + rules + .decide("Read", &Subject::Paths(vec![".env".to_string()]), &ws) + .map(|v| v.effect), + Some(Effect::Deny), + ); +} + // --- a turn's allowlist --------------------------------------------------- fn listed(texts: &[&str]) -> Allowlist { diff --git a/src/plugins.rs b/src/plugins.rs index 5690a0f..ca380c8 100644 --- a/src/plugins.rs +++ b/src/plugins.rs @@ -200,7 +200,7 @@ pub fn discover_in(project: Option<&Path>, user: &Path) -> Found { /// One plugin folder. fn load_one(root: &Path, scope: Scope) -> Result { let path = root.join(MANIFEST); - let text = std::fs::read_to_string(&path).map_err(|e| { + let text = crate::text::read_authored(&path).map_err(|e| { format!( "plugin at {} has no readable {MANIFEST}: {e}", root.display() @@ -318,7 +318,7 @@ impl Plugin { .file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default(); - let text = std::fs::read_to_string(dir.join("SKILL.md")).unwrap_or_default(); + let text = crate::text::read_authored(dir.join("SKILL.md")).unwrap_or_default(); let description = crate::subagents::split_front_matter(&text) .and_then(|(front, _)| crate::subagents::parse_fields(front).ok()) .and_then(|fields| { @@ -341,7 +341,7 @@ impl Plugin { /// Its hook commands, with [`ROOT_VAR`] filled in. pub fn hooks(&self) -> Result { let path = self.root.join("hooks.json"); - let Ok(text) = std::fs::read_to_string(&path) else { + let Ok(text) = crate::text::read_authored(&path) else { return Ok(crate::hooks::HookConfig::default()); }; let mut config: crate::hooks::HookConfig = @@ -366,7 +366,7 @@ impl Plugin { return Ok(Vec::new()); }; let text = - std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?; + crate::text::read_authored(&path).map_err(|e| format!("{}: {e}", path.display()))?; let value: serde_json::Value = serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; let Some(servers) = value @@ -973,6 +973,25 @@ mod tests { assert_eq!(m.description, "say \"hi\""); } + /// **A manifest saved with a byte-order mark installs.** The mark kept + /// the first key from reading as `name`, and the plugin was refused with + /// "`name` is missing" (found by hand under #22). + #[test] + fn a_manifest_with_a_byte_order_mark_installs() { + let dir = tempfile::tempdir().expect("dir"); + let source = dir.path().join("src").join("demo"); + std::fs::create_dir_all(&source).expect("mkdir"); + std::fs::write( + source.join(MANIFEST), + b"\xEF\xBB\xBFname = \"demo\"\r\nversion = \"1.0.0\"\r\n", + ) + .expect("write"); + let target = dir.path().join("plugins"); + let added = add(&source.to_string_lossy(), &target).expect("installs"); + assert_eq!(added.manifest.name, "demo"); + assert_eq!(added.manifest.version, "1.0.0"); + } + #[test] fn a_manifest_without_a_usable_name_is_refused() { assert!(parse_manifest("version = \"1\"\n").is_err()); diff --git a/src/research.rs b/src/research.rs index 3fa3d5b..23e3a7d 100644 --- a/src/research.rs +++ b/src/research.rs @@ -126,7 +126,7 @@ pub fn declared_dependencies(workspace: &Path) -> Vec { if !path.is_file() { continue; } - if let Ok(text) = std::fs::read_to_string(&path) { + if let Ok(text) = crate::text::read_authored(&path) { add((kind.parse)(&text)); } } @@ -141,7 +141,7 @@ pub fn declared_dependencies(workspace: &Path) -> Vec { .and_then(|e| e.to_str()) .is_some_and(|e| matches!(e, "csproj" | "fsproj" | "vbproj")); if is_project { - if let Ok(text) = std::fs::read_to_string(&path) { + if let Ok(text) = crate::text::read_authored(&path) { add(dotnet_dependencies(&text)); } } diff --git a/src/runtime/lifecycle.rs b/src/runtime/lifecycle.rs index 8835e45..9c7a2d0 100644 --- a/src/runtime/lifecycle.rs +++ b/src/runtime/lifecycle.rs @@ -87,7 +87,7 @@ impl LifecycleHooks { /// Load from a JSON file (e.g. `.incode/lifecycle.json`). pub fn load_from_file(path: &Path) -> Option { - let content = std::fs::read_to_string(path).ok()?; + let content = crate::text::read_authored(path).ok()?; serde_json::from_str(&content).ok() } @@ -104,7 +104,7 @@ impl LifecycleHooks { // Try workspace/.incode.json (lifecycle section) let config_path = workspace.join(".incode.json"); - if let Ok(content) = std::fs::read_to_string(&config_path) { + if let Ok(content) = crate::text::read_authored(&config_path) { if let Ok(config) = serde_json::from_str::(&content) { if let Some(lifecycle) = config.get("lifecycle") { if let Ok(hooks) = serde_json::from_value::(lifecycle.clone()) { diff --git a/src/spec.rs b/src/spec.rs index f685bba..5cba4c6 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -698,7 +698,7 @@ pub fn exists(workspace: &Path, file: &SpecFile) -> bool { /// The document's text, when it is written: present, not blank, and — for /// JSON — parseable. A corrupted JSON file does not satisfy the spec contract. fn written(workspace: &Path, file: &SpecFile) -> Option { - let text = std::fs::read_to_string(workspace.join(file.path)).ok()?; + let text = crate::text::read_authored(workspace.join(file.path)).ok()?; if text.trim().is_empty() { return None; } diff --git a/src/styles.rs b/src/styles.rs index 9562a50..8fe278b 100644 --- a/src/styles.rs +++ b/src/styles.rs @@ -188,7 +188,7 @@ fn read_style(path: &Path, name: &str, origin: Origin) -> Result "{size} bytes, over the {MAX_FILE_BYTES}-byte limit" )); } - let text = std::fs::read_to_string(path).map_err(|e| e.to_string())?; + let text = crate::text::read_authored(path).map_err(|e| e.to_string())?; let (description, body) = match crate::subagents::split_front_matter(&text) { Some((front, body)) => { let mut description = None; diff --git a/src/text.rs b/src/text.rs index 3cc3221..c6f0931 100644 --- a/src/text.rs +++ b/src/text.rs @@ -10,6 +10,44 @@ //! These helpers cut on a character boundary instead. Use [`truncate`] when the //! limit is about how much a human should read, and [`head_bytes`] when it is //! about how many bytes may be sent onward. +//! +//! [`read_authored`] is the reading side of the same concern: text a person +//! wrote by hand arrives with whatever their editor put in front of it. + +/// The byte-order mark some Windows tools write at the start of a UTF-8 file. +const BOM: char = '\u{feff}'; + +/// A file a person writes by hand (a setting, a manifest, a definition), as +/// text with any leading byte-order mark dropped. +/// +/// Windows PowerShell 5.1's `Set-Content -Encoding utf8` and some editors +/// start a UTF-8 file with U+FEFF. `str::trim` keeps it, and serde_json and +/// the toml crate refuse it, so without this a hand-written file is rejected +/// or, worse, half-read: a command's front matter stopped being recognised +/// and its `allowed-tools` were never applied. +/// +/// Errors are `std::fs::read_to_string`'s, unchanged, so a caller's +/// `NotFound` handling still works. +/// +/// Wired to: every reader of a file a person authors — admin.rs (admin.json), +/// mcp_config.rs (mcp.json), plugins.rs (plugin.toml, hooks.json, mcp.json, +/// SKILL.md), custom_commands.rs, styles.rs, cli/keys.rs, guards.rs, +/// contracts.rs, permissions/rules/set.rs (settings.json), +/// permissions/managed.rs, runtime/lifecycle.rs, harness.rs, spec.rs, goal.rs, +/// brief.rs, folders.rs (.gitignore), research.rs (project manifests). +/// Sibling: subagents.rs `parse` strips the mark from text it is handed. +/// Not for files incode writes itself, nor for content passed through as is. +/// +/// # Errors +/// +/// The file could not be read, or is not UTF-8. +pub fn read_authored(path: impl AsRef) -> std::io::Result { + let mut text = std::fs::read_to_string(path)?; + if text.starts_with(BOM) { + text.drain(..BOM.len_utf8()); + } + Ok(text) +} /// Longest prefix of `s` holding at most `max_chars` characters. #[must_use] @@ -234,6 +272,30 @@ mod tests { assert!(!reply.is_empty()); } + /// **A hand-written file reads the same with or without a byte-order + /// mark**, and only a LEADING mark goes: one inside the text is the + /// person's own and stays. + #[test] + fn a_leading_byte_order_mark_is_dropped_and_nothing_else() { + let dir = tempfile::tempdir().unwrap(); + let with = dir.path().join("with.json"); + let without = dir.path().join("without.json"); + std::fs::write(&with, b"\xEF\xBB\xBF{\"a\": \"x\xEF\xBB\xBFy\"}").unwrap(); + std::fs::write(&without, b"{\"a\": \"x\xEF\xBB\xBFy\"}").unwrap(); + let read = read_authored(&with).unwrap(); + assert_eq!(read, read_authored(&without).unwrap()); + assert_eq!(read, "{\"a\": \"x\u{feff}y\"}"); + assert!(serde_json::from_str::(&read).is_ok()); + } + + /// A missing file is still `NotFound`, which callers treat as "none". + #[test] + fn a_missing_authored_file_is_not_found() { + let dir = tempfile::tempdir().unwrap(); + let err = read_authored(dir.path().join("absent.json")).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); + } + #[test] fn the_greeting_reply_names_a_workspace_with_no_file_name() { let reply = greeting_reply(std::path::Path::new("/"));