Fix CI and incremental indexing file boundaries

README.md

@@ -60,8 +60,9 @@ The authored evaluation benchmark is located in `evaluation/questions.json`, wit
 ## Deliberate boundaries & non-claims
 
 - **No In-Binary Hybrid or Vector Search**: Vector embeddings and hybrid search algorithms exist only in the offline Python script `evaluation/evaluate_hybrid.py`. They are **not** integrated into the Rust product binary, library, or HTTP API.
-- **No Runtime Citation Verification**: In `--answer` mode, the prompt asks the external model to cite lines. The Rust product binary does **not** parse, validate, or cryptographically prove citations at runtime; citation presence is checked only as an offline test assertion via `./scripts/llm_answer_smoke.sh`.
-- **No Deep Binary Detection**: File filtering relies strictly on path names (`.git`, `target`, `node_modules`) and file extension matching. General byte-level content sniffing, null-byte scanning, or MIME detection are not implemented.
+- **Heuristic Citation Screening, Not Answer Verification**: The CLI screens some plain `path:line` tokens against retrieved text. This is not a complete citation parser and does not prove that an answer is supported by its sources. The lexical evaluator reports retrieval Recall@5 and MRR only, not answer or citation accuracy.
+- **File Access Policy**: Initial scans and incremental updates exclude hidden path components, common build/dependency directories, `.pem`/`.key` files, and named SSH private keys. Incremental updates reject absolute/parent paths and symlink components. This name-based policy is not a secret scanner and does not defend against a hostile concurrent filesystem mutation.
+- **Working-Tree Prototype**: Files are read from the working tree, not an immutable Git snapshot. A `-dirty` label is diagnostic, not a reproducible snapshot identifier. Commit-consistent indexing and complete rename handling remain unfinished.
 - **No Background Watcher**: Background filesystem events are not monitored; synchronization is triggered explicitly via CLI or HTTP `/reload`.
 - **No Production Hardening**: Authentication, TLS, rate limiting, and multi-tenant isolation remain out of scope for this local development prototype.
 
@@ -70,4 +71,3 @@ The authored evaluation benchmark is located in `evaluation/questions.json`, wit
 MIT. The evaluation corpus in this repository is authored specifically for this project.
 
 Architecture and limitations are documented in `docs/architecture.md`; the current baseline is summarized in `RELEASE_NOTES.md`. Run `python3 scripts/benchmark.py` for the local fixed-corpus latency measurement.
-

evaluation/evaluate.py

@@ -22,7 +22,7 @@ def main() -> int:
             ranks.append(rank)
     recall = len(ranks) / len(questions)
     mrr = sum(1 / rank for rank in ranks) / len(questions)
-    print(json.dumps({"questions": len(questions), "recall_at_5": recall, "mrr": mrr, "citation_file_accuracy": recall}, indent=2))
+    print(json.dumps({"questions": len(questions), "recall_at_5": recall, "mrr": mrr}, indent=2))
     return 0
 
 

src/lib.rs

@@ -82,6 +82,19 @@ impl Index {
 
     pub fn update_file(&mut self, root: &Path, relative: &Path) -> io::Result<()> {
         self.remove_file(relative);
+        if !is_safe_relative(relative) {
+            return Ok(());
+        }
+        let mut checked = root.to_path_buf();
+        for component in relative.components() {
+            checked.push(component);
+            match fs::symlink_metadata(&checked) {
+                Ok(metadata) if metadata.file_type().is_symlink() => return Ok(()),
+                Ok(_) => {}
+                Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
+                Err(error) => return Err(error),
+            }
+        }
         let path = root.join(relative);
         if path.is_file() && is_indexable(relative) {
             self.add_file(relative.to_path_buf(), fs::read_to_string(path)?);
@@ -106,7 +119,7 @@ impl Index {
             .iter()
             .map(|(k, v)| (k.clone(), v.len()))
             .collect();
-        top_terms.sort_by(|a, b| b.1.cmp(&a.1));
+        top_terms.sort_by_key(|(term, count)| (std::cmp::Reverse(*count), term.clone()));
         top_terms.truncate(5);
         let top_terms_str = top_terms
             .iter()
@@ -159,7 +172,7 @@ impl Index {
         for entry in fs::read_dir(dir)? {
             let entry = entry?;
             let path = entry.path();
-            
+
             // Skip symlinks
             if let Ok(meta) = fs::symlink_metadata(&path) {
                 if meta.file_type().is_symlink() {
@@ -168,24 +181,38 @@ impl Index {
             }
 
             let rel = path.strip_prefix(root).expect("walked under root");
+            if !is_safe_relative(rel) {
+                continue;
+            }
             let file_name = rel.file_name().and_then(|n| n.to_str()).unwrap_or("");
-            
+
             if path.is_dir() {
                 // Sensitive dirs & typical ignores
-                if file_name.starts_with('.') || matches!(
-                    file_name,
-                    "target" | "node_modules" | "build" | "dist"
-                ) {
+                if file_name.starts_with('.')
+                    || matches!(file_name, "target" | "node_modules" | "build" | "dist")
+                {
                     continue;
                 }
                 self.walk(root, &path)?;
             } else if is_indexable(rel) {
                 // Sensitive files
-                if (file_name.starts_with('.') && file_name != ".github" && file_name != ".gitignore") || 
-                   file_name.ends_with(".pem") || file_name.ends_with(".key") || file_name == "id_rsa" ||
-                   file_name.ends_with(".p12") || file_name.ends_with(".pfx") || file_name.ends_with(".keystore") ||
-                   file_name == "credentials.json" || file_name == "service-account.json" || file_name == ".npmrc" || file_name == ".netrc" ||
-                   file_name == ".env" || file_name.starts_with(".env.") || file_name == "id_ed25519" {
+                if (file_name.starts_with('.')
+                    && file_name != ".github"
+                    && file_name != ".gitignore")
+                    || file_name.ends_with(".pem")
+                    || file_name.ends_with(".key")
+                    || file_name == "id_rsa"
+                    || file_name.ends_with(".p12")
+                    || file_name.ends_with(".pfx")
+                    || file_name.ends_with(".keystore")
+                    || file_name == "credentials.json"
+                    || file_name == "service-account.json"
+                    || file_name == ".npmrc"
+                    || file_name == ".netrc"
+                    || file_name == ".env"
+                    || file_name.starts_with(".env.")
+                    || file_name == "id_ed25519"
+                {
                     continue;
                 }
                 self.add_file(rel.to_path_buf(), fs::read_to_string(path)?);
@@ -217,10 +244,15 @@ fn git_revision(root: &Path) -> Option<String> {
         return None;
     }
     let mut revision = String::from_utf8(output.stdout).ok()?.trim().to_owned();
-    if revision.is_empty() { return None; }
-
+    if revision.is_empty() {
+        return None;
+    }
+
     // Check if dirty
-    if let Ok(status) = Command::new("git").args(["-C", root.to_str()?, "status", "--porcelain"]).output() {
+    if let Ok(status) = Command::new("git")
+        .args(["-C", root.to_str()?, "status", "--porcelain"])
+        .output()
+    {
         if !status.stdout.is_empty() {
             revision.push_str("-dirty");
         }
@@ -232,6 +264,25 @@ pub fn current_git_revision(root: &Path) -> Option<String> {
     git_revision(root)
 }
 
+fn is_safe_relative(path: &Path) -> bool {
+    !path.as_os_str().is_empty()
+        && path.components().all(|component| {
+            let std::path::Component::Normal(name) = component else {
+                return false;
+            };
+            let Some(name) = name.to_str() else {
+                return false;
+            };
+            !name.starts_with('.')
+                && !matches!(
+                    name,
+                    "target" | "node_modules" | "build" | "dist" | "id_rsa" | "id_ed25519"
+                )
+                && !name.ends_with(".pem")
+                && !name.ends_with(".key")
+        })
+}
+
 fn is_indexable(path: &Path) -> bool {
     !matches!(
         path.extension().and_then(|x| x.to_str()),
@@ -324,4 +375,36 @@ mod tests {
         assert!(stats.contains(r#""files": 1"#));
         assert!(stats.contains(r#""lines": 1"#));
     }
+
+    #[test]
+    fn incremental_updates_reject_sensitive_and_outside_paths() {
+        let root = fixture();
+        let mut index = Index::build(&root).unwrap();
+        for name in [".env", "private.key", "id_ed25519"] {
+            fs::write(root.join(name), "sensitivecanary").unwrap();
+            index.update_file(&root, Path::new(name)).unwrap();
+        }
+        index
+            .update_file(&root, Path::new("../outside.rs"))
+            .unwrap();
+        assert!(index.search("sensitivecanary", 5).is_empty());
+        assert!(Index::build(&root)
+            .unwrap()
+            .search("sensitivecanary", 5)
+            .is_empty());
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn incremental_updates_skip_symlink_parents() {
+        let root = fixture();
+        let outside = fixture();
+        fs::write(outside.join("secret.rs"), "outsidecanary").unwrap();
+        std::os::unix::fs::symlink(&outside, root.join("linked")).unwrap();
+        let mut index = Index::build(&root).unwrap();
+        index
+            .update_file(&root, Path::new("linked/secret.rs"))
+            .unwrap();
+        assert!(index.search("outsidecanary", 5).is_empty());
+    }
 }

src/main.rs

@@ -35,7 +35,7 @@ fn main() {
         let mut answer = provider
             .answer(&question, &evidence)
             .expect("run LLM provider");
-            
+
         // Runtime citation validation
         let _evidence_lines: Vec<&str> = evidence.lines().collect();
         let mut verified_text = String::new();
@@ -46,25 +46,37 @@ fn main() {
             let mut all_citations_valid = true;
             for word in line.split_whitespace() {
                 // If the word looks like a citation [path:line]
-                let cleaned = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '.' && c != ':' && c != '/' && c != '_' && c != '-');
+                let cleaned = word.trim_matches(|c: char| {
+                    !c.is_alphanumeric() && c != '.' && c != ':' && c != '/' && c != '_' && c != '-'
+                });
                 if cleaned.contains(':') && cleaned.chars().filter(|&c| c == ':').count() == 1 {
                     let parts: Vec<&str> = cleaned.split(':').collect();
-                    if parts.len() == 2 && parts[1].parse::<u32>().is_ok() && parts[0].contains('.') {
+                    if parts.len() == 2 && parts[1].parse::<u32>().is_ok() && parts[0].contains('.')
+                    {
                         // It's formatted as path:line. Does evidence contain this exact string bounded by newline or space?
                         // A simple contains is not enough (e.g. src/main.rs:99 matches src/main.rs:999)
                         let target = format!("{}:{}", parts[0], parts[1]);
                         let target_with_space = format!("{} ", target);
                         let target_with_newline = format!("{}\n", target);
-
-                        if !evidence.contains(&target_with_space) && !evidence.contains(&target_with_newline) && !evidence.ends_with(&target) {
+
+                        if !evidence.contains(&target_with_space)
+                            && !evidence.contains(&target_with_newline)
+                            && !evidence.ends_with(&target)
+                        {
                             all_citations_valid = false;
                             break;
                         }
                     }
                 }
             }
             if !all_citations_valid {
-                verified_text.push_str(&format!("{} [WARNING: Unverified citation removed]\n", line.split_whitespace().filter(|w| !w.contains(":")).collect::<Vec<_>>().join(" ")));
+                verified_text.push_str(&format!(
+                    "{} [WARNING: Unverified citation removed]\n",
+                    line.split_whitespace()
+                        .filter(|w| !w.contains(":"))
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                ));
             } else {
                 verified_text.push_str(line);
                 verified_text.push('\n');