diff --git a/src/languages/go.rs b/src/languages/go.rs index ccb0e42..2b00940 100644 --- a/src/languages/go.rs +++ b/src/languages/go.rs @@ -31,12 +31,7 @@ impl LanguageParser for GoParser { return ParserRecognition::NotRecognized; } - let window = file.first_content_window(); - if window.content_kind == ContentKind::Text && !window.truncated { - ParserRecognition::Recognized - } else { - ParserRecognition::NotRecognized - } + ParserRecognition::Recognized } fn parse(&self, file: &AnalyzedFile) -> ParserOutput { @@ -62,6 +57,17 @@ impl LanguageParser for GoParser { } }; + if window.content_kind != ContentKind::Text { + output.diagnostics.push(ParserDiagnostic { + code: "unsupported_content_kind".to_owned(), + message: format!( + "Go parser requires text source, found {}", + content_kind_label(window.content_kind) + ), + }); + return output; + } + let mut parser = tree_sitter::Parser::new(); if let Err(source) = parser.set_language(&tree_sitter_go::LANGUAGE.into()) { output.diagnostics.push(ParserDiagnostic { @@ -98,6 +104,14 @@ fn has_go_extension(path: &Path) -> bool { .is_some_and(|extension| extension.eq_ignore_ascii_case("go")) } +fn content_kind_label(kind: ContentKind) -> &'static str { + match kind { + ContentKind::Text => "text", + ContentKind::Binary => "binary", + ContentKind::Unknown => "unknown", + } +} + fn empty_output(language_id: &str) -> ParserOutput { ParserOutput { language_id: language_id.to_owned(), diff --git a/src/pipeline/file_analyzer.rs b/src/pipeline/file_analyzer.rs index d69aaa8..40ee443 100644 --- a/src/pipeline/file_analyzer.rs +++ b/src/pipeline/file_analyzer.rs @@ -37,6 +37,7 @@ impl FileAnalyzer { let window = file.first_content_window(); let mut diagnostics = window.diagnostics.clone(); let parser = self.parse(&file); + append_parser_notes(&mut diagnostics, parser.output.as_ref()); let line_count = line_count_from_window(&window, &mut diagnostics); let parser_metrics = parser.output.as_ref().map(parser_metrics); @@ -306,6 +307,21 @@ struct ParserMetrics { max_function_complexity_pressure: u64, } +fn append_parser_notes(diagnostics: &mut Vec, output: Option<&ParserOutput>) { + let Some(output) = output else { + return; + }; + + diagnostics.extend(output.diagnostics.iter().map(|diagnostic| FileDiagnostic { + code: diagnostic.code.clone(), + message: diagnostic.message.clone(), + })); + diagnostics.extend(output.limitations.iter().map(|limitation| FileDiagnostic { + code: limitation.code.clone(), + message: limitation.message.clone(), + })); +} + fn parser_metrics(output: &ParserOutput) -> ParserMetrics { let complexity = CodeMetricsAnalyzer::new().analyze(&output.metrics_input); @@ -669,6 +685,48 @@ mod tests { .any(|diagnostic| diagnostic.code == "line_count_skipped")); } + #[test] + fn default_go_parser_surfaces_parser_diagnostics_in_file_result() { + let fixture = Fixture::new("go-parser-diagnostics"); + let invalid_path = fixture.write("bad.go", &[b'a', 0xff]); + let truncated_path = fixture.write("large.go", b"package main\nfunc main() {}\n"); + let invalid_result = FileAnalyzer::new().analyze(FileAnalysisInput { path: invalid_path }); + let truncated_result = FileAnalyzer::with_options(FileAnalyzerOptions { + content_window_bytes: 8, + ..FileAnalyzerOptions::default() + }) + .analyze(FileAnalysisInput { + path: truncated_path, + }); + + assert_eq!(invalid_result.parser_status, FileParserStatus::Parsed); + assert_eq!(invalid_result.language_id.as_deref(), Some("go")); + assert!(invalid_result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "invalid_utf8")); + assert!(invalid_result + .parser_output + .as_ref() + .is_some_and(|output| output + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "invalid_utf8"))); + + assert_eq!(truncated_result.language_id.as_deref(), Some("go")); + assert!(truncated_result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "truncated_source")); + assert!(truncated_result + .parser_output + .as_ref() + .is_some_and(|output| output + .limitations + .iter() + .any(|limitation| limitation.code == "truncated_source"))); + } + #[test] fn analyzer_classifies_generated_and_vendor_files() { let fixture = Fixture::new("path-classification"); diff --git a/src/pipeline/store_reducer.rs b/src/pipeline/store_reducer.rs index 4a451dc..ae02462 100644 --- a/src/pipeline/store_reducer.rs +++ b/src/pipeline/store_reducer.rs @@ -13,9 +13,7 @@ use serde_json::json; use std::collections::{BTreeMap, BTreeSet}; use crate::pipeline::events::PipelineEvent; -use crate::pipeline::file_analyzer::{ - ContentKind, FileAnalysisResult, FileDiagnostic, FileParserStatus, -}; +use crate::pipeline::file_analyzer::{ContentKind, FileAnalysisResult, FileParserStatus}; use crate::pipeline::file_risk_assessor::{ FileRiskAssessment, FileRiskAssessor, FileRiskInput, RepositoryRiskContext, FORMULA_ID, }; @@ -1365,7 +1363,7 @@ fn flush_batch( result .max_function_complexity_pressure .map(|value| value as i64), - diagnostics_json(&result.diagnostics), + diagnostics_json(result), ]) .map_err(StoreReducerError::WriteDatabase)?; } @@ -3283,8 +3281,9 @@ fn parser_status_name(status: FileParserStatus) -> &'static str { } } -fn diagnostics_json(diagnostics: &[FileDiagnostic]) -> String { - let diagnostics: Vec<_> = diagnostics +fn diagnostics_json(result: &FileAnalysisResult) -> String { + let mut diagnostics: Vec<_> = result + .diagnostics .iter() .map(|diagnostic| { json!({ @@ -3293,9 +3292,35 @@ fn diagnostics_json(diagnostics: &[FileDiagnostic]) -> String { }) }) .collect(); + + if let Some(parser_output) = result.parser_output.as_ref() { + for diagnostic in &parser_output.diagnostics { + push_diagnostic_json(&mut diagnostics, &diagnostic.code, &diagnostic.message); + } + for limitation in &parser_output.limitations { + push_diagnostic_json(&mut diagnostics, &limitation.code, &limitation.message); + } + } + serde_json::to_string(&diagnostics).unwrap_or_else(|_| "[]".to_owned()) } +fn push_diagnostic_json(diagnostics: &mut Vec, code: &str, message: &str) { + let exists = diagnostics.iter().any(|diagnostic| { + diagnostic.get("code").and_then(serde_json::Value::as_str) == Some(code) + && diagnostic + .get("message") + .and_then(serde_json::Value::as_str) + == Some(message) + }); + if !exists { + diagnostics.push(json!({ + "code": code, + "message": message, + })); + } +} + fn bool_to_i64(value: bool) -> i64 { if value { 1 @@ -3316,7 +3341,10 @@ mod tests { use super::{ GitRepositorySummaryInput, StoreReducer, StoreReducerOptions, DEFAULT_STORE_QUEUE_CAPACITY, }; - use crate::languages::{ParserOutput, UniversalCodeMetricsInput, UniversalReference}; + use crate::languages::{ + ParserDiagnostic, ParserLimitation, ParserOutput, UniversalCodeMetricsInput, + UniversalReference, + }; use crate::pipeline::events::PipelineEvent; use crate::pipeline::file_analyzer::{ContentKind, FileAnalysisResult, FileParserStatus}; use crate::pipeline::git_history_analyzer::{ @@ -3441,6 +3469,49 @@ mod tests { ))); } + #[test] + fn stores_parser_diagnostics_and_limitations_in_index_diagnostics() { + let fixture = Fixture::new("parser-diagnostics"); + let (event_sender, _event_receiver) = mpsc::channel(); + let reducer = + StoreReducer::start(&fixture.path, StoreReducerOptions::default(), event_sender) + .expect("reducer should start"); + let mut result = file_result("bad.go"); + result.parser_status = FileParserStatus::Parsed; + result.language_id = Some("go".to_owned()); + result.parser_output = Some(ParserOutput { + language_id: "go".to_owned(), + symbols: Vec::new(), + references: Vec::new(), + metrics_input: UniversalCodeMetricsInput::default(), + diagnostics: vec![ParserDiagnostic { + code: "parse_error".to_owned(), + message: "Go source contains syntax errors".to_owned(), + }], + limitations: vec![ParserLimitation { + code: "truncated_source".to_owned(), + message: "Go parser skipped content beyond the active file window".to_owned(), + }], + }); + + reducer + .handle() + .store_file_analysis(result) + .expect("file result should enqueue"); + reducer.finish().expect("reducer should finish"); + + let connection = Connection::open(fixture.db_path()).expect("db should open"); + let diagnostics: String = connection + .query_row( + "SELECT diagnostics FROM file_analysis WHERE relative_path = 'bad.go'", + [], + |row| row.get(0), + ) + .expect("diagnostics should be stored"); + assert!(diagnostics.contains("parse_error")); + assert!(diagnostics.contains("truncated_source")); + } + #[test] fn finish_flushes_partial_batch() { let fixture = Fixture::new("finish-flush"); diff --git a/src/tui.rs b/src/tui.rs index d82a4b4..6965aea 100644 --- a/src/tui.rs +++ b/src/tui.rs @@ -151,6 +151,7 @@ pub struct RiskRow { pub terms: Vec, pub facts: Vec, pub limitations: Vec, + pub parser_diagnostics: Vec, pub owners: Vec, pub tags: Vec, } @@ -749,6 +750,22 @@ fn inspector_lines(row: &RiskRow, width: u16) -> Vec> { lines.push(Line::styled(tags, style(TuiSeverity::Muted))); } + if !row.parser_diagnostics.is_empty() { + lines.push(Line::raw("")); + lines.push(section_divider(width)); + lines.push(Line::raw("")); + lines.push(Line::styled( + "Parser diagnostics", + style(TuiSeverity::Medium).add_modifier(Modifier::BOLD), + )); + for diagnostic in &row.parser_diagnostics { + lines.push(Line::styled( + format!(" - {}: {}", diagnostic.code, diagnostic.message), + style(TuiSeverity::Muted), + )); + } + } + lines.push(Line::raw("")); lines.push(section_divider(width)); lines.extend(risk_driver_lines(row)); @@ -865,7 +882,8 @@ fn load_risk_rows(connection: &Connection) -> rusqlite::Result> { facts.dominant_owner, facts.dominant_owner_share, facts.owner_count, - facts.author_count + facts.author_count, + facts.diagnostics FROM file_risk_scores score LEFT JOIN file_facts facts ON facts.relative_path = score.relative_path @@ -901,6 +919,7 @@ fn load_risk_rows(connection: &Connection) -> rusqlite::Result> { terms: Vec::new(), facts: Vec::new(), limitations: Vec::new(), + parser_diagnostics: parse_diagnostics_json(row.get::<_, String>(24)?.as_str()), owners: Vec::new(), tags: Vec::new(), }) @@ -1017,6 +1036,22 @@ fn load_owners(connection: &Connection) -> rusqlite::Result Vec { + let Ok(items) = serde_json::from_str::>(value) else { + return Vec::new(); + }; + + items + .into_iter() + .filter_map(|item| { + Some(RiskLimitation { + code: item.get("code")?.as_str()?.to_owned(), + message: item.get("message")?.as_str()?.to_owned(), + }) + }) + .collect() +} + fn find_index_root(current_dir: &Path) -> Option { current_dir .ancestors() @@ -1036,6 +1071,9 @@ fn inspector_tags(row: &RiskRow) -> Vec { if row.is_vendor { signals.push(("VENDOR", 1.0, 10)); } + if !row.parser_diagnostics.is_empty() { + signals.push(("PARSER", 1.0, 95)); + } for term in &row.terms { let value = term.normalized_value.unwrap_or_default(); if value < 0.60 { @@ -1661,6 +1699,8 @@ mod tests { assert_eq!(snapshot.rows[0].terms.len(), 2); assert_eq!(snapshot.rows[0].facts.len(), 1); assert_eq!(snapshot.rows[0].limitations.len(), 1); + assert_eq!(snapshot.rows[0].parser_diagnostics.len(), 1); + assert_eq!(snapshot.rows[0].parser_diagnostics[0].code, "parse_error"); assert_eq!(snapshot.rows[0].owners.len(), 1); } @@ -1717,6 +1757,8 @@ mod tests { assert!(output.contains("Inspector")); assert!(output.contains("src/risky.go")); assert!(output.contains("CHURN")); + assert!(output.contains("Parser diagnostics")); + assert!(output.contains("parse_error")); } #[test] @@ -1785,7 +1827,8 @@ mod tests { dominant_owner TEXT, dominant_owner_share REAL, owner_count INTEGER, - author_count INTEGER NOT NULL + author_count INTEGER NOT NULL, + diagnostics TEXT NOT NULL ); CREATE TABLE file_risk_terms ( relative_path TEXT NOT NULL, @@ -1891,7 +1934,8 @@ mod tests { dominant_owner TEXT, dominant_owner_share REAL, owner_count INTEGER, - author_count INTEGER NOT NULL + author_count INTEGER NOT NULL, + diagnostics TEXT NOT NULL ); CREATE TABLE file_risk_terms ( relative_path TEXT NOT NULL, @@ -1931,8 +1975,8 @@ mod tests { ('src/risky.go', 'C:/repo/src/risky.go', 1, 'hotpath.score.go.v1', 1, 0.9, 9.0, 'extreme', 1, 0), ('src/safe.go', 'C:/repo/src/safe.go', 1, 'hotpath.score.go.v1', 2, 0.2, 2.0, 'low', 0, 0); INSERT INTO file_facts VALUES - ('src/risky.go', 1200, 48000, 'go', 220, 45, 12, 8, 20, 2300, 1000, 3, 'Alice ', 0.9, 1, 1), - ('src/safe.go', 10, 400, 'go', 1, 1, 0, 0, 0, 1, 0, 1, NULL, NULL, NULL, 1); + ('src/risky.go', 1200, 48000, 'go', 220, 45, 12, 8, 20, 2300, 1000, 3, 'Alice ', 0.9, 1, 1, '[{\"code\":\"parse_error\",\"message\":\"Go source contains syntax errors\"}]'), + ('src/safe.go', 10, 400, 'go', 1, 1, 0, 0, 0, 1, 0, 1, NULL, NULL, NULL, 1, '[]'); INSERT INTO file_risk_terms VALUES ('src/risky.go', 'hotpath.score.go.v1', 'churn', 2300, 1.0, 0.18, 0.18), ('src/risky.go', 'hotpath.score.go.v1', 'complexity_pressure', 220, 1.0, 0.16, 0.16),