Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/languages/go.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,7 @@ impl LanguageParser for GoParser {
return ParserRecognition::NotRecognized;
}

let window = file.first_content_window();
if window.content_kind == ContentKind::Text && !window.truncated {
ParserRecognition::Recognized
} else {
ParserRecognition::NotRecognized
}
ParserRecognition::Recognized
}

fn parse(&self, file: &AnalyzedFile) -> ParserOutput {
Expand All @@ -62,6 +57,17 @@ impl LanguageParser for GoParser {
}
};

if window.content_kind != ContentKind::Text {
output.diagnostics.push(ParserDiagnostic {
code: "unsupported_content_kind".to_owned(),
message: format!(
"Go parser requires text source, found {}",
content_kind_label(window.content_kind)
),
});
return output;
}

let mut parser = tree_sitter::Parser::new();
if let Err(source) = parser.set_language(&tree_sitter_go::LANGUAGE.into()) {
output.diagnostics.push(ParserDiagnostic {
Expand Down Expand Up @@ -98,6 +104,14 @@ fn has_go_extension(path: &Path) -> bool {
.is_some_and(|extension| extension.eq_ignore_ascii_case("go"))
}

fn content_kind_label(kind: ContentKind) -> &'static str {
match kind {
ContentKind::Text => "text",
ContentKind::Binary => "binary",
ContentKind::Unknown => "unknown",
}
}

fn empty_output(language_id: &str) -> ParserOutput {
ParserOutput {
language_id: language_id.to_owned(),
Expand Down
58 changes: 58 additions & 0 deletions src/pipeline/file_analyzer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ impl FileAnalyzer {
let window = file.first_content_window();
let mut diagnostics = window.diagnostics.clone();
let parser = self.parse(&file);
append_parser_notes(&mut diagnostics, parser.output.as_ref());
let line_count = line_count_from_window(&window, &mut diagnostics);
let parser_metrics = parser.output.as_ref().map(parser_metrics);

Expand Down Expand Up @@ -306,6 +307,21 @@ struct ParserMetrics {
max_function_complexity_pressure: u64,
}

fn append_parser_notes(diagnostics: &mut Vec<FileDiagnostic>, output: Option<&ParserOutput>) {
let Some(output) = output else {
return;
};

diagnostics.extend(output.diagnostics.iter().map(|diagnostic| FileDiagnostic {
code: diagnostic.code.clone(),
message: diagnostic.message.clone(),
}));
diagnostics.extend(output.limitations.iter().map(|limitation| FileDiagnostic {
code: limitation.code.clone(),
message: limitation.message.clone(),
}));
}

fn parser_metrics(output: &ParserOutput) -> ParserMetrics {
let complexity = CodeMetricsAnalyzer::new().analyze(&output.metrics_input);

Expand Down Expand Up @@ -669,6 +685,48 @@ mod tests {
.any(|diagnostic| diagnostic.code == "line_count_skipped"));
}

#[test]
fn default_go_parser_surfaces_parser_diagnostics_in_file_result() {
let fixture = Fixture::new("go-parser-diagnostics");
let invalid_path = fixture.write("bad.go", &[b'a', 0xff]);
let truncated_path = fixture.write("large.go", b"package main\nfunc main() {}\n");
let invalid_result = FileAnalyzer::new().analyze(FileAnalysisInput { path: invalid_path });
let truncated_result = FileAnalyzer::with_options(FileAnalyzerOptions {
content_window_bytes: 8,
..FileAnalyzerOptions::default()
})
.analyze(FileAnalysisInput {
path: truncated_path,
});

assert_eq!(invalid_result.parser_status, FileParserStatus::Parsed);
assert_eq!(invalid_result.language_id.as_deref(), Some("go"));
assert!(invalid_result
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "invalid_utf8"));
assert!(invalid_result
.parser_output
.as_ref()
.is_some_and(|output| output
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "invalid_utf8")));

assert_eq!(truncated_result.language_id.as_deref(), Some("go"));
assert!(truncated_result
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "truncated_source"));
assert!(truncated_result
.parser_output
.as_ref()
.is_some_and(|output| output
.limitations
.iter()
.any(|limitation| limitation.code == "truncated_source")));
}

#[test]
fn analyzer_classifies_generated_and_vendor_files() {
let fixture = Fixture::new("path-classification");
Expand Down
85 changes: 78 additions & 7 deletions src/pipeline/store_reducer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ use serde_json::json;
use std::collections::{BTreeMap, BTreeSet};

use crate::pipeline::events::PipelineEvent;
use crate::pipeline::file_analyzer::{
ContentKind, FileAnalysisResult, FileDiagnostic, FileParserStatus,
};
use crate::pipeline::file_analyzer::{ContentKind, FileAnalysisResult, FileParserStatus};
use crate::pipeline::file_risk_assessor::{
FileRiskAssessment, FileRiskAssessor, FileRiskInput, RepositoryRiskContext, FORMULA_ID,
};
Expand Down Expand Up @@ -1365,7 +1363,7 @@ fn flush_batch(
result
.max_function_complexity_pressure
.map(|value| value as i64),
diagnostics_json(&result.diagnostics),
diagnostics_json(result),
])
.map_err(StoreReducerError::WriteDatabase)?;
}
Expand Down Expand Up @@ -3283,8 +3281,9 @@ fn parser_status_name(status: FileParserStatus) -> &'static str {
}
}

fn diagnostics_json(diagnostics: &[FileDiagnostic]) -> String {
let diagnostics: Vec<_> = diagnostics
fn diagnostics_json(result: &FileAnalysisResult) -> String {
let mut diagnostics: Vec<_> = result
.diagnostics
.iter()
.map(|diagnostic| {
json!({
Expand All @@ -3293,9 +3292,35 @@ fn diagnostics_json(diagnostics: &[FileDiagnostic]) -> String {
})
})
.collect();

if let Some(parser_output) = result.parser_output.as_ref() {
for diagnostic in &parser_output.diagnostics {
push_diagnostic_json(&mut diagnostics, &diagnostic.code, &diagnostic.message);
}
for limitation in &parser_output.limitations {
push_diagnostic_json(&mut diagnostics, &limitation.code, &limitation.message);
}
}

serde_json::to_string(&diagnostics).unwrap_or_else(|_| "[]".to_owned())
}

fn push_diagnostic_json(diagnostics: &mut Vec<serde_json::Value>, code: &str, message: &str) {
let exists = diagnostics.iter().any(|diagnostic| {
diagnostic.get("code").and_then(serde_json::Value::as_str) == Some(code)
&& diagnostic
.get("message")
.and_then(serde_json::Value::as_str)
== Some(message)
});
if !exists {
diagnostics.push(json!({
"code": code,
"message": message,
}));
}
}

fn bool_to_i64(value: bool) -> i64 {
if value {
1
Expand All @@ -3316,7 +3341,10 @@ mod tests {
use super::{
GitRepositorySummaryInput, StoreReducer, StoreReducerOptions, DEFAULT_STORE_QUEUE_CAPACITY,
};
use crate::languages::{ParserOutput, UniversalCodeMetricsInput, UniversalReference};
use crate::languages::{
ParserDiagnostic, ParserLimitation, ParserOutput, UniversalCodeMetricsInput,
UniversalReference,
};
use crate::pipeline::events::PipelineEvent;
use crate::pipeline::file_analyzer::{ContentKind, FileAnalysisResult, FileParserStatus};
use crate::pipeline::git_history_analyzer::{
Expand Down Expand Up @@ -3441,6 +3469,49 @@ mod tests {
)));
}

#[test]
fn stores_parser_diagnostics_and_limitations_in_index_diagnostics() {
let fixture = Fixture::new("parser-diagnostics");
let (event_sender, _event_receiver) = mpsc::channel();
let reducer =
StoreReducer::start(&fixture.path, StoreReducerOptions::default(), event_sender)
.expect("reducer should start");
let mut result = file_result("bad.go");
result.parser_status = FileParserStatus::Parsed;
result.language_id = Some("go".to_owned());
result.parser_output = Some(ParserOutput {
language_id: "go".to_owned(),
symbols: Vec::new(),
references: Vec::new(),
metrics_input: UniversalCodeMetricsInput::default(),
diagnostics: vec![ParserDiagnostic {
code: "parse_error".to_owned(),
message: "Go source contains syntax errors".to_owned(),
}],
limitations: vec![ParserLimitation {
code: "truncated_source".to_owned(),
message: "Go parser skipped content beyond the active file window".to_owned(),
}],
});

reducer
.handle()
.store_file_analysis(result)
.expect("file result should enqueue");
reducer.finish().expect("reducer should finish");

let connection = Connection::open(fixture.db_path()).expect("db should open");
let diagnostics: String = connection
.query_row(
"SELECT diagnostics FROM file_analysis WHERE relative_path = 'bad.go'",
[],
|row| row.get(0),
)
.expect("diagnostics should be stored");
assert!(diagnostics.contains("parse_error"));
assert!(diagnostics.contains("truncated_source"));
}

#[test]
fn finish_flushes_partial_batch() {
let fixture = Fixture::new("finish-flush");
Expand Down
54 changes: 49 additions & 5 deletions src/tui.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ pub struct RiskRow {
pub terms: Vec<RiskTerm>,
pub facts: Vec<RiskFact>,
pub limitations: Vec<RiskLimitation>,
pub parser_diagnostics: Vec<RiskLimitation>,
pub owners: Vec<RiskOwner>,
pub tags: Vec<String>,
}
Expand Down Expand Up @@ -749,6 +750,22 @@ fn inspector_lines(row: &RiskRow, width: u16) -> Vec<Line<'static>> {
lines.push(Line::styled(tags, style(TuiSeverity::Muted)));
}

if !row.parser_diagnostics.is_empty() {
lines.push(Line::raw(""));
lines.push(section_divider(width));
lines.push(Line::raw(""));
lines.push(Line::styled(
"Parser diagnostics",
style(TuiSeverity::Medium).add_modifier(Modifier::BOLD),
));
for diagnostic in &row.parser_diagnostics {
lines.push(Line::styled(
format!(" - {}: {}", diagnostic.code, diagnostic.message),
style(TuiSeverity::Muted),
));
}
}

lines.push(Line::raw(""));
lines.push(section_divider(width));
lines.extend(risk_driver_lines(row));
Expand Down Expand Up @@ -865,7 +882,8 @@ fn load_risk_rows(connection: &Connection) -> rusqlite::Result<Vec<RiskRow>> {
facts.dominant_owner,
facts.dominant_owner_share,
facts.owner_count,
facts.author_count
facts.author_count,
facts.diagnostics
FROM file_risk_scores score
LEFT JOIN file_facts facts
ON facts.relative_path = score.relative_path
Expand Down Expand Up @@ -901,6 +919,7 @@ fn load_risk_rows(connection: &Connection) -> rusqlite::Result<Vec<RiskRow>> {
terms: Vec::new(),
facts: Vec::new(),
limitations: Vec::new(),
parser_diagnostics: parse_diagnostics_json(row.get::<_, String>(24)?.as_str()),
owners: Vec::new(),
tags: Vec::new(),
})
Expand Down Expand Up @@ -1017,6 +1036,22 @@ fn load_owners(connection: &Connection) -> rusqlite::Result<BTreeMap<String, Vec
Ok(grouped)
}

fn parse_diagnostics_json(value: &str) -> Vec<RiskLimitation> {
let Ok(items) = serde_json::from_str::<Vec<serde_json::Value>>(value) else {
return Vec::new();
};

items
.into_iter()
.filter_map(|item| {
Some(RiskLimitation {
code: item.get("code")?.as_str()?.to_owned(),
message: item.get("message")?.as_str()?.to_owned(),
})
})
.collect()
}

fn find_index_root(current_dir: &Path) -> Option<PathBuf> {
current_dir
.ancestors()
Expand All @@ -1036,6 +1071,9 @@ fn inspector_tags(row: &RiskRow) -> Vec<String> {
if row.is_vendor {
signals.push(("VENDOR", 1.0, 10));
}
if !row.parser_diagnostics.is_empty() {
signals.push(("PARSER", 1.0, 95));
}
for term in &row.terms {
let value = term.normalized_value.unwrap_or_default();
if value < 0.60 {
Expand Down Expand Up @@ -1661,6 +1699,8 @@ mod tests {
assert_eq!(snapshot.rows[0].terms.len(), 2);
assert_eq!(snapshot.rows[0].facts.len(), 1);
assert_eq!(snapshot.rows[0].limitations.len(), 1);
assert_eq!(snapshot.rows[0].parser_diagnostics.len(), 1);
assert_eq!(snapshot.rows[0].parser_diagnostics[0].code, "parse_error");
assert_eq!(snapshot.rows[0].owners.len(), 1);
}

Expand Down Expand Up @@ -1717,6 +1757,8 @@ mod tests {
assert!(output.contains("Inspector"));
assert!(output.contains("src/risky.go"));
assert!(output.contains("CHURN"));
assert!(output.contains("Parser diagnostics"));
assert!(output.contains("parse_error"));
}

#[test]
Expand Down Expand Up @@ -1785,7 +1827,8 @@ mod tests {
dominant_owner TEXT,
dominant_owner_share REAL,
owner_count INTEGER,
author_count INTEGER NOT NULL
author_count INTEGER NOT NULL,
diagnostics TEXT NOT NULL
);
CREATE TABLE file_risk_terms (
relative_path TEXT NOT NULL,
Expand Down Expand Up @@ -1891,7 +1934,8 @@ mod tests {
dominant_owner TEXT,
dominant_owner_share REAL,
owner_count INTEGER,
author_count INTEGER NOT NULL
author_count INTEGER NOT NULL,
diagnostics TEXT NOT NULL
);
CREATE TABLE file_risk_terms (
relative_path TEXT NOT NULL,
Expand Down Expand Up @@ -1931,8 +1975,8 @@ mod tests {
('src/risky.go', 'C:/repo/src/risky.go', 1, 'hotpath.score.go.v1', 1, 0.9, 9.0, 'extreme', 1, 0),
('src/safe.go', 'C:/repo/src/safe.go', 1, 'hotpath.score.go.v1', 2, 0.2, 2.0, 'low', 0, 0);
INSERT INTO file_facts VALUES
('src/risky.go', 1200, 48000, 'go', 220, 45, 12, 8, 20, 2300, 1000, 3, 'Alice <a@example.invalid>', 0.9, 1, 1),
('src/safe.go', 10, 400, 'go', 1, 1, 0, 0, 0, 1, 0, 1, NULL, NULL, NULL, 1);
('src/risky.go', 1200, 48000, 'go', 220, 45, 12, 8, 20, 2300, 1000, 3, 'Alice <a@example.invalid>', 0.9, 1, 1, '[{\"code\":\"parse_error\",\"message\":\"Go source contains syntax errors\"}]'),
('src/safe.go', 10, 400, 'go', 1, 1, 0, 0, 0, 1, 0, 1, NULL, NULL, NULL, 1, '[]');
INSERT INTO file_risk_terms VALUES
('src/risky.go', 'hotpath.score.go.v1', 'churn', 2300, 1.0, 0.18, 0.18),
('src/risky.go', 'hotpath.score.go.v1', 'complexity_pressure', 220, 1.0, 0.16, 0.16),
Expand Down
Loading