Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ language-derived metrics or Go file risk scores.
Current Go processing is intentionally limited:

- Go recognition is extension-based: only paths ending in `.go` are considered.
- Go files ending in `_test.go` are tagged as test files in the local index so
their churn, size, and complexity can be interpreted separately from
production source files.
- Go files must be readable as UTF-8 text.
- Files larger than the active content window are not parsed. The default
content window is 1 MiB.
Expand Down
29 changes: 28 additions & 1 deletion src/pipeline/file_analyzer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ impl FileAnalyzer {
line_count,
is_generated: is_generated_file(file.path(), &window),
is_vendor: is_vendor_path(file.path()),
is_test: is_test_path(file.path()),
diagnostics,
parser_status: parser.status,
parser_output: parser.output,
Expand Down Expand Up @@ -142,7 +143,7 @@ impl Default for FileAnalyzerOptions {

pub fn file_analyzer_options_signature(options: &FileAnalyzerOptions) -> String {
format!(
"file-local-v3-source-refs;content-window={};parsers={}",
"file-local-v4-test-files;content-window={};parsers={}",
options.content_window_bytes,
options
.parsers
Expand All @@ -167,6 +168,7 @@ pub struct FileAnalysisResult {
pub line_count: Option<u64>,
pub is_generated: bool,
pub is_vendor: bool,
pub is_test: bool,
pub diagnostics: Vec<FileDiagnostic>,
pub parser_status: FileParserStatus,
pub parser_output: Option<ParserOutput>,
Expand Down Expand Up @@ -430,6 +432,12 @@ fn is_vendor_path(path: &Path) -> bool {
})
}

fn is_test_path(path: &Path) -> bool {
path.file_name()
.and_then(|file_name| file_name.to_str())
.is_some_and(|file_name| file_name.to_ascii_lowercase().ends_with("_test.go"))
}

fn is_generated_file(path: &Path, window: &FileContentWindow) -> bool {
is_generated_path(path) || has_go_generated_code_comment(window)
}
Expand Down Expand Up @@ -695,6 +703,25 @@ mod tests {
assert!(vendor.is_vendor);
}

#[test]
fn analyzer_tags_go_test_files() {
let fixture = Fixture::new("test-file-classification");
let test_path = fixture.write("service_test.go", b"package main\n");
let regular_path = fixture.write("service.go", b"package main\n");
let similarly_named_path = fixture.write("service_test.rs", b"fn main() {}\n");
let analyzer = FileAnalyzer::new();

let test_file = analyzer.analyze(FileAnalysisInput { path: test_path });
let regular_file = analyzer.analyze(FileAnalysisInput { path: regular_path });
let similarly_named = analyzer.analyze(FileAnalysisInput {
path: similarly_named_path,
});

assert!(test_file.is_test);
assert!(!regular_file.is_test);
assert!(!similarly_named.is_test);
}

#[test]
fn empty_parser_registry_reports_unsupported_without_attempts() {
let fixture = Fixture::new("empty-parser-registry");
Expand Down
57 changes: 44 additions & 13 deletions src/pipeline/store_reducer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,7 @@ fn initialize_database(connection: &Connection) -> Result<(), StoreReducerError>
line_count INTEGER,
is_generated INTEGER NOT NULL,
is_vendor INTEGER NOT NULL,
is_test INTEGER NOT NULL DEFAULT 0,
parser_status TEXT NOT NULL,
parser_recognition_attempts INTEGER NOT NULL,
language_id TEXT,
Expand Down Expand Up @@ -877,6 +878,7 @@ fn initialize_database(connection: &Connection) -> Result<(), StoreReducerError>
line_count INTEGER,
is_generated INTEGER NOT NULL,
is_vendor INTEGER NOT NULL,
is_test INTEGER NOT NULL DEFAULT 0,
parser_status TEXT NOT NULL,
parser_recognition_attempts INTEGER NOT NULL,
language_id TEXT,
Expand Down Expand Up @@ -927,6 +929,7 @@ fn initialize_database(connection: &Connection) -> Result<(), StoreReducerError>
risk_band TEXT NOT NULL,
is_generated INTEGER NOT NULL,
is_vendor INTEGER NOT NULL,
is_test INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (relative_path, formula_id)
);

Expand Down Expand Up @@ -1045,6 +1048,12 @@ fn initialize_database(connection: &Connection) -> Result<(), StoreReducerError>
"is_active",
"INTEGER NOT NULL DEFAULT 1",
)?;
add_column_if_missing(
connection,
"file_analysis",
"is_test",
"INTEGER NOT NULL DEFAULT 0",
)?;
add_column_if_missing(connection, "file_analysis", "language_id", "TEXT")?;
add_column_if_missing(
connection,
Expand Down Expand Up @@ -1088,6 +1097,12 @@ fn initialize_database(connection: &Connection) -> Result<(), StoreReducerError>
"max_function_complexity_pressure",
"INTEGER",
)?;
add_column_if_missing(
connection,
"file_facts",
"is_test",
"INTEGER NOT NULL DEFAULT 0",
)?;
add_column_if_missing(connection, "file_facts", "language_id", "TEXT")?;
add_column_if_missing(
connection,
Expand Down Expand Up @@ -1138,6 +1153,12 @@ fn initialize_database(connection: &Connection) -> Result<(), StoreReducerError>
"source_coupling_pressure_out",
"INTEGER",
)?;
add_column_if_missing(
connection,
"file_risk_scores",
"is_test",
"INTEGER NOT NULL DEFAULT 0",
)?;
add_column_if_missing(
connection,
"git_chunks",
Expand Down Expand Up @@ -1318,6 +1339,7 @@ fn flush_batch(
line_count,
is_generated,
is_vendor,
is_test,
parser_status,
parser_recognition_attempts,
language_id,
Expand All @@ -1329,7 +1351,7 @@ fn flush_batch(
complexity_pressure,
max_function_complexity_pressure,
diagnostics
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22)
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23)
",
)
.map_err(StoreReducerError::WriteDatabase)?;
Expand All @@ -1353,6 +1375,7 @@ fn flush_batch(
result.line_count.map(|value| value as i64),
bool_to_i64(result.is_generated),
bool_to_i64(result.is_vendor),
bool_to_i64(result.is_test),
parser_status_name(result.parser_status),
result.parser_recognition_attempts as i64,
result.language_id.as_deref(),
Expand Down Expand Up @@ -2188,6 +2211,7 @@ fn materialize_file_facts(
line_count,
is_generated,
is_vendor,
is_test,
parser_status,
parser_recognition_attempts,
language_id,
Expand Down Expand Up @@ -2228,6 +2252,7 @@ fn materialize_file_facts(
file_analysis.line_count,
file_analysis.is_generated,
file_analysis.is_vendor,
file_analysis.is_test,
file_analysis.parser_status,
file_analysis.parser_recognition_attempts,
file_analysis.language_id,
Expand Down Expand Up @@ -2295,6 +2320,7 @@ struct FileRiskRow {
active_scan_id: i64,
is_generated: bool,
is_vendor: bool,
is_test: bool,
input: FileRiskInput,
assessment: FileRiskAssessment,
}
Expand All @@ -2315,6 +2341,7 @@ fn materialize_file_risk_scores(
line_count,
is_generated,
is_vendor,
is_test,
total_churn_lines,
recent_churn_lines,
owner_count,
Expand All @@ -2341,17 +2368,17 @@ fn materialize_file_risk_scores(
relative_path: relative_path.clone(),
line_count: optional_i64_to_u64(row.get::<_, Option<i64>>(4)?),
byte_size: optional_i64_to_u64(row.get::<_, Option<i64>>(3)?),
total_churn_lines: i64_to_u64(row.get::<_, i64>(7)?),
recent_churn_lines: i64_to_u64(row.get::<_, i64>(8)?),
owner_count: optional_i64_to_u64(row.get::<_, Option<i64>>(9)?),
dominant_owner_share: row.get::<_, Option<f64>>(10)?,
co_changed_file_count: i64_to_u64(row.get::<_, i64>(11)?),
file_age_days: optional_i64_to_u64(row.get::<_, Option<i64>>(12)?),
source_coupling_pressure_in: optional_i64_to_u64(row.get::<_, Option<i64>>(13)?),
source_coupling_pressure_out: optional_i64_to_u64(row.get::<_, Option<i64>>(14)?),
complexity_pressure: optional_i64_to_u64(row.get::<_, Option<i64>>(15)?),
total_churn_lines: i64_to_u64(row.get::<_, i64>(8)?),
recent_churn_lines: i64_to_u64(row.get::<_, i64>(9)?),
owner_count: optional_i64_to_u64(row.get::<_, Option<i64>>(10)?),
dominant_owner_share: row.get::<_, Option<f64>>(11)?,
co_changed_file_count: i64_to_u64(row.get::<_, i64>(12)?),
file_age_days: optional_i64_to_u64(row.get::<_, Option<i64>>(13)?),
source_coupling_pressure_in: optional_i64_to_u64(row.get::<_, Option<i64>>(14)?),
source_coupling_pressure_out: optional_i64_to_u64(row.get::<_, Option<i64>>(15)?),
complexity_pressure: optional_i64_to_u64(row.get::<_, Option<i64>>(16)?),
max_function_complexity_pressure: optional_i64_to_u64(
row.get::<_, Option<i64>>(16)?,
row.get::<_, Option<i64>>(17)?,
),
};
Ok(FileRiskRow {
Expand All @@ -2360,6 +2387,7 @@ fn materialize_file_risk_scores(
active_scan_id: row.get(2)?,
is_generated: row.get::<_, i64>(5)? != 0,
is_vendor: row.get::<_, i64>(6)? != 0,
is_test: row.get::<_, i64>(7)? != 0,
input,
assessment: FileRiskAssessment {
formula_id: FORMULA_ID,
Expand Down Expand Up @@ -2413,8 +2441,9 @@ fn materialize_file_risk_scores(
risk_10,
risk_band,
is_generated,
is_vendor
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
is_vendor,
is_test
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
",
)
.map_err(StoreReducerError::WriteDatabase)?;
Expand Down Expand Up @@ -2474,6 +2503,7 @@ fn materialize_file_risk_scores(
risk_row.assessment.risk_band,
bool_to_i64(risk_row.is_generated),
bool_to_i64(risk_row.is_vendor),
bool_to_i64(risk_row.is_test),
])
.map_err(StoreReducerError::WriteDatabase)?;

Expand Down Expand Up @@ -4366,6 +4396,7 @@ mod tests {
line_count: Some(1),
is_generated: false,
is_vendor: false,
is_test: false,
diagnostics: Vec::new(),
parser_status: FileParserStatus::Unsupported,
parser_output: None,
Expand Down
53 changes: 53 additions & 0 deletions tests/scanner_cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,53 @@ fn scan_prints_file_and_git_progress_summary() {
assert_eq!(row_count(&connection, "file_analysis"), 2);
}

#[test]
fn scan_tags_go_test_files_in_index_facts_and_risk_rows() {
let fixture = Fixture::new("scan-go-test-files");
fixture.write("service.go", "package main\n\nfunc Service() {}\n");
fixture.write("service_test.go", "package main\n\nfunc TestService() {}\n");

let output = hotpath(&["scan"], &fixture.path);

assert!(
output.status.success(),
"hotpath failed\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);

let connection =
Connection::open(fixture.path.join(".hotpath").join("index.sqlite")).expect("db opens");
assert_eq!(
scalar_i64(
&connection,
"SELECT is_test FROM file_analysis WHERE relative_path = 'service_test.go'",
),
1
);
assert_eq!(
scalar_i64(
&connection,
"SELECT is_test FROM file_facts WHERE relative_path = 'service_test.go'",
),
1
);
assert_eq!(
scalar_i64(
&connection,
"SELECT is_test FROM file_risk_scores WHERE relative_path = 'service_test.go'",
),
1
);
assert_eq!(
scalar_i64(
&connection,
"SELECT is_test FROM file_facts WHERE relative_path = 'service.go'",
),
0
);
}

#[test]
fn scan_respects_ignore_rules_in_file_count() {
let fixture = GitFixture::new("scan-ignore");
Expand Down Expand Up @@ -329,3 +376,9 @@ fn scalar_text(connection: &Connection, sql: &str) -> String {
.query_row(sql, [], |row| row.get(0))
.expect("scalar query should run")
}

fn scalar_i64(connection: &Connection, sql: &str) -> i64 {
connection
.query_row(sql, [], |row| row.get(0))
.expect("scalar query should run")
}
Loading