From fe9f02614b7d8419cb48845be1a493df643d721e Mon Sep 17 00:00:00 2001 From: monozoide Date: Fri, 17 Oct 2025 22:24:43 +0200 Subject: [PATCH] Improve NULL handling for NOT NULL integer columns #65 Refines the logic in format_sql_value to treat columns as nullable only if 'NOT NULL' is absent from the SQL type definition. Adds stricter validation to prevent empty strings from being converted to integers for NOT NULL columns, and introduces a new test case to verify data conversion failure for NOT NULL integer fields. --- lib/maillogsentinel/sql_exporter.py | 31 +++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/maillogsentinel/sql_exporter.py b/lib/maillogsentinel/sql_exporter.py index b451c5e..527d140 100644 --- a/lib/maillogsentinel/sql_exporter.py +++ b/lib/maillogsentinel/sql_exporter.py @@ -212,8 +212,9 @@ def format_sql_value(value: Any, sql_type_def: str) -> str: Returns: SQL-formatted string representation of the value. """ - is_not_null = "NOT NULL" in sql_type_def.upper() - is_nullable = not is_not_null or "DEFAULT NULL" in sql_type_def.upper() + # A column is considered nullable if "NOT NULL" is absent from its definition. + # The presence of "DEFAULT NULL" implies nullability, but the absence of "NOT NULL" is the key check. + is_nullable = "NOT NULL" not in sql_type_def.upper() if value is None or str(value).strip().lower() in ["null", "na", "n/a", ""]: if is_nullable: @@ -228,6 +229,9 @@ def format_sql_value(value: Any, sql_type_def: str) -> str: if "int" in sql_type_lower or "serial" in sql_type_lower: try: + # Ensure that empty strings or other non-numeric values are not converted to NULL for NOT NULL columns + if str(value).strip() == "": + raise ValueError("Empty string cannot be converted to integer.") return str(int(value)) except (ValueError, TypeError): if is_nullable: @@ -893,6 +897,29 @@ def _reset_offset_file(config_obj: DummyTestConfig): ) all_tests_passed = False + # --- Test Case 5: Data conversion failure for NOT NULL integer --- + test_runner_logger.info( + "\n--- Test Case 5: Data conversion failure for NOT NULL integer ---" + ) + config_case5 = DummyTestConfig(base_dir_name="maillog_test_case5_") + test_configs_to_clean.append(config_case5) + # Create a CSV with an empty string for 'asn', which maps to 'asn_int' (NOT NULL) + invalid_row_vals = DUMMY_CSV_DATA_ROW_1_VALS.copy() + invalid_row_vals["asn"] = "" # This should fail conversion for a NOT NULL int + invalid_row_data = [_make_dummy_csv_data_row(DUMMY_CSV_HEADERS, invalid_row_vals)] + _create_dummy_csv(config_case5, DUMMY_CSV_HEADERS, invalid_row_data) + _reset_offset_file(config_case5) + success_case5 = run_sql_export(config_case5) + if not success_case5: + test_runner_logger.info( + "Test Case 5 Result (Bad Data for NOT NULL Int): SUCCESS (aborted as expected)" + ) + else: + test_runner_logger.error( + "Test Case 5 Result (Bad Data for NOT NULL Int): FAIL (should have aborted)" + ) + all_tests_passed = False + except Exception as e: test_runner_logger.error( f"An unexpected error occurred during testing: {e}", exc_info=True