diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 742d7ff071f9..8cb65b4560ee 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -230,47 +230,10 @@ USER openmetadata # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is # what keeps scanners reporting CVE-2026-59890. -# Keep this the LAST pip layer that can compile anything -- a later layer that builds a -# package needing pkg_resources would fail here, and the error would not look like a -# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt. +# Keep this the LAST pip layer -- a later layer that compiles a package needing +# pkg_resources would fail here, and the error would not look like a setuptools problem. RUN pip install --upgrade "setuptools>=83" -# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893, -# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in -# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no -# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us: -# collate-sqllineage 2.1.4 sqlparse==0.5.4 -# dbt-core (transitive via collate-data-diff) sqlparse<0.6.0 -# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with -# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core -- -# the two releases are byte-identical apart from the version string, so 0.6.0 is a version -# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and -# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete -# this layer and raise the floors in ingestion/setup.py instead. -# -# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install` -# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two -# unsatisfied pins for the life of the image, so the import gate is what actually keeps -# this honest. It asserts the two things that would otherwise fail silently: that we really -# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites. -# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a -# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage -# comes back quietly truncated with nothing failing. -RUN pip install --no-deps "sqlparse==0.6.0" \ - && python -W ignore -c "\ -import sqlparse; \ -from sqlparse.engine import grouping; \ -from sqlparse.keywords import KEYWORDS; \ -import collate_sqllineage.core.parser.sqlparse; \ -from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \ -from collate_sqllineage.runner import LineageRunner; \ -assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \ -assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \ -assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \ -r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \ -assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \ -assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables" - # Strip spaCy's bundled test fixture, which scanners misreport as an installed black. # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index c7fac3356237..c4ad1cbd0769 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -238,47 +238,10 @@ USER openmetadata # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is # what keeps scanners reporting CVE-2026-59890. -# Keep this the LAST pip layer that can compile anything -- a later layer that builds a -# package needing pkg_resources would fail here, and the error would not look like a -# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt. +# Keep this the LAST pip layer -- a later layer that compiles a package needing +# pkg_resources would fail here, and the error would not look like a setuptools problem. RUN pip install --upgrade "setuptools>=83" -# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893, -# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in -# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no -# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us: -# collate-sqllineage 2.1.4 sqlparse==0.5.4 -# dbt-core (transitive via collate-data-diff) sqlparse<0.6.0 -# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with -# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core -- -# the two releases are byte-identical apart from the version string, so 0.6.0 is a version -# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and -# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete -# this layer and raise the floors in ingestion/setup.py instead. -# -# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install` -# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two -# unsatisfied pins for the life of the image, so the import gate is what actually keeps -# this honest. It asserts the two things that would otherwise fail silently: that we really -# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites. -# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a -# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage -# comes back quietly truncated with nothing failing. -RUN pip install --no-deps "sqlparse==0.6.0" \ - && python -W ignore -c "\ -import sqlparse; \ -from sqlparse.engine import grouping; \ -from sqlparse.keywords import KEYWORDS; \ -import collate_sqllineage.core.parser.sqlparse; \ -from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \ -from collate_sqllineage.runner import LineageRunner; \ -assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \ -assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \ -assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \ -r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \ -assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \ -assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables" - # Strip spaCy's bundled test fixture, which scanners misreport as an installed black. # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the diff --git a/ingestion/setup.py b/ingestion/setup.py index a734cc234375..18f477bf68b5 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -178,7 +178,7 @@ "requests>=2.32.4", "requests-aws4auth~=1.1", # Only depends on requests as external package. Leaving as base. "sqlalchemy>=2.0.0,<3", - "collate-sqllineage==2.1.4", + "collate-sqllineage==2.1.7", "tabulate==0.9.0", "tenacity>=8.0,<10", "typing-inspect", diff --git a/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py b/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py index fb1ad25144e4..85705298aa3f 100644 --- a/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py +++ b/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py @@ -1077,19 +1077,22 @@ def test_update_with_join_and_cte(self): {"price_history"}, {"products"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Capturing only "latest_prices" CTE as source table that too is wrong - # SqlFluff: Not capturing target table "products" - test_sqlglot=False, - test_sqlfluff=False, ) - # UPDATE with CTE - parsers may have different column lineage extraction + # SET sources resolve through the latest_prices CTE to price_history assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("price", "price_history"), + TestColumnQualifierTuple("current_price", "products"), + ), + ( + TestColumnQualifierTuple("effective_date", "price_history"), + TestColumnQualifierTuple("last_price_update", "products"), + ), + ], dialect=Dialect.POSTGRES.value, - # Graph: SqlGlot (3n/2e) vs SqlFluff (13n/14e) - skip_graph_check=True, ) def test_create_table_as_select_complex(self): @@ -5800,11 +5803,26 @@ def test_update_merge_01_update_with_join_and_column_mapping(self): dialect=Dialect.POSTGRES.value, ) - # UPDATE column lineage - parsers may differ + # price_change_percent reads lp.new_price both directly and via p.current_price, + # which the parsers emit as the chain + # latest_prices.new_price -> products.current_price -> products.price_change_percent. + # assert_column_lineage compares only the ends of each path, so that chain and the + # direct read collapse onto the same pair. assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("new_price", "latest_prices"), + TestColumnQualifierTuple("price_change_percent", "products"), + ), + ( + TestColumnQualifierTuple("update_date", "latest_prices"), + TestColumnQualifierTuple("last_updated", "products"), + ), + ], dialect=Dialect.POSTGRES.value, + # SqlFluff: adds latest_prices.* and products.* wildcard edges + test_sqlfluff=False, ) def test_update_merge_02_update_with_cte(self): @@ -5835,19 +5853,26 @@ def test_update_merge_02_update_with_cte(self): {"sales"}, {"product_stats"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Treats CTE (aggregated_sales) as a source table instead of tracing through to sales - # SqlFluff: Doesn't detect target table in UPDATE statements with CTEs - # SqlParse: Also has issues with UPDATE + CTE - test_sqlglot=False, - test_sqlfluff=False, - test_sqlparse=False, ) + # SET sources resolve through the aggregated_sales CTE to sales assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("*", "sales"), + TestColumnQualifierTuple("ytd_sales_count", "product_stats"), + ), + ( + TestColumnQualifierTuple("quantity", "sales"), + TestColumnQualifierTuple("ytd_quantity_sold", "product_stats"), + ), + ( + TestColumnQualifierTuple("amount", "sales"), + TestColumnQualifierTuple("ytd_revenue", "product_stats"), + ), + ], dialect=Dialect.POSTGRES.value, - skip_graph_check=True, ) def test_update_merge_03_merge_with_insert_update(self): @@ -5939,18 +5964,27 @@ def test_update_merge_04_update_from_multiple_tables(self): {"sales", "products", "suppliers"}, {"inventory"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Cannot parse INTERVAL syntax in subquery - # SqlFluff: Doesn't trace through subquery to detect sales table - test_sqlglot=False, - test_sqlfluff=False, ) + # reorder_level resolves through the "s" subquery to sales.quantity, and + # supplier_lead_time to the joined suppliers table assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("quantity", "sales"), + TestColumnQualifierTuple("reorder_level", "inventory"), + ), + ( + TestColumnQualifierTuple("lead_time_days", "suppliers"), + TestColumnQualifierTuple("supplier_lead_time", "inventory"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlGlot: Cannot parse INTERVAL syntax in subquery - test_sqlglot=False, + # SqlFluff: adds inventory.*, products.*, suppliers.* and s.* wildcard edges + # SqlParse: resolves reorder_level but not the joined suppliers.lead_time_days + test_sqlfluff=False, + test_sqlparse=False, skip_graph_check=True, ) @@ -6046,18 +6080,21 @@ def test_update_merge_06_update_with_window_functions(self): {"employees"}, {"employee_rankings"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Doesn't detect any source or target tables in UPDATE with window functions - # SqlFluff: Doesn't track target table (employee_rankings) as source - test_sqlglot=False, - test_sqlfluff=False, ) + # All three window expressions read employees.salary through the "ranked" subquery. + # Only AVG(salary) resolves, because it reads salary as a function argument. RANK() + # and PERCENT_RANK() read it through OVER (ORDER BY salary), which no parser treats + # as a source column, so salary_rank and salary_percentile have no edge yet. assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("salary", "employees"), + TestColumnQualifierTuple("dept_avg_salary", "employee_rankings"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlGlot: Doesn't detect any source or target tables in UPDATE with window functions - test_sqlglot=False, skip_graph_check=True, ) @@ -6155,17 +6192,29 @@ def test_update_merge_08_update_with_correlated_subquery(self): {"reviews"}, {"products"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Doesn't include target table (products) as a source in UPDATE statements # Graph: Parsers create different graph structures (table lineage is correct) - test_sqlglot=False, skip_graph_check=True, ) + # Each correlated subquery reads one reviews column into its own products column assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("rating", "reviews"), + TestColumnQualifierTuple("avg_rating", "products"), + ), + ( + TestColumnQualifierTuple("*", "reviews"), + TestColumnQualifierTuple("review_count", "products"), + ), + ( + TestColumnQualifierTuple("review_date", "reviews"), + TestColumnQualifierTuple("last_review_date", "products"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlFluff: Tracks extra column lineages from correlated subquery + # SqlFluff: collapses all three targets onto a single products."avg(rating)" column # Graph: Parsers create different graph structures (column lineage is correct) test_sqlfluff=False, skip_graph_check=True, @@ -6277,21 +6326,25 @@ def test_update_merge_10_update_with_recursive_cte(self): {"employees"}, {"employee_hierarchy"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Treats recursive CTE (manager_chain) as a source table instead of tracing to employees - # SqlFluff: Doesn't track target table (employee_hierarchy) as source in UPDATE - # SqlParse: Includes recursive CTE (manager_chain) as a source table in UPDATE with recursive CTE - test_sqlglot=False, - test_sqlfluff=False, + # SqlParse: still reports the manager_chain recursive CTE as a source table + # Graph: SqlGlot (7n/5e) and SqlFluff (8n/4e) build different internal shapes + # for the recursion, though both resolve the same source and target tables test_sqlparse=False, + skip_graph_check=True, ) + # Correct lineage would be employees.employee_id to reporting_chain and to + # top_level_manager, since chain accumulates employee_id through the recursion and + # management_level derives from the literal counter. No parser produces that yet. assert_column_lineage_equal( query, [], dialect=Dialect.POSTGRES.value, - # SqlGlot: Treats CTE as source, doesn't produce column lineages - # SqlFluff/SqlParse: May produce different graph structures + # SqlGlot: traces to employees but adds manager_id as a source of the chain + # columns and gives management_level a source it does not have + # SqlParse: stops at the manager_chain CTE instead of tracing to employees test_sqlglot=False, + test_sqlparse=False, skip_graph_check=True, ) diff --git a/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py b/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py index bfb794ecc4c6..9a2956faef40 100644 --- a/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py +++ b/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py @@ -663,7 +663,11 @@ def test_postgres_ddl_statements(self): set(), # DDL statements don't have target tables for lineage dialect=Dialect.POSTGRES.value, # SqlFluff raises UnsupportedStatementException for SET and ALTER SEQUENCE statements + # SqlGlot: since collate-sqllineage 2.1.5 it also raises rather than silently + # returning empty lineage for statements it cannot parse, such as + # "SET client_min_messages=notice" test_sqlfluff=False, + test_sqlglot=False, ) # No column lineage expected - DDL statements with no source or target tables @@ -672,6 +676,7 @@ def test_postgres_ddl_statements(self): [], dialect=Dialect.POSTGRES.value, test_sqlfluff=False, + test_sqlglot=False, ) def test_snowflake_insert_with_cte_and_sequence(self):