Fixes #31790: upgrade collate-sqllineage to 2.1.7 for UPDATE and MERGE column lineage

ingestion/operators/docker/Dockerfile

@@ -230,47 +230,10 @@ USER openmetadata
 # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and
 # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is
 # what keeps scanners reporting CVE-2026-59890.
-# Keep this the LAST pip layer that can compile anything -- a later layer that builds a
-# package needing pkg_resources would fail here, and the error would not look like a
-# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt.
+# Keep this the LAST pip layer -- a later layer that compiles a package needing
+# pkg_resources would fail here, and the error would not look like a setuptools problem.
 RUN pip install --upgrade "setuptools>=83"
 
-# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893,
-# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in
-# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no
-# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us:
-#     collate-sqllineage 2.1.4                     sqlparse==0.5.4
-#     dbt-core (transitive via collate-data-diff)  sqlparse<0.6.0
-# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with
-# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core --
-# the two releases are byte-identical apart from the version string, so 0.6.0 is a version
-# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and
-# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete
-# this layer and raise the floors in ingestion/setup.py instead.
-#
-# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install`
-# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two
-# unsatisfied pins for the life of the image, so the import gate is what actually keeps
-# this honest. It asserts the two things that would otherwise fail silently: that we really
-# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites.
-# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a
-# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage
-# comes back quietly truncated with nothing failing.
-RUN pip install --no-deps "sqlparse==0.6.0" \
- && python -W ignore -c "\
-import sqlparse; \
-from sqlparse.engine import grouping; \
-from sqlparse.keywords import KEYWORDS; \
-import collate_sqllineage.core.parser.sqlparse; \
-from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \
-from collate_sqllineage.runner import LineageRunner; \
-assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \
-assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \
-assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \
-r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \
-assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \
-assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables"
-
 
 # Strip spaCy's bundled test fixture, which scanners misreport as an installed black.
 # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the

ingestion/operators/docker/Dockerfile.ci

@@ -238,47 +238,10 @@ USER openmetadata
 # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and
 # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is
 # what keeps scanners reporting CVE-2026-59890.
-# Keep this the LAST pip layer that can compile anything -- a later layer that builds a
-# package needing pkg_resources would fail here, and the error would not look like a
-# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt.
+# Keep this the LAST pip layer -- a later layer that compiles a package needing
+# pkg_resources would fail here, and the error would not look like a setuptools problem.
 RUN pip install --upgrade "setuptools>=83"
 
-# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893,
-# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in
-# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no
-# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us:
-#     collate-sqllineage 2.1.4                     sqlparse==0.5.4
-#     dbt-core (transitive via collate-data-diff)  sqlparse<0.6.0
-# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with
-# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core --
-# the two releases are byte-identical apart from the version string, so 0.6.0 is a version
-# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and
-# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete
-# this layer and raise the floors in ingestion/setup.py instead.
-#
-# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install`
-# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two
-# unsatisfied pins for the life of the image, so the import gate is what actually keeps
-# this honest. It asserts the two things that would otherwise fail silently: that we really
-# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites.
-# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a
-# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage
-# comes back quietly truncated with nothing failing.
-RUN pip install --no-deps "sqlparse==0.6.0" \
- && python -W ignore -c "\
-import sqlparse; \
-from sqlparse.engine import grouping; \
-from sqlparse.keywords import KEYWORDS; \
-import collate_sqllineage.core.parser.sqlparse; \
-from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \
-from collate_sqllineage.runner import LineageRunner; \
-assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \
-assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \
-assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \
-r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \
-assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \
-assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables"
-
 
 # Strip spaCy's bundled test fixture, which scanners misreport as an installed black.
 # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the

ingestion/setup.py

@@ -178,7 +178,7 @@
     "requests>=2.32.4",
     "requests-aws4auth~=1.1",  # Only depends on requests as external package. Leaving as base.
     "sqlalchemy>=2.0.0,<3",
-    "collate-sqllineage==2.1.4",
+    "collate-sqllineage==2.1.7",
     "tabulate==0.9.0",
     "tenacity>=8.0,<10",
     "typing-inspect",

ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py

@@ -1077,19 +1077,22 @@ def test_update_with_join_and_cte(self):
             {"price_history"},
             {"products"},
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Capturing only "latest_prices" CTE as source table that too is wrong
-            # SqlFluff: Not capturing target table "products"
-            test_sqlglot=False,
-            test_sqlfluff=False,
         )
 
-        # UPDATE with CTE - parsers may have different column lineage extraction
+        # SET sources resolve through the latest_prices CTE to price_history
         assert_column_lineage_equal(
             query,
-            [],
+            [
+                (
+                    TestColumnQualifierTuple("price", "price_history"),
+                    TestColumnQualifierTuple("current_price", "products"),
+                ),
+                (
+                    TestColumnQualifierTuple("effective_date", "price_history"),
+                    TestColumnQualifierTuple("last_price_update", "products"),
+                ),
+            ],
             dialect=Dialect.POSTGRES.value,
-            # Graph: SqlGlot (3n/2e) vs SqlFluff (13n/14e)
-            skip_graph_check=True,
         )
 
     def test_create_table_as_select_complex(self):
@@ -5800,11 +5803,26 @@ def test_update_merge_01_update_with_join_and_column_mapping(self):
             dialect=Dialect.POSTGRES.value,
         )
 
-        # UPDATE column lineage - parsers may differ
+        # price_change_percent reads lp.new_price both directly and via p.current_price,
+        # which the parsers emit as the chain
+        # latest_prices.new_price -> products.current_price -> products.price_change_percent.
+        # assert_column_lineage compares only the ends of each path, so that chain and the
+        # direct read collapse onto the same pair.
         assert_column_lineage_equal(
             query,
-            [],
+            [
+                (
+                    TestColumnQualifierTuple("new_price", "latest_prices"),
+                    TestColumnQualifierTuple("price_change_percent", "products"),
+                ),
+                (
+                    TestColumnQualifierTuple("update_date", "latest_prices"),
+                    TestColumnQualifierTuple("last_updated", "products"),
+                ),
+            ],
             dialect=Dialect.POSTGRES.value,
+            # SqlFluff: adds latest_prices.* and products.* wildcard edges
+            test_sqlfluff=False,
         )
 
     def test_update_merge_02_update_with_cte(self):
@@ -5835,19 +5853,26 @@ def test_update_merge_02_update_with_cte(self):
             {"sales"},
             {"product_stats"},
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Treats CTE (aggregated_sales) as a source table instead of tracing through to sales
-            # SqlFluff: Doesn't detect target table in UPDATE statements with CTEs
-            # SqlParse: Also has issues with UPDATE + CTE
-            test_sqlglot=False,
-            test_sqlfluff=False,
-            test_sqlparse=False,
         )
 
+        # SET sources resolve through the aggregated_sales CTE to sales
         assert_column_lineage_equal(
             query,
-            [],
+            [
+                (
+                    TestColumnQualifierTuple("*", "sales"),
+                    TestColumnQualifierTuple("ytd_sales_count", "product_stats"),
+                ),
+                (
+                    TestColumnQualifierTuple("quantity", "sales"),
+                    TestColumnQualifierTuple("ytd_quantity_sold", "product_stats"),
+                ),
+                (
+                    TestColumnQualifierTuple("amount", "sales"),
+                    TestColumnQualifierTuple("ytd_revenue", "product_stats"),
+                ),
+            ],
             dialect=Dialect.POSTGRES.value,
-            skip_graph_check=True,
         )
 
     def test_update_merge_03_merge_with_insert_update(self):
@@ -5939,18 +5964,27 @@ def test_update_merge_04_update_from_multiple_tables(self):
             {"sales", "products", "suppliers"},
             {"inventory"},
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Cannot parse INTERVAL syntax in subquery
-            # SqlFluff: Doesn't trace through subquery to detect sales table
-            test_sqlglot=False,
-            test_sqlfluff=False,
         )
 
+        # reorder_level resolves through the "s" subquery to sales.quantity, and
+        # supplier_lead_time to the joined suppliers table
         assert_column_lineage_equal(
             query,
-            [],
+            [
+                (
+                    TestColumnQualifierTuple("quantity", "sales"),
+                    TestColumnQualifierTuple("reorder_level", "inventory"),
+                ),
+                (
+                    TestColumnQualifierTuple("lead_time_days", "suppliers"),
+                    TestColumnQualifierTuple("supplier_lead_time", "inventory"),
+                ),
+            ],
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Cannot parse INTERVAL syntax in subquery
-            test_sqlglot=False,
+            # SqlFluff: adds inventory.*, products.*, suppliers.* and s.* wildcard edges
+            # SqlParse: resolves reorder_level but not the joined suppliers.lead_time_days
+            test_sqlfluff=False,
+            test_sqlparse=False,
             skip_graph_check=True,
         )
 
@@ -6046,18 +6080,21 @@ def test_update_merge_06_update_with_window_functions(self):
             {"employees"},
             {"employee_rankings"},
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Doesn't detect any source or target tables in UPDATE with window functions
-            # SqlFluff: Doesn't track target table (employee_rankings) as source
-            test_sqlglot=False,
-            test_sqlfluff=False,
         )
 
+        # All three window expressions read employees.salary through the "ranked" subquery.
+        # Only AVG(salary) resolves, because it reads salary as a function argument. RANK()
+        # and PERCENT_RANK() read it through OVER (ORDER BY salary), which no parser treats
+        # as a source column, so salary_rank and salary_percentile have no edge yet.
         assert_column_lineage_equal(
             query,
-            [],
+            [
+                (
+                    TestColumnQualifierTuple("salary", "employees"),
+                    TestColumnQualifierTuple("dept_avg_salary", "employee_rankings"),
+                ),
+            ],
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Doesn't detect any source or target tables in UPDATE with window functions
-            test_sqlglot=False,
             skip_graph_check=True,
         )
 
@@ -6155,17 +6192,29 @@ def test_update_merge_08_update_with_correlated_subquery(self):
             {"reviews"},
             {"products"},
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Doesn't include target table (products) as a source in UPDATE statements
             # Graph: Parsers create different graph structures (table lineage is correct)
-            test_sqlglot=False,
             skip_graph_check=True,
         )
 
+        # Each correlated subquery reads one reviews column into its own products column
         assert_column_lineage_equal(
             query,
-            [],
+            [
+                (
+                    TestColumnQualifierTuple("rating", "reviews"),
+                    TestColumnQualifierTuple("avg_rating", "products"),
+                ),
+                (
+                    TestColumnQualifierTuple("*", "reviews"),
+                    TestColumnQualifierTuple("review_count", "products"),
+                ),
+                (
+                    TestColumnQualifierTuple("review_date", "reviews"),
+                    TestColumnQualifierTuple("last_review_date", "products"),
+                ),
+            ],
             dialect=Dialect.POSTGRES.value,
-            # SqlFluff: Tracks extra column lineages from correlated subquery
+            # SqlFluff: collapses all three targets onto a single products."avg(rating)" column
             # Graph: Parsers create different graph structures (column lineage is correct)
             test_sqlfluff=False,
             skip_graph_check=True,
@@ -6277,21 +6326,25 @@ def test_update_merge_10_update_with_recursive_cte(self):
             {"employees"},
             {"employee_hierarchy"},
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Treats recursive CTE (manager_chain) as a source table instead of tracing to employees
-            # SqlFluff: Doesn't track target table (employee_hierarchy) as source in UPDATE
-            # SqlParse: Includes recursive CTE (manager_chain) as a source table in UPDATE with recursive CTE
-            test_sqlglot=False,
-            test_sqlfluff=False,
+            # SqlParse: still reports the manager_chain recursive CTE as a source table
+            # Graph: SqlGlot (7n/5e) and SqlFluff (8n/4e) build different internal shapes
+            # for the recursion, though both resolve the same source and target tables
             test_sqlparse=False,
+            skip_graph_check=True,
         )
 
+        # Correct lineage would be employees.employee_id to reporting_chain and to
+        # top_level_manager, since chain accumulates employee_id through the recursion and
+        # management_level derives from the literal counter. No parser produces that yet.
         assert_column_lineage_equal(
             query,
             [],
             dialect=Dialect.POSTGRES.value,
-            # SqlGlot: Treats CTE as source, doesn't produce column lineages
-            # SqlFluff/SqlParse: May produce different graph structures
+            # SqlGlot: traces to employees but adds manager_id as a source of the chain
+            #   columns and gives management_level a source it does not have
+            # SqlParse: stops at the manager_chain CTE instead of tracing to employees
             test_sqlglot=False,
+            test_sqlparse=False,
             skip_graph_check=True,
         )
 

ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py

@@ -663,7 +663,11 @@ def test_postgres_ddl_statements(self):
             set(),  # DDL statements don't have target tables for lineage
             dialect=Dialect.POSTGRES.value,
             # SqlFluff raises UnsupportedStatementException for SET and ALTER SEQUENCE statements
+            # SqlGlot: since collate-sqllineage 2.1.5 it also raises rather than silently
+            # returning empty lineage for statements it cannot parse, such as
+            # "SET client_min_messages=notice"
             test_sqlfluff=False,
+            test_sqlglot=False,
         )
 
         # No column lineage expected - DDL statements with no source or target tables
@@ -672,6 +676,7 @@ def test_postgres_ddl_statements(self):
             [],
             dialect=Dialect.POSTGRES.value,
             test_sqlfluff=False,
+            test_sqlglot=False,
         )
 
     def test_snowflake_insert_with_cte_and_sequence(self):