From 9216bad231758c2da28ac4b211992316464fddbb Mon Sep 17 00:00:00 2001 From: Mohit Tilala Date: Fri, 21 Aug 2026 15:50:20 +0530 Subject: [PATCH 1/3] Fixes #31790: upgrade collate-sqllineage to 2.1.7 2.1.4 produces no column lineage for UPDATE and MERGE. Table lineage still resolves, so the gap is quiet: the graph looks populated while every column edge on those statements is missing. 2.1.7 also clears one of the sqlparse>=0.6.0 ceilings tracked in the issue. Update the lineage tests to assert the correct lineage instead of the empty result that encoded the old limitation. Two shapes are still unresolved upstream and stay marked xfail with the correct lineage asserted. --- ingestion/setup.py | 2 +- .../queries/test_complex_query_patterns.py | 165 +++++++++++++----- .../queries/test_specific_dialect_queries.py | 5 + 3 files changed, 126 insertions(+), 46 deletions(-) diff --git a/ingestion/setup.py b/ingestion/setup.py index a734cc234375..18f477bf68b5 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -178,7 +178,7 @@ "requests>=2.32.4", "requests-aws4auth~=1.1", # Only depends on requests as external package. Leaving as base. "sqlalchemy>=2.0.0,<3", - "collate-sqllineage==2.1.4", + "collate-sqllineage==2.1.7", "tabulate==0.9.0", "tenacity>=8.0,<10", "typing-inspect", diff --git a/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py b/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py index fb1ad25144e4..4392cdfccb55 100644 --- a/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py +++ b/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py @@ -6,6 +6,7 @@ capabilities. """ +import pytest from collate_sqllineage.core.models import DataFunction from ingestion.tests.unit.lineage.queries.helpers import ( @@ -1077,19 +1078,22 @@ def test_update_with_join_and_cte(self): {"price_history"}, {"products"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Capturing only "latest_prices" CTE as source table that too is wrong - # SqlFluff: Not capturing target table "products" - test_sqlglot=False, - test_sqlfluff=False, ) - # UPDATE with CTE - parsers may have different column lineage extraction + # SET sources resolve through the latest_prices CTE to price_history assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("price", "price_history"), + TestColumnQualifierTuple("current_price", "products"), + ), + ( + TestColumnQualifierTuple("effective_date", "price_history"), + TestColumnQualifierTuple("last_price_update", "products"), + ), + ], dialect=Dialect.POSTGRES.value, - # Graph: SqlGlot (3n/2e) vs SqlFluff (13n/14e) - skip_graph_check=True, ) def test_create_table_as_select_complex(self): @@ -5800,11 +5804,26 @@ def test_update_merge_01_update_with_join_and_column_mapping(self): dialect=Dialect.POSTGRES.value, ) - # UPDATE column lineage - parsers may differ + # price_change_percent reads lp.new_price both directly and via p.current_price, + # which the parsers emit as the chain + # latest_prices.new_price -> products.current_price -> products.price_change_percent. + # assert_column_lineage compares only the ends of each path, so that chain and the + # direct read collapse onto the same pair. assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("new_price", "latest_prices"), + TestColumnQualifierTuple("price_change_percent", "products"), + ), + ( + TestColumnQualifierTuple("update_date", "latest_prices"), + TestColumnQualifierTuple("last_updated", "products"), + ), + ], dialect=Dialect.POSTGRES.value, + # SqlFluff: adds latest_prices.* and products.* wildcard edges + test_sqlfluff=False, ) def test_update_merge_02_update_with_cte(self): @@ -5835,19 +5854,26 @@ def test_update_merge_02_update_with_cte(self): {"sales"}, {"product_stats"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Treats CTE (aggregated_sales) as a source table instead of tracing through to sales - # SqlFluff: Doesn't detect target table in UPDATE statements with CTEs - # SqlParse: Also has issues with UPDATE + CTE - test_sqlglot=False, - test_sqlfluff=False, - test_sqlparse=False, ) + # SET sources resolve through the aggregated_sales CTE to sales assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("*", "sales"), + TestColumnQualifierTuple("ytd_sales_count", "product_stats"), + ), + ( + TestColumnQualifierTuple("quantity", "sales"), + TestColumnQualifierTuple("ytd_quantity_sold", "product_stats"), + ), + ( + TestColumnQualifierTuple("amount", "sales"), + TestColumnQualifierTuple("ytd_revenue", "product_stats"), + ), + ], dialect=Dialect.POSTGRES.value, - skip_graph_check=True, ) def test_update_merge_03_merge_with_insert_update(self): @@ -5939,18 +5965,27 @@ def test_update_merge_04_update_from_multiple_tables(self): {"sales", "products", "suppliers"}, {"inventory"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Cannot parse INTERVAL syntax in subquery - # SqlFluff: Doesn't trace through subquery to detect sales table - test_sqlglot=False, - test_sqlfluff=False, ) + # reorder_level resolves through the "s" subquery to sales.quantity, and + # supplier_lead_time to the joined suppliers table assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("quantity", "sales"), + TestColumnQualifierTuple("reorder_level", "inventory"), + ), + ( + TestColumnQualifierTuple("lead_time_days", "suppliers"), + TestColumnQualifierTuple("supplier_lead_time", "inventory"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlGlot: Cannot parse INTERVAL syntax in subquery - test_sqlglot=False, + # SqlFluff: adds inventory.*, products.*, suppliers.* and s.* wildcard edges + # SqlParse: resolves reorder_level but not the joined suppliers.lead_time_days + test_sqlfluff=False, + test_sqlparse=False, skip_graph_check=True, ) @@ -6021,6 +6056,12 @@ def test_update_merge_05_merge_with_complex_matching(self): test_sqlparse=False, ) + @pytest.mark.xfail( + reason="collate-sqllineage 2.1.7 resolves only the AVG(salary) window expression " + "back to employees.salary. The RANK() and PERCENT_RANK() expressions, which read " + "salary through ORDER BY rather than as an argument, produce no edge.", + strict=False, + ) def test_update_merge_06_update_with_window_functions(self): """Test UPDATE using window functions in subquery""" query = """ @@ -6046,18 +6087,26 @@ def test_update_merge_06_update_with_window_functions(self): {"employees"}, {"employee_rankings"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Doesn't detect any source or target tables in UPDATE with window functions - # SqlFluff: Doesn't track target table (employee_rankings) as source - test_sqlglot=False, - test_sqlfluff=False, ) + # All three window expressions read employees.salary through the "ranked" subquery assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("salary", "employees"), + TestColumnQualifierTuple("salary_rank", "employee_rankings"), + ), + ( + TestColumnQualifierTuple("salary", "employees"), + TestColumnQualifierTuple("salary_percentile", "employee_rankings"), + ), + ( + TestColumnQualifierTuple("salary", "employees"), + TestColumnQualifierTuple("dept_avg_salary", "employee_rankings"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlGlot: Doesn't detect any source or target tables in UPDATE with window functions - test_sqlglot=False, skip_graph_check=True, ) @@ -6155,17 +6204,29 @@ def test_update_merge_08_update_with_correlated_subquery(self): {"reviews"}, {"products"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Doesn't include target table (products) as a source in UPDATE statements # Graph: Parsers create different graph structures (table lineage is correct) - test_sqlglot=False, skip_graph_check=True, ) + # Each correlated subquery reads one reviews column into its own products column assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("rating", "reviews"), + TestColumnQualifierTuple("avg_rating", "products"), + ), + ( + TestColumnQualifierTuple("*", "reviews"), + TestColumnQualifierTuple("review_count", "products"), + ), + ( + TestColumnQualifierTuple("review_date", "reviews"), + TestColumnQualifierTuple("last_review_date", "products"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlFluff: Tracks extra column lineages from correlated subquery + # SqlFluff: collapses all three targets onto a single products."avg(rating)" column # Graph: Parsers create different graph structures (column lineage is correct) test_sqlfluff=False, skip_graph_check=True, @@ -6240,6 +6301,12 @@ def test_update_merge_09_merge_with_computed_columns(self): test_sqlparse=False, ) + @pytest.mark.xfail( + reason="collate-sqllineage 2.1.7 traces the recursive CTE back to employees but " + "over-reports: it adds employees.manager_id as a source of the chain columns and " + "gives management_level a source even though it derives from the literal counter.", + strict=False, + ) def test_update_merge_10_update_with_recursive_cte(self): """Test UPDATE with recursive CTE for hierarchical updates""" query = """ @@ -6277,21 +6344,29 @@ def test_update_merge_10_update_with_recursive_cte(self): {"employees"}, {"employee_hierarchy"}, dialect=Dialect.POSTGRES.value, - # SqlGlot: Treats recursive CTE (manager_chain) as a source table instead of tracing to employees - # SqlFluff: Doesn't track target table (employee_hierarchy) as source in UPDATE - # SqlParse: Includes recursive CTE (manager_chain) as a source table in UPDATE with recursive CTE - test_sqlglot=False, - test_sqlfluff=False, + # SqlParse: still reports the manager_chain recursive CTE as a source table test_sqlparse=False, ) + # chain accumulates employee_id through the recursion. management_level derives + # from the literal level counter, so it has no source column. assert_column_lineage_equal( query, - [], + [ + ( + TestColumnQualifierTuple("employee_id", "employees"), + TestColumnQualifierTuple("reporting_chain", "employee_hierarchy"), + ), + ( + TestColumnQualifierTuple("employee_id", "employees"), + TestColumnQualifierTuple("top_level_manager", "employee_hierarchy"), + ), + ], dialect=Dialect.POSTGRES.value, - # SqlGlot: Treats CTE as source, doesn't produce column lineages - # SqlFluff/SqlParse: May produce different graph structures - test_sqlglot=False, + # SqlFluff: produces no column lineage for this shape + # SqlParse: stops at the manager_chain CTE instead of tracing to employees + test_sqlfluff=False, + test_sqlparse=False, skip_graph_check=True, ) diff --git a/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py b/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py index bfb794ecc4c6..9a2956faef40 100644 --- a/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py +++ b/ingestion/tests/unit/lineage/queries/test_specific_dialect_queries.py @@ -663,7 +663,11 @@ def test_postgres_ddl_statements(self): set(), # DDL statements don't have target tables for lineage dialect=Dialect.POSTGRES.value, # SqlFluff raises UnsupportedStatementException for SET and ALTER SEQUENCE statements + # SqlGlot: since collate-sqllineage 2.1.5 it also raises rather than silently + # returning empty lineage for statements it cannot parse, such as + # "SET client_min_messages=notice" test_sqlfluff=False, + test_sqlglot=False, ) # No column lineage expected - DDL statements with no source or target tables @@ -672,6 +676,7 @@ def test_postgres_ddl_statements(self): [], dialect=Dialect.POSTGRES.value, test_sqlfluff=False, + test_sqlglot=False, ) def test_snowflake_insert_with_cte_and_sequence(self): From 5b589cc5935afae028caabae34c3753d3f6bef61 Mon Sep 17 00:00:00 2001 From: Mohit Tilala Date: Fri, 21 Aug 2026 16:31:03 +0530 Subject: [PATCH 2/3] Revert "Force sqlparse 0.6.0 in the ingestion operator images to clear 4 CVEs (#31791)" collate-sqllineage 2.1.7 carries sqlparse 0.6.0 in its own metadata, so the resolver handles this without a --no-deps override and an import gate to keep it honest. Reverts 2e8f04552a. --- ingestion/operators/docker/Dockerfile | 41 ++---------------------- ingestion/operators/docker/Dockerfile.ci | 41 ++---------------------- 2 files changed, 4 insertions(+), 78 deletions(-) diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 742d7ff071f9..8cb65b4560ee 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -230,47 +230,10 @@ USER openmetadata # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is # what keeps scanners reporting CVE-2026-59890. -# Keep this the LAST pip layer that can compile anything -- a later layer that builds a -# package needing pkg_resources would fail here, and the error would not look like a -# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt. +# Keep this the LAST pip layer -- a later layer that compiles a package needing +# pkg_resources would fail here, and the error would not look like a setuptools problem. RUN pip install --upgrade "setuptools>=83" -# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893, -# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in -# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no -# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us: -# collate-sqllineage 2.1.4 sqlparse==0.5.4 -# dbt-core (transitive via collate-data-diff) sqlparse<0.6.0 -# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with -# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core -- -# the two releases are byte-identical apart from the version string, so 0.6.0 is a version -# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and -# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete -# this layer and raise the floors in ingestion/setup.py instead. -# -# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install` -# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two -# unsatisfied pins for the life of the image, so the import gate is what actually keeps -# this honest. It asserts the two things that would otherwise fail silently: that we really -# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites. -# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a -# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage -# comes back quietly truncated with nothing failing. -RUN pip install --no-deps "sqlparse==0.6.0" \ - && python -W ignore -c "\ -import sqlparse; \ -from sqlparse.engine import grouping; \ -from sqlparse.keywords import KEYWORDS; \ -import collate_sqllineage.core.parser.sqlparse; \ -from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \ -from collate_sqllineage.runner import LineageRunner; \ -assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \ -assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \ -assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \ -r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \ -assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \ -assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables" - # Strip spaCy's bundled test fixture, which scanners misreport as an installed black. # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index c7fac3356237..c4ad1cbd0769 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -238,47 +238,10 @@ USER openmetadata # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is # what keeps scanners reporting CVE-2026-59890. -# Keep this the LAST pip layer that can compile anything -- a later layer that builds a -# package needing pkg_resources would fail here, and the error would not look like a -# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt. +# Keep this the LAST pip layer -- a later layer that compiles a package needing +# pkg_resources would fail here, and the error would not look like a setuptools problem. RUN pip install --upgrade "setuptools>=83" -# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893, -# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in -# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no -# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us: -# collate-sqllineage 2.1.4 sqlparse==0.5.4 -# dbt-core (transitive via collate-data-diff) sqlparse<0.6.0 -# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with -# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core -- -# the two releases are byte-identical apart from the version string, so 0.6.0 is a version -# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and -# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete -# this layer and raise the floors in ingestion/setup.py instead. -# -# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install` -# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two -# unsatisfied pins for the life of the image, so the import gate is what actually keeps -# this honest. It asserts the two things that would otherwise fail silently: that we really -# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites. -# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a -# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage -# comes back quietly truncated with nothing failing. -RUN pip install --no-deps "sqlparse==0.6.0" \ - && python -W ignore -c "\ -import sqlparse; \ -from sqlparse.engine import grouping; \ -from sqlparse.keywords import KEYWORDS; \ -import collate_sqllineage.core.parser.sqlparse; \ -from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \ -from collate_sqllineage.runner import LineageRunner; \ -assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \ -assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \ -assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \ -r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \ -assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \ -assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables" - # Strip spaCy's bundled test fixture, which scanners misreport as an installed black. # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the From 0316dc7a2137d9f2d25eb75507dd545754f533e7 Mon Sep 17 00:00:00 2001 From: Mohit Tilala Date: Fri, 21 Aug 2026 17:40:53 +0530 Subject: [PATCH 3/3] Address review: drop xfail marks, use the existing parser-disable convention Greptile flagged that whole-test xfails also cover the table-lineage and parser checks, so an unrelated regression reports as XFAIL instead of failing. Use the per-parser flags the rest of the file already uses. merge_06 asserts the one edge all three parsers resolve, with a comment on why the two window ORDER BY edges are missing. merge_10 goes back to an empty expectation with SqlGlot and SqlParse disabled and their specific errors named, plus skip_graph_check on the table assertion since SqlGlot and SqlFluff build different internal shapes for the recursion. --- .../queries/test_complex_query_patterns.py | 50 ++++++------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py b/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py index 4392cdfccb55..85705298aa3f 100644 --- a/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py +++ b/ingestion/tests/unit/lineage/queries/test_complex_query_patterns.py @@ -6,7 +6,6 @@ capabilities. """ -import pytest from collate_sqllineage.core.models import DataFunction from ingestion.tests.unit.lineage.queries.helpers import ( @@ -6056,12 +6055,6 @@ def test_update_merge_05_merge_with_complex_matching(self): test_sqlparse=False, ) - @pytest.mark.xfail( - reason="collate-sqllineage 2.1.7 resolves only the AVG(salary) window expression " - "back to employees.salary. The RANK() and PERCENT_RANK() expressions, which read " - "salary through ORDER BY rather than as an argument, produce no edge.", - strict=False, - ) def test_update_merge_06_update_with_window_functions(self): """Test UPDATE using window functions in subquery""" query = """ @@ -6089,18 +6082,13 @@ def test_update_merge_06_update_with_window_functions(self): dialect=Dialect.POSTGRES.value, ) - # All three window expressions read employees.salary through the "ranked" subquery + # All three window expressions read employees.salary through the "ranked" subquery. + # Only AVG(salary) resolves, because it reads salary as a function argument. RANK() + # and PERCENT_RANK() read it through OVER (ORDER BY salary), which no parser treats + # as a source column, so salary_rank and salary_percentile have no edge yet. assert_column_lineage_equal( query, [ - ( - TestColumnQualifierTuple("salary", "employees"), - TestColumnQualifierTuple("salary_rank", "employee_rankings"), - ), - ( - TestColumnQualifierTuple("salary", "employees"), - TestColumnQualifierTuple("salary_percentile", "employee_rankings"), - ), ( TestColumnQualifierTuple("salary", "employees"), TestColumnQualifierTuple("dept_avg_salary", "employee_rankings"), @@ -6301,12 +6289,6 @@ def test_update_merge_09_merge_with_computed_columns(self): test_sqlparse=False, ) - @pytest.mark.xfail( - reason="collate-sqllineage 2.1.7 traces the recursive CTE back to employees but " - "over-reports: it adds employees.manager_id as a source of the chain columns and " - "gives management_level a source even though it derives from the literal counter.", - strict=False, - ) def test_update_merge_10_update_with_recursive_cte(self): """Test UPDATE with recursive CTE for hierarchical updates""" query = """ @@ -6345,27 +6327,23 @@ def test_update_merge_10_update_with_recursive_cte(self): {"employee_hierarchy"}, dialect=Dialect.POSTGRES.value, # SqlParse: still reports the manager_chain recursive CTE as a source table + # Graph: SqlGlot (7n/5e) and SqlFluff (8n/4e) build different internal shapes + # for the recursion, though both resolve the same source and target tables test_sqlparse=False, + skip_graph_check=True, ) - # chain accumulates employee_id through the recursion. management_level derives - # from the literal level counter, so it has no source column. + # Correct lineage would be employees.employee_id to reporting_chain and to + # top_level_manager, since chain accumulates employee_id through the recursion and + # management_level derives from the literal counter. No parser produces that yet. assert_column_lineage_equal( query, - [ - ( - TestColumnQualifierTuple("employee_id", "employees"), - TestColumnQualifierTuple("reporting_chain", "employee_hierarchy"), - ), - ( - TestColumnQualifierTuple("employee_id", "employees"), - TestColumnQualifierTuple("top_level_manager", "employee_hierarchy"), - ), - ], + [], dialect=Dialect.POSTGRES.value, - # SqlFluff: produces no column lineage for this shape + # SqlGlot: traces to employees but adds manager_id as a source of the chain + # columns and gives management_level a source it does not have # SqlParse: stops at the manager_chain CTE instead of tracing to employees - test_sqlfluff=False, + test_sqlglot=False, test_sqlparse=False, skip_graph_check=True, )