Fixes #31790: [1.13] upgrade collate-sqllineage and require Python 3.10

ingestion/README.md

@@ -2,12 +2,12 @@
 This guide will help you setup the Ingestion framework and connectors
 ---
 
-![Python version 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)
+![Python version 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)
 
 OpenMetadata Ingestion is a simple framework to build connectors and ingest metadata of various systems through OpenMetadata APIs. It could be used in an orchestration framework(e.g. Apache Airflow) to ingest metadata.
 **Prerequisites**
 
-- Python >= 3.9.x
+- Python >= 3.10.x
 
 ### Docs
 

ingestion/noxfile.py

@@ -11,6 +11,7 @@
 """
 Nox sessions for testing and formatting checks.
 """
+
 import os
 
 import nox
@@ -21,8 +22,6 @@
 #    - Add integration tests
 #    - Address the TODOs in the code
 
-# TODO: Add python 3.9. PYTHON 3.9 fails in Mac os due to problem with `psycopg2-binary` package
-
 SUPPORTED_PYTHON_VERSIONS = ["3.10", "3.11", "3.12"]
 
 

ingestion/operators/docker/Dockerfile

@@ -230,47 +230,10 @@ USER openmetadata
 # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and
 # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is
 # what keeps scanners reporting CVE-2026-59890.
-# Keep this the LAST pip layer that can compile anything -- a later layer that builds a
-# package needing pkg_resources would fail here, and the error would not look like a
-# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt.
+# Keep this the LAST pip layer -- a later layer that compiles a package needing
+# pkg_resources would fail here, and the error would not look like a setuptools problem.
 RUN pip install --upgrade "setuptools>=83"
 
-# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893,
-# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in
-# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no
-# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us:
-#     collate-sqllineage 2.1.4                     sqlparse==0.5.4
-#     dbt-core (transitive via collate-data-diff)  sqlparse<0.6.0
-# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with
-# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core --
-# the two releases are byte-identical apart from the version string, so 0.6.0 is a version
-# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and
-# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete
-# this layer and raise the floors in ingestion/setup.py instead.
-#
-# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install`
-# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two
-# unsatisfied pins for the life of the image, so the import gate is what actually keeps
-# this honest. It asserts the two things that would otherwise fail silently: that we really
-# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites.
-# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a
-# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage
-# comes back quietly truncated with nothing failing.
-RUN pip install --no-deps "sqlparse==0.6.0" \
- && python -W ignore -c "\
-import sqlparse; \
-from sqlparse.engine import grouping; \
-from sqlparse.keywords import KEYWORDS; \
-import collate_sqllineage.core.parser.sqlparse; \
-from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \
-from collate_sqllineage.runner import LineageRunner; \
-assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \
-assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \
-assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \
-r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \
-assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \
-assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables"
-
 
 # Strip spaCy's bundled test fixture, which scanners misreport as an installed black.
 # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the

ingestion/operators/docker/Dockerfile.ci

@@ -238,47 +238,10 @@ USER openmetadata
 # build-time only: cx_Oracle and mysqlclient import pkg_resources from their setup.py and
 # 81.0.0 removed it, but once built both import fine against 83+. Leaving 80.x on disk is
 # what keeps scanners reporting CVE-2026-59890.
-# Keep this the LAST pip layer that can compile anything -- a later layer that builds a
-# package needing pkg_resources would fail here, and the error would not look like a
-# setuptools problem. The sqlparse override below is a pure-Python wheel, so it is exempt.
+# Keep this the LAST pip layer -- a later layer that compiles a package needing
+# pkg_resources would fail here, and the error would not look like a setuptools problem.
 RUN pip install --upgrade "setuptools>=83"
 
-# Force sqlparse past two declared ceilings to clear CVE-2026-54284, CVE-2026-59893,
-# CVE-2026-71491 (parser CPU-exhaustion DoS) and CVE-2026-59894 (SQL string breakout in
-# the python/php output formats). All four are fixed only in 0.6.0 -- OSV reports no
-# patched 0.5.x -- so no in-range version is clean and the resolver cannot help us:
-#     collate-sqllineage 2.1.4                     sqlparse==0.5.4
-#     dbt-core (transitive via collate-data-diff)  sqlparse<0.6.0
-# Both ceilings are stale rather than substantive. collate-sqllineage 2.1.5 shipped with
-# sqlparse==0.6.0 and 2.1.6 reverted only the pin to stay co-installable with dbt-core --
-# the two releases are byte-identical apart from the version string, so 0.6.0 is a version
-# upstream already released against. dbt-core's ceiling predates 0.6.0 by nine months and
-# is tracked at https://github.com/dbt-labs/dbt-core/issues/15988. Once that lands, delete
-# this layer and raise the floors in ingestion/setup.py instead.
-#
-# --no-deps because pip would otherwise backtrack on the declared conflict. `pip install`
-# exits 0 while printing the resolver-conflict ERROR, and `pip check` will report the two
-# unsatisfied pins for the life of the image, so the import gate is what actually keeps
-# this honest. It asserts the two things that would otherwise fail silently: that we really
-# got 0.6.x, and that collate-sqllineage's monkeypatch of sqlparse internals still bites.
-# That patch raises MAX_GROUPING_DEPTH/MAX_GROUPING_TOKENS 100x and retypes STRING as a
-# builtin; if a future sqlparse renames either, the patch degrades to a no-op and lineage
-# comes back quietly truncated with nothing failing.
-RUN pip install --no-deps "sqlparse==0.6.0" \
- && python -W ignore -c "\
-import sqlparse; \
-from sqlparse.engine import grouping; \
-from sqlparse.keywords import KEYWORDS; \
-import collate_sqllineage.core.parser.sqlparse; \
-from collate_sqllineage.core.parser.sqlparse.analyzer import SqlParseLineageAnalyzer; \
-from collate_sqllineage.runner import LineageRunner; \
-assert sqlparse.__version__.startswith('0.6.'), sqlparse.__version__; \
-assert (grouping.MAX_GROUPING_DEPTH, grouping.MAX_GROUPING_TOKENS) == (10000, 1000000), 'sqllineage grouping patch is a no-op'; \
-assert str(KEYWORDS['STRING']) == 'Token.Name.Builtin', 'sqllineage keyword patch is a no-op'; \
-r = LineageRunner('INSERT INTO db.sch.tgt SELECT c FROM db.sch.src', analyzer=SqlParseLineageAnalyzer); \
-assert [str(t) for t in r.source_tables] == ['db.sch.src'], r.source_tables; \
-assert [str(t) for t in r.target_tables] == ['db.sch.tgt'], r.target_tables"
-
 
 # Strip spaCy's bundled test fixture, which scanners misreport as an installed black.
 # See ingestion/scripts/strip_spacy_test_fixture.sh for the rationale. Must run after the

ingestion/pyproject.toml

@@ -13,7 +13,7 @@ authors = [
 ]
 license = { file = "LICENSE" }
 description = "Ingestion Framework for OpenMetadata"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 [project.urls]
 Homepage = "https://open-metadata.org/"
@@ -346,4 +346,3 @@ allowedUntypedLibraries = []
 # [[tool.basedpyright.executionEnvironments]]
 # root = "src/metadata/utils"
 # reportMissingParameterType = "error"
-

ingestion/setup.py

@@ -180,7 +180,7 @@
     "requests>=2.23",
     "requests-aws4auth~=1.1",  # Only depends on requests as external package. Leaving as base.
     "sqlalchemy>=2.0.0,<3",
-    "collate-sqllineage==2.1.4",
+    "collate-sqllineage==2.1.7",
     "tabulate==0.9.0",
     "tenacity>=8.0,<10",
     "typing-inspect",