From 7ccce2262674e89b110592f3e5ba4f2527701794 Mon Sep 17 00:00:00 2001 From: Khairajani Date: Fri, 21 Aug 2026 18:15:15 +0530 Subject: [PATCH 1/5] fix(security): pull expat 2.8.3-1~deb13u1 into the ingestion-base image The image reports expat 2.8.2 for CVE-2026-72522 (DSA-6446-1: out-of-bounds read and resultant infinite loop in libexpat's *_toUtf16 surrogate handling); trixie-security has 2.8.3-1~deb13u1. expat is not in the python:3.12-slim-trixie base at all -- it comes in transitively from the top-of-file apt install -- and that layer is exactly why the image is stuck at 2.8.2: its cache key never changes, so it keeps resolving against the Debian index that was current when it was first built. Adding expat there would freeze the fix the same way. It goes in the late root layer instead, below the COPY and pip layers, where the index is re-read on every build -- the same reasoning that put util-linux there. expat rides in the existing dpkg source-package query rather than a second RUN, so one apt call covers both source packages and the next OS CVE is a one-line edit. The package set stays computed from dpkg rather than hand-listed for the reason already documented there: scanners report each binary of a source package separately, so a hand-written list silently leaves one behind. util-linux keeps its non-empty assert and expat deliberately does not get one. util-linux is Essential, so an empty query there means dpkg-query misbehaved and the build must stop; expat is transitive, so an empty query is a legitimate image with nothing to patch and failing on it would break the build for no reason. Asserting one of the two still covers the hazard the check exists for: `apt-get install --only-upgrade` with no package arguments exits 0, so an all-empty query would give a green build that shipped the vulnerable packages. Verified in a real build layer on python:3.12-slim-trixie: libexpat1 2.7.1-2 -> 2.8.3-1~deb13u1 and all nine util-linux binaries -> 2.41.5-0+deb13u1, exit 0. The expat-absent path was exercised on the bare base and also exits 0, upgrading util-linux alone. ingestion/Dockerfile gets no equivalent change because there is nothing to upgrade to, not because it is clean: apache/airflow:3.3.0-python3.12 is bookworm, Debian still marks bookworm and bookworm-security vulnerable, and the image already carries 2.5.0-1+deb12u2 -- the newest build either suite offers. It needs revisiting when Debian ships a bookworm fix. --- ingestion/operators/docker/Dockerfile | 33 +++++++++++++++--------- ingestion/operators/docker/Dockerfile.ci | 33 +++++++++++++++--------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 742d7ff071f9..989104b00272 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -197,29 +197,38 @@ RUN pip install psycopg2 mysqlclient==2.1.1 # apt-mark manual pins them before autoremove runs so the cleanup doesn't take # them out too -- verified end-to-end by the `import MySQLdb` gate check. # -# The util-linux upgrade rides along in the same root layer. trixie's base ships -# 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and CVE-2026-27456; -# trixie-security has 2.41.5-0+deb13u1. -# The package set is computed from dpkg rather than hand-listed. One source +# The OS security upgrades ride along in the same root layer. trixie's base ships +# util-linux 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and +# CVE-2026-27456; trixie-security has 2.41.5-0+deb13u1. +# expat is the same story one release later. It is not in the trixie base at all +# -- it arrives transitively from the top-of-file apt layer -- and that layer's +# frozen index is exactly why the image keeps shipping 2.8.2 and being reported +# for CVE-2026-72522 while trixie-security has 2.8.3-1~deb13u1. +# The package sets are computed from dpkg rather than hand-listed. One source # package produces many binaries -- here util-linux, bsdutils, login, mount, # liblastlog2-2, libblkid1, libmount1, libsmartcols1 and libuuid1 -- and scanners # report each separately, so a hand-written list silently leaves behind whichever # binary it forgot, and liblastlog2-2 is exactly the one that is easy to forget. -# Asking dpkg which installed packages came from the util-linux source cannot -# miss one, and stays correct if Debian splits the source differently later. -# The empty check is not defensive noise: `apt-get install --only-upgrade` with -# no package arguments exits 0, so a query that silently returned nothing would -# give a green build that shipped the vulnerable packages anyway. Fail closed. +# Asking dpkg which installed packages came from the source cannot miss one, and +# stays correct if Debian splits the source differently later. +# util-linux is asserted non-empty and expat deliberately is not: util-linux is +# Essential, so an empty query there means dpkg-query misbehaved and the build +# must not continue, whereas expat is transitive and an empty query is a +# legitimate image with nothing to patch. Asserting one of the two is what the +# check is for -- `apt-get install --only-upgrade` with no package arguments +# exits 0, so an all-empty query would give a green build that shipped the +# vulnerable packages anyway. Fail closed. # This deliberately does NOT go in the top-of-file apt RUN: that layer's cache # key never changes, so an upgrade placed there freezes its Debian index with it # and the image keeps shipping whatever was current when the layer was first # built. Below the COPY and pip layers, the index is re-read on every build. USER root RUN set -eu; \ - pkgs="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ - [ -n "$pkgs" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ul="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ + [ -n "$ul" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ex="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="expat"{print $2}')"; \ apt-get -qq update; \ - apt-get -qq install -y --only-upgrade $pkgs; \ + apt-get -qq install -y --only-upgrade $ul $ex; \ apt-get -qq purge -y libmariadb-dev libmariadb-dev-compat libunbound8; \ apt-mark manual libmariadb3 mariadb-common; \ apt-get -qq autoremove -y --purge; \ diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index c7fac3356237..ede6cd033875 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -205,29 +205,38 @@ RUN pip install psycopg2 mysqlclient==2.1.1 # apt-mark manual pins them before autoremove runs so the cleanup doesn't take # them out too -- verified end-to-end by the `import MySQLdb` gate check. # -# The util-linux upgrade rides along in the same root layer. trixie's base ships -# 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and CVE-2026-27456; -# trixie-security has 2.41.5-0+deb13u1. -# The package set is computed from dpkg rather than hand-listed. One source +# The OS security upgrades ride along in the same root layer. trixie's base ships +# util-linux 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and +# CVE-2026-27456; trixie-security has 2.41.5-0+deb13u1. +# expat is the same story one release later. It is not in the trixie base at all +# -- it arrives transitively from the top-of-file apt layer -- and that layer's +# frozen index is exactly why the image keeps shipping 2.8.2 and being reported +# for CVE-2026-72522 while trixie-security has 2.8.3-1~deb13u1. +# The package sets are computed from dpkg rather than hand-listed. One source # package produces many binaries -- here util-linux, bsdutils, login, mount, # liblastlog2-2, libblkid1, libmount1, libsmartcols1 and libuuid1 -- and scanners # report each separately, so a hand-written list silently leaves behind whichever # binary it forgot, and liblastlog2-2 is exactly the one that is easy to forget. -# Asking dpkg which installed packages came from the util-linux source cannot -# miss one, and stays correct if Debian splits the source differently later. -# The empty check is not defensive noise: `apt-get install --only-upgrade` with -# no package arguments exits 0, so a query that silently returned nothing would -# give a green build that shipped the vulnerable packages anyway. Fail closed. +# Asking dpkg which installed packages came from the source cannot miss one, and +# stays correct if Debian splits the source differently later. +# util-linux is asserted non-empty and expat deliberately is not: util-linux is +# Essential, so an empty query there means dpkg-query misbehaved and the build +# must not continue, whereas expat is transitive and an empty query is a +# legitimate image with nothing to patch. Asserting one of the two is what the +# check is for -- `apt-get install --only-upgrade` with no package arguments +# exits 0, so an all-empty query would give a green build that shipped the +# vulnerable packages anyway. Fail closed. # This deliberately does NOT go in the top-of-file apt RUN: that layer's cache # key never changes, so an upgrade placed there freezes its Debian index with it # and the image keeps shipping whatever was current when the layer was first # built. Below the COPY and pip layers, the index is re-read on every build. USER root RUN set -eu; \ - pkgs="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ - [ -n "$pkgs" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ul="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ + [ -n "$ul" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ex="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="expat"{print $2}')"; \ apt-get -qq update; \ - apt-get -qq install -y --only-upgrade $pkgs; \ + apt-get -qq install -y --only-upgrade $ul $ex; \ apt-get -qq purge -y libmariadb-dev libmariadb-dev-compat libunbound8; \ apt-mark manual libmariadb3 mariadb-common; \ apt-get -qq autoremove -y --purge; \ From 5d283f5562fbc7dd2c2f93611004a63be28f5cee Mon Sep 17 00:00:00 2001 From: Khairajani Date: Fri, 21 Aug 2026 18:15:53 +0530 Subject: [PATCH 2/5] fix(security): bump apache-airflow 3.3.0 -> 3.3.1 for CVE-2026-67587 / CVE-2026-54183 CVE-2026-67587 (High) -- the Task SDK rebuilt a Serde `Callback` by re-running its constructor, which imports the module named by the stored callback path. `SyncCallback` is an Airflow class, so it passes the default `allowed_deserialization_classes` allow-list and tightening that setting does not help. A Dag author controls a task instance's `next_kwargs`, so they can get an arbitrary module imported inside the scheduler process when the `awaiting_input` timeout sweep deserializes that value. No non-default configuration is required. CVE-2026-54183 (Medium) -- the secrets masker's recursion-depth limit did not descend into values nested inside a list, tuple or set, so a Variable holding a deeply nested sensitive value rendered unmasked in the Variables UI. Anyone who can see the Variable in the UI can already read it through the Variables REST API, so this is a shoulder-surfing defense rather than a disclosure boundary -- hence Medium. Both are fixed in 3.3.1 and nothing in the 3.3.0 line is clean. The pin, both base image tags, the vendored constraints file and the integration-test image move together, as they did for 3.2.2 -> 3.3.0 in #31338. The constraints file is a straight re-download this time. It carried two hand-patches marked "keep on regeneration" -- impyla and thrift at 0.24.0 for CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586, against upstream constraints-3.3.0 shipping impyla==0.22.0 and thrift==0.16.0 -- and upstream constraints-3.3.1 now ships both at 0.24.0 natively. Diffing the vendored 3.3.0 file against upstream constraints-3.3.0 confirms those two blocks were the only divergence, and the new file is byte-identical to upstream constraints-3.3.1 outside comments. The comments are kept, reworded to say the pin is now upstream-native, so a future regeneration off a branch that regressed either version still gets caught. Two comments that name a constraint were checked rather than blind-renumbered: - tests/integration/airflow/Dockerfile explains the constraints are deliberately not applied because they pin chardet against openmetadata-ingestion's chardet==4.0.0. Still true; the version moved 6.0.0.post1 -> 7.5.1, so the number is updated. - Dockerfile.ci's universal-pathlib workaround cites Airflow's >=0.3.8 floor. apache-airflow-core 3.3.1 still declares `universal-pathlib>=0.3.8` and constraints still pin 0.3.10, so only the Airflow version in the prose changes. Resolution verified with `uv pip compile --python-version 3.12 --extra airflow`: apache-airflow==3.3.1 co-installs with every transitive floor pin in the airflow extra (providers-http, -opensearch, -elasticsearch, tornado, Werkzeug, starlette, python-multipart) with no conflict. sqlparse resolves to 0.5.4 there via collate-sqllineage, unchanged by this PR -- the release images force 0.6.0 in ingestion/operators/docker/Dockerfile, which does not consume this constraints file. Folded in: drop the stale croniter<3 ceiling from the dagster extra `uv pip install ingestion[all]` + `[test]` in the CI test environment fails to resolve against apache-airflow 3.3.1: Because apache-airflow-core>=3.3.1 depends on croniter>=6.2.2 [...] and because openmetadata-ingestion[test]==2.0.0.0.dev0 depends on apache-airflow==3.3.1 and croniter<3, we can conclude that [...] requirements are unsatisfiable. apache-airflow-core raised its croniter floor from >=2.0.2 to >=6.2.2 between 3.3.0 and 3.3.1, so the ceiling that used to co-exist with airflow now excludes it. The ceiling is dead weight and predates this conflict -- it has been in the dagster extra since #6416 and survived the #15679 dependency cleanup: - dagster 1.13.18 (what `dagster_graphql>=1.8.0` resolves to) declares no croniter dependency at all; resolving dagster_graphql alone produces no croniter. - Nothing under ingestion/ imports croniter. The only importer in the repo is openmetadata-airflow-apis, a separate distribution that does not declare it and picks it up transitively from airflow. - That importer is already running croniter 6.2.x: constraints-3.3.0 pins croniter==6.2.2 and the airflow images install under it, so 6.x compatibility for `croniter.is_valid` / `get_prev` is established by the shipped image, not assumed here. The call site is also behind an `is_airflow_3_or_higher()` guard and unreachable on Airflow 3. Only the CI test environment resolved croniter 2.x, because it installs from the extras without the constraints file. Removing the ceiling makes it match the images. Verified with `uv pip compile --extra all --extra test` on both 3.10 and 3.12: resolves to apache-airflow==3.3.1, croniter==6.2.4, dagster==1.13.18 with no conflict. This was the only pin in the tree that collided with the airflow bump. --- ingestion/Dockerfile | 4 +- ingestion/Dockerfile.ci | 10 +- ....3.0.txt => airflow-constraints-3.3.1.txt} | 527 +++++++++--------- ingestion/setup.py | 11 +- .../tests/integration/airflow/Dockerfile | 6 +- .../tests/integration/airflow/conftest.py | 2 +- 6 files changed, 291 insertions(+), 269 deletions(-) rename ingestion/{airflow-constraints-3.3.0.txt => airflow-constraints-3.3.1.txt} (61%) diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index e357849bba58..bafcefec5751 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -1,6 +1,6 @@ FROM mysql:8.3 AS mysql -FROM apache/airflow:3.3.0-python3.12 +FROM apache/airflow:3.3.1-python3.12 USER root RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list @@ -122,7 +122,7 @@ ARG RI_VERSION="2.0.0.0.dev0" RUN pip install --upgrade "pip>=26.2,<27" "setuptools<81" # Pre-install cx-Oracle without build isolation to use the pinned setuptools RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" -RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.3.0/constraints-3.12.txt" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.3.1/constraints-3.12.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index 83895664e419..d72cce355f91 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -1,6 +1,6 @@ FROM mysql:8.3 AS mysql -FROM apache/airflow:3.3.0-python3.12 +FROM apache/airflow:3.3.1-python3.12 USER root RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list @@ -94,7 +94,7 @@ COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airf # Required for Airflow DAGs of Sample Data COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags COPY --chown=airflow:0 ingestion/examples/airflow/test_dags /opt/airflow/dags -COPY --chown=airflow:0 ingestion/airflow-constraints-3.3.0.txt /home/airflow/airflow-constraints-3.3.0.txt +COPY --chown=airflow:0 ingestion/airflow-constraints-3.3.1.txt /home/airflow/airflow-constraints-3.3.1.txt # Refresh the interpreter-level pip before dropping privileges. The per-user # pip upgrade below only reaches ~/.local, leaving the base image's @@ -122,11 +122,11 @@ RUN pip install --upgrade "pip>=26.2,<27" "setuptools<81" RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" # Install FAB provider for Airflow 3.x Flask Blueprint compatibility -RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.3.0.txt" || true +RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.3.1.txt" || true WORKDIR /home/airflow/openmetadata-airflow-apis -RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.3.0.txt" +RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.3.1.txt" WORKDIR /home/airflow/ingestion @@ -169,7 +169,7 @@ RUN pip install psycopg2 mysqlclient==2.1.1 RUN mkdir -p /opt/airflow/dag_generated_configs EXPOSE 8080 -# Airflow 3.3.0 requires universal-pathlib>=0.3.8, but prior installs in this image +# Airflow 3.3.1 requires universal-pathlib>=0.3.8, but prior installs in this image # can leave stale 0.2.6 `upath` module files in site-packages that cause import # errors at runtime. Force-remove the stale registration then pin to the required version. RUN pip uninstall upath -y && pip install "universal-pathlib==0.3.10" diff --git a/ingestion/airflow-constraints-3.3.0.txt b/ingestion/airflow-constraints-3.3.1.txt similarity index 61% rename from ingestion/airflow-constraints-3.3.0.txt rename to ingestion/airflow-constraints-3.3.1.txt index dfffbbe5452f..3921501f8693 100644 --- a/ingestion/airflow-constraints-3.3.0.txt +++ b/ingestion/airflow-constraints-3.3.1.txt @@ -1,6 +1,6 @@ # -# This constraints file was automatically generated on 2026-07-06T12:27:13.807760 +# This constraints file was automatically generated on 2026-08-12T08:36:03.248024 # via `uv pip install --resolution highest` for the "v3-3-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. @@ -35,19 +35,19 @@ Authlib==1.7.2 Deprecated==1.3.1 Events==0.5 Flask-JWT-Extended==4.7.4 -Flask-Limiter==3.12 +Flask-Limiter==4.1.1 Flask-Login==0.6.3 Flask-SQLAlchemy==3.1.1 Flask-Session==0.8.0 Flask-WTF==1.3.0 Flask==3.1.3 -GitPython==3.1.50 +GitPython==3.1.58 JayDeBeApi==1.2.3 Jinja2==3.1.6 -Mako==1.3.12 -Markdown==3.10.2 +Mako==1.4.1 +Markdown==3.10.3 MarkupSafe==3.0.3 -PyAthena==3.32.0 +PyAthena==3.35.4 PyGithub==2.9.1 PyHive==0.7.0 PyJWT==2.13.0 @@ -62,170 +62,174 @@ WTForms==3.2.2 Werkzeug==3.1.8 a2wsgi==1.10.10 adal==1.2.7 +adbc-driver-manager==1.12.0 +adbc-driver-postgresql==1.12.0 +adbc-driver-sqlite==1.12.0 adlfs==2026.5.0 aenum==3.1.17 -aiobotocore==3.7.0 +aiobotocore==3.9.0 aiofiles==24.1.0 aiohappyeyeballs==2.7.1 aiohttp-cors==0.8.1 -aiohttp==3.14.1 +aiohttp==3.14.3 aioitertools==0.13.0 aiomysql==0.3.2 aiosignal==1.4.0 aiosmtplib==5.1.2 -aiosqlite==0.21.0 +aiosqlite==0.22.1 airbyte-api==1.0.1 akeyless==5.0.28 -alembic==1.18.5 +alembic==1.19.0 alibabacloud-adb20211201==3.7.0 alibabacloud-credentials-api==1.0.1 -alibabacloud-credentials==1.0.9 +alibabacloud-credentials==1.0.11 alibabacloud-gateway-spi==0.0.4 alibabacloud-openapi-util==0.2.4 -alibabacloud-oss-v2==1.3.1 -alibabacloud-tea-util==0.3.14 +alibabacloud-oss-v2==1.3.2 +alibabacloud-tea-util==0.3.15 alibabacloud-tea==0.4.3 alibabacloud_endpoint_util==0.0.4 alibabacloud_tea_openapi==0.3.16 alibabacloud_tea_xml==0.0.3 amqp==5.3.1 -annotated-doc==0.0.4 -annotated-types==0.7.0 -anyio==4.14.1 -apache-airflow-providers-airbyte==5.5.1 -apache-airflow-providers-akeyless==0.2.0 -apache-airflow-providers-alibaba==3.3.9 -apache-airflow-providers-amazon==9.31.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.14.2 +apache-airflow-providers-airbyte==6.0.1 +apache-airflow-providers-akeyless==0.3.0 +apache-airflow-providers-alibaba==3.4.0 +apache-airflow-providers-amazon==9.34.0 apache-airflow-providers-apache-cassandra==3.9.5 apache-airflow-providers-apache-drill==3.3.3 apache-airflow-providers-apache-druid==4.5.2 apache-airflow-providers-apache-flink==1.8.5 -apache-airflow-providers-apache-hdfs==4.12.1 -apache-airflow-providers-apache-hive==9.5.0 -apache-airflow-providers-apache-iceberg==1.4.1 +apache-airflow-providers-apache-hdfs==4.12.2 +apache-airflow-providers-apache-hive==9.6.1 +apache-airflow-providers-apache-iceberg==2.0.3 apache-airflow-providers-apache-impala==1.9.3 -apache-airflow-providers-apache-kafka==1.14.0 +apache-airflow-providers-apache-kafka==1.15.1 apache-airflow-providers-apache-kylin==3.10.5 -apache-airflow-providers-apache-livy==4.5.7 +apache-airflow-providers-apache-livy==4.6.0 apache-airflow-providers-apache-pig==4.8.5 apache-airflow-providers-apache-pinot==4.10.3 -apache-airflow-providers-apache-spark==6.2.0 +apache-airflow-providers-apache-spark==6.3.1 apache-airflow-providers-apache-tinkerpop==1.1.4 apache-airflow-providers-apprise==2.3.4 -apache-airflow-providers-arangodb==2.9.5 +apache-airflow-providers-arangodb==2.9.6 apache-airflow-providers-asana==2.11.4 -apache-airflow-providers-atlassian-jira==3.3.4 -apache-airflow-providers-celery==3.21.0 +apache-airflow-providers-atlassian-jira==3.3.5 +apache-airflow-providers-celery==3.23.1 apache-airflow-providers-clickhousedb==1.0.0 apache-airflow-providers-cloudant==4.3.5 -apache-airflow-providers-cncf-kubernetes==10.19.0 -apache-airflow-providers-cohere==1.6.6 -apache-airflow-providers-common-ai==0.5.0 -apache-airflow-providers-common-compat==1.15.0 +apache-airflow-providers-cncf-kubernetes==10.21.0 +apache-airflow-providers-cohere==1.6.7 +apache-airflow-providers-common-ai==0.7.0 +apache-airflow-providers-common-compat==1.18.0 apache-airflow-providers-common-io==1.8.0 apache-airflow-providers-common-messaging==2.0.4 -apache-airflow-providers-common-sql==2.0.1 -apache-airflow-providers-databricks==7.16.1 +apache-airflow-providers-common-sql==2.1.0 +apache-airflow-providers-databricks==7.18.1 apache-airflow-providers-datadog==3.10.5 -apache-airflow-providers-dbt-cloud==4.9.2 +apache-airflow-providers-dbt-cloud==4.9.3 apache-airflow-providers-dingding==3.9.5 apache-airflow-providers-discord==3.12.3 -apache-airflow-providers-docker==4.5.7 -apache-airflow-providers-edge3==4.0.0 -apache-airflow-providers-elasticsearch==6.7.0 -apache-airflow-providers-exasol==4.10.3 -apache-airflow-providers-fab==3.7.1 +apache-airflow-providers-docker==4.5.9 +apache-airflow-providers-edge3==4.3.0 +apache-airflow-providers-elasticsearch==6.9.0 +apache-airflow-providers-exasol==4.10.5 +apache-airflow-providers-fab==3.8.0 apache-airflow-providers-facebook==3.9.5 -apache-airflow-providers-ftp==3.15.1 -apache-airflow-providers-git==0.4.0 +apache-airflow-providers-ftp==3.15.2 +apache-airflow-providers-git==0.4.2 apache-airflow-providers-github==2.11.3 -apache-airflow-providers-google==22.1.0 +apache-airflow-providers-google==22.3.0 apache-airflow-providers-grpc==3.9.5 -apache-airflow-providers-hashicorp==4.7.1 -apache-airflow-providers-http==6.0.4 -apache-airflow-providers-imap==3.11.3 +apache-airflow-providers-hashicorp==4.8.0 +apache-airflow-providers-http==6.0.5 +apache-airflow-providers-imap==3.12.1 apache-airflow-providers-influxdb==2.11.0 apache-airflow-providers-informatica==0.2.0 apache-airflow-providers-jdbc==5.5.0 apache-airflow-providers-jenkins==4.2.6 -apache-airflow-providers-keycloak==0.8.1 -apache-airflow-providers-microsoft-azure==13.5.0 +apache-airflow-providers-keycloak==0.8.2 +apache-airflow-providers-microsoft-azure==14.1.0 apache-airflow-providers-microsoft-mssql==4.7.0 apache-airflow-providers-microsoft-psrp==3.2.6 -apache-airflow-providers-microsoft-winrm==3.14.3 +apache-airflow-providers-microsoft-winrm==3.14.4 apache-airflow-providers-mongo==5.4.0 apache-airflow-providers-mysql==6.6.1 -apache-airflow-providers-neo4j==3.11.6 +apache-airflow-providers-neo4j==3.12.1 apache-airflow-providers-odbc==4.12.3 -apache-airflow-providers-openai==1.7.5 +apache-airflow-providers-openai==1.8.2 apache-airflow-providers-openfaas==3.9.5 -apache-airflow-providers-openlineage==2.18.1 -apache-airflow-providers-opensearch==1.10.0 +apache-airflow-providers-openlineage==2.20.0 +apache-airflow-providers-opensearch==1.12.0 apache-airflow-providers-opsgenie==5.10.4 -apache-airflow-providers-oracle==4.6.1 +apache-airflow-providers-oracle==4.6.2 apache-airflow-providers-pagerduty==5.2.6 -apache-airflow-providers-papermill==3.13.1 -apache-airflow-providers-pgvector==1.7.2 +apache-airflow-providers-papermill==3.13.2 +apache-airflow-providers-pgvector==1.7.3 apache-airflow-providers-pinecone==2.4.5 -apache-airflow-providers-postgres==6.8.0 -apache-airflow-providers-presto==5.12.0 +apache-airflow-providers-postgres==7.0.1 +apache-airflow-providers-presto==5.12.1 apache-airflow-providers-qdrant==1.5.6 apache-airflow-providers-redis==4.5.0 -apache-airflow-providers-salesforce==5.14.1 +apache-airflow-providers-salesforce==5.14.2 apache-airflow-providers-samba==4.12.6 apache-airflow-providers-segment==3.9.5 apache-airflow-providers-sendgrid==4.2.4 -apache-airflow-providers-sftp==5.8.2 +apache-airflow-providers-sftp==6.0.1 apache-airflow-providers-singularity==3.9.4 apache-airflow-providers-slack==9.10.2 -apache-airflow-providers-smtp==3.0.1 -apache-airflow-providers-snowflake==6.4.0 +apache-airflow-providers-smtp==3.0.3 +apache-airflow-providers-snowflake==6.16.0 apache-airflow-providers-sqlite==4.3.3 -apache-airflow-providers-ssh==5.0.3 -apache-airflow-providers-standard==1.15.0 -apache-airflow-providers-tableau==5.5.0 -apache-airflow-providers-telegram==4.9.5 -apache-airflow-providers-teradata==3.6.0 -apache-airflow-providers-trino==6.6.0 +apache-airflow-providers-ssh==6.0.1 +apache-airflow-providers-standard==1.17.0 +apache-airflow-providers-tableau==5.6.0 +apache-airflow-providers-telegram==4.9.6 +apache-airflow-providers-teradata==3.6.2 +apache-airflow-providers-trino==6.6.1 apache-airflow-providers-vertica==4.4.0 apache-airflow-providers-vespa==0.1.1 -apache-airflow-providers-weaviate==3.3.5 -apache-airflow-providers-yandex==4.5.0 +apache-airflow-providers-weaviate==3.4.1 +apache-airflow-providers-yandex==4.5.1 apache-airflow-providers-ydb==2.5.3 apache-airflow-providers-zendesk==4.12.0 apispec==6.10.0 -apprise==1.11.0 -argcomplete==3.7.0 +apprise==1.12.0 +argcomplete==3.7.2 arrow==1.4.0 asana==5.2.5 -asgiref==3.11.1 +asgiref==3.12.1 asn1crypto==1.5.1 -asttokens==3.0.1 +asttokens==3.0.2 async-timeout==4.0.3 asyncpg==0.31.0 asyncssh==2.24.0 atlasclient==1.0.0 atlassian-python-api==4.0.7 attrs==26.1.0 +azure-ai-projects==2.4.0 azure-batch==14.2.0 azure-common==1.1.28 azure-core==1.41.0 -azure-cosmos==4.16.1 +azure-cosmos==4.16.3 azure-datalake-store==0.0.53 azure-identity==1.25.3 azure-keyvault-secrets==4.11.0 azure-kusto-data==6.0.4 -azure-mgmt-compute==38.1.0 +azure-mgmt-compute==38.2.0 azure-mgmt-containerinstance==10.1.0 azure-mgmt-containerregistry==15.0.0 azure-mgmt-core==1.6.0 -azure-mgmt-cosmosdb==9.9.0 -azure-mgmt-datafactory==9.3.0 +azure-mgmt-cosmosdb==10.0.0 +azure-mgmt-datafactory==10.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==26.0.0 -azure-mgmt-storage==25.0.0 +azure-mgmt-storage==25.1.0 azure-nspkg==3.0.2 azure-servicebus==7.14.3 azure-storage-blob==12.30.0 @@ -235,47 +239,48 @@ azure-synapse-artifacts==0.22.0 azure-synapse-spark==0.7.0 babel==2.18.0 backoff==2.2.1 +backports.zstd==1.6.0 bcrypt==5.0.0 beautifulsoup4==4.15.0 billiard==4.2.4 -bitarray==3.8.2 +bitarray==3.10.1 black==26.5.1 bleach==6.4.0 blinker==1.9.0 -boto3==1.43.0 -botocore==1.43.0 -cachelib==0.14.0 -cachetools==7.1.4 +boto3==1.43.56 +botocore==1.43.56 +cachelib==0.15.2 +cachetools==6.2.6 cadwyn==7.0.0 capi_param_builder_python==1.3.0 -cassandra-driver==3.30.0 +cassandra-driver==3.30.1 cattrs==26.1.0 celery==5.6.3 -certifi==2026.6.17 -cffi==2.0.0 -chardet==6.0.0.post1 -charset-normalizer==3.4.7 +certifi==2026.7.22 +cffi==2.1.1 +chardet==7.5.1 +charset-normalizer==3.4.9 ciso8601==2.3.3 click-didyoumean==0.3.1 click-plugins==1.1.1.2 click-repl==0.3.0 click==8.4.2 -clickhouse-connect==1.4.1 -cloudpickle==3.1.2 -cohere==7.0.5 +clickhouse-connect==1.6.0 +cloudpickle==3.1.1 +cohere==7.0.8 colorama==0.4.6 colorful==0.5.8 -colorlog==6.10.1 +colorlog==6.12.0 comm==0.2.3 confluent-kafka==2.15.0 crcmod-plus==2.3.1 cron_descriptor==2.1.0 -croniter==6.2.2 -cryptography==48.0.1 +croniter==6.2.4 +cryptography==50.0.0 curlify==3.0.0 -databricks-sql-connector==4.2.5 -datadog==0.52.2 -db-dtypes==1.7.0 +databricks-sql-connector==4.4.0 +datadog==0.53.0 +db-dtypes==1.7.1 debugpy==1.8.21 decorator==5.3.1 defusedxml==0.7.1 @@ -284,153 +289,156 @@ dill==0.4.1 distlib==0.4.3 distro==1.9.0 dnspython==2.8.0 -docker==7.1.0 +docker==7.2.0 docopt==0.6.2 docstring_parser==0.18.0 durationpy==0.10 elastic-transport==9.4.2 -elasticsearch==9.4.1 +elasticsearch==9.5.0 email-validator==2.3.0 entrypoints==0.4 +et_xmlfile==2.0.0 eventlet==0.41.0 executing==2.2.1 -facebook_business==25.0.2 -fastapi-cli==0.0.27 +facebook_business==26.0.0 +fastapi-cli==0.0.32 fastapi==0.136.3 fastavro==1.12.2 -fastcore==1.13.9 -fastjsonschema==2.21.2 +fastcore==2.2.5 +fastjsonschema==2.22.1 fastuuid==0.14.0 -filelock==3.29.4 -flask-appbuilder==5.2.1 +filelock==3.32.2 +flask-appbuilder==5.2.2 flask-babel==4.0.0 flower==2.0.1 frozenlist==1.8.0 -fsspec==2026.6.0 +fsspec==2026.7.0 future==1.0.0 -gcloud-aio-auth==5.4.4 +gcloud-aio-auth==5.5.0 gcloud-aio-bigquery==7.1.0 gcloud-aio-storage==9.6.4 -gcsfs==2026.6.0 -genai-prices==0.0.69 +gcsfs==2026.7.0 +genai-prices==0.1.1 geomet==1.1.0 -gevent==26.5.0 +gevent==26.7.0 gitdb==4.0.12 -google-ads==31.1.0 +google-ads==31.2.0 google-analytics-admin==0.30.1 -google-api-core==2.31.0 +google-api-core==2.34.0 google-api-python-client==2.198.0 -google-auth-httplib2==0.4.0 +google-auth-httplib2==0.4.1 google-auth-oauthlib==1.4.0 -google-auth==2.55.1 -google-cloud-aiplatform==1.148.1 +google-auth==2.56.3 +google-cloud-aiplatform==1.163.0 google-cloud-alloydb==0.11.0 google-cloud-appengine-logging==1.10.0 -google-cloud-audit-log==0.6.0 +google-cloud-audit-log==0.6.1 google-cloud-automl==2.20.0 -google-cloud-batch==0.22.0 +google-cloud-batch==0.22.2 google-cloud-bigquery-datatransfer==3.23.0 -google-cloud-bigquery-storage==2.39.0 -google-cloud-bigquery==3.42.1 -google-cloud-bigtable==2.40.0 -google-cloud-build==3.38.0 -google-cloud-compute==1.49.0 +google-cloud-bigquery-storage==2.40.0 +google-cloud-bigquery==3.43.0 +google-cloud-bigtable==2.41.0 +google-cloud-build==3.38.1 +google-cloud-compute==1.50.0 google-cloud-container==2.65.0 -google-cloud-core==2.6.0 +google-cloud-core==2.6.1 google-cloud-datacatalog==3.31.0 google-cloud-dataflow-client==0.14.0 -google-cloud-dataform==0.11.1 +google-cloud-dataform==0.11.2 google-cloud-dataplex==2.20.0 google-cloud-dataproc-metastore==1.23.0 -google-cloud-dataproc==5.29.0 +google-cloud-dataproc==5.30.0 google-cloud-dlp==3.38.0 -google-cloud-kms==3.14.0 +google-cloud-kms==3.16.0 google-cloud-language==2.21.0 -google-cloud-logging==3.16.0 +google-cloud-logging==3.16.2 google-cloud-managedkafka==0.4.1 google-cloud-memcache==1.16.0 google-cloud-monitoring==2.31.0 google-cloud-orchestration-airflow==1.22.0 google-cloud-os-login==2.22.0 -google-cloud-pubsub==2.39.0 +google-cloud-pubsub==2.39.1 google-cloud-redis==2.22.0 google-cloud-resource-manager==1.18.0 google-cloud-run==0.16.1 -google-cloud-secret-manager==2.29.0 -google-cloud-spanner==3.69.0 +google-cloud-secret-manager==2.30.0 +google-cloud-spanner==3.69.1 google-cloud-speech==2.40.0 -google-cloud-storage-control==1.12.0 +google-cloud-storage-control==1.13.0 google-cloud-storage-transfer==1.21.0 -google-cloud-storage==3.12.0 -google-cloud-tasks==2.23.0 +google-cloud-storage==3.13.1 +google-cloud-tasks==2.24.0 google-cloud-texttospeech==2.37.0 google-cloud-translate==3.27.0 google-cloud-videointelligence==2.20.0 google-cloud-vision==3.15.0 google-cloud-workflows==1.23.0 google-crc32c==1.8.0 -google-genai==1.75.0 -google-resumable-media==2.10.0 -googleapis-common-protos==1.75.0 +google-genai==2.17.0 +google-resumable-media==2.10.1 +googleapis-common-protos==1.75.1 graphviz==0.21 greenback==1.3.0 -greenlet==3.5.3 +greenlet==3.5.4 gremlinpython==3.8.1 griffelib==2.1.0 -grpc-google-iam-v1==0.14.4 +grpc-google-iam-v1==0.14.5 grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 -grpcio-health-checking==1.81.1 -grpcio-status==1.81.1 -grpcio-tools==1.81.1 -grpcio==1.81.1 +grpcio-status==1.78.0 +grpcio-tools==1.78.0 +grpcio==1.78.0 gssapi==1.11.1 gunicorn==26.0.0 h11==0.16.0 -h2==4.3.0 +h2==4.4.1 hdfs==2.7.3 -hf-xet==1.5.1 +hf-xet==1.6.0 hmsclient==0.1.1 hpack==4.2.0 -httpcore2==2.5.0 +httpcore2==2.9.1 httpcore==1.0.9 httplib2==0.32.0 -httpr==0.4.8 +httpr==0.6.0 httptools==0.8.0 -httpx2==2.5.0 +httpx2==2.9.1 httpx==0.28.1 -huggingface_hub==1.21.0 +huggingface_hub==1.27.0 humanize==4.16.0 hvac==2.4.0 hyperframe==6.1.0 -ibm-cloud-sdk-core==3.24.4 -ibmcloudant==0.11.8 +ibm-cloud-sdk-core==3.25.0 +ibmcloudant==0.11.9 idna==3.18 ijson==3.4.0.post0 immutabledict==4.3.1 -importlib_metadata==9.0.0 -# Hand-patched (keep on regeneration): impyla <0.24.0 hard-pins thrift==0.16.0, which holds -# thrift below the 0.24.0 release that fixes CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. +importlib_metadata==8.9.0 +importlib_resources==7.1.0 +# Upstream constraints-3.3.1 now ships this natively, but keep the pin on regeneration: +# impyla <0.24.0 hard-pins thrift==0.16.0, which holds thrift below the 0.24.0 release +# that fixes CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. # Must stay in sync with the hive/impala extras in ingestion/setup.py. impyla==0.24.0 inflection==0.5.1 influxdb-client==1.50.0 influxdb3-python==0.20.0 +invoke==3.0.3 ipykernel==7.3.0 -ipython==9.15.0 +ipython==9.16.1 ipython_pygments_lexers==1.1.1 isodate==0.7.2 isoduration==20.11.0 itsdangerous==2.2.0 jaraco.classes==3.4.0 jaraco.context==6.1.2 -jaraco.functools==4.5.0 +jaraco.functools==4.6.0 jedi==0.20.0 jeepney==0.9.0 jiter==0.16.0 jmespath==1.1.0 joblib==1.5.3 -joserfc==1.7.2 +joserfc==1.7.4 jpype1==1.7.1 jsonpath-ng==1.8.0 jsonschema-specifications==2025.9.1 @@ -442,17 +450,17 @@ jwcrypto==1.5.8 keyring==25.7.0 kombu==5.6.2 krb5==0.9.0 -kubernetes==36.0.2 +kubernetes==36.0.3 kubernetes_asyncio==36.1.0 kylinpy==2.8.4 lazy-object-proxy==1.12.0 -libcst==1.8.6 +libcst==1.9.0 limits==5.8.0 linkify-it-py==2.1.0 -litellm==1.82.6 +litellm==1.85.7 lockfile==0.12.2 -logfire-api==4.37.0 -looker_sdk==26.10.0 +logfire-api==4.40.0 +looker_sdk==26.12.0 lxml==6.1.1 lz4==4.4.5 markdown-it-py==4.2.0 @@ -461,7 +469,7 @@ marshmallow==4.3.0 matplotlib-inline==0.2.2 mdit-py-plugins==0.6.1 mdurl==0.1.2 -memray==1.19.3 +memray==1.20.0 mergedeep==1.3.4 methodtools==0.4.7 microsoft-kiota-abstractions==1.11.7 @@ -469,13 +477,13 @@ microsoft-kiota-authentication-azure==1.11.7 microsoft-kiota-http==1.11.7 microsoft-kiota-serialization-json==1.11.7 microsoft-kiota-serialization-text==1.11.7 -mistune==3.3.2 +mistune==3.3.4 mmh3==5.2.1 more-itertools==11.1.0 msal-extensions==1.3.1 msal==1.37.0 msgpack==1.2.1 -msgraph-core==1.4.0 +msgraph-core==1.5.1 msgraphfs==0.5 msgspec==0.21.1 msrest==0.7.1 @@ -483,93 +491,97 @@ msrestazure==0.6.4.post1 multi_key_dict==2.0.3 multidict==6.7.1 mypy_extensions==1.1.0 -mysql-connector-python==9.6.0 +mysql-connector-python==26.7.0 mysqlclient==2.2.8 -narwhals==2.23.0 +narwhals==2.24.0 natsort==8.4.0 nbclient==0.11.0 nbconvert==7.17.1 -nbformat==5.10.4 +nbformat==5.11.0 neo4j==6.2.0 nest-asyncio2==1.7.2 nest-asyncio==1.6.0 -numpy==1.26.4 +numpy==2.5.1 oauthlib==3.3.1 -openai==2.44.0 +openai==2.53.0 opencensus-context==0.1.3 opencensus==0.11.4 -openlineage-integration-common==1.50.0 -openlineage-python==1.50.0 -openlineage_sql==1.50.0 +openlineage-integration-common==1.52.0 +openlineage-python==1.52.0 +openlineage_sql==1.52.0 +openpyxl==3.1.5 opensearch-protobufs==1.2.0 opensearch-py==3.2.0 -opentelemetry-api==1.43.0 -opentelemetry-exporter-otlp-proto-common==1.43.0 -opentelemetry-exporter-otlp-proto-grpc==1.43.0 -opentelemetry-exporter-otlp-proto-http==1.43.0 -opentelemetry-exporter-otlp==1.43.0 -opentelemetry-exporter-prometheus==0.64b0 -opentelemetry-proto==1.43.0 -opentelemetry-resourcedetector-gcp==1.12.0a0 -opentelemetry-sdk==1.43.0 -opentelemetry-semantic-conventions==0.64b0 +opentelemetry-api==1.44.0 +opentelemetry-exporter-otlp-proto-common==1.44.0 +opentelemetry-exporter-otlp-proto-grpc==1.44.0 +opentelemetry-exporter-otlp-proto-http==1.44.0 +opentelemetry-exporter-otlp==1.44.0 +opentelemetry-exporter-prometheus==0.65b0 +opentelemetry-proto==1.44.0 +opentelemetry-resourcedetector-gcp==1.14.0 +opentelemetry-sdk==1.44.0 +opentelemetry-semantic-conventions==0.65b0 opsgenie-sdk==2.1.5 -oracledb==4.0.1 +oracledb==4.0.2 ordered-set==4.1.0 orjson==3.11.9 outcome==1.3.0.post0 -packaging==26.2 -pagerduty==6.3.0 -pandas-gbq==0.35.0 -pandas-stubs==3.0.3.260530 -pandas==2.1.4 +packaging==26.3 +pagerduty==7.0.0 +pandas-gbq==0.35.1 +pandas==3.0.5 pandocfilters==1.5.1 papermill==2.7.0 -paramiko==3.5.1 +paramiko==5.0.0 parso==0.8.7 pathlib_abc==0.5.2 pathspec==1.1.1 pbr==7.0.3 pendulum==3.2.0 pexpect==4.9.0 -pgvector==0.4.2 +pgvector==0.5.0 pinecone==9.1.0 pinotdb==9.1.2 -platformdirs==4.10.0 +platformdirs==4.11.1 pluggy==1.6.0 -polars-runtime-32==1.42.1 -polars==1.42.1 +polars-runtime-32==1.43.2 +polars==1.43.2 portalocker==3.2.0 presto-python-client==0.8.4 prison==0.2.1 -prometheus_client==0.25.0 -prompt_toolkit==3.0.52 +prometheus_client==0.26.0 +prompt_toolkit==3.0.53 propcache==0.5.2 -proto-plus==1.28.0 +proto-plus==1.28.3 protobuf==6.33.6 psutil==7.2.2 +psycopg-binary==3.3.4 psycopg2-binary==2.9.12 +psycopg==3.3.4 ptyprocess==0.7.0 pure-sasl==0.6.2 pure_eval==0.2.3 py-spy==0.4.2 -pyOpenSSL==26.2.0 -pyarrow==24.0.0 -pyasn1==0.6.3 +pyOpenSSL==26.4.0 +pyarrow==25.0.0 +pyasn1==0.6.4 pyasn1_modules==0.4.2 +pybreaker==1.4.1 pycountry==26.2.16 pycparser==3.0 pycryptodome==3.23.0 -pydantic-ai-slim==2.3.0 +pydantic-ai-slim==2.27.0 pydantic-extra-types==2.11.1 -pydantic-graph==2.3.0 -pydantic-settings==2.14.2 +pydantic-graph==2.27.0 +pydantic-settings==2.15.0 pydantic==2.13.4 pydantic_core==2.46.4 pydata-google-auth==1.9.1 pydruid==0.6.9 -pyexasol==2.2.2 +pyexasol==1.3.0 pygtrie==2.5.0 +pyiceberg==0.11.1 pykerberos==1.2.4 pymongo==4.17.0 pymssql==2.3.13 @@ -577,12 +589,13 @@ pyodbc==5.3.0 pyodps==0.13.0 pyparsing==3.3.2 pypsrp==0.9.1 -pyspark-client==4.0.3 +pyroaring==1.1.0 +pyspark-client==4.2.0 pyspnego==0.12.1 python-arango==8.3.3 python-daemon==3.1.2 python-dateutil==2.9.0.post0 -python-discovery==1.4.2 +python-discovery==1.5.1 python-dotenv==1.2.2 python-http-client==3.3.7 python-jenkins==1.8.3 @@ -593,17 +606,17 @@ python-slugify==8.0.4 python-telegram-bot==22.8 python3-saml==1.16.0 pytokens==0.4.1 -pytz==2026.2 -pyvespa==1.2.3 +pytz==2026.3.post1 +pyvespa==1.2.4 pywinrm==0.5.0 pyzmq==27.1.0 -qdrant-client==1.18.0 -ray==2.56.0 -reactivex==4.1.0 +qdrant-client==1.19.0 +ray==2.56.1 +reactivex==5.1.0 redis==6.4.0 -redshift_connector==2.1.15 +redshift_connector==2.1.16 referencing==0.37.0 -regex==2026.6.28 +regex==2026.7.19 requests-file==3.0.1 requests-kerberos==0.15.0 requests-oauthlib==2.0.0 @@ -612,104 +625,108 @@ requests==2.34.2 requests_ntlm==1.3.0 retryhttp==1.5.0 rich-argparse==1.8.0 -rich-toolkit==0.20.1 -rich==13.9.4 +rich-toolkit==0.20.3 +rich==14.3.4 rpds-py==2026.6.3 rsa==4.9.1 ruamel.yaml==0.19.1 -s3fs==2026.6.0 -s3transfer==0.17.1 -sagemaker_studio==1.0.26 +s3fs==2026.7.0 +s3transfer==0.19.2 +sagemaker_studio==1.0.27 scikit-learn==1.9.0 -scipy==1.17.1 -scramp==1.4.10 +scipy==1.18.0 +scramp==1.4.17 scrapbook==0.5.0 segment-analytics-python==2.3.6 sendgrid==6.12.5 -sentry-sdk==2.64.0 +sentry-sdk==2.66.1 setproctitle==1.3.7 -setuptools==82.0.1 +setuptools==83.0.0 shellingham==1.5.4 -simple-salesforce==1.12.9 +simple-salesforce==1.12.10 six==1.17.0 slack_sdk==3.43.0 -smart_open==8.0.0 -smbprotocol==1.16.1 +smart_open==8.0.1 +smbprotocol==1.17.0 smmap==5.0.3 sniffio==1.3.1 -snowflake-connector-python==4.6.0 -snowflake-sqlalchemy==1.10.2 +snowflake-connector-python==4.7.2 +snowflake-snowpark-python==1.54.0 +snowflake-sqlalchemy==1.11.0 sortedcontainers==2.4.0 -soupsieve==2.8.4 +soupsieve==2.9.2 spython==0.3.14 -sqlalchemy-bigquery==1.17.0 +sqlalchemy-bigquery==1.17.2 sqlalchemy-spanner==1.19.0 sqlalchemy_drill==1.1.10 -sqlglot==30.12.0 +sqlglot==30.15.0 sqlparse==0.5.5 stack-data==0.6.3 -starlette==1.3.1 +starlette==1.5.0 statsd==4.0.1 -std-uritemplate==2.0.10 +std-uritemplate==2.0.12 +strictyaml==1.7.3 structlog==26.1.0 -svcs==25.1.0 +svcs==26.1.0 tableauserverclient==0.41 tabulate==0.10.0 tenacity==9.1.4 -teradatasql==20.0.0.62 +teradatasql==20.0.0.64 teradatasqlalchemy==20.0.0.9 termcolor==3.3.0 text-unidecode==1.3 -textual==6.2.1 +textual==8.2.8 threadpoolctl==3.6.0 thrift-sasl==0.4.3 -# Hand-patched (keep on regeneration): see the impyla note above. thrift <0.24.0 is vulnerable -# to CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. +# Upstream constraints-3.3.1 now ships this natively, but keep the pin on regeneration: +# see the impyla note above. thrift <0.24.0 is vulnerable to CVE-2026-66053 / +# CVE-2026-41608 / CVE-2026-48586. thrift==0.24.0 tiktoken==0.13.0 tinycss2==1.5.1 tokenizers==0.23.1 -tomlkit==0.15.0 -tornado==6.5.7 -tqdm==4.68.3 -traitlets==5.15.1 +tomlkit==0.15.1 +tornado==6.5.8 +tqdm==4.70.0 +traitlets==5.16.1 trino==0.338.0 truststore==0.10.4 -typer==0.25.1 +typer==0.27.1 types-protobuf==7.34.1.20260518 -types-requests==2.33.0.20260518 +types-requests==2.33.0.20260712 typing-inspection==0.4.2 typing_extensions==4.16.0 -tzdata==2026.2 +tzdata==2026.3 tzlocal==5.4.4 uc-micro-py==2.0.0 universal_pathlib==0.3.10 uritemplate==4.2.0 urllib3==2.7.0 uuid6==2025.0.1 -uv==0.11.26 -uvicorn==0.49.0 +uv==0.12.3 +uvicorn==0.52.1 uvloop==0.22.1 validators==0.35.0 vertica-python==1.4.0 vine==5.1.0 -virtualenv==21.5.1 +virtualenv==21.7.2 watchfiles==1.2.0 watchtower==3.4.0 wcwidth==0.8.2 -weaviate-client==4.16.2 +weaviate-client==4.22.0 webencodings==0.5.1 -websocket-client==1.9.0 -websockets==16.0 +websocket-client==1.8.0 +websockets==16.1.1 +wheel==0.47.0 wirerope==1.0.0 -wrapt==2.2.2 +wrapt==2.3.0 xmlsec==1.3.17 xmltodict==1.0.4 yandex-query-client==0.1.4 -yandexcloud==0.397.0 -yarl==1.24.2 +yandexcloud==0.402.0 +yarl==1.24.5 ydb-dbapi==0.1.22 -ydb==3.29.6 +ydb==3.31.2 zeep==4.3.3 zenpy==2.0.57 zipp==4.1.0 diff --git a/ingestion/setup.py b/ingestion/setup.py index a734cc234375..ddac2c4b2cd6 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -20,8 +20,10 @@ # Add here versions required for multiple plugins VERSIONS = { # CVE-2026-42252 BashOperator Jinja2 injection; CVE-2026-48891 /ui/dependencies leaks - # Dag IDs the caller cannot read (residual gap in the CVE-2026-28563 fix, needs 3.3.0) - "airflow": "apache-airflow==3.3.0", + # Dag IDs the caller cannot read (residual gap in the CVE-2026-28563 fix, needs 3.3.0); + # CVE-2026-67587 Dag-author RCE on the Scheduler via a Serde Callback deserialization + # gadget and CVE-2026-54183 Variables unmasked in the UI (both need 3.3.1) + "airflow": "apache-airflow==3.3.1", "adlfs": "adlfs>=2023.1.0", "aiobotocore": "aiobotocore~=2.26.0", "avro": "avro>=1.11.4,<1.12", @@ -236,7 +238,10 @@ DATA_DIFF["clickhouse"], }, "dagster": { - "croniter<3", + # No croniter ceiling here: dagster 1.13 declares no croniter dependency at all, + # nothing under ingestion/ imports it, and apache-airflow-core 3.3.1 raised its + # floor to croniter>=6.2.2 -- a stale "croniter<3" makes the two uninstallable + # together. The airflow images already run croniter 6.2.x via the constraints file. VERSIONS["pymysql"], "psycopg2-binary", VERSIONS["geoalchemy2"], diff --git a/ingestion/tests/integration/airflow/Dockerfile b/ingestion/tests/integration/airflow/Dockerfile index f14867ee5a59..6f560750f6ec 100644 --- a/ingestion/tests/integration/airflow/Dockerfile +++ b/ingestion/tests/integration/airflow/Dockerfile @@ -12,16 +12,16 @@ # Airflow carrying the working tree's OpenMetadata lineage provider. Build context is # ingestion/, so src/metadata/generated must already exist (make generate). -FROM apache/airflow:3.3.0-python3.10 +FROM apache/airflow:3.3.1-python3.10 USER airflow COPY --chown=airflow:0 . /tmp/ingestion -# airflow-constraints-3.3.0.txt is deliberately NOT applied: it pins chardet==6.0.0.post1 +# airflow-constraints-3.3.1.txt is deliberately NOT applied: it pins chardet==7.5.1 # against openmetadata-ingestion's chardet==4.0.0. Dockerfile.ci installs the package # unconstrained for the same reason. apache-airflow is pinned so the resolver cannot move it. RUN pip install --no-cache-dir uv \ - && uv pip install --no-cache "apache-airflow==3.3.0" /tmp/ingestion \ + && uv pip install --no-cache "apache-airflow==3.3.1" /tmp/ingestion \ && rm -rf /tmp/ingestion # Migrating at build time keeps container start to a few seconds. diff --git a/ingestion/tests/integration/airflow/conftest.py b/ingestion/tests/integration/airflow/conftest.py index ffbb3259fa64..387d3475564b 100644 --- a/ingestion/tests/integration/airflow/conftest.py +++ b/ingestion/tests/integration/airflow/conftest.py @@ -26,7 +26,7 @@ from testcontainers.core.container import DockerContainer from testcontainers.core.docker_client import DockerClient -AIRFLOW_BASE_IMAGE = "apache/airflow:3.3.0-python3.10" +AIRFLOW_BASE_IMAGE = "apache/airflow:3.3.1-python3.10" AIRFLOW_TEST_IMAGE = "om-airflow-lineage-test:local" # Fixed network name declared by docker/development/docker-compose.yml, so the container From 85592d6beec1be24e232a66060ea4db4999b26aa Mon Sep 17 00:00:00 2001 From: Pablo Takara Date: Fri, 21 Aug 2026 18:57:15 +0200 Subject: [PATCH 3/5] fix(ingestion): migrate NumPy and Pydantic dependency stack --- .github/workflows/py-tests-shared.yml | 9 ++-- ingestion/setup.py | 14 +++---- .../ingestion/models/custom_pydantic.py | 12 ++---- .../src/metadata/ingestion/models/topology.py | 2 +- .../metadata/pii/algorithms/presidio_utils.py | 23 +++++++++++ .../src/metadata/pii/algorithms/utils.py | 2 +- .../src/metadata/pii/scanners/ner_scanner.py | 23 ++--------- .../profiler/metrics/hybrid/histogram.py | 4 +- .../test_custom_basemodel_validation.py | 11 +++++ .../pii/algorithms/test_feature_extraction.py | 30 +++++++++++++- .../tests/unit/pii/algorithms/test_utils.py | 19 +++++++++ .../pii/test_cases/customers_sensitive.py | 10 +++++ ingestion/tests/unit/pii/test_ner_scanner.py | 2 +- ingestion/tests/unit/test_mf4_reader.py | 20 +++++++++ ingestion/tests/unit/test_pydantic_v2.py | 41 +++++++++++++++++++ .../tests/unit/utils/test_deprecation.py | 4 +- 16 files changed, 179 insertions(+), 47 deletions(-) create mode 100644 ingestion/tests/unit/pii/algorithms/test_utils.py diff --git a/.github/workflows/py-tests-shared.yml b/.github/workflows/py-tests-shared.yml index 9fbe73c70bc7..c0e39e30c909 100644 --- a/.github/workflows/py-tests-shared.yml +++ b/.github/workflows/py-tests-shared.yml @@ -44,10 +44,9 @@ on: env: # matrix can't use 'env'. When updating it, update it for both jobs. MAIN_PYTHON_VERSION: "3.10" - # Hugging Face uses a moving wheel name, so pin its immutable revision and content hash. - SPACY_MODEL_VERSION: "3.7.1" - SPACY_MODEL_REVISION: "22f17ee20cda126e498ea2fb92dc504bea0d111c" - SPACY_MODEL_SHA256: "6a0f857a2b4d219c6fa17d455f82430b365bf53171a2d919b9376e5dc9be032e" + # Pin the model release and verify its content before installation. + SPACY_MODEL_VERSION: "3.8.0" + SPACY_MODEL_SHA256: "5e6329fe3fecedb1d1a02c3ea2172ee0fede6cea6e4aefb6a02d832dba78a310" SONAR_OPTS: >- -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} -Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }} @@ -176,7 +175,7 @@ jobs: wheel_path=".cache/spacy-model/en_core_web_md-${SPACY_MODEL_VERSION}-py3-none-any.whl" curl --fail --location --retry 10 --retry-all-errors --connect-timeout 20 --max-time 600 \ --output "${wheel_path}" \ - "https://huggingface.co/spacy/en_core_web_md/resolve/${SPACY_MODEL_REVISION}/en_core_web_md-any-py3-none-any.whl" + "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-${SPACY_MODEL_VERSION}/en_core_web_md-${SPACY_MODEL_VERSION}-py3-none-any.whl" echo "${SPACY_MODEL_SHA256} ${wheel_path}" | sha256sum --check --strict shell: bash diff --git a/ingestion/setup.py b/ingestion/setup.py index ddac2c4b2cd6..156629ae49e6 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -40,15 +40,15 @@ "ijson": "ijson~=3.4", "msal": "msal~=1.2", "neo4j": "neo4j~=5.3", - "pandas": "pandas~=2.1.4", + "pandas": "pandas>=2.2.2,<3", "pyarrow": "pyarrow>=23.0.1,<26", # CVE-2026-25087 / CVE-2024-52338 IPC pre-buffer use-after-free (fixed in 23.0.1) - "pydantic": "pydantic~=2.0,>=2.7.0,<2.12", # Pin down to <2.12 due to breaking changes in 2.12.0 + "pydantic": "pydantic>=2.12.5,<3", "pydantic-settings": "pydantic-settings~=2.0,>=2.14.2", # GHSA-4xgf-cpjx-pc3j secrets_dir symlink escape "pydomo": "pydomo~=0.3", "pymysql": "pymysql~=1.0", "pyodbc": "pyodbc~=5.3.0", - "numpy": "numpy<2", - "scikit-learn": "scikit-learn>=1.3,<2", + "numpy": "numpy>=2,<3", + "scikit-learn": "scikit-learn>=1.4.2,<2", "packaging": "packaging", "azure-storage-blob": "azure-storage-blob~=12.14", "azure-identity": "azure-identity~=1.12", @@ -56,7 +56,7 @@ "databricks-sql-connector": "databricks-sql-connector>=4.0.0", "databricks-sqlalchemy": "databricks-sqlalchemy~=2.0.9", "trino": "trino[sqlalchemy]", - "spacy": "spacy<3.8", + "spacy": "spacy>=3.8.2,<3.9", "looker-sdk": "looker-sdk>=22.20.0,!=24.18.0", "lkml": "lkml~=1.3", "tableau": "tableauserverclient==0.40", # pre-0.37 pins urllib3<2, which conflicts with collate-data-diff's urllib3>=2.7 @@ -77,7 +77,7 @@ "s3fs": "s3fs~=2026.3", "sqlalchemy-bigquery": "sqlalchemy-bigquery>=1.15.0", "presidio-analyzer": "presidio-analyzer==2.2.358", - "asammdf": "asammdf~=7.4.5", + "asammdf": "asammdf~=8.1.0", "kafka-connect": "kafka-connect-py==0.10.11", "griffe2md": "griffe2md~=1.2", "factory-boy": "factory-boy~=3.3.3", @@ -450,7 +450,7 @@ "google-api-python-client-stubs", "google-auth-stubs", "types-requests", - "pandas-stubs~=2.1.4", + "pandas-stubs~=2.2", "scipy-stubs", "nox", "pre-commit", diff --git a/ingestion/src/metadata/ingestion/models/custom_pydantic.py b/ingestion/src/metadata/ingestion/models/custom_pydantic.py index c76180a09804..4bee6dc50b92 100644 --- a/ingestion/src/metadata/ingestion/models/custom_pydantic.py +++ b/ingestion/src/metadata/ingestion/models/custom_pydantic.py @@ -117,21 +117,15 @@ def model_post_init(self, context: Any, /): logger.warning(f"Exception while parsing FilterPattern: {exc}") @model_validator(mode="after") - @classmethod - def parse_name(cls, values): # pylint: disable=inconsistent-return-statements + def parse_name(self): # pylint: disable=inconsistent-return-statements """ Transform entity names using hybrid configuration system. """ - - if not values: - return values - try: - # Try new hybrid system first - return transform_entity_names(entity=values, model=cls) + return transform_entity_names(entity=self, model=type(self)) except Exception as exc: logger.warning("Exception while parsing Basemodel: %s", exc) - return values + return self def model_dump_json( # pylint: disable=too-many-arguments self, diff --git a/ingestion/src/metadata/ingestion/models/topology.py b/ingestion/src/metadata/ingestion/models/topology.py index 2321d5c16d4e..6401fd3cd249 100644 --- a/ingestion/src/metadata/ingestion/models/topology.py +++ b/ingestion/src/metadata/ingestion/models/topology.py @@ -145,7 +145,7 @@ def create(cls, topology: ServiceTopology) -> "TopologyContext": :return: TopologyContext """ nodes = get_topology_nodes(topology) - ctx_fields = { + ctx_fields: dict[str, Any] = { stage.context: (Optional[stage.type_], None) # noqa: UP045 for node in nodes for stage in node.stages diff --git a/ingestion/src/metadata/pii/algorithms/presidio_utils.py b/ingestion/src/metadata/pii/algorithms/presidio_utils.py index ddab0da33223..ec7a01c12b5a 100644 --- a/ingestion/src/metadata/pii/algorithms/presidio_utils.py +++ b/ingestion/src/metadata/pii/algorithms/presidio_utils.py @@ -26,6 +26,7 @@ from presidio_analyzer import ( AnalyzerEngine, EntityRecognizer, + Pattern, PatternRecognizer, RecognizerRegistry, RecognizerResult, @@ -36,6 +37,7 @@ AuTfnRecognizer, CreditCardRecognizer, DateRecognizer, + InAadhaarRecognizer, NhsRecognizer, UsBankRecognizer, UsLicenseRecognizer, @@ -233,6 +235,27 @@ def au_tfn_factory( ) +@recognizer_factories.add( # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] + InAadhaarRecognizer +) +def in_aadhaar_factory( + *, + supported_language: str = SUPPORTED_LANG, + context: list[str] | None = None, +) -> InAadhaarRecognizer: + return InAadhaarRecognizer( + patterns=[ + Pattern( + "AADHAAR", + r"(? Sequ Get the top n scores from the scores mapping that are above the threshold. The classes are sorted in descending order of their scores. """ - sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True) + sorted_scores = sorted(scores.items(), key=lambda item: (-item[1], str(item[0]))) top_classes = [key for key, score in sorted_scores if score >= threshold] return top_classes[:n] diff --git a/ingestion/src/metadata/pii/scanners/ner_scanner.py b/ingestion/src/metadata/pii/scanners/ner_scanner.py index 34527d963f05..d7b2c3188ae6 100644 --- a/ingestion/src/metadata/pii/scanners/ner_scanner.py +++ b/ingestion/src/metadata/pii/scanners/ner_scanner.py @@ -20,12 +20,12 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple, Union # noqa: UP035 -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel from metadata.generated.schema.entity.classification.tag import Tag from metadata.pii.algorithms.preprocessing import MAX_NLP_TEXT_LENGTH -from metadata.pii.algorithms.presidio_utils import _load_spacy_model -from metadata.pii.constants import PII, SPACY_EN_MODEL +from metadata.pii.algorithms.presidio_utils import build_analyzer_engine +from metadata.pii.constants import PII from metadata.pii.models import TagAndConfidence from metadata.pii.ner import NEREntity from metadata.pii.scanners.base import BaseScanner @@ -46,32 +46,17 @@ class StringAnalysis(BaseModel): appearances: int -class NLPEngineModel(BaseModel): - """Required to pass the nlp_engine as {"lang_code": "en", "model_name": "en_core_web_lg"}""" - - model_config = ConfigDict(protected_namespaces=()) - lang_code: str - model_name: str - - # pylint: disable=import-outside-toplevel class NERScanner(BaseScanner): """Based on https://microsoft.github.io/presidio/""" def __init__(self): - from presidio_analyzer import AnalyzerEngine - from presidio_analyzer.nlp_engine.spacy_nlp_engine import SpacyNlpEngine - - _load_spacy_model(SPACY_EN_MODEL) - - nlp_engine_model = NLPEngineModel(lang_code=SUPPORTED_LANG, model_name=SPACY_EN_MODEL) - # Set the presidio logger to talk less about internal entities unless we are debugging logging.getLogger(PRESIDIO_LOGGER).setLevel( logging.INFO if logging.getLogger(METADATA_LOGGER).level == logging.DEBUG else logging.ERROR ) - self.analyzer = AnalyzerEngine(nlp_engine=SpacyNlpEngine(models=[nlp_engine_model.model_dump()])) + self.analyzer = build_analyzer_engine() @staticmethod def get_highest_score_label(entities_score: Dict[str, StringAnalysis]) -> Tuple[str, float]: # noqa: UP006 diff --git a/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py b/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py index 622b45d204d6..ab9babcf2f52 100644 --- a/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py +++ b/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py @@ -252,11 +252,11 @@ def df_fn( for df in dfs: if not frequencies.any(): frequencies = ( - pd.cut(df[self.col.name], bins, right=False).value_counts().values + pd.cut(df[self.col.name], bins, right=False).value_counts().to_numpy() ) # right boundary is exclusive continue frequencies += ( - pd.cut(df[self.col.name], bins, right=False).value_counts().values + pd.cut(df[self.col.name], bins, right=False).value_counts().to_numpy() ) # right boundary is exclusive if frequencies.size > 0: # pyright: ignore[reportAttributeAccessIssue] diff --git a/ingestion/tests/unit/models/test_custom_basemodel_validation.py b/ingestion/tests/unit/models/test_custom_basemodel_validation.py index 485908f6d397..8649e83c4cf5 100644 --- a/ingestion/tests/unit/models/test_custom_basemodel_validation.py +++ b/ingestion/tests/unit/models/test_custom_basemodel_validation.py @@ -58,6 +58,17 @@ from metadata.utils.entity_link import CustomColumnName +def test_generated_model_transforms_name(): + """Validate automatic name transformation on generated models.""" + request = CreateTableRequest( + name=EntityName('my::table>with"special_chars'), + columns=[Column(name=ColumnName("column"), dataType=DataType.STRING)], + databaseSchema=FullyQualifiedEntityName("database.schema"), + ) + + assert request.name.root == "my__reserved__colon__table__reserved__arrow__with__reserved__quote__special_chars" + + class TestCustomBasemodelValidation(TestCase): """Comprehensive test suite for custom basemodel validation functionality.""" diff --git a/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py b/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py index aed62ced83de..8eb770c5710a 100644 --- a/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py +++ b/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py @@ -10,6 +10,8 @@ # limitations under the License. from typing import Mapping, Optional # noqa: UP035 +import pytest + from metadata.pii.algorithms.column_patterns import get_pii_column_name_patterns from metadata.pii.algorithms.feature_extraction import ( extract_pii_from_column_names, @@ -232,7 +234,12 @@ def test_aadhaar_extraction(analyzer): "0249-3285-1294", ] context = ["aadhaar", "govt id", "uidai"] - extracted = extract_pii_tags(analyzer, samples, context=context) + extracted = extract_pii_tags( + analyzer, + samples, + context=context, + recognizer_result_patcher=date_time_patcher, + ) assert get_top_pii_tag(extracted) == PIITag.IN_AADHAAR, ( PIITag.IN_AADHAAR, samples, @@ -240,6 +247,27 @@ def test_aadhaar_extraction(analyzer): ) +@pytest.mark.parametrize("sample", ["2161 6729 3627", "8384-2795-9970"]) +def test_aadhaar_extraction_accepts_supported_separators(analyzer, sample): + extracted = extract_pii_tags( + analyzer, + [sample], + context=["aadhaar", "govt id", "uidai"], + ) + + assert get_top_pii_tag(extracted) == PIITag.IN_AADHAAR, extracted + + +def test_aadhaar_extraction_does_not_match_dashed_credit_card(analyzer): + extracted = extract_pii_tags( + analyzer, + ["5105-1051-0510-5100"], + context=["card", "number"], + ) + + assert PIITag.IN_AADHAAR not in extracted + + def test_indian_passport_extraction(analyzer): # Randomly generated valid Indian passport numbers samples = [ diff --git a/ingestion/tests/unit/pii/algorithms/test_utils.py b/ingestion/tests/unit/pii/algorithms/test_utils.py new file mode 100644 index 000000000000..214da0e0c0aa --- /dev/null +++ b/ingestion/tests/unit/pii/algorithms/test_utils.py @@ -0,0 +1,19 @@ +# Copyright 2025 Collate +# Licensed under the Collate Community License, Version 1.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from metadata.pii.algorithms.utils import get_top_classes + + +def test_get_top_classes_breaks_score_ties_deterministically(): + scores = {"NRP": 0.425, "LOCATION": 0.425} + + assert get_top_classes(scores, n=1, threshold=0.0) == ["LOCATION"] + assert get_top_classes(dict(reversed(scores.items())), n=1, threshold=0.0) == ["LOCATION"] diff --git a/ingestion/tests/unit/pii/test_cases/customers_sensitive.py b/ingestion/tests/unit/pii/test_cases/customers_sensitive.py index e4911c45b457..b55a53948dc2 100644 --- a/ingestion/tests/unit/pii/test_cases/customers_sensitive.py +++ b/ingestion/tests/unit/pii/test_cases/customers_sensitive.py @@ -128,4 +128,14 @@ state=State.Suggested, ), ), + ColumnTag( + column_fqn="Service.database.schema.example_table.address", + tag_label=TagLabel( + name="NonSensitive", + tagFQN=TagFQN(root="PII.NonSensitive"), + source=TagSource.Classification, + labelType=LabelType.Generated, + state=State.Suggested, + ), + ), ] diff --git a/ingestion/tests/unit/pii/test_ner_scanner.py b/ingestion/tests/unit/pii/test_ner_scanner.py index b8d5f318e625..b90c60668c3e 100644 --- a/ingestion/tests/unit/pii/test_ner_scanner.py +++ b/ingestion/tests/unit/pii/test_ner_scanner.py @@ -157,5 +157,5 @@ def test_scan_entities(scanner): ] assert scanner.scan(ssn_numbers).tag_fqn == "PII.Sensitive" - nif_numbers = ["12345678A", "87654321B", "23456789C", "98765432D", "34567890E"] + nif_numbers = ["12345678Z", "87654321X", "23456789D", "98765432M", "34567890V"] assert scanner.scan(nif_numbers).tag_fqn == "PII.Sensitive" diff --git a/ingestion/tests/unit/test_mf4_reader.py b/ingestion/tests/unit/test_mf4_reader.py index ee390b3eaf86..97ee50843064 100644 --- a/ingestion/tests/unit/test_mf4_reader.py +++ b/ingestion/tests/unit/test_mf4_reader.py @@ -13,6 +13,8 @@ MF4 reader tests """ +from pathlib import Path +from tempfile import TemporaryDirectory from unittest import TestCase from unittest.mock import MagicMock, patch @@ -20,6 +22,24 @@ from metadata.readers.dataframe.models import DatalakeColumnWrapper +def test_local_mf4_reading_with_installed_asammdf(): + from asammdf import MDF + + from metadata.generated.schema.entity.services.connections.database.datalakeConnection import ( + LocalConfig, + ) + + with TemporaryDirectory() as tmp_dir: + file_path = Path(tmp_dir) / "empty.mf4" + with MDF(version="4.10") as mdf: + mdf.save(file_path) + + reader = MF4DataFrameReader(LocalConfig(), None) + result = reader._read(key=str(file_path), bucket_name="") + + assert list(result.dataframes()) == [] + + class TestMF4DataFrameReader(TestCase): """ Test MF4DataFrameReader functionality diff --git a/ingestion/tests/unit/test_pydantic_v2.py b/ingestion/tests/unit/test_pydantic_v2.py index 305e1a701371..f1881d43631d 100644 --- a/ingestion/tests/unit/test_pydantic_v2.py +++ b/ingestion/tests/unit/test_pydantic_v2.py @@ -10,6 +10,8 @@ # limitations under the License. """Test pydantic v2 models serialize data as pydantic v1""" +import subprocess +import sys from datetime import datetime from pydantic import AnyUrl @@ -19,6 +21,27 @@ from metadata.ingestion.models.custom_pydantic import BaseModel +def test_custom_base_model_imports_without_pydantic_212_deprecations(): + """Keep import-time validators on supported Pydantic APIs.""" + script = """ +import warnings + +from pydantic.warnings import PydanticDeprecatedSince212 + +warnings.simplefilter("error", PydanticDeprecatedSince212) +import metadata.ingestion.models.custom_pydantic +""" + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0, result.stderr + + def test_simple_dump(): """ Compare V1 and custom V2 serialization, @@ -76,6 +99,24 @@ class ModelV2(BaseModel): assert json_v1 == json_v2 +def test_serialize_as_any_includes_subclass_fields(): + """Preserve polymorphic serialization exposed by the custom base model.""" + + class Parent(BaseModel): + name: str + + class Child(Parent): + token: str + + class Container(BaseModel): + user: Parent + + model = Container(user=Child(name="name", token="token")) + + assert model.model_dump() == {"user": {"name": "name"}} + assert model.model_dump(serialize_as_any=True) == {"user": {"name": "name", "token": "token"}} + + def test_tz_aware_date(): """Validate how we can create "aware" datetime objects""" diff --git a/ingestion/tests/unit/utils/test_deprecation.py b/ingestion/tests/unit/utils/test_deprecation.py index 540ec4d2c18b..0e46524f45c9 100644 --- a/ingestion/tests/unit/utils/test_deprecation.py +++ b/ingestion/tests/unit/utils/test_deprecation.py @@ -29,6 +29,9 @@ def deprecated_call(self) -> None: def test_deprecation_warning(self) -> None: """Test that deprecation warnings are controlled by logger level.""" + metadata_logger = logging.getLogger("metadata") + self.addCleanup(metadata_logger.setLevel, metadata_logger.level) + logger_levels = [ logging.DEBUG, logging.INFO, @@ -44,7 +47,6 @@ def test_deprecation_warning(self) -> None: # Capture logging output log_capture = StringIO() handler = logging.StreamHandler(log_capture) - metadata_logger = logging.getLogger("metadata") metadata_logger.addHandler(handler) # Create and call a deprecated function From baefb8b233bb80b34975f66f6abd2b2eab5b4568 Mon Sep 17 00:00:00 2001 From: Pablo Takara Date: Fri, 21 Aug 2026 19:47:43 +0200 Subject: [PATCH 4/5] fix(ingestion): separate Aadhaar recognizer change --- .../metadata/pii/algorithms/presidio_utils.py | 23 ------------------- .../pii/algorithms/test_feature_extraction.py | 23 ------------------- 2 files changed, 46 deletions(-) diff --git a/ingestion/src/metadata/pii/algorithms/presidio_utils.py b/ingestion/src/metadata/pii/algorithms/presidio_utils.py index ec7a01c12b5a..ddab0da33223 100644 --- a/ingestion/src/metadata/pii/algorithms/presidio_utils.py +++ b/ingestion/src/metadata/pii/algorithms/presidio_utils.py @@ -26,7 +26,6 @@ from presidio_analyzer import ( AnalyzerEngine, EntityRecognizer, - Pattern, PatternRecognizer, RecognizerRegistry, RecognizerResult, @@ -37,7 +36,6 @@ AuTfnRecognizer, CreditCardRecognizer, DateRecognizer, - InAadhaarRecognizer, NhsRecognizer, UsBankRecognizer, UsLicenseRecognizer, @@ -235,27 +233,6 @@ def au_tfn_factory( ) -@recognizer_factories.add( # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] - InAadhaarRecognizer -) -def in_aadhaar_factory( - *, - supported_language: str = SUPPORTED_LANG, - context: list[str] | None = None, -) -> InAadhaarRecognizer: - return InAadhaarRecognizer( - patterns=[ - Pattern( - "AADHAAR", - r"(? Date: Fri, 21 Aug 2026 21:40:52 +0200 Subject: [PATCH 5/5] fix: filter invalid date recognizer results --- ingestion/src/metadata/pii/tag_analyzer.py | 38 +++++++++++-------- .../unit/metadata/pii/test_tag_scoring.py | 38 +++++++++++++++++++ 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/ingestion/src/metadata/pii/tag_analyzer.py b/ingestion/src/metadata/pii/tag_analyzer.py index 64ec87d45790..52944ecff794 100644 --- a/ingestion/src/metadata/pii/tag_analyzer.py +++ b/ingestion/src/metadata/pii/tag_analyzer.py @@ -19,6 +19,10 @@ ) from metadata.generated.schema.type.recognizer import RecognizerException from metadata.pii.algorithms.feature_extraction import split_column_name +from metadata.pii.algorithms.presidio_patches import ( + PresidioRecognizerResultPatcher, + date_time_patcher, +) from metadata.pii.algorithms.presidio_recognizer_factory import ( PresidioRecognizerFactory, ) @@ -161,6 +165,7 @@ def _analyze_with( text_or_values: str | Sequence[str], recognizers: list[EntityRecognizer], context: Optional[list[str]] = None, # noqa: UP045 + result_patcher: Optional[PresidioRecognizerResultPatcher] = None, # noqa: UP045 ) -> list[RecognizerResult]: values = [text_or_values] if isinstance(text_or_values, str) else list(text_or_values) results: list[RecognizerResult] = [] @@ -168,14 +173,13 @@ def _analyze_with( if self._language is not ClassificationLanguage.any: analyzer = self.build_analyzer_with(recognizers) for value in values: - results.extend( - analyzer.analyze( - value, - language=self._language.value, - context=context, - return_decision_process=True, - ) + value_results = analyzer.analyze( + value, + language=self._language.value, + context=context, + return_decision_process=True, ) + results.extend(result_patcher(value_results, value) if result_patcher else value_results) return results sorted_recs = sorted(recognizers, key=lambda r: r.supported_language) @@ -194,14 +198,13 @@ def _analyze_with( effective_language=effective_lang, ) for value in values: - results.extend( - analyzer.analyze( - value, - language=effective_lang, - context=context, - return_decision_process=True, - ) + value_results = analyzer.analyze( + value, + language=effective_lang, + context=context, + return_decision_process=True, ) + results.extend(result_patcher(value_results, value) if result_patcher else value_results) return results def analyze( @@ -215,7 +218,12 @@ def analyze( content_recognizers = self.content_recognizers if content_recognizers: context = split_column_name(self._column_name) - content_results = self._analyze_with(str_values, content_recognizers, context=context) + content_results = self._analyze_with( + str_values, + content_recognizers, + context=context, + result_patcher=date_time_patcher, + ) content_score = min(sum(r.score for r in content_results) / len(str_values), 1.0) column_results: list[RecognizerResult] = [] diff --git a/ingestion/tests/unit/metadata/pii/test_tag_scoring.py b/ingestion/tests/unit/metadata/pii/test_tag_scoring.py index 6f4a9ec9d7f1..ba71a5701e00 100644 --- a/ingestion/tests/unit/metadata/pii/test_tag_scoring.py +++ b/ingestion/tests/unit/metadata/pii/test_tag_scoring.py @@ -24,6 +24,7 @@ from _openmetadata_testutils.factories.metadata.generated.schema.type.recognizer import ( PatternFactory, PatternRecognizerFactory, + PredefinedRecognizerFactory, RecognizerFactory, ) from metadata.generated.schema.entity.classification.classification import ( @@ -35,7 +36,10 @@ from metadata.generated.schema.type.classificationLanguages import ( ClassificationLanguage, ) +from metadata.generated.schema.type.piiEntity import PIIEntity +from metadata.generated.schema.type.predefinedRecognizer import Name from metadata.generated.schema.type.recognizer import RecognizerException, Target +from metadata.pii.algorithms.presidio_utils import load_nlp_engine from metadata.pii.algorithms.tag_scoring import TagScorer from metadata.pii.models import ScoredTag from metadata.pii.tag_analyzer import TagAnalysis, TagAnalyzer @@ -318,6 +322,40 @@ def tag_analyzer(self, email_tag, column, nlp_engine): """Create a TagAnalyzer instance""" return TagAnalyzer(tag=email_tag, column=column, nlp_engine=nlp_engine) + @pytest.fixture + def date_tag_analyzer(self, column: Column) -> TagAnalyzer: + spacy_recognizer = RecognizerFactory.create( + name="SpacyRecognizer", + recognizerConfig=PredefinedRecognizerFactory.create( + name=Name.SpacyRecognizer, + supportedEntities=[PIIEntity.DATE_TIME], + ), + target=Target.content, + ) + date_tag = TagFactory.create( + tag_name="Date", + autoClassificationEnabled=True, + recognizers=[spacy_recognizer], + description="Date field", + ) + return TagAnalyzer( + tag=date_tag, + column=column, + nlp_engine=load_nlp_engine(), + ) + + def test_analyze_content_rejects_epoch_timestamp_as_date(self, date_tag_analyzer: TagAnalyzer): + analysis = date_tag_analyzer.analyze(str_values=["1760000000123"]) + + assert analysis.score == 0.0 + assert analysis.recognizer_results == [] + + def test_analyze_content_preserves_textual_date(self, date_tag_analyzer: TagAnalyzer): + analysis = date_tag_analyzer.analyze(str_values=["2025-01-15"]) + + assert analysis.score > 0.0 + assert [result.entity_type for result in analysis.recognizer_results] == [PIIEntity.DATE_TIME.value] + def test_analyze_content_with_emails(self, tag_analyzer, email_tag: Tag): """Test content analysis with email data""" values = ["john@example.com", "jane@test.org", "bob@company.co.uk"]