diff --git a/.github/workflows/py-tests-shared.yml b/.github/workflows/py-tests-shared.yml index 9fbe73c70bc7..c0e39e30c909 100644 --- a/.github/workflows/py-tests-shared.yml +++ b/.github/workflows/py-tests-shared.yml @@ -44,10 +44,9 @@ on: env: # matrix can't use 'env'. When updating it, update it for both jobs. MAIN_PYTHON_VERSION: "3.10" - # Hugging Face uses a moving wheel name, so pin its immutable revision and content hash. - SPACY_MODEL_VERSION: "3.7.1" - SPACY_MODEL_REVISION: "22f17ee20cda126e498ea2fb92dc504bea0d111c" - SPACY_MODEL_SHA256: "6a0f857a2b4d219c6fa17d455f82430b365bf53171a2d919b9376e5dc9be032e" + # Pin the model release and verify its content before installation. + SPACY_MODEL_VERSION: "3.8.0" + SPACY_MODEL_SHA256: "5e6329fe3fecedb1d1a02c3ea2172ee0fede6cea6e4aefb6a02d832dba78a310" SONAR_OPTS: >- -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} -Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }} @@ -176,7 +175,7 @@ jobs: wheel_path=".cache/spacy-model/en_core_web_md-${SPACY_MODEL_VERSION}-py3-none-any.whl" curl --fail --location --retry 10 --retry-all-errors --connect-timeout 20 --max-time 600 \ --output "${wheel_path}" \ - "https://huggingface.co/spacy/en_core_web_md/resolve/${SPACY_MODEL_REVISION}/en_core_web_md-any-py3-none-any.whl" + "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-${SPACY_MODEL_VERSION}/en_core_web_md-${SPACY_MODEL_VERSION}-py3-none-any.whl" echo "${SPACY_MODEL_SHA256} ${wheel_path}" | sha256sum --check --strict shell: bash diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index e357849bba58..bafcefec5751 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -1,6 +1,6 @@ FROM mysql:8.3 AS mysql -FROM apache/airflow:3.3.0-python3.12 +FROM apache/airflow:3.3.1-python3.12 USER root RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list @@ -122,7 +122,7 @@ ARG RI_VERSION="2.0.0.0.dev0" RUN pip install --upgrade "pip>=26.2,<27" "setuptools<81" # Pre-install cx-Oracle without build isolation to use the pinned setuptools RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" -RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.3.0/constraints-3.12.txt" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.3.1/constraints-3.12.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index 83895664e419..d72cce355f91 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -1,6 +1,6 @@ FROM mysql:8.3 AS mysql -FROM apache/airflow:3.3.0-python3.12 +FROM apache/airflow:3.3.1-python3.12 USER root RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list @@ -94,7 +94,7 @@ COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airf # Required for Airflow DAGs of Sample Data COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags COPY --chown=airflow:0 ingestion/examples/airflow/test_dags /opt/airflow/dags -COPY --chown=airflow:0 ingestion/airflow-constraints-3.3.0.txt /home/airflow/airflow-constraints-3.3.0.txt +COPY --chown=airflow:0 ingestion/airflow-constraints-3.3.1.txt /home/airflow/airflow-constraints-3.3.1.txt # Refresh the interpreter-level pip before dropping privileges. The per-user # pip upgrade below only reaches ~/.local, leaving the base image's @@ -122,11 +122,11 @@ RUN pip install --upgrade "pip>=26.2,<27" "setuptools<81" RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" # Install FAB provider for Airflow 3.x Flask Blueprint compatibility -RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.3.0.txt" || true +RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.3.1.txt" || true WORKDIR /home/airflow/openmetadata-airflow-apis -RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.3.0.txt" +RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.3.1.txt" WORKDIR /home/airflow/ingestion @@ -169,7 +169,7 @@ RUN pip install psycopg2 mysqlclient==2.1.1 RUN mkdir -p /opt/airflow/dag_generated_configs EXPOSE 8080 -# Airflow 3.3.0 requires universal-pathlib>=0.3.8, but prior installs in this image +# Airflow 3.3.1 requires universal-pathlib>=0.3.8, but prior installs in this image # can leave stale 0.2.6 `upath` module files in site-packages that cause import # errors at runtime. Force-remove the stale registration then pin to the required version. RUN pip uninstall upath -y && pip install "universal-pathlib==0.3.10" diff --git a/ingestion/airflow-constraints-3.3.0.txt b/ingestion/airflow-constraints-3.3.1.txt similarity index 61% rename from ingestion/airflow-constraints-3.3.0.txt rename to ingestion/airflow-constraints-3.3.1.txt index dfffbbe5452f..3921501f8693 100644 --- a/ingestion/airflow-constraints-3.3.0.txt +++ b/ingestion/airflow-constraints-3.3.1.txt @@ -1,6 +1,6 @@ # -# This constraints file was automatically generated on 2026-07-06T12:27:13.807760 +# This constraints file was automatically generated on 2026-08-12T08:36:03.248024 # via `uv pip install --resolution highest` for the "v3-3-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. @@ -35,19 +35,19 @@ Authlib==1.7.2 Deprecated==1.3.1 Events==0.5 Flask-JWT-Extended==4.7.4 -Flask-Limiter==3.12 +Flask-Limiter==4.1.1 Flask-Login==0.6.3 Flask-SQLAlchemy==3.1.1 Flask-Session==0.8.0 Flask-WTF==1.3.0 Flask==3.1.3 -GitPython==3.1.50 +GitPython==3.1.58 JayDeBeApi==1.2.3 Jinja2==3.1.6 -Mako==1.3.12 -Markdown==3.10.2 +Mako==1.4.1 +Markdown==3.10.3 MarkupSafe==3.0.3 -PyAthena==3.32.0 +PyAthena==3.35.4 PyGithub==2.9.1 PyHive==0.7.0 PyJWT==2.13.0 @@ -62,170 +62,174 @@ WTForms==3.2.2 Werkzeug==3.1.8 a2wsgi==1.10.10 adal==1.2.7 +adbc-driver-manager==1.12.0 +adbc-driver-postgresql==1.12.0 +adbc-driver-sqlite==1.12.0 adlfs==2026.5.0 aenum==3.1.17 -aiobotocore==3.7.0 +aiobotocore==3.9.0 aiofiles==24.1.0 aiohappyeyeballs==2.7.1 aiohttp-cors==0.8.1 -aiohttp==3.14.1 +aiohttp==3.14.3 aioitertools==0.13.0 aiomysql==0.3.2 aiosignal==1.4.0 aiosmtplib==5.1.2 -aiosqlite==0.21.0 +aiosqlite==0.22.1 airbyte-api==1.0.1 akeyless==5.0.28 -alembic==1.18.5 +alembic==1.19.0 alibabacloud-adb20211201==3.7.0 alibabacloud-credentials-api==1.0.1 -alibabacloud-credentials==1.0.9 +alibabacloud-credentials==1.0.11 alibabacloud-gateway-spi==0.0.4 alibabacloud-openapi-util==0.2.4 -alibabacloud-oss-v2==1.3.1 -alibabacloud-tea-util==0.3.14 +alibabacloud-oss-v2==1.3.2 +alibabacloud-tea-util==0.3.15 alibabacloud-tea==0.4.3 alibabacloud_endpoint_util==0.0.4 alibabacloud_tea_openapi==0.3.16 alibabacloud_tea_xml==0.0.3 amqp==5.3.1 -annotated-doc==0.0.4 -annotated-types==0.7.0 -anyio==4.14.1 -apache-airflow-providers-airbyte==5.5.1 -apache-airflow-providers-akeyless==0.2.0 -apache-airflow-providers-alibaba==3.3.9 -apache-airflow-providers-amazon==9.31.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.14.2 +apache-airflow-providers-airbyte==6.0.1 +apache-airflow-providers-akeyless==0.3.0 +apache-airflow-providers-alibaba==3.4.0 +apache-airflow-providers-amazon==9.34.0 apache-airflow-providers-apache-cassandra==3.9.5 apache-airflow-providers-apache-drill==3.3.3 apache-airflow-providers-apache-druid==4.5.2 apache-airflow-providers-apache-flink==1.8.5 -apache-airflow-providers-apache-hdfs==4.12.1 -apache-airflow-providers-apache-hive==9.5.0 -apache-airflow-providers-apache-iceberg==1.4.1 +apache-airflow-providers-apache-hdfs==4.12.2 +apache-airflow-providers-apache-hive==9.6.1 +apache-airflow-providers-apache-iceberg==2.0.3 apache-airflow-providers-apache-impala==1.9.3 -apache-airflow-providers-apache-kafka==1.14.0 +apache-airflow-providers-apache-kafka==1.15.1 apache-airflow-providers-apache-kylin==3.10.5 -apache-airflow-providers-apache-livy==4.5.7 +apache-airflow-providers-apache-livy==4.6.0 apache-airflow-providers-apache-pig==4.8.5 apache-airflow-providers-apache-pinot==4.10.3 -apache-airflow-providers-apache-spark==6.2.0 +apache-airflow-providers-apache-spark==6.3.1 apache-airflow-providers-apache-tinkerpop==1.1.4 apache-airflow-providers-apprise==2.3.4 -apache-airflow-providers-arangodb==2.9.5 +apache-airflow-providers-arangodb==2.9.6 apache-airflow-providers-asana==2.11.4 -apache-airflow-providers-atlassian-jira==3.3.4 -apache-airflow-providers-celery==3.21.0 +apache-airflow-providers-atlassian-jira==3.3.5 +apache-airflow-providers-celery==3.23.1 apache-airflow-providers-clickhousedb==1.0.0 apache-airflow-providers-cloudant==4.3.5 -apache-airflow-providers-cncf-kubernetes==10.19.0 -apache-airflow-providers-cohere==1.6.6 -apache-airflow-providers-common-ai==0.5.0 -apache-airflow-providers-common-compat==1.15.0 +apache-airflow-providers-cncf-kubernetes==10.21.0 +apache-airflow-providers-cohere==1.6.7 +apache-airflow-providers-common-ai==0.7.0 +apache-airflow-providers-common-compat==1.18.0 apache-airflow-providers-common-io==1.8.0 apache-airflow-providers-common-messaging==2.0.4 -apache-airflow-providers-common-sql==2.0.1 -apache-airflow-providers-databricks==7.16.1 +apache-airflow-providers-common-sql==2.1.0 +apache-airflow-providers-databricks==7.18.1 apache-airflow-providers-datadog==3.10.5 -apache-airflow-providers-dbt-cloud==4.9.2 +apache-airflow-providers-dbt-cloud==4.9.3 apache-airflow-providers-dingding==3.9.5 apache-airflow-providers-discord==3.12.3 -apache-airflow-providers-docker==4.5.7 -apache-airflow-providers-edge3==4.0.0 -apache-airflow-providers-elasticsearch==6.7.0 -apache-airflow-providers-exasol==4.10.3 -apache-airflow-providers-fab==3.7.1 +apache-airflow-providers-docker==4.5.9 +apache-airflow-providers-edge3==4.3.0 +apache-airflow-providers-elasticsearch==6.9.0 +apache-airflow-providers-exasol==4.10.5 +apache-airflow-providers-fab==3.8.0 apache-airflow-providers-facebook==3.9.5 -apache-airflow-providers-ftp==3.15.1 -apache-airflow-providers-git==0.4.0 +apache-airflow-providers-ftp==3.15.2 +apache-airflow-providers-git==0.4.2 apache-airflow-providers-github==2.11.3 -apache-airflow-providers-google==22.1.0 +apache-airflow-providers-google==22.3.0 apache-airflow-providers-grpc==3.9.5 -apache-airflow-providers-hashicorp==4.7.1 -apache-airflow-providers-http==6.0.4 -apache-airflow-providers-imap==3.11.3 +apache-airflow-providers-hashicorp==4.8.0 +apache-airflow-providers-http==6.0.5 +apache-airflow-providers-imap==3.12.1 apache-airflow-providers-influxdb==2.11.0 apache-airflow-providers-informatica==0.2.0 apache-airflow-providers-jdbc==5.5.0 apache-airflow-providers-jenkins==4.2.6 -apache-airflow-providers-keycloak==0.8.1 -apache-airflow-providers-microsoft-azure==13.5.0 +apache-airflow-providers-keycloak==0.8.2 +apache-airflow-providers-microsoft-azure==14.1.0 apache-airflow-providers-microsoft-mssql==4.7.0 apache-airflow-providers-microsoft-psrp==3.2.6 -apache-airflow-providers-microsoft-winrm==3.14.3 +apache-airflow-providers-microsoft-winrm==3.14.4 apache-airflow-providers-mongo==5.4.0 apache-airflow-providers-mysql==6.6.1 -apache-airflow-providers-neo4j==3.11.6 +apache-airflow-providers-neo4j==3.12.1 apache-airflow-providers-odbc==4.12.3 -apache-airflow-providers-openai==1.7.5 +apache-airflow-providers-openai==1.8.2 apache-airflow-providers-openfaas==3.9.5 -apache-airflow-providers-openlineage==2.18.1 -apache-airflow-providers-opensearch==1.10.0 +apache-airflow-providers-openlineage==2.20.0 +apache-airflow-providers-opensearch==1.12.0 apache-airflow-providers-opsgenie==5.10.4 -apache-airflow-providers-oracle==4.6.1 +apache-airflow-providers-oracle==4.6.2 apache-airflow-providers-pagerduty==5.2.6 -apache-airflow-providers-papermill==3.13.1 -apache-airflow-providers-pgvector==1.7.2 +apache-airflow-providers-papermill==3.13.2 +apache-airflow-providers-pgvector==1.7.3 apache-airflow-providers-pinecone==2.4.5 -apache-airflow-providers-postgres==6.8.0 -apache-airflow-providers-presto==5.12.0 +apache-airflow-providers-postgres==7.0.1 +apache-airflow-providers-presto==5.12.1 apache-airflow-providers-qdrant==1.5.6 apache-airflow-providers-redis==4.5.0 -apache-airflow-providers-salesforce==5.14.1 +apache-airflow-providers-salesforce==5.14.2 apache-airflow-providers-samba==4.12.6 apache-airflow-providers-segment==3.9.5 apache-airflow-providers-sendgrid==4.2.4 -apache-airflow-providers-sftp==5.8.2 +apache-airflow-providers-sftp==6.0.1 apache-airflow-providers-singularity==3.9.4 apache-airflow-providers-slack==9.10.2 -apache-airflow-providers-smtp==3.0.1 -apache-airflow-providers-snowflake==6.4.0 +apache-airflow-providers-smtp==3.0.3 +apache-airflow-providers-snowflake==6.16.0 apache-airflow-providers-sqlite==4.3.3 -apache-airflow-providers-ssh==5.0.3 -apache-airflow-providers-standard==1.15.0 -apache-airflow-providers-tableau==5.5.0 -apache-airflow-providers-telegram==4.9.5 -apache-airflow-providers-teradata==3.6.0 -apache-airflow-providers-trino==6.6.0 +apache-airflow-providers-ssh==6.0.1 +apache-airflow-providers-standard==1.17.0 +apache-airflow-providers-tableau==5.6.0 +apache-airflow-providers-telegram==4.9.6 +apache-airflow-providers-teradata==3.6.2 +apache-airflow-providers-trino==6.6.1 apache-airflow-providers-vertica==4.4.0 apache-airflow-providers-vespa==0.1.1 -apache-airflow-providers-weaviate==3.3.5 -apache-airflow-providers-yandex==4.5.0 +apache-airflow-providers-weaviate==3.4.1 +apache-airflow-providers-yandex==4.5.1 apache-airflow-providers-ydb==2.5.3 apache-airflow-providers-zendesk==4.12.0 apispec==6.10.0 -apprise==1.11.0 -argcomplete==3.7.0 +apprise==1.12.0 +argcomplete==3.7.2 arrow==1.4.0 asana==5.2.5 -asgiref==3.11.1 +asgiref==3.12.1 asn1crypto==1.5.1 -asttokens==3.0.1 +asttokens==3.0.2 async-timeout==4.0.3 asyncpg==0.31.0 asyncssh==2.24.0 atlasclient==1.0.0 atlassian-python-api==4.0.7 attrs==26.1.0 +azure-ai-projects==2.4.0 azure-batch==14.2.0 azure-common==1.1.28 azure-core==1.41.0 -azure-cosmos==4.16.1 +azure-cosmos==4.16.3 azure-datalake-store==0.0.53 azure-identity==1.25.3 azure-keyvault-secrets==4.11.0 azure-kusto-data==6.0.4 -azure-mgmt-compute==38.1.0 +azure-mgmt-compute==38.2.0 azure-mgmt-containerinstance==10.1.0 azure-mgmt-containerregistry==15.0.0 azure-mgmt-core==1.6.0 -azure-mgmt-cosmosdb==9.9.0 -azure-mgmt-datafactory==9.3.0 +azure-mgmt-cosmosdb==10.0.0 +azure-mgmt-datafactory==10.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==26.0.0 -azure-mgmt-storage==25.0.0 +azure-mgmt-storage==25.1.0 azure-nspkg==3.0.2 azure-servicebus==7.14.3 azure-storage-blob==12.30.0 @@ -235,47 +239,48 @@ azure-synapse-artifacts==0.22.0 azure-synapse-spark==0.7.0 babel==2.18.0 backoff==2.2.1 +backports.zstd==1.6.0 bcrypt==5.0.0 beautifulsoup4==4.15.0 billiard==4.2.4 -bitarray==3.8.2 +bitarray==3.10.1 black==26.5.1 bleach==6.4.0 blinker==1.9.0 -boto3==1.43.0 -botocore==1.43.0 -cachelib==0.14.0 -cachetools==7.1.4 +boto3==1.43.56 +botocore==1.43.56 +cachelib==0.15.2 +cachetools==6.2.6 cadwyn==7.0.0 capi_param_builder_python==1.3.0 -cassandra-driver==3.30.0 +cassandra-driver==3.30.1 cattrs==26.1.0 celery==5.6.3 -certifi==2026.6.17 -cffi==2.0.0 -chardet==6.0.0.post1 -charset-normalizer==3.4.7 +certifi==2026.7.22 +cffi==2.1.1 +chardet==7.5.1 +charset-normalizer==3.4.9 ciso8601==2.3.3 click-didyoumean==0.3.1 click-plugins==1.1.1.2 click-repl==0.3.0 click==8.4.2 -clickhouse-connect==1.4.1 -cloudpickle==3.1.2 -cohere==7.0.5 +clickhouse-connect==1.6.0 +cloudpickle==3.1.1 +cohere==7.0.8 colorama==0.4.6 colorful==0.5.8 -colorlog==6.10.1 +colorlog==6.12.0 comm==0.2.3 confluent-kafka==2.15.0 crcmod-plus==2.3.1 cron_descriptor==2.1.0 -croniter==6.2.2 -cryptography==48.0.1 +croniter==6.2.4 +cryptography==50.0.0 curlify==3.0.0 -databricks-sql-connector==4.2.5 -datadog==0.52.2 -db-dtypes==1.7.0 +databricks-sql-connector==4.4.0 +datadog==0.53.0 +db-dtypes==1.7.1 debugpy==1.8.21 decorator==5.3.1 defusedxml==0.7.1 @@ -284,153 +289,156 @@ dill==0.4.1 distlib==0.4.3 distro==1.9.0 dnspython==2.8.0 -docker==7.1.0 +docker==7.2.0 docopt==0.6.2 docstring_parser==0.18.0 durationpy==0.10 elastic-transport==9.4.2 -elasticsearch==9.4.1 +elasticsearch==9.5.0 email-validator==2.3.0 entrypoints==0.4 +et_xmlfile==2.0.0 eventlet==0.41.0 executing==2.2.1 -facebook_business==25.0.2 -fastapi-cli==0.0.27 +facebook_business==26.0.0 +fastapi-cli==0.0.32 fastapi==0.136.3 fastavro==1.12.2 -fastcore==1.13.9 -fastjsonschema==2.21.2 +fastcore==2.2.5 +fastjsonschema==2.22.1 fastuuid==0.14.0 -filelock==3.29.4 -flask-appbuilder==5.2.1 +filelock==3.32.2 +flask-appbuilder==5.2.2 flask-babel==4.0.0 flower==2.0.1 frozenlist==1.8.0 -fsspec==2026.6.0 +fsspec==2026.7.0 future==1.0.0 -gcloud-aio-auth==5.4.4 +gcloud-aio-auth==5.5.0 gcloud-aio-bigquery==7.1.0 gcloud-aio-storage==9.6.4 -gcsfs==2026.6.0 -genai-prices==0.0.69 +gcsfs==2026.7.0 +genai-prices==0.1.1 geomet==1.1.0 -gevent==26.5.0 +gevent==26.7.0 gitdb==4.0.12 -google-ads==31.1.0 +google-ads==31.2.0 google-analytics-admin==0.30.1 -google-api-core==2.31.0 +google-api-core==2.34.0 google-api-python-client==2.198.0 -google-auth-httplib2==0.4.0 +google-auth-httplib2==0.4.1 google-auth-oauthlib==1.4.0 -google-auth==2.55.1 -google-cloud-aiplatform==1.148.1 +google-auth==2.56.3 +google-cloud-aiplatform==1.163.0 google-cloud-alloydb==0.11.0 google-cloud-appengine-logging==1.10.0 -google-cloud-audit-log==0.6.0 +google-cloud-audit-log==0.6.1 google-cloud-automl==2.20.0 -google-cloud-batch==0.22.0 +google-cloud-batch==0.22.2 google-cloud-bigquery-datatransfer==3.23.0 -google-cloud-bigquery-storage==2.39.0 -google-cloud-bigquery==3.42.1 -google-cloud-bigtable==2.40.0 -google-cloud-build==3.38.0 -google-cloud-compute==1.49.0 +google-cloud-bigquery-storage==2.40.0 +google-cloud-bigquery==3.43.0 +google-cloud-bigtable==2.41.0 +google-cloud-build==3.38.1 +google-cloud-compute==1.50.0 google-cloud-container==2.65.0 -google-cloud-core==2.6.0 +google-cloud-core==2.6.1 google-cloud-datacatalog==3.31.0 google-cloud-dataflow-client==0.14.0 -google-cloud-dataform==0.11.1 +google-cloud-dataform==0.11.2 google-cloud-dataplex==2.20.0 google-cloud-dataproc-metastore==1.23.0 -google-cloud-dataproc==5.29.0 +google-cloud-dataproc==5.30.0 google-cloud-dlp==3.38.0 -google-cloud-kms==3.14.0 +google-cloud-kms==3.16.0 google-cloud-language==2.21.0 -google-cloud-logging==3.16.0 +google-cloud-logging==3.16.2 google-cloud-managedkafka==0.4.1 google-cloud-memcache==1.16.0 google-cloud-monitoring==2.31.0 google-cloud-orchestration-airflow==1.22.0 google-cloud-os-login==2.22.0 -google-cloud-pubsub==2.39.0 +google-cloud-pubsub==2.39.1 google-cloud-redis==2.22.0 google-cloud-resource-manager==1.18.0 google-cloud-run==0.16.1 -google-cloud-secret-manager==2.29.0 -google-cloud-spanner==3.69.0 +google-cloud-secret-manager==2.30.0 +google-cloud-spanner==3.69.1 google-cloud-speech==2.40.0 -google-cloud-storage-control==1.12.0 +google-cloud-storage-control==1.13.0 google-cloud-storage-transfer==1.21.0 -google-cloud-storage==3.12.0 -google-cloud-tasks==2.23.0 +google-cloud-storage==3.13.1 +google-cloud-tasks==2.24.0 google-cloud-texttospeech==2.37.0 google-cloud-translate==3.27.0 google-cloud-videointelligence==2.20.0 google-cloud-vision==3.15.0 google-cloud-workflows==1.23.0 google-crc32c==1.8.0 -google-genai==1.75.0 -google-resumable-media==2.10.0 -googleapis-common-protos==1.75.0 +google-genai==2.17.0 +google-resumable-media==2.10.1 +googleapis-common-protos==1.75.1 graphviz==0.21 greenback==1.3.0 -greenlet==3.5.3 +greenlet==3.5.4 gremlinpython==3.8.1 griffelib==2.1.0 -grpc-google-iam-v1==0.14.4 +grpc-google-iam-v1==0.14.5 grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 -grpcio-health-checking==1.81.1 -grpcio-status==1.81.1 -grpcio-tools==1.81.1 -grpcio==1.81.1 +grpcio-status==1.78.0 +grpcio-tools==1.78.0 +grpcio==1.78.0 gssapi==1.11.1 gunicorn==26.0.0 h11==0.16.0 -h2==4.3.0 +h2==4.4.1 hdfs==2.7.3 -hf-xet==1.5.1 +hf-xet==1.6.0 hmsclient==0.1.1 hpack==4.2.0 -httpcore2==2.5.0 +httpcore2==2.9.1 httpcore==1.0.9 httplib2==0.32.0 -httpr==0.4.8 +httpr==0.6.0 httptools==0.8.0 -httpx2==2.5.0 +httpx2==2.9.1 httpx==0.28.1 -huggingface_hub==1.21.0 +huggingface_hub==1.27.0 humanize==4.16.0 hvac==2.4.0 hyperframe==6.1.0 -ibm-cloud-sdk-core==3.24.4 -ibmcloudant==0.11.8 +ibm-cloud-sdk-core==3.25.0 +ibmcloudant==0.11.9 idna==3.18 ijson==3.4.0.post0 immutabledict==4.3.1 -importlib_metadata==9.0.0 -# Hand-patched (keep on regeneration): impyla <0.24.0 hard-pins thrift==0.16.0, which holds -# thrift below the 0.24.0 release that fixes CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. +importlib_metadata==8.9.0 +importlib_resources==7.1.0 +# Upstream constraints-3.3.1 now ships this natively, but keep the pin on regeneration: +# impyla <0.24.0 hard-pins thrift==0.16.0, which holds thrift below the 0.24.0 release +# that fixes CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. # Must stay in sync with the hive/impala extras in ingestion/setup.py. impyla==0.24.0 inflection==0.5.1 influxdb-client==1.50.0 influxdb3-python==0.20.0 +invoke==3.0.3 ipykernel==7.3.0 -ipython==9.15.0 +ipython==9.16.1 ipython_pygments_lexers==1.1.1 isodate==0.7.2 isoduration==20.11.0 itsdangerous==2.2.0 jaraco.classes==3.4.0 jaraco.context==6.1.2 -jaraco.functools==4.5.0 +jaraco.functools==4.6.0 jedi==0.20.0 jeepney==0.9.0 jiter==0.16.0 jmespath==1.1.0 joblib==1.5.3 -joserfc==1.7.2 +joserfc==1.7.4 jpype1==1.7.1 jsonpath-ng==1.8.0 jsonschema-specifications==2025.9.1 @@ -442,17 +450,17 @@ jwcrypto==1.5.8 keyring==25.7.0 kombu==5.6.2 krb5==0.9.0 -kubernetes==36.0.2 +kubernetes==36.0.3 kubernetes_asyncio==36.1.0 kylinpy==2.8.4 lazy-object-proxy==1.12.0 -libcst==1.8.6 +libcst==1.9.0 limits==5.8.0 linkify-it-py==2.1.0 -litellm==1.82.6 +litellm==1.85.7 lockfile==0.12.2 -logfire-api==4.37.0 -looker_sdk==26.10.0 +logfire-api==4.40.0 +looker_sdk==26.12.0 lxml==6.1.1 lz4==4.4.5 markdown-it-py==4.2.0 @@ -461,7 +469,7 @@ marshmallow==4.3.0 matplotlib-inline==0.2.2 mdit-py-plugins==0.6.1 mdurl==0.1.2 -memray==1.19.3 +memray==1.20.0 mergedeep==1.3.4 methodtools==0.4.7 microsoft-kiota-abstractions==1.11.7 @@ -469,13 +477,13 @@ microsoft-kiota-authentication-azure==1.11.7 microsoft-kiota-http==1.11.7 microsoft-kiota-serialization-json==1.11.7 microsoft-kiota-serialization-text==1.11.7 -mistune==3.3.2 +mistune==3.3.4 mmh3==5.2.1 more-itertools==11.1.0 msal-extensions==1.3.1 msal==1.37.0 msgpack==1.2.1 -msgraph-core==1.4.0 +msgraph-core==1.5.1 msgraphfs==0.5 msgspec==0.21.1 msrest==0.7.1 @@ -483,93 +491,97 @@ msrestazure==0.6.4.post1 multi_key_dict==2.0.3 multidict==6.7.1 mypy_extensions==1.1.0 -mysql-connector-python==9.6.0 +mysql-connector-python==26.7.0 mysqlclient==2.2.8 -narwhals==2.23.0 +narwhals==2.24.0 natsort==8.4.0 nbclient==0.11.0 nbconvert==7.17.1 -nbformat==5.10.4 +nbformat==5.11.0 neo4j==6.2.0 nest-asyncio2==1.7.2 nest-asyncio==1.6.0 -numpy==1.26.4 +numpy==2.5.1 oauthlib==3.3.1 -openai==2.44.0 +openai==2.53.0 opencensus-context==0.1.3 opencensus==0.11.4 -openlineage-integration-common==1.50.0 -openlineage-python==1.50.0 -openlineage_sql==1.50.0 +openlineage-integration-common==1.52.0 +openlineage-python==1.52.0 +openlineage_sql==1.52.0 +openpyxl==3.1.5 opensearch-protobufs==1.2.0 opensearch-py==3.2.0 -opentelemetry-api==1.43.0 -opentelemetry-exporter-otlp-proto-common==1.43.0 -opentelemetry-exporter-otlp-proto-grpc==1.43.0 -opentelemetry-exporter-otlp-proto-http==1.43.0 -opentelemetry-exporter-otlp==1.43.0 -opentelemetry-exporter-prometheus==0.64b0 -opentelemetry-proto==1.43.0 -opentelemetry-resourcedetector-gcp==1.12.0a0 -opentelemetry-sdk==1.43.0 -opentelemetry-semantic-conventions==0.64b0 +opentelemetry-api==1.44.0 +opentelemetry-exporter-otlp-proto-common==1.44.0 +opentelemetry-exporter-otlp-proto-grpc==1.44.0 +opentelemetry-exporter-otlp-proto-http==1.44.0 +opentelemetry-exporter-otlp==1.44.0 +opentelemetry-exporter-prometheus==0.65b0 +opentelemetry-proto==1.44.0 +opentelemetry-resourcedetector-gcp==1.14.0 +opentelemetry-sdk==1.44.0 +opentelemetry-semantic-conventions==0.65b0 opsgenie-sdk==2.1.5 -oracledb==4.0.1 +oracledb==4.0.2 ordered-set==4.1.0 orjson==3.11.9 outcome==1.3.0.post0 -packaging==26.2 -pagerduty==6.3.0 -pandas-gbq==0.35.0 -pandas-stubs==3.0.3.260530 -pandas==2.1.4 +packaging==26.3 +pagerduty==7.0.0 +pandas-gbq==0.35.1 +pandas==3.0.5 pandocfilters==1.5.1 papermill==2.7.0 -paramiko==3.5.1 +paramiko==5.0.0 parso==0.8.7 pathlib_abc==0.5.2 pathspec==1.1.1 pbr==7.0.3 pendulum==3.2.0 pexpect==4.9.0 -pgvector==0.4.2 +pgvector==0.5.0 pinecone==9.1.0 pinotdb==9.1.2 -platformdirs==4.10.0 +platformdirs==4.11.1 pluggy==1.6.0 -polars-runtime-32==1.42.1 -polars==1.42.1 +polars-runtime-32==1.43.2 +polars==1.43.2 portalocker==3.2.0 presto-python-client==0.8.4 prison==0.2.1 -prometheus_client==0.25.0 -prompt_toolkit==3.0.52 +prometheus_client==0.26.0 +prompt_toolkit==3.0.53 propcache==0.5.2 -proto-plus==1.28.0 +proto-plus==1.28.3 protobuf==6.33.6 psutil==7.2.2 +psycopg-binary==3.3.4 psycopg2-binary==2.9.12 +psycopg==3.3.4 ptyprocess==0.7.0 pure-sasl==0.6.2 pure_eval==0.2.3 py-spy==0.4.2 -pyOpenSSL==26.2.0 -pyarrow==24.0.0 -pyasn1==0.6.3 +pyOpenSSL==26.4.0 +pyarrow==25.0.0 +pyasn1==0.6.4 pyasn1_modules==0.4.2 +pybreaker==1.4.1 pycountry==26.2.16 pycparser==3.0 pycryptodome==3.23.0 -pydantic-ai-slim==2.3.0 +pydantic-ai-slim==2.27.0 pydantic-extra-types==2.11.1 -pydantic-graph==2.3.0 -pydantic-settings==2.14.2 +pydantic-graph==2.27.0 +pydantic-settings==2.15.0 pydantic==2.13.4 pydantic_core==2.46.4 pydata-google-auth==1.9.1 pydruid==0.6.9 -pyexasol==2.2.2 +pyexasol==1.3.0 pygtrie==2.5.0 +pyiceberg==0.11.1 pykerberos==1.2.4 pymongo==4.17.0 pymssql==2.3.13 @@ -577,12 +589,13 @@ pyodbc==5.3.0 pyodps==0.13.0 pyparsing==3.3.2 pypsrp==0.9.1 -pyspark-client==4.0.3 +pyroaring==1.1.0 +pyspark-client==4.2.0 pyspnego==0.12.1 python-arango==8.3.3 python-daemon==3.1.2 python-dateutil==2.9.0.post0 -python-discovery==1.4.2 +python-discovery==1.5.1 python-dotenv==1.2.2 python-http-client==3.3.7 python-jenkins==1.8.3 @@ -593,17 +606,17 @@ python-slugify==8.0.4 python-telegram-bot==22.8 python3-saml==1.16.0 pytokens==0.4.1 -pytz==2026.2 -pyvespa==1.2.3 +pytz==2026.3.post1 +pyvespa==1.2.4 pywinrm==0.5.0 pyzmq==27.1.0 -qdrant-client==1.18.0 -ray==2.56.0 -reactivex==4.1.0 +qdrant-client==1.19.0 +ray==2.56.1 +reactivex==5.1.0 redis==6.4.0 -redshift_connector==2.1.15 +redshift_connector==2.1.16 referencing==0.37.0 -regex==2026.6.28 +regex==2026.7.19 requests-file==3.0.1 requests-kerberos==0.15.0 requests-oauthlib==2.0.0 @@ -612,104 +625,108 @@ requests==2.34.2 requests_ntlm==1.3.0 retryhttp==1.5.0 rich-argparse==1.8.0 -rich-toolkit==0.20.1 -rich==13.9.4 +rich-toolkit==0.20.3 +rich==14.3.4 rpds-py==2026.6.3 rsa==4.9.1 ruamel.yaml==0.19.1 -s3fs==2026.6.0 -s3transfer==0.17.1 -sagemaker_studio==1.0.26 +s3fs==2026.7.0 +s3transfer==0.19.2 +sagemaker_studio==1.0.27 scikit-learn==1.9.0 -scipy==1.17.1 -scramp==1.4.10 +scipy==1.18.0 +scramp==1.4.17 scrapbook==0.5.0 segment-analytics-python==2.3.6 sendgrid==6.12.5 -sentry-sdk==2.64.0 +sentry-sdk==2.66.1 setproctitle==1.3.7 -setuptools==82.0.1 +setuptools==83.0.0 shellingham==1.5.4 -simple-salesforce==1.12.9 +simple-salesforce==1.12.10 six==1.17.0 slack_sdk==3.43.0 -smart_open==8.0.0 -smbprotocol==1.16.1 +smart_open==8.0.1 +smbprotocol==1.17.0 smmap==5.0.3 sniffio==1.3.1 -snowflake-connector-python==4.6.0 -snowflake-sqlalchemy==1.10.2 +snowflake-connector-python==4.7.2 +snowflake-snowpark-python==1.54.0 +snowflake-sqlalchemy==1.11.0 sortedcontainers==2.4.0 -soupsieve==2.8.4 +soupsieve==2.9.2 spython==0.3.14 -sqlalchemy-bigquery==1.17.0 +sqlalchemy-bigquery==1.17.2 sqlalchemy-spanner==1.19.0 sqlalchemy_drill==1.1.10 -sqlglot==30.12.0 +sqlglot==30.15.0 sqlparse==0.5.5 stack-data==0.6.3 -starlette==1.3.1 +starlette==1.5.0 statsd==4.0.1 -std-uritemplate==2.0.10 +std-uritemplate==2.0.12 +strictyaml==1.7.3 structlog==26.1.0 -svcs==25.1.0 +svcs==26.1.0 tableauserverclient==0.41 tabulate==0.10.0 tenacity==9.1.4 -teradatasql==20.0.0.62 +teradatasql==20.0.0.64 teradatasqlalchemy==20.0.0.9 termcolor==3.3.0 text-unidecode==1.3 -textual==6.2.1 +textual==8.2.8 threadpoolctl==3.6.0 thrift-sasl==0.4.3 -# Hand-patched (keep on regeneration): see the impyla note above. thrift <0.24.0 is vulnerable -# to CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. +# Upstream constraints-3.3.1 now ships this natively, but keep the pin on regeneration: +# see the impyla note above. thrift <0.24.0 is vulnerable to CVE-2026-66053 / +# CVE-2026-41608 / CVE-2026-48586. thrift==0.24.0 tiktoken==0.13.0 tinycss2==1.5.1 tokenizers==0.23.1 -tomlkit==0.15.0 -tornado==6.5.7 -tqdm==4.68.3 -traitlets==5.15.1 +tomlkit==0.15.1 +tornado==6.5.8 +tqdm==4.70.0 +traitlets==5.16.1 trino==0.338.0 truststore==0.10.4 -typer==0.25.1 +typer==0.27.1 types-protobuf==7.34.1.20260518 -types-requests==2.33.0.20260518 +types-requests==2.33.0.20260712 typing-inspection==0.4.2 typing_extensions==4.16.0 -tzdata==2026.2 +tzdata==2026.3 tzlocal==5.4.4 uc-micro-py==2.0.0 universal_pathlib==0.3.10 uritemplate==4.2.0 urllib3==2.7.0 uuid6==2025.0.1 -uv==0.11.26 -uvicorn==0.49.0 +uv==0.12.3 +uvicorn==0.52.1 uvloop==0.22.1 validators==0.35.0 vertica-python==1.4.0 vine==5.1.0 -virtualenv==21.5.1 +virtualenv==21.7.2 watchfiles==1.2.0 watchtower==3.4.0 wcwidth==0.8.2 -weaviate-client==4.16.2 +weaviate-client==4.22.0 webencodings==0.5.1 -websocket-client==1.9.0 -websockets==16.0 +websocket-client==1.8.0 +websockets==16.1.1 +wheel==0.47.0 wirerope==1.0.0 -wrapt==2.2.2 +wrapt==2.3.0 xmlsec==1.3.17 xmltodict==1.0.4 yandex-query-client==0.1.4 -yandexcloud==0.397.0 -yarl==1.24.2 +yandexcloud==0.402.0 +yarl==1.24.5 ydb-dbapi==0.1.22 -ydb==3.29.6 +ydb==3.31.2 zeep==4.3.3 zenpy==2.0.57 zipp==4.1.0 diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 742d7ff071f9..989104b00272 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -197,29 +197,38 @@ RUN pip install psycopg2 mysqlclient==2.1.1 # apt-mark manual pins them before autoremove runs so the cleanup doesn't take # them out too -- verified end-to-end by the `import MySQLdb` gate check. # -# The util-linux upgrade rides along in the same root layer. trixie's base ships -# 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and CVE-2026-27456; -# trixie-security has 2.41.5-0+deb13u1. -# The package set is computed from dpkg rather than hand-listed. One source +# The OS security upgrades ride along in the same root layer. trixie's base ships +# util-linux 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and +# CVE-2026-27456; trixie-security has 2.41.5-0+deb13u1. +# expat is the same story one release later. It is not in the trixie base at all +# -- it arrives transitively from the top-of-file apt layer -- and that layer's +# frozen index is exactly why the image keeps shipping 2.8.2 and being reported +# for CVE-2026-72522 while trixie-security has 2.8.3-1~deb13u1. +# The package sets are computed from dpkg rather than hand-listed. One source # package produces many binaries -- here util-linux, bsdutils, login, mount, # liblastlog2-2, libblkid1, libmount1, libsmartcols1 and libuuid1 -- and scanners # report each separately, so a hand-written list silently leaves behind whichever # binary it forgot, and liblastlog2-2 is exactly the one that is easy to forget. -# Asking dpkg which installed packages came from the util-linux source cannot -# miss one, and stays correct if Debian splits the source differently later. -# The empty check is not defensive noise: `apt-get install --only-upgrade` with -# no package arguments exits 0, so a query that silently returned nothing would -# give a green build that shipped the vulnerable packages anyway. Fail closed. +# Asking dpkg which installed packages came from the source cannot miss one, and +# stays correct if Debian splits the source differently later. +# util-linux is asserted non-empty and expat deliberately is not: util-linux is +# Essential, so an empty query there means dpkg-query misbehaved and the build +# must not continue, whereas expat is transitive and an empty query is a +# legitimate image with nothing to patch. Asserting one of the two is what the +# check is for -- `apt-get install --only-upgrade` with no package arguments +# exits 0, so an all-empty query would give a green build that shipped the +# vulnerable packages anyway. Fail closed. # This deliberately does NOT go in the top-of-file apt RUN: that layer's cache # key never changes, so an upgrade placed there freezes its Debian index with it # and the image keeps shipping whatever was current when the layer was first # built. Below the COPY and pip layers, the index is re-read on every build. USER root RUN set -eu; \ - pkgs="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ - [ -n "$pkgs" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ul="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ + [ -n "$ul" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ex="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="expat"{print $2}')"; \ apt-get -qq update; \ - apt-get -qq install -y --only-upgrade $pkgs; \ + apt-get -qq install -y --only-upgrade $ul $ex; \ apt-get -qq purge -y libmariadb-dev libmariadb-dev-compat libunbound8; \ apt-mark manual libmariadb3 mariadb-common; \ apt-get -qq autoremove -y --purge; \ diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index c7fac3356237..ede6cd033875 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -205,29 +205,38 @@ RUN pip install psycopg2 mysqlclient==2.1.1 # apt-mark manual pins them before autoremove runs so the cleanup doesn't take # them out too -- verified end-to-end by the `import MySQLdb` gate check. # -# The util-linux upgrade rides along in the same root layer. trixie's base ships -# 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and CVE-2026-27456; -# trixie-security has 2.41.5-0+deb13u1. -# The package set is computed from dpkg rather than hand-listed. One source +# The OS security upgrades ride along in the same root layer. trixie's base ships +# util-linux 2.41-5, which carries CVE-2025-14104, CVE-2026-13595 and +# CVE-2026-27456; trixie-security has 2.41.5-0+deb13u1. +# expat is the same story one release later. It is not in the trixie base at all +# -- it arrives transitively from the top-of-file apt layer -- and that layer's +# frozen index is exactly why the image keeps shipping 2.8.2 and being reported +# for CVE-2026-72522 while trixie-security has 2.8.3-1~deb13u1. +# The package sets are computed from dpkg rather than hand-listed. One source # package produces many binaries -- here util-linux, bsdutils, login, mount, # liblastlog2-2, libblkid1, libmount1, libsmartcols1 and libuuid1 -- and scanners # report each separately, so a hand-written list silently leaves behind whichever # binary it forgot, and liblastlog2-2 is exactly the one that is easy to forget. -# Asking dpkg which installed packages came from the util-linux source cannot -# miss one, and stays correct if Debian splits the source differently later. -# The empty check is not defensive noise: `apt-get install --only-upgrade` with -# no package arguments exits 0, so a query that silently returned nothing would -# give a green build that shipped the vulnerable packages anyway. Fail closed. +# Asking dpkg which installed packages came from the source cannot miss one, and +# stays correct if Debian splits the source differently later. +# util-linux is asserted non-empty and expat deliberately is not: util-linux is +# Essential, so an empty query there means dpkg-query misbehaved and the build +# must not continue, whereas expat is transitive and an empty query is a +# legitimate image with nothing to patch. Asserting one of the two is what the +# check is for -- `apt-get install --only-upgrade` with no package arguments +# exits 0, so an all-empty query would give a green build that shipped the +# vulnerable packages anyway. Fail closed. # This deliberately does NOT go in the top-of-file apt RUN: that layer's cache # key never changes, so an upgrade placed there freezes its Debian index with it # and the image keeps shipping whatever was current when the layer was first # built. Below the COPY and pip layers, the index is re-read on every build. USER root RUN set -eu; \ - pkgs="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ - [ -n "$pkgs" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ul="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="util-linux"{print $2}')"; \ + [ -n "$ul" ] || { echo "no src:util-linux packages found; refusing to skip the CVE patch" >&2; exit 1; }; \ + ex="$(dpkg-query -W -f='${source:Package} ${Package}\n' | awk '$1=="expat"{print $2}')"; \ apt-get -qq update; \ - apt-get -qq install -y --only-upgrade $pkgs; \ + apt-get -qq install -y --only-upgrade $ul $ex; \ apt-get -qq purge -y libmariadb-dev libmariadb-dev-compat libunbound8; \ apt-mark manual libmariadb3 mariadb-common; \ apt-get -qq autoremove -y --purge; \ diff --git a/ingestion/setup.py b/ingestion/setup.py index a734cc234375..156629ae49e6 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -20,8 +20,10 @@ # Add here versions required for multiple plugins VERSIONS = { # CVE-2026-42252 BashOperator Jinja2 injection; CVE-2026-48891 /ui/dependencies leaks - # Dag IDs the caller cannot read (residual gap in the CVE-2026-28563 fix, needs 3.3.0) - "airflow": "apache-airflow==3.3.0", + # Dag IDs the caller cannot read (residual gap in the CVE-2026-28563 fix, needs 3.3.0); + # CVE-2026-67587 Dag-author RCE on the Scheduler via a Serde Callback deserialization + # gadget and CVE-2026-54183 Variables unmasked in the UI (both need 3.3.1) + "airflow": "apache-airflow==3.3.1", "adlfs": "adlfs>=2023.1.0", "aiobotocore": "aiobotocore~=2.26.0", "avro": "avro>=1.11.4,<1.12", @@ -38,15 +40,15 @@ "ijson": "ijson~=3.4", "msal": "msal~=1.2", "neo4j": "neo4j~=5.3", - "pandas": "pandas~=2.1.4", + "pandas": "pandas>=2.2.2,<3", "pyarrow": "pyarrow>=23.0.1,<26", # CVE-2026-25087 / CVE-2024-52338 IPC pre-buffer use-after-free (fixed in 23.0.1) - "pydantic": "pydantic~=2.0,>=2.7.0,<2.12", # Pin down to <2.12 due to breaking changes in 2.12.0 + "pydantic": "pydantic>=2.12.5,<3", "pydantic-settings": "pydantic-settings~=2.0,>=2.14.2", # GHSA-4xgf-cpjx-pc3j secrets_dir symlink escape "pydomo": "pydomo~=0.3", "pymysql": "pymysql~=1.0", "pyodbc": "pyodbc~=5.3.0", - "numpy": "numpy<2", - "scikit-learn": "scikit-learn>=1.3,<2", + "numpy": "numpy>=2,<3", + "scikit-learn": "scikit-learn>=1.4.2,<2", "packaging": "packaging", "azure-storage-blob": "azure-storage-blob~=12.14", "azure-identity": "azure-identity~=1.12", @@ -54,7 +56,7 @@ "databricks-sql-connector": "databricks-sql-connector>=4.0.0", "databricks-sqlalchemy": "databricks-sqlalchemy~=2.0.9", "trino": "trino[sqlalchemy]", - "spacy": "spacy<3.8", + "spacy": "spacy>=3.8.2,<3.9", "looker-sdk": "looker-sdk>=22.20.0,!=24.18.0", "lkml": "lkml~=1.3", "tableau": "tableauserverclient==0.40", # pre-0.37 pins urllib3<2, which conflicts with collate-data-diff's urllib3>=2.7 @@ -75,7 +77,7 @@ "s3fs": "s3fs~=2026.3", "sqlalchemy-bigquery": "sqlalchemy-bigquery>=1.15.0", "presidio-analyzer": "presidio-analyzer==2.2.358", - "asammdf": "asammdf~=7.4.5", + "asammdf": "asammdf~=8.1.0", "kafka-connect": "kafka-connect-py==0.10.11", "griffe2md": "griffe2md~=1.2", "factory-boy": "factory-boy~=3.3.3", @@ -236,7 +238,10 @@ DATA_DIFF["clickhouse"], }, "dagster": { - "croniter<3", + # No croniter ceiling here: dagster 1.13 declares no croniter dependency at all, + # nothing under ingestion/ imports it, and apache-airflow-core 3.3.1 raised its + # floor to croniter>=6.2.2 -- a stale "croniter<3" makes the two uninstallable + # together. The airflow images already run croniter 6.2.x via the constraints file. VERSIONS["pymysql"], "psycopg2-binary", VERSIONS["geoalchemy2"], @@ -445,7 +450,7 @@ "google-api-python-client-stubs", "google-auth-stubs", "types-requests", - "pandas-stubs~=2.1.4", + "pandas-stubs~=2.2", "scipy-stubs", "nox", "pre-commit", diff --git a/ingestion/src/metadata/ingestion/models/custom_pydantic.py b/ingestion/src/metadata/ingestion/models/custom_pydantic.py index c76180a09804..4bee6dc50b92 100644 --- a/ingestion/src/metadata/ingestion/models/custom_pydantic.py +++ b/ingestion/src/metadata/ingestion/models/custom_pydantic.py @@ -117,21 +117,15 @@ def model_post_init(self, context: Any, /): logger.warning(f"Exception while parsing FilterPattern: {exc}") @model_validator(mode="after") - @classmethod - def parse_name(cls, values): # pylint: disable=inconsistent-return-statements + def parse_name(self): # pylint: disable=inconsistent-return-statements """ Transform entity names using hybrid configuration system. """ - - if not values: - return values - try: - # Try new hybrid system first - return transform_entity_names(entity=values, model=cls) + return transform_entity_names(entity=self, model=type(self)) except Exception as exc: logger.warning("Exception while parsing Basemodel: %s", exc) - return values + return self def model_dump_json( # pylint: disable=too-many-arguments self, diff --git a/ingestion/src/metadata/ingestion/models/topology.py b/ingestion/src/metadata/ingestion/models/topology.py index 2321d5c16d4e..6401fd3cd249 100644 --- a/ingestion/src/metadata/ingestion/models/topology.py +++ b/ingestion/src/metadata/ingestion/models/topology.py @@ -145,7 +145,7 @@ def create(cls, topology: ServiceTopology) -> "TopologyContext": :return: TopologyContext """ nodes = get_topology_nodes(topology) - ctx_fields = { + ctx_fields: dict[str, Any] = { stage.context: (Optional[stage.type_], None) # noqa: UP045 for node in nodes for stage in node.stages diff --git a/ingestion/src/metadata/pii/algorithms/utils.py b/ingestion/src/metadata/pii/algorithms/utils.py index 922e501cc6a5..8d52b514884d 100644 --- a/ingestion/src/metadata/pii/algorithms/utils.py +++ b/ingestion/src/metadata/pii/algorithms/utils.py @@ -34,7 +34,7 @@ def get_top_classes(scores: Mapping[T, float], n: int, threshold: float) -> Sequ Get the top n scores from the scores mapping that are above the threshold. The classes are sorted in descending order of their scores. """ - sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True) + sorted_scores = sorted(scores.items(), key=lambda item: (-item[1], str(item[0]))) top_classes = [key for key, score in sorted_scores if score >= threshold] return top_classes[:n] diff --git a/ingestion/src/metadata/pii/scanners/ner_scanner.py b/ingestion/src/metadata/pii/scanners/ner_scanner.py index 34527d963f05..d7b2c3188ae6 100644 --- a/ingestion/src/metadata/pii/scanners/ner_scanner.py +++ b/ingestion/src/metadata/pii/scanners/ner_scanner.py @@ -20,12 +20,12 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple, Union # noqa: UP035 -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel from metadata.generated.schema.entity.classification.tag import Tag from metadata.pii.algorithms.preprocessing import MAX_NLP_TEXT_LENGTH -from metadata.pii.algorithms.presidio_utils import _load_spacy_model -from metadata.pii.constants import PII, SPACY_EN_MODEL +from metadata.pii.algorithms.presidio_utils import build_analyzer_engine +from metadata.pii.constants import PII from metadata.pii.models import TagAndConfidence from metadata.pii.ner import NEREntity from metadata.pii.scanners.base import BaseScanner @@ -46,32 +46,17 @@ class StringAnalysis(BaseModel): appearances: int -class NLPEngineModel(BaseModel): - """Required to pass the nlp_engine as {"lang_code": "en", "model_name": "en_core_web_lg"}""" - - model_config = ConfigDict(protected_namespaces=()) - lang_code: str - model_name: str - - # pylint: disable=import-outside-toplevel class NERScanner(BaseScanner): """Based on https://microsoft.github.io/presidio/""" def __init__(self): - from presidio_analyzer import AnalyzerEngine - from presidio_analyzer.nlp_engine.spacy_nlp_engine import SpacyNlpEngine - - _load_spacy_model(SPACY_EN_MODEL) - - nlp_engine_model = NLPEngineModel(lang_code=SUPPORTED_LANG, model_name=SPACY_EN_MODEL) - # Set the presidio logger to talk less about internal entities unless we are debugging logging.getLogger(PRESIDIO_LOGGER).setLevel( logging.INFO if logging.getLogger(METADATA_LOGGER).level == logging.DEBUG else logging.ERROR ) - self.analyzer = AnalyzerEngine(nlp_engine=SpacyNlpEngine(models=[nlp_engine_model.model_dump()])) + self.analyzer = build_analyzer_engine() @staticmethod def get_highest_score_label(entities_score: Dict[str, StringAnalysis]) -> Tuple[str, float]: # noqa: UP006 diff --git a/ingestion/src/metadata/pii/tag_analyzer.py b/ingestion/src/metadata/pii/tag_analyzer.py index 64ec87d45790..52944ecff794 100644 --- a/ingestion/src/metadata/pii/tag_analyzer.py +++ b/ingestion/src/metadata/pii/tag_analyzer.py @@ -19,6 +19,10 @@ ) from metadata.generated.schema.type.recognizer import RecognizerException from metadata.pii.algorithms.feature_extraction import split_column_name +from metadata.pii.algorithms.presidio_patches import ( + PresidioRecognizerResultPatcher, + date_time_patcher, +) from metadata.pii.algorithms.presidio_recognizer_factory import ( PresidioRecognizerFactory, ) @@ -161,6 +165,7 @@ def _analyze_with( text_or_values: str | Sequence[str], recognizers: list[EntityRecognizer], context: Optional[list[str]] = None, # noqa: UP045 + result_patcher: Optional[PresidioRecognizerResultPatcher] = None, # noqa: UP045 ) -> list[RecognizerResult]: values = [text_or_values] if isinstance(text_or_values, str) else list(text_or_values) results: list[RecognizerResult] = [] @@ -168,14 +173,13 @@ def _analyze_with( if self._language is not ClassificationLanguage.any: analyzer = self.build_analyzer_with(recognizers) for value in values: - results.extend( - analyzer.analyze( - value, - language=self._language.value, - context=context, - return_decision_process=True, - ) + value_results = analyzer.analyze( + value, + language=self._language.value, + context=context, + return_decision_process=True, ) + results.extend(result_patcher(value_results, value) if result_patcher else value_results) return results sorted_recs = sorted(recognizers, key=lambda r: r.supported_language) @@ -194,14 +198,13 @@ def _analyze_with( effective_language=effective_lang, ) for value in values: - results.extend( - analyzer.analyze( - value, - language=effective_lang, - context=context, - return_decision_process=True, - ) + value_results = analyzer.analyze( + value, + language=effective_lang, + context=context, + return_decision_process=True, ) + results.extend(result_patcher(value_results, value) if result_patcher else value_results) return results def analyze( @@ -215,7 +218,12 @@ def analyze( content_recognizers = self.content_recognizers if content_recognizers: context = split_column_name(self._column_name) - content_results = self._analyze_with(str_values, content_recognizers, context=context) + content_results = self._analyze_with( + str_values, + content_recognizers, + context=context, + result_patcher=date_time_patcher, + ) content_score = min(sum(r.score for r in content_results) / len(str_values), 1.0) column_results: list[RecognizerResult] = [] diff --git a/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py b/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py index 622b45d204d6..ab9babcf2f52 100644 --- a/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py +++ b/ingestion/src/metadata/profiler/metrics/hybrid/histogram.py @@ -252,11 +252,11 @@ def df_fn( for df in dfs: if not frequencies.any(): frequencies = ( - pd.cut(df[self.col.name], bins, right=False).value_counts().values + pd.cut(df[self.col.name], bins, right=False).value_counts().to_numpy() ) # right boundary is exclusive continue frequencies += ( - pd.cut(df[self.col.name], bins, right=False).value_counts().values + pd.cut(df[self.col.name], bins, right=False).value_counts().to_numpy() ) # right boundary is exclusive if frequencies.size > 0: # pyright: ignore[reportAttributeAccessIssue] diff --git a/ingestion/tests/integration/airflow/Dockerfile b/ingestion/tests/integration/airflow/Dockerfile index f14867ee5a59..6f560750f6ec 100644 --- a/ingestion/tests/integration/airflow/Dockerfile +++ b/ingestion/tests/integration/airflow/Dockerfile @@ -12,16 +12,16 @@ # Airflow carrying the working tree's OpenMetadata lineage provider. Build context is # ingestion/, so src/metadata/generated must already exist (make generate). -FROM apache/airflow:3.3.0-python3.10 +FROM apache/airflow:3.3.1-python3.10 USER airflow COPY --chown=airflow:0 . /tmp/ingestion -# airflow-constraints-3.3.0.txt is deliberately NOT applied: it pins chardet==6.0.0.post1 +# airflow-constraints-3.3.1.txt is deliberately NOT applied: it pins chardet==7.5.1 # against openmetadata-ingestion's chardet==4.0.0. Dockerfile.ci installs the package # unconstrained for the same reason. apache-airflow is pinned so the resolver cannot move it. RUN pip install --no-cache-dir uv \ - && uv pip install --no-cache "apache-airflow==3.3.0" /tmp/ingestion \ + && uv pip install --no-cache "apache-airflow==3.3.1" /tmp/ingestion \ && rm -rf /tmp/ingestion # Migrating at build time keeps container start to a few seconds. diff --git a/ingestion/tests/integration/airflow/conftest.py b/ingestion/tests/integration/airflow/conftest.py index ffbb3259fa64..387d3475564b 100644 --- a/ingestion/tests/integration/airflow/conftest.py +++ b/ingestion/tests/integration/airflow/conftest.py @@ -26,7 +26,7 @@ from testcontainers.core.container import DockerContainer from testcontainers.core.docker_client import DockerClient -AIRFLOW_BASE_IMAGE = "apache/airflow:3.3.0-python3.10" +AIRFLOW_BASE_IMAGE = "apache/airflow:3.3.1-python3.10" AIRFLOW_TEST_IMAGE = "om-airflow-lineage-test:local" # Fixed network name declared by docker/development/docker-compose.yml, so the container diff --git a/ingestion/tests/unit/metadata/pii/test_tag_scoring.py b/ingestion/tests/unit/metadata/pii/test_tag_scoring.py index 6f4a9ec9d7f1..ba71a5701e00 100644 --- a/ingestion/tests/unit/metadata/pii/test_tag_scoring.py +++ b/ingestion/tests/unit/metadata/pii/test_tag_scoring.py @@ -24,6 +24,7 @@ from _openmetadata_testutils.factories.metadata.generated.schema.type.recognizer import ( PatternFactory, PatternRecognizerFactory, + PredefinedRecognizerFactory, RecognizerFactory, ) from metadata.generated.schema.entity.classification.classification import ( @@ -35,7 +36,10 @@ from metadata.generated.schema.type.classificationLanguages import ( ClassificationLanguage, ) +from metadata.generated.schema.type.piiEntity import PIIEntity +from metadata.generated.schema.type.predefinedRecognizer import Name from metadata.generated.schema.type.recognizer import RecognizerException, Target +from metadata.pii.algorithms.presidio_utils import load_nlp_engine from metadata.pii.algorithms.tag_scoring import TagScorer from metadata.pii.models import ScoredTag from metadata.pii.tag_analyzer import TagAnalysis, TagAnalyzer @@ -318,6 +322,40 @@ def tag_analyzer(self, email_tag, column, nlp_engine): """Create a TagAnalyzer instance""" return TagAnalyzer(tag=email_tag, column=column, nlp_engine=nlp_engine) + @pytest.fixture + def date_tag_analyzer(self, column: Column) -> TagAnalyzer: + spacy_recognizer = RecognizerFactory.create( + name="SpacyRecognizer", + recognizerConfig=PredefinedRecognizerFactory.create( + name=Name.SpacyRecognizer, + supportedEntities=[PIIEntity.DATE_TIME], + ), + target=Target.content, + ) + date_tag = TagFactory.create( + tag_name="Date", + autoClassificationEnabled=True, + recognizers=[spacy_recognizer], + description="Date field", + ) + return TagAnalyzer( + tag=date_tag, + column=column, + nlp_engine=load_nlp_engine(), + ) + + def test_analyze_content_rejects_epoch_timestamp_as_date(self, date_tag_analyzer: TagAnalyzer): + analysis = date_tag_analyzer.analyze(str_values=["1760000000123"]) + + assert analysis.score == 0.0 + assert analysis.recognizer_results == [] + + def test_analyze_content_preserves_textual_date(self, date_tag_analyzer: TagAnalyzer): + analysis = date_tag_analyzer.analyze(str_values=["2025-01-15"]) + + assert analysis.score > 0.0 + assert [result.entity_type for result in analysis.recognizer_results] == [PIIEntity.DATE_TIME.value] + def test_analyze_content_with_emails(self, tag_analyzer, email_tag: Tag): """Test content analysis with email data""" values = ["john@example.com", "jane@test.org", "bob@company.co.uk"] diff --git a/ingestion/tests/unit/models/test_custom_basemodel_validation.py b/ingestion/tests/unit/models/test_custom_basemodel_validation.py index 485908f6d397..8649e83c4cf5 100644 --- a/ingestion/tests/unit/models/test_custom_basemodel_validation.py +++ b/ingestion/tests/unit/models/test_custom_basemodel_validation.py @@ -58,6 +58,17 @@ from metadata.utils.entity_link import CustomColumnName +def test_generated_model_transforms_name(): + """Validate automatic name transformation on generated models.""" + request = CreateTableRequest( + name=EntityName('my::table>with"special_chars'), + columns=[Column(name=ColumnName("column"), dataType=DataType.STRING)], + databaseSchema=FullyQualifiedEntityName("database.schema"), + ) + + assert request.name.root == "my__reserved__colon__table__reserved__arrow__with__reserved__quote__special_chars" + + class TestCustomBasemodelValidation(TestCase): """Comprehensive test suite for custom basemodel validation functionality.""" diff --git a/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py b/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py index aed62ced83de..cc7931a35526 100644 --- a/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py +++ b/ingestion/tests/unit/pii/algorithms/test_feature_extraction.py @@ -232,7 +232,12 @@ def test_aadhaar_extraction(analyzer): "0249-3285-1294", ] context = ["aadhaar", "govt id", "uidai"] - extracted = extract_pii_tags(analyzer, samples, context=context) + extracted = extract_pii_tags( + analyzer, + samples, + context=context, + recognizer_result_patcher=date_time_patcher, + ) assert get_top_pii_tag(extracted) == PIITag.IN_AADHAAR, ( PIITag.IN_AADHAAR, samples, diff --git a/ingestion/tests/unit/pii/algorithms/test_utils.py b/ingestion/tests/unit/pii/algorithms/test_utils.py new file mode 100644 index 000000000000..214da0e0c0aa --- /dev/null +++ b/ingestion/tests/unit/pii/algorithms/test_utils.py @@ -0,0 +1,19 @@ +# Copyright 2025 Collate +# Licensed under the Collate Community License, Version 1.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from metadata.pii.algorithms.utils import get_top_classes + + +def test_get_top_classes_breaks_score_ties_deterministically(): + scores = {"NRP": 0.425, "LOCATION": 0.425} + + assert get_top_classes(scores, n=1, threshold=0.0) == ["LOCATION"] + assert get_top_classes(dict(reversed(scores.items())), n=1, threshold=0.0) == ["LOCATION"] diff --git a/ingestion/tests/unit/pii/test_cases/customers_sensitive.py b/ingestion/tests/unit/pii/test_cases/customers_sensitive.py index e4911c45b457..b55a53948dc2 100644 --- a/ingestion/tests/unit/pii/test_cases/customers_sensitive.py +++ b/ingestion/tests/unit/pii/test_cases/customers_sensitive.py @@ -128,4 +128,14 @@ state=State.Suggested, ), ), + ColumnTag( + column_fqn="Service.database.schema.example_table.address", + tag_label=TagLabel( + name="NonSensitive", + tagFQN=TagFQN(root="PII.NonSensitive"), + source=TagSource.Classification, + labelType=LabelType.Generated, + state=State.Suggested, + ), + ), ] diff --git a/ingestion/tests/unit/pii/test_ner_scanner.py b/ingestion/tests/unit/pii/test_ner_scanner.py index b8d5f318e625..b90c60668c3e 100644 --- a/ingestion/tests/unit/pii/test_ner_scanner.py +++ b/ingestion/tests/unit/pii/test_ner_scanner.py @@ -157,5 +157,5 @@ def test_scan_entities(scanner): ] assert scanner.scan(ssn_numbers).tag_fqn == "PII.Sensitive" - nif_numbers = ["12345678A", "87654321B", "23456789C", "98765432D", "34567890E"] + nif_numbers = ["12345678Z", "87654321X", "23456789D", "98765432M", "34567890V"] assert scanner.scan(nif_numbers).tag_fqn == "PII.Sensitive" diff --git a/ingestion/tests/unit/test_mf4_reader.py b/ingestion/tests/unit/test_mf4_reader.py index ee390b3eaf86..97ee50843064 100644 --- a/ingestion/tests/unit/test_mf4_reader.py +++ b/ingestion/tests/unit/test_mf4_reader.py @@ -13,6 +13,8 @@ MF4 reader tests """ +from pathlib import Path +from tempfile import TemporaryDirectory from unittest import TestCase from unittest.mock import MagicMock, patch @@ -20,6 +22,24 @@ from metadata.readers.dataframe.models import DatalakeColumnWrapper +def test_local_mf4_reading_with_installed_asammdf(): + from asammdf import MDF + + from metadata.generated.schema.entity.services.connections.database.datalakeConnection import ( + LocalConfig, + ) + + with TemporaryDirectory() as tmp_dir: + file_path = Path(tmp_dir) / "empty.mf4" + with MDF(version="4.10") as mdf: + mdf.save(file_path) + + reader = MF4DataFrameReader(LocalConfig(), None) + result = reader._read(key=str(file_path), bucket_name="") + + assert list(result.dataframes()) == [] + + class TestMF4DataFrameReader(TestCase): """ Test MF4DataFrameReader functionality diff --git a/ingestion/tests/unit/test_pydantic_v2.py b/ingestion/tests/unit/test_pydantic_v2.py index 305e1a701371..f1881d43631d 100644 --- a/ingestion/tests/unit/test_pydantic_v2.py +++ b/ingestion/tests/unit/test_pydantic_v2.py @@ -10,6 +10,8 @@ # limitations under the License. """Test pydantic v2 models serialize data as pydantic v1""" +import subprocess +import sys from datetime import datetime from pydantic import AnyUrl @@ -19,6 +21,27 @@ from metadata.ingestion.models.custom_pydantic import BaseModel +def test_custom_base_model_imports_without_pydantic_212_deprecations(): + """Keep import-time validators on supported Pydantic APIs.""" + script = """ +import warnings + +from pydantic.warnings import PydanticDeprecatedSince212 + +warnings.simplefilter("error", PydanticDeprecatedSince212) +import metadata.ingestion.models.custom_pydantic +""" + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0, result.stderr + + def test_simple_dump(): """ Compare V1 and custom V2 serialization, @@ -76,6 +99,24 @@ class ModelV2(BaseModel): assert json_v1 == json_v2 +def test_serialize_as_any_includes_subclass_fields(): + """Preserve polymorphic serialization exposed by the custom base model.""" + + class Parent(BaseModel): + name: str + + class Child(Parent): + token: str + + class Container(BaseModel): + user: Parent + + model = Container(user=Child(name="name", token="token")) + + assert model.model_dump() == {"user": {"name": "name"}} + assert model.model_dump(serialize_as_any=True) == {"user": {"name": "name", "token": "token"}} + + def test_tz_aware_date(): """Validate how we can create "aware" datetime objects""" diff --git a/ingestion/tests/unit/utils/test_deprecation.py b/ingestion/tests/unit/utils/test_deprecation.py index 540ec4d2c18b..0e46524f45c9 100644 --- a/ingestion/tests/unit/utils/test_deprecation.py +++ b/ingestion/tests/unit/utils/test_deprecation.py @@ -29,6 +29,9 @@ def deprecated_call(self) -> None: def test_deprecation_warning(self) -> None: """Test that deprecation warnings are controlled by logger level.""" + metadata_logger = logging.getLogger("metadata") + self.addCleanup(metadata_logger.setLevel, metadata_logger.level) + logger_levels = [ logging.DEBUG, logging.INFO, @@ -44,7 +47,6 @@ def test_deprecation_warning(self) -> None: # Capture logging output log_capture = StringIO() handler = logging.StreamHandler(log_capture) - metadata_logger = logging.getLogger("metadata") metadata_logger.addHandler(handler) # Create and call a deprecated function