diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index e357849bba58..bafcefec5751 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -1,6 +1,6 @@ FROM mysql:8.3 AS mysql -FROM apache/airflow:3.3.0-python3.12 +FROM apache/airflow:3.3.1-python3.12 USER root RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list @@ -122,7 +122,7 @@ ARG RI_VERSION="2.0.0.0.dev0" RUN pip install --upgrade "pip>=26.2,<27" "setuptools<81" # Pre-install cx-Oracle without build isolation to use the pinned setuptools RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" -RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.3.0/constraints-3.12.txt" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.3.1/constraints-3.12.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index 83895664e419..d72cce355f91 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -1,6 +1,6 @@ FROM mysql:8.3 AS mysql -FROM apache/airflow:3.3.0-python3.12 +FROM apache/airflow:3.3.1-python3.12 USER root RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ && echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list @@ -94,7 +94,7 @@ COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airf # Required for Airflow DAGs of Sample Data COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags COPY --chown=airflow:0 ingestion/examples/airflow/test_dags /opt/airflow/dags -COPY --chown=airflow:0 ingestion/airflow-constraints-3.3.0.txt /home/airflow/airflow-constraints-3.3.0.txt +COPY --chown=airflow:0 ingestion/airflow-constraints-3.3.1.txt /home/airflow/airflow-constraints-3.3.1.txt # Refresh the interpreter-level pip before dropping privileges. The per-user # pip upgrade below only reaches ~/.local, leaving the base image's @@ -122,11 +122,11 @@ RUN pip install --upgrade "pip>=26.2,<27" "setuptools<81" RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" # Install FAB provider for Airflow 3.x Flask Blueprint compatibility -RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.3.0.txt" || true +RUN pip install "apache-airflow-providers-fab>=1.0.0" --constraint "/home/airflow/airflow-constraints-3.3.1.txt" || true WORKDIR /home/airflow/openmetadata-airflow-apis -RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.3.0.txt" +RUN pip install "." --constraint "/home/airflow/airflow-constraints-3.3.1.txt" WORKDIR /home/airflow/ingestion @@ -169,7 +169,7 @@ RUN pip install psycopg2 mysqlclient==2.1.1 RUN mkdir -p /opt/airflow/dag_generated_configs EXPOSE 8080 -# Airflow 3.3.0 requires universal-pathlib>=0.3.8, but prior installs in this image +# Airflow 3.3.1 requires universal-pathlib>=0.3.8, but prior installs in this image # can leave stale 0.2.6 `upath` module files in site-packages that cause import # errors at runtime. Force-remove the stale registration then pin to the required version. RUN pip uninstall upath -y && pip install "universal-pathlib==0.3.10" diff --git a/ingestion/airflow-constraints-3.3.0.txt b/ingestion/airflow-constraints-3.3.1.txt similarity index 61% rename from ingestion/airflow-constraints-3.3.0.txt rename to ingestion/airflow-constraints-3.3.1.txt index dfffbbe5452f..3921501f8693 100644 --- a/ingestion/airflow-constraints-3.3.0.txt +++ b/ingestion/airflow-constraints-3.3.1.txt @@ -1,6 +1,6 @@ # -# This constraints file was automatically generated on 2026-07-06T12:27:13.807760 +# This constraints file was automatically generated on 2026-08-12T08:36:03.248024 # via `uv pip install --resolution highest` for the "v3-3-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. @@ -35,19 +35,19 @@ Authlib==1.7.2 Deprecated==1.3.1 Events==0.5 Flask-JWT-Extended==4.7.4 -Flask-Limiter==3.12 +Flask-Limiter==4.1.1 Flask-Login==0.6.3 Flask-SQLAlchemy==3.1.1 Flask-Session==0.8.0 Flask-WTF==1.3.0 Flask==3.1.3 -GitPython==3.1.50 +GitPython==3.1.58 JayDeBeApi==1.2.3 Jinja2==3.1.6 -Mako==1.3.12 -Markdown==3.10.2 +Mako==1.4.1 +Markdown==3.10.3 MarkupSafe==3.0.3 -PyAthena==3.32.0 +PyAthena==3.35.4 PyGithub==2.9.1 PyHive==0.7.0 PyJWT==2.13.0 @@ -62,170 +62,174 @@ WTForms==3.2.2 Werkzeug==3.1.8 a2wsgi==1.10.10 adal==1.2.7 +adbc-driver-manager==1.12.0 +adbc-driver-postgresql==1.12.0 +adbc-driver-sqlite==1.12.0 adlfs==2026.5.0 aenum==3.1.17 -aiobotocore==3.7.0 +aiobotocore==3.9.0 aiofiles==24.1.0 aiohappyeyeballs==2.7.1 aiohttp-cors==0.8.1 -aiohttp==3.14.1 +aiohttp==3.14.3 aioitertools==0.13.0 aiomysql==0.3.2 aiosignal==1.4.0 aiosmtplib==5.1.2 -aiosqlite==0.21.0 +aiosqlite==0.22.1 airbyte-api==1.0.1 akeyless==5.0.28 -alembic==1.18.5 +alembic==1.19.0 alibabacloud-adb20211201==3.7.0 alibabacloud-credentials-api==1.0.1 -alibabacloud-credentials==1.0.9 +alibabacloud-credentials==1.0.11 alibabacloud-gateway-spi==0.0.4 alibabacloud-openapi-util==0.2.4 -alibabacloud-oss-v2==1.3.1 -alibabacloud-tea-util==0.3.14 +alibabacloud-oss-v2==1.3.2 +alibabacloud-tea-util==0.3.15 alibabacloud-tea==0.4.3 alibabacloud_endpoint_util==0.0.4 alibabacloud_tea_openapi==0.3.16 alibabacloud_tea_xml==0.0.3 amqp==5.3.1 -annotated-doc==0.0.4 -annotated-types==0.7.0 -anyio==4.14.1 -apache-airflow-providers-airbyte==5.5.1 -apache-airflow-providers-akeyless==0.2.0 -apache-airflow-providers-alibaba==3.3.9 -apache-airflow-providers-amazon==9.31.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.14.2 +apache-airflow-providers-airbyte==6.0.1 +apache-airflow-providers-akeyless==0.3.0 +apache-airflow-providers-alibaba==3.4.0 +apache-airflow-providers-amazon==9.34.0 apache-airflow-providers-apache-cassandra==3.9.5 apache-airflow-providers-apache-drill==3.3.3 apache-airflow-providers-apache-druid==4.5.2 apache-airflow-providers-apache-flink==1.8.5 -apache-airflow-providers-apache-hdfs==4.12.1 -apache-airflow-providers-apache-hive==9.5.0 -apache-airflow-providers-apache-iceberg==1.4.1 +apache-airflow-providers-apache-hdfs==4.12.2 +apache-airflow-providers-apache-hive==9.6.1 +apache-airflow-providers-apache-iceberg==2.0.3 apache-airflow-providers-apache-impala==1.9.3 -apache-airflow-providers-apache-kafka==1.14.0 +apache-airflow-providers-apache-kafka==1.15.1 apache-airflow-providers-apache-kylin==3.10.5 -apache-airflow-providers-apache-livy==4.5.7 +apache-airflow-providers-apache-livy==4.6.0 apache-airflow-providers-apache-pig==4.8.5 apache-airflow-providers-apache-pinot==4.10.3 -apache-airflow-providers-apache-spark==6.2.0 +apache-airflow-providers-apache-spark==6.3.1 apache-airflow-providers-apache-tinkerpop==1.1.4 apache-airflow-providers-apprise==2.3.4 -apache-airflow-providers-arangodb==2.9.5 +apache-airflow-providers-arangodb==2.9.6 apache-airflow-providers-asana==2.11.4 -apache-airflow-providers-atlassian-jira==3.3.4 -apache-airflow-providers-celery==3.21.0 +apache-airflow-providers-atlassian-jira==3.3.5 +apache-airflow-providers-celery==3.23.1 apache-airflow-providers-clickhousedb==1.0.0 apache-airflow-providers-cloudant==4.3.5 -apache-airflow-providers-cncf-kubernetes==10.19.0 -apache-airflow-providers-cohere==1.6.6 -apache-airflow-providers-common-ai==0.5.0 -apache-airflow-providers-common-compat==1.15.0 +apache-airflow-providers-cncf-kubernetes==10.21.0 +apache-airflow-providers-cohere==1.6.7 +apache-airflow-providers-common-ai==0.7.0 +apache-airflow-providers-common-compat==1.18.0 apache-airflow-providers-common-io==1.8.0 apache-airflow-providers-common-messaging==2.0.4 -apache-airflow-providers-common-sql==2.0.1 -apache-airflow-providers-databricks==7.16.1 +apache-airflow-providers-common-sql==2.1.0 +apache-airflow-providers-databricks==7.18.1 apache-airflow-providers-datadog==3.10.5 -apache-airflow-providers-dbt-cloud==4.9.2 +apache-airflow-providers-dbt-cloud==4.9.3 apache-airflow-providers-dingding==3.9.5 apache-airflow-providers-discord==3.12.3 -apache-airflow-providers-docker==4.5.7 -apache-airflow-providers-edge3==4.0.0 -apache-airflow-providers-elasticsearch==6.7.0 -apache-airflow-providers-exasol==4.10.3 -apache-airflow-providers-fab==3.7.1 +apache-airflow-providers-docker==4.5.9 +apache-airflow-providers-edge3==4.3.0 +apache-airflow-providers-elasticsearch==6.9.0 +apache-airflow-providers-exasol==4.10.5 +apache-airflow-providers-fab==3.8.0 apache-airflow-providers-facebook==3.9.5 -apache-airflow-providers-ftp==3.15.1 -apache-airflow-providers-git==0.4.0 +apache-airflow-providers-ftp==3.15.2 +apache-airflow-providers-git==0.4.2 apache-airflow-providers-github==2.11.3 -apache-airflow-providers-google==22.1.0 +apache-airflow-providers-google==22.3.0 apache-airflow-providers-grpc==3.9.5 -apache-airflow-providers-hashicorp==4.7.1 -apache-airflow-providers-http==6.0.4 -apache-airflow-providers-imap==3.11.3 +apache-airflow-providers-hashicorp==4.8.0 +apache-airflow-providers-http==6.0.5 +apache-airflow-providers-imap==3.12.1 apache-airflow-providers-influxdb==2.11.0 apache-airflow-providers-informatica==0.2.0 apache-airflow-providers-jdbc==5.5.0 apache-airflow-providers-jenkins==4.2.6 -apache-airflow-providers-keycloak==0.8.1 -apache-airflow-providers-microsoft-azure==13.5.0 +apache-airflow-providers-keycloak==0.8.2 +apache-airflow-providers-microsoft-azure==14.1.0 apache-airflow-providers-microsoft-mssql==4.7.0 apache-airflow-providers-microsoft-psrp==3.2.6 -apache-airflow-providers-microsoft-winrm==3.14.3 +apache-airflow-providers-microsoft-winrm==3.14.4 apache-airflow-providers-mongo==5.4.0 apache-airflow-providers-mysql==6.6.1 -apache-airflow-providers-neo4j==3.11.6 +apache-airflow-providers-neo4j==3.12.1 apache-airflow-providers-odbc==4.12.3 -apache-airflow-providers-openai==1.7.5 +apache-airflow-providers-openai==1.8.2 apache-airflow-providers-openfaas==3.9.5 -apache-airflow-providers-openlineage==2.18.1 -apache-airflow-providers-opensearch==1.10.0 +apache-airflow-providers-openlineage==2.20.0 +apache-airflow-providers-opensearch==1.12.0 apache-airflow-providers-opsgenie==5.10.4 -apache-airflow-providers-oracle==4.6.1 +apache-airflow-providers-oracle==4.6.2 apache-airflow-providers-pagerduty==5.2.6 -apache-airflow-providers-papermill==3.13.1 -apache-airflow-providers-pgvector==1.7.2 +apache-airflow-providers-papermill==3.13.2 +apache-airflow-providers-pgvector==1.7.3 apache-airflow-providers-pinecone==2.4.5 -apache-airflow-providers-postgres==6.8.0 -apache-airflow-providers-presto==5.12.0 +apache-airflow-providers-postgres==7.0.1 +apache-airflow-providers-presto==5.12.1 apache-airflow-providers-qdrant==1.5.6 apache-airflow-providers-redis==4.5.0 -apache-airflow-providers-salesforce==5.14.1 +apache-airflow-providers-salesforce==5.14.2 apache-airflow-providers-samba==4.12.6 apache-airflow-providers-segment==3.9.5 apache-airflow-providers-sendgrid==4.2.4 -apache-airflow-providers-sftp==5.8.2 +apache-airflow-providers-sftp==6.0.1 apache-airflow-providers-singularity==3.9.4 apache-airflow-providers-slack==9.10.2 -apache-airflow-providers-smtp==3.0.1 -apache-airflow-providers-snowflake==6.4.0 +apache-airflow-providers-smtp==3.0.3 +apache-airflow-providers-snowflake==6.16.0 apache-airflow-providers-sqlite==4.3.3 -apache-airflow-providers-ssh==5.0.3 -apache-airflow-providers-standard==1.15.0 -apache-airflow-providers-tableau==5.5.0 -apache-airflow-providers-telegram==4.9.5 -apache-airflow-providers-teradata==3.6.0 -apache-airflow-providers-trino==6.6.0 +apache-airflow-providers-ssh==6.0.1 +apache-airflow-providers-standard==1.17.0 +apache-airflow-providers-tableau==5.6.0 +apache-airflow-providers-telegram==4.9.6 +apache-airflow-providers-teradata==3.6.2 +apache-airflow-providers-trino==6.6.1 apache-airflow-providers-vertica==4.4.0 apache-airflow-providers-vespa==0.1.1 -apache-airflow-providers-weaviate==3.3.5 -apache-airflow-providers-yandex==4.5.0 +apache-airflow-providers-weaviate==3.4.1 +apache-airflow-providers-yandex==4.5.1 apache-airflow-providers-ydb==2.5.3 apache-airflow-providers-zendesk==4.12.0 apispec==6.10.0 -apprise==1.11.0 -argcomplete==3.7.0 +apprise==1.12.0 +argcomplete==3.7.2 arrow==1.4.0 asana==5.2.5 -asgiref==3.11.1 +asgiref==3.12.1 asn1crypto==1.5.1 -asttokens==3.0.1 +asttokens==3.0.2 async-timeout==4.0.3 asyncpg==0.31.0 asyncssh==2.24.0 atlasclient==1.0.0 atlassian-python-api==4.0.7 attrs==26.1.0 +azure-ai-projects==2.4.0 azure-batch==14.2.0 azure-common==1.1.28 azure-core==1.41.0 -azure-cosmos==4.16.1 +azure-cosmos==4.16.3 azure-datalake-store==0.0.53 azure-identity==1.25.3 azure-keyvault-secrets==4.11.0 azure-kusto-data==6.0.4 -azure-mgmt-compute==38.1.0 +azure-mgmt-compute==38.2.0 azure-mgmt-containerinstance==10.1.0 azure-mgmt-containerregistry==15.0.0 azure-mgmt-core==1.6.0 -azure-mgmt-cosmosdb==9.9.0 -azure-mgmt-datafactory==9.3.0 +azure-mgmt-cosmosdb==10.0.0 +azure-mgmt-datafactory==10.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==26.0.0 -azure-mgmt-storage==25.0.0 +azure-mgmt-storage==25.1.0 azure-nspkg==3.0.2 azure-servicebus==7.14.3 azure-storage-blob==12.30.0 @@ -235,47 +239,48 @@ azure-synapse-artifacts==0.22.0 azure-synapse-spark==0.7.0 babel==2.18.0 backoff==2.2.1 +backports.zstd==1.6.0 bcrypt==5.0.0 beautifulsoup4==4.15.0 billiard==4.2.4 -bitarray==3.8.2 +bitarray==3.10.1 black==26.5.1 bleach==6.4.0 blinker==1.9.0 -boto3==1.43.0 -botocore==1.43.0 -cachelib==0.14.0 -cachetools==7.1.4 +boto3==1.43.56 +botocore==1.43.56 +cachelib==0.15.2 +cachetools==6.2.6 cadwyn==7.0.0 capi_param_builder_python==1.3.0 -cassandra-driver==3.30.0 +cassandra-driver==3.30.1 cattrs==26.1.0 celery==5.6.3 -certifi==2026.6.17 -cffi==2.0.0 -chardet==6.0.0.post1 -charset-normalizer==3.4.7 +certifi==2026.7.22 +cffi==2.1.1 +chardet==7.5.1 +charset-normalizer==3.4.9 ciso8601==2.3.3 click-didyoumean==0.3.1 click-plugins==1.1.1.2 click-repl==0.3.0 click==8.4.2 -clickhouse-connect==1.4.1 -cloudpickle==3.1.2 -cohere==7.0.5 +clickhouse-connect==1.6.0 +cloudpickle==3.1.1 +cohere==7.0.8 colorama==0.4.6 colorful==0.5.8 -colorlog==6.10.1 +colorlog==6.12.0 comm==0.2.3 confluent-kafka==2.15.0 crcmod-plus==2.3.1 cron_descriptor==2.1.0 -croniter==6.2.2 -cryptography==48.0.1 +croniter==6.2.4 +cryptography==50.0.0 curlify==3.0.0 -databricks-sql-connector==4.2.5 -datadog==0.52.2 -db-dtypes==1.7.0 +databricks-sql-connector==4.4.0 +datadog==0.53.0 +db-dtypes==1.7.1 debugpy==1.8.21 decorator==5.3.1 defusedxml==0.7.1 @@ -284,153 +289,156 @@ dill==0.4.1 distlib==0.4.3 distro==1.9.0 dnspython==2.8.0 -docker==7.1.0 +docker==7.2.0 docopt==0.6.2 docstring_parser==0.18.0 durationpy==0.10 elastic-transport==9.4.2 -elasticsearch==9.4.1 +elasticsearch==9.5.0 email-validator==2.3.0 entrypoints==0.4 +et_xmlfile==2.0.0 eventlet==0.41.0 executing==2.2.1 -facebook_business==25.0.2 -fastapi-cli==0.0.27 +facebook_business==26.0.0 +fastapi-cli==0.0.32 fastapi==0.136.3 fastavro==1.12.2 -fastcore==1.13.9 -fastjsonschema==2.21.2 +fastcore==2.2.5 +fastjsonschema==2.22.1 fastuuid==0.14.0 -filelock==3.29.4 -flask-appbuilder==5.2.1 +filelock==3.32.2 +flask-appbuilder==5.2.2 flask-babel==4.0.0 flower==2.0.1 frozenlist==1.8.0 -fsspec==2026.6.0 +fsspec==2026.7.0 future==1.0.0 -gcloud-aio-auth==5.4.4 +gcloud-aio-auth==5.5.0 gcloud-aio-bigquery==7.1.0 gcloud-aio-storage==9.6.4 -gcsfs==2026.6.0 -genai-prices==0.0.69 +gcsfs==2026.7.0 +genai-prices==0.1.1 geomet==1.1.0 -gevent==26.5.0 +gevent==26.7.0 gitdb==4.0.12 -google-ads==31.1.0 +google-ads==31.2.0 google-analytics-admin==0.30.1 -google-api-core==2.31.0 +google-api-core==2.34.0 google-api-python-client==2.198.0 -google-auth-httplib2==0.4.0 +google-auth-httplib2==0.4.1 google-auth-oauthlib==1.4.0 -google-auth==2.55.1 -google-cloud-aiplatform==1.148.1 +google-auth==2.56.3 +google-cloud-aiplatform==1.163.0 google-cloud-alloydb==0.11.0 google-cloud-appengine-logging==1.10.0 -google-cloud-audit-log==0.6.0 +google-cloud-audit-log==0.6.1 google-cloud-automl==2.20.0 -google-cloud-batch==0.22.0 +google-cloud-batch==0.22.2 google-cloud-bigquery-datatransfer==3.23.0 -google-cloud-bigquery-storage==2.39.0 -google-cloud-bigquery==3.42.1 -google-cloud-bigtable==2.40.0 -google-cloud-build==3.38.0 -google-cloud-compute==1.49.0 +google-cloud-bigquery-storage==2.40.0 +google-cloud-bigquery==3.43.0 +google-cloud-bigtable==2.41.0 +google-cloud-build==3.38.1 +google-cloud-compute==1.50.0 google-cloud-container==2.65.0 -google-cloud-core==2.6.0 +google-cloud-core==2.6.1 google-cloud-datacatalog==3.31.0 google-cloud-dataflow-client==0.14.0 -google-cloud-dataform==0.11.1 +google-cloud-dataform==0.11.2 google-cloud-dataplex==2.20.0 google-cloud-dataproc-metastore==1.23.0 -google-cloud-dataproc==5.29.0 +google-cloud-dataproc==5.30.0 google-cloud-dlp==3.38.0 -google-cloud-kms==3.14.0 +google-cloud-kms==3.16.0 google-cloud-language==2.21.0 -google-cloud-logging==3.16.0 +google-cloud-logging==3.16.2 google-cloud-managedkafka==0.4.1 google-cloud-memcache==1.16.0 google-cloud-monitoring==2.31.0 google-cloud-orchestration-airflow==1.22.0 google-cloud-os-login==2.22.0 -google-cloud-pubsub==2.39.0 +google-cloud-pubsub==2.39.1 google-cloud-redis==2.22.0 google-cloud-resource-manager==1.18.0 google-cloud-run==0.16.1 -google-cloud-secret-manager==2.29.0 -google-cloud-spanner==3.69.0 +google-cloud-secret-manager==2.30.0 +google-cloud-spanner==3.69.1 google-cloud-speech==2.40.0 -google-cloud-storage-control==1.12.0 +google-cloud-storage-control==1.13.0 google-cloud-storage-transfer==1.21.0 -google-cloud-storage==3.12.0 -google-cloud-tasks==2.23.0 +google-cloud-storage==3.13.1 +google-cloud-tasks==2.24.0 google-cloud-texttospeech==2.37.0 google-cloud-translate==3.27.0 google-cloud-videointelligence==2.20.0 google-cloud-vision==3.15.0 google-cloud-workflows==1.23.0 google-crc32c==1.8.0 -google-genai==1.75.0 -google-resumable-media==2.10.0 -googleapis-common-protos==1.75.0 +google-genai==2.17.0 +google-resumable-media==2.10.1 +googleapis-common-protos==1.75.1 graphviz==0.21 greenback==1.3.0 -greenlet==3.5.3 +greenlet==3.5.4 gremlinpython==3.8.1 griffelib==2.1.0 -grpc-google-iam-v1==0.14.4 +grpc-google-iam-v1==0.14.5 grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 -grpcio-health-checking==1.81.1 -grpcio-status==1.81.1 -grpcio-tools==1.81.1 -grpcio==1.81.1 +grpcio-status==1.78.0 +grpcio-tools==1.78.0 +grpcio==1.78.0 gssapi==1.11.1 gunicorn==26.0.0 h11==0.16.0 -h2==4.3.0 +h2==4.4.1 hdfs==2.7.3 -hf-xet==1.5.1 +hf-xet==1.6.0 hmsclient==0.1.1 hpack==4.2.0 -httpcore2==2.5.0 +httpcore2==2.9.1 httpcore==1.0.9 httplib2==0.32.0 -httpr==0.4.8 +httpr==0.6.0 httptools==0.8.0 -httpx2==2.5.0 +httpx2==2.9.1 httpx==0.28.1 -huggingface_hub==1.21.0 +huggingface_hub==1.27.0 humanize==4.16.0 hvac==2.4.0 hyperframe==6.1.0 -ibm-cloud-sdk-core==3.24.4 -ibmcloudant==0.11.8 +ibm-cloud-sdk-core==3.25.0 +ibmcloudant==0.11.9 idna==3.18 ijson==3.4.0.post0 immutabledict==4.3.1 -importlib_metadata==9.0.0 -# Hand-patched (keep on regeneration): impyla <0.24.0 hard-pins thrift==0.16.0, which holds -# thrift below the 0.24.0 release that fixes CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. +importlib_metadata==8.9.0 +importlib_resources==7.1.0 +# Upstream constraints-3.3.1 now ships this natively, but keep the pin on regeneration: +# impyla <0.24.0 hard-pins thrift==0.16.0, which holds thrift below the 0.24.0 release +# that fixes CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. # Must stay in sync with the hive/impala extras in ingestion/setup.py. impyla==0.24.0 inflection==0.5.1 influxdb-client==1.50.0 influxdb3-python==0.20.0 +invoke==3.0.3 ipykernel==7.3.0 -ipython==9.15.0 +ipython==9.16.1 ipython_pygments_lexers==1.1.1 isodate==0.7.2 isoduration==20.11.0 itsdangerous==2.2.0 jaraco.classes==3.4.0 jaraco.context==6.1.2 -jaraco.functools==4.5.0 +jaraco.functools==4.6.0 jedi==0.20.0 jeepney==0.9.0 jiter==0.16.0 jmespath==1.1.0 joblib==1.5.3 -joserfc==1.7.2 +joserfc==1.7.4 jpype1==1.7.1 jsonpath-ng==1.8.0 jsonschema-specifications==2025.9.1 @@ -442,17 +450,17 @@ jwcrypto==1.5.8 keyring==25.7.0 kombu==5.6.2 krb5==0.9.0 -kubernetes==36.0.2 +kubernetes==36.0.3 kubernetes_asyncio==36.1.0 kylinpy==2.8.4 lazy-object-proxy==1.12.0 -libcst==1.8.6 +libcst==1.9.0 limits==5.8.0 linkify-it-py==2.1.0 -litellm==1.82.6 +litellm==1.85.7 lockfile==0.12.2 -logfire-api==4.37.0 -looker_sdk==26.10.0 +logfire-api==4.40.0 +looker_sdk==26.12.0 lxml==6.1.1 lz4==4.4.5 markdown-it-py==4.2.0 @@ -461,7 +469,7 @@ marshmallow==4.3.0 matplotlib-inline==0.2.2 mdit-py-plugins==0.6.1 mdurl==0.1.2 -memray==1.19.3 +memray==1.20.0 mergedeep==1.3.4 methodtools==0.4.7 microsoft-kiota-abstractions==1.11.7 @@ -469,13 +477,13 @@ microsoft-kiota-authentication-azure==1.11.7 microsoft-kiota-http==1.11.7 microsoft-kiota-serialization-json==1.11.7 microsoft-kiota-serialization-text==1.11.7 -mistune==3.3.2 +mistune==3.3.4 mmh3==5.2.1 more-itertools==11.1.0 msal-extensions==1.3.1 msal==1.37.0 msgpack==1.2.1 -msgraph-core==1.4.0 +msgraph-core==1.5.1 msgraphfs==0.5 msgspec==0.21.1 msrest==0.7.1 @@ -483,93 +491,97 @@ msrestazure==0.6.4.post1 multi_key_dict==2.0.3 multidict==6.7.1 mypy_extensions==1.1.0 -mysql-connector-python==9.6.0 +mysql-connector-python==26.7.0 mysqlclient==2.2.8 -narwhals==2.23.0 +narwhals==2.24.0 natsort==8.4.0 nbclient==0.11.0 nbconvert==7.17.1 -nbformat==5.10.4 +nbformat==5.11.0 neo4j==6.2.0 nest-asyncio2==1.7.2 nest-asyncio==1.6.0 -numpy==1.26.4 +numpy==2.5.1 oauthlib==3.3.1 -openai==2.44.0 +openai==2.53.0 opencensus-context==0.1.3 opencensus==0.11.4 -openlineage-integration-common==1.50.0 -openlineage-python==1.50.0 -openlineage_sql==1.50.0 +openlineage-integration-common==1.52.0 +openlineage-python==1.52.0 +openlineage_sql==1.52.0 +openpyxl==3.1.5 opensearch-protobufs==1.2.0 opensearch-py==3.2.0 -opentelemetry-api==1.43.0 -opentelemetry-exporter-otlp-proto-common==1.43.0 -opentelemetry-exporter-otlp-proto-grpc==1.43.0 -opentelemetry-exporter-otlp-proto-http==1.43.0 -opentelemetry-exporter-otlp==1.43.0 -opentelemetry-exporter-prometheus==0.64b0 -opentelemetry-proto==1.43.0 -opentelemetry-resourcedetector-gcp==1.12.0a0 -opentelemetry-sdk==1.43.0 -opentelemetry-semantic-conventions==0.64b0 +opentelemetry-api==1.44.0 +opentelemetry-exporter-otlp-proto-common==1.44.0 +opentelemetry-exporter-otlp-proto-grpc==1.44.0 +opentelemetry-exporter-otlp-proto-http==1.44.0 +opentelemetry-exporter-otlp==1.44.0 +opentelemetry-exporter-prometheus==0.65b0 +opentelemetry-proto==1.44.0 +opentelemetry-resourcedetector-gcp==1.14.0 +opentelemetry-sdk==1.44.0 +opentelemetry-semantic-conventions==0.65b0 opsgenie-sdk==2.1.5 -oracledb==4.0.1 +oracledb==4.0.2 ordered-set==4.1.0 orjson==3.11.9 outcome==1.3.0.post0 -packaging==26.2 -pagerduty==6.3.0 -pandas-gbq==0.35.0 -pandas-stubs==3.0.3.260530 -pandas==2.1.4 +packaging==26.3 +pagerduty==7.0.0 +pandas-gbq==0.35.1 +pandas==3.0.5 pandocfilters==1.5.1 papermill==2.7.0 -paramiko==3.5.1 +paramiko==5.0.0 parso==0.8.7 pathlib_abc==0.5.2 pathspec==1.1.1 pbr==7.0.3 pendulum==3.2.0 pexpect==4.9.0 -pgvector==0.4.2 +pgvector==0.5.0 pinecone==9.1.0 pinotdb==9.1.2 -platformdirs==4.10.0 +platformdirs==4.11.1 pluggy==1.6.0 -polars-runtime-32==1.42.1 -polars==1.42.1 +polars-runtime-32==1.43.2 +polars==1.43.2 portalocker==3.2.0 presto-python-client==0.8.4 prison==0.2.1 -prometheus_client==0.25.0 -prompt_toolkit==3.0.52 +prometheus_client==0.26.0 +prompt_toolkit==3.0.53 propcache==0.5.2 -proto-plus==1.28.0 +proto-plus==1.28.3 protobuf==6.33.6 psutil==7.2.2 +psycopg-binary==3.3.4 psycopg2-binary==2.9.12 +psycopg==3.3.4 ptyprocess==0.7.0 pure-sasl==0.6.2 pure_eval==0.2.3 py-spy==0.4.2 -pyOpenSSL==26.2.0 -pyarrow==24.0.0 -pyasn1==0.6.3 +pyOpenSSL==26.4.0 +pyarrow==25.0.0 +pyasn1==0.6.4 pyasn1_modules==0.4.2 +pybreaker==1.4.1 pycountry==26.2.16 pycparser==3.0 pycryptodome==3.23.0 -pydantic-ai-slim==2.3.0 +pydantic-ai-slim==2.27.0 pydantic-extra-types==2.11.1 -pydantic-graph==2.3.0 -pydantic-settings==2.14.2 +pydantic-graph==2.27.0 +pydantic-settings==2.15.0 pydantic==2.13.4 pydantic_core==2.46.4 pydata-google-auth==1.9.1 pydruid==0.6.9 -pyexasol==2.2.2 +pyexasol==1.3.0 pygtrie==2.5.0 +pyiceberg==0.11.1 pykerberos==1.2.4 pymongo==4.17.0 pymssql==2.3.13 @@ -577,12 +589,13 @@ pyodbc==5.3.0 pyodps==0.13.0 pyparsing==3.3.2 pypsrp==0.9.1 -pyspark-client==4.0.3 +pyroaring==1.1.0 +pyspark-client==4.2.0 pyspnego==0.12.1 python-arango==8.3.3 python-daemon==3.1.2 python-dateutil==2.9.0.post0 -python-discovery==1.4.2 +python-discovery==1.5.1 python-dotenv==1.2.2 python-http-client==3.3.7 python-jenkins==1.8.3 @@ -593,17 +606,17 @@ python-slugify==8.0.4 python-telegram-bot==22.8 python3-saml==1.16.0 pytokens==0.4.1 -pytz==2026.2 -pyvespa==1.2.3 +pytz==2026.3.post1 +pyvespa==1.2.4 pywinrm==0.5.0 pyzmq==27.1.0 -qdrant-client==1.18.0 -ray==2.56.0 -reactivex==4.1.0 +qdrant-client==1.19.0 +ray==2.56.1 +reactivex==5.1.0 redis==6.4.0 -redshift_connector==2.1.15 +redshift_connector==2.1.16 referencing==0.37.0 -regex==2026.6.28 +regex==2026.7.19 requests-file==3.0.1 requests-kerberos==0.15.0 requests-oauthlib==2.0.0 @@ -612,104 +625,108 @@ requests==2.34.2 requests_ntlm==1.3.0 retryhttp==1.5.0 rich-argparse==1.8.0 -rich-toolkit==0.20.1 -rich==13.9.4 +rich-toolkit==0.20.3 +rich==14.3.4 rpds-py==2026.6.3 rsa==4.9.1 ruamel.yaml==0.19.1 -s3fs==2026.6.0 -s3transfer==0.17.1 -sagemaker_studio==1.0.26 +s3fs==2026.7.0 +s3transfer==0.19.2 +sagemaker_studio==1.0.27 scikit-learn==1.9.0 -scipy==1.17.1 -scramp==1.4.10 +scipy==1.18.0 +scramp==1.4.17 scrapbook==0.5.0 segment-analytics-python==2.3.6 sendgrid==6.12.5 -sentry-sdk==2.64.0 +sentry-sdk==2.66.1 setproctitle==1.3.7 -setuptools==82.0.1 +setuptools==83.0.0 shellingham==1.5.4 -simple-salesforce==1.12.9 +simple-salesforce==1.12.10 six==1.17.0 slack_sdk==3.43.0 -smart_open==8.0.0 -smbprotocol==1.16.1 +smart_open==8.0.1 +smbprotocol==1.17.0 smmap==5.0.3 sniffio==1.3.1 -snowflake-connector-python==4.6.0 -snowflake-sqlalchemy==1.10.2 +snowflake-connector-python==4.7.2 +snowflake-snowpark-python==1.54.0 +snowflake-sqlalchemy==1.11.0 sortedcontainers==2.4.0 -soupsieve==2.8.4 +soupsieve==2.9.2 spython==0.3.14 -sqlalchemy-bigquery==1.17.0 +sqlalchemy-bigquery==1.17.2 sqlalchemy-spanner==1.19.0 sqlalchemy_drill==1.1.10 -sqlglot==30.12.0 +sqlglot==30.15.0 sqlparse==0.5.5 stack-data==0.6.3 -starlette==1.3.1 +starlette==1.5.0 statsd==4.0.1 -std-uritemplate==2.0.10 +std-uritemplate==2.0.12 +strictyaml==1.7.3 structlog==26.1.0 -svcs==25.1.0 +svcs==26.1.0 tableauserverclient==0.41 tabulate==0.10.0 tenacity==9.1.4 -teradatasql==20.0.0.62 +teradatasql==20.0.0.64 teradatasqlalchemy==20.0.0.9 termcolor==3.3.0 text-unidecode==1.3 -textual==6.2.1 +textual==8.2.8 threadpoolctl==3.6.0 thrift-sasl==0.4.3 -# Hand-patched (keep on regeneration): see the impyla note above. thrift <0.24.0 is vulnerable -# to CVE-2026-66053 / CVE-2026-41608 / CVE-2026-48586. +# Upstream constraints-3.3.1 now ships this natively, but keep the pin on regeneration: +# see the impyla note above. thrift <0.24.0 is vulnerable to CVE-2026-66053 / +# CVE-2026-41608 / CVE-2026-48586. thrift==0.24.0 tiktoken==0.13.0 tinycss2==1.5.1 tokenizers==0.23.1 -tomlkit==0.15.0 -tornado==6.5.7 -tqdm==4.68.3 -traitlets==5.15.1 +tomlkit==0.15.1 +tornado==6.5.8 +tqdm==4.70.0 +traitlets==5.16.1 trino==0.338.0 truststore==0.10.4 -typer==0.25.1 +typer==0.27.1 types-protobuf==7.34.1.20260518 -types-requests==2.33.0.20260518 +types-requests==2.33.0.20260712 typing-inspection==0.4.2 typing_extensions==4.16.0 -tzdata==2026.2 +tzdata==2026.3 tzlocal==5.4.4 uc-micro-py==2.0.0 universal_pathlib==0.3.10 uritemplate==4.2.0 urllib3==2.7.0 uuid6==2025.0.1 -uv==0.11.26 -uvicorn==0.49.0 +uv==0.12.3 +uvicorn==0.52.1 uvloop==0.22.1 validators==0.35.0 vertica-python==1.4.0 vine==5.1.0 -virtualenv==21.5.1 +virtualenv==21.7.2 watchfiles==1.2.0 watchtower==3.4.0 wcwidth==0.8.2 -weaviate-client==4.16.2 +weaviate-client==4.22.0 webencodings==0.5.1 -websocket-client==1.9.0 -websockets==16.0 +websocket-client==1.8.0 +websockets==16.1.1 +wheel==0.47.0 wirerope==1.0.0 -wrapt==2.2.2 +wrapt==2.3.0 xmlsec==1.3.17 xmltodict==1.0.4 yandex-query-client==0.1.4 -yandexcloud==0.397.0 -yarl==1.24.2 +yandexcloud==0.402.0 +yarl==1.24.5 ydb-dbapi==0.1.22 -ydb==3.29.6 +ydb==3.31.2 zeep==4.3.3 zenpy==2.0.57 zipp==4.1.0 diff --git a/ingestion/setup.py b/ingestion/setup.py index a734cc234375..ddac2c4b2cd6 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -20,8 +20,10 @@ # Add here versions required for multiple plugins VERSIONS = { # CVE-2026-42252 BashOperator Jinja2 injection; CVE-2026-48891 /ui/dependencies leaks - # Dag IDs the caller cannot read (residual gap in the CVE-2026-28563 fix, needs 3.3.0) - "airflow": "apache-airflow==3.3.0", + # Dag IDs the caller cannot read (residual gap in the CVE-2026-28563 fix, needs 3.3.0); + # CVE-2026-67587 Dag-author RCE on the Scheduler via a Serde Callback deserialization + # gadget and CVE-2026-54183 Variables unmasked in the UI (both need 3.3.1) + "airflow": "apache-airflow==3.3.1", "adlfs": "adlfs>=2023.1.0", "aiobotocore": "aiobotocore~=2.26.0", "avro": "avro>=1.11.4,<1.12", @@ -236,7 +238,10 @@ DATA_DIFF["clickhouse"], }, "dagster": { - "croniter<3", + # No croniter ceiling here: dagster 1.13 declares no croniter dependency at all, + # nothing under ingestion/ imports it, and apache-airflow-core 3.3.1 raised its + # floor to croniter>=6.2.2 -- a stale "croniter<3" makes the two uninstallable + # together. The airflow images already run croniter 6.2.x via the constraints file. VERSIONS["pymysql"], "psycopg2-binary", VERSIONS["geoalchemy2"], diff --git a/ingestion/tests/integration/airflow/Dockerfile b/ingestion/tests/integration/airflow/Dockerfile index f14867ee5a59..6f560750f6ec 100644 --- a/ingestion/tests/integration/airflow/Dockerfile +++ b/ingestion/tests/integration/airflow/Dockerfile @@ -12,16 +12,16 @@ # Airflow carrying the working tree's OpenMetadata lineage provider. Build context is # ingestion/, so src/metadata/generated must already exist (make generate). -FROM apache/airflow:3.3.0-python3.10 +FROM apache/airflow:3.3.1-python3.10 USER airflow COPY --chown=airflow:0 . /tmp/ingestion -# airflow-constraints-3.3.0.txt is deliberately NOT applied: it pins chardet==6.0.0.post1 +# airflow-constraints-3.3.1.txt is deliberately NOT applied: it pins chardet==7.5.1 # against openmetadata-ingestion's chardet==4.0.0. Dockerfile.ci installs the package # unconstrained for the same reason. apache-airflow is pinned so the resolver cannot move it. RUN pip install --no-cache-dir uv \ - && uv pip install --no-cache "apache-airflow==3.3.0" /tmp/ingestion \ + && uv pip install --no-cache "apache-airflow==3.3.1" /tmp/ingestion \ && rm -rf /tmp/ingestion # Migrating at build time keeps container start to a few seconds. diff --git a/ingestion/tests/integration/airflow/conftest.py b/ingestion/tests/integration/airflow/conftest.py index ffbb3259fa64..387d3475564b 100644 --- a/ingestion/tests/integration/airflow/conftest.py +++ b/ingestion/tests/integration/airflow/conftest.py @@ -26,7 +26,7 @@ from testcontainers.core.container import DockerContainer from testcontainers.core.docker_client import DockerClient -AIRFLOW_BASE_IMAGE = "apache/airflow:3.3.0-python3.10" +AIRFLOW_BASE_IMAGE = "apache/airflow:3.3.1-python3.10" AIRFLOW_TEST_IMAGE = "om-airflow-lineage-test:local" # Fixed network name declared by docker/development/docker-compose.yml, so the container