diff --git a/.github/actions/migration_tests/action.yml b/.github/actions/migration_tests/action.yml index ae05e647899d7..f3e17b4b18203 100644 --- a/.github/actions/migration_tests/action.yml +++ b/.github/actions/migration_tests/action.yml @@ -31,9 +31,13 @@ runs: MIN_AIRFLOW_VERSION="$( python ./scripts/ci/testing/get_min_airflow_version_for_python.py "${PYTHON_VERSION}" )" + AIRFLOW_EXTRAS="" + if [[ "${MIN_AIRFLOW_VERSION}" =~ ^2\. ]]; then + AIRFLOW_EXTRAS="--airflow-extras pydantic" + fi breeze shell "${AIRFLOW_2_CMD}" \ --use-airflow-version "${MIN_AIRFLOW_VERSION}" \ - --airflow-extras pydantic \ + ${AIRFLOW_EXTRAS} \ --answer y && breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${DB_MANGERS} ${AIRFLOW_3_CMD}" --no-db-cleanup @@ -61,9 +65,13 @@ runs: MIN_AIRFLOW_VERSION="$( python ./scripts/ci/testing/get_min_airflow_version_for_python.py "${PYTHON_VERSION}" )" + AIRFLOW_EXTRAS="" + if [[ "${MIN_AIRFLOW_VERSION}" =~ ^2\. ]]; then + AIRFLOW_EXTRAS="--airflow-extras pydantic" + fi breeze shell "${AIRFLOW_2_CMD}" \ --use-airflow-version "${MIN_AIRFLOW_VERSION}" \ - --airflow-extras pydantic \ + ${AIRFLOW_EXTRAS} \ --answer y && breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${DB_MANGERS} ${AIRFLOW_3_CMD}" --no-db-cleanup diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 5b23a3be28fd0..883290172986c 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -203,7 +203,9 @@ jobs: uses: ./.github/actions/migration_tests with: python-version: ${{ matrix.python-version }} - if: inputs.run-migration-tests == 'true' && inputs.test-group == 'core' + # Any new python version should be disabled below via `&& matrix.python-version != '3.xx'` until the first + # Airflow version that supports it is released - otherwise there's nothing to migrate back to. + if: inputs.run-migration-tests == 'true' && inputs.test-group == 'core' && matrix.python-version != '3.14' - name: > ${{ inputs.test-group }}:${{ inputs.test-scope }} Tests ${{ inputs.test-name }} ${{ matrix.backend-version }} Py${{ matrix.python-version }}:${{ env.PARALLEL_TEST_TYPES }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 879fb1246e137..abb8b12679f9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -585,6 +585,7 @@ repos: exclude: > (?x) ^README\.md$| + ^pyproject\.toml$| ^generated/PYPI_README\.md$| ^airflow-core/docs/.*commits\.rst$| ^airflow-core/newsfragments/41368\.significant\.rst$| @@ -808,6 +809,17 @@ repos: ^providers/.*/provider\.yaml$ pass_filenames: false require_serial: true + - id: check-excluded-provider-markers + name: Check excluded-provider python_version markers in pyproject.toml + language: python + entry: ./scripts/ci/prek/check_excluded_provider_markers.py + files: > + (?x) + ^pyproject\.toml$| + ^providers/.*/provider\.yaml$ + pass_filenames: false + require_serial: true + additional_dependencies: ['packaging>=25', 'pyyaml', 'tomli>=2.0.1', 'rich>=13.6.0'] - id: update-reproducible-source-date-epoch name: Update Source Date Epoch for reproducible builds language: python diff --git a/Dockerfile b/Dockerfile index c7d0b8c9f1812..db08779031d6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -810,7 +810,34 @@ function common::get_constraints_location() { echo echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}" echo - curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}" + local http_code + if http_code=$(curl -sS -L -o "${HOME}/constraints.txt" -w "%{http_code}" "${AIRFLOW_CONSTRAINTS_LOCATION}"); then + if [[ ${http_code} == "200" ]]; then + return + fi + if [[ ${http_code} == "404" \ + && ${ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE:="false"} == "true" \ + && ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION:="false"} == "true" ]]; then + echo + echo "${COLOR_YELLOW}Constraints file not found at ${AIRFLOW_CONSTRAINTS_LOCATION}.${COLOR_RESET}" + echo "${COLOR_YELLOW}Using an empty constraints file because bootstrap mode was explicitly enabled.${COLOR_RESET}" + echo + AIRFLOW_CONSTRAINTS_LOCATION="" + : > "${HOME}/constraints.txt" + return + fi + echo + echo "${COLOR_RED}Failed to download constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} (HTTP ${http_code}).${COLOR_RESET}" + if [[ ${http_code} == "404" ]]; then + echo "${COLOR_RED}Only bootstrap builds should set both ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE=true and AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=true.${COLOR_RESET}" + fi + echo + return 1 + fi + echo + echo "${COLOR_RED}Failed to download constraints from ${AIRFLOW_CONSTRAINTS_LOCATION}.${COLOR_RESET}" + echo + return 1 else echo echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}" diff --git a/Dockerfile.ci b/Dockerfile.ci index 5d86908a01467..5e438a44603ad 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -750,7 +750,34 @@ function common::get_constraints_location() { echo echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}" echo - curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}" + local http_code + if http_code=$(curl -sS -L -o "${HOME}/constraints.txt" -w "%{http_code}" "${AIRFLOW_CONSTRAINTS_LOCATION}"); then + if [[ ${http_code} == "200" ]]; then + return + fi + if [[ ${http_code} == "404" \ + && ${ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE:="false"} == "true" \ + && ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION:="false"} == "true" ]]; then + echo + echo "${COLOR_YELLOW}Constraints file not found at ${AIRFLOW_CONSTRAINTS_LOCATION}.${COLOR_RESET}" + echo "${COLOR_YELLOW}Using an empty constraints file because bootstrap mode was explicitly enabled.${COLOR_RESET}" + echo + AIRFLOW_CONSTRAINTS_LOCATION="" + : > "${HOME}/constraints.txt" + return + fi + echo + echo "${COLOR_RED}Failed to download constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} (HTTP ${http_code}).${COLOR_RESET}" + if [[ ${http_code} == "404" ]]; then + echo "${COLOR_RED}Only bootstrap builds should set both ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE=true and AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=true.${COLOR_RESET}" + fi + echo + return 1 + fi + echo + echo "${COLOR_RED}Failed to download constraints from ${AIRFLOW_CONSTRAINTS_LOCATION}.${COLOR_RESET}" + echo + return 1 else echo echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}" @@ -1680,6 +1707,7 @@ ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers" ARG AIRFLOW_CONSTRAINTS_REFERENCE="" ARG AIRFLOW_CONSTRAINTS_LOCATION="" +ARG ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE="false" ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" # By default fallback to installation without constraints because in CI image it should always be tried ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="true" @@ -1712,6 +1740,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ + ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE=${ALLOW_MISSING_PREVIOUS_CONSTRAINTS_FILE} \ AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \ @@ -1729,6 +1758,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_VERSION_SPECIFICATION="" \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ + INCLUDE_PRE_RELEASE="true" \ CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \ CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON} diff --git a/airflow-core/docs/installation/prerequisites.rst b/airflow-core/docs/installation/prerequisites.rst index 17dd6a2e7e6d1..359450d3eeb09 100644 --- a/airflow-core/docs/installation/prerequisites.rst +++ b/airflow-core/docs/installation/prerequisites.rst @@ -20,7 +20,7 @@ Prerequisites Airflow® is tested with: -* Python: 3.10, 3.11, 3.12, 3.13 +* Python: 3.10, 3.11, 3.12, 3.13, 3.14 * Databases: diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index c853c7de833f8..a9fb7321e8ca8 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -36,12 +36,11 @@ description = "Core packages for Apache Airflow, schedule and API server" readme = { file = "README.md", content-type = "text/markdown" } license = "Apache-2.0" license-files = ["LICENSE", "NOTICE"] -# We know that it will take a while before we can support Python 3.14 because of all our dependencies -# It takes about 4-7 months after Python release before we can support it, so we limit it to <3.14 -# proactively. This way we also have a chance to test it with Python 3.14 and bump the upper binding -# and manually mark providers that do not support it yet with !-3.14 - until they support it - which will -# also exclude resolving uv workspace dependencies for those providers. -requires-python = ">=3.10,!=3.14" +# Supporting new Python releases typically takes 4-7 months due to all our dependencies. +# We proactively exclude the next major version to avoid dependency conflicts, then test it and +# bump the upper binding once ready. Providers that don't support it yet are marked with +# != constraint - until they support it - which also excludes resolving uv workspace dependencies. +requires-python = ">=3.10,!=3.15" authors = [ { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, ] @@ -60,6 +59,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: System :: Monitoring", ] @@ -80,7 +80,8 @@ dependencies = [ # The 1.13.0 of alembic marked some migration code as SQLAlchemy 2+ only so we limit it to 1.13.1 "alembic>=1.13.1, <2.0", "argcomplete>=1.10", - "asgiref>=2.3.0", + "asgiref>=2.3.0; python_version < '3.14'", + "asgiref>=3.11.1; python_version >= '3.14'", "attrs>=22.1.0, !=25.2.0", "cadwyn>=6.0.4", "colorlog>=6.8.2", @@ -103,7 +104,8 @@ dependencies = [ "jinja2>=3.1.5", "jsonschema>=4.19.1", "lazy-object-proxy>=1.2.0", - 'libcst >=1.8.2', + 'libcst >=1.8.2; python_version < "3.14"', + 'libcst >=1.8.6; python_version >= "3.14"', "linkify-it-py>=2.0.0", "lockfile>=0.12.2", "methodtools>=0.4.7", @@ -138,8 +140,7 @@ dependencies = [ "rich-argparse>=1.0.0", "rich>=13.6.0", "setproctitle>=1.3.3", - # SQLAlchemy >=2.0.36 fixes Python 3.13 TypingOnly import AssertionError caused by new typing attributes (__static_attributes__, __firstlineno__) - "sqlalchemy[asyncio]>=2.0.36", + "sqlalchemy[asyncio]>=2.0.48", "svcs>=25.1.0", "tabulate>=0.9.0", "tenacity>=8.3.0", @@ -173,7 +174,8 @@ dependencies = [ "async" = [ "eventlet>=0.37.0", "gevent>=25.4.1", - "greenlet>=3.1.0", + "greenlet>=3.1.0; python_version < '3.14'", + "greenlet>=3.3.2; python_version >= '3.14'", "greenback>=1.2.1", ] "graphviz" = [ diff --git a/airflow-core/src/airflow/models/connection.py b/airflow-core/src/airflow/models/connection.py index 0e1505f4667dd..d58f2b3202dbd 100644 --- a/airflow-core/src/airflow/models/connection.py +++ b/airflow-core/src/airflow/models/connection.py @@ -32,7 +32,6 @@ from airflow._shared.module_loading import import_string from airflow._shared.secrets_masker import mask_secret -from airflow.configuration import conf, ensure_secrets_loaded from airflow.exceptions import AirflowException, AirflowNotFoundException from airflow.models.base import ID_LEN, Base from airflow.models.crypto import get_fernet @@ -528,6 +527,8 @@ def get_connection_from_secrets(cls, conn_id: str, team_name: str | None = None) raise AirflowNotFoundException(f"The conn_id `{conn_id}` isn't defined") from None raise + from airflow.configuration import conf, ensure_secrets_loaded + if team_name and not conf.getboolean("core", "multi_team"): raise ValueError( "Multi-team mode is not configured in the Airflow environment but the task trying to access the connection belongs to a team" diff --git a/airflow-core/src/airflow/typing_compat.py b/airflow-core/src/airflow/typing_compat.py index 31b6e6ecaa5a0..1f9fd0b217f12 100644 --- a/airflow-core/src/airflow/typing_compat.py +++ b/airflow-core/src/airflow/typing_compat.py @@ -29,4 +29,5 @@ if sys.version_info >= (3, 11): from typing import Self, Unpack, assert_never else: + # TODO: Remove once Python 3.10 support is dropped (EOL 2026) from typing_extensions import Self, Unpack, assert_never diff --git a/airflow-core/tests/unit/always/test_example_dags.py b/airflow-core/tests/unit/always/test_example_dags.py index b2fca80e8700d..995436b73bc42 100644 --- a/airflow-core/tests/unit/always/test_example_dags.py +++ b/airflow-core/tests/unit/always/test_example_dags.py @@ -66,6 +66,9 @@ IGNORE_EXAMPLE_DAGS: tuple[str, ...] = ( # These example dags require suspended providers, eg: google dataflow dependent on the Apache Beam provider, # but it's in the suspended list, we can't import the dag + # Ray uses pydantic v1 internally, which fails to infer types in Python 3.14. + # TODO: remove once ray releases a version with Python 3.14 support. + "providers/google/tests/system/google/cloud/ray/example_ray_job.py", "providers/google/tests/system/google/cloud/dataflow/example_dataflow_go.py", "providers/google/tests/system/google/cloud/dataflow/example_dataflow_java_streaming.py", "providers/google/tests/system/google/cloud/dataflow/example_dataflow_native_java.py", diff --git a/airflow-core/tests/unit/api_fastapi/auth/managers/simple/conftest.py b/airflow-core/tests/unit/api_fastapi/auth/managers/simple/conftest.py index 54808788a3d1b..92cc89f581f5b 100644 --- a/airflow-core/tests/unit/api_fastapi/auth/managers/simple/conftest.py +++ b/airflow-core/tests/unit/api_fastapi/auth/managers/simple/conftest.py @@ -18,6 +18,7 @@ from __future__ import annotations +import contextlib import os import pytest @@ -33,7 +34,7 @@ @pytest.fixture def auth_manager(): auth_manager = SimpleAuthManager() - if os.path.exists(auth_manager.get_generated_password_file()): + with contextlib.suppress(FileNotFoundError): os.remove(auth_manager.get_generated_password_file()) return auth_manager diff --git a/airflow-core/tests/unit/api_fastapi/common/test_http_access_log.py b/airflow-core/tests/unit/api_fastapi/common/test_http_access_log.py index 3547fdf22adc2..5e7e9e6e760d3 100644 --- a/airflow-core/tests/unit/api_fastapi/common/test_http_access_log.py +++ b/airflow-core/tests/unit/api_fastapi/common/test_http_access_log.py @@ -123,7 +123,7 @@ async def lifespan_app(scope, receive, send): import asyncio with structlog.testing.capture_logs() as logs: - asyncio.get_event_loop().run_until_complete(middleware({"type": "lifespan"}, None, None)) + asyncio.run(middleware({"type": "lifespan"}, None, None)) assert logs == [] diff --git a/airflow-core/tests/unit/executors/test_local_executor.py b/airflow-core/tests/unit/executors/test_local_executor.py index 34e8f818aa94c..59afffe6833fe 100644 --- a/airflow-core/tests/unit/executors/test_local_executor.py +++ b/airflow-core/tests/unit/executors/test_local_executor.py @@ -41,10 +41,16 @@ pytestmark = pytest.mark.db_test -# Runtime is fine, we just can't run the tests on macOS -skip_spawn_mp_start = pytest.mark.skipif( - multiprocessing.get_context().get_start_method() == "spawn", - reason="mock patching in test don't work with 'spawn' mode (default on macOS)", +# Mock patching doesn't work across process boundaries with 'spawn' (default on macOS) +# or 'forkserver' (default on Linux with Python 3.14+). +skip_non_fork_mp_start = pytest.mark.skipif( + multiprocessing.get_start_method() != "fork", + reason="mock patching in test doesn't work with non-fork multiprocessing start methods", +) + +skip_fork_mp_start = pytest.mark.skipif( + multiprocessing.get_start_method() == "fork", + reason="tests non-fork (lazy-spawning) behavior", ) @@ -68,7 +74,7 @@ def test_supports_multi_team(self): def test_serve_logs_default_value(self): assert LocalExecutor.serve_logs - @skip_spawn_mp_start + @skip_non_fork_mp_start @mock.patch.object(gc, "unfreeze") @mock.patch.object(gc, "freeze") def test_executor_worker_spawned(self, mock_freeze, mock_unfreeze): @@ -82,6 +88,33 @@ def test_executor_worker_spawned(self, mock_freeze, mock_unfreeze): executor.end() + @skip_fork_mp_start + @mock.patch.object(gc, "unfreeze") + @mock.patch.object(gc, "freeze") + def test_executor_lazy_worker_spawning(self, mock_freeze, mock_unfreeze): + """On non-fork start methods, workers are spawned lazily and gc.freeze is not called.""" + executor = LocalExecutor(parallelism=3) + executor.start() + + try: + # No workers should be pre-spawned + assert len(executor.workers) == 0 + mock_freeze.assert_not_called() + mock_unfreeze.assert_not_called() + + # Simulate a queued message so _check_workers spawns one worker on demand + with executor._unread_messages: + executor._unread_messages.value = 1 + executor.activity_queue.put(None) # poison pill so the worker exits cleanly + executor._check_workers() + + assert len(executor.workers) == 1 + # gc.freeze is still not used for non-fork + mock_freeze.assert_not_called() + finally: + executor.end() + + @skip_non_fork_mp_start @mock.patch("airflow.sdk.execution_time.supervisor.supervise") def test_execution(self, mock_supervise): success_tis = [ @@ -182,7 +215,7 @@ def test_gauge_executor_metrics(self, mock_stats_gauge, mock_trigger_tasks, mock mock_stats_gauge.assert_has_calls(calls) @skip_if_force_lowest_dependencies_marker - @pytest.mark.execution_timeout(5) + @pytest.mark.execution_timeout(30) def test_clean_stop_on_signal(self): import signal @@ -303,8 +336,15 @@ def test_multiple_team_executors_isolation(self): # Verify each executor has its own workers dict assert team_a_executor.workers is not team_b_executor.workers - assert len(team_a_executor.workers) == 2 - assert len(team_b_executor.workers) == 3 + + if LocalExecutor.is_mp_using_fork: + # fork pre-spawns all workers at start() + assert len(team_a_executor.workers) == 2 + assert len(team_b_executor.workers) == 3 + else: + # forkserver/spawn use lazy spawning + assert len(team_a_executor.workers) == 0 + assert len(team_b_executor.workers) == 0 # Verify each executor has its own unread_messages counter assert team_a_executor._unread_messages is not team_b_executor._unread_messages @@ -327,8 +367,11 @@ def test_global_executor_without_team_name(self): executor.start() - # Verify workers were created - assert len(executor.workers) == 2 + if LocalExecutor.is_mp_using_fork: + assert len(executor.workers) == 2 + else: + # forkserver/spawn use lazy spawning + assert len(executor.workers) == 0 executor.end() @@ -338,7 +381,7 @@ def test_supports_callbacks_flag_is_true(self): executor = LocalExecutor() assert executor.supports_callbacks is True - @skip_spawn_mp_start + @skip_non_fork_mp_start @mock.patch("airflow.executors.workloads.callback.execute_callback_workload") def test_process_callback_workload(self, mock_execute_callback): mock_execute_callback.return_value = (True, None) diff --git a/airflow-core/tests/unit/utils/test_process_utils.py b/airflow-core/tests/unit/utils/test_process_utils.py index 47ffc9245f3e8..5d054db778609 100644 --- a/airflow-core/tests/unit/utils/test_process_utils.py +++ b/airflow-core/tests/unit/utils/test_process_utils.py @@ -18,11 +18,10 @@ from __future__ import annotations import logging -import multiprocessing import os import signal import subprocess -import time +import sys from contextlib import suppress from subprocess import CalledProcessError from time import sleep @@ -42,61 +41,51 @@ class TestReapProcessGroup: - @staticmethod - def _ignores_sigterm(child_pid, child_setup_done): - def signal_handler(unused_signum, unused_frame): - pass - - signal.signal(signal.SIGTERM, signal_handler) - child_pid.value = os.getpid() - child_setup_done.release() - while True: - time.sleep(1) - - @staticmethod - def _parent_of_ignores_sigterm(parent_pid, child_pid, setup_done): - def signal_handler(unused_signum, unused_frame): - pass - - os.setsid() - signal.signal(signal.SIGTERM, signal_handler) - child_setup_done = multiprocessing.Semaphore(0) - child = multiprocessing.Process( - target=TestReapProcessGroup._ignores_sigterm, args=[child_pid, child_setup_done] - ) - child.start() - child_setup_done.acquire(timeout=5.0) - parent_pid.value = os.getpid() - setup_done.release() - while True: - time.sleep(1) + # Inline script that creates a new session, spawns a SIGTERM-ignoring child, + # prints both PIDs to stdout, then loops forever. Uses subprocess.Popen + # instead of multiprocessing.Process to avoid fork()-ing the (large) test + # worker process — which causes OOM under xdist on Python 3.14. + _PARENT_SCRIPT = """\ +import os, signal, subprocess, sys, time +os.setsid() +signal.signal(signal.SIGTERM, lambda s, f: None) +child = subprocess.Popen( + [sys.executable, "-c", + "import signal, time; signal.signal(signal.SIGTERM, lambda s,f: None); time.sleep(300)"] +) +print(f"{os.getpid()} {child.pid}", flush=True) +time.sleep(300) +""" def test_reap_process_group(self): """ Spin up a process that can't be killed by SIGTERM and make sure it gets killed anyway. """ - parent_setup_done = multiprocessing.Semaphore(0) - parent_pid = multiprocessing.Value("i", 0) - child_pid = multiprocessing.Value("i", 0) - args = [parent_pid, child_pid, parent_setup_done] - parent = multiprocessing.Process(target=TestReapProcessGroup._parent_of_ignores_sigterm, args=args) + parent = subprocess.Popen( + [sys.executable, "-c", self._PARENT_SCRIPT], + stdout=subprocess.PIPE, + text=True, + ) try: - parent.start() - assert parent_setup_done.acquire(timeout=5.0) - assert psutil.pid_exists(parent_pid.value) - assert psutil.pid_exists(child_pid.value) + line = parent.stdout.readline() + assert line, "Parent script did not print PIDs" + parent_pid, child_pid = map(int, line.strip().split()) - process_utils.reap_process_group(parent_pid.value, logging.getLogger(), timeout=1) + assert psutil.pid_exists(parent_pid) + assert psutil.pid_exists(child_pid) - assert not psutil.pid_exists(parent_pid.value) - assert not psutil.pid_exists(child_pid.value) + process_utils.reap_process_group(parent_pid, logging.getLogger(), timeout=1) + + assert not psutil.pid_exists(parent_pid) + assert not psutil.pid_exists(child_pid) finally: try: - os.kill(parent_pid.value, signal.SIGKILL) # terminate doesn't work here - os.kill(child_pid.value, signal.SIGKILL) # terminate doesn't work here + os.kill(parent.pid, signal.SIGKILL) except OSError: pass + parent.stdout.close() + parent.wait() @pytest.mark.db_test @@ -133,48 +122,42 @@ def test_using_env_as_kwarg_works(self, caplog): assert "My value is 1" in caplog.text -def my_sleep_subprocess(): - sleep(100) - - -def my_sleep_subprocess_with_signals(): - signal.signal(signal.SIGINT, lambda signum, frame: None) - signal.signal(signal.SIGTERM, lambda signum, frame: None) - sleep(100) - - @pytest.mark.db_test class TestKillChildProcessesByPids: def test_should_kill_process(self): - before_num_process = subprocess.check_output(["ps", "-ax", "-o", "pid="]).decode().count("\n") - - process = multiprocessing.Process(target=my_sleep_subprocess, args=()) - process.start() - sleep(0) + process = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) - num_process = subprocess.check_output(["ps", "-ax", "-o", "pid="]).decode().count("\n") - assert before_num_process + 1 == num_process + assert psutil.pid_exists(process.pid) process_utils.kill_child_processes_by_pids([process.pid]) - num_process = subprocess.check_output(["ps", "-ax", "-o", "pid="]).decode().count("\n") - assert before_num_process == num_process + assert not psutil.pid_exists(process.pid) def test_should_force_kill_process(self, caplog): - process = multiprocessing.Process(target=my_sleep_subprocess_with_signals, args=()) - process.start() - sleep(0) + process = subprocess.Popen( + [ + sys.executable, + "-c", + "import signal, time; " + "signal.signal(signal.SIGINT, lambda s,f: None); " + "signal.signal(signal.SIGTERM, lambda s,f: None); " + "print('ready', flush=True); " + "time.sleep(300)", + ], + stdout=subprocess.PIPE, + text=True, + ) + # Wait until signal handlers are installed before sending signals + process.stdout.readline() - all_processes = subprocess.check_output(["ps", "-ax", "-o", "pid="]).decode().splitlines() - assert str(process.pid) in (x.strip() for x in all_processes) + assert psutil.pid_exists(process.pid) with caplog.at_level(logging.INFO, logger=process_utils.log.name): caplog.clear() process_utils.kill_child_processes_by_pids([process.pid], timeout=0) assert f"Killing child PID: {process.pid}" in caplog.messages sleep(0) - all_processes = subprocess.check_output(["ps", "-ax", "-o", "pid="]).decode().splitlines() - assert str(process.pid) not in (x.strip() for x in all_processes) + assert not psutil.pid_exists(process.pid) class TestPatchEnviron: diff --git a/airflow-ctl-tests/pyproject.toml b/airflow-ctl-tests/pyproject.toml index 6f6ccd16424ed..be43fb4cf5c1d 100644 --- a/airflow-ctl-tests/pyproject.toml +++ b/airflow-ctl-tests/pyproject.toml @@ -33,7 +33,7 @@ description = "Airflow CTL tests for Apache Airflow" classifiers = [ "Private :: Do Not Upload", ] -requires-python = ">=3.10,!=3.14" +requires-python = ">=3.10,!=3.15" authors = [ { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, ] diff --git a/airflow-ctl/docs/installation/prerequisites.rst b/airflow-ctl/docs/installation/prerequisites.rst index b01ff4c3cc41b..9d2efa3641c06 100644 --- a/airflow-ctl/docs/installation/prerequisites.rst +++ b/airflow-ctl/docs/installation/prerequisites.rst @@ -69,8 +69,6 @@ use cases. Simply install them to make them available: Python Version Compatibility ---------------------------- ``airflowctl`` is compatible with versions of Python 3.10 through Python 3.14. -Currently, Python 3.14 is not supported. Thanks for your understanding! -We will work on adding support for Python 3.14. .. list-table:: :widths: 15 85 @@ -87,4 +85,6 @@ We will work on adding support for Python 3.14. * - 3.13 - Yes * - 3.14 + - Yes + * - 3.15 - No diff --git a/airflow-ctl/pyproject.toml b/airflow-ctl/pyproject.toml index fec7206a23092..1fc37efc1668f 100644 --- a/airflow-ctl/pyproject.toml +++ b/airflow-ctl/pyproject.toml @@ -22,11 +22,10 @@ description = "Apache Airflow command line tool for communicating with an Apache readme = { file = "README.md", content-type = "text/markdown" } license = "Apache-2.0" license-files = ["LICENSE", "NOTICE"] -# We do not want to upper-bind Python version, as we do not know if we will support Python 3.14+ -# out-of-the box. Airflow-ctl is a small tool that does not have many dependencies and does not use -# sophisticated features of Python, so it should work with Python 3.14+ once all it's dependencies are -# updated to support it. -requires-python = ">=3.10,!=3.14" +# We do not want to upper-bind Python version unnecessarily. Airflow-ctl is a small tool with +# minimal dependencies and does not use sophisticated Python features, so it should work with +# new Python versions once all dependencies are updated to support them. +requires-python = ">=3.10,!=3.15" dependencies = [ # TODO there could be still missing deps such as airflow-core "argcomplete>=1.10", diff --git a/airflow-e2e-tests/pyproject.toml b/airflow-e2e-tests/pyproject.toml index 20cc5bcad55c6..882a6e176f914 100644 --- a/airflow-e2e-tests/pyproject.toml +++ b/airflow-e2e-tests/pyproject.toml @@ -32,7 +32,7 @@ description = "E2E tests for Apache Airflow" classifiers = [ "Private :: Do Not Upload", ] -requires-python = ">=3.10,!=3.14" +requires-python = ">=3.10,!=3.15" authors = [ { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, ] diff --git a/chart/pyproject.toml b/chart/pyproject.toml index f8d2eed8bbabc..fc903a8c070ee 100644 --- a/chart/pyproject.toml +++ b/chart/pyproject.toml @@ -29,7 +29,7 @@ build-backend = "hatchling.build" [project] name = "apache-airflow-helm-chart" description = "Programmatically author, schedule and monitor data pipelines" -requires-python = ">=3.10,!=3.14" +requires-python = ">=3.10,!=3.15" authors = [ { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, ] @@ -48,6 +48,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: System :: Monitoring", "Topic :: System :: Monitoring", ] diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 628f36e13d66f..36d4fc8db1d69 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -33,7 +33,7 @@ description = "Apache Airflow API (Stable)" readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE", "NOTICE"] -requires-python = ">=3.10,!=3.14" +requires-python = ">=3.10,!=3.15" authors = [ { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, ] @@ -53,6 +53,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: System :: Monitoring", ] diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 1fed95c89acc5..d6cd7ad22845c 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -1,4 +1,4 @@ - +