From 2e4517ffe98e9136990e565db578724860e95cd1 Mon Sep 17 00:00:00 2001 From: Maksim Moiseenkov Date: Wed, 22 Jan 2025 11:57:41 +0000 Subject: [PATCH] Add config option [secrets]backends_order --- .../secrets/secrets-backend/index.rst | 23 +++- .../core_api/openapi/_private_ui.yaml | 98 ++++++++++++++ .../api_fastapi/core_api/routes/ui/config.py | 38 +++++- .../src/airflow/config_templates/config.yml | 34 +++++ airflow-core/src/airflow/configuration.py | 109 +++++++++++++--- airflow-core/src/airflow/secrets/__init__.py | 9 +- .../airflow/ui/openapi-gen/queries/common.ts | 6 + .../ui/openapi-gen/queries/ensureQueryData.ts | 10 ++ .../ui/openapi-gen/queries/prefetch.ts | 10 ++ .../airflow/ui/openapi-gen/queries/queries.ts | 10 ++ .../ui/openapi-gen/queries/suspense.ts | 10 ++ .../ui/openapi-gen/requests/services.gen.ts | 24 +++- .../ui/openapi-gen/requests/types.gen.ts | 29 +++++ .../ui/public/i18n/locales/en/admin.json | 1 + .../pages/Variables/BackendsOrderButton.tsx | 76 +++++++++++ .../src/pages/Variables/BackendsOrderCard.tsx | 42 ++++++ .../pages/Variables/BackendsOrderModal.tsx | 67 ++++++++++ .../ui/src/pages/Variables/Variables.tsx | 4 + .../tests/unit/always/test_secrets.py | 123 ++++++++++++++++++ task-sdk/src/airflow/sdk/configuration.py | 119 +++++++++++++---- .../task_sdk/execution_time/test_context.py | 10 -- 21 files changed, 792 insertions(+), 60 deletions(-) create mode 100644 airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderButton.tsx create mode 100644 airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderCard.tsx create mode 100644 airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderModal.tsx diff --git a/airflow-core/docs/security/secrets/secrets-backend/index.rst b/airflow-core/docs/security/secrets/secrets-backend/index.rst index 029dadcfb875a..45ea494433986 100644 --- a/airflow-core/docs/security/secrets/secrets-backend/index.rst +++ b/airflow-core/docs/security/secrets/secrets-backend/index.rst @@ -39,13 +39,15 @@ When looking up a connection/variable, by default Airflow will search environmen database second. If you enable an alternative secrets backend, it will be searched first, followed by environment variables, -then metastore. This search ordering is not configurable. Though, in some alternative secrets backend you might have +then metastore. Though, in some alternative secrets backend you might have the option to filter which connection/variable/config is searched in the secret backend. Please look at the documentation of the secret backend you are using to see if such option is available. On the other hand, if a workers secrets backend is defined, the order of lookup has higher priority for the workers secrets backend and then the secrets backend. +The secrets backends search ordering is also configurable via the configuration option ``[secrets]backends_order``. + .. warning:: When using environment variables or an alternative secrets backend to store secrets or variables, it is possible to create key collisions. @@ -64,12 +66,21 @@ The ``[secrets]`` section has the following options: [secrets] backend = backend_kwargs = + backends_order = Set ``backend`` to the fully qualified class name of the backend you want to enable. You can provide ``backend_kwargs`` with json and it will be passed as kwargs to the ``__init__`` method of your secrets backend. +``backends_order`` is a comma-separated list of secret backends. These backends will be used in the order they are specified. +Please note that the ``environment_variable`` and ``metastore`` are required values and cannot be removed +from the list. Supported values are: + +* ``custom``: Custom secret backend specified in the ``secrets[backend]`` configuration option. +* ``environment_variable``: Standard environment variable backend ``airflow.secrets.environment_variables.EnvironmentVariablesBackend``. +* ``metastore``: Standard metastore backend ``airflow.secrets.metastore.MetastoreBackend``. + If you want to check which secret backend is currently set, you can use ``airflow config get-value secrets backend`` command as in the example below. @@ -112,13 +123,21 @@ configure separate secrets backend for workers, you can do that using: [workers] secrets_backend = secrets_backend_kwargs = - + backends_order = Set ``secrets_backend`` to the fully qualified class name of the backend you want to enable. You can provide ``secrets_backend_kwargs`` with json and it will be passed as kwargs to the ``__init__`` method of your secrets backend for the workers. +``backends_order`` is a comma-separated list of secret backends for workers. These backends will be used in the order they are specified. +Please note that the ``environment_variable`` and ``execution_api`` are required values and cannot be removed +from the list. Supported values are: + +* ``custom``: Custom secret backend specified in the ``workers[secrets_backend]`` configuration option. +* ``environment_variable``: Standard environment variable backend ``airflow.secrets.environment_variables.EnvironmentVariablesBackend``. +* ``execution_api``: Standard execution_api backend ``airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend``. + If you want to check which secret backend is currently set, you can use ``airflow config get-value workers secrets_backend`` command as in the example below. diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index 6c38b7ce2d6c3..de9ea5f6f4aee 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -199,6 +199,52 @@ paths: security: - OAuth2PasswordBearer: [] - HTTPBearer: [] + /ui/backends_order: + get: + tags: + - Config + summary: Get Backends Order Value + operationId: get_backends_order_value + security: + - OAuth2PasswordBearer: [] + - HTTPBearer: [] + parameters: + - name: accept + in: header + required: false + schema: + type: string + enum: + - application/json + - text/plain + - '*/*' + default: '*/*' + title: Accept + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Config' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '406': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Acceptable + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /ui/connections/hook_meta: get: tags: @@ -1567,6 +1613,41 @@ components: - count title: CalendarTimeRangeResponse description: Represents a summary of DAG runs for a specific calendar time range. + Config: + properties: + sections: + items: + $ref: '#/components/schemas/ConfigSection' + type: array + title: Sections + additionalProperties: false + type: object + required: + - sections + title: Config + description: List of config sections with their options. + ConfigOption: + properties: + key: + type: string + title: Key + value: + anyOf: + - type: string + - prefixItems: + - type: string + - type: string + type: array + maxItems: 2 + minItems: 2 + title: Value + additionalProperties: false + type: object + required: + - key + - value + title: ConfigOption + description: Config option. ConfigResponse: properties: fallback_page_limit: @@ -1629,6 +1710,23 @@ components: - multi_team title: ConfigResponse description: configuration serializer. + ConfigSection: + properties: + name: + type: string + title: Name + options: + items: + $ref: '#/components/schemas/ConfigOption' + type: array + title: Options + additionalProperties: false + type: object + required: + - name + - options + title: ConfigSection + description: Config Section Schema. ConnectionHookFieldBehavior: properties: hidden: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/config.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/config.py index 9510938232015..2643a4dcb85e8 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/config.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/config.py @@ -19,13 +19,20 @@ from json import loads from typing import Any -from fastapi import Depends, status +from fastapi import Depends, HTTPException, status +from airflow.api_fastapi.common.headers import HeaderAcceptJsonOrText from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.common.types import UIAlert +from airflow.api_fastapi.core_api.datamodels.config import ( + Config, + ConfigOption, + ConfigSection, +) from airflow.api_fastapi.core_api.datamodels.ui.config import ConfigResponse from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.api_fastapi.core_api.security import requires_authenticated +from airflow.api_fastapi.core_api.services.public.config import _response_based_on_accept from airflow.configuration import conf from airflow.settings import DASHBOARD_UIALERTS from airflow.utils.log.log_reader import TaskLogReader @@ -67,3 +74,32 @@ def get_configs() -> ConfigResponse: config.update({key: value for key, value in additional_config.items()}) return ConfigResponse.model_validate(config) + + +@config_router.get( + "/backends_order", + responses={ + **create_openapi_http_exception_doc( + [ + status.HTTP_404_NOT_FOUND, + status.HTTP_406_NOT_ACCEPTABLE, + ] + ), + }, + response_model=Config, + dependencies=[Depends(requires_authenticated())], +) +def get_backends_order_value( + accept: HeaderAcceptJsonOrText, +): + section, option = "secrets", "backends_order" + if not conf.has_option(section, option): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Option [{section}/{option}] not found.", + ) + + value = conf.get(section, option) + + config = Config(sections=[ConfigSection(name=section, options=[ConfigOption(key=option, value=value)])]) + return _response_based_on_accept(accept, config) diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index f931fedc473fb..492ce99a2ba85 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -1469,6 +1469,22 @@ secrets: sensitive: true example: ~ default: "" + backends_order: + description: | + .. note:: |experimental| + + Comma-separated list of secret backends. These backends will be used in the order they are specified. + Please note that the `environment_variable` and `metastore` are required values and cannot be removed + from the list. Supported values are: + + * ``custom``: Custom secret backend specified in the ``secrets[backend]`` configuration option. + * ``environment_variable``: Standard environment variable backend + ``airflow.secrets.environment_variables.EnvironmentVariablesBackend``. + * ``metastore``: Standard metastore backend ``airflow.secrets.metastore.MetastoreBackend``. + version_added: 3.2.0 + type: string + example: ~ + default: "custom,environment_variable,metastore" use_cache: description: | .. note:: |experimental| @@ -1833,6 +1849,24 @@ workers: sensitive: true example: ~ default: "" + backends_order: + description: | + .. note:: |experimental| + + Comma-separated list of secret backends for workers. These backends will be used in the order they are + specified. Please note that the ``environment_variable`` and ``execution_api`` are required values and + cannot be removed from the list. Supported values are: + + * ``custom``: Custom secret backend specified in the ``workers[secrets_backend]`` configuration + option. + * ``environment_variable``: Standard environment variable backend + ``airflow.secrets.environment_variables.EnvironmentVariablesBackend``. + * ``execution_api``: Standard execution_api backend + ``airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend``. + version_added: 3.2.0 + type: string + example: ~ + default: "custom,environment_variable,execution_api" min_heartbeat_interval: description: | The minimum interval (in seconds) at which the worker checks the task instance's diff --git a/airflow-core/src/airflow/configuration.py b/airflow-core/src/airflow/configuration.py index 69d936be01d82..8409dc650ba9a 100644 --- a/airflow-core/src/airflow/configuration.py +++ b/airflow-core/src/airflow/configuration.py @@ -30,6 +30,7 @@ from collections.abc import Callable from configparser import ConfigParser from copy import deepcopy +from enum import Enum from inspect import ismodule from io import StringIO from re import Pattern @@ -46,7 +47,6 @@ ) from airflow._shared.module_loading import import_string from airflow.exceptions import AirflowConfigException, RemovedInAirflow4Warning -from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH from airflow.task.weight_rule import WeightRule from airflow.utils import yaml @@ -841,7 +841,7 @@ def make_group_other_inaccessible(file_path: str): def ensure_secrets_loaded( - default_backends: list[str] = DEFAULT_SECRETS_SEARCH_PATH, + default_backends: list[str] | None = None, ) -> list[BaseSecretsBackend]: """ Ensure that all secrets backends are loaded. @@ -850,9 +850,8 @@ def ensure_secrets_loaded( """ # Check if the secrets_backend_list contains only 2 default backends. - # Check if we are loading the backends for worker too by checking if the default_backends is equal - # to DEFAULT_SECRETS_SEARCH_PATH. - if len(secrets_backend_list) == 2 or default_backends != DEFAULT_SECRETS_SEARCH_PATH: + # Check if we are loading the backends for worker too by checking if the default_backends is not None + if len(secrets_backend_list) == 2 or default_backends is not None: return initialize_secrets_backends(default_backends=default_backends) return secrets_backend_list @@ -868,8 +867,25 @@ def get_custom_secret_backend(worker_mode: bool = False) -> BaseSecretsBackend | return conf._get_custom_secret_backend(worker_mode=worker_mode) +def get_importable_secret_backend(class_name: str | None) -> BaseSecretsBackend | None: + """Get secret backend defined in the given class name.""" + if class_name is not None: + secrets_backend_cls = import_string(class_name) + return secrets_backend_cls() + return None + + +class Backends(Enum): + """Type of the secrets backend.""" + + ENVIRONMENT_VARIABLE = "environment_variable" + EXECUTION_API = "execution_api" + CUSTOM = "custom" + METASTORE = "metastore" + + def initialize_secrets_backends( - default_backends: list[str] = DEFAULT_SECRETS_SEARCH_PATH, + default_backends: list[str] | None = None, ) -> list[BaseSecretsBackend]: """ Initialize secrets backend. @@ -877,26 +893,81 @@ def initialize_secrets_backends( * import secrets backend classes * instantiate them and return them in a list """ - backend_list = [] worker_mode = False - if default_backends != DEFAULT_SECRETS_SEARCH_PATH: + search_section = "secrets" + environment_variable_args: str | None = ( + "airflow.secrets.environment_variables.EnvironmentVariablesBackend" + ) + metastore_args: str | None = "airflow.secrets.metastore.MetastoreBackend" + execution_args: str | None = None + + if default_backends is not None: worker_mode = True + search_section = "workers" + environment_variable_args = ( + environment_variable_args if environment_variable_args in default_backends else None + ) + metastore_args = metastore_args if metastore_args in default_backends else None + execution_args = ( + "airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend" + if "airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend" + in default_backends + else None + ) + + backends_map: dict[str, dict[str, Any]] = { + "environment_variable": { + "callback": get_importable_secret_backend, + "args": (environment_variable_args,), + }, + "metastore": { + "callback": get_importable_secret_backend, + "args": (metastore_args,), + }, + "custom": { + "callback": get_custom_secret_backend, + "args": (worker_mode,), + }, + "execution_api": { + "callback": get_importable_secret_backend, + "args": (execution_args,), + }, + } - custom_secret_backend = get_custom_secret_backend(worker_mode) + backends_order = conf.getlist(search_section, "backends_order", delimiter=",") - if custom_secret_backend is not None: - from airflow.models import Connection + required_backends = ( + [Backends.ENVIRONMENT_VARIABLE, Backends.EXECUTION_API] + if worker_mode + else [Backends.METASTORE, Backends.ENVIRONMENT_VARIABLE] + ) - custom_secret_backend._set_connection_class(Connection) - backend_list.append(custom_secret_backend) + if missing_backends := [b.value for b in required_backends if b.value not in backends_order]: + raise AirflowConfigException( + f"The configuration option [{search_section}]backends_order is misconfigured. " + f"The following backend types are missing: {missing_backends}", + search_section, + missing_backends, + ) - for class_name in default_backends: - from airflow.models import Connection + if unsupported_backends := [b for b in backends_order if b not in backends_map.keys()]: + raise AirflowConfigException( + f"The configuration option [{search_section}]backends_order is misconfigured. " + f"The following backend types are unsupported: {unsupported_backends}", + search_section, + unsupported_backends, + ) - secrets_backend_cls = import_string(class_name) - backend = secrets_backend_cls() - backend._set_connection_class(Connection) - backend_list.append(backend) + backend_list = [] + for backend_type in backends_order: + backend_item = backends_map[backend_type] + callback, args = backend_item["callback"], backend_item["args"] + backend = callback(*args) if args else callback() + if backend: + from airflow.models import Connection + + backend._set_connection_class(Connection) + backend_list.append(backend) return backend_list diff --git a/airflow-core/src/airflow/secrets/__init__.py b/airflow-core/src/airflow/secrets/__init__.py index f9b8e20ba0ce2..91a7ea4191e4f 100644 --- a/airflow-core/src/airflow/secrets/__init__.py +++ b/airflow-core/src/airflow/secrets/__init__.py @@ -29,12 +29,11 @@ from airflow.utils.deprecation_tools import add_deprecated_classes -__all__ = ["BaseSecretsBackend", "DEFAULT_SECRETS_SEARCH_PATH"] +__all__ = [ + "BaseSecretsBackend", +] -from airflow.secrets.base_secrets import ( - DEFAULT_SECRETS_SEARCH_PATH as DEFAULT_SECRETS_SEARCH_PATH, - BaseSecretsBackend, -) +from airflow.secrets.base_secrets import BaseSecretsBackend __deprecated_classes = { "cache": { diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index d74622f5a8ca2..bb725bcf4fe53 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -233,6 +233,12 @@ export type ConfigServiceGetConfigsDefaultResponse = Awaited = UseQueryResult; export const useConfigServiceGetConfigsKey = "ConfigServiceGetConfigs"; export const UseConfigServiceGetConfigsKeyFn = (queryKey?: Array) => [useConfigServiceGetConfigsKey, ...(queryKey ?? [])]; +export type ConfigServiceGetBackendsOrderValueDefaultResponse = Awaited>; +export type ConfigServiceGetBackendsOrderValueQueryResult = UseQueryResult; +export const useConfigServiceGetBackendsOrderValueKey = "ConfigServiceGetBackendsOrderValue"; +export const UseConfigServiceGetBackendsOrderValueKeyFn = ({ accept }: { + accept?: "application/json" | "text/plain" | "*/*"; +} = {}, queryKey?: Array) => [useConfigServiceGetBackendsOrderValueKey, ...(queryKey ?? [{ accept }])]; export type DagWarningServiceListDagWarningsDefaultResponse = Awaited>; export type DagWarningServiceListDagWarningsQueryResult = UseQueryResult; export const useDagWarningServiceListDagWarningsKey = "DagWarningServiceListDagWarnings"; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index 8ccaf76a47a83..65003a2d6d934 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -435,6 +435,16 @@ export const ensureUseConfigServiceGetConfigValueData = (queryClient: QueryClien */ export const ensureUseConfigServiceGetConfigsData = (queryClient: QueryClient) => queryClient.ensureQueryData({ queryKey: Common.UseConfigServiceGetConfigsKeyFn(), queryFn: () => ConfigService.getConfigs() }); /** +* Get Backends Order Value +* @param data The data for the request. +* @param data.accept +* @returns Config Successful Response +* @throws ApiError +*/ +export const ensureUseConfigServiceGetBackendsOrderValueData = (queryClient: QueryClient, { accept }: { + accept?: "application/json" | "text/plain" | "*/*"; +} = {}) => queryClient.ensureQueryData({ queryKey: Common.UseConfigServiceGetBackendsOrderValueKeyFn({ accept }), queryFn: () => ConfigService.getBackendsOrderValue({ accept }) }); +/** * List Dag Warnings * Get a list of DAG warnings. * @param data The data for the request. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index 57c4174683dfa..d230c3342bc1e 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -435,6 +435,16 @@ export const prefetchUseConfigServiceGetConfigValue = (queryClient: QueryClient, */ export const prefetchUseConfigServiceGetConfigs = (queryClient: QueryClient) => queryClient.prefetchQuery({ queryKey: Common.UseConfigServiceGetConfigsKeyFn(), queryFn: () => ConfigService.getConfigs() }); /** +* Get Backends Order Value +* @param data The data for the request. +* @param data.accept +* @returns Config Successful Response +* @throws ApiError +*/ +export const prefetchUseConfigServiceGetBackendsOrderValue = (queryClient: QueryClient, { accept }: { + accept?: "application/json" | "text/plain" | "*/*"; +} = {}) => queryClient.prefetchQuery({ queryKey: Common.UseConfigServiceGetBackendsOrderValueKeyFn({ accept }), queryFn: () => ConfigService.getBackendsOrderValue({ accept }) }); +/** * List Dag Warnings * Get a list of DAG warnings. * @param data The data for the request. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index 33a5ca388165a..9ed1e725495c5 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -435,6 +435,16 @@ export const useConfigServiceGetConfigValue = = unknown[]>(queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseConfigServiceGetConfigsKeyFn(queryKey), queryFn: () => ConfigService.getConfigs() as TData, ...options }); /** +* Get Backends Order Value +* @param data The data for the request. +* @param data.accept +* @returns Config Successful Response +* @throws ApiError +*/ +export const useConfigServiceGetBackendsOrderValue = = unknown[]>({ accept }: { + accept?: "application/json" | "text/plain" | "*/*"; +} = {}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseConfigServiceGetBackendsOrderValueKeyFn({ accept }, queryKey), queryFn: () => ConfigService.getBackendsOrderValue({ accept }) as TData, ...options }); +/** * List Dag Warnings * Get a list of DAG warnings. * @param data The data for the request. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index 2d01abdb58f4f..738541e887733 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -435,6 +435,16 @@ export const useConfigServiceGetConfigValueSuspense = = unknown[]>(queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseConfigServiceGetConfigsKeyFn(queryKey), queryFn: () => ConfigService.getConfigs() as TData, ...options }); /** +* Get Backends Order Value +* @param data The data for the request. +* @param data.accept +* @returns Config Successful Response +* @throws ApiError +*/ +export const useConfigServiceGetBackendsOrderValueSuspense = = unknown[]>({ accept }: { + accept?: "application/json" | "text/plain" | "*/*"; +} = {}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseConfigServiceGetBackendsOrderValueKeyFn({ accept }, queryKey), queryFn: () => ConfigService.getBackendsOrderValue({ accept }) as TData, ...options }); +/** * List Dag Warnings * Get a list of DAG warnings. * @param data The data for the request. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index d36012e372c2a..5d1c55b491385 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, DeleteDagAssetQueuedEventsData, DeleteDagAssetQueuedEventsResponse, GetDagAssetQueuedEventData, GetDagAssetQueuedEventResponse, DeleteDagAssetQueuedEventData, DeleteDagAssetQueuedEventResponse, NextRunAssetsData, NextRunAssetsResponse, ListBackfillsData, ListBackfillsResponse, CreateBackfillData, CreateBackfillResponse, GetBackfillData, GetBackfillResponse, PauseBackfillData, PauseBackfillResponse, UnpauseBackfillData, UnpauseBackfillResponse, CancelBackfillData, CancelBackfillResponse, CreateBackfillDryRunData, CreateBackfillDryRunResponse, ListBackfillsUiData, ListBackfillsUiResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, PatchConnectionData, PatchConnectionResponse, GetConnectionsData, GetConnectionsResponse, PostConnectionData, PostConnectionResponse, BulkConnectionsData, BulkConnectionsResponse, TestConnectionData, TestConnectionResponse, CreateDefaultConnectionsResponse, HookMetaDataResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, PatchDagRunData, PatchDagRunResponse, GetUpstreamAssetEventsData, GetUpstreamAssetEventsResponse, ClearDagRunData, ClearDagRunResponse, GetDagRunsData, GetDagRunsResponse, TriggerDagRunData, TriggerDagRunResponse, WaitDagRunUntilFinishedData, WaitDagRunUntilFinishedResponse, GetListDagRunsBatchData, GetListDagRunsBatchResponse, GetDagSourceData, GetDagSourceResponse, GetDagStatsData, GetDagStatsResponse, GetConfigData, GetConfigResponse, GetConfigValueData, GetConfigValueResponse, GetConfigsResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, GetDagsResponse, PatchDagsData, PatchDagsResponse, GetDagData, GetDagResponse, PatchDagData, PatchDagResponse, DeleteDagData, DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, FavoriteDagData, FavoriteDagResponse, UnfavoriteDagData, UnfavoriteDagResponse, GetDagTagsData, GetDagTagsResponse, GetDagsUiData, GetDagsUiResponse, GetLatestRunInfoData, GetLatestRunInfoResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, GetEventLogsResponse, GetExtraLinksData, GetExtraLinksResponse, GetTaskInstanceData, GetTaskInstanceResponse, PatchTaskInstanceData, PatchTaskInstanceResponse, DeleteTaskInstanceData, DeleteTaskInstanceResponse, GetMappedTaskInstancesData, GetMappedTaskInstancesResponse, GetTaskInstanceDependenciesByMapIndexData, GetTaskInstanceDependenciesByMapIndexResponse, GetTaskInstanceDependenciesData, GetTaskInstanceDependenciesResponse, GetTaskInstanceTriesData, GetTaskInstanceTriesResponse, GetMappedTaskInstanceTriesData, GetMappedTaskInstanceTriesResponse, GetMappedTaskInstanceData, GetMappedTaskInstanceResponse, PatchTaskInstanceByMapIndexData, PatchTaskInstanceByMapIndexResponse, GetTaskInstancesData, GetTaskInstancesResponse, BulkTaskInstancesData, BulkTaskInstancesResponse, GetTaskInstancesBatchData, GetTaskInstancesBatchResponse, GetTaskInstanceTryDetailsData, GetTaskInstanceTryDetailsResponse, GetMappedTaskInstanceTryDetailsData, GetMappedTaskInstanceTryDetailsResponse, PostClearTaskInstancesData, PostClearTaskInstancesResponse, PatchTaskInstanceDryRunByMapIndexData, PatchTaskInstanceDryRunByMapIndexResponse, PatchTaskInstanceDryRunData, PatchTaskInstanceDryRunResponse, GetLogData, GetLogResponse, GetExternalLogUrlData, GetExternalLogUrlResponse, UpdateHitlDetailData, UpdateHitlDetailResponse, GetHitlDetailData, GetHitlDetailResponse, GetHitlDetailTryDetailData, GetHitlDetailTryDetailResponse, GetHitlDetailsData, GetHitlDetailsResponse, GetImportErrorData, GetImportErrorResponse, GetImportErrorsData, GetImportErrorsResponse, GetJobsData, GetJobsResponse, GetPluginsData, GetPluginsResponse, ImportErrorsResponse, DeletePoolData, DeletePoolResponse, GetPoolData, GetPoolResponse, PatchPoolData, PatchPoolResponse, GetPoolsData, GetPoolsResponse, PostPoolData, PostPoolResponse, BulkPoolsData, BulkPoolsResponse, GetProvidersData, GetProvidersResponse, GetXcomEntryData, GetXcomEntryResponse, UpdateXcomEntryData, UpdateXcomEntryResponse, DeleteXcomEntryData, DeleteXcomEntryResponse, GetXcomEntriesData, GetXcomEntriesResponse, CreateXcomEntryData, CreateXcomEntryResponse, GetTasksData, GetTasksResponse, GetTaskData, GetTaskResponse, DeleteVariableData, DeleteVariableResponse, GetVariableData, GetVariableResponse, PatchVariableData, PatchVariableResponse, GetVariablesData, GetVariablesResponse, PostVariableData, PostVariableResponse, BulkVariablesData, BulkVariablesResponse, ReparseDagFileData, ReparseDagFileResponse, GetDagVersionData, GetDagVersionResponse, GetDagVersionsData, GetDagVersionsResponse, GetHealthResponse, GetVersionResponse, LoginData, LoginResponse, LogoutResponse, GetAuthMenusResponse, GetCurrentUserInfoResponse, GenerateTokenData, GenerateTokenResponse2, GetPartitionedDagRunsData, GetPartitionedDagRunsResponse, GetPendingPartitionedDagRunData, GetPendingPartitionedDagRunResponse, GetDependenciesData, GetDependenciesResponse, HistoricalMetricsData, HistoricalMetricsResponse, DagStatsResponse2, GetDagRunDeadlinesData, GetDagRunDeadlinesResponse, StructureDataData, StructureDataResponse2, GetDagStructureData, GetDagStructureResponse, GetGridRunsData, GetGridRunsResponse, GetGridTiSummariesData, GetGridTiSummariesResponse, GetGanttDataData, GetGanttDataResponse, GetCalendarData, GetCalendarResponse, ListTeamsData, ListTeamsResponse } from './types.gen'; +import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, DeleteDagAssetQueuedEventsData, DeleteDagAssetQueuedEventsResponse, GetDagAssetQueuedEventData, GetDagAssetQueuedEventResponse, DeleteDagAssetQueuedEventData, DeleteDagAssetQueuedEventResponse, NextRunAssetsData, NextRunAssetsResponse, ListBackfillsData, ListBackfillsResponse, CreateBackfillData, CreateBackfillResponse, GetBackfillData, GetBackfillResponse, PauseBackfillData, PauseBackfillResponse, UnpauseBackfillData, UnpauseBackfillResponse, CancelBackfillData, CancelBackfillResponse, CreateBackfillDryRunData, CreateBackfillDryRunResponse, ListBackfillsUiData, ListBackfillsUiResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, PatchConnectionData, PatchConnectionResponse, GetConnectionsData, GetConnectionsResponse, PostConnectionData, PostConnectionResponse, BulkConnectionsData, BulkConnectionsResponse, TestConnectionData, TestConnectionResponse, CreateDefaultConnectionsResponse, HookMetaDataResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, PatchDagRunData, PatchDagRunResponse, GetUpstreamAssetEventsData, GetUpstreamAssetEventsResponse, ClearDagRunData, ClearDagRunResponse, GetDagRunsData, GetDagRunsResponse, TriggerDagRunData, TriggerDagRunResponse, WaitDagRunUntilFinishedData, WaitDagRunUntilFinishedResponse, GetListDagRunsBatchData, GetListDagRunsBatchResponse, GetDagSourceData, GetDagSourceResponse, GetDagStatsData, GetDagStatsResponse, GetConfigData, GetConfigResponse, GetConfigValueData, GetConfigValueResponse, GetConfigsResponse, GetBackendsOrderValueData, GetBackendsOrderValueResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, GetDagsResponse, PatchDagsData, PatchDagsResponse, GetDagData, GetDagResponse, PatchDagData, PatchDagResponse, DeleteDagData, DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, FavoriteDagData, FavoriteDagResponse, UnfavoriteDagData, UnfavoriteDagResponse, GetDagTagsData, GetDagTagsResponse, GetDagsUiData, GetDagsUiResponse, GetLatestRunInfoData, GetLatestRunInfoResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, GetEventLogsResponse, GetExtraLinksData, GetExtraLinksResponse, GetTaskInstanceData, GetTaskInstanceResponse, PatchTaskInstanceData, PatchTaskInstanceResponse, DeleteTaskInstanceData, DeleteTaskInstanceResponse, GetMappedTaskInstancesData, GetMappedTaskInstancesResponse, GetTaskInstanceDependenciesByMapIndexData, GetTaskInstanceDependenciesByMapIndexResponse, GetTaskInstanceDependenciesData, GetTaskInstanceDependenciesResponse, GetTaskInstanceTriesData, GetTaskInstanceTriesResponse, GetMappedTaskInstanceTriesData, GetMappedTaskInstanceTriesResponse, GetMappedTaskInstanceData, GetMappedTaskInstanceResponse, PatchTaskInstanceByMapIndexData, PatchTaskInstanceByMapIndexResponse, GetTaskInstancesData, GetTaskInstancesResponse, BulkTaskInstancesData, BulkTaskInstancesResponse, GetTaskInstancesBatchData, GetTaskInstancesBatchResponse, GetTaskInstanceTryDetailsData, GetTaskInstanceTryDetailsResponse, GetMappedTaskInstanceTryDetailsData, GetMappedTaskInstanceTryDetailsResponse, PostClearTaskInstancesData, PostClearTaskInstancesResponse, PatchTaskInstanceDryRunByMapIndexData, PatchTaskInstanceDryRunByMapIndexResponse, PatchTaskInstanceDryRunData, PatchTaskInstanceDryRunResponse, GetLogData, GetLogResponse, GetExternalLogUrlData, GetExternalLogUrlResponse, UpdateHitlDetailData, UpdateHitlDetailResponse, GetHitlDetailData, GetHitlDetailResponse, GetHitlDetailTryDetailData, GetHitlDetailTryDetailResponse, GetHitlDetailsData, GetHitlDetailsResponse, GetImportErrorData, GetImportErrorResponse, GetImportErrorsData, GetImportErrorsResponse, GetJobsData, GetJobsResponse, GetPluginsData, GetPluginsResponse, ImportErrorsResponse, DeletePoolData, DeletePoolResponse, GetPoolData, GetPoolResponse, PatchPoolData, PatchPoolResponse, GetPoolsData, GetPoolsResponse, PostPoolData, PostPoolResponse, BulkPoolsData, BulkPoolsResponse, GetProvidersData, GetProvidersResponse, GetXcomEntryData, GetXcomEntryResponse, UpdateXcomEntryData, UpdateXcomEntryResponse, DeleteXcomEntryData, DeleteXcomEntryResponse, GetXcomEntriesData, GetXcomEntriesResponse, CreateXcomEntryData, CreateXcomEntryResponse, GetTasksData, GetTasksResponse, GetTaskData, GetTaskResponse, DeleteVariableData, DeleteVariableResponse, GetVariableData, GetVariableResponse, PatchVariableData, PatchVariableResponse, GetVariablesData, GetVariablesResponse, PostVariableData, PostVariableResponse, BulkVariablesData, BulkVariablesResponse, ReparseDagFileData, ReparseDagFileResponse, GetDagVersionData, GetDagVersionResponse, GetDagVersionsData, GetDagVersionsResponse, GetHealthResponse, GetVersionResponse, LoginData, LoginResponse, LogoutResponse, GetAuthMenusResponse, GetCurrentUserInfoResponse, GenerateTokenData, GenerateTokenResponse2, GetPartitionedDagRunsData, GetPartitionedDagRunsResponse, GetPendingPartitionedDagRunData, GetPendingPartitionedDagRunResponse, GetDependenciesData, GetDependenciesResponse, HistoricalMetricsData, HistoricalMetricsResponse, DagStatsResponse2, GetDagRunDeadlinesData, GetDagRunDeadlinesResponse, StructureDataData, StructureDataResponse2, GetDagStructureData, GetDagStructureResponse, GetGridRunsData, GetGridRunsResponse, GetGridTiSummariesData, GetGridTiSummariesResponse, GetGanttDataData, GetGanttDataResponse, GetCalendarData, GetCalendarResponse, ListTeamsData, ListTeamsResponse } from './types.gen'; export class AssetService { /** @@ -1332,6 +1332,28 @@ export class ConfigService { }); } + /** + * Get Backends Order Value + * @param data The data for the request. + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ + public static getBackendsOrderValue(data: GetBackendsOrderValueData = {}): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/ui/backends_order', + headers: { + accept: data.accept + }, + errors: { + 404: 'Not Found', + 406: 'Not Acceptable', + 422: 'Validation Error' + } + }); + } + } export class DagWarningService { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 038c02ee2cd0b..4f4f3e603930d 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -2651,6 +2651,12 @@ export type GetConfigValueResponse = Config; export type GetConfigsResponse = ConfigResponse; +export type GetBackendsOrderValueData = { + accept?: 'application/json' | 'text/plain' | '*/*'; +}; + +export type GetBackendsOrderValueResponse = Config; + export type ListDagWarningsData = { dagId?: string | null; limit?: number; @@ -4894,6 +4900,29 @@ export type $OpenApiTs = { }; }; }; + '/ui/backends_order': { + get: { + req: GetBackendsOrderValueData; + res: { + /** + * Successful Response + */ + 200: Config; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Not Acceptable + */ + 406: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; '/api/v2/dagWarnings': { get: { req: ListDagWarningsData; diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/admin.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/admin.json index 1f0136b906d25..b3ef07115d25d 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/admin.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/admin.json @@ -140,6 +140,7 @@ }, "variables": { "add": "Add Variable", + "backendsOrder": "Secret backends order", "columns": { "isEncrypted": "Is Encrypted" }, diff --git a/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderButton.tsx b/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderButton.tsx new file mode 100644 index 0000000000000..03f033c98f3f8 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderButton.tsx @@ -0,0 +1,76 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, HStack, Skeleton, Text } from "@chakra-ui/react"; +import { FiChevronRight } from "react-icons/fi"; +import { Link as RouterLink } from "react-router-dom"; + +import type { TaskInstanceState } from "openapi/requests/types.gen"; +import { StateBadge } from "src/components/StateBadge"; + +export const BackendsOrderButton = ({ + colorScheme, + icon, + isLoading = false, + label, + link, + onClick, + state, +}: { + readonly colorScheme: string; + readonly icon?: React.ReactNode; + readonly isLoading?: boolean; + readonly label: string; + readonly link?: string; + readonly onClick?: () => void; + readonly state?: TaskInstanceState | null; +}) => { + if (isLoading) { + return ; + } + + const content = ( + + + {icon} + + + + {label} + + + + ); + + if (onClick) { + return ( + + {content} + + ); + } + + return {content}; +}; diff --git a/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderCard.tsx b/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderCard.tsx new file mode 100644 index 0000000000000..8988190ae4ecf --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderCard.tsx @@ -0,0 +1,42 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, useDisclosure } from "@chakra-ui/react"; +import { useTranslation } from "react-i18next"; +import { LuSettings } from "react-icons/lu"; + +import { BackendsOrderButton } from "src/pages/Variables/BackendsOrderButton"; +import { BackendsOrderModal } from "src/pages/Variables/BackendsOrderModal"; + +export const BackendsOrderCard = () => { + const { t: translate } = useTranslation("admin"); + const { onClose, onOpen, open } = useDisclosure(); + + return ( + + } + isLoading={false} + label={translate("variables.backendsOrder")} + onClick={onOpen} + /> + + + ); +}; diff --git a/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderModal.tsx b/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderModal.tsx new file mode 100644 index 0000000000000..bb5fa993945df --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Variables/BackendsOrderModal.tsx @@ -0,0 +1,67 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Heading, Text, HStack } from "@chakra-ui/react"; +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { LuSettings } from "react-icons/lu"; + +import { useConfigServiceGetBackendsOrderValue } from "openapi/queries"; +import { ErrorAlert } from "src/components/ErrorAlert"; +import { Dialog } from "src/components/ui"; + +type BackendsOrderModalProps = { + onClose: () => void; + open: boolean; +}; + +export const BackendsOrderModal: React.FC = ({ onClose, open }) => { + const { t: translate } = useTranslation("admin"); + const [backendsOrder, setBackendsOrder] = useState | string>(); + const { data, error } = useConfigServiceGetBackendsOrderValue(); + + const onOpenChange = () => { + onClose(); + }; + + useEffect(() => { + setBackendsOrder(data?.sections[0]?.options[0]?.value ?? ""); + }, [data, open]); + + return ( + + + + + + {translate("variables.backendsOrder")} + + + + + + + {Boolean(error) ? : null} + + {backendsOrder} + + + + + ); +}; diff --git a/airflow-core/src/airflow/ui/src/pages/Variables/Variables.tsx b/airflow-core/src/airflow/ui/src/pages/Variables/Variables.tsx index 053b432e106c5..4929f632948fa 100644 --- a/airflow-core/src/airflow/ui/src/pages/Variables/Variables.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Variables/Variables.tsx @@ -38,6 +38,7 @@ import { SearchParamsKeys, type SearchParamsKeysType } from "src/constants/searc import { useConfig } from "src/queries/useConfig.tsx"; import { TrimText } from "src/utils/TrimText"; +import { BackendsOrderCard } from "./BackendsOrderCard"; import DeleteVariablesButton from "./DeleteVariablesButton"; import ImportVariablesButton from "./ImportVariablesButton"; import AddVariableButton from "./ManageVariable/AddVariableButton"; @@ -207,6 +208,9 @@ export const Variables = () => { placeholder={translate("variables.searchPlaceholder")} /> + 0} /> + + Any | None: + """Get secret backend defined in the given class name.""" + from airflow.sdk._shared.module_loading import import_string + + if class_name is not None: + secrets_backend_cls = import_string(class_name) + return secrets_backend_cls() + return None + + def initialize_secrets_backends( - default_backends: list[str] = _SERVER_DEFAULT_SECRETS_SEARCH_PATH, + default_backends: list[str] | None = None, ): """ Initialize secrets backend. @@ -215,36 +235,92 @@ def initialize_secrets_backends( Uses SDK's conf instead of Core's conf. """ - from airflow.sdk._shared.module_loading import import_string - - backend_list = [] - worker_mode = False # Determine worker mode - if default_backends is not the server default, it's worker mode # This is a simplified check; in practice, worker mode is determined by the caller - if default_backends != _SERVER_DEFAULT_SECRETS_SEARCH_PATH: + from airflow.sdk.configuration import conf + + worker_mode = False + search_section = "secrets" + environment_variable_args: str | None = ( + "airflow.secrets.environment_variables.EnvironmentVariablesBackend" + ) + metastore_args: str | None = "airflow.secrets.metastore.MetastoreBackend" + execution_args: str | None = None + + if default_backends is not None: worker_mode = True + search_section = "workers" + environment_variable_args = ( + environment_variable_args if environment_variable_args in default_backends else None + ) + metastore_args = metastore_args if metastore_args in default_backends else None + execution_args = ( + "airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend" + if "airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend" + in default_backends + else None + ) + + backends_map: dict[str, dict[str, Any]] = { + "environment_variable": { + "callback": get_importable_secret_backend, + "args": (environment_variable_args,), + }, + "metastore": { + "callback": get_importable_secret_backend, + "args": (metastore_args,), + }, + "custom": { + "callback": get_custom_secret_backend, + "args": (worker_mode,), + }, + "execution_api": { + "callback": get_importable_secret_backend, + "args": (execution_args,), + }, + } - custom_secret_backend = get_custom_secret_backend(worker_mode) + backends_order = conf.getlist(search_section, "backends_order", delimiter=",") - if custom_secret_backend is not None: - from airflow.sdk.definitions.connection import Connection + required_backends = ( + [Backends.ENVIRONMENT_VARIABLE, Backends.EXECUTION_API] + if worker_mode + else [Backends.METASTORE, Backends.ENVIRONMENT_VARIABLE] + ) - custom_secret_backend._set_connection_class(Connection) - backend_list.append(custom_secret_backend) + if missing_backends := [b.value for b in required_backends if b.value not in backends_order]: + raise AirflowConfigException( + f"The configuration option [{search_section}]backends_order is misconfigured. " + f"The following backend types are missing: {missing_backends}", + search_section, + missing_backends, + ) + + if unsupported_backends := [b for b in backends_order if b not in backends_map.keys()]: + raise AirflowConfigException( + f"The configuration option [{search_section}]backends_order is misconfigured. " + f"The following backend types are unsupported: {unsupported_backends}", + search_section, + unsupported_backends, + ) - for class_name in default_backends: - from airflow.sdk.definitions.connection import Connection + backend_list = [] - secrets_backend_cls = import_string(class_name) - backend = secrets_backend_cls() - backend._set_connection_class(Connection) - backend_list.append(backend) + for backend_type in backends_order: + backend_item = backends_map[backend_type] + callback, args = backend_item["callback"], backend_item["args"] + backend = callback(*args) if args else callback() + if backend: + from airflow.sdk.definitions.connection import Connection + + backend._set_connection_class(Connection) + backend_list.append(backend) return backend_list def ensure_secrets_loaded( - default_backends: list[str] = _SERVER_DEFAULT_SECRETS_SEARCH_PATH, + default_backends: list[str] | None = None, ) -> list: """ Ensure that all secrets backends are loaded. @@ -253,10 +329,9 @@ def ensure_secrets_loaded( """ # Check if the secrets_backend_list contains only 2 default backends. - # Check if we are loading the backends for worker too by checking if the default_backends is equal - # to _SERVER_DEFAULT_SECRETS_SEARCH_PATH. + # Check if we are loading the backends for worker too by checking if the default_backends is not None secrets_backend_list = initialize_secrets_backends() - if len(secrets_backend_list) == 2 or default_backends != _SERVER_DEFAULT_SECRETS_SEARCH_PATH: + if len(secrets_backend_list) == 2 or default_backends is not None: return initialize_secrets_backends(default_backends=default_backends) return secrets_backend_list diff --git a/task-sdk/tests/task_sdk/execution_time/test_context.py b/task-sdk/tests/task_sdk/execution_time/test_context.py index a2c3310822b5f..3a058f11abf72 100644 --- a/task-sdk/tests/task_sdk/execution_time/test_context.py +++ b/task-sdk/tests/task_sdk/execution_time/test_context.py @@ -936,16 +936,6 @@ def test_execution_api_backend_in_worker_chain(self): in DEFAULT_SECRETS_SEARCH_PATH_WORKERS ) - def test_metastore_backend_in_server_chain(self): - """Test that MetastoreBackend is in the API server search path.""" - from airflow.sdk.execution_time.secrets import _SERVER_DEFAULT_SECRETS_SEARCH_PATH - - assert "airflow.secrets.metastore.MetastoreBackend" in _SERVER_DEFAULT_SECRETS_SEARCH_PATH - assert ( - "airflow.sdk.execution_time.secrets.execution_api.ExecutionAPISecretsBackend" - not in _SERVER_DEFAULT_SECRETS_SEARCH_PATH - ) - def test_get_connection_uses_backend_chain(self, mock_supervisor_comms): """Test that _get_connection properly iterates through backends.""" from airflow.sdk.api.datamodels._generated import ConnectionResponse