diff --git a/poetry.lock b/poetry.lock index eb623a130..ecd1af293 100644 --- a/poetry.lock +++ b/poetry.lock @@ -597,6 +597,21 @@ pytest = ">=6.2.5" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "pyyaml" version = "6.0.2" @@ -954,4 +969,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.13" -content-hash = "1c87af20b623a70bb2ce19cb6051568cce5a4f6d2d7f996799ff0400fac50bbf" +content-hash = "be2f500ed6d1e0968c6aa0fea3512e7347d60632ec303ad3c1e8de8db6e490db" diff --git a/pyproject.toml b/pyproject.toml index 5d15d9b93..b347dabc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ astor = ">=0.8.1" shortuuid = ">=1.0.11" dacite = ">=1.8.1" deprecated = ">=1.2.14" +python-dateutil = "^2.8.2" [tool.poetry.group.dev.dependencies] pylint = ">=2.17.5" @@ -120,6 +121,14 @@ ignore = [ "W191", "E501", "B011", + # too-many-arguments + "PLR0913", + # collapsible-else-if + "PLR5501", + # too-many-branches + "PLR0912", + # too-many-return-statements + "PLR0911" ] [tool.ruff.lint.isort] diff --git a/src/conductor/client/ai/orchestrator.py b/src/conductor/client/ai/orchestrator.py index 5d63f9de6..fe81dde29 100644 --- a/src/conductor/client/ai/orchestrator.py +++ b/src/conductor/client/ai/orchestrator.py @@ -1,19 +1,22 @@ from __future__ import annotations -from typing import Optional, List +from typing import Optional, List, TYPE_CHECKING from uuid import uuid4 from typing_extensions import Self -from conductor.client.ai.configuration import LLMProvider, VectorDB -from conductor.client.ai.integrations import IntegrationConfig -from conductor.client.configuration.configuration import Configuration from conductor.client.http.models.integration_api_update import IntegrationApiUpdate from conductor.client.http.models.integration_update import IntegrationUpdate -from conductor.client.http.models.prompt_template import PromptTemplate from conductor.client.http.rest import ApiException from conductor.client.orkes_clients import OrkesClients +if TYPE_CHECKING: + from conductor.client.http.models.prompt_template import PromptTemplate + from conductor.client.configuration.configuration import Configuration + from conductor.client.ai.integrations import IntegrationConfig + from conductor.client.ai.configuration import LLMProvider, VectorDB + +NOT_FOUND_STATUS = 404 class AIOrchestrator: def __init__(self, api_configuration: Configuration, prompt_test_workflow_name: str = '') -> Self: @@ -36,7 +39,7 @@ def get_prompt_template(self, template_name: str) -> PromptTemplate: try: return self.prompt_client.get_prompt(template_name) except ApiException as e: - if e.code == 404: + if e.code == NOT_FOUND_STATUS: return None raise e @@ -93,7 +96,6 @@ def add_vector_store(self, db_integration_name: str, provider: VectorDB, indices existing_integration_api = self.integration_client.get_integration_api(db_integration_name, index) if existing_integration_api is None or overwrite: self.integration_client.save_integration_api(db_integration_name, index, api_details) - pass def get_token_used(self, ai_integration: str) -> dict: return self.integration_client.get_token_usage_for_integration_provider(ai_integration) diff --git a/src/conductor/client/automator/task_handler.py b/src/conductor/client/automator/task_handler.py index 796a51aa7..932a7cc43 100644 --- a/src/conductor/client/automator/task_handler.py +++ b/src/conductor/client/automator/task_handler.py @@ -66,8 +66,7 @@ def __init__( elif not isinstance(workers, list): workers = [workers] if scan_for_annotated_workers is True: - for (task_def_name, domain) in _decorated_functions: - record = _decorated_functions[(task_def_name, domain)] + for (task_def_name, domain), record in _decorated_functions.items(): fn = record['func'] worker_id = record['worker_id'] poll_interval = record['poll_interval'] diff --git a/src/conductor/client/automator/utils.py b/src/conductor/client/automator/utils.py index 7d0029351..0914c6613 100644 --- a/src/conductor/client/automator/utils.py +++ b/src/conductor/client/automator/utils.py @@ -109,10 +109,7 @@ def get_value(typ: type, val: object) -> object: if typ in simple_types: return val elif str(typ).startswith('typing.List[') or str(typ).startswith('typing.Set[') or str(typ).startswith('list['): - values = [] - for item in val: - converted = get_value(type(item), item) - values.append(converted) + values = [get_value(type(item), item) for item in val] return values elif str(typ).startswith('dict[') or str(typ).startswith( 'typing.Dict[') or str(typ).startswith('requests.structures.CaseInsensitiveDict[') or typ is dict: diff --git a/src/conductor/client/configuration/settings/metrics_settings.py b/src/conductor/client/configuration/settings/metrics_settings.py index e450d053c..622aaccee 100644 --- a/src/conductor/client/configuration/settings/metrics_settings.py +++ b/src/conductor/client/configuration/settings/metrics_settings.py @@ -35,5 +35,5 @@ def __set_dir(self, dir: str) -> None: os.mkdir(dir) except Exception as e: logger.warning( - 'Failed to create metrics temporary folder, reason: ', e) + 'Failed to create metrics temporary folder, reason: %s', e) self.directory = dir diff --git a/src/conductor/client/exceptions/api_exception_handler.py b/src/conductor/client/exceptions/api_exception_handler.py index e3d0e8b36..27a9ee087 100644 --- a/src/conductor/client/exceptions/api_exception_handler.py +++ b/src/conductor/client/exceptions/api_exception_handler.py @@ -3,12 +3,18 @@ from conductor.client.exceptions.api_error import APIError, APIErrorCode from conductor.client.http.rest import ApiException +BAD_REQUEST_STATUS = 400 +FORBIDDEN_STATUS = 403 +NOT_FOUND_STATUS = 404 +REQUEST_TIMEOUT_STATUS = 408 +CONFLICT_STATUS = 409 + STATUS_TO_MESSAGE_DEFAULT_MAPPING = { - 400: "Invalid request", - 403: "Access forbidden", - 404: "Resource not found", - 408: "Request timed out", - 409: "Resource exists already", + BAD_REQUEST_STATUS: "Invalid request", + FORBIDDEN_STATUS: "Access forbidden", + NOT_FOUND_STATUS: "Resource not found", + REQUEST_TIMEOUT_STATUS: "Request timed out", + CONFLICT_STATUS: "Resource exists already", } @@ -18,20 +24,20 @@ def inner_function(*args, **kwargs): return function(*args, **kwargs) except ApiException as e: - if e.status == 404: + if e.status == NOT_FOUND_STATUS: code = APIErrorCode.NOT_FOUND - elif e.status == 403: + elif e.status == FORBIDDEN_STATUS: code = APIErrorCode.FORBIDDEN - elif e.status == 409: + elif e.status == CONFLICT_STATUS: code = APIErrorCode.CONFLICT - elif e.status == 400: + elif e.status == BAD_REQUEST_STATUS: code = APIErrorCode.BAD_REQUEST - elif e.status == 408: + elif e.status == REQUEST_TIMEOUT_STATUS: code = APIErrorCode.REQUEST_TIMEOUT else: code = APIErrorCode.UNKNOWN - message = STATUS_TO_MESSAGE_DEFAULT_MAPPING[e.status] + message = STATUS_TO_MESSAGE_DEFAULT_MAPPING.get(e.status, "Unknown error") try: if e.body: diff --git a/src/conductor/client/helpers/helper.py b/src/conductor/client/helpers/helper.py index 6bea33cb2..4ff2a2d37 100644 --- a/src/conductor/client/helpers/helper.py +++ b/src/conductor/client/helpers/helper.py @@ -1,6 +1,7 @@ import datetime import logging import re +from dateutil.parser import parse from typing import Any, ClassVar, Dict, Tuple import six @@ -21,7 +22,7 @@ class ObjectMapper(object): PRIMITIVE_TYPES: ClassVar[Tuple[Any, ...]] = (float, bool, bytes, six.text_type, *six.integer_types) NATIVE_TYPES_MAPPING: ClassVar[Dict[str, Any]] = { 'int': int, - 'long': int if six.PY3 else long, # noqa: F821, RUF100 + 'long': int if six.PY3 else long, # noqa: F821, RUF100, YTT202 'float': float, 'str': str, 'bool': bool, @@ -31,39 +32,29 @@ class ObjectMapper(object): } def to_json(self, obj): - if obj is None: return None elif isinstance(obj, self.PRIMITIVE_TYPES): return obj elif isinstance(obj, list): - return [self.to_json(sub_obj) - for sub_obj in obj] + return [self.to_json(sub_obj) for sub_obj in obj] elif isinstance(obj, tuple): - return tuple(self.to_json(sub_obj) - for sub_obj in obj) + return tuple(self.to_json(sub_obj) for sub_obj in obj) elif isinstance(obj, (datetime.datetime, datetime.date)): return obj.isoformat() - - if isinstance(obj, dict) or isinstance(obj, CaseInsensitiveDict): + elif isinstance(obj, dict) or isinstance(obj, CaseInsensitiveDict): obj_dict = obj + elif hasattr(obj, 'attribute_map') and hasattr(obj, 'swagger_types'): + obj_dict = {obj.attribute_map[attr]: getattr(obj, attr) + for attr, _ in six.iteritems(obj.swagger_types) + if getattr(obj, attr) is not None} else: - # Convert model obj to dict except - # attributes `swagger_types`, `attribute_map` - # and attributes which value is not None. - # Convert attribute name to json key in - # model definition for request. - if hasattr(obj, 'attribute_map') and hasattr(obj, 'swagger_types'): - obj_dict = {obj.attribute_map[attr]: getattr(obj, attr) - for attr, _ in six.iteritems(obj.swagger_types) - if getattr(obj, attr) is not None} - else: - obj_dict = {name: getattr(obj, name) - for name in vars(obj) - if getattr(obj, name) is not None} + obj_dict = {name: getattr(obj, name) + for name in vars(obj) + if getattr(obj, name) is not None} return {key: self.to_json(val) - for key, val in six.iteritems(obj_dict)} + for key, val in six.iteritems(obj_dict)} def from_json(self, data, klass): return self.__deserialize(data, klass) @@ -134,7 +125,6 @@ def __deserialize_date(self, string): :return: date. """ try: - from dateutil.parser import parse return parse(string).date() except ImportError: return string @@ -153,7 +143,6 @@ def __deserialize_datatime(self, string): :return: datetime. """ try: - from dateutil.parser import parse return parse(string) except ImportError: return string diff --git a/src/conductor/client/integration_client.py b/src/conductor/client/integration_client.py index 8772d4e68..b9756c4d6 100644 --- a/src/conductor/client/integration_client.py +++ b/src/conductor/client/integration_client.py @@ -35,82 +35,82 @@ class IntegrationClient(ABC): @abstractmethod def associate_prompt_with_integration(self, ai_integration:str, model_name:str, prompt_name:str): """Associate a prompt with an AI integration and model""" - pass + ... @abstractmethod def delete_integration_api(self, api_name:str, integration_name:str): """Delete a specific integration api for a given integration""" - pass + ... @abstractmethod def delete_integration(self, integration_name:str): """Delete an integration""" - pass + @abstractmethod def get_integration_api(self, api_name:str, integration_name:str) -> IntegrationApi: - pass + ... @abstractmethod def get_integration_apis(self, integration_name:str) -> List[IntegrationApi]: - pass + ... @abstractmethod def get_integration(self, integration_name:str) -> Integration: - pass + ... @abstractmethod def get_integrations(self) -> List[Integration]: """Returns the list of all the available integrations""" - pass + @abstractmethod def get_prompts_with_integration(self, ai_integration:str, model_name:str) -> List[PromptTemplate]: - pass + ... @abstractmethod def get_token_usage_for_integration(self, name, integration_name) -> int: - pass + ... @abstractmethod def get_token_usage_for_integration_provider(self, name) -> dict: - pass + ... @abstractmethod def register_token_usage(self, body, name, integration_name): - pass + ... @abstractmethod def save_integration_api(self, integration_name, api_name, api_details: IntegrationApiUpdate): - pass + ... @abstractmethod def save_integration(self, integration_name, integration_details: IntegrationUpdate): - pass + ... # Tags @abstractmethod def delete_tag_for_integration(self, body, tag_name, integration_name): """Delete an integration""" - pass + @abstractmethod def delete_tag_for_integration_provider(self, body, name): - pass + ... @abstractmethod def put_tag_for_integration(self, body, name, integration_name): - pass + ... @abstractmethod def put_tag_for_integration_provider(self, body, name): - pass + ... @abstractmethod def get_tags_for_integration(self, name, integration_name): - pass + ... @abstractmethod def get_tags_for_integration_provider(self, name): - pass + ... diff --git a/src/conductor/client/orkes/models/access_key.py b/src/conductor/client/orkes/models/access_key.py index 137b81010..4c25b96fd 100644 --- a/src/conductor/client/orkes/models/access_key.py +++ b/src/conductor/client/orkes/models/access_key.py @@ -3,7 +3,7 @@ from conductor.client.orkes.models.access_key_status import AccessKeyStatus -class AccessKey: +class AccessKey: # noqa: PLW1641 def __init__(self, id: str, status: AccessKeyStatus, created_at: int) -> Self: self._id = id self._status = status diff --git a/src/conductor/client/orkes/models/created_access_key.py b/src/conductor/client/orkes/models/created_access_key.py index c9b5e5544..10f7982e2 100644 --- a/src/conductor/client/orkes/models/created_access_key.py +++ b/src/conductor/client/orkes/models/created_access_key.py @@ -1,7 +1,7 @@ from typing_extensions import Self -class CreatedAccessKey: +class CreatedAccessKey: # noqa: PLW1641 def __init__(self, id: str, secret: str) -> Self: self._id = id self._secret = secret diff --git a/src/conductor/client/orkes/models/granted_permission.py b/src/conductor/client/orkes/models/granted_permission.py index 92f6b3f4c..29510b6b8 100644 --- a/src/conductor/client/orkes/models/granted_permission.py +++ b/src/conductor/client/orkes/models/granted_permission.py @@ -5,7 +5,7 @@ from conductor.client.http.models.target_ref import TargetRef -class GrantedPermission: +class GrantedPermission: # noqa: PLW1641 def __init__(self, target: TargetRef, access: List[str]) -> Self: self._target = target self._access = access diff --git a/src/conductor/client/orkes/orkes_integration_client.py b/src/conductor/client/orkes/orkes_integration_client.py index a56d9a08a..edbc42edf 100644 --- a/src/conductor/client/orkes/orkes_integration_client.py +++ b/src/conductor/client/orkes/orkes_integration_client.py @@ -65,25 +65,25 @@ def get_token_usage_for_integration_provider(self, name) -> dict: return self.integrationApi.get_token_usage_for_integration_provider(name) def register_token_usage(self, body, name, integration_name): - pass + ... # Tags def delete_tag_for_integration(self, body, tag_name, integration_name): """Delete an integration""" - pass + def delete_tag_for_integration_provider(self, body, name): - pass + ... def put_tag_for_integration(self, body, name, integration_name): - pass + ... def put_tag_for_integration_provider(self, body, name): - pass + ... def get_tags_for_integration(self, name, integration_name): - pass + ... def get_tags_for_integration_provider(self, name): - pass + ... diff --git a/src/conductor/client/schema_client.py b/src/conductor/client/schema_client.py index 317c4e450..46b269c46 100644 --- a/src/conductor/client/schema_client.py +++ b/src/conductor/client/schema_client.py @@ -15,32 +15,32 @@ def register_schema(self, schema: SchemaDef) -> None: """ Register a new schema. """ - pass + ... @abstractmethod def get_schema(self, schema_name: str, version: int) -> SchemaDef: """ Retrieve a schema by its name and version. """ - pass + ... @abstractmethod def get_all_schemas(self) -> List[SchemaDef]: """ Retrieve all schemas. """ - pass + ... @abstractmethod def delete_schema(self, schema_name: str, version: int) -> None: """ Delete a schema by its name and version. """ - pass + ... @abstractmethod def delete_schema_by_name(self, schema_name: str) -> None: """ Delete all the versions of a schema by its name """ - pass + ... diff --git a/src/conductor/client/worker/worker_interface.py b/src/conductor/client/worker/worker_interface.py index e31795613..acb5f20f9 100644 --- a/src/conductor/client/worker/worker_interface.py +++ b/src/conductor/client/worker/worker_interface.py @@ -26,7 +26,7 @@ def execute(self, task: Task) -> TaskResult: :return: TaskResult If the task is not completed yet, return with the status as IN_PROGRESS. """ - pass + ... def get_identity(self) -> str: """ diff --git a/src/conductor/client/workflow/task/task.py b/src/conductor/client/workflow/task/task.py index a883beae7..bd7ca1565 100644 --- a/src/conductor/client/workflow/task/task.py +++ b/src/conductor/client/workflow/task/task.py @@ -155,6 +155,8 @@ def to_workflow_task(self) -> WorkflowTask: def output(self, json_path: Optional[str] = None) -> str: if json_path is None: return '${' + f'{self.task_reference_name}.output' + '}' + elif json_path.startswith('.'): + return '${' + f'{self.task_reference_name}.output{json_path}' + '}' else: return '${' + f'{self.task_reference_name}.output.{json_path}' + '}' @@ -174,7 +176,7 @@ def input(self, json_path: Optional[str] = None, key: Optional[str] = None, valu else: return '${' + f'{self.task_reference_name}.input.{json_path}' + '}' - def __getattribute__(self, __name: str) -> Any: + def __getattribute__(self, __name: str, /) -> Any: try: val = super().__getattribute__(__name) return val